/** * Output Validator — 输出验证器 * * 在 Agent 输出最终答案之前进行验证: * 1. 格式验证(Markdown/JSON 是否合法) * 2. 内容安全检测(敏感词、PII 泄露) * 3. 事实一致性检查(与工具结果是否矛盾) * 4. 幻觉检测(无根据的断言) * * v0.3.0 增强: * - 实现事实一致性检查(基于工具结果验证输出声明) * - 实现幻觉检测(检测上下文中无依据的具体声明) * * @see docs/生产级通用 AI Agent 智能体桌面应用:完整设计与构建指南.html — 第五章 */ export interface ValidationResult { valid: boolean; issues: ValidationIssue[]; score: number; // 0-1, 越高越好 } export interface ValidationIssue { severity: 'error' | 'warning' | 'info'; type: string; message: string; } export interface ValidationOptions { /** 工具调用结果文本列表,用于事实一致性检查 */ toolResults?: string[]; /** 对话上下文文本,用于幻觉检测 */ context?: string; } /** 敏感词模式 */ const SENSITIVE_PATTERNS = [ { pattern: /\b\d{4}[\s-]?\d{4}[\s-]?\d{4}[\s-]?\d{4}\b/, type: 'credit_card', message: 'Possible credit card number detected' }, // v0.3.0 修复:移除字符类中的 | 字面字符([A-Z|a-z] → [A-Za-z]) { pattern: /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/, type: 'email', message: 'Email address detected' }, { pattern: /\b\d{3}[-.]?\d{3}[-.]?\d{4}\b/, type: 'phone', message: 'Phone number detected' }, { pattern: /\b(sk-[a-zA-Z0-9]{20,})\b/, type: 'api_key', message: 'Possible API key detected' }, ]; /** 不安全内容模式 */ const UNSAFE_PATTERNS = [ { pattern: /ignore\s+(all\s+)?(previous|above|prior)\s+(instructions|commands)/i, message: 'Possible prompt injection attempt' }, { pattern: /\brm\s+-rf\s+\//, message: 'Dangerous filesystem command detected' }, { pattern: /\bsudo\b/, message: 'Privilege escalation command detected' }, ]; export class OutputValidator { /** * 验证输出内容 * * v0.3.0: 新增事实一致性检查和幻觉检测 */ async validate(output: string, options?: ValidationOptions): Promise { const issues: ValidationIssue[] = []; // 1. 格式验证 issues.push(...this.checkFormat(output)); // 2. 内容安全检测 issues.push(...this.checkSafety(output)); // 3. 事实一致性检查(v0.3.0 新增) if (options?.toolResults && options.toolResults.length > 0) { issues.push(...this.checkFactConsistency(output, options.toolResults)); } // 4. 幻觉检测(v0.3.0 新增) if (options?.context) { issues.push(...this.checkHallucination(output, options.context)); } // 5. 空内容检查 if (!output.trim()) { issues.push({ severity: 'error', type: 'empty', message: 'Output is empty' }); } // 6. 过短内容检查 if (output.trim().length < 10 && output.trim().length > 0) { issues.push({ severity: 'warning', type: 'short', message: 'Output is suspiciously short' }); } return { valid: issues.filter((i) => i.severity === 'error').length === 0, issues, score: this.calculateScore(issues), }; } /** * 事实一致性检查 — 验证输出中的声明是否与工具结果矛盾 * * v0.3.0 实现的检查策略: * 1. 工具报告错误但输出声称成功 * 2. 工具报告文件不存在但输出引用文件内容 * 3. 工具报告的数值与输出中的数值矛盾 * * v0.3.0 修复: * - errorIndicators 改为更精确的匹配,避免常见词 "error" 误报 * 使用 "error:" 或 "error occurred" 等上下文限定 */ private checkFactConsistency(output: string, toolResults: string[]): ValidationIssue[] { const issues: ValidationIssue[] = []; const toolContext = toolResults.join('\n'); // 检查1:工具报告错误但输出声称成功 // v0.3.0 修复:errorIndicators 改为更精确的匹配,避免 "error" 单独出现导致误报 // 要求 error 后跟冒号、消息或特定错误模式 const errorIndicators = /(?:error\s*[:\)]|error\s+occurred|failed\s+to|not\s+found|does\s+not\s+exist|enoent|permission\s+denied|cannot\s+access|no\s+such\s+file|exception|traceback|exit\s+code\s+[1-9])/i; const hasErrorInTools = errorIndicators.test(toolContext); if (hasErrorInTools) { const successClaims = [ /\b(?:successfully|success|done|completed|created|written|deleted|installed|updated|finished)\b/i, /(?:已(?:完成|创建|写入|删除|安装|更新|成功))/, ]; for (const pattern of successClaims) { const match = output.match(pattern); if (match) { issues.push({ severity: 'warning', type: 'fact_inconsistency', message: `Output claims "${match[0]}" but tool results contain errors`, }); break; } } } // 检查2:工具报告文件不存在但输出引用文件内容 const notFoundPattern = /(?:not\s+found|no\s+such\s+file|does\s+not\s+exist|文件(?:不存在|未找到|找不到))/i; if (notFoundPattern.test(toolContext)) { const contentClaimPatterns = [ /(?:file\s+(?:contains|says|shows)|文件(?:内容|包含|显示))/i, /(?:the\s+content\s+is|内容是)/i, ]; for (const pattern of contentClaimPatterns) { if (pattern.test(output)) { issues.push({ severity: 'error', type: 'fact_inconsistency', message: 'Output references file content but tool reported file not found', }); break; } } } // 检查3:工具报告退出码非零但输出声称命令成功 const exitCodePattern = /exit\s+code[:\s]+(\d+)/i; const exitMatch = toolContext.match(exitCodePattern); if (exitMatch && parseInt(exitMatch[1], 10) !== 0) { const commandSuccessPattern = /\b(?:command\s+(?:succeeded|completed\s+successfully|ran\s+successfully))\b/i; if (commandSuccessPattern.test(output)) { issues.push({ severity: 'error', type: 'fact_inconsistency', message: `Output claims command success but tool reported exit code ${exitMatch[1]}`, }); } } return issues; } /** * 幻觉检测 — 检测输出中无依据的具体声明 * * v0.3.0 实现的检查策略: * 1. 提取输出中的文件路径,检查是否在上下文中出现 * 2. 提取输出中的 URL,检查是否在上下文中出现 * 3. 检测虚构的 API 响应模式 * * v0.3.0 修复: * - Windows 路径正则包含反斜杠字符类,正确匹配 C:\Users\test 等完整路径 * - apiResponsePattern 在循环前重置 lastIndex,避免间歇性漏检 * - 路径比较改为大小写不敏感(Windows 路径不敏感) */ private checkHallucination(output: string, context: string): ValidationIssue[] { const issues: ValidationIssue[] = []; // 检查1:文件路径声称但上下文中不存在 // v0.3.0 修复:Windows 路径正则 [A-Z]:\\[\w\\][\w.-]+ 不含反斜杠,改为 [A-Z]:\\[\w\\.-]+(?:\\[\w\\.-]+)* const pathPattern = /(?:\/[\w][\w.-]*){2,}|[A-Z]:\\[\w\\.-]+(?:\\[\w\\.-]+)*/g; const paths = output.match(pathPattern) || []; const seenPaths = new Set(); // v0.3.0 修复:上下文比较改为小写(Windows 路径大小写不敏感) const contextLower = context.toLowerCase(); for (const path of paths) { if (seenPaths.has(path)) continue; seenPaths.add(path); // 检查输出是否声称读取/访问了该路径 const pathIndex = output.indexOf(path); const surrounding = output.substring( Math.max(0, pathIndex - 80), Math.min(output.length, pathIndex + path.length + 80), ); const accessClaim = /(?:read|loaded|opened|accessed|found|exists|contains|读取|加载|打开|访问|存在|包含)/i; // v0.3.0 修复:路径比较改为大小写不敏感 if (accessClaim.test(surrounding) && !contextLower.includes(path.toLowerCase()) && path.length > 8) { issues.push({ severity: 'warning', type: 'hallucination', message: `Path "${path.substring(0, 60)}" claimed in output but not found in context`, }); } } // 检查2:URL 声称但上下文中不存在 const urlPattern = /https?:\/\/[^\s<>"]{8,}/g; const urls = output.match(urlPattern) || []; const seenUrls = new Set(); for (const url of urls) { if (seenUrls.has(url)) continue; seenUrls.add(url); // URL 在上下文中不存在且输出声称访问了它 const urlIndex = output.indexOf(url); const surrounding = output.substring( Math.max(0, urlIndex - 60), Math.min(output.length, urlIndex + url.length + 60), ); const fetchClaim = /(?:fetched|retrieved|loaded|returned|visited|获取|抓取|访问|返回)/i; // v0.3.0 修复: URL 比较改为大小写不敏感,因为 URL host 部分大小写不敏感 if (fetchClaim.test(surrounding) && !context.toLowerCase().includes(url.toLowerCase())) { issues.push({ severity: 'info', type: 'hallucination', message: `URL in output not found in conversation context`, }); } } // 检查3:检测虚构的 API 响应(声称返回了特定 JSON 但上下文中没有) const apiResponsePattern = /(?:API\s+(?:returned|responded)|接口(?:返回|响应))[:\s]*(\{[^}]{10,}\})/gi; // v0.3.0 修复:循环前重置 lastIndex,避免带 g flag 的正则在 break 后 lastIndex 未重置导致漏检 apiResponsePattern.lastIndex = 0; let apiMatch; while ((apiMatch = apiResponsePattern.exec(output)) !== null) { const jsonSnippet = apiMatch[1]; if (!context.includes(jsonSnippet.substring(0, 20))) { issues.push({ severity: 'warning', type: 'hallucination', message: 'API response in output not found in tool results', }); break; } } return issues; } /** * 格式验证 */ private checkFormat(output: string): ValidationIssue[] { const issues: ValidationIssue[] = []; // 未闭合的代码块 const openCodeBlocks = (output.match(/```/g) || []).length; if (openCodeBlocks % 2 !== 0) { issues.push({ severity: 'warning', type: 'format', message: 'Unclosed code block detected' }); } // 未闭合的 HTML 标签 const openTags = (output.match(/<[a-z][a-z0-9]*[^/]*>/gi) || []).length; const closeTags = (output.match(/<\/[a-z][a-z0-9]*>/gi) || []).length; if (openTags > closeTags + 5) { issues.push({ severity: 'info', type: 'format', message: 'Possible unclosed HTML tags' }); } // 未闭合的括号 const openParens = (output.match(/\(/g) || []).length; const closeParens = (output.match(/\)/g) || []).length; if (Math.abs(openParens - closeParens) > 3) { issues.push({ severity: 'info', type: 'format', message: 'Mismatched parentheses detected' }); } return issues; } /** * 内容安全检测 */ private checkSafety(output: string): ValidationIssue[] { const issues: ValidationIssue[] = []; // 敏感信息检测 for (const { pattern, type, message } of SENSITIVE_PATTERNS) { if (pattern.test(output)) { issues.push({ severity: 'warning', type: `sensitive_${type}`, message }); } } // 不安全内容检测 for (const { pattern, message } of UNSAFE_PATTERNS) { if (pattern.test(output)) { issues.push({ severity: 'error', type: 'unsafe', message }); } } return issues; } /** * 计算质量分数 */ private calculateScore(issues: ValidationIssue[]): number { let score = 1.0; for (const issue of issues) { switch (issue.severity) { case 'error': score -= 0.3; break; case 'warning': score -= 0.1; break; case 'info': score -= 0.02; break; } } return Math.max(0, Math.min(1, score)); } }