/** * Output Validator — 输出验证器 * * 在 Agent 输出最终答案之前进行验证: * 1. 格式验证(Markdown/JSON 是否合法) * 2. 内容安全检测(敏感词、PII 泄露) * 3. 事实一致性检查(与工具结果是否矛盾) * 4. 幻觉检测(无根据的断言) * * @see docs/生产级通用 AI Agent 智能体桌面应用:完整设计与构建指南.html — 第五章 */ export interface ValidationResult { valid: boolean; issues: ValidationIssue[]; score: number; // 0-1, 越高越好 } export interface ValidationIssue { severity: 'error' | 'warning' | 'info'; type: string; message: string; } /** 敏感词模式 */ const SENSITIVE_PATTERNS = [ { pattern: /\b\d{4}[\s-]?\d{4}[\s-]?\d{4}[\s-]?\d{4}\b/, type: 'credit_card', message: 'Possible credit card number detected' }, { pattern: /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/, type: 'email', message: 'Email address detected' }, { pattern: /\b\d{3}[-.]?\d{3}[-.]?\d{4}\b/, type: 'phone', message: 'Phone number detected' }, { pattern: /\b(sk-[a-zA-Z0-9]{20,})\b/, type: 'api_key', message: 'Possible API key detected' }, ]; /** 不安全内容模式 */ const UNSAFE_PATTERNS = [ { pattern: /ignore\s+(all\s+)?(previous|above|prior)\s+(instructions|commands)/i, message: 'Possible prompt injection attempt' }, { pattern: /\brm\s+-rf\s+\//, message: 'Dangerous filesystem command detected' }, { pattern: /\bsudo\b/, message: 'Privilege escalation command detected' }, ]; export class OutputValidator { /** * 验证输出内容 */ async validate(output: string): Promise { const issues: ValidationIssue[] = []; // 1. 格式验证 issues.push(...this.checkFormat(output)); // 2. 内容安全检测 issues.push(...this.checkSafety(output)); // 3. 空内容检查 if (!output.trim()) { issues.push({ severity: 'error', type: 'empty', message: 'Output is empty' }); } // 4. 过短内容检查 if (output.trim().length < 10 && output.trim().length > 0) { issues.push({ severity: 'warning', type: 'short', message: 'Output is suspiciously short' }); } return { valid: issues.filter((i) => i.severity === 'error').length === 0, issues, score: this.calculateScore(issues), }; } /** * 格式验证 */ private checkFormat(output: string): ValidationIssue[] { const issues: ValidationIssue[] = []; // 未闭合的代码块 const openCodeBlocks = (output.match(/```/g) || []).length; if (openCodeBlocks % 2 !== 0) { issues.push({ severity: 'warning', type: 'format', message: 'Unclosed code block detected' }); } // 未闭合的 HTML 标签 const openTags = (output.match(/<[a-z][a-z0-9]*[^/]*>/gi) || []).length; const closeTags = (output.match(/<\/[a-z][a-z0-9]*>/gi) || []).length; if (openTags > closeTags + 5) { issues.push({ severity: 'info', type: 'format', message: 'Possible unclosed HTML tags' }); } // 未闭合的括号 const openParens = (output.match(/\(/g) || []).length; const closeParens = (output.match(/\)/g) || []).length; if (Math.abs(openParens - closeParens) > 3) { issues.push({ severity: 'info', type: 'format', message: 'Mismatched parentheses detected' }); } return issues; } /** * 内容安全检测 */ private checkSafety(output: string): ValidationIssue[] { const issues: ValidationIssue[] = []; // 敏感信息检测 for (const { pattern, type, message } of SENSITIVE_PATTERNS) { if (pattern.test(output)) { issues.push({ severity: 'warning', type: `sensitive_${type}`, message }); } } // 不安全内容检测 for (const { pattern, message } of UNSAFE_PATTERNS) { if (pattern.test(output)) { issues.push({ severity: 'error', type: 'unsafe', message }); } } return issues; } /** * 计算质量分数 */ private calculateScore(issues: ValidationIssue[]): number { let score = 1.0; for (const issue of issues) { switch (issue.severity) { case 'error': score -= 0.3; break; case 'warning': score -= 0.1; break; case 'info': score -= 0.02; break; } } return Math.max(0, Math.min(1, score)); } }