metona-ai-desktop/electron/harness/verification/output-validator.ts

/**
 * Output Validator — 输出验证器
 *
 * 在 Agent 输出最终答案之前进行验证：
 * 1. 格式验证（Markdown/JSON 是否合法）
 * 2. 内容安全检测（敏感词、PII 泄露）
 * 3. 事实一致性检查（与工具结果是否矛盾）
 * 4. 幻觉检测（无根据的断言）
 *
 * v0.3.0 增强：
 * - 实现事实一致性检查（基于工具结果验证输出声明）
 * - 实现幻觉检测（检测上下文中无依据的具体声明）
 *
 * @see docs/生产级通用 AI Agent 智能体桌面应用：完整设计与构建指南.html — 第五章
 */

export interface ValidationResult {
  valid: boolean;
  issues: ValidationIssue[];
  score: number; // 0-1, 越高越好
}

export interface ValidationIssue {
  severity: 'error' | 'warning' | 'info';
  type: string;
  message: string;
}

export interface ValidationOptions {
  /** 工具调用结果文本列表，用于事实一致性检查 */
  toolResults?: string[];
  /** 对话上下文文本，用于幻觉检测 */
  context?: string;
}

/** 敏感词模式 */
const SENSITIVE_PATTERNS = [
  { pattern: /\b\d{4}[\s-]?\d{4}[\s-]?\d{4}[\s-]?\d{4}\b/, type: 'credit_card', message: 'Possible credit card number detected' },
  // v0.3.0 修复：移除字符类中的 | 字面字符（[A-Z|a-z] → [A-Za-z]）
  { pattern: /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/, type: 'email', message: 'Email address detected' },
  { pattern: /\b\d{3}[-.]?\d{3}[-.]?\d{4}\b/, type: 'phone', message: 'Phone number detected' },
  { pattern: /\b(sk-[a-zA-Z0-9]{20,})\b/, type: 'api_key', message: 'Possible API key detected' },
];

/** 不安全内容模式 */
const UNSAFE_PATTERNS = [
  { pattern: /ignore\s+(all\s+)?(previous|above|prior)\s+(instructions|commands)/i, message: 'Possible prompt injection attempt' },
  { pattern: /\brm\s+-rf\s+\//, message: 'Dangerous filesystem command detected' },
  { pattern: /\bsudo\b/, message: 'Privilege escalation command detected' },
];

export class OutputValidator {
  /**
   * 验证输出内容
   *
   * v0.3.0: 新增事实一致性检查和幻觉检测
   */
  async validate(output: string, options?: ValidationOptions): Promise<ValidationResult> {
    const issues: ValidationIssue[] = [];

    // 1. 格式验证
    issues.push(...this.checkFormat(output));

    // 2. 内容安全检测
    issues.push(...this.checkSafety(output));

    // 3. 事实一致性检查（v0.3.0 新增）
    if (options?.toolResults && options.toolResults.length > 0) {
      issues.push(...this.checkFactConsistency(output, options.toolResults));
    }

    // 4. 幻觉检测（v0.3.0 新增）
    if (options?.context) {
      issues.push(...this.checkHallucination(output, options.context));
    }

    // 5. 空内容检查
    if (!output.trim()) {
      issues.push({ severity: 'error', type: 'empty', message: 'Output is empty' });
    }

    // 6. 过短内容检查
    if (output.trim().length < 10 && output.trim().length > 0) {
      issues.push({ severity: 'warning', type: 'short', message: 'Output is suspiciously short' });
    }

    return {
      valid: issues.filter((i) => i.severity === 'error').length === 0,
      issues,
      score: this.calculateScore(issues),
    };
  }

  /**
   * 事实一致性检查 — 验证输出中的声明是否与工具结果矛盾
   *
   * v0.3.0 实现的检查策略：
   * 1. 工具报告错误但输出声称成功
   * 2. 工具报告文件不存在但输出引用文件内容
   * 3. 工具报告的数值与输出中的数值矛盾
   *
   * v0.3.0 修复：
   * - errorIndicators 改为更精确的匹配，避免常见词 "error" 误报
   *   使用 "error:" 或 "error occurred" 等上下文限定
   */
  private checkFactConsistency(output: string, toolResults: string[]): ValidationIssue[] {
    const issues: ValidationIssue[] = [];
    const toolContext = toolResults.join('\n');

    // 检查1：工具报告错误但输出声称成功
    // v0.3.0 修复：errorIndicators 改为更精确的匹配，避免 "error" 单独出现导致误报
    // 要求 error 后跟冒号、消息或特定错误模式
    const errorIndicators = /(?:error\s*[:)]|error\s+occurred|failed\s+to|not\s+found|does\s+not\s+exist|enoent|permission\s+denied|cannot\s+access|no\s+such\s+file|exception|traceback|exit\s+code\s+[1-9])/i;
    const hasErrorInTools = errorIndicators.test(toolContext);

    if (hasErrorInTools) {
      const successClaims = [
        /\b(?:successfully|success|done|completed|created|written|deleted|installed|updated|finished)\b/i,
        /(?:已(?:完成|创建|写入|删除|安装|更新|成功))/,
      ];
      for (const pattern of successClaims) {
        const match = output.match(pattern);
        if (match) {
          issues.push({
            severity: 'warning',
            type: 'fact_inconsistency',
            message: `Output claims "${match[0]}" but tool results contain errors`,
          });
          break;
        }
      }
    }

    // 检查2：工具报告文件不存在但输出引用文件内容
    const notFoundPattern = /(?:not\s+found|no\s+such\s+file|does\s+not\s+exist|文件(?:不存在|未找到|找不到))/i;
    if (notFoundPattern.test(toolContext)) {
      const contentClaimPatterns = [
        /(?:file\s+(?:contains|says|shows)|文件(?:内容|包含|显示))/i,
        /(?:the\s+content\s+is|内容是)/i,
      ];
      for (const pattern of contentClaimPatterns) {
        if (pattern.test(output)) {
          issues.push({
            severity: 'error',
            type: 'fact_inconsistency',
            message: 'Output references file content but tool reported file not found',
          });
          break;
        }
      }
    }

    // 检查3：工具报告退出码非零但输出声称命令成功
    const exitCodePattern = /exit\s+code[:\s]+(\d+)/i;
    const exitMatch = toolContext.match(exitCodePattern);
    if (exitMatch && parseInt(exitMatch[1], 10) !== 0) {
      const commandSuccessPattern = /\b(?:command\s+(?:succeeded|completed\s+successfully|ran\s+successfully))\b/i;
      if (commandSuccessPattern.test(output)) {
        issues.push({
          severity: 'error',
          type: 'fact_inconsistency',
          message: `Output claims command success but tool reported exit code ${exitMatch[1]}`,
        });
      }
    }

    return issues;
  }

  /**
   * 幻觉检测 — 检测输出中无依据的具体声明
   *
   * v0.3.0 实现的检查策略：
   * 1. 提取输出中的文件路径，检查是否在上下文中出现
   * 2. 提取输出中的 URL，检查是否在上下文中出现
   * 3. 检测虚构的 API 响应模式
   *
   * v0.3.0 修复：
   * - Windows 路径正则包含反斜杠字符类，正确匹配 C:\Users\test 等完整路径
   * - apiResponsePattern 在循环前重置 lastIndex，避免间歇性漏检
   * - 路径比较改为大小写不敏感（Windows 路径不敏感）
   */
  private checkHallucination(output: string, context: string): ValidationIssue[] {
    const issues: ValidationIssue[] = [];

    // 检查1：文件路径声称但上下文中不存在
    // v0.3.0 修复：Windows 路径正则 [A-Z]:\\[\w\\][\w.-]+ 不含反斜杠，改为 [A-Z]:\\[\w\\.-]+(?:\\[\w\\.-]+)*
    const pathPattern = /(?:\/[\w][\w.-]*){2,}|[A-Z]:\\[\w\\.-]+(?:\\[\w\\.-]+)*/g;
    const paths = output.match(pathPattern) || [];
    const seenPaths = new Set<string>();
    // v0.3.0 修复：上下文比较改为小写（Windows 路径大小写不敏感）
    const contextLower = context.toLowerCase();
    for (const path of paths) {
      if (seenPaths.has(path)) continue;
      seenPaths.add(path);

      // 检查输出是否声称读取/访问了该路径
      const pathIndex = output.indexOf(path);
      const surrounding = output.substring(
        Math.max(0, pathIndex - 80),
        Math.min(output.length, pathIndex + path.length + 80),
      );
      const accessClaim = /(?:read|loaded|opened|accessed|found|exists|contains|读取|加载|打开|访问|存在|包含)/i;

      // v0.3.0 修复：路径比较改为大小写不敏感
      if (accessClaim.test(surrounding) && !contextLower.includes(path.toLowerCase()) && path.length > 8) {
        issues.push({
          severity: 'warning',
          type: 'hallucination',
          message: `Path "${path.substring(0, 60)}" claimed in output but not found in context`,
        });
      }
    }

    // 检查2：URL 声称但上下文中不存在
    const urlPattern = /https?:\/\/[^\s<>"]{8,}/g;
    const urls = output.match(urlPattern) || [];
    const seenUrls = new Set<string>();
    for (const url of urls) {
      if (seenUrls.has(url)) continue;
      seenUrls.add(url);

      // URL 在上下文中不存在且输出声称访问了它
      const urlIndex = output.indexOf(url);
      const surrounding = output.substring(
        Math.max(0, urlIndex - 60),
        Math.min(output.length, urlIndex + url.length + 60),
      );
      const fetchClaim = /(?:fetched|retrieved|loaded|returned|visited|获取|抓取|访问|返回)/i;

      // v0.3.0 修复: URL 比较改为大小写不敏感，因为 URL host 部分大小写不敏感
      if (fetchClaim.test(surrounding) && !context.toLowerCase().includes(url.toLowerCase())) {
        issues.push({
          severity: 'info',
          type: 'hallucination',
          message: `URL in output not found in conversation context`,
        });
      }
    }

    // 检查3：检测虚构的 API 响应（声称返回了特定 JSON 但上下文中没有）
    const apiResponsePattern = /(?:API\s+(?:returned|responded)|接口(?:返回|响应))[:\s]*(\{[^}]{10,}\})/gi;
    // v0.3.0 修复：循环前重置 lastIndex，避免带 g flag 的正则在 break 后 lastIndex 未重置导致漏检
    apiResponsePattern.lastIndex = 0;
    let apiMatch;
    while ((apiMatch = apiResponsePattern.exec(output)) !== null) {
      const jsonSnippet = apiMatch[1];
      if (!context.includes(jsonSnippet.substring(0, 20))) {
        issues.push({
          severity: 'warning',
          type: 'hallucination',
          message: 'API response in output not found in tool results',
        });
        break;
      }
    }

    return issues;
  }

  /**
   * 格式验证
   */
  private checkFormat(output: string): ValidationIssue[] {
    const issues: ValidationIssue[] = [];

    // 未闭合的代码块
    const openCodeBlocks = (output.match(/```/g) || []).length;
    if (openCodeBlocks % 2 !== 0) {
      issues.push({ severity: 'warning', type: 'format', message: 'Unclosed code block detected' });
    }

    // 未闭合的 HTML 标签
    const openTags = (output.match(/<[a-z][a-z0-9]*[^/]*>/gi) || []).length;
    const closeTags = (output.match(/<\/[a-z][a-z0-9]*>/gi) || []).length;
    if (openTags > closeTags + 5) {
      issues.push({ severity: 'info', type: 'format', message: 'Possible unclosed HTML tags' });
    }

    // 未闭合的括号
    const openParens = (output.match(/\(/g) || []).length;
    const closeParens = (output.match(/\)/g) || []).length;
    if (Math.abs(openParens - closeParens) > 3) {
      issues.push({ severity: 'info', type: 'format', message: 'Mismatched parentheses detected' });
    }

    return issues;
  }

  /**
   * 内容安全检测
   */
  private checkSafety(output: string): ValidationIssue[] {
    const issues: ValidationIssue[] = [];

    // 敏感信息检测
    for (const { pattern, type, message } of SENSITIVE_PATTERNS) {
      if (pattern.test(output)) {
        issues.push({ severity: 'warning', type: `sensitive_${type}`, message });
      }
    }

    // 不安全内容检测
    for (const { pattern, message } of UNSAFE_PATTERNS) {
      if (pattern.test(output)) {
        issues.push({ severity: 'error', type: 'unsafe', message });
      }
    }

    return issues;
  }

  /**
   * 计算质量分数
   */
  private calculateScore(issues: ValidationIssue[]): number {
    let score = 1.0;
    for (const issue of issues) {
      switch (issue.severity) {
        case 'error': score -= 0.3; break;
        case 'warning': score -= 0.1; break;
        case 'info': score -= 0.02; break;
      }
    }
    return Math.max(0, Math.min(1, score));
  }
}