metona-ai-desktop/electron/harness/tools/built-in/web-search.ts

/**
 * web_search — 网络搜索工具
 *
 * 双模式搜索：SearXNG 元搜索 / 内置四引擎并行搜索
 * 内置引擎：Bing + 百度 + 搜狗 + 360 搜索（Promise.allSettled 容错并发）
 * 智能排序：引擎权重(50%) + 可达性(30%) + 摘要质量(20%)
 * 自动抓取：对前 N 条结果调用 web_fetch 获取完整正文
 *
 * @see docs/Agent网络工具通用设计-v2.md — 第 2 章 web_search 搜索设计
 */

import type { IMetonaTool, ToolExecutionContext } from '../../types/metona-tool';
import type { MetonaToolDef } from '../../../harness/types';
import { MetonaToolCategory, MetonaRiskLevel } from '../../../harness/types';
import type { ConfigService } from '../../../services/config.service';
import {
  searchCache,
  normalizeUrl,
  fetchWithTimeout,
  buildSearXNGAuthHeaders,
  logTool,
} from './network-utils';
import type { WebFetchTool } from './web-fetch';

// ===== 类型定义 =====

interface SearchResult {
  title: string;
  url: string;
  snippet: string;
  engine: string;
  weight: number;
  reachable?: boolean;
  _score?: number;
  _enhanced?: boolean;
}

interface SearXNGConfig {
  enabled: boolean;
  url: string;
  engines: string;
  language: string;
  safesearch: number;
  time_range: string;
  max_results: number;
  auth_key: string;
  auth_type: string;
  format: string;
  fetch_count: number;
  fetch_mode: string;
}

// ===== 引擎定义 =====

interface EngineDef {
  name: string;
  weight: number;
  searchUrl: (query: string, timeRange?: string) => string;
  parse: (html: string) => SearchResult[];
}

const ENGINES: EngineDef[] = [
  {
    name: 'bing',
    weight: 90,
    searchUrl: (q, tr) => {
      const freshness = tr ? `&filters=ex1:"ez${tr === 'day' ? '1' : tr === 'week' ? '2' : tr === 'month' ? '3' : '4'}"` : '';
      return `https://www.bing.com/search?q=${encodeURIComponent(q)}${freshness}&count=20`;
    },
    parse: parseBing,
  },
  {
    name: '百度',
    weight: 80,
    searchUrl: (q) => `https://www.baidu.com/s?wd=${encodeURIComponent(q)}&rn=20`,
    parse: parseBaidu,
  },
  {
    name: '搜狗',
    weight: 75,
    searchUrl: (q) => `https://www.sogou.com/web?query=${encodeURIComponent(q)}&num=20`,
    parse: parseSogou,
  },
  {
    name: '360搜索',
    weight: 75,
    searchUrl: (q) => `https://www.so.com/s?q=${encodeURIComponent(q)}&pn=20`,
    parse: parse360,
  },
];

// ===== HTML 解析器（正则实现，后续可迁移至 cheerio） =====

function parseBing(html: string): SearchResult[] {
  const results: SearchResult[] = [];
  const blocks = html.split(/<li[^>]*class="b_algo"/i).slice(1);
  for (const block of blocks) {
    const titleMatch = block.match(/<a[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
    if (!titleMatch) continue;
    const url = titleMatch[1];
    const title = titleMatch[2].replace(/<[^>]+>/g, '').trim();
    const snippetMatch = block.match(/<p[^>]*>([\s\S]*?)<\/p>/i) || block.match(/class="b_caption"[^>]*>([\s\S]*?)<\/div>/i);
    const snippet = snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '').trim() : '';
    if (title && url && !url.includes('bing.com')) {
      results.push({ title, url, snippet, engine: 'bing', weight: 90 });
    }
  }
  return results;
}

function parseBaidu(html: string): SearchResult[] {
  const results: SearchResult[] = [];
  const blocks = html.split(/<div[^>]*class="result[^"]*"/i).slice(1);
  for (const block of blocks) {
    const titleMatch = block.match(/<a[^>]*data-url="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i)
      || block.match(/<a[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
    if (!titleMatch) continue;
    const url = titleMatch[1].startsWith('http') ? titleMatch[1] : `https://${titleMatch[1]}`;
    const title = titleMatch[2].replace(/<[^>]+>/g, '').trim();
    const snippetMatch = block.match(/class="c-abstract[^"]*"[^>]*>([\s\S]*?)<\/span>/i)
      || block.match(/class="content-right[^"]*"[^>]*>([\s\S]*?)<\/div>/i);
    const snippet = snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '').trim() : '';
    if (title && url && !url.includes('baidu.com/link')) {
      results.push({ title, url, snippet, engine: '百度', weight: 80 });
    }
  }
  return results;
}

function parseSogou(html: string): SearchResult[] {
  const results: SearchResult[] = [];
  const blocks = html.split(/<div[^>]*class="vrwrap"/i).slice(1)
    .concat(html.split(/<div[^>]*class="rb"/i).slice(1));
  for (const block of blocks) {
    const titleMatch = block.match(/<a[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
    if (!titleMatch) continue;
    const url = titleMatch[1].startsWith('http') ? titleMatch[1] : `https://www.sogou.com${titleMatch[1]}`;
    const title = titleMatch[2].replace(/<[^>]+>/g, '').trim();
    const snippetMatch = block.match(/class="star-wiki[^"]*"[^>]*>([\s\S]*?)<\/div>/i)
      || block.match(/class="space-txt[^"]*"[^>]*>([\s\S]*?)<\/p>/i)
      || block.match(/class="str_info[^"]*"[^>]*>([\s\S]*?)<\/p>/i);
    const snippet = snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '').trim() : '';
    if (title && url && !url.includes('sogou.com')) {
      results.push({ title, url, snippet, engine: '搜狗', weight: 75 });
    }
  }
  return results;
}

function parse360(html: string): SearchResult[] {
  const results: SearchResult[] = [];
  const blocks = html.split(/<li[^>]*class="res-list"/i).slice(1)
    .concat(html.split(/<div[^>]*class="result"/i).slice(1));
  for (const block of blocks) {
    const titleMatch = block.match(/<a[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
    if (!titleMatch) continue;
    const url = titleMatch[1];
    const title = titleMatch[2].replace(/<[^>]+>/g, '').trim();
    const snippetMatch = block.match(/class="res-desc[^"]*"[^>]*>([\s\S]*?)<\/p>/i)
      || block.match(/class="res-rich[^"]*"[^>]*>([\s\S]*?)<\/div>/i)
      || block.match(/class="res-summary[^"]*"[^>]*>([\s\S]*?)<\/p>/i)
      || block.match(/<dd[^>]*>([\s\S]*?)<\/dd>/i);
    const snippet = snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '').trim() : '';
    if (title && url && !url.includes('so.com')) {
      results.push({ title, url, snippet, engine: '360搜索', weight: 75 });
    }
  }
  return results;
}

// ===== 可达性预检 =====

async function checkReachability(urls: string[], concurrency = 5): Promise<Map<string, boolean>> {
  const result = new Map<string, boolean>();
  for (let i = 0; i < urls.length; i += concurrency) {
    const batch = urls.slice(i, i + concurrency);
    const checks = batch.map(async (url) => {
      try {
        const resp = await fetchWithTimeout(url, { method: 'HEAD', redirect: 'follow' }, 3_000);
        result.set(url, resp.ok);
      } catch {
        result.set(url, false);
      }
    });
    await Promise.allSettled(checks);
  }
  return result;
}

// ===== 智能排序 =====

function smartSort(results: SearchResult[]): SearchResult[] {
  for (const r of results) {
    const reachability = r.reachable ? 30 : -20;
    const snippetQuality = Math.min(r.snippet.length, 100) / 100 * 20;
    const weightScore = (r.weight / 100) * 50;
    r._score = weightScore + reachability + snippetQuality;
  }
  return results.sort((a, b) => (b._score ?? 0) - (a._score ?? 0));
}

// ===== 相关性过滤（用于自动抓取） =====

function computeRelevance(query: string, result: SearchResult): number {
  const terms: string[] = [];
  // 中文双字/三字片段
  const chinese = query.match(/[\u4e00-\u9fa5]{2,3}/g);
  if (chinese) terms.push(...chinese);
  // 英文单词
  const english = query.match(/[a-zA-Z]{2,}/g);
  if (english) terms.push(...english);

  let score = 0;
  const text = `${result.title} ${result.snippet}`.toLowerCase();
  for (const term of terms) {
    if (text.includes(term.toLowerCase())) score += 10;
  }
  return Math.min(score, 100);
}

// ===== WebSearchTool =====

export class WebSearchTool implements IMetonaTool {
  readonly definition: MetonaToolDef = {
    name: 'web_search',
    description: 'Search the web for information. Returns titles, snippets, and URLs. When SearXNG is enabled, uses the configured SearXNG instance; otherwise uses built-in engines (Bing, Baidu, Sogou, 360). Automatically fetches full content for top results.',
    parameters: {
      type: 'object',
      properties: {
        query: { type: 'string', description: 'Search query keywords' },
        time_range: { type: 'string', description: 'Time filter: day, week, month, year (optional)' },
        enhance_snippets: { type: 'boolean', description: 'Auto-enhance short snippets (default true)' },
      },
      required: ['query'],
    },
    category: MetonaToolCategory.NETWORK,
    riskLevel: MetonaRiskLevel.LOW,
    requiresPermission: false,
    timeoutMs: 300_000,
  };

  constructor(
    private configService: ConfigService,
    private webFetchTool: WebFetchTool,
  ) {}

  async execute(args: Record<string, unknown>, _context: ToolExecutionContext): Promise<unknown> {
    const query = args.query as string;

    // 读取 SearXNG 配置
    const searxngConfig = this.readSearXNGConfig();
    const useSearXNG = searxngConfig.enabled && !!searxngConfig.url;

    // 两个模式参数完全独立：
    // - 内置模式：固定默认值 maxResults=30, fetchTop=5
    // - SearXNG 模式：由配置面板控制
    const maxResults = useSearXNG
      ? Math.min(50, Math.max(1, searxngConfig.max_results > 0 ? searxngConfig.max_results : 30))
      : 30;

    const timeRange = (args.time_range as string) ?? '';
    const enhanceSnippets = (args.enhance_snippets as boolean) ?? true;

    const fetchTop = useSearXNG
      ? Math.min(8, Math.max(3, searxngConfig.fetch_count > 0 ? searxngConfig.fetch_count : 5))
      : 5;

    logTool('web_search', `Mode=${useSearXNG ? 'searxng' : 'builtin'}, maxResults=${maxResults}, fetchTop=${fetchTop}`);

    // 缓存检查（key 含模式 + maxResults + fetchTop，避免配置变更后返回旧缓存）
    const cacheKey = `${searxngConfig.enabled ? 'searxng' : 'builtin'}:${maxResults}:${fetchTop}:${normalizeUrl(query).toLowerCase()}`;
    const cached = searchCache.get(cacheKey);
    if (cached) {
      logTool('web_search', `Cache hit: "${query}"`);
      return { ...(cached as Record<string, unknown>), from_cache: true };
    }

    let results: SearchResult[];
    let mode: string;
    let engineStats: Record<string, string>;

    if (useSearXNG) {
      const searxResult = await this.searchSearXNG(query, searxngConfig, maxResults, timeRange);
      results = searxResult.results;
      mode = 'searxng';
      engineStats = searxResult.engineStats;
    } else {
      const builtinResult = await this.searchBuiltinEngines(query, maxResults, timeRange);
      results = builtinResult.results;
      mode = 'builtin';
      engineStats = builtinResult.engineStats;
    }

    // 合并去重
    const deduped = this.deduplicate(results);

    // 可达性预检
    const topUrls = deduped.slice(0, Math.min(20, deduped.length)).map((r) => r.url);
    const reachabilityMap = await checkReachability(topUrls);
    for (const r of deduped) {
      r.reachable = reachabilityMap.get(r.url) ?? false;
    }

    // 智能排序
    const sorted = smartSort(deduped).slice(0, maxResults);

    // 摘要增强
    if (enhanceSnippets) {
      await this.enhanceSnippets(sorted, 3);
    }

    // 自动抓取完整内容
    const fetchedContent = await this.autoFetch(query, sorted, searxngConfig.fetch_mode, fetchTop);

    // 格式化输出
    const formatted = this.formatResults(query, sorted);

    const output = {
      success: true,
      query,
      results: sorted.map((r) => ({
        title: r.title,
        url: r.url,
        snippet: r.snippet,
        engine: r.engine,
        reachable: r.reachable,
        _score: r._score ? Math.round(r._score * 10) / 10 : undefined,
        _enhanced: r._enhanced,
      })),
      total: sorted.length,
      formatted,
      from_cache: false,
      engine_stats: engineStats,
      _mode: mode,
      _fetched: fetchedContent.map((f) => ({ url: f.url, title: f.title, content: f.content })),
      _fetched_count: fetchedContent.length,
    };

    // 写入缓存
    searchCache.set(cacheKey, output);
    logTool('web_search', `Completed: ${sorted.length} results, ${fetchedContent.length} fetched, mode=${mode}`);

    return output;
  }

  // ===== SearXNG 搜索 =====

  private async searchSearXNG(
    query: string,
    config: SearXNGConfig,
    maxResults: number,
    timeRange: string,
  ): Promise<{ results: SearchResult[]; engineStats: Record<string, string> }> {
    const results: SearchResult[] = [];
    const engineStats: Record<string, string> = {};
    const engineCounts = new Map<string, number>();
    const baseUrl = config.url.replace(/\/$/, '');
    const headers = buildSearXNGAuthHeaders(config.auth_key, config.auth_type);
    const tr = timeRange || config.time_range;

    // SearXNG 标准分页：每页由实例配置决定（通常 10 条），用 pageno 翻页直到达到 maxResults
    const maxPages = Math.ceil(maxResults / 5) + 1; // 保守估计，每页至少 5 条
    let page = 1;

    while (results.length < maxResults && page <= maxPages) {
      const params = new URLSearchParams();
      params.set('q', query);
      params.set('format', config.format || 'json');
      params.set('pageno', String(page));
      if (config.engines) params.set('engines', config.engines);
      if (config.language && config.language !== 'auto') params.set('language', config.language);
      params.set('safesearch', String(config.safesearch));
      if (tr) params.set('time_range', tr);

      const searchUrl = `${baseUrl}/search?${params.toString()}`;
      logTool('web_search', `[SearXNG] Fetching page ${page}: ${searchUrl}`);

      const pageResults: SearchResult[] = [];

      try {
        const response = await fetchWithTimeout(searchUrl, { headers }, 15_000);
        if (!response.ok) {
          throw new Error(`SearXNG API error: ${response.status} ${response.statusText}`);
        }

        if (config.format === 'html') {
          const html = await response.text();
          const matches = html.matchAll(/<a[^>]*href="(https?:\/\/[^"]+)"[^>]*>([\s\S]*?)<\/a>/gi);
          for (const m of matches) {
            const url = m[1];
            const title = m[2].replace(/<[^>]+>/g, '').trim();
            if (title && url && !url.includes(config.url)) {
              pageResults.push({ title, url, snippet: '', engine: 'searxng', weight: 85 });
            }
          }
        } else {
          const data = await response.json() as { results?: Array<Record<string, unknown>> };
          for (const item of data.results ?? []) {
            const url = item.url as string;
            const title = item.title as string;
            const snippet = item.content as string;
            const engine = (item.engine as string) ?? 'searxng';
            pageResults.push({ title, url, snippet: snippet ?? '', engine, weight: 85 });
            engineCounts.set(engine, (engineCounts.get(engine) ?? 0) + 1);
          }
        }
      } catch (err) {
        logTool('web_search', `[SearXNG] Page ${page} error: ${(err as Error).message}`);
        // 第一页就失败则抛出，后续页失败则用已获取的结果
        if (page === 1) throw err;
        break;
      }

      // 本页无新结果 → 已到末尾
      if (pageResults.length === 0) {
        logTool('web_search', `[SearXNG] Page ${page} returned 0 results, stopping pagination`);
        break;
      }

      results.push(...pageResults);
      logTool('web_search', `[SearXNG] Page ${page}: +${pageResults.length} (total ${results.length})`);
      page++;
    }

    for (const [engine, count] of engineCounts) {
      engineStats[engine] = `${count} 条`;
    }

    logTool('web_search', `[SearXNG] Total: ${results.length} raw results (target ${maxResults})`);
    return { results, engineStats };
  }

  // ===== 内置四引擎并行搜索 =====

  private async searchBuiltinEngines(
    query: string,
    maxResults: number,
    timeRange: string,
  ): Promise<{ results: SearchResult[]; engineStats: Record<string, string> }> {
    const searchPromises = ENGINES.map(async (engine) => {
      try {
        const url = engine.searchUrl(query, timeRange);
        const response = await fetchWithTimeout(url, {
          headers: {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36',
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
          },
        }, 8_000);

        if (!response.ok) {
          logTool('web_search', `[内置] ${engine.name} HTTP ${response.status}`);
          return [];
        }

        const html = await response.text();
        const parsed = engine.parse(html);
        logTool('web_search', `[内置] ${engine.name}: ${parsed.length} results`);
        return parsed;
      } catch (err) {
        logTool('web_search', `[内置] ${engine.name} error: ${(err as Error).message}`);
        return [];
      }
    });

    const settled = await Promise.allSettled(searchPromises);
    const allResults: SearchResult[] = [];
    const engineStats: Record<string, string> = {};

    for (let i = 0; i < ENGINES.length; i++) {
      const result = settled[i];
      const engineName = ENGINES[i].name;
      if (result.status === 'fulfilled') {
        allResults.push(...result.value);
        engineStats[engineName] = `${result.value.length} 条`;
      } else {
        engineStats[engineName] = '失败';
      }
    }

    return { results: allResults, engineStats };
  }

  // ===== 合并去重 =====

  private deduplicate(results: SearchResult[]): SearchResult[] {
    const seen = new Map<string, SearchResult>();
    for (const r of results) {
      const normalized = normalizeUrl(r.url);
      if (!seen.has(normalized)) {
        seen.set(normalized, r);
      }
    }
    return Array.from(seen.values());
  }

  // ===== 摘要增强（委托给 WebFetchTool） =====

  private async enhanceSnippets(results: SearchResult[], maxEnhance: number): Promise<void> {
    let enhanced = 0;
    for (const r of results) {
      if (enhanced >= maxEnhance) break;
      if (r.snippet.length < 30 && r.reachable) {
        try {
          const fetchResult = await this.webFetchTool.execute(
            { url: r.url },
            { sessionId: '', workspacePath: '', iteration: 0, requestId: '' },
          ) as { success: boolean; content?: string };

          if (fetchResult.success && fetchResult.content) {
            const text = fetchResult.content.slice(0, 200);
            if (text.length > r.snippet.length) {
              r.snippet = text;
              r._enhanced = true;
              enhanced++;
            }
          }
        } catch {
          // 忽略增强失败
        }
      }
    }
  }

  // ===== 自动抓取完整内容（委托给 WebFetchTool） =====

  /**
   * P2-12: 自动抓取改为并行（批次并发 3）
   *
   * 原实现逐条串行抓取（单个 web_fetch 最长 120s 超时），top5 结果最坏耗时
   * 逼近 web_search 的 300s 工具超时上限。并行批次化后总耗时约降至 1/3。
   * 失败结果直接跳过（原"随机补充重试"逻辑收益边际，复杂度高，已移除）。
   */
  private async autoFetch(
    query: string,
    results: SearchResult[],
    fetchMode: string,
    fetchTop: number,
  ): Promise<Array<{ url: string; title: string; content: string }>> {
    // 相关性评分（不过滤，relevance=0 的结果也参与抓取候选）
    const withRelevance = results.map((r) => ({ result: r, relevance: computeRelevance(query, r) }));
    const filtered = withRelevance.length > 0 ? withRelevance : [];

    // 确定抓取数量：fetchTop 已在 execute() 中综合了配置面板和工具参数
    const topN = Math.min(fetchTop, 8, filtered.length);

    // 构建抓取列表
    let toFetch: typeof filtered;
    if (fetchMode === 'random') {
      // Fisher-Yates 洗牌
      const shuffled = [...filtered];
      for (let i = shuffled.length - 1; i > 0; i--) {
        const j = Math.floor(Math.random() * (i + 1));
        [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
      }
      toFetch = shuffled.slice(0, topN);
    } else {
      toFetch = filtered
        .sort((a, b) => b.relevance - a.relevance)
        .slice(0, topN);
    }

    const fetched: Array<{ url: string; title: string; content: string }> = [];

    const fetchOne = async (item: { result: SearchResult }): Promise<{ url: string; title: string; content: string } | null> => {
      try {
        // 委托给 WebFetchTool — 享受三阶段回退策略（HTTP + 反爬 + 浏览器渲染）
        const fetchResult = await this.webFetchTool.execute(
          { url: item.result.url },
          { sessionId: '', workspacePath: '', iteration: 0, requestId: '' },
        ) as { success: boolean; content?: string };

        if (fetchResult.success && fetchResult.content) {
          return { url: item.result.url, title: item.result.title, content: fetchResult.content };
        }
        return null;
      } catch (err) {
        logTool('web_search', `Auto-fetch failed for ${item.result.url}: ${(err as Error).message}`);
        return null;
      }
    };

    // 并行批次抓取（并发 3）
    const CONCURRENCY = 3;
    for (let i = 0; i < toFetch.length; i += CONCURRENCY) {
      const batch = toFetch.slice(i, i + CONCURRENCY);
      const settled = await Promise.allSettled(batch.map((item) => fetchOne(item)));
      for (const r of settled) {
        if (r.status === 'fulfilled' && r.value) fetched.push(r.value);
      }
    }

    return fetched;
  }

  // ===== 格式化人类可读输出 =====

  private formatResults(query: string, results: SearchResult[]): string {
    if (results.length === 0) return `搜索 "${query}" 无结果。`;
    const lines = [`搜索 "${query}" — ${results.length} 条结果：\n`];
    results.forEach((r, i) => {
      lines.push(`${i + 1}. ${r.title}`);
      lines.push(`   URL: ${r.url}`);
      if (r.snippet) lines.push(`   摘要: ${r.snippet.slice(0, 150)}`);
      lines.push(`   来源: ${r.engine}${r.reachable === false ? ' (不可达)' : ''}${r._enhanced ? ' [已增强]' : ''}\n`);
    });
    return lines.join('\n');
  }

  // ===== 读取 SearXNG 配置 =====

  private readSearXNGConfig(): SearXNGConfig {
    const cs = this.configService;
    return {
      enabled: cs.get<boolean>('searxng.enabled') ?? false,
      url: cs.get<string>('searxng.url') ?? '',
      engines: cs.get<string>('searxng.engines') ?? '',
      language: cs.get<string>('searxng.language') ?? 'zh-CN',
      safesearch: cs.get<number>('searxng.safesearch') ?? 1,
      time_range: cs.get<string>('searxng.time_range') ?? '',
      max_results: cs.get<number>('searxng.max_results') ?? 0,
      auth_key: cs.get<string>('searxng.auth_key') ?? '',
      auth_type: cs.get<string>('searxng.auth_type') ?? 'bearer',
      format: cs.get<string>('searxng.format') ?? 'json',
      fetch_count: cs.get<number>('searxng.fetch_count') ?? 0,
      fetch_mode: cs.get<string>('searxng.fetch_mode') ?? 'sequential',
    };
  }
}