/** * web_search — 网络搜索工具 * * 双模式搜索:SearXNG 元搜索 / 内置四引擎并行搜索 * 内置引擎:Bing + 百度 + 搜狗 + 360 搜索(Promise.allSettled 容错并发) * 智能排序:引擎权重(50%) + 可达性(30%) + 摘要质量(20%) * 自动抓取:对前 N 条结果调用 web_fetch 获取完整正文 * * v0.4.1: HTML 解析迁移至 node-html-parser(结构化解析) * 主层使用 DOM 结构解析(引擎改版时选择器更精确、可维护性远优于正则), * 正则解析保留为降级路径(结构化解析无结果时兜底)。 * 此前纯正则方案违反项目开发规范第一铁律(HTML 解析应使用成熟库)。 * * @see docs/Agent网络工具通用设计-v2.md — 第 2 章 web_search 搜索设计 */ import { parse as parseHtmlDom, type HTMLElement } from 'node-html-parser'; import type { IMetonaTool, ToolExecutionContext } from '../../types/metona-tool'; import type { MetonaToolDef } from '../../../harness/types'; import { MetonaToolCategory, MetonaRiskLevel } from '../../../harness/types'; import type { ConfigService } from '../../../services/config.service'; import { searchCache, normalizeUrl, fetchWithTimeout, buildSearXNGAuthHeaders, logTool, } from './network-utils'; // v0.7.3 P2-1: 可达性预检经 SSRF 校验 + DNS pinning(结果 URL 是不可信外部输入) import { safeValidateSSRF, assertSafeConfigTargetDeep, DeepCheckSoftFailure } from './ssrf-guard'; import { ssrfPinnedFetch } from './ssrf-dispatcher'; import type { WebFetchTool } from './web-fetch'; // ===== 类型定义 ===== interface SearchResult { title: string; url: string; snippet: string; engine: string; weight: number; reachable?: boolean; _score?: number; _enhanced?: boolean; } interface SearXNGConfig { enabled: boolean; url: string; engines: string; language: string; safesearch: number; time_range: string; max_results: number; auth_key: string; auth_type: string; format: string; fetch_count: number; fetch_mode: string; } // ===== 引擎定义 ===== interface EngineDef { name: string; weight: number; searchUrl: (query: string, timeRange?: string) => string; parse: (html: string) => SearchResult[]; } const ENGINES: EngineDef[] = [ { name: 'bing', weight: 90, searchUrl: (q, tr) => { const freshness = tr ? `&filters=ex1:"ez${tr === 'day' ? '1' : tr === 'week' ? '2' : tr === 'month' ? '3' : '4'}"` : ''; return `https://www.bing.com/search?q=${encodeURIComponent(q)}${freshness}&count=20`; }, parse: parseBing, }, { name: '百度', weight: 80, searchUrl: (q) => `https://www.baidu.com/s?wd=${encodeURIComponent(q)}&rn=20`, parse: parseBaidu, }, { name: '搜狗', weight: 75, searchUrl: (q) => `https://www.sogou.com/web?query=${encodeURIComponent(q)}&num=20`, parse: parseSogou, }, { name: '360搜索', weight: 75, searchUrl: (q) => `https://www.so.com/s?q=${encodeURIComponent(q)}&pn=20`, parse: parse360, }, ]; // ===== HTML 解析器(v0.4.1: node-html-parser 结构化解析为主层,正则为降级层) ===== /** * v0.4.1: 从结果块中提取标题链接 — 跳过指向搜索引擎自身域名的链接(favicon/子导航等) */ function extractTitleLink( block: HTMLElement, selfDomain: string, ): { url: string; title: string } | null { for (const a of block.querySelectorAll('a[href]')) { const url = a.getAttribute('href') ?? ''; const title = a.text.trim(); if (title && url && !url.includes(selfDomain) && url.startsWith('http')) { return { url, title }; } } return null; } /** v0.4.1: 提取第一个非空文本的选择器(按优先级尝试多个候选选择器) */ function extractText(block: HTMLElement, selectors: string[]): string { for (const sel of selectors) { const el = block.querySelector(sel); if (el) { const text = el.text.trim(); if (text) return text; } } return ''; } /** v0.4.1: Bing 结构化解析 — li.b_algo 结果块 */ function parseBingStructured(html: string): SearchResult[] { const results: SearchResult[] = []; const root = parseHtmlDom(html); for (const block of root.querySelectorAll('li.b_algo')) { const link = extractTitleLink(block, 'bing.com'); if (!link) continue; const snippet = extractText(block, ['p', '.b_caption']); results.push({ title: link.title, url: link.url, snippet, engine: 'bing', weight: 90 }); } return results; } /** v0.4.1: 百度结构化解析 — div.result / div.c-container 结果块,优先 a[data-url] 真实链接 */ function parseBaiduStructured(html: string): SearchResult[] { const results: SearchResult[] = []; const root = parseHtmlDom(html); // 复合选择器去重:class="result c-container" 的元素同时命中两个类名, // 分别查询再拼接会重复收录同一结果块 const blocks = root.querySelectorAll('div.result, div.c-container'); for (const block of blocks) { // 百度标题链接: 优先 data-url 属性(真实目标 URL),href 通常是 baidu.com/link 跳转 const dataUrlLink = block.querySelector('a[data-url]'); let url = dataUrlLink?.getAttribute('data-url') ?? ''; let title = dataUrlLink?.text.trim() ?? ''; if (!url || !title) { const fallback = block.querySelector('h3 a[href]') ?? block.querySelector('a[href]'); if (fallback) { const href = fallback.getAttribute('href') ?? ''; url = href.startsWith('http') ? href : href ? `https://${href}` : ''; title = fallback.text.trim(); } } const snippet = extractText(block, ['.c-abstract', '[class^="content-right"]']); if (title && url && !url.includes('baidu.com/link')) { results.push({ title, url, snippet, engine: '百度', weight: 80 }); } } return results; } /** * v0.4.1: 搜狗结构化解析 — div.vrwrap / div.rb 结果块(相对链接补全 sogou.com 前缀) * * v0.4.1 修复(原正则实现遗留缺陷): 搜狗结果链接是 sogou.com/link?url=... 跳转形式, * 原 `!url.includes('sogou.com')` 过滤条件把所有跳转结果一并丢弃(相对链接补全后必含 sogou.com), * 导致搜狗引擎基本无法返回结果。现仅过滤 sogou 自身页面链接,保留 /link 跳转结果 * (可达性预检会跟随重定向验证)。 */ function parseSogouStructured(html: string): SearchResult[] { const results: SearchResult[] = []; const root = parseHtmlDom(html); // 复合选择器避免同一元素命中两个类名时重复收录 const blocks = root.querySelectorAll('div.vrwrap, div.rb'); for (const block of blocks) { const a = block.querySelector('h3 a[href]') ?? block.querySelector('a[href]'); if (!a) continue; const href = a.getAttribute('href') ?? ''; const url = href.startsWith('http') ? href : `https://www.sogou.com${href}`; const title = a.text.trim(); const snippet = extractText(block, ['.star-wiki', '.space-txt', '.str_info']); // 过滤搜狗自身页面(保留 /link 跳转结果) const isSelfPage = url.includes('sogou.com') && !url.includes('/link'); if (title && url && !isSelfPage) { results.push({ title, url, snippet, engine: '搜狗', weight: 75 }); } } return results; } /** v0.4.1: 360 结构化解析 — li.res-list / div.result 结果块 */ function parse360Structured(html: string): SearchResult[] { const results: SearchResult[] = []; const root = parseHtmlDom(html); // 复合选择器避免同一元素命中多个类名时重复收录 const blocks = root.querySelectorAll('li.res-list, div.result'); for (const block of blocks) { const link = extractTitleLink(block, 'so.com'); if (!link) continue; const snippet = extractText(block, ['.res-desc', '.res-rich', '.res-summary', 'dd']); results.push({ title: link.title, url: link.url, snippet, engine: '360搜索', weight: 75 }); } return results; } /** v0.4.1: 结构化解析 + 正则降级的组合入口(供 ENGINES 引用,测试导出) */ export function parseBing(html: string): SearchResult[] { const structured = parseBingStructured(html); if (structured.length > 0) return structured; return parseBingRegex(html); } export function parseBaidu(html: string): SearchResult[] { const structured = parseBaiduStructured(html); if (structured.length > 0) return structured; return parseBaiduRegex(html); } export function parseSogou(html: string): SearchResult[] { const structured = parseSogouStructured(html); if (structured.length > 0) return structured; return parseSogouRegex(html); } export function parse360(html: string): SearchResult[] { const structured = parse360Structured(html); if (structured.length > 0) return structured; return parse360Regex(html); } // ===== 正则降级解析器(v0.4.1 前的主实现,结构化解析无结果时兜底) ===== function parseBingRegex(html: string): SearchResult[] { const results: SearchResult[] = []; const blocks = html.split(/]*class="b_algo"/i).slice(1); for (const block of blocks) { const titleMatch = block.match(/]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i); if (!titleMatch) continue; const url = titleMatch[1]; const title = titleMatch[2].replace(/<[^>]+>/g, '').trim(); const snippetMatch = block.match(/]*>([\s\S]*?)<\/p>/i) || block.match(/class="b_caption"[^>]*>([\s\S]*?)<\/div>/i); const snippet = snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '').trim() : ''; if (title && url && !url.includes('bing.com')) { results.push({ title, url, snippet, engine: 'bing', weight: 90 }); } } return results; } function parseBaiduRegex(html: string): SearchResult[] { const results: SearchResult[] = []; const blocks = html.split(/]*class="result[^"]*"/i).slice(1); for (const block of blocks) { const titleMatch = block.match(/]*data-url="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i) || block.match(/]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i); if (!titleMatch) continue; const url = titleMatch[1].startsWith('http') ? titleMatch[1] : `https://${titleMatch[1]}`; const title = titleMatch[2].replace(/<[^>]+>/g, '').trim(); const snippetMatch = block.match(/class="c-abstract[^"]*"[^>]*>([\s\S]*?)<\/span>/i) || block.match(/class="content-right[^"]*"[^>]*>([\s\S]*?)<\/div>/i); const snippet = snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '').trim() : ''; if (title && url && !url.includes('baidu.com/link')) { results.push({ title, url, snippet, engine: '百度', weight: 80 }); } } return results; } function parseSogouRegex(html: string): SearchResult[] { const results: SearchResult[] = []; const blocks = html .split(/]*class="vrwrap"/i) .slice(1) .concat(html.split(/]*class="rb"/i).slice(1)); for (const block of blocks) { const titleMatch = block.match(/]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i); if (!titleMatch) continue; const url = titleMatch[1].startsWith('http') ? titleMatch[1] : `https://www.sogou.com${titleMatch[1]}`; const title = titleMatch[2].replace(/<[^>]+>/g, '').trim(); const snippetMatch = block.match(/class="star-wiki[^"]*"[^>]*>([\s\S]*?)<\/div>/i) || block.match(/class="space-txt[^"]*"[^>]*>([\s\S]*?)<\/p>/i) || block.match(/class="str_info[^"]*"[^>]*>([\s\S]*?)<\/p>/i); const snippet = snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '').trim() : ''; if (title && url && !url.includes('sogou.com')) { results.push({ title, url, snippet, engine: '搜狗', weight: 75 }); } } return results; } function parse360Regex(html: string): SearchResult[] { const results: SearchResult[] = []; const blocks = html .split(/]*class="res-list"/i) .slice(1) .concat(html.split(/]*class="result"/i).slice(1)); for (const block of blocks) { const titleMatch = block.match(/]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i); if (!titleMatch) continue; const url = titleMatch[1]; const title = titleMatch[2].replace(/<[^>]+>/g, '').trim(); const snippetMatch = block.match(/class="res-desc[^"]*"[^>]*>([\s\S]*?)<\/p>/i) || block.match(/class="res-rich[^"]*"[^>]*>([\s\S]*?)<\/div>/i) || block.match(/class="res-summary[^"]*"[^>]*>([\s\S]*?)<\/p>/i) || block.match(/]*>([\s\S]*?)<\/dd>/i); const snippet = snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '').trim() : ''; if (title && url && !url.includes('so.com')) { results.push({ title, url, snippet, engine: '360搜索', weight: 75 }); } } return results; } // ===== 可达性预检 ===== // v0.7.3 P2-1: 可达性预检的 URL 来自不可信的搜索结果 —— HEAD 探测同样不得 // 触达内网/元数据地址。校验失败的 URL 直接标记不可达(不发起任何请求); // 探测经 ssrfPinnedFetch(DNS pinning),重定向不自动跟随(3xx 即视为可达 —— // 链接活性已证明,且跟跳目标不再绕过校验)。 async function checkReachability( urls: string[], signal?: AbortSignal, concurrency = 5, ): Promise> { const result = new Map(); for (let i = 0; i < urls.length; i += concurrency) { // v0.8.0 P1-2: abort 后立即终止预检批次 if (signal?.aborted) break; const batch = urls.slice(i, i + concurrency); const checks = batch.map(async (url) => { try { const ssrf = await safeValidateSSRF(url); if (!ssrf.ok) { result.set(url, false); return; } // v0.8.0 P1-2: 透传引擎级 abort signal const resp = await ssrfPinnedFetch( url, { method: 'HEAD', redirect: 'manual' }, 3_000, signal, ); result.set(url, resp.ok || (resp.status >= 300 && resp.status < 400)); } catch { result.set(url, false); } }); await Promise.allSettled(checks); } return result; } // ===== 智能排序 ===== function smartSort(results: SearchResult[]): SearchResult[] { for (const r of results) { const reachability = r.reachable ? 30 : -20; const snippetQuality = (Math.min(r.snippet.length, 100) / 100) * 20; const weightScore = (r.weight / 100) * 50; r._score = weightScore + reachability + snippetQuality; } return results.sort((a, b) => (b._score ?? 0) - (a._score ?? 0)); } // ===== 相关性过滤(用于自动抓取) ===== function computeRelevance(query: string, result: SearchResult): number { const terms: string[] = []; // 中文双字/三字片段 const chinese = query.match(/[\u4e00-\u9fa5]{2,3}/g); if (chinese) terms.push(...chinese); // 英文单词 const english = query.match(/[a-zA-Z]{2,}/g); if (english) terms.push(...english); let score = 0; const text = `${result.title} ${result.snippet}`.toLowerCase(); for (const term of terms) { if (text.includes(term.toLowerCase())) score += 10; } return Math.min(score, 100); } // ===== WebSearchTool ===== export class WebSearchTool implements IMetonaTool { readonly definition: MetonaToolDef = { name: 'web_search', description: 'Search the web for information. Returns titles, snippets, and URLs. When SearXNG is enabled, uses the configured SearXNG instance; otherwise uses built-in engines (Bing, Baidu, Sogou, 360). Automatically fetches full content for top results.', parameters: { type: 'object', properties: { query: { type: 'string', description: 'Search query keywords' }, time_range: { type: 'string', description: 'Time filter: day, week, month, year (optional)', }, enhance_snippets: { type: 'boolean', description: 'Auto-enhance short snippets (default true)', }, }, required: ['query'], }, category: MetonaToolCategory.NETWORK, riskLevel: MetonaRiskLevel.LOW, requiresPermission: false, timeoutMs: 300_000, }; constructor( private configService: ConfigService, private webFetchTool: WebFetchTool, ) {} async execute(args: Record, context: ToolExecutionContext): Promise { const query = args.query as string; // v0.8.0 P1-2: abort 立即生效(后续各阶段均携带本 signal) if (context.signal?.aborted) { return { success: false, query, results: [], error: 'Search aborted' }; } // 读取 SearXNG 配置 const searxngConfig = this.readSearXNGConfig(); const useSearXNG = searxngConfig.enabled && !!searxngConfig.url; // 两个模式参数完全独立: // - 内置模式:固定默认值 maxResults=30, fetchTop=5 // - SearXNG 模式:由配置面板控制 const maxResults = useSearXNG ? Math.min(50, Math.max(1, searxngConfig.max_results > 0 ? searxngConfig.max_results : 30)) : 30; const timeRange = (args.time_range as string) ?? ''; const enhanceSnippets = (args.enhance_snippets as boolean) ?? true; const fetchTop = useSearXNG ? Math.min(8, Math.max(3, searxngConfig.fetch_count > 0 ? searxngConfig.fetch_count : 5)) : 5; logTool( 'web_search', `Mode=${useSearXNG ? 'searxng' : 'builtin'}, maxResults=${maxResults}, fetchTop=${fetchTop}`, ); // 缓存检查(key 含模式 + maxResults + fetchTop,避免配置变更后返回旧缓存) const cacheKey = `${searxngConfig.enabled ? 'searxng' : 'builtin'}:${maxResults}:${fetchTop}:${normalizeUrl(query).toLowerCase()}`; const cached = searchCache.get(cacheKey); if (cached) { logTool('web_search', `Cache hit: "${query}"`); return { ...(cached as Record), from_cache: true }; } let results: SearchResult[]; let mode: string; let engineStats: Record; if (useSearXNG) { const searxResult = await this.searchSearXNG( query, searxngConfig, maxResults, timeRange, context.signal, ); results = searxResult.results; mode = 'searxng'; engineStats = searxResult.engineStats; } else { const builtinResult = await this.searchBuiltinEngines( query, maxResults, timeRange, context.signal, ); results = builtinResult.results; mode = 'builtin'; engineStats = builtinResult.engineStats; } // 合并去重 const deduped = this.deduplicate(results); // 可达性预检 const topUrls = deduped.slice(0, Math.min(20, deduped.length)).map((r) => r.url); const reachabilityMap = await checkReachability(topUrls, context.signal); for (const r of deduped) { r.reachable = reachabilityMap.get(r.url) ?? false; } // 智能排序 const sorted = smartSort(deduped).slice(0, maxResults); // 摘要增强 if (enhanceSnippets) { await this.enhanceSnippets(sorted, 3, context); } // 自动抓取完整内容 const fetchedContent = await this.autoFetch( query, sorted, searxngConfig.fetch_mode, fetchTop, context, ); // 格式化输出 const formatted = this.formatResults(query, sorted); const output = { success: true, query, results: sorted.map((r) => ({ title: r.title, url: r.url, snippet: r.snippet, engine: r.engine, reachable: r.reachable, _score: r._score ? Math.round(r._score * 10) / 10 : undefined, _enhanced: r._enhanced, })), total: sorted.length, formatted, from_cache: false, engine_stats: engineStats, _mode: mode, _fetched: fetchedContent.map((f) => ({ url: f.url, title: f.title, content: f.content })), _fetched_count: fetchedContent.length, }; // 写入缓存 searchCache.set(cacheKey, output); logTool( 'web_search', `Completed: ${sorted.length} results, ${fetchedContent.length} fetched, mode=${mode}`, ); return output; } // ===== SearXNG 搜索 ===== private async searchSearXNG( query: string, config: SearXNGConfig, maxResults: number, timeRange: string, signal?: AbortSignal, ): Promise<{ results: SearchResult[]; engineStats: Record }> { const results: SearchResult[] = []; const engineStats: Record = {}; const engineCounts = new Map(); const baseUrl = config.url.replace(/\/$/, ''); const headers = buildSearXNGAuthHeaders(config.auth_key, config.auth_type); const tr = timeRange || config.time_range; // v0.8.2 P1-6: SearXNG 运行时请求的 SSRF 纵深校验。 // 配置期已有 assertSafeConfigTarget(ipc/shared 写入链),但 DNS 记录可在配置 // 之后被切换(指向云元数据/链路本地)—— 运行时请求此前完全无校验。此处对 // 每次搜索会话做同口径静态校验 + DNS 深校验;本地回环/RFC1918 合法放行 // (SearXNG 常部署本机/内网),DNS 解析失败按 DeepCheckSoftFailure 留痕放行 // (离线实例合法,与配置期深校验语义一致)。 try { await assertSafeConfigTargetDeep(baseUrl); } catch (err) { if (err instanceof DeepCheckSoftFailure) { logTool('web_search', `[SearXNG] DNS deep check skipped (soft-fail): ${err.message}`); } else { throw new Error(`SearXNG target blocked by security policy: ${(err as Error).message}`); } } // SearXNG 标准分页:每页由实例配置决定(通常 10 条),用 pageno 翻页直到达到 maxResults const maxPages = Math.ceil(maxResults / 5) + 1; // 保守估计,每页至少 5 条 let page = 1; while (results.length < maxResults && page <= maxPages) { // v0.8.0 P1-2: abort 后停止翻页 if (signal?.aborted) break; const params = new URLSearchParams(); params.set('q', query); params.set('format', config.format || 'json'); params.set('pageno', String(page)); if (config.engines) params.set('engines', config.engines); if (config.language && config.language !== 'auto') params.set('language', config.language); params.set('safesearch', String(config.safesearch)); if (tr) params.set('time_range', tr); const searchUrl = `${baseUrl}/search?${params.toString()}`; logTool('web_search', `[SearXNG] Fetching page ${page}: ${searchUrl}`); const pageResults: SearchResult[] = []; try { // v0.8.0 P1-2: SearXNG 请求携带引擎级 abort signal const response = await fetchWithTimeout(searchUrl, { headers }, 15_000, signal); if (!response.ok) { throw new Error(`SearXNG API error: ${response.status} ${response.statusText}`); } if (config.format === 'html') { const html = await response.text(); const matches = html.matchAll(/]*href="(https?:\/\/[^"]+)"[^>]*>([\s\S]*?)<\/a>/gi); for (const m of matches) { const url = m[1]; const title = m[2].replace(/<[^>]+>/g, '').trim(); if (title && url && !url.includes(config.url)) { pageResults.push({ title, url, snippet: '', engine: 'searxng', weight: 85 }); } } } else { const data = (await response.json()) as { results?: Array> }; for (const item of data.results ?? []) { const url = item.url as string; const title = item.title as string; const snippet = item.content as string; const engine = (item.engine as string) ?? 'searxng'; pageResults.push({ title, url, snippet: snippet ?? '', engine, weight: 85 }); engineCounts.set(engine, (engineCounts.get(engine) ?? 0) + 1); } } } catch (err) { logTool('web_search', `[SearXNG] Page ${page} error: ${(err as Error).message}`); // 第一页就失败则抛出,后续页失败则用已获取的结果 if (page === 1) throw err; break; } // 本页无新结果 → 已到末尾 if (pageResults.length === 0) { logTool('web_search', `[SearXNG] Page ${page} returned 0 results, stopping pagination`); break; } results.push(...pageResults); logTool( 'web_search', `[SearXNG] Page ${page}: +${pageResults.length} (total ${results.length})`, ); page++; } for (const [engine, count] of engineCounts) { engineStats[engine] = `${count} 条`; } logTool('web_search', `[SearXNG] Total: ${results.length} raw results (target ${maxResults})`); return { results, engineStats }; } // ===== 内置四引擎并行搜索 ===== private async searchBuiltinEngines( query: string, maxResults: number, timeRange: string, signal?: AbortSignal, ): Promise<{ results: SearchResult[]; engineStats: Record }> { const searchPromises = ENGINES.map(async (engine) => { try { const url = engine.searchUrl(query, timeRange); const response = await fetchWithTimeout( url, { headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36', 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8', }, }, 8_000, signal, ); if (!response.ok) { logTool('web_search', `[内置] ${engine.name} HTTP ${response.status}`); return []; } const html = await response.text(); const parsed = engine.parse(html); logTool('web_search', `[内置] ${engine.name}: ${parsed.length} results`); return parsed; } catch (err) { logTool('web_search', `[内置] ${engine.name} error: ${(err as Error).message}`); return []; } }); const settled = await Promise.allSettled(searchPromises); const allResults: SearchResult[] = []; const engineStats: Record = {}; for (let i = 0; i < ENGINES.length; i++) { const result = settled[i]; const engineName = ENGINES[i].name; if (result.status === 'fulfilled') { allResults.push(...result.value); engineStats[engineName] = `${result.value.length} 条`; } else { engineStats[engineName] = '失败'; } } return { results: allResults, engineStats }; } // ===== 合并去重 ===== private deduplicate(results: SearchResult[]): SearchResult[] { const seen = new Map(); for (const r of results) { const normalized = normalizeUrl(r.url); if (!seen.has(normalized)) { seen.set(normalized, r); } } return Array.from(seen.values()); } // ===== 摘要增强(委托给 WebFetchTool) ===== private async enhanceSnippets( results: SearchResult[], maxEnhance: number, context: ToolExecutionContext, ): Promise { let enhanced = 0; for (const r of results) { // v0.8.0 P1-2: abort 后停止增强 if (context.signal?.aborted) break; if (enhanced >= maxEnhance) break; if (r.snippet.length < 30 && r.reachable) { try { // v0.8.0 P1-2: 透传真实执行上下文(signal)—— 移除"伪造空上下文" const fetchResult = (await this.webFetchTool.execute({ url: r.url }, context)) as { success: boolean; content?: string; }; if (fetchResult.success && fetchResult.content) { const text = fetchResult.content.slice(0, 200); if (text.length > r.snippet.length) { r.snippet = text; r._enhanced = true; enhanced++; } } } catch { // 忽略增强失败 } } } } // ===== 自动抓取完整内容(委托给 WebFetchTool) ===== /** * P2-12: 自动抓取改为并行(批次并发 3) * * 原实现逐条串行抓取(单个 web_fetch 最长 120s 超时),top5 结果最坏耗时 * 逼近 web_search 的 300s 工具超时上限。并行批次化后总耗时约降至 1/3。 * 失败结果直接跳过(原"随机补充重试"逻辑收益边际,复杂度高,已移除)。 */ private async autoFetch( query: string, results: SearchResult[], fetchMode: string, fetchTop: number, context: ToolExecutionContext, ): Promise> { // v0.8.0 P1-2 根治: 时间预算收敛 —— 旧实现最坏耗时(8 条 × 3 并发 × 单次 // web_fetch 240s ≈ 720s)远超工具自身 300s 超时,registry 层超时后整次调用 // 作废(已抓取内容全部丢弃)。现以 deadline 为界:剩余预算不足一次最小 // 抓取(30s)即停,已抓取内容照常返回。 const BUDGET_RATIO = 0.8; const MIN_FETCH_BUDGET_MS = 30_000; const deadlineMs = Date.now() + 300_000 * BUDGET_RATIO; // 相关性评分(不过滤,relevance=0 的结果也参与抓取候选) const filtered = results.map((r) => ({ result: r, relevance: computeRelevance(query, r), })); // 确定抓取数量:fetchTop 已在 execute() 中综合了配置面板和工具参数 const topN = Math.min(fetchTop, 8, filtered.length); // 构建抓取列表 let toFetch: typeof filtered; if (fetchMode === 'random') { // Fisher-Yates 洗牌 const shuffled = [...filtered]; for (let i = shuffled.length - 1; i > 0; i--) { const j = Math.floor(Math.random() * (i + 1)); [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]]; } toFetch = shuffled.slice(0, topN); } else { toFetch = filtered.sort((a, b) => b.relevance - a.relevance).slice(0, topN); } const fetched: Array<{ url: string; title: string; content: string }> = []; const fetchOne = async (item: { result: SearchResult; }): Promise<{ url: string; title: string; content: string } | null> => { try { // 委托给 WebFetchTool — 享受三阶段回退策略(HTTP + 反爬 + 浏览器渲染) // v0.8.0 P1-2: 透传真实执行上下文(signal)—— 移除"伪造空上下文" const fetchResult = (await this.webFetchTool.execute( { url: item.result.url }, context, )) as { success: boolean; content?: string }; if (fetchResult.success && fetchResult.content) { return { url: item.result.url, title: item.result.title, content: fetchResult.content }; } return null; } catch (err) { logTool( 'web_search', `Auto-fetch failed for ${item.result.url}: ${(err as Error).message}`, ); return null; } }; // 并行批次抓取(并发 3,时间预算收敛) const CONCURRENCY = 3; for (let i = 0; i < toFetch.length; i += CONCURRENCY) { // v0.8.0 P1-2: 剩余预算不足一次最小抓取 → 停止,保留已抓取内容 const remaining = deadlineMs - Date.now(); if (context.signal?.aborted || remaining < MIN_FETCH_BUDGET_MS) { logTool( 'web_search', `Auto-fetch budget stop: fetched=${fetched.length}, remaining=${Math.max(0, remaining)}ms, aborted=${!!context.signal?.aborted}`, ); break; } const batch = toFetch.slice(i, i + CONCURRENCY); const settled = await Promise.allSettled(batch.map((item) => fetchOne(item))); for (const r of settled) { if (r.status === 'fulfilled' && r.value) fetched.push(r.value); } } return fetched; } // ===== 格式化人类可读输出 ===== private formatResults(query: string, results: SearchResult[]): string { if (results.length === 0) return `搜索 "${query}" 无结果。`; const lines = [`搜索 "${query}" — ${results.length} 条结果:\n`]; results.forEach((r, i) => { lines.push(`${i + 1}. ${r.title}`); lines.push(` URL: ${r.url}`); if (r.snippet) lines.push(` 摘要: ${r.snippet.slice(0, 150)}`); lines.push( ` 来源: ${r.engine}${r.reachable === false ? ' (不可达)' : ''}${r._enhanced ? ' [已增强]' : ''}\n`, ); }); return lines.join('\n'); } // ===== 读取 SearXNG 配置 ===== private readSearXNGConfig(): SearXNGConfig { const cs = this.configService; return { enabled: cs.get('searxng.enabled') ?? false, url: cs.get('searxng.url') ?? '', engines: cs.get('searxng.engines') ?? '', language: cs.get('searxng.language') ?? 'zh-CN', safesearch: cs.get('searxng.safesearch') ?? 1, time_range: cs.get('searxng.time_range') ?? '', max_results: cs.get('searxng.max_results') ?? 0, auth_key: cs.get('searxng.auth_key') ?? '', auth_type: cs.get('searxng.auth_type') ?? 'bearer', format: cs.get('searxng.format') ?? 'json', fetch_count: cs.get('searxng.fetch_count') ?? 0, fetch_mode: cs.get('searxng.fetch_mode') ?? 'sequential', }; } }