/** * web_search — 网络搜索工具 * * 双模式搜索:SearXNG 元搜索 / 内置四引擎并行搜索 * 内置引擎:Bing + 百度 + 搜狗 + 360 搜索(Promise.allSettled 容错并发) * 智能排序:引擎权重(50%) + 可达性(30%) + 摘要质量(20%) * 自动抓取:对前 N 条结果调用 web_fetch 获取完整正文 * * v0.4.1: HTML 解析迁移至 node-html-parser(结构化解析) * 主层使用 DOM 结构解析(引擎改版时选择器更精确、可维护性远优于正则), * 正则解析保留为降级路径(结构化解析无结果时兜底)。 * 此前纯正则方案违反项目开发规范第一铁律(HTML 解析应使用成熟库)。 * * @see docs/Agent网络工具通用设计-v2.md — 第 2 章 web_search 搜索设计 */ import { parse as parseHtmlDom, type HTMLElement } from 'node-html-parser'; import type { IMetonaTool, ToolExecutionContext } from '../../types/metona-tool'; import type { MetonaToolDef } from '../../../harness/types'; import { MetonaToolCategory, MetonaRiskLevel } from '../../../harness/types'; import type { ConfigService } from '../../../services/config.service'; import { searchCache, normalizeUrl, fetchWithTimeout, buildSearXNGAuthHeaders, logTool, } from './network-utils'; // v0.7.3 P2-1: 可达性预检经 SSRF 校验 + DNS pinning(结果 URL 是不可信外部输入) import { safeValidateSSRF, assertSafeConfigTargetDeep, DeepCheckSoftFailure } from './ssrf-guard'; import { ssrfPinnedFetch } from './ssrf-dispatcher'; import type { WebFetchTool } from './web-fetch'; // ===== 类型定义 ===== interface SearchResult { title: string; url: string; snippet: string; engine: string; weight: number; reachable?: boolean; _score?: number; _enhanced?: boolean; } interface SearXNGConfig { enabled: boolean; url: string; engines: string; language: string; safesearch: number; time_range: string; max_results: number; auth_key: string; auth_type: string; format: string; fetch_count: number; fetch_mode: string; } // ===== 引擎定义 ===== interface EngineDef { name: string; weight: number; searchUrl: (query: string, timeRange?: string) => string; parse: (html: string) => SearchResult[]; } const ENGINES: EngineDef[] = [ { name: 'bing', weight: 90, searchUrl: (q, tr) => { const freshness = tr ? `&filters=ex1:"ez${tr === 'day' ? '1' : tr === 'week' ? '2' : tr === 'month' ? '3' : '4'}"` : ''; return `https://www.bing.com/search?q=${encodeURIComponent(q)}${freshness}&count=20`; }, parse: parseBing, }, { name: '百度', weight: 80, searchUrl: (q) => `https://www.baidu.com/s?wd=${encodeURIComponent(q)}&rn=20`, parse: parseBaidu, }, { name: '搜狗', weight: 75, searchUrl: (q) => `https://www.sogou.com/web?query=${encodeURIComponent(q)}&num=20`, parse: parseSogou, }, { name: '360搜索', weight: 75, searchUrl: (q) => `https://www.so.com/s?q=${encodeURIComponent(q)}&pn=20`, parse: parse360, }, ]; // ===== HTML 解析器(v0.4.1: node-html-parser 结构化解析为主层,正则为降级层) ===== /** * v0.4.1: 从结果块中提取标题链接 — 跳过指向搜索引擎自身域名的链接(favicon/子导航等) */ function extractTitleLink( block: HTMLElement, selfDomain: string, ): { url: string; title: string } | null { for (const a of block.querySelectorAll('a[href]')) { const url = a.getAttribute('href') ?? ''; const title = a.text.trim(); if (title && url && !url.includes(selfDomain) && url.startsWith('http')) { return { url, title }; } } return null; } /** v0.4.1: 提取第一个非空文本的选择器(按优先级尝试多个候选选择器) */ function extractText(block: HTMLElement, selectors: string[]): string { for (const sel of selectors) { const el = block.querySelector(sel); if (el) { const text = el.text.trim(); if (text) return text; } } return ''; } /** v0.4.1: Bing 结构化解析 — li.b_algo 结果块 */ function parseBingStructured(html: string): SearchResult[] { const results: SearchResult[] = []; const root = parseHtmlDom(html); for (const block of root.querySelectorAll('li.b_algo')) { const link = extractTitleLink(block, 'bing.com'); if (!link) continue; const snippet = extractText(block, ['p', '.b_caption']); results.push({ title: link.title, url: link.url, snippet, engine: 'bing', weight: 90 }); } return results; } /** v0.4.1: 百度结构化解析 — div.result / div.c-container 结果块,优先 a[data-url] 真实链接 */ function parseBaiduStructured(html: string): SearchResult[] { const results: SearchResult[] = []; const root = parseHtmlDom(html); // 复合选择器去重:class="result c-container" 的元素同时命中两个类名, // 分别查询再拼接会重复收录同一结果块 const blocks = root.querySelectorAll('div.result, div.c-container'); for (const block of blocks) { // 百度标题链接: 优先 data-url 属性(真实目标 URL),href 通常是 baidu.com/link 跳转 const dataUrlLink = block.querySelector('a[data-url]'); let url = dataUrlLink?.getAttribute('data-url') ?? ''; let title = dataUrlLink?.text.trim() ?? ''; if (!url || !title) { const fallback = block.querySelector('h3 a[href]') ?? block.querySelector('a[href]'); if (fallback) { const href = fallback.getAttribute('href') ?? ''; url = href.startsWith('http') ? href : href ? `https://${href}` : ''; title = fallback.text.trim(); } } const snippet = extractText(block, ['.c-abstract', '[class^="content-right"]']); if (title && url && !url.includes('baidu.com/link')) { results.push({ title, url, snippet, engine: '百度', weight: 80 }); } } return results; } /** * v0.4.1: 搜狗结构化解析 — div.vrwrap / div.rb 结果块(相对链接补全 sogou.com 前缀) * * v0.4.1 修复(原正则实现遗留缺陷): 搜狗结果链接是 sogou.com/link?url=... 跳转形式, * 原 `!url.includes('sogou.com')` 过滤条件把所有跳转结果一并丢弃(相对链接补全后必含 sogou.com), * 导致搜狗引擎基本无法返回结果。现仅过滤 sogou 自身页面链接,保留 /link 跳转结果 * (可达性预检会跟随重定向验证)。 */ function parseSogouStructured(html: string): SearchResult[] { const results: SearchResult[] = []; const root = parseHtmlDom(html); // 复合选择器避免同一元素命中两个类名时重复收录 const blocks = root.querySelectorAll('div.vrwrap, div.rb'); for (const block of blocks) { const a = block.querySelector('h3 a[href]') ?? block.querySelector('a[href]'); if (!a) continue; const href = a.getAttribute('href') ?? ''; const url = href.startsWith('http') ? href : `https://www.sogou.com${href}`; const title = a.text.trim(); const snippet = extractText(block, ['.star-wiki', '.space-txt', '.str_info']); // 过滤搜狗自身页面(保留 /link 跳转结果) const isSelfPage = url.includes('sogou.com') && !url.includes('/link'); if (title && url && !isSelfPage) { results.push({ title, url, snippet, engine: '搜狗', weight: 75 }); } } return results; } /** v0.4.1: 360 结构化解析 — li.res-list / div.result 结果块 */ function parse360Structured(html: string): SearchResult[] { const results: SearchResult[] = []; const root = parseHtmlDom(html); // 复合选择器避免同一元素命中多个类名时重复收录 const blocks = root.querySelectorAll('li.res-list, div.result'); for (const block of blocks) { const link = extractTitleLink(block, 'so.com'); if (!link) continue; const snippet = extractText(block, ['.res-desc', '.res-rich', '.res-summary', 'dd']); results.push({ title: link.title, url: link.url, snippet, engine: '360搜索', weight: 75 }); } return results; } /** v0.4.1: 结构化解析 + 正则降级的组合入口(供 ENGINES 引用,测试导出) */ export function parseBing(html: string): SearchResult[] { const structured = parseBingStructured(html); if (structured.length > 0) return structured; return parseBingRegex(html); } export function parseBaidu(html: string): SearchResult[] { const structured = parseBaiduStructured(html); if (structured.length > 0) return structured; return parseBaiduRegex(html); } export function parseSogou(html: string): SearchResult[] { const structured = parseSogouStructured(html); if (structured.length > 0) return structured; return parseSogouRegex(html); } export function parse360(html: string): SearchResult[] { const structured = parse360Structured(html); if (structured.length > 0) return structured; return parse360Regex(html); } // ===== 正则降级解析器(v0.4.1 前的主实现,结构化解析无结果时兜底) ===== function parseBingRegex(html: string): SearchResult[] { const results: SearchResult[] = []; const blocks = html.split(/
]*>([\s\S]*?)<\/p>/i) || block.match(/class="b_caption"[^>]*>([\s\S]*?)<\/div>/i); const snippet = snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '').trim() : ''; if (title && url && !url.includes('bing.com')) { results.push({ title, url, snippet, engine: 'bing', weight: 90 }); } } return results; } function parseBaiduRegex(html: string): SearchResult[] { const results: SearchResult[] = []; const blocks = html.split(/