/** * web_search — 网络搜索工具 * * 双模式搜索:SearXNG 元搜索 / 内置四引擎并行搜索 * 内置引擎:Bing + 百度 + 搜狗 + 360 搜索(Promise.allSettled 容错并发) * 智能排序:引擎权重(50%) + 可达性(30%) + 摘要质量(20%) * 自动抓取:对前 N 条结果调用 web_fetch 获取完整正文 * * @see docs/Agent网络工具通用设计-v2.md — 第 2 章 web_search 搜索设计 */ import type { IMetonaTool, ToolExecutionContext } from '../../types/metona-tool'; import type { MetonaToolDef } from '../../../harness/types'; import { MetonaToolCategory, MetonaRiskLevel } from '../../../harness/types'; import type { ConfigService } from '../../../services/config.service'; import { searchCache, normalizeUrl, fetchWithTimeout, buildSearXNGAuthHeaders, logTool, } from './network-utils'; import type { WebFetchTool } from './web-fetch'; // ===== 类型定义 ===== interface SearchResult { title: string; url: string; snippet: string; engine: string; weight: number; reachable?: boolean; _score?: number; _enhanced?: boolean; } interface SearXNGConfig { enabled: boolean; url: string; engines: string; language: string; safesearch: number; time_range: string; max_results: number; auth_key: string; auth_type: string; format: string; fetch_count: number; fetch_mode: string; } // ===== 引擎定义 ===== interface EngineDef { name: string; weight: number; searchUrl: (query: string, timeRange?: string) => string; parse: (html: string) => SearchResult[]; } const ENGINES: EngineDef[] = [ { name: 'bing', weight: 90, searchUrl: (q, tr) => { const freshness = tr ? `&filters=ex1:"ez${tr === 'day' ? '1' : tr === 'week' ? '2' : tr === 'month' ? '3' : '4'}"` : ''; return `https://www.bing.com/search?q=${encodeURIComponent(q)}${freshness}&count=20`; }, parse: parseBing, }, { name: '百度', weight: 80, searchUrl: (q) => `https://www.baidu.com/s?wd=${encodeURIComponent(q)}&rn=20`, parse: parseBaidu, }, { name: '搜狗', weight: 75, searchUrl: (q) => `https://www.sogou.com/web?query=${encodeURIComponent(q)}&num=20`, parse: parseSogou, }, { name: '360搜索', weight: 75, searchUrl: (q) => `https://www.so.com/s?q=${encodeURIComponent(q)}&pn=20`, parse: parse360, }, ]; // ===== HTML 解析器(正则实现,后续可迁移至 cheerio) ===== function parseBing(html: string): SearchResult[] { const results: SearchResult[] = []; const blocks = html.split(/]*class="b_algo"/i).slice(1); for (const block of blocks) { const titleMatch = block.match(/]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i); if (!titleMatch) continue; const url = titleMatch[1]; const title = titleMatch[2].replace(/<[^>]+>/g, '').trim(); const snippetMatch = block.match(/]*>([\s\S]*?)<\/p>/i) || block.match(/class="b_caption"[^>]*>([\s\S]*?)<\/div>/i); const snippet = snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '').trim() : ''; if (title && url && !url.includes('bing.com')) { results.push({ title, url, snippet, engine: 'bing', weight: 90 }); } } return results; } function parseBaidu(html: string): SearchResult[] { const results: SearchResult[] = []; const blocks = html.split(/]*class="result[^"]*"/i).slice(1); for (const block of blocks) { const titleMatch = block.match(/]*data-url="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i) || block.match(/]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i); if (!titleMatch) continue; const url = titleMatch[1].startsWith('http') ? titleMatch[1] : `https://${titleMatch[1]}`; const title = titleMatch[2].replace(/<[^>]+>/g, '').trim(); const snippetMatch = block.match(/class="c-abstract[^"]*"[^>]*>([\s\S]*?)<\/span>/i) || block.match(/class="content-right[^"]*"[^>]*>([\s\S]*?)<\/div>/i); const snippet = snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '').trim() : ''; if (title && url && !url.includes('baidu.com/link')) { results.push({ title, url, snippet, engine: '百度', weight: 80 }); } } return results; } function parseSogou(html: string): SearchResult[] { const results: SearchResult[] = []; const blocks = html.split(/]*class="vrwrap"/i).slice(1) .concat(html.split(/]*class="rb"/i).slice(1)); for (const block of blocks) { const titleMatch = block.match(/]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i); if (!titleMatch) continue; const url = titleMatch[1].startsWith('http') ? titleMatch[1] : `https://www.sogou.com${titleMatch[1]}`; const title = titleMatch[2].replace(/<[^>]+>/g, '').trim(); const snippetMatch = block.match(/class="star-wiki[^"]*"[^>]*>([\s\S]*?)<\/div>/i) || block.match(/class="space-txt[^"]*"[^>]*>([\s\S]*?)<\/p>/i) || block.match(/class="str_info[^"]*"[^>]*>([\s\S]*?)<\/p>/i); const snippet = snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '').trim() : ''; if (title && url && !url.includes('sogou.com')) { results.push({ title, url, snippet, engine: '搜狗', weight: 75 }); } } return results; } function parse360(html: string): SearchResult[] { const results: SearchResult[] = []; const blocks = html.split(/]*class="res-list"/i).slice(1) .concat(html.split(/]*class="result"/i).slice(1)); for (const block of blocks) { const titleMatch = block.match(/]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i); if (!titleMatch) continue; const url = titleMatch[1]; const title = titleMatch[2].replace(/<[^>]+>/g, '').trim(); const snippetMatch = block.match(/class="res-desc[^"]*"[^>]*>([\s\S]*?)<\/p>/i) || block.match(/class="res-rich[^"]*"[^>]*>([\s\S]*?)<\/div>/i) || block.match(/class="res-summary[^"]*"[^>]*>([\s\S]*?)<\/p>/i) || block.match(/]*>([\s\S]*?)<\/dd>/i); const snippet = snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '').trim() : ''; if (title && url && !url.includes('so.com')) { results.push({ title, url, snippet, engine: '360搜索', weight: 75 }); } } return results; } // ===== 可达性预检 ===== async function checkReachability(urls: string[], concurrency = 5): Promise> { const result = new Map(); for (let i = 0; i < urls.length; i += concurrency) { const batch = urls.slice(i, i + concurrency); const checks = batch.map(async (url) => { try { const resp = await fetchWithTimeout(url, { method: 'HEAD', redirect: 'follow' }, 3_000); result.set(url, resp.ok); } catch { result.set(url, false); } }); await Promise.allSettled(checks); } return result; } // ===== 智能排序 ===== function smartSort(results: SearchResult[], reachabilityMap: Map): SearchResult[] { for (const r of results) { const reachability = r.reachable ? 30 : -20; const snippetQuality = Math.min(r.snippet.length, 100) / 100 * 20; const weightScore = (r.weight / 100) * 50; r._score = weightScore + reachability + snippetQuality; } return results.sort((a, b) => (b._score ?? 0) - (a._score ?? 0)); } // ===== 相关性过滤(用于自动抓取) ===== function computeRelevance(query: string, result: SearchResult): number { const terms: string[] = []; // 中文双字/三字片段 const chinese = query.match(/[\u4e00-\u9fa5]{2,3}/g); if (chinese) terms.push(...chinese); // 英文单词 const english = query.match(/[a-zA-Z]{2,}/g); if (english) terms.push(...english); let score = 0; const text = `${result.title} ${result.snippet}`.toLowerCase(); for (const term of terms) { if (text.includes(term.toLowerCase())) score += 10; } return Math.min(score, 100); } // ===== WebSearchTool ===== export class WebSearchTool implements IMetonaTool { readonly definition: MetonaToolDef = { name: 'web_search', description: 'Search the web for information. Returns titles, snippets, and URLs. When SearXNG is enabled, uses the configured SearXNG instance; otherwise uses built-in engines (Bing, Baidu, Sogou, 360). Automatically fetches full content for top results.', parameters: { type: 'object', properties: { query: { type: 'string', description: 'Search query keywords' }, time_range: { type: 'string', description: 'Time filter: day, week, month, year (optional)' }, enhance_snippets: { type: 'boolean', description: 'Auto-enhance short snippets (default true)' }, }, required: ['query'], }, category: MetonaToolCategory.NETWORK, riskLevel: MetonaRiskLevel.LOW, requiresPermission: false, timeoutMs: 300_000, }; constructor( private configService: ConfigService, private webFetchTool: WebFetchTool, ) {} async execute(args: Record, _context: ToolExecutionContext): Promise { const query = args.query as string; // 读取 SearXNG 配置(提前读取,使工具参数默认值与配置一致) const searxngConfig = this.readSearXNGConfig(); // max_results: 仅由配置面板控制,AI 不可覆盖 const maxResults = Math.min(50, Math.max(1, searxngConfig.max_results > 0 ? searxngConfig.max_results : 30)); const timeRange = (args.time_range as string) ?? ''; const enhanceSnippets = (args.enhance_snippets as boolean) ?? true; // fetch_top: 仅由配置面板控制,AI 不可覆盖 const fetchTop = Math.min(8, Math.max(3, searxngConfig.fetch_count > 0 ? searxngConfig.fetch_count : 5)); logTool('web_search', `Config: maxResults=${maxResults}, fetchTop=${fetchTop}, searxng=${searxngConfig.enabled}`); // 缓存检查(key 含模式 + maxResults + fetchTop,避免配置变更后返回旧缓存) const cacheKey = `${searxngConfig.enabled ? 'searxng' : 'builtin'}:${maxResults}:${fetchTop}:${normalizeUrl(query).toLowerCase()}`; const cached = searchCache.get(cacheKey); if (cached) { logTool('web_search', `Cache hit: "${query}"`); return { ...(cached as Record), from_cache: true }; } let results: SearchResult[]; let mode: string; let engineStats: Record; if (searxngConfig.enabled && searxngConfig.url) { const searxResult = await this.searchSearXNG(query, searxngConfig, maxResults, timeRange); results = searxResult.results; mode = 'searxng'; engineStats = searxResult.engineStats; } else { const builtinResult = await this.searchBuiltinEngines(query, maxResults, timeRange); results = builtinResult.results; mode = 'builtin'; engineStats = builtinResult.engineStats; } // 合并去重 const deduped = this.deduplicate(results); // 可达性预检 const topUrls = deduped.slice(0, Math.min(20, deduped.length)).map((r) => r.url); const reachabilityMap = await checkReachability(topUrls); for (const r of deduped) { r.reachable = reachabilityMap.get(r.url) ?? false; } // 智能排序 const sorted = smartSort(deduped, reachabilityMap).slice(0, maxResults); // 摘要增强 if (enhanceSnippets) { await this.enhanceSnippets(sorted, 3); } // 自动抓取完整内容 const fetchedContent = await this.autoFetch(query, sorted, searxngConfig.fetch_mode, fetchTop); // 格式化输出 const formatted = this.formatResults(query, sorted); const output = { success: true, query, results: sorted.map((r) => ({ title: r.title, url: r.url, snippet: r.snippet, engine: r.engine, reachable: r.reachable, _score: r._score ? Math.round(r._score * 10) / 10 : undefined, _enhanced: r._enhanced, })), total: sorted.length, formatted, from_cache: false, engine_stats: engineStats, _mode: mode, _fetched: fetchedContent.map((f) => ({ url: f.url, title: f.title, content: f.content })), _fetched_count: fetchedContent.length, }; // 写入缓存 searchCache.set(cacheKey, output); logTool('web_search', `Completed: ${sorted.length} results, ${fetchedContent.length} fetched, mode=${mode}`); return output; } // ===== SearXNG 搜索 ===== private async searchSearXNG( query: string, config: SearXNGConfig, maxResults: number, timeRange: string, ): Promise<{ results: SearchResult[]; engineStats: Record }> { const results: SearchResult[] = []; const engineStats: Record = {}; const engineCounts = new Map(); const baseUrl = config.url.replace(/\/$/, ''); const headers = buildSearXNGAuthHeaders(config.auth_key, config.auth_type); const tr = timeRange || config.time_range; // SearXNG 标准分页:每页由实例配置决定(通常 10 条),用 pageno 翻页直到达到 maxResults const maxPages = Math.ceil(maxResults / 5) + 1; // 保守估计,每页至少 5 条 let page = 1; while (results.length < maxResults && page <= maxPages) { const params = new URLSearchParams(); params.set('q', query); params.set('format', config.format || 'json'); params.set('pageno', String(page)); if (config.engines) params.set('engines', config.engines); if (config.language && config.language !== 'auto') params.set('language', config.language); params.set('safesearch', String(config.safesearch)); if (tr) params.set('time_range', tr); const searchUrl = `${baseUrl}/search?${params.toString()}`; logTool('web_search', `[SearXNG] Fetching page ${page}: ${searchUrl}`); let pageResults: SearchResult[] = []; try { const response = await fetchWithTimeout(searchUrl, { headers }, 15_000); if (!response.ok) { throw new Error(`SearXNG API error: ${response.status} ${response.statusText}`); } if (config.format === 'html') { const html = await response.text(); const matches = html.matchAll(/]*href="(https?:\/\/[^"]+)"[^>]*>([\s\S]*?)<\/a>/gi); for (const m of matches) { const url = m[1]; const title = m[2].replace(/<[^>]+>/g, '').trim(); if (title && url && !url.includes(config.url)) { pageResults.push({ title, url, snippet: '', engine: 'searxng', weight: 85 }); } } } else { const data = await response.json() as { results?: Array> }; for (const item of data.results ?? []) { const url = item.url as string; const title = item.title as string; const snippet = item.content as string; const engine = (item.engine as string) ?? 'searxng'; pageResults.push({ title, url, snippet: snippet ?? '', engine, weight: 85 }); engineCounts.set(engine, (engineCounts.get(engine) ?? 0) + 1); } } } catch (err) { logTool('web_search', `[SearXNG] Page ${page} error: ${(err as Error).message}`); // 第一页就失败则抛出,后续页失败则用已获取的结果 if (page === 1) throw err; break; } // 本页无新结果 → 已到末尾 if (pageResults.length === 0) { logTool('web_search', `[SearXNG] Page ${page} returned 0 results, stopping pagination`); break; } results.push(...pageResults); logTool('web_search', `[SearXNG] Page ${page}: +${pageResults.length} (total ${results.length})`); page++; } for (const [engine, count] of engineCounts) { engineStats[engine] = `${count} 条`; } logTool('web_search', `[SearXNG] Total: ${results.length} raw results (target ${maxResults})`); return { results, engineStats }; } // ===== 内置四引擎并行搜索 ===== private async searchBuiltinEngines( query: string, maxResults: number, timeRange: string, ): Promise<{ results: SearchResult[]; engineStats: Record }> { const searchPromises = ENGINES.map(async (engine) => { try { const url = engine.searchUrl(query, timeRange); const response = await fetchWithTimeout(url, { headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36', 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8', }, }, 8_000); if (!response.ok) { logTool('web_search', `[内置] ${engine.name} HTTP ${response.status}`); return []; } const html = await response.text(); const parsed = engine.parse(html); logTool('web_search', `[内置] ${engine.name}: ${parsed.length} results`); return parsed; } catch (err) { logTool('web_search', `[内置] ${engine.name} error: ${(err as Error).message}`); return []; } }); const settled = await Promise.allSettled(searchPromises); const allResults: SearchResult[] = []; const engineStats: Record = {}; for (let i = 0; i < ENGINES.length; i++) { const result = settled[i]; const engineName = ENGINES[i].name; if (result.status === 'fulfilled') { allResults.push(...result.value); engineStats[engineName] = `${result.value.length} 条`; } else { engineStats[engineName] = '失败'; } } return { results: allResults, engineStats }; } // ===== 合并去重 ===== private deduplicate(results: SearchResult[]): SearchResult[] { const seen = new Map(); for (const r of results) { const normalized = normalizeUrl(r.url); if (!seen.has(normalized)) { seen.set(normalized, r); } } return Array.from(seen.values()); } // ===== 摘要增强(委托给 WebFetchTool) ===== private async enhanceSnippets(results: SearchResult[], maxEnhance: number): Promise { let enhanced = 0; for (const r of results) { if (enhanced >= maxEnhance) break; if (r.snippet.length < 30 && r.reachable) { try { const fetchResult = await this.webFetchTool.execute( { url: r.url }, { sessionId: '', workspacePath: '', iteration: 0, requestId: '' }, ) as { success: boolean; content?: string }; if (fetchResult.success && fetchResult.content) { const text = fetchResult.content.slice(0, 200); if (text.length > r.snippet.length) { r.snippet = text; r._enhanced = true; enhanced++; } } } catch { // 忽略增强失败 } } } } // ===== 自动抓取完整内容(委托给 WebFetchTool) ===== private async autoFetch( query: string, results: SearchResult[], fetchMode: string, fetchTop: number, ): Promise> { // 相关性评分(不过滤,relevance=0 的结果也参与抓取候选) const withRelevance = results.map((r) => ({ result: r, relevance: computeRelevance(query, r) })); const filtered = withRelevance.length > 0 ? withRelevance : []; // 确定抓取数量:fetchTop 已在 execute() 中综合了配置面板和工具参数 const topN = Math.min(fetchTop, 8, filtered.length); // 构建抓取列表 let toFetch: typeof filtered; if (fetchMode === 'random') { // Fisher-Yates 洗牌 const shuffled = [...filtered]; for (let i = shuffled.length - 1; i > 0; i--) { const j = Math.floor(Math.random() * (i + 1)); [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]]; } toFetch = shuffled.slice(0, topN); } else { toFetch = filtered .sort((a, b) => b.relevance - a.relevance) .slice(0, topN); } const fetched: Array<{ url: string; title: string; content: string }> = []; for (const item of toFetch) { try { // 委托给 WebFetchTool — 享受三阶段回退策略(HTTP + 反爬 + 浏览器渲染) const fetchResult = await this.webFetchTool.execute( { url: item.result.url }, { sessionId: '', workspacePath: '', iteration: 0, requestId: '' }, ) as { success: boolean; content?: string; method?: string }; if (fetchResult.success && fetchResult.content) { fetched.push({ url: item.result.url, title: item.result.title, content: fetchResult.content }); } } catch (err) { logTool('web_search', `Auto-fetch failed for ${item.result.url}: ${(err as Error).message}`); // 从剩余结果中随机补充 const remaining = withRelevance.filter((x) => !toFetch.includes(x) && !fetched.some((f) => f.url === x.result.url)); if (remaining.length > 0) { const randomPick = remaining[Math.floor(Math.random() * remaining.length)]; try { const fetchResult2 = await this.webFetchTool.execute( { url: randomPick.result.url }, { sessionId: '', workspacePath: '', iteration: 0, requestId: '' }, ) as { success: boolean; content?: string }; if (fetchResult2.success && fetchResult2.content) { fetched.push({ url: randomPick.result.url, title: randomPick.result.title, content: fetchResult2.content }); } } catch { // 忽略补充失败 } } } } return fetched; } // ===== 格式化人类可读输出 ===== private formatResults(query: string, results: SearchResult[]): string { if (results.length === 0) return `搜索 "${query}" 无结果。`; const lines = [`搜索 "${query}" — ${results.length} 条结果:\n`]; results.forEach((r, i) => { lines.push(`${i + 1}. ${r.title}`); lines.push(` URL: ${r.url}`); if (r.snippet) lines.push(` 摘要: ${r.snippet.slice(0, 150)}`); lines.push(` 来源: ${r.engine}${r.reachable === false ? ' (不可达)' : ''}${r._enhanced ? ' [已增强]' : ''}\n`); }); return lines.join('\n'); } // ===== 读取 SearXNG 配置 ===== private readSearXNGConfig(): SearXNGConfig { const cs = this.configService; return { enabled: cs.get('searxng.enabled') ?? false, url: cs.get('searxng.url') ?? '', engines: cs.get('searxng.engines') ?? '', language: cs.get('searxng.language') ?? 'zh-CN', safesearch: cs.get('searxng.safesearch') ?? 1, time_range: cs.get('searxng.time_range') ?? '', max_results: cs.get('searxng.max_results') ?? 0, auth_key: cs.get('searxng.auth_key') ?? '', auth_type: cs.get('searxng.auth_type') ?? 'bearer', format: cs.get('searxng.format') ?? 'json', fetch_count: cs.get('searxng.fetch_count') ?? 0, fetch_mode: cs.get('searxng.fetch_mode') ?? 'sequential', }; } }