912 lines
33 KiB
TypeScript
912 lines
33 KiB
TypeScript
/**
|
||
* web_search — 网络搜索工具
|
||
*
|
||
* 双模式搜索：SearXNG 元搜索 / 内置四引擎并行搜索
|
||
* 内置引擎：Bing + 百度 + 搜狗 + 360 搜索（Promise.allSettled 容错并发）
|
||
* 智能排序：引擎权重(50%) + 可达性(30%) + 摘要质量(20%)
|
||
* 自动抓取：对前 N 条结果调用 web_fetch 获取完整正文
|
||
*
|
||
* v0.4.1: HTML 解析迁移至 node-html-parser（结构化解析）
|
||
* 主层使用 DOM 结构解析（引擎改版时选择器更精确、可维护性远优于正则），
|
||
* 正则解析保留为降级路径（结构化解析无结果时兜底）。
|
||
* 此前纯正则方案违反项目开发规范第一铁律（HTML 解析应使用成熟库）。
|
||
*
|
||
* @see docs/Agent网络工具通用设计-v2.md — 第 2 章 web_search 搜索设计
|
||
*/
|
||
|
||
import { parse as parseHtmlDom, type HTMLElement } from 'node-html-parser';
|
||
import type { IMetonaTool, ToolExecutionContext } from '../../types/metona-tool';
|
||
import type { MetonaToolDef } from '../../../harness/types';
|
||
import { MetonaToolCategory, MetonaRiskLevel } from '../../../harness/types';
|
||
import type { ConfigService } from '../../../services/config.service';
|
||
import {
|
||
searchCache,
|
||
normalizeUrl,
|
||
fetchWithTimeout,
|
||
buildSearXNGAuthHeaders,
|
||
logTool,
|
||
} from './network-utils';
|
||
// v0.7.3 P2-1: 可达性预检经 SSRF 校验 + DNS pinning（结果 URL 是不可信外部输入）
|
||
import { safeValidateSSRF, assertSafeConfigTargetDeep, DeepCheckSoftFailure } from './ssrf-guard';
|
||
import { ssrfPinnedFetch } from './ssrf-dispatcher';
|
||
import type { WebFetchTool } from './web-fetch';
|
||
|
||
// ===== 类型定义 =====
|
||
|
||
interface SearchResult {
|
||
title: string;
|
||
url: string;
|
||
snippet: string;
|
||
engine: string;
|
||
weight: number;
|
||
reachable?: boolean;
|
||
_score?: number;
|
||
_enhanced?: boolean;
|
||
}
|
||
|
||
interface SearXNGConfig {
|
||
enabled: boolean;
|
||
url: string;
|
||
engines: string;
|
||
language: string;
|
||
safesearch: number;
|
||
time_range: string;
|
||
max_results: number;
|
||
auth_key: string;
|
||
auth_type: string;
|
||
format: string;
|
||
fetch_count: number;
|
||
fetch_mode: string;
|
||
}
|
||
|
||
// ===== 引擎定义 =====
|
||
|
||
interface EngineDef {
|
||
name: string;
|
||
weight: number;
|
||
searchUrl: (query: string, timeRange?: string) => string;
|
||
parse: (html: string) => SearchResult[];
|
||
}
|
||
|
||
const ENGINES: EngineDef[] = [
|
||
{
|
||
name: 'bing',
|
||
weight: 90,
|
||
searchUrl: (q, tr) => {
|
||
const freshness = tr
|
||
? `&filters=ex1:"ez${tr === 'day' ? '1' : tr === 'week' ? '2' : tr === 'month' ? '3' : '4'}"`
|
||
: '';
|
||
return `https://www.bing.com/search?q=${encodeURIComponent(q)}${freshness}&count=20`;
|
||
},
|
||
parse: parseBing,
|
||
},
|
||
{
|
||
name: '百度',
|
||
weight: 80,
|
||
searchUrl: (q) => `https://www.baidu.com/s?wd=${encodeURIComponent(q)}&rn=20`,
|
||
parse: parseBaidu,
|
||
},
|
||
{
|
||
name: '搜狗',
|
||
weight: 75,
|
||
searchUrl: (q) => `https://www.sogou.com/web?query=${encodeURIComponent(q)}&num=20`,
|
||
parse: parseSogou,
|
||
},
|
||
{
|
||
name: '360搜索',
|
||
weight: 75,
|
||
searchUrl: (q) => `https://www.so.com/s?q=${encodeURIComponent(q)}&pn=20`,
|
||
parse: parse360,
|
||
},
|
||
];
|
||
|
||
// ===== HTML 解析器（v0.4.1: node-html-parser 结构化解析为主层，正则为降级层） =====
|
||
|
||
/**
|
||
* v0.4.1: 从结果块中提取标题链接 — 跳过指向搜索引擎自身域名的链接（favicon/子导航等）
|
||
*/
|
||
function extractTitleLink(
|
||
block: HTMLElement,
|
||
selfDomain: string,
|
||
): { url: string; title: string } | null {
|
||
for (const a of block.querySelectorAll('a[href]')) {
|
||
const url = a.getAttribute('href') ?? '';
|
||
const title = a.text.trim();
|
||
if (title && url && !url.includes(selfDomain) && url.startsWith('http')) {
|
||
return { url, title };
|
||
}
|
||
}
|
||
return null;
|
||
}
|
||
|
||
/** v0.4.1: 提取第一个非空文本的选择器（按优先级尝试多个候选选择器） */
|
||
function extractText(block: HTMLElement, selectors: string[]): string {
|
||
for (const sel of selectors) {
|
||
const el = block.querySelector(sel);
|
||
if (el) {
|
||
const text = el.text.trim();
|
||
if (text) return text;
|
||
}
|
||
}
|
||
return '';
|
||
}
|
||
|
||
/** v0.4.1: Bing 结构化解析 — li.b_algo 结果块 */
|
||
function parseBingStructured(html: string): SearchResult[] {
|
||
const results: SearchResult[] = [];
|
||
const root = parseHtmlDom(html);
|
||
for (const block of root.querySelectorAll('li.b_algo')) {
|
||
const link = extractTitleLink(block, 'bing.com');
|
||
if (!link) continue;
|
||
const snippet = extractText(block, ['p', '.b_caption']);
|
||
results.push({ title: link.title, url: link.url, snippet, engine: 'bing', weight: 90 });
|
||
}
|
||
return results;
|
||
}
|
||
|
||
/** v0.4.1: 百度结构化解析 — div.result / div.c-container 结果块，优先 a[data-url] 真实链接 */
|
||
function parseBaiduStructured(html: string): SearchResult[] {
|
||
const results: SearchResult[] = [];
|
||
const root = parseHtmlDom(html);
|
||
// 复合选择器去重：class="result c-container" 的元素同时命中两个类名，
|
||
// 分别查询再拼接会重复收录同一结果块
|
||
const blocks = root.querySelectorAll('div.result, div.c-container');
|
||
for (const block of blocks) {
|
||
// 百度标题链接: 优先 data-url 属性（真实目标 URL），href 通常是 baidu.com/link 跳转
|
||
const dataUrlLink = block.querySelector('a[data-url]');
|
||
let url = dataUrlLink?.getAttribute('data-url') ?? '';
|
||
let title = dataUrlLink?.text.trim() ?? '';
|
||
if (!url || !title) {
|
||
const fallback = block.querySelector('h3 a[href]') ?? block.querySelector('a[href]');
|
||
if (fallback) {
|
||
const href = fallback.getAttribute('href') ?? '';
|
||
url = href.startsWith('http') ? href : href ? `https://${href}` : '';
|
||
title = fallback.text.trim();
|
||
}
|
||
}
|
||
const snippet = extractText(block, ['.c-abstract', '[class^="content-right"]']);
|
||
if (title && url && !url.includes('baidu.com/link')) {
|
||
results.push({ title, url, snippet, engine: '百度', weight: 80 });
|
||
}
|
||
}
|
||
return results;
|
||
}
|
||
|
||
/**
|
||
* v0.4.1: 搜狗结构化解析 — div.vrwrap / div.rb 结果块（相对链接补全 sogou.com 前缀）
|
||
*
|
||
* v0.4.1 修复（原正则实现遗留缺陷）: 搜狗结果链接是 sogou.com/link?url=... 跳转形式，
|
||
* 原 `!url.includes('sogou.com')` 过滤条件把所有跳转结果一并丢弃（相对链接补全后必含 sogou.com），
|
||
* 导致搜狗引擎基本无法返回结果。现仅过滤 sogou 自身页面链接，保留 /link 跳转结果
|
||
* （可达性预检会跟随重定向验证）。
|
||
*/
|
||
function parseSogouStructured(html: string): SearchResult[] {
|
||
const results: SearchResult[] = [];
|
||
const root = parseHtmlDom(html);
|
||
// 复合选择器避免同一元素命中两个类名时重复收录
|
||
const blocks = root.querySelectorAll('div.vrwrap, div.rb');
|
||
for (const block of blocks) {
|
||
const a = block.querySelector('h3 a[href]') ?? block.querySelector('a[href]');
|
||
if (!a) continue;
|
||
const href = a.getAttribute('href') ?? '';
|
||
const url = href.startsWith('http') ? href : `https://www.sogou.com${href}`;
|
||
const title = a.text.trim();
|
||
const snippet = extractText(block, ['.star-wiki', '.space-txt', '.str_info']);
|
||
// 过滤搜狗自身页面（保留 /link 跳转结果）
|
||
const isSelfPage = url.includes('sogou.com') && !url.includes('/link');
|
||
if (title && url && !isSelfPage) {
|
||
results.push({ title, url, snippet, engine: '搜狗', weight: 75 });
|
||
}
|
||
}
|
||
return results;
|
||
}
|
||
|
||
/** v0.4.1: 360 结构化解析 — li.res-list / div.result 结果块 */
|
||
function parse360Structured(html: string): SearchResult[] {
|
||
const results: SearchResult[] = [];
|
||
const root = parseHtmlDom(html);
|
||
// 复合选择器避免同一元素命中多个类名时重复收录
|
||
const blocks = root.querySelectorAll('li.res-list, div.result');
|
||
for (const block of blocks) {
|
||
const link = extractTitleLink(block, 'so.com');
|
||
if (!link) continue;
|
||
const snippet = extractText(block, ['.res-desc', '.res-rich', '.res-summary', 'dd']);
|
||
results.push({ title: link.title, url: link.url, snippet, engine: '360搜索', weight: 75 });
|
||
}
|
||
return results;
|
||
}
|
||
|
||
/** v0.4.1: 结构化解析 + 正则降级的组合入口（供 ENGINES 引用，测试导出） */
|
||
export function parseBing(html: string): SearchResult[] {
|
||
const structured = parseBingStructured(html);
|
||
if (structured.length > 0) return structured;
|
||
return parseBingRegex(html);
|
||
}
|
||
|
||
export function parseBaidu(html: string): SearchResult[] {
|
||
const structured = parseBaiduStructured(html);
|
||
if (structured.length > 0) return structured;
|
||
return parseBaiduRegex(html);
|
||
}
|
||
|
||
export function parseSogou(html: string): SearchResult[] {
|
||
const structured = parseSogouStructured(html);
|
||
if (structured.length > 0) return structured;
|
||
return parseSogouRegex(html);
|
||
}
|
||
|
||
export function parse360(html: string): SearchResult[] {
|
||
const structured = parse360Structured(html);
|
||
if (structured.length > 0) return structured;
|
||
return parse360Regex(html);
|
||
}
|
||
|
||
// ===== 正则降级解析器（v0.4.1 前的主实现，结构化解析无结果时兜底） =====
|
||
|
||
function parseBingRegex(html: string): SearchResult[] {
|
||
const results: SearchResult[] = [];
|
||
const blocks = html.split(/<li[^>]*class="b_algo"/i).slice(1);
|
||
for (const block of blocks) {
|
||
const titleMatch = block.match(/<a[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
|
||
if (!titleMatch) continue;
|
||
const url = titleMatch[1];
|
||
const title = titleMatch[2].replace(/<[^>]+>/g, '').trim();
|
||
const snippetMatch =
|
||
block.match(/<p[^>]*>([\s\S]*?)<\/p>/i) ||
|
||
block.match(/class="b_caption"[^>]*>([\s\S]*?)<\/div>/i);
|
||
const snippet = snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '').trim() : '';
|
||
if (title && url && !url.includes('bing.com')) {
|
||
results.push({ title, url, snippet, engine: 'bing', weight: 90 });
|
||
}
|
||
}
|
||
return results;
|
||
}
|
||
|
||
function parseBaiduRegex(html: string): SearchResult[] {
|
||
const results: SearchResult[] = [];
|
||
const blocks = html.split(/<div[^>]*class="result[^"]*"/i).slice(1);
|
||
for (const block of blocks) {
|
||
const titleMatch =
|
||
block.match(/<a[^>]*data-url="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i) ||
|
||
block.match(/<a[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
|
||
if (!titleMatch) continue;
|
||
const url = titleMatch[1].startsWith('http') ? titleMatch[1] : `https://${titleMatch[1]}`;
|
||
const title = titleMatch[2].replace(/<[^>]+>/g, '').trim();
|
||
const snippetMatch =
|
||
block.match(/class="c-abstract[^"]*"[^>]*>([\s\S]*?)<\/span>/i) ||
|
||
block.match(/class="content-right[^"]*"[^>]*>([\s\S]*?)<\/div>/i);
|
||
const snippet = snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '').trim() : '';
|
||
if (title && url && !url.includes('baidu.com/link')) {
|
||
results.push({ title, url, snippet, engine: '百度', weight: 80 });
|
||
}
|
||
}
|
||
return results;
|
||
}
|
||
|
||
function parseSogouRegex(html: string): SearchResult[] {
|
||
const results: SearchResult[] = [];
|
||
const blocks = html
|
||
.split(/<div[^>]*class="vrwrap"/i)
|
||
.slice(1)
|
||
.concat(html.split(/<div[^>]*class="rb"/i).slice(1));
|
||
for (const block of blocks) {
|
||
const titleMatch = block.match(/<a[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
|
||
if (!titleMatch) continue;
|
||
const url = titleMatch[1].startsWith('http')
|
||
? titleMatch[1]
|
||
: `https://www.sogou.com${titleMatch[1]}`;
|
||
const title = titleMatch[2].replace(/<[^>]+>/g, '').trim();
|
||
const snippetMatch =
|
||
block.match(/class="star-wiki[^"]*"[^>]*>([\s\S]*?)<\/div>/i) ||
|
||
block.match(/class="space-txt[^"]*"[^>]*>([\s\S]*?)<\/p>/i) ||
|
||
block.match(/class="str_info[^"]*"[^>]*>([\s\S]*?)<\/p>/i);
|
||
const snippet = snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '').trim() : '';
|
||
if (title && url && !url.includes('sogou.com')) {
|
||
results.push({ title, url, snippet, engine: '搜狗', weight: 75 });
|
||
}
|
||
}
|
||
return results;
|
||
}
|
||
|
||
function parse360Regex(html: string): SearchResult[] {
|
||
const results: SearchResult[] = [];
|
||
const blocks = html
|
||
.split(/<li[^>]*class="res-list"/i)
|
||
.slice(1)
|
||
.concat(html.split(/<div[^>]*class="result"/i).slice(1));
|
||
for (const block of blocks) {
|
||
const titleMatch = block.match(/<a[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
|
||
if (!titleMatch) continue;
|
||
const url = titleMatch[1];
|
||
const title = titleMatch[2].replace(/<[^>]+>/g, '').trim();
|
||
const snippetMatch =
|
||
block.match(/class="res-desc[^"]*"[^>]*>([\s\S]*?)<\/p>/i) ||
|
||
block.match(/class="res-rich[^"]*"[^>]*>([\s\S]*?)<\/div>/i) ||
|
||
block.match(/class="res-summary[^"]*"[^>]*>([\s\S]*?)<\/p>/i) ||
|
||
block.match(/<dd[^>]*>([\s\S]*?)<\/dd>/i);
|
||
const snippet = snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '').trim() : '';
|
||
if (title && url && !url.includes('so.com')) {
|
||
results.push({ title, url, snippet, engine: '360搜索', weight: 75 });
|
||
}
|
||
}
|
||
return results;
|
||
}
|
||
|
||
// ===== 可达性预检 =====
|
||
// v0.7.3 P2-1: 可达性预检的 URL 来自不可信的搜索结果 —— HEAD 探测同样不得
|
||
// 触达内网/元数据地址。校验失败的 URL 直接标记不可达（不发起任何请求）；
|
||
// 探测经 ssrfPinnedFetch（DNS pinning），重定向不自动跟随（3xx 即视为可达 ——
|
||
// 链接活性已证明，且跟跳目标不再绕过校验）。
|
||
|
||
async function checkReachability(
|
||
urls: string[],
|
||
signal?: AbortSignal,
|
||
concurrency = 5,
|
||
): Promise<Map<string, boolean>> {
|
||
const result = new Map<string, boolean>();
|
||
for (let i = 0; i < urls.length; i += concurrency) {
|
||
// v0.8.0 P1-2: abort 后立即终止预检批次
|
||
if (signal?.aborted) break;
|
||
const batch = urls.slice(i, i + concurrency);
|
||
const checks = batch.map(async (url) => {
|
||
try {
|
||
const ssrf = await safeValidateSSRF(url);
|
||
if (!ssrf.ok) {
|
||
result.set(url, false);
|
||
return;
|
||
}
|
||
// v0.8.0 P1-2: 透传引擎级 abort signal
|
||
const resp = await ssrfPinnedFetch(
|
||
url,
|
||
{ method: 'HEAD', redirect: 'manual' },
|
||
3_000,
|
||
signal,
|
||
);
|
||
result.set(url, resp.ok || (resp.status >= 300 && resp.status < 400));
|
||
} catch {
|
||
result.set(url, false);
|
||
}
|
||
});
|
||
await Promise.allSettled(checks);
|
||
}
|
||
return result;
|
||
}
|
||
|
||
// ===== 智能排序 =====
|
||
|
||
function smartSort(results: SearchResult[]): SearchResult[] {
|
||
for (const r of results) {
|
||
const reachability = r.reachable ? 30 : -20;
|
||
const snippetQuality = (Math.min(r.snippet.length, 100) / 100) * 20;
|
||
const weightScore = (r.weight / 100) * 50;
|
||
r._score = weightScore + reachability + snippetQuality;
|
||
}
|
||
return results.sort((a, b) => (b._score ?? 0) - (a._score ?? 0));
|
||
}
|
||
|
||
// ===== 相关性过滤（用于自动抓取） =====
|
||
|
||
function computeRelevance(query: string, result: SearchResult): number {
|
||
const terms: string[] = [];
|
||
// 中文双字/三字片段
|
||
const chinese = query.match(/[\u4e00-\u9fa5]{2,3}/g);
|
||
if (chinese) terms.push(...chinese);
|
||
// 英文单词
|
||
const english = query.match(/[a-zA-Z]{2,}/g);
|
||
if (english) terms.push(...english);
|
||
|
||
let score = 0;
|
||
const text = `${result.title} ${result.snippet}`.toLowerCase();
|
||
for (const term of terms) {
|
||
if (text.includes(term.toLowerCase())) score += 10;
|
||
}
|
||
return Math.min(score, 100);
|
||
}
|
||
|
||
// ===== WebSearchTool =====
|
||
|
||
export class WebSearchTool implements IMetonaTool {
|
||
readonly definition: MetonaToolDef = {
|
||
name: 'web_search',
|
||
description:
|
||
'Search the web for information. Returns titles, snippets, and URLs. When SearXNG is enabled, uses the configured SearXNG instance; otherwise uses built-in engines (Bing, Baidu, Sogou, 360). Automatically fetches full content for top results.',
|
||
parameters: {
|
||
type: 'object',
|
||
properties: {
|
||
query: { type: 'string', description: 'Search query keywords' },
|
||
time_range: {
|
||
type: 'string',
|
||
description: 'Time filter: day, week, month, year (optional)',
|
||
},
|
||
enhance_snippets: {
|
||
type: 'boolean',
|
||
description: 'Auto-enhance short snippets (default true)',
|
||
},
|
||
},
|
||
required: ['query'],
|
||
},
|
||
category: MetonaToolCategory.NETWORK,
|
||
riskLevel: MetonaRiskLevel.LOW,
|
||
requiresPermission: false,
|
||
timeoutMs: 300_000,
|
||
};
|
||
|
||
constructor(
|
||
private configService: ConfigService,
|
||
private webFetchTool: WebFetchTool,
|
||
) {}
|
||
|
||
async execute(args: Record<string, unknown>, context: ToolExecutionContext): Promise<unknown> {
|
||
const query = args.query as string;
|
||
|
||
// v0.8.0 P1-2: abort 立即生效（后续各阶段均携带本 signal）
|
||
if (context.signal?.aborted) {
|
||
return { success: false, query, results: [], error: 'Search aborted' };
|
||
}
|
||
|
||
// 读取 SearXNG 配置
|
||
const searxngConfig = this.readSearXNGConfig();
|
||
const useSearXNG = searxngConfig.enabled && !!searxngConfig.url;
|
||
|
||
// 两个模式参数完全独立：
|
||
// - 内置模式：固定默认值 maxResults=30, fetchTop=5
|
||
// - SearXNG 模式：由配置面板控制
|
||
const maxResults = useSearXNG
|
||
? Math.min(50, Math.max(1, searxngConfig.max_results > 0 ? searxngConfig.max_results : 30))
|
||
: 30;
|
||
|
||
const timeRange = (args.time_range as string) ?? '';
|
||
const enhanceSnippets = (args.enhance_snippets as boolean) ?? true;
|
||
|
||
const fetchTop = useSearXNG
|
||
? Math.min(8, Math.max(3, searxngConfig.fetch_count > 0 ? searxngConfig.fetch_count : 5))
|
||
: 5;
|
||
|
||
logTool(
|
||
'web_search',
|
||
`Mode=${useSearXNG ? 'searxng' : 'builtin'}, maxResults=${maxResults}, fetchTop=${fetchTop}`,
|
||
);
|
||
|
||
// 缓存检查（key 含模式 + maxResults + fetchTop，避免配置变更后返回旧缓存）
|
||
const cacheKey = `${searxngConfig.enabled ? 'searxng' : 'builtin'}:${maxResults}:${fetchTop}:${normalizeUrl(query).toLowerCase()}`;
|
||
const cached = searchCache.get(cacheKey);
|
||
if (cached) {
|
||
logTool('web_search', `Cache hit: "${query}"`);
|
||
return { ...(cached as Record<string, unknown>), from_cache: true };
|
||
}
|
||
|
||
let results: SearchResult[];
|
||
let mode: string;
|
||
let engineStats: Record<string, string>;
|
||
|
||
if (useSearXNG) {
|
||
const searxResult = await this.searchSearXNG(
|
||
query,
|
||
searxngConfig,
|
||
maxResults,
|
||
timeRange,
|
||
context.signal,
|
||
);
|
||
results = searxResult.results;
|
||
mode = 'searxng';
|
||
engineStats = searxResult.engineStats;
|
||
} else {
|
||
const builtinResult = await this.searchBuiltinEngines(
|
||
query,
|
||
maxResults,
|
||
timeRange,
|
||
context.signal,
|
||
);
|
||
results = builtinResult.results;
|
||
mode = 'builtin';
|
||
engineStats = builtinResult.engineStats;
|
||
}
|
||
|
||
// 合并去重
|
||
const deduped = this.deduplicate(results);
|
||
|
||
// 可达性预检
|
||
const topUrls = deduped.slice(0, Math.min(20, deduped.length)).map((r) => r.url);
|
||
const reachabilityMap = await checkReachability(topUrls, context.signal);
|
||
for (const r of deduped) {
|
||
r.reachable = reachabilityMap.get(r.url) ?? false;
|
||
}
|
||
|
||
// 智能排序
|
||
const sorted = smartSort(deduped).slice(0, maxResults);
|
||
|
||
// 摘要增强
|
||
if (enhanceSnippets) {
|
||
await this.enhanceSnippets(sorted, 3, context);
|
||
}
|
||
|
||
// 自动抓取完整内容
|
||
const fetchedContent = await this.autoFetch(
|
||
query,
|
||
sorted,
|
||
searxngConfig.fetch_mode,
|
||
fetchTop,
|
||
context,
|
||
);
|
||
|
||
// 格式化输出
|
||
const formatted = this.formatResults(query, sorted);
|
||
|
||
const output = {
|
||
success: true,
|
||
query,
|
||
results: sorted.map((r) => ({
|
||
title: r.title,
|
||
url: r.url,
|
||
snippet: r.snippet,
|
||
engine: r.engine,
|
||
reachable: r.reachable,
|
||
_score: r._score ? Math.round(r._score * 10) / 10 : undefined,
|
||
_enhanced: r._enhanced,
|
||
})),
|
||
total: sorted.length,
|
||
formatted,
|
||
from_cache: false,
|
||
engine_stats: engineStats,
|
||
_mode: mode,
|
||
_fetched: fetchedContent.map((f) => ({ url: f.url, title: f.title, content: f.content })),
|
||
_fetched_count: fetchedContent.length,
|
||
};
|
||
|
||
// 写入缓存
|
||
searchCache.set(cacheKey, output);
|
||
logTool(
|
||
'web_search',
|
||
`Completed: ${sorted.length} results, ${fetchedContent.length} fetched, mode=${mode}`,
|
||
);
|
||
|
||
return output;
|
||
}
|
||
|
||
// ===== SearXNG 搜索 =====
|
||
|
||
private async searchSearXNG(
|
||
query: string,
|
||
config: SearXNGConfig,
|
||
maxResults: number,
|
||
timeRange: string,
|
||
signal?: AbortSignal,
|
||
): Promise<{ results: SearchResult[]; engineStats: Record<string, string> }> {
|
||
const results: SearchResult[] = [];
|
||
const engineStats: Record<string, string> = {};
|
||
const engineCounts = new Map<string, number>();
|
||
const baseUrl = config.url.replace(/\/$/, '');
|
||
const headers = buildSearXNGAuthHeaders(config.auth_key, config.auth_type);
|
||
const tr = timeRange || config.time_range;
|
||
|
||
// v0.8.2 P1-6: SearXNG 运行时请求的 SSRF 纵深校验。
|
||
// 配置期已有 assertSafeConfigTarget（ipc/shared 写入链），但 DNS 记录可在配置
|
||
// 之后被切换（指向云元数据/链路本地）—— 运行时请求此前完全无校验。此处对
|
||
// 每次搜索会话做同口径静态校验 + DNS 深校验；本地回环/RFC1918 合法放行
|
||
// （SearXNG 常部署本机/内网），DNS 解析失败按 DeepCheckSoftFailure 留痕放行
|
||
// （离线实例合法，与配置期深校验语义一致）。
|
||
try {
|
||
await assertSafeConfigTargetDeep(baseUrl);
|
||
} catch (err) {
|
||
if (err instanceof DeepCheckSoftFailure) {
|
||
logTool('web_search', `[SearXNG] DNS deep check skipped (soft-fail): ${err.message}`);
|
||
} else {
|
||
throw new Error(`SearXNG target blocked by security policy: ${(err as Error).message}`);
|
||
}
|
||
}
|
||
|
||
// SearXNG 标准分页：每页由实例配置决定（通常 10 条），用 pageno 翻页直到达到 maxResults
|
||
const maxPages = Math.ceil(maxResults / 5) + 1; // 保守估计，每页至少 5 条
|
||
let page = 1;
|
||
|
||
while (results.length < maxResults && page <= maxPages) {
|
||
// v0.8.0 P1-2: abort 后停止翻页
|
||
if (signal?.aborted) break;
|
||
const params = new URLSearchParams();
|
||
params.set('q', query);
|
||
params.set('format', config.format || 'json');
|
||
params.set('pageno', String(page));
|
||
if (config.engines) params.set('engines', config.engines);
|
||
if (config.language && config.language !== 'auto') params.set('language', config.language);
|
||
params.set('safesearch', String(config.safesearch));
|
||
if (tr) params.set('time_range', tr);
|
||
|
||
const searchUrl = `${baseUrl}/search?${params.toString()}`;
|
||
logTool('web_search', `[SearXNG] Fetching page ${page}: ${searchUrl}`);
|
||
|
||
const pageResults: SearchResult[] = [];
|
||
|
||
try {
|
||
// v0.8.0 P1-2: SearXNG 请求携带引擎级 abort signal
|
||
const response = await fetchWithTimeout(searchUrl, { headers }, 15_000, signal);
|
||
if (!response.ok) {
|
||
throw new Error(`SearXNG API error: ${response.status} ${response.statusText}`);
|
||
}
|
||
|
||
if (config.format === 'html') {
|
||
const html = await response.text();
|
||
const matches = html.matchAll(/<a[^>]*href="(https?:\/\/[^"]+)"[^>]*>([\s\S]*?)<\/a>/gi);
|
||
for (const m of matches) {
|
||
const url = m[1];
|
||
const title = m[2].replace(/<[^>]+>/g, '').trim();
|
||
if (title && url && !url.includes(config.url)) {
|
||
pageResults.push({ title, url, snippet: '', engine: 'searxng', weight: 85 });
|
||
}
|
||
}
|
||
} else {
|
||
const data = (await response.json()) as { results?: Array<Record<string, unknown>> };
|
||
for (const item of data.results ?? []) {
|
||
const url = item.url as string;
|
||
const title = item.title as string;
|
||
const snippet = item.content as string;
|
||
const engine = (item.engine as string) ?? 'searxng';
|
||
pageResults.push({ title, url, snippet: snippet ?? '', engine, weight: 85 });
|
||
engineCounts.set(engine, (engineCounts.get(engine) ?? 0) + 1);
|
||
}
|
||
}
|
||
} catch (err) {
|
||
logTool('web_search', `[SearXNG] Page ${page} error: ${(err as Error).message}`);
|
||
// 第一页就失败则抛出，后续页失败则用已获取的结果
|
||
if (page === 1) throw err;
|
||
break;
|
||
}
|
||
|
||
// 本页无新结果 → 已到末尾
|
||
if (pageResults.length === 0) {
|
||
logTool('web_search', `[SearXNG] Page ${page} returned 0 results, stopping pagination`);
|
||
break;
|
||
}
|
||
|
||
results.push(...pageResults);
|
||
logTool(
|
||
'web_search',
|
||
`[SearXNG] Page ${page}: +${pageResults.length} (total ${results.length})`,
|
||
);
|
||
page++;
|
||
}
|
||
|
||
for (const [engine, count] of engineCounts) {
|
||
engineStats[engine] = `${count} 条`;
|
||
}
|
||
|
||
logTool('web_search', `[SearXNG] Total: ${results.length} raw results (target ${maxResults})`);
|
||
return { results, engineStats };
|
||
}
|
||
|
||
// ===== 内置四引擎并行搜索 =====
|
||
|
||
private async searchBuiltinEngines(
|
||
query: string,
|
||
maxResults: number,
|
||
timeRange: string,
|
||
signal?: AbortSignal,
|
||
): Promise<{ results: SearchResult[]; engineStats: Record<string, string> }> {
|
||
const searchPromises = ENGINES.map(async (engine) => {
|
||
try {
|
||
const url = engine.searchUrl(query, timeRange);
|
||
const response = await fetchWithTimeout(
|
||
url,
|
||
{
|
||
headers: {
|
||
'User-Agent':
|
||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36',
|
||
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
|
||
},
|
||
},
|
||
8_000,
|
||
signal,
|
||
);
|
||
|
||
if (!response.ok) {
|
||
logTool('web_search', `[内置] ${engine.name} HTTP ${response.status}`);
|
||
return [];
|
||
}
|
||
|
||
const html = await response.text();
|
||
const parsed = engine.parse(html);
|
||
logTool('web_search', `[内置] ${engine.name}: ${parsed.length} results`);
|
||
return parsed;
|
||
} catch (err) {
|
||
logTool('web_search', `[内置] ${engine.name} error: ${(err as Error).message}`);
|
||
return [];
|
||
}
|
||
});
|
||
|
||
const settled = await Promise.allSettled(searchPromises);
|
||
const allResults: SearchResult[] = [];
|
||
const engineStats: Record<string, string> = {};
|
||
|
||
for (let i = 0; i < ENGINES.length; i++) {
|
||
const result = settled[i];
|
||
const engineName = ENGINES[i].name;
|
||
if (result.status === 'fulfilled') {
|
||
allResults.push(...result.value);
|
||
engineStats[engineName] = `${result.value.length} 条`;
|
||
} else {
|
||
engineStats[engineName] = '失败';
|
||
}
|
||
}
|
||
|
||
return { results: allResults, engineStats };
|
||
}
|
||
|
||
// ===== 合并去重 =====
|
||
|
||
private deduplicate(results: SearchResult[]): SearchResult[] {
|
||
const seen = new Map<string, SearchResult>();
|
||
for (const r of results) {
|
||
const normalized = normalizeUrl(r.url);
|
||
if (!seen.has(normalized)) {
|
||
seen.set(normalized, r);
|
||
}
|
||
}
|
||
return Array.from(seen.values());
|
||
}
|
||
|
||
// ===== 摘要增强（委托给 WebFetchTool） =====
|
||
|
||
private async enhanceSnippets(
|
||
results: SearchResult[],
|
||
maxEnhance: number,
|
||
context: ToolExecutionContext,
|
||
): Promise<void> {
|
||
let enhanced = 0;
|
||
for (const r of results) {
|
||
// v0.8.0 P1-2: abort 后停止增强
|
||
if (context.signal?.aborted) break;
|
||
if (enhanced >= maxEnhance) break;
|
||
if (r.snippet.length < 30 && r.reachable) {
|
||
try {
|
||
// v0.8.0 P1-2: 透传真实执行上下文（signal）—— 移除"伪造空上下文"
|
||
const fetchResult = (await this.webFetchTool.execute({ url: r.url }, context)) as {
|
||
success: boolean;
|
||
content?: string;
|
||
};
|
||
|
||
if (fetchResult.success && fetchResult.content) {
|
||
const text = fetchResult.content.slice(0, 200);
|
||
if (text.length > r.snippet.length) {
|
||
r.snippet = text;
|
||
r._enhanced = true;
|
||
enhanced++;
|
||
}
|
||
}
|
||
} catch {
|
||
// 忽略增强失败
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// ===== 自动抓取完整内容（委托给 WebFetchTool） =====
|
||
|
||
/**
|
||
* P2-12: 自动抓取改为并行（批次并发 3）
|
||
*
|
||
* 原实现逐条串行抓取（单个 web_fetch 最长 120s 超时），top5 结果最坏耗时
|
||
* 逼近 web_search 的 300s 工具超时上限。并行批次化后总耗时约降至 1/3。
|
||
* 失败结果直接跳过（原"随机补充重试"逻辑收益边际，复杂度高，已移除）。
|
||
*/
|
||
private async autoFetch(
|
||
query: string,
|
||
results: SearchResult[],
|
||
fetchMode: string,
|
||
fetchTop: number,
|
||
context: ToolExecutionContext,
|
||
): Promise<Array<{ url: string; title: string; content: string }>> {
|
||
// v0.8.0 P1-2 根治: 时间预算收敛 —— 旧实现最坏耗时（8 条 × 3 并发 × 单次
|
||
// web_fetch 240s ≈ 720s）远超工具自身 300s 超时，registry 层超时后整次调用
|
||
// 作废（已抓取内容全部丢弃）。现以 deadline 为界：剩余预算不足一次最小
|
||
// 抓取（30s）即停，已抓取内容照常返回。
|
||
const BUDGET_RATIO = 0.8;
|
||
const MIN_FETCH_BUDGET_MS = 30_000;
|
||
const deadlineMs = Date.now() + 300_000 * BUDGET_RATIO;
|
||
|
||
// 相关性评分（不过滤，relevance=0 的结果也参与抓取候选）
|
||
const filtered = results.map((r) => ({
|
||
result: r,
|
||
relevance: computeRelevance(query, r),
|
||
}));
|
||
|
||
// 确定抓取数量：fetchTop 已在 execute() 中综合了配置面板和工具参数
|
||
const topN = Math.min(fetchTop, 8, filtered.length);
|
||
|
||
// 构建抓取列表
|
||
let toFetch: typeof filtered;
|
||
if (fetchMode === 'random') {
|
||
// Fisher-Yates 洗牌
|
||
const shuffled = [...filtered];
|
||
for (let i = shuffled.length - 1; i > 0; i--) {
|
||
const j = Math.floor(Math.random() * (i + 1));
|
||
[shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
|
||
}
|
||
toFetch = shuffled.slice(0, topN);
|
||
} else {
|
||
toFetch = filtered.sort((a, b) => b.relevance - a.relevance).slice(0, topN);
|
||
}
|
||
|
||
const fetched: Array<{ url: string; title: string; content: string }> = [];
|
||
|
||
const fetchOne = async (item: {
|
||
result: SearchResult;
|
||
}): Promise<{ url: string; title: string; content: string } | null> => {
|
||
try {
|
||
// 委托给 WebFetchTool — 享受三阶段回退策略（HTTP + 反爬 + 浏览器渲染）
|
||
// v0.8.0 P1-2: 透传真实执行上下文（signal）—— 移除"伪造空上下文"
|
||
const fetchResult = (await this.webFetchTool.execute(
|
||
{ url: item.result.url },
|
||
context,
|
||
)) as { success: boolean; content?: string };
|
||
|
||
if (fetchResult.success && fetchResult.content) {
|
||
return { url: item.result.url, title: item.result.title, content: fetchResult.content };
|
||
}
|
||
return null;
|
||
} catch (err) {
|
||
logTool(
|
||
'web_search',
|
||
`Auto-fetch failed for ${item.result.url}: ${(err as Error).message}`,
|
||
);
|
||
return null;
|
||
}
|
||
};
|
||
|
||
// 并行批次抓取（并发 3，时间预算收敛）
|
||
const CONCURRENCY = 3;
|
||
for (let i = 0; i < toFetch.length; i += CONCURRENCY) {
|
||
// v0.8.0 P1-2: 剩余预算不足一次最小抓取 → 停止，保留已抓取内容
|
||
const remaining = deadlineMs - Date.now();
|
||
if (context.signal?.aborted || remaining < MIN_FETCH_BUDGET_MS) {
|
||
logTool(
|
||
'web_search',
|
||
`Auto-fetch budget stop: fetched=${fetched.length}, remaining=${Math.max(0, remaining)}ms, aborted=${!!context.signal?.aborted}`,
|
||
);
|
||
break;
|
||
}
|
||
const batch = toFetch.slice(i, i + CONCURRENCY);
|
||
const settled = await Promise.allSettled(batch.map((item) => fetchOne(item)));
|
||
for (const r of settled) {
|
||
if (r.status === 'fulfilled' && r.value) fetched.push(r.value);
|
||
}
|
||
}
|
||
|
||
return fetched;
|
||
}
|
||
|
||
// ===== 格式化人类可读输出 =====
|
||
|
||
private formatResults(query: string, results: SearchResult[]): string {
|
||
if (results.length === 0) return `搜索 "${query}" 无结果。`;
|
||
const lines = [`搜索 "${query}" — ${results.length} 条结果：\n`];
|
||
results.forEach((r, i) => {
|
||
lines.push(`${i + 1}. ${r.title}`);
|
||
lines.push(` URL: ${r.url}`);
|
||
if (r.snippet) lines.push(` 摘要: ${r.snippet.slice(0, 150)}`);
|
||
lines.push(
|
||
` 来源: ${r.engine}${r.reachable === false ? ' (不可达)' : ''}${r._enhanced ? ' [已增强]' : ''}\n`,
|
||
);
|
||
});
|
||
return lines.join('\n');
|
||
}
|
||
|
||
// ===== 读取 SearXNG 配置 =====
|
||
|
||
private readSearXNGConfig(): SearXNGConfig {
|
||
const cs = this.configService;
|
||
return {
|
||
enabled: cs.get<boolean>('searxng.enabled') ?? false,
|
||
url: cs.get<string>('searxng.url') ?? '',
|
||
engines: cs.get<string>('searxng.engines') ?? '',
|
||
language: cs.get<string>('searxng.language') ?? 'zh-CN',
|
||
safesearch: cs.get<number>('searxng.safesearch') ?? 1,
|
||
time_range: cs.get<string>('searxng.time_range') ?? '',
|
||
max_results: cs.get<number>('searxng.max_results') ?? 0,
|
||
auth_key: cs.get<string>('searxng.auth_key') ?? '',
|
||
auth_type: cs.get<string>('searxng.auth_type') ?? 'bearer',
|
||
format: cs.get<string>('searxng.format') ?? 'json',
|
||
fetch_count: cs.get<number>('searxng.fetch_count') ?? 0,
|
||
fetch_mode: cs.get<string>('searxng.fetch_mode') ?? 'sequential',
|
||
};
|
||
}
|
||
}
|