thzxx
ebe45482b0
P1 修复面收口: Prompt Cache 根治(日期/记忆/附件三类易变内容出 system 入用户消息 前置块 user-context.ts, system 跨 run 字节级稳定; Anthropic system 块数组化 + cache_control ephemeral 断言, DeepSeek 自动缓存前缀命中 — 多轮对话输入 token 成本降数量级); 编辑重发/重新生成幽灵 Trace 双侧根治(DB truncateMessagesAfter 同步过滤 metadata.traceSteps + 前端 trimTraceStepsByAnchor 镜像, 严格小于锚点 时间戳, 同毫秒等值判废); sessions:deleteMessage 死通道全链路删除(渲染层零调用 + message_count 漂移面); Ollama vision 能力门控全链路(MetonaModelInfo .supportsVision 贯穿 adapter/IPC/store/UI, model-capabilities.ts 三道判定纯函数, 未知保守放行); 记忆固化节流(consolidation-policy 纯函数: 总开关 + 内容门控 [回答>=200字符或存在成功工具调用] + 会话级 10 分钟频率窗口, 三 memory.* 配置键) P2 安全纵深: SSRF DNS Pinning 关闭 rebinding 窗口(ssrf-guard 重构 resolvePublicAddresses 单源; ssrf-dispatcher 以 undici Agent.connect.lookup 钉死校验 IP, TLS SNI 保持原域名, 一次性 dispatcher 用后即毁; 代理激活显式 退化为仅入口校验); web_fetch 重写手动逐跳重定向循环(每跳先校验后连接, 替代 redirect:follow 内核跟跳的中间跳裸奔, 上限 5 跳); http_request 换用 pinned fetch; web_search 可达性预检加固(私有 URL 零请求 + 不跟跳, 3xx 视为 可达); Agent 浏览器 CORS 通配收紧为 Origin 回显 + Vary: Origin; ConfirmationHook.forgetSession 会话终态清理(会话删除/abort 联动/SubAgent 终结三处接线, 根治 rememberedDecisions 泄漏) P3 架构还债: agent.enableReflection 死配置全链路接线(main→shared→引擎→ Orchestrator→设置开关, REFLECTING 状态真实可达); AgentLoopConfig.timeoutMs 死字段删除; MemoryManager.cleanupExpired 挂入健康检查周期(expires_at 回收 管道真实化); buildSafeEnv 收敛 utils/safe-env.ts 单源(run_command 与 MCP stdio 共用, 终结双实现漂移); Trace 生命周期治理(metadata 只保留最近 20 个 run — keepRecentRuns 纯函数; JSONL 录制启动自动清理保留 200 个 + 设置页 手动清理); SLO/健康快照可视化(app:healthSnapshot IPC + 设置页只读卡片 + 审计链一键校验) P4 能力演进: 会话标题 LLM 自动生成(TitleGenerator — 每会话幂等/并发重入复用 同一 Promise/自定义标题不覆盖/失败静默回退, Sidebar 经 config:changed 实时 刷新); MCP 自动重连(5s/15s/60s 退避最多 3 次, reconnecting 状态机, teardownConnection 内部拆除保留簿记 — 用户断开/开关关闭即时取消, 设置页 显示第 N/3 次); 死循环检测 ABAB 乒乓模式(最近4轮 A→B→A→B 交替判定, 补齐 docs 第五章"两状态反复切换"检测契约); i18n 第三阶段(ChatInput/LLMSettings/ OnboardingWizard/MemoryViewer 主链路文案出层, zh-CN + en-US 双字典补齐) 测试: 737 → 824 用例(+87, 新增 8 个测试文件 + 扩展 3 个)。新覆盖: user-context 分组/空值收缩/拼接契约、context-builder 字节级稳定性、Anthropic cache_control 四态、consolidation-policy 九路判定矩阵、ssrf-dispatcher(pinned lookup/重定向 解析/IP 校验)、forget-session 会话隔离、trace-lifecycle run 淘汰、 trace-trim 严格小于边界、safe-env 净化矩阵、mcp-reconnect 退避状态机 (fake timers)、title-generator 并发重入、SQLite 侧 truncate×TRACE 联动 (Electron ABI)。测试驱动修复: GIT_*/ 注释终止块注释、重连计数被自身重试 前置断开重置(拆 teardownConnection 保留簿记)、TitleGenerator 幂等占位与 并发去重的检查顺序竞态(去重先于幂等) 版本: 0.7.3; README 同步(配置表新增 agent.enableReflection/memory.*/mcp.autoReconnect) 回归: typecheck 双端 0 错误; ESLint 0/0; 系统 Node 771 通过 53 跳过 (better-sqlite3 ABI); Electron ABI 全量 824/824 零跳过
831 lines
30 KiB
TypeScript
831 lines
30 KiB
TypeScript
/**
|
||
* web_search — 网络搜索工具
|
||
*
|
||
* 双模式搜索：SearXNG 元搜索 / 内置四引擎并行搜索
|
||
* 内置引擎：Bing + 百度 + 搜狗 + 360 搜索（Promise.allSettled 容错并发）
|
||
* 智能排序：引擎权重(50%) + 可达性(30%) + 摘要质量(20%)
|
||
* 自动抓取：对前 N 条结果调用 web_fetch 获取完整正文
|
||
*
|
||
* v0.4.1: HTML 解析迁移至 node-html-parser（结构化解析）
|
||
* 主层使用 DOM 结构解析（引擎改版时选择器更精确、可维护性远优于正则），
|
||
* 正则解析保留为降级路径（结构化解析无结果时兜底）。
|
||
* 此前纯正则方案违反项目开发规范第一铁律（HTML 解析应使用成熟库）。
|
||
*
|
||
* @see docs/Agent网络工具通用设计-v2.md — 第 2 章 web_search 搜索设计
|
||
*/
|
||
|
||
import { parse as parseHtmlDom, type HTMLElement } from 'node-html-parser';
|
||
import type { IMetonaTool, ToolExecutionContext } from '../../types/metona-tool';
|
||
import type { MetonaToolDef } from '../../../harness/types';
|
||
import { MetonaToolCategory, MetonaRiskLevel } from '../../../harness/types';
|
||
import type { ConfigService } from '../../../services/config.service';
|
||
import {
|
||
searchCache,
|
||
normalizeUrl,
|
||
fetchWithTimeout,
|
||
buildSearXNGAuthHeaders,
|
||
logTool,
|
||
} from './network-utils';
|
||
// v0.7.3 P2-1: 可达性预检经 SSRF 校验 + DNS pinning（结果 URL 是不可信外部输入）
|
||
import { safeValidateSSRF } from './ssrf-guard';
|
||
import { ssrfPinnedFetch } from './ssrf-dispatcher';
|
||
import type { WebFetchTool } from './web-fetch';
|
||
|
||
// ===== 类型定义 =====
|
||
|
||
interface SearchResult {
|
||
title: string;
|
||
url: string;
|
||
snippet: string;
|
||
engine: string;
|
||
weight: number;
|
||
reachable?: boolean;
|
||
_score?: number;
|
||
_enhanced?: boolean;
|
||
}
|
||
|
||
interface SearXNGConfig {
|
||
enabled: boolean;
|
||
url: string;
|
||
engines: string;
|
||
language: string;
|
||
safesearch: number;
|
||
time_range: string;
|
||
max_results: number;
|
||
auth_key: string;
|
||
auth_type: string;
|
||
format: string;
|
||
fetch_count: number;
|
||
fetch_mode: string;
|
||
}
|
||
|
||
// ===== 引擎定义 =====
|
||
|
||
interface EngineDef {
|
||
name: string;
|
||
weight: number;
|
||
searchUrl: (query: string, timeRange?: string) => string;
|
||
parse: (html: string) => SearchResult[];
|
||
}
|
||
|
||
const ENGINES: EngineDef[] = [
|
||
{
|
||
name: 'bing',
|
||
weight: 90,
|
||
searchUrl: (q, tr) => {
|
||
const freshness = tr
|
||
? `&filters=ex1:"ez${tr === 'day' ? '1' : tr === 'week' ? '2' : tr === 'month' ? '3' : '4'}"`
|
||
: '';
|
||
return `https://www.bing.com/search?q=${encodeURIComponent(q)}${freshness}&count=20`;
|
||
},
|
||
parse: parseBing,
|
||
},
|
||
{
|
||
name: '百度',
|
||
weight: 80,
|
||
searchUrl: (q) => `https://www.baidu.com/s?wd=${encodeURIComponent(q)}&rn=20`,
|
||
parse: parseBaidu,
|
||
},
|
||
{
|
||
name: '搜狗',
|
||
weight: 75,
|
||
searchUrl: (q) => `https://www.sogou.com/web?query=${encodeURIComponent(q)}&num=20`,
|
||
parse: parseSogou,
|
||
},
|
||
{
|
||
name: '360搜索',
|
||
weight: 75,
|
||
searchUrl: (q) => `https://www.so.com/s?q=${encodeURIComponent(q)}&pn=20`,
|
||
parse: parse360,
|
||
},
|
||
];
|
||
|
||
// ===== HTML 解析器（v0.4.1: node-html-parser 结构化解析为主层，正则为降级层） =====
|
||
|
||
/**
|
||
* v0.4.1: 从结果块中提取标题链接 — 跳过指向搜索引擎自身域名的链接（favicon/子导航等）
|
||
*/
|
||
function extractTitleLink(
|
||
block: HTMLElement,
|
||
selfDomain: string,
|
||
): { url: string; title: string } | null {
|
||
for (const a of block.querySelectorAll('a[href]')) {
|
||
const url = a.getAttribute('href') ?? '';
|
||
const title = a.text.trim();
|
||
if (title && url && !url.includes(selfDomain) && url.startsWith('http')) {
|
||
return { url, title };
|
||
}
|
||
}
|
||
return null;
|
||
}
|
||
|
||
/** v0.4.1: 提取第一个非空文本的选择器（按优先级尝试多个候选选择器） */
|
||
function extractText(block: HTMLElement, selectors: string[]): string {
|
||
for (const sel of selectors) {
|
||
const el = block.querySelector(sel);
|
||
if (el) {
|
||
const text = el.text.trim();
|
||
if (text) return text;
|
||
}
|
||
}
|
||
return '';
|
||
}
|
||
|
||
/** v0.4.1: Bing 结构化解析 — li.b_algo 结果块 */
|
||
function parseBingStructured(html: string): SearchResult[] {
|
||
const results: SearchResult[] = [];
|
||
const root = parseHtmlDom(html);
|
||
for (const block of root.querySelectorAll('li.b_algo')) {
|
||
const link = extractTitleLink(block, 'bing.com');
|
||
if (!link) continue;
|
||
const snippet = extractText(block, ['p', '.b_caption']);
|
||
results.push({ title: link.title, url: link.url, snippet, engine: 'bing', weight: 90 });
|
||
}
|
||
return results;
|
||
}
|
||
|
||
/** v0.4.1: 百度结构化解析 — div.result / div.c-container 结果块，优先 a[data-url] 真实链接 */
|
||
function parseBaiduStructured(html: string): SearchResult[] {
|
||
const results: SearchResult[] = [];
|
||
const root = parseHtmlDom(html);
|
||
// 复合选择器去重：class="result c-container" 的元素同时命中两个类名，
|
||
// 分别查询再拼接会重复收录同一结果块
|
||
const blocks = root.querySelectorAll('div.result, div.c-container');
|
||
for (const block of blocks) {
|
||
// 百度标题链接: 优先 data-url 属性（真实目标 URL），href 通常是 baidu.com/link 跳转
|
||
const dataUrlLink = block.querySelector('a[data-url]');
|
||
let url = dataUrlLink?.getAttribute('data-url') ?? '';
|
||
let title = dataUrlLink?.text.trim() ?? '';
|
||
if (!url || !title) {
|
||
const fallback = block.querySelector('h3 a[href]') ?? block.querySelector('a[href]');
|
||
if (fallback) {
|
||
const href = fallback.getAttribute('href') ?? '';
|
||
url = href.startsWith('http') ? href : href ? `https://${href}` : '';
|
||
title = fallback.text.trim();
|
||
}
|
||
}
|
||
const snippet = extractText(block, ['.c-abstract', '[class^="content-right"]']);
|
||
if (title && url && !url.includes('baidu.com/link')) {
|
||
results.push({ title, url, snippet, engine: '百度', weight: 80 });
|
||
}
|
||
}
|
||
return results;
|
||
}
|
||
|
||
/**
|
||
* v0.4.1: 搜狗结构化解析 — div.vrwrap / div.rb 结果块（相对链接补全 sogou.com 前缀）
|
||
*
|
||
* v0.4.1 修复（原正则实现遗留缺陷）: 搜狗结果链接是 sogou.com/link?url=... 跳转形式，
|
||
* 原 `!url.includes('sogou.com')` 过滤条件把所有跳转结果一并丢弃（相对链接补全后必含 sogou.com），
|
||
* 导致搜狗引擎基本无法返回结果。现仅过滤 sogou 自身页面链接，保留 /link 跳转结果
|
||
* （可达性预检会跟随重定向验证）。
|
||
*/
|
||
function parseSogouStructured(html: string): SearchResult[] {
|
||
const results: SearchResult[] = [];
|
||
const root = parseHtmlDom(html);
|
||
// 复合选择器避免同一元素命中两个类名时重复收录
|
||
const blocks = root.querySelectorAll('div.vrwrap, div.rb');
|
||
for (const block of blocks) {
|
||
const a = block.querySelector('h3 a[href]') ?? block.querySelector('a[href]');
|
||
if (!a) continue;
|
||
const href = a.getAttribute('href') ?? '';
|
||
const url = href.startsWith('http') ? href : `https://www.sogou.com${href}`;
|
||
const title = a.text.trim();
|
||
const snippet = extractText(block, ['.star-wiki', '.space-txt', '.str_info']);
|
||
// 过滤搜狗自身页面（保留 /link 跳转结果）
|
||
const isSelfPage = url.includes('sogou.com') && !url.includes('/link');
|
||
if (title && url && !isSelfPage) {
|
||
results.push({ title, url, snippet, engine: '搜狗', weight: 75 });
|
||
}
|
||
}
|
||
return results;
|
||
}
|
||
|
||
/** v0.4.1: 360 结构化解析 — li.res-list / div.result 结果块 */
|
||
function parse360Structured(html: string): SearchResult[] {
|
||
const results: SearchResult[] = [];
|
||
const root = parseHtmlDom(html);
|
||
// 复合选择器避免同一元素命中多个类名时重复收录
|
||
const blocks = root.querySelectorAll('li.res-list, div.result');
|
||
for (const block of blocks) {
|
||
const link = extractTitleLink(block, 'so.com');
|
||
if (!link) continue;
|
||
const snippet = extractText(block, ['.res-desc', '.res-rich', '.res-summary', 'dd']);
|
||
results.push({ title: link.title, url: link.url, snippet, engine: '360搜索', weight: 75 });
|
||
}
|
||
return results;
|
||
}
|
||
|
||
/** v0.4.1: 结构化解析 + 正则降级的组合入口（供 ENGINES 引用，测试导出） */
|
||
export function parseBing(html: string): SearchResult[] {
|
||
const structured = parseBingStructured(html);
|
||
if (structured.length > 0) return structured;
|
||
return parseBingRegex(html);
|
||
}
|
||
|
||
export function parseBaidu(html: string): SearchResult[] {
|
||
const structured = parseBaiduStructured(html);
|
||
if (structured.length > 0) return structured;
|
||
return parseBaiduRegex(html);
|
||
}
|
||
|
||
export function parseSogou(html: string): SearchResult[] {
|
||
const structured = parseSogouStructured(html);
|
||
if (structured.length > 0) return structured;
|
||
return parseSogouRegex(html);
|
||
}
|
||
|
||
export function parse360(html: string): SearchResult[] {
|
||
const structured = parse360Structured(html);
|
||
if (structured.length > 0) return structured;
|
||
return parse360Regex(html);
|
||
}
|
||
|
||
// ===== 正则降级解析器（v0.4.1 前的主实现，结构化解析无结果时兜底） =====
|
||
|
||
function parseBingRegex(html: string): SearchResult[] {
|
||
const results: SearchResult[] = [];
|
||
const blocks = html.split(/<li[^>]*class="b_algo"/i).slice(1);
|
||
for (const block of blocks) {
|
||
const titleMatch = block.match(/<a[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
|
||
if (!titleMatch) continue;
|
||
const url = titleMatch[1];
|
||
const title = titleMatch[2].replace(/<[^>]+>/g, '').trim();
|
||
const snippetMatch =
|
||
block.match(/<p[^>]*>([\s\S]*?)<\/p>/i) ||
|
||
block.match(/class="b_caption"[^>]*>([\s\S]*?)<\/div>/i);
|
||
const snippet = snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '').trim() : '';
|
||
if (title && url && !url.includes('bing.com')) {
|
||
results.push({ title, url, snippet, engine: 'bing', weight: 90 });
|
||
}
|
||
}
|
||
return results;
|
||
}
|
||
|
||
function parseBaiduRegex(html: string): SearchResult[] {
|
||
const results: SearchResult[] = [];
|
||
const blocks = html.split(/<div[^>]*class="result[^"]*"/i).slice(1);
|
||
for (const block of blocks) {
|
||
const titleMatch =
|
||
block.match(/<a[^>]*data-url="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i) ||
|
||
block.match(/<a[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
|
||
if (!titleMatch) continue;
|
||
const url = titleMatch[1].startsWith('http') ? titleMatch[1] : `https://${titleMatch[1]}`;
|
||
const title = titleMatch[2].replace(/<[^>]+>/g, '').trim();
|
||
const snippetMatch =
|
||
block.match(/class="c-abstract[^"]*"[^>]*>([\s\S]*?)<\/span>/i) ||
|
||
block.match(/class="content-right[^"]*"[^>]*>([\s\S]*?)<\/div>/i);
|
||
const snippet = snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '').trim() : '';
|
||
if (title && url && !url.includes('baidu.com/link')) {
|
||
results.push({ title, url, snippet, engine: '百度', weight: 80 });
|
||
}
|
||
}
|
||
return results;
|
||
}
|
||
|
||
function parseSogouRegex(html: string): SearchResult[] {
|
||
const results: SearchResult[] = [];
|
||
const blocks = html
|
||
.split(/<div[^>]*class="vrwrap"/i)
|
||
.slice(1)
|
||
.concat(html.split(/<div[^>]*class="rb"/i).slice(1));
|
||
for (const block of blocks) {
|
||
const titleMatch = block.match(/<a[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
|
||
if (!titleMatch) continue;
|
||
const url = titleMatch[1].startsWith('http')
|
||
? titleMatch[1]
|
||
: `https://www.sogou.com${titleMatch[1]}`;
|
||
const title = titleMatch[2].replace(/<[^>]+>/g, '').trim();
|
||
const snippetMatch =
|
||
block.match(/class="star-wiki[^"]*"[^>]*>([\s\S]*?)<\/div>/i) ||
|
||
block.match(/class="space-txt[^"]*"[^>]*>([\s\S]*?)<\/p>/i) ||
|
||
block.match(/class="str_info[^"]*"[^>]*>([\s\S]*?)<\/p>/i);
|
||
const snippet = snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '').trim() : '';
|
||
if (title && url && !url.includes('sogou.com')) {
|
||
results.push({ title, url, snippet, engine: '搜狗', weight: 75 });
|
||
}
|
||
}
|
||
return results;
|
||
}
|
||
|
||
function parse360Regex(html: string): SearchResult[] {
|
||
const results: SearchResult[] = [];
|
||
const blocks = html
|
||
.split(/<li[^>]*class="res-list"/i)
|
||
.slice(1)
|
||
.concat(html.split(/<div[^>]*class="result"/i).slice(1));
|
||
for (const block of blocks) {
|
||
const titleMatch = block.match(/<a[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
|
||
if (!titleMatch) continue;
|
||
const url = titleMatch[1];
|
||
const title = titleMatch[2].replace(/<[^>]+>/g, '').trim();
|
||
const snippetMatch =
|
||
block.match(/class="res-desc[^"]*"[^>]*>([\s\S]*?)<\/p>/i) ||
|
||
block.match(/class="res-rich[^"]*"[^>]*>([\s\S]*?)<\/div>/i) ||
|
||
block.match(/class="res-summary[^"]*"[^>]*>([\s\S]*?)<\/p>/i) ||
|
||
block.match(/<dd[^>]*>([\s\S]*?)<\/dd>/i);
|
||
const snippet = snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '').trim() : '';
|
||
if (title && url && !url.includes('so.com')) {
|
||
results.push({ title, url, snippet, engine: '360搜索', weight: 75 });
|
||
}
|
||
}
|
||
return results;
|
||
}
|
||
|
||
// ===== 可达性预检 =====
|
||
// v0.7.3 P2-1: 可达性预检的 URL 来自不可信的搜索结果 —— HEAD 探测同样不得
|
||
// 触达内网/元数据地址。校验失败的 URL 直接标记不可达（不发起任何请求）；
|
||
// 探测经 ssrfPinnedFetch（DNS pinning），重定向不自动跟随（3xx 即视为可达 ——
|
||
// 链接活性已证明，且跟跳目标不再绕过校验）。
|
||
|
||
async function checkReachability(urls: string[], concurrency = 5): Promise<Map<string, boolean>> {
|
||
const result = new Map<string, boolean>();
|
||
for (let i = 0; i < urls.length; i += concurrency) {
|
||
const batch = urls.slice(i, i + concurrency);
|
||
const checks = batch.map(async (url) => {
|
||
try {
|
||
const ssrf = await safeValidateSSRF(url);
|
||
if (!ssrf.ok) {
|
||
result.set(url, false);
|
||
return;
|
||
}
|
||
const resp = await ssrfPinnedFetch(url, { method: 'HEAD', redirect: 'manual' }, 3_000);
|
||
result.set(url, resp.ok || (resp.status >= 300 && resp.status < 400));
|
||
} catch {
|
||
result.set(url, false);
|
||
}
|
||
});
|
||
await Promise.allSettled(checks);
|
||
}
|
||
return result;
|
||
}
|
||
|
||
// ===== 智能排序 =====
|
||
|
||
function smartSort(results: SearchResult[]): SearchResult[] {
|
||
for (const r of results) {
|
||
const reachability = r.reachable ? 30 : -20;
|
||
const snippetQuality = (Math.min(r.snippet.length, 100) / 100) * 20;
|
||
const weightScore = (r.weight / 100) * 50;
|
||
r._score = weightScore + reachability + snippetQuality;
|
||
}
|
||
return results.sort((a, b) => (b._score ?? 0) - (a._score ?? 0));
|
||
}
|
||
|
||
// ===== 相关性过滤（用于自动抓取） =====
|
||
|
||
function computeRelevance(query: string, result: SearchResult): number {
|
||
const terms: string[] = [];
|
||
// 中文双字/三字片段
|
||
const chinese = query.match(/[\u4e00-\u9fa5]{2,3}/g);
|
||
if (chinese) terms.push(...chinese);
|
||
// 英文单词
|
||
const english = query.match(/[a-zA-Z]{2,}/g);
|
||
if (english) terms.push(...english);
|
||
|
||
let score = 0;
|
||
const text = `${result.title} ${result.snippet}`.toLowerCase();
|
||
for (const term of terms) {
|
||
if (text.includes(term.toLowerCase())) score += 10;
|
||
}
|
||
return Math.min(score, 100);
|
||
}
|
||
|
||
// ===== WebSearchTool =====
|
||
|
||
export class WebSearchTool implements IMetonaTool {
|
||
readonly definition: MetonaToolDef = {
|
||
name: 'web_search',
|
||
description:
|
||
'Search the web for information. Returns titles, snippets, and URLs. When SearXNG is enabled, uses the configured SearXNG instance; otherwise uses built-in engines (Bing, Baidu, Sogou, 360). Automatically fetches full content for top results.',
|
||
parameters: {
|
||
type: 'object',
|
||
properties: {
|
||
query: { type: 'string', description: 'Search query keywords' },
|
||
time_range: {
|
||
type: 'string',
|
||
description: 'Time filter: day, week, month, year (optional)',
|
||
},
|
||
enhance_snippets: {
|
||
type: 'boolean',
|
||
description: 'Auto-enhance short snippets (default true)',
|
||
},
|
||
},
|
||
required: ['query'],
|
||
},
|
||
category: MetonaToolCategory.NETWORK,
|
||
riskLevel: MetonaRiskLevel.LOW,
|
||
requiresPermission: false,
|
||
timeoutMs: 300_000,
|
||
};
|
||
|
||
constructor(
|
||
private configService: ConfigService,
|
||
private webFetchTool: WebFetchTool,
|
||
) {}
|
||
|
||
async execute(args: Record<string, unknown>, _context: ToolExecutionContext): Promise<unknown> {
|
||
const query = args.query as string;
|
||
|
||
// 读取 SearXNG 配置
|
||
const searxngConfig = this.readSearXNGConfig();
|
||
const useSearXNG = searxngConfig.enabled && !!searxngConfig.url;
|
||
|
||
// 两个模式参数完全独立：
|
||
// - 内置模式：固定默认值 maxResults=30, fetchTop=5
|
||
// - SearXNG 模式：由配置面板控制
|
||
const maxResults = useSearXNG
|
||
? Math.min(50, Math.max(1, searxngConfig.max_results > 0 ? searxngConfig.max_results : 30))
|
||
: 30;
|
||
|
||
const timeRange = (args.time_range as string) ?? '';
|
||
const enhanceSnippets = (args.enhance_snippets as boolean) ?? true;
|
||
|
||
const fetchTop = useSearXNG
|
||
? Math.min(8, Math.max(3, searxngConfig.fetch_count > 0 ? searxngConfig.fetch_count : 5))
|
||
: 5;
|
||
|
||
logTool(
|
||
'web_search',
|
||
`Mode=${useSearXNG ? 'searxng' : 'builtin'}, maxResults=${maxResults}, fetchTop=${fetchTop}`,
|
||
);
|
||
|
||
// 缓存检查（key 含模式 + maxResults + fetchTop，避免配置变更后返回旧缓存）
|
||
const cacheKey = `${searxngConfig.enabled ? 'searxng' : 'builtin'}:${maxResults}:${fetchTop}:${normalizeUrl(query).toLowerCase()}`;
|
||
const cached = searchCache.get(cacheKey);
|
||
if (cached) {
|
||
logTool('web_search', `Cache hit: "${query}"`);
|
||
return { ...(cached as Record<string, unknown>), from_cache: true };
|
||
}
|
||
|
||
let results: SearchResult[];
|
||
let mode: string;
|
||
let engineStats: Record<string, string>;
|
||
|
||
if (useSearXNG) {
|
||
const searxResult = await this.searchSearXNG(query, searxngConfig, maxResults, timeRange);
|
||
results = searxResult.results;
|
||
mode = 'searxng';
|
||
engineStats = searxResult.engineStats;
|
||
} else {
|
||
const builtinResult = await this.searchBuiltinEngines(query, maxResults, timeRange);
|
||
results = builtinResult.results;
|
||
mode = 'builtin';
|
||
engineStats = builtinResult.engineStats;
|
||
}
|
||
|
||
// 合并去重
|
||
const deduped = this.deduplicate(results);
|
||
|
||
// 可达性预检
|
||
const topUrls = deduped.slice(0, Math.min(20, deduped.length)).map((r) => r.url);
|
||
const reachabilityMap = await checkReachability(topUrls);
|
||
for (const r of deduped) {
|
||
r.reachable = reachabilityMap.get(r.url) ?? false;
|
||
}
|
||
|
||
// 智能排序
|
||
const sorted = smartSort(deduped).slice(0, maxResults);
|
||
|
||
// 摘要增强
|
||
if (enhanceSnippets) {
|
||
await this.enhanceSnippets(sorted, 3);
|
||
}
|
||
|
||
// 自动抓取完整内容
|
||
const fetchedContent = await this.autoFetch(query, sorted, searxngConfig.fetch_mode, fetchTop);
|
||
|
||
// 格式化输出
|
||
const formatted = this.formatResults(query, sorted);
|
||
|
||
const output = {
|
||
success: true,
|
||
query,
|
||
results: sorted.map((r) => ({
|
||
title: r.title,
|
||
url: r.url,
|
||
snippet: r.snippet,
|
||
engine: r.engine,
|
||
reachable: r.reachable,
|
||
_score: r._score ? Math.round(r._score * 10) / 10 : undefined,
|
||
_enhanced: r._enhanced,
|
||
})),
|
||
total: sorted.length,
|
||
formatted,
|
||
from_cache: false,
|
||
engine_stats: engineStats,
|
||
_mode: mode,
|
||
_fetched: fetchedContent.map((f) => ({ url: f.url, title: f.title, content: f.content })),
|
||
_fetched_count: fetchedContent.length,
|
||
};
|
||
|
||
// 写入缓存
|
||
searchCache.set(cacheKey, output);
|
||
logTool(
|
||
'web_search',
|
||
`Completed: ${sorted.length} results, ${fetchedContent.length} fetched, mode=${mode}`,
|
||
);
|
||
|
||
return output;
|
||
}
|
||
|
||
// ===== SearXNG 搜索 =====
|
||
|
||
private async searchSearXNG(
|
||
query: string,
|
||
config: SearXNGConfig,
|
||
maxResults: number,
|
||
timeRange: string,
|
||
): Promise<{ results: SearchResult[]; engineStats: Record<string, string> }> {
|
||
const results: SearchResult[] = [];
|
||
const engineStats: Record<string, string> = {};
|
||
const engineCounts = new Map<string, number>();
|
||
const baseUrl = config.url.replace(/\/$/, '');
|
||
const headers = buildSearXNGAuthHeaders(config.auth_key, config.auth_type);
|
||
const tr = timeRange || config.time_range;
|
||
|
||
// SearXNG 标准分页：每页由实例配置决定（通常 10 条），用 pageno 翻页直到达到 maxResults
|
||
const maxPages = Math.ceil(maxResults / 5) + 1; // 保守估计，每页至少 5 条
|
||
let page = 1;
|
||
|
||
while (results.length < maxResults && page <= maxPages) {
|
||
const params = new URLSearchParams();
|
||
params.set('q', query);
|
||
params.set('format', config.format || 'json');
|
||
params.set('pageno', String(page));
|
||
if (config.engines) params.set('engines', config.engines);
|
||
if (config.language && config.language !== 'auto') params.set('language', config.language);
|
||
params.set('safesearch', String(config.safesearch));
|
||
if (tr) params.set('time_range', tr);
|
||
|
||
const searchUrl = `${baseUrl}/search?${params.toString()}`;
|
||
logTool('web_search', `[SearXNG] Fetching page ${page}: ${searchUrl}`);
|
||
|
||
const pageResults: SearchResult[] = [];
|
||
|
||
try {
|
||
const response = await fetchWithTimeout(searchUrl, { headers }, 15_000);
|
||
if (!response.ok) {
|
||
throw new Error(`SearXNG API error: ${response.status} ${response.statusText}`);
|
||
}
|
||
|
||
if (config.format === 'html') {
|
||
const html = await response.text();
|
||
const matches = html.matchAll(/<a[^>]*href="(https?:\/\/[^"]+)"[^>]*>([\s\S]*?)<\/a>/gi);
|
||
for (const m of matches) {
|
||
const url = m[1];
|
||
const title = m[2].replace(/<[^>]+>/g, '').trim();
|
||
if (title && url && !url.includes(config.url)) {
|
||
pageResults.push({ title, url, snippet: '', engine: 'searxng', weight: 85 });
|
||
}
|
||
}
|
||
} else {
|
||
const data = (await response.json()) as { results?: Array<Record<string, unknown>> };
|
||
for (const item of data.results ?? []) {
|
||
const url = item.url as string;
|
||
const title = item.title as string;
|
||
const snippet = item.content as string;
|
||
const engine = (item.engine as string) ?? 'searxng';
|
||
pageResults.push({ title, url, snippet: snippet ?? '', engine, weight: 85 });
|
||
engineCounts.set(engine, (engineCounts.get(engine) ?? 0) + 1);
|
||
}
|
||
}
|
||
} catch (err) {
|
||
logTool('web_search', `[SearXNG] Page ${page} error: ${(err as Error).message}`);
|
||
// 第一页就失败则抛出，后续页失败则用已获取的结果
|
||
if (page === 1) throw err;
|
||
break;
|
||
}
|
||
|
||
// 本页无新结果 → 已到末尾
|
||
if (pageResults.length === 0) {
|
||
logTool('web_search', `[SearXNG] Page ${page} returned 0 results, stopping pagination`);
|
||
break;
|
||
}
|
||
|
||
results.push(...pageResults);
|
||
logTool(
|
||
'web_search',
|
||
`[SearXNG] Page ${page}: +${pageResults.length} (total ${results.length})`,
|
||
);
|
||
page++;
|
||
}
|
||
|
||
for (const [engine, count] of engineCounts) {
|
||
engineStats[engine] = `${count} 条`;
|
||
}
|
||
|
||
logTool('web_search', `[SearXNG] Total: ${results.length} raw results (target ${maxResults})`);
|
||
return { results, engineStats };
|
||
}
|
||
|
||
// ===== 内置四引擎并行搜索 =====
|
||
|
||
private async searchBuiltinEngines(
|
||
query: string,
|
||
maxResults: number,
|
||
timeRange: string,
|
||
): Promise<{ results: SearchResult[]; engineStats: Record<string, string> }> {
|
||
const searchPromises = ENGINES.map(async (engine) => {
|
||
try {
|
||
const url = engine.searchUrl(query, timeRange);
|
||
const response = await fetchWithTimeout(
|
||
url,
|
||
{
|
||
headers: {
|
||
'User-Agent':
|
||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36',
|
||
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
|
||
},
|
||
},
|
||
8_000,
|
||
);
|
||
|
||
if (!response.ok) {
|
||
logTool('web_search', `[内置] ${engine.name} HTTP ${response.status}`);
|
||
return [];
|
||
}
|
||
|
||
const html = await response.text();
|
||
const parsed = engine.parse(html);
|
||
logTool('web_search', `[内置] ${engine.name}: ${parsed.length} results`);
|
||
return parsed;
|
||
} catch (err) {
|
||
logTool('web_search', `[内置] ${engine.name} error: ${(err as Error).message}`);
|
||
return [];
|
||
}
|
||
});
|
||
|
||
const settled = await Promise.allSettled(searchPromises);
|
||
const allResults: SearchResult[] = [];
|
||
const engineStats: Record<string, string> = {};
|
||
|
||
for (let i = 0; i < ENGINES.length; i++) {
|
||
const result = settled[i];
|
||
const engineName = ENGINES[i].name;
|
||
if (result.status === 'fulfilled') {
|
||
allResults.push(...result.value);
|
||
engineStats[engineName] = `${result.value.length} 条`;
|
||
} else {
|
||
engineStats[engineName] = '失败';
|
||
}
|
||
}
|
||
|
||
return { results: allResults, engineStats };
|
||
}
|
||
|
||
// ===== 合并去重 =====
|
||
|
||
private deduplicate(results: SearchResult[]): SearchResult[] {
|
||
const seen = new Map<string, SearchResult>();
|
||
for (const r of results) {
|
||
const normalized = normalizeUrl(r.url);
|
||
if (!seen.has(normalized)) {
|
||
seen.set(normalized, r);
|
||
}
|
||
}
|
||
return Array.from(seen.values());
|
||
}
|
||
|
||
// ===== 摘要增强（委托给 WebFetchTool） =====
|
||
|
||
private async enhanceSnippets(results: SearchResult[], maxEnhance: number): Promise<void> {
|
||
let enhanced = 0;
|
||
for (const r of results) {
|
||
if (enhanced >= maxEnhance) break;
|
||
if (r.snippet.length < 30 && r.reachable) {
|
||
try {
|
||
const fetchResult = (await this.webFetchTool.execute(
|
||
{ url: r.url },
|
||
{ sessionId: '', workspacePath: '', iteration: 0, requestId: '' },
|
||
)) as { success: boolean; content?: string };
|
||
|
||
if (fetchResult.success && fetchResult.content) {
|
||
const text = fetchResult.content.slice(0, 200);
|
||
if (text.length > r.snippet.length) {
|
||
r.snippet = text;
|
||
r._enhanced = true;
|
||
enhanced++;
|
||
}
|
||
}
|
||
} catch {
|
||
// 忽略增强失败
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// ===== 自动抓取完整内容（委托给 WebFetchTool） =====
|
||
|
||
/**
|
||
* P2-12: 自动抓取改为并行（批次并发 3）
|
||
*
|
||
* 原实现逐条串行抓取（单个 web_fetch 最长 120s 超时），top5 结果最坏耗时
|
||
* 逼近 web_search 的 300s 工具超时上限。并行批次化后总耗时约降至 1/3。
|
||
* 失败结果直接跳过（原"随机补充重试"逻辑收益边际，复杂度高，已移除）。
|
||
*/
|
||
private async autoFetch(
|
||
query: string,
|
||
results: SearchResult[],
|
||
fetchMode: string,
|
||
fetchTop: number,
|
||
): Promise<Array<{ url: string; title: string; content: string }>> {
|
||
// 相关性评分（不过滤，relevance=0 的结果也参与抓取候选）
|
||
const withRelevance = results.map((r) => ({
|
||
result: r,
|
||
relevance: computeRelevance(query, r),
|
||
}));
|
||
const filtered = withRelevance.length > 0 ? withRelevance : [];
|
||
|
||
// 确定抓取数量：fetchTop 已在 execute() 中综合了配置面板和工具参数
|
||
const topN = Math.min(fetchTop, 8, filtered.length);
|
||
|
||
// 构建抓取列表
|
||
let toFetch: typeof filtered;
|
||
if (fetchMode === 'random') {
|
||
// Fisher-Yates 洗牌
|
||
const shuffled = [...filtered];
|
||
for (let i = shuffled.length - 1; i > 0; i--) {
|
||
const j = Math.floor(Math.random() * (i + 1));
|
||
[shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
|
||
}
|
||
toFetch = shuffled.slice(0, topN);
|
||
} else {
|
||
toFetch = filtered.sort((a, b) => b.relevance - a.relevance).slice(0, topN);
|
||
}
|
||
|
||
const fetched: Array<{ url: string; title: string; content: string }> = [];
|
||
|
||
const fetchOne = async (item: {
|
||
result: SearchResult;
|
||
}): Promise<{ url: string; title: string; content: string } | null> => {
|
||
try {
|
||
// 委托给 WebFetchTool — 享受三阶段回退策略（HTTP + 反爬 + 浏览器渲染）
|
||
const fetchResult = (await this.webFetchTool.execute(
|
||
{ url: item.result.url },
|
||
{ sessionId: '', workspacePath: '', iteration: 0, requestId: '' },
|
||
)) as { success: boolean; content?: string };
|
||
|
||
if (fetchResult.success && fetchResult.content) {
|
||
return { url: item.result.url, title: item.result.title, content: fetchResult.content };
|
||
}
|
||
return null;
|
||
} catch (err) {
|
||
logTool(
|
||
'web_search',
|
||
`Auto-fetch failed for ${item.result.url}: ${(err as Error).message}`,
|
||
);
|
||
return null;
|
||
}
|
||
};
|
||
|
||
// 并行批次抓取（并发 3）
|
||
const CONCURRENCY = 3;
|
||
for (let i = 0; i < toFetch.length; i += CONCURRENCY) {
|
||
const batch = toFetch.slice(i, i + CONCURRENCY);
|
||
const settled = await Promise.allSettled(batch.map((item) => fetchOne(item)));
|
||
for (const r of settled) {
|
||
if (r.status === 'fulfilled' && r.value) fetched.push(r.value);
|
||
}
|
||
}
|
||
|
||
return fetched;
|
||
}
|
||
|
||
// ===== 格式化人类可读输出 =====
|
||
|
||
private formatResults(query: string, results: SearchResult[]): string {
|
||
if (results.length === 0) return `搜索 "${query}" 无结果。`;
|
||
const lines = [`搜索 "${query}" — ${results.length} 条结果：\n`];
|
||
results.forEach((r, i) => {
|
||
lines.push(`${i + 1}. ${r.title}`);
|
||
lines.push(` URL: ${r.url}`);
|
||
if (r.snippet) lines.push(` 摘要: ${r.snippet.slice(0, 150)}`);
|
||
lines.push(
|
||
` 来源: ${r.engine}${r.reachable === false ? ' (不可达)' : ''}${r._enhanced ? ' [已增强]' : ''}\n`,
|
||
);
|
||
});
|
||
return lines.join('\n');
|
||
}
|
||
|
||
// ===== 读取 SearXNG 配置 =====
|
||
|
||
private readSearXNGConfig(): SearXNGConfig {
|
||
const cs = this.configService;
|
||
return {
|
||
enabled: cs.get<boolean>('searxng.enabled') ?? false,
|
||
url: cs.get<string>('searxng.url') ?? '',
|
||
engines: cs.get<string>('searxng.engines') ?? '',
|
||
language: cs.get<string>('searxng.language') ?? 'zh-CN',
|
||
safesearch: cs.get<number>('searxng.safesearch') ?? 1,
|
||
time_range: cs.get<string>('searxng.time_range') ?? '',
|
||
max_results: cs.get<number>('searxng.max_results') ?? 0,
|
||
auth_key: cs.get<string>('searxng.auth_key') ?? '',
|
||
auth_type: cs.get<string>('searxng.auth_type') ?? 'bearer',
|
||
format: cs.get<string>('searxng.format') ?? 'json',
|
||
fetch_count: cs.get<number>('searxng.fetch_count') ?? 0,
|
||
fetch_mode: cs.get<string>('searxng.fetch_mode') ?? 'sequential',
|
||
};
|
||
}
|
||
}
|