thzxx
49c9b25538
工程化（从零到一）: - 新增 Gitea Actions CI（debian-latest）：类型检查 + Lint + 单元测试 + 产物编译验证 - 新增 husky + lint-staged 预提交钩子（lint-staged + typecheck 门禁） - 移除坏脚本 test:e2e（无 Playwright 配置必失败）；prebuild 改用内置 fs.rmSync - 依赖清理：移除死依赖 sql.js(2MB)/@playwright/test，@types/shell-quote 移至 devDependencies 安全加固: - PolicyEngine 频率限制按会话隔离（多会话并发不再互抢配额） - ConfirmationHook 拒绝记忆加 10 分钟 TTL + 恢复询问入口（新增 2 个 IPC 通道） - Windows run_command 白名单工具（git/node/npm/npx/pnpm/yarn/tsc）改走 cmd.exe /c + 参数数组执行，收窄 shell 注入面 - web_search 四引擎 HTML 解析迁移 node-html-parser（结构化主层 + 正则降级） 缺陷修复（测试驱动发现）: - mapError 大小写缺陷：网络错误码永远落入 UNKNOWN 无法触发重试 - 搜狗解析器自我过滤：相对链接补全后又被 sogou.com 过滤导致结果全丢 - 百度复合类名重复收录：class="result c-container" 被双重匹配 测试补齐（113 → 194 用例）: - 新增 5 个测试文件：sse-stream / base-adapter / confirmation-hook / ipc-agent 编排链路 / web-search 解析器 - 覆盖 sendMessage 全分支、SSE 流解析、错误映射、确认钩子竞态/超时/批量审批 体验升级: - OutputValidator 验证结果可见化（VALIDATION 流事件 → 聊天流提示卡） - SettingsModal 巨型组件拆分（1503 行 → 10 个文件，可独立维护） - MessageList 接入 react-virtuoso 真虚拟滚动（千条消息恒定开销） - MCP 新增 streamable HTTP 传输支持（SDK 内置传输 + DB 迁移 6 + UI 双模式）
819 lines
29 KiB
TypeScript
819 lines
29 KiB
TypeScript
/**
|
||
* web_search — 网络搜索工具
|
||
*
|
||
* 双模式搜索：SearXNG 元搜索 / 内置四引擎并行搜索
|
||
* 内置引擎：Bing + 百度 + 搜狗 + 360 搜索（Promise.allSettled 容错并发）
|
||
* 智能排序：引擎权重(50%) + 可达性(30%) + 摘要质量(20%)
|
||
* 自动抓取：对前 N 条结果调用 web_fetch 获取完整正文
|
||
*
|
||
* v0.4.1: HTML 解析迁移至 node-html-parser（结构化解析）
|
||
* 主层使用 DOM 结构解析（引擎改版时选择器更精确、可维护性远优于正则），
|
||
* 正则解析保留为降级路径（结构化解析无结果时兜底）。
|
||
* 此前纯正则方案违反项目开发规范第一铁律（HTML 解析应使用成熟库）。
|
||
*
|
||
* @see docs/Agent网络工具通用设计-v2.md — 第 2 章 web_search 搜索设计
|
||
*/
|
||
|
||
import { parse as parseHtmlDom, type HTMLElement } from 'node-html-parser';
|
||
import type { IMetonaTool, ToolExecutionContext } from '../../types/metona-tool';
|
||
import type { MetonaToolDef } from '../../../harness/types';
|
||
import { MetonaToolCategory, MetonaRiskLevel } from '../../../harness/types';
|
||
import type { ConfigService } from '../../../services/config.service';
|
||
import {
|
||
searchCache,
|
||
normalizeUrl,
|
||
fetchWithTimeout,
|
||
buildSearXNGAuthHeaders,
|
||
logTool,
|
||
} from './network-utils';
|
||
import type { WebFetchTool } from './web-fetch';
|
||
|
||
// ===== 类型定义 =====
|
||
|
||
interface SearchResult {
|
||
title: string;
|
||
url: string;
|
||
snippet: string;
|
||
engine: string;
|
||
weight: number;
|
||
reachable?: boolean;
|
||
_score?: number;
|
||
_enhanced?: boolean;
|
||
}
|
||
|
||
interface SearXNGConfig {
|
||
enabled: boolean;
|
||
url: string;
|
||
engines: string;
|
||
language: string;
|
||
safesearch: number;
|
||
time_range: string;
|
||
max_results: number;
|
||
auth_key: string;
|
||
auth_type: string;
|
||
format: string;
|
||
fetch_count: number;
|
||
fetch_mode: string;
|
||
}
|
||
|
||
// ===== 引擎定义 =====
|
||
|
||
interface EngineDef {
|
||
name: string;
|
||
weight: number;
|
||
searchUrl: (query: string, timeRange?: string) => string;
|
||
parse: (html: string) => SearchResult[];
|
||
}
|
||
|
||
const ENGINES: EngineDef[] = [
|
||
{
|
||
name: 'bing',
|
||
weight: 90,
|
||
searchUrl: (q, tr) => {
|
||
const freshness = tr
|
||
? `&filters=ex1:"ez${tr === 'day' ? '1' : tr === 'week' ? '2' : tr === 'month' ? '3' : '4'}"`
|
||
: '';
|
||
return `https://www.bing.com/search?q=${encodeURIComponent(q)}${freshness}&count=20`;
|
||
},
|
||
parse: parseBing,
|
||
},
|
||
{
|
||
name: '百度',
|
||
weight: 80,
|
||
searchUrl: (q) => `https://www.baidu.com/s?wd=${encodeURIComponent(q)}&rn=20`,
|
||
parse: parseBaidu,
|
||
},
|
||
{
|
||
name: '搜狗',
|
||
weight: 75,
|
||
searchUrl: (q) => `https://www.sogou.com/web?query=${encodeURIComponent(q)}&num=20`,
|
||
parse: parseSogou,
|
||
},
|
||
{
|
||
name: '360搜索',
|
||
weight: 75,
|
||
searchUrl: (q) => `https://www.so.com/s?q=${encodeURIComponent(q)}&pn=20`,
|
||
parse: parse360,
|
||
},
|
||
];
|
||
|
||
// ===== HTML 解析器（v0.4.1: node-html-parser 结构化解析为主层，正则为降级层） =====
|
||
|
||
/**
|
||
* v0.4.1: 从结果块中提取标题链接 — 跳过指向搜索引擎自身域名的链接（favicon/子导航等）
|
||
*/
|
||
function extractTitleLink(
|
||
block: HTMLElement,
|
||
selfDomain: string,
|
||
): { url: string; title: string } | null {
|
||
for (const a of block.querySelectorAll('a[href]')) {
|
||
const url = a.getAttribute('href') ?? '';
|
||
const title = a.text.trim();
|
||
if (title && url && !url.includes(selfDomain) && url.startsWith('http')) {
|
||
return { url, title };
|
||
}
|
||
}
|
||
return null;
|
||
}
|
||
|
||
/** v0.4.1: 提取第一个非空文本的选择器（按优先级尝试多个候选选择器） */
|
||
function extractText(block: HTMLElement, selectors: string[]): string {
|
||
for (const sel of selectors) {
|
||
const el = block.querySelector(sel);
|
||
if (el) {
|
||
const text = el.text.trim();
|
||
if (text) return text;
|
||
}
|
||
}
|
||
return '';
|
||
}
|
||
|
||
/** v0.4.1: Bing 结构化解析 — li.b_algo 结果块 */
|
||
function parseBingStructured(html: string): SearchResult[] {
|
||
const results: SearchResult[] = [];
|
||
const root = parseHtmlDom(html);
|
||
for (const block of root.querySelectorAll('li.b_algo')) {
|
||
const link = extractTitleLink(block, 'bing.com');
|
||
if (!link) continue;
|
||
const snippet = extractText(block, ['p', '.b_caption']);
|
||
results.push({ title: link.title, url: link.url, snippet, engine: 'bing', weight: 90 });
|
||
}
|
||
return results;
|
||
}
|
||
|
||
/** v0.4.1: 百度结构化解析 — div.result / div.c-container 结果块，优先 a[data-url] 真实链接 */
|
||
function parseBaiduStructured(html: string): SearchResult[] {
|
||
const results: SearchResult[] = [];
|
||
const root = parseHtmlDom(html);
|
||
// 复合选择器去重：class="result c-container" 的元素同时命中两个类名，
|
||
// 分别查询再拼接会重复收录同一结果块
|
||
const blocks = root.querySelectorAll('div.result, div.c-container');
|
||
for (const block of blocks) {
|
||
// 百度标题链接: 优先 data-url 属性（真实目标 URL），href 通常是 baidu.com/link 跳转
|
||
const dataUrlLink = block.querySelector('a[data-url]');
|
||
let url = dataUrlLink?.getAttribute('data-url') ?? '';
|
||
let title = dataUrlLink?.text.trim() ?? '';
|
||
if (!url || !title) {
|
||
const fallback = block.querySelector('h3 a[href]') ?? block.querySelector('a[href]');
|
||
if (fallback) {
|
||
const href = fallback.getAttribute('href') ?? '';
|
||
url = href.startsWith('http') ? href : href ? `https://${href}` : '';
|
||
title = fallback.text.trim();
|
||
}
|
||
}
|
||
const snippet = extractText(block, ['.c-abstract', '[class^="content-right"]']);
|
||
if (title && url && !url.includes('baidu.com/link')) {
|
||
results.push({ title, url, snippet, engine: '百度', weight: 80 });
|
||
}
|
||
}
|
||
return results;
|
||
}
|
||
|
||
/**
|
||
* v0.4.1: 搜狗结构化解析 — div.vrwrap / div.rb 结果块（相对链接补全 sogou.com 前缀）
|
||
*
|
||
* v0.4.1 修复（原正则实现遗留缺陷）: 搜狗结果链接是 sogou.com/link?url=... 跳转形式，
|
||
* 原 `!url.includes('sogou.com')` 过滤条件把所有跳转结果一并丢弃（相对链接补全后必含 sogou.com），
|
||
* 导致搜狗引擎基本无法返回结果。现仅过滤 sogou 自身页面链接，保留 /link 跳转结果
|
||
* （可达性预检会跟随重定向验证）。
|
||
*/
|
||
function parseSogouStructured(html: string): SearchResult[] {
|
||
const results: SearchResult[] = [];
|
||
const root = parseHtmlDom(html);
|
||
// 复合选择器避免同一元素命中两个类名时重复收录
|
||
const blocks = root.querySelectorAll('div.vrwrap, div.rb');
|
||
for (const block of blocks) {
|
||
const a = block.querySelector('h3 a[href]') ?? block.querySelector('a[href]');
|
||
if (!a) continue;
|
||
const href = a.getAttribute('href') ?? '';
|
||
const url = href.startsWith('http') ? href : `https://www.sogou.com${href}`;
|
||
const title = a.text.trim();
|
||
const snippet = extractText(block, ['.star-wiki', '.space-txt', '.str_info']);
|
||
// 过滤搜狗自身页面（保留 /link 跳转结果）
|
||
const isSelfPage = url.includes('sogou.com') && !url.includes('/link');
|
||
if (title && url && !isSelfPage) {
|
||
results.push({ title, url, snippet, engine: '搜狗', weight: 75 });
|
||
}
|
||
}
|
||
return results;
|
||
}
|
||
|
||
/** v0.4.1: 360 结构化解析 — li.res-list / div.result 结果块 */
|
||
function parse360Structured(html: string): SearchResult[] {
|
||
const results: SearchResult[] = [];
|
||
const root = parseHtmlDom(html);
|
||
// 复合选择器避免同一元素命中多个类名时重复收录
|
||
const blocks = root.querySelectorAll('li.res-list, div.result');
|
||
for (const block of blocks) {
|
||
const link = extractTitleLink(block, 'so.com');
|
||
if (!link) continue;
|
||
const snippet = extractText(block, ['.res-desc', '.res-rich', '.res-summary', 'dd']);
|
||
results.push({ title: link.title, url: link.url, snippet, engine: '360搜索', weight: 75 });
|
||
}
|
||
return results;
|
||
}
|
||
|
||
/** v0.4.1: 结构化解析 + 正则降级的组合入口（供 ENGINES 引用，测试导出） */
|
||
export function parseBing(html: string): SearchResult[] {
|
||
const structured = parseBingStructured(html);
|
||
if (structured.length > 0) return structured;
|
||
return parseBingRegex(html);
|
||
}
|
||
|
||
export function parseBaidu(html: string): SearchResult[] {
|
||
const structured = parseBaiduStructured(html);
|
||
if (structured.length > 0) return structured;
|
||
return parseBaiduRegex(html);
|
||
}
|
||
|
||
export function parseSogou(html: string): SearchResult[] {
|
||
const structured = parseSogouStructured(html);
|
||
if (structured.length > 0) return structured;
|
||
return parseSogouRegex(html);
|
||
}
|
||
|
||
export function parse360(html: string): SearchResult[] {
|
||
const structured = parse360Structured(html);
|
||
if (structured.length > 0) return structured;
|
||
return parse360Regex(html);
|
||
}
|
||
|
||
// ===== 正则降级解析器（v0.4.1 前的主实现，结构化解析无结果时兜底） =====
|
||
|
||
function parseBingRegex(html: string): SearchResult[] {
|
||
const results: SearchResult[] = [];
|
||
const blocks = html.split(/<li[^>]*class="b_algo"/i).slice(1);
|
||
for (const block of blocks) {
|
||
const titleMatch = block.match(/<a[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
|
||
if (!titleMatch) continue;
|
||
const url = titleMatch[1];
|
||
const title = titleMatch[2].replace(/<[^>]+>/g, '').trim();
|
||
const snippetMatch =
|
||
block.match(/<p[^>]*>([\s\S]*?)<\/p>/i) ||
|
||
block.match(/class="b_caption"[^>]*>([\s\S]*?)<\/div>/i);
|
||
const snippet = snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '').trim() : '';
|
||
if (title && url && !url.includes('bing.com')) {
|
||
results.push({ title, url, snippet, engine: 'bing', weight: 90 });
|
||
}
|
||
}
|
||
return results;
|
||
}
|
||
|
||
function parseBaiduRegex(html: string): SearchResult[] {
|
||
const results: SearchResult[] = [];
|
||
const blocks = html.split(/<div[^>]*class="result[^"]*"/i).slice(1);
|
||
for (const block of blocks) {
|
||
const titleMatch =
|
||
block.match(/<a[^>]*data-url="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i) ||
|
||
block.match(/<a[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
|
||
if (!titleMatch) continue;
|
||
const url = titleMatch[1].startsWith('http') ? titleMatch[1] : `https://${titleMatch[1]}`;
|
||
const title = titleMatch[2].replace(/<[^>]+>/g, '').trim();
|
||
const snippetMatch =
|
||
block.match(/class="c-abstract[^"]*"[^>]*>([\s\S]*?)<\/span>/i) ||
|
||
block.match(/class="content-right[^"]*"[^>]*>([\s\S]*?)<\/div>/i);
|
||
const snippet = snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '').trim() : '';
|
||
if (title && url && !url.includes('baidu.com/link')) {
|
||
results.push({ title, url, snippet, engine: '百度', weight: 80 });
|
||
}
|
||
}
|
||
return results;
|
||
}
|
||
|
||
function parseSogouRegex(html: string): SearchResult[] {
|
||
const results: SearchResult[] = [];
|
||
const blocks = html
|
||
.split(/<div[^>]*class="vrwrap"/i)
|
||
.slice(1)
|
||
.concat(html.split(/<div[^>]*class="rb"/i).slice(1));
|
||
for (const block of blocks) {
|
||
const titleMatch = block.match(/<a[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
|
||
if (!titleMatch) continue;
|
||
const url = titleMatch[1].startsWith('http')
|
||
? titleMatch[1]
|
||
: `https://www.sogou.com${titleMatch[1]}`;
|
||
const title = titleMatch[2].replace(/<[^>]+>/g, '').trim();
|
||
const snippetMatch =
|
||
block.match(/class="star-wiki[^"]*"[^>]*>([\s\S]*?)<\/div>/i) ||
|
||
block.match(/class="space-txt[^"]*"[^>]*>([\s\S]*?)<\/p>/i) ||
|
||
block.match(/class="str_info[^"]*"[^>]*>([\s\S]*?)<\/p>/i);
|
||
const snippet = snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '').trim() : '';
|
||
if (title && url && !url.includes('sogou.com')) {
|
||
results.push({ title, url, snippet, engine: '搜狗', weight: 75 });
|
||
}
|
||
}
|
||
return results;
|
||
}
|
||
|
||
function parse360Regex(html: string): SearchResult[] {
|
||
const results: SearchResult[] = [];
|
||
const blocks = html
|
||
.split(/<li[^>]*class="res-list"/i)
|
||
.slice(1)
|
||
.concat(html.split(/<div[^>]*class="result"/i).slice(1));
|
||
for (const block of blocks) {
|
||
const titleMatch = block.match(/<a[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
|
||
if (!titleMatch) continue;
|
||
const url = titleMatch[1];
|
||
const title = titleMatch[2].replace(/<[^>]+>/g, '').trim();
|
||
const snippetMatch =
|
||
block.match(/class="res-desc[^"]*"[^>]*>([\s\S]*?)<\/p>/i) ||
|
||
block.match(/class="res-rich[^"]*"[^>]*>([\s\S]*?)<\/div>/i) ||
|
||
block.match(/class="res-summary[^"]*"[^>]*>([\s\S]*?)<\/p>/i) ||
|
||
block.match(/<dd[^>]*>([\s\S]*?)<\/dd>/i);
|
||
const snippet = snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '').trim() : '';
|
||
if (title && url && !url.includes('so.com')) {
|
||
results.push({ title, url, snippet, engine: '360搜索', weight: 75 });
|
||
}
|
||
}
|
||
return results;
|
||
}
|
||
|
||
// ===== 可达性预检 =====
|
||
|
||
async function checkReachability(urls: string[], concurrency = 5): Promise<Map<string, boolean>> {
|
||
const result = new Map<string, boolean>();
|
||
for (let i = 0; i < urls.length; i += concurrency) {
|
||
const batch = urls.slice(i, i + concurrency);
|
||
const checks = batch.map(async (url) => {
|
||
try {
|
||
const resp = await fetchWithTimeout(url, { method: 'HEAD', redirect: 'follow' }, 3_000);
|
||
result.set(url, resp.ok);
|
||
} catch {
|
||
result.set(url, false);
|
||
}
|
||
});
|
||
await Promise.allSettled(checks);
|
||
}
|
||
return result;
|
||
}
|
||
|
||
// ===== 智能排序 =====
|
||
|
||
function smartSort(results: SearchResult[]): SearchResult[] {
|
||
for (const r of results) {
|
||
const reachability = r.reachable ? 30 : -20;
|
||
const snippetQuality = (Math.min(r.snippet.length, 100) / 100) * 20;
|
||
const weightScore = (r.weight / 100) * 50;
|
||
r._score = weightScore + reachability + snippetQuality;
|
||
}
|
||
return results.sort((a, b) => (b._score ?? 0) - (a._score ?? 0));
|
||
}
|
||
|
||
// ===== 相关性过滤（用于自动抓取） =====
|
||
|
||
function computeRelevance(query: string, result: SearchResult): number {
|
||
const terms: string[] = [];
|
||
// 中文双字/三字片段
|
||
const chinese = query.match(/[\u4e00-\u9fa5]{2,3}/g);
|
||
if (chinese) terms.push(...chinese);
|
||
// 英文单词
|
||
const english = query.match(/[a-zA-Z]{2,}/g);
|
||
if (english) terms.push(...english);
|
||
|
||
let score = 0;
|
||
const text = `${result.title} ${result.snippet}`.toLowerCase();
|
||
for (const term of terms) {
|
||
if (text.includes(term.toLowerCase())) score += 10;
|
||
}
|
||
return Math.min(score, 100);
|
||
}
|
||
|
||
// ===== WebSearchTool =====
|
||
|
||
export class WebSearchTool implements IMetonaTool {
|
||
readonly definition: MetonaToolDef = {
|
||
name: 'web_search',
|
||
description:
|
||
'Search the web for information. Returns titles, snippets, and URLs. When SearXNG is enabled, uses the configured SearXNG instance; otherwise uses built-in engines (Bing, Baidu, Sogou, 360). Automatically fetches full content for top results.',
|
||
parameters: {
|
||
type: 'object',
|
||
properties: {
|
||
query: { type: 'string', description: 'Search query keywords' },
|
||
time_range: {
|
||
type: 'string',
|
||
description: 'Time filter: day, week, month, year (optional)',
|
||
},
|
||
enhance_snippets: {
|
||
type: 'boolean',
|
||
description: 'Auto-enhance short snippets (default true)',
|
||
},
|
||
},
|
||
required: ['query'],
|
||
},
|
||
category: MetonaToolCategory.NETWORK,
|
||
riskLevel: MetonaRiskLevel.LOW,
|
||
requiresPermission: false,
|
||
timeoutMs: 300_000,
|
||
};
|
||
|
||
constructor(
|
||
private configService: ConfigService,
|
||
private webFetchTool: WebFetchTool,
|
||
) {}
|
||
|
||
async execute(args: Record<string, unknown>, _context: ToolExecutionContext): Promise<unknown> {
|
||
const query = args.query as string;
|
||
|
||
// 读取 SearXNG 配置
|
||
const searxngConfig = this.readSearXNGConfig();
|
||
const useSearXNG = searxngConfig.enabled && !!searxngConfig.url;
|
||
|
||
// 两个模式参数完全独立：
|
||
// - 内置模式：固定默认值 maxResults=30, fetchTop=5
|
||
// - SearXNG 模式：由配置面板控制
|
||
const maxResults = useSearXNG
|
||
? Math.min(50, Math.max(1, searxngConfig.max_results > 0 ? searxngConfig.max_results : 30))
|
||
: 30;
|
||
|
||
const timeRange = (args.time_range as string) ?? '';
|
||
const enhanceSnippets = (args.enhance_snippets as boolean) ?? true;
|
||
|
||
const fetchTop = useSearXNG
|
||
? Math.min(8, Math.max(3, searxngConfig.fetch_count > 0 ? searxngConfig.fetch_count : 5))
|
||
: 5;
|
||
|
||
logTool(
|
||
'web_search',
|
||
`Mode=${useSearXNG ? 'searxng' : 'builtin'}, maxResults=${maxResults}, fetchTop=${fetchTop}`,
|
||
);
|
||
|
||
// 缓存检查（key 含模式 + maxResults + fetchTop，避免配置变更后返回旧缓存）
|
||
const cacheKey = `${searxngConfig.enabled ? 'searxng' : 'builtin'}:${maxResults}:${fetchTop}:${normalizeUrl(query).toLowerCase()}`;
|
||
const cached = searchCache.get(cacheKey);
|
||
if (cached) {
|
||
logTool('web_search', `Cache hit: "${query}"`);
|
||
return { ...(cached as Record<string, unknown>), from_cache: true };
|
||
}
|
||
|
||
let results: SearchResult[];
|
||
let mode: string;
|
||
let engineStats: Record<string, string>;
|
||
|
||
if (useSearXNG) {
|
||
const searxResult = await this.searchSearXNG(query, searxngConfig, maxResults, timeRange);
|
||
results = searxResult.results;
|
||
mode = 'searxng';
|
||
engineStats = searxResult.engineStats;
|
||
} else {
|
||
const builtinResult = await this.searchBuiltinEngines(query, maxResults, timeRange);
|
||
results = builtinResult.results;
|
||
mode = 'builtin';
|
||
engineStats = builtinResult.engineStats;
|
||
}
|
||
|
||
// 合并去重
|
||
const deduped = this.deduplicate(results);
|
||
|
||
// 可达性预检
|
||
const topUrls = deduped.slice(0, Math.min(20, deduped.length)).map((r) => r.url);
|
||
const reachabilityMap = await checkReachability(topUrls);
|
||
for (const r of deduped) {
|
||
r.reachable = reachabilityMap.get(r.url) ?? false;
|
||
}
|
||
|
||
// 智能排序
|
||
const sorted = smartSort(deduped).slice(0, maxResults);
|
||
|
||
// 摘要增强
|
||
if (enhanceSnippets) {
|
||
await this.enhanceSnippets(sorted, 3);
|
||
}
|
||
|
||
// 自动抓取完整内容
|
||
const fetchedContent = await this.autoFetch(query, sorted, searxngConfig.fetch_mode, fetchTop);
|
||
|
||
// 格式化输出
|
||
const formatted = this.formatResults(query, sorted);
|
||
|
||
const output = {
|
||
success: true,
|
||
query,
|
||
results: sorted.map((r) => ({
|
||
title: r.title,
|
||
url: r.url,
|
||
snippet: r.snippet,
|
||
engine: r.engine,
|
||
reachable: r.reachable,
|
||
_score: r._score ? Math.round(r._score * 10) / 10 : undefined,
|
||
_enhanced: r._enhanced,
|
||
})),
|
||
total: sorted.length,
|
||
formatted,
|
||
from_cache: false,
|
||
engine_stats: engineStats,
|
||
_mode: mode,
|
||
_fetched: fetchedContent.map((f) => ({ url: f.url, title: f.title, content: f.content })),
|
||
_fetched_count: fetchedContent.length,
|
||
};
|
||
|
||
// 写入缓存
|
||
searchCache.set(cacheKey, output);
|
||
logTool(
|
||
'web_search',
|
||
`Completed: ${sorted.length} results, ${fetchedContent.length} fetched, mode=${mode}`,
|
||
);
|
||
|
||
return output;
|
||
}
|
||
|
||
// ===== SearXNG 搜索 =====
|
||
|
||
private async searchSearXNG(
|
||
query: string,
|
||
config: SearXNGConfig,
|
||
maxResults: number,
|
||
timeRange: string,
|
||
): Promise<{ results: SearchResult[]; engineStats: Record<string, string> }> {
|
||
const results: SearchResult[] = [];
|
||
const engineStats: Record<string, string> = {};
|
||
const engineCounts = new Map<string, number>();
|
||
const baseUrl = config.url.replace(/\/$/, '');
|
||
const headers = buildSearXNGAuthHeaders(config.auth_key, config.auth_type);
|
||
const tr = timeRange || config.time_range;
|
||
|
||
// SearXNG 标准分页：每页由实例配置决定（通常 10 条），用 pageno 翻页直到达到 maxResults
|
||
const maxPages = Math.ceil(maxResults / 5) + 1; // 保守估计，每页至少 5 条
|
||
let page = 1;
|
||
|
||
while (results.length < maxResults && page <= maxPages) {
|
||
const params = new URLSearchParams();
|
||
params.set('q', query);
|
||
params.set('format', config.format || 'json');
|
||
params.set('pageno', String(page));
|
||
if (config.engines) params.set('engines', config.engines);
|
||
if (config.language && config.language !== 'auto') params.set('language', config.language);
|
||
params.set('safesearch', String(config.safesearch));
|
||
if (tr) params.set('time_range', tr);
|
||
|
||
const searchUrl = `${baseUrl}/search?${params.toString()}`;
|
||
logTool('web_search', `[SearXNG] Fetching page ${page}: ${searchUrl}`);
|
||
|
||
const pageResults: SearchResult[] = [];
|
||
|
||
try {
|
||
const response = await fetchWithTimeout(searchUrl, { headers }, 15_000);
|
||
if (!response.ok) {
|
||
throw new Error(`SearXNG API error: ${response.status} ${response.statusText}`);
|
||
}
|
||
|
||
if (config.format === 'html') {
|
||
const html = await response.text();
|
||
const matches = html.matchAll(/<a[^>]*href="(https?:\/\/[^"]+)"[^>]*>([\s\S]*?)<\/a>/gi);
|
||
for (const m of matches) {
|
||
const url = m[1];
|
||
const title = m[2].replace(/<[^>]+>/g, '').trim();
|
||
if (title && url && !url.includes(config.url)) {
|
||
pageResults.push({ title, url, snippet: '', engine: 'searxng', weight: 85 });
|
||
}
|
||
}
|
||
} else {
|
||
const data = (await response.json()) as { results?: Array<Record<string, unknown>> };
|
||
for (const item of data.results ?? []) {
|
||
const url = item.url as string;
|
||
const title = item.title as string;
|
||
const snippet = item.content as string;
|
||
const engine = (item.engine as string) ?? 'searxng';
|
||
pageResults.push({ title, url, snippet: snippet ?? '', engine, weight: 85 });
|
||
engineCounts.set(engine, (engineCounts.get(engine) ?? 0) + 1);
|
||
}
|
||
}
|
||
} catch (err) {
|
||
logTool('web_search', `[SearXNG] Page ${page} error: ${(err as Error).message}`);
|
||
// 第一页就失败则抛出，后续页失败则用已获取的结果
|
||
if (page === 1) throw err;
|
||
break;
|
||
}
|
||
|
||
// 本页无新结果 → 已到末尾
|
||
if (pageResults.length === 0) {
|
||
logTool('web_search', `[SearXNG] Page ${page} returned 0 results, stopping pagination`);
|
||
break;
|
||
}
|
||
|
||
results.push(...pageResults);
|
||
logTool(
|
||
'web_search',
|
||
`[SearXNG] Page ${page}: +${pageResults.length} (total ${results.length})`,
|
||
);
|
||
page++;
|
||
}
|
||
|
||
for (const [engine, count] of engineCounts) {
|
||
engineStats[engine] = `${count} 条`;
|
||
}
|
||
|
||
logTool('web_search', `[SearXNG] Total: ${results.length} raw results (target ${maxResults})`);
|
||
return { results, engineStats };
|
||
}
|
||
|
||
// ===== 内置四引擎并行搜索 =====
|
||
|
||
private async searchBuiltinEngines(
|
||
query: string,
|
||
maxResults: number,
|
||
timeRange: string,
|
||
): Promise<{ results: SearchResult[]; engineStats: Record<string, string> }> {
|
||
const searchPromises = ENGINES.map(async (engine) => {
|
||
try {
|
||
const url = engine.searchUrl(query, timeRange);
|
||
const response = await fetchWithTimeout(
|
||
url,
|
||
{
|
||
headers: {
|
||
'User-Agent':
|
||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36',
|
||
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
|
||
},
|
||
},
|
||
8_000,
|
||
);
|
||
|
||
if (!response.ok) {
|
||
logTool('web_search', `[内置] ${engine.name} HTTP ${response.status}`);
|
||
return [];
|
||
}
|
||
|
||
const html = await response.text();
|
||
const parsed = engine.parse(html);
|
||
logTool('web_search', `[内置] ${engine.name}: ${parsed.length} results`);
|
||
return parsed;
|
||
} catch (err) {
|
||
logTool('web_search', `[内置] ${engine.name} error: ${(err as Error).message}`);
|
||
return [];
|
||
}
|
||
});
|
||
|
||
const settled = await Promise.allSettled(searchPromises);
|
||
const allResults: SearchResult[] = [];
|
||
const engineStats: Record<string, string> = {};
|
||
|
||
for (let i = 0; i < ENGINES.length; i++) {
|
||
const result = settled[i];
|
||
const engineName = ENGINES[i].name;
|
||
if (result.status === 'fulfilled') {
|
||
allResults.push(...result.value);
|
||
engineStats[engineName] = `${result.value.length} 条`;
|
||
} else {
|
||
engineStats[engineName] = '失败';
|
||
}
|
||
}
|
||
|
||
return { results: allResults, engineStats };
|
||
}
|
||
|
||
// ===== 合并去重 =====
|
||
|
||
private deduplicate(results: SearchResult[]): SearchResult[] {
|
||
const seen = new Map<string, SearchResult>();
|
||
for (const r of results) {
|
||
const normalized = normalizeUrl(r.url);
|
||
if (!seen.has(normalized)) {
|
||
seen.set(normalized, r);
|
||
}
|
||
}
|
||
return Array.from(seen.values());
|
||
}
|
||
|
||
// ===== 摘要增强（委托给 WebFetchTool） =====
|
||
|
||
private async enhanceSnippets(results: SearchResult[], maxEnhance: number): Promise<void> {
|
||
let enhanced = 0;
|
||
for (const r of results) {
|
||
if (enhanced >= maxEnhance) break;
|
||
if (r.snippet.length < 30 && r.reachable) {
|
||
try {
|
||
const fetchResult = (await this.webFetchTool.execute(
|
||
{ url: r.url },
|
||
{ sessionId: '', workspacePath: '', iteration: 0, requestId: '' },
|
||
)) as { success: boolean; content?: string };
|
||
|
||
if (fetchResult.success && fetchResult.content) {
|
||
const text = fetchResult.content.slice(0, 200);
|
||
if (text.length > r.snippet.length) {
|
||
r.snippet = text;
|
||
r._enhanced = true;
|
||
enhanced++;
|
||
}
|
||
}
|
||
} catch {
|
||
// 忽略增强失败
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// ===== 自动抓取完整内容（委托给 WebFetchTool） =====
|
||
|
||
/**
|
||
* P2-12: 自动抓取改为并行（批次并发 3）
|
||
*
|
||
* 原实现逐条串行抓取（单个 web_fetch 最长 120s 超时），top5 结果最坏耗时
|
||
* 逼近 web_search 的 300s 工具超时上限。并行批次化后总耗时约降至 1/3。
|
||
* 失败结果直接跳过（原"随机补充重试"逻辑收益边际，复杂度高，已移除）。
|
||
*/
|
||
private async autoFetch(
|
||
query: string,
|
||
results: SearchResult[],
|
||
fetchMode: string,
|
||
fetchTop: number,
|
||
): Promise<Array<{ url: string; title: string; content: string }>> {
|
||
// 相关性评分（不过滤，relevance=0 的结果也参与抓取候选）
|
||
const withRelevance = results.map((r) => ({
|
||
result: r,
|
||
relevance: computeRelevance(query, r),
|
||
}));
|
||
const filtered = withRelevance.length > 0 ? withRelevance : [];
|
||
|
||
// 确定抓取数量：fetchTop 已在 execute() 中综合了配置面板和工具参数
|
||
const topN = Math.min(fetchTop, 8, filtered.length);
|
||
|
||
// 构建抓取列表
|
||
let toFetch: typeof filtered;
|
||
if (fetchMode === 'random') {
|
||
// Fisher-Yates 洗牌
|
||
const shuffled = [...filtered];
|
||
for (let i = shuffled.length - 1; i > 0; i--) {
|
||
const j = Math.floor(Math.random() * (i + 1));
|
||
[shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
|
||
}
|
||
toFetch = shuffled.slice(0, topN);
|
||
} else {
|
||
toFetch = filtered.sort((a, b) => b.relevance - a.relevance).slice(0, topN);
|
||
}
|
||
|
||
const fetched: Array<{ url: string; title: string; content: string }> = [];
|
||
|
||
const fetchOne = async (item: {
|
||
result: SearchResult;
|
||
}): Promise<{ url: string; title: string; content: string } | null> => {
|
||
try {
|
||
// 委托给 WebFetchTool — 享受三阶段回退策略（HTTP + 反爬 + 浏览器渲染）
|
||
const fetchResult = (await this.webFetchTool.execute(
|
||
{ url: item.result.url },
|
||
{ sessionId: '', workspacePath: '', iteration: 0, requestId: '' },
|
||
)) as { success: boolean; content?: string };
|
||
|
||
if (fetchResult.success && fetchResult.content) {
|
||
return { url: item.result.url, title: item.result.title, content: fetchResult.content };
|
||
}
|
||
return null;
|
||
} catch (err) {
|
||
logTool(
|
||
'web_search',
|
||
`Auto-fetch failed for ${item.result.url}: ${(err as Error).message}`,
|
||
);
|
||
return null;
|
||
}
|
||
};
|
||
|
||
// 并行批次抓取（并发 3）
|
||
const CONCURRENCY = 3;
|
||
for (let i = 0; i < toFetch.length; i += CONCURRENCY) {
|
||
const batch = toFetch.slice(i, i + CONCURRENCY);
|
||
const settled = await Promise.allSettled(batch.map((item) => fetchOne(item)));
|
||
for (const r of settled) {
|
||
if (r.status === 'fulfilled' && r.value) fetched.push(r.value);
|
||
}
|
||
}
|
||
|
||
return fetched;
|
||
}
|
||
|
||
// ===== 格式化人类可读输出 =====
|
||
|
||
private formatResults(query: string, results: SearchResult[]): string {
|
||
if (results.length === 0) return `搜索 "${query}" 无结果。`;
|
||
const lines = [`搜索 "${query}" — ${results.length} 条结果：\n`];
|
||
results.forEach((r, i) => {
|
||
lines.push(`${i + 1}. ${r.title}`);
|
||
lines.push(` URL: ${r.url}`);
|
||
if (r.snippet) lines.push(` 摘要: ${r.snippet.slice(0, 150)}`);
|
||
lines.push(
|
||
` 来源: ${r.engine}${r.reachable === false ? ' (不可达)' : ''}${r._enhanced ? ' [已增强]' : ''}\n`,
|
||
);
|
||
});
|
||
return lines.join('\n');
|
||
}
|
||
|
||
// ===== 读取 SearXNG 配置 =====
|
||
|
||
private readSearXNGConfig(): SearXNGConfig {
|
||
const cs = this.configService;
|
||
return {
|
||
enabled: cs.get<boolean>('searxng.enabled') ?? false,
|
||
url: cs.get<string>('searxng.url') ?? '',
|
||
engines: cs.get<string>('searxng.engines') ?? '',
|
||
language: cs.get<string>('searxng.language') ?? 'zh-CN',
|
||
safesearch: cs.get<number>('searxng.safesearch') ?? 1,
|
||
time_range: cs.get<string>('searxng.time_range') ?? '',
|
||
max_results: cs.get<number>('searxng.max_results') ?? 0,
|
||
auth_key: cs.get<string>('searxng.auth_key') ?? '',
|
||
auth_type: cs.get<string>('searxng.auth_type') ?? 'bearer',
|
||
format: cs.get<string>('searxng.format') ?? 'json',
|
||
fetch_count: cs.get<number>('searxng.fetch_count') ?? 0,
|
||
fetch_mode: cs.get<string>('searxng.fetch_mode') ?? 'sequential',
|
||
};
|
||
}
|
||
}
|