thzxx
dc5f8aedcd
内置模式: maxResults=30, fetchTop=5 固定默认值, 不读取 SearXNG 配置 SearXNG 模式: max_results/fetch_count 由配置面板控制 统一使用 useSearXNG 变量判断模式
627 lines
23 KiB
TypeScript
627 lines
23 KiB
TypeScript
/**
|
||
* web_search — 网络搜索工具
|
||
*
|
||
* 双模式搜索：SearXNG 元搜索 / 内置四引擎并行搜索
|
||
* 内置引擎：Bing + 百度 + 搜狗 + 360 搜索（Promise.allSettled 容错并发）
|
||
* 智能排序：引擎权重(50%) + 可达性(30%) + 摘要质量(20%)
|
||
* 自动抓取：对前 N 条结果调用 web_fetch 获取完整正文
|
||
*
|
||
* @see docs/Agent网络工具通用设计-v2.md — 第 2 章 web_search 搜索设计
|
||
*/
|
||
|
||
import type { IMetonaTool, ToolExecutionContext } from '../../types/metona-tool';
|
||
import type { MetonaToolDef } from '../../../harness/types';
|
||
import { MetonaToolCategory, MetonaRiskLevel } from '../../../harness/types';
|
||
import type { ConfigService } from '../../../services/config.service';
|
||
import {
|
||
searchCache,
|
||
normalizeUrl,
|
||
fetchWithTimeout,
|
||
buildSearXNGAuthHeaders,
|
||
logTool,
|
||
} from './network-utils';
|
||
import type { WebFetchTool } from './web-fetch';
|
||
|
||
// ===== 类型定义 =====
|
||
|
||
interface SearchResult {
|
||
title: string;
|
||
url: string;
|
||
snippet: string;
|
||
engine: string;
|
||
weight: number;
|
||
reachable?: boolean;
|
||
_score?: number;
|
||
_enhanced?: boolean;
|
||
}
|
||
|
||
interface SearXNGConfig {
|
||
enabled: boolean;
|
||
url: string;
|
||
engines: string;
|
||
language: string;
|
||
safesearch: number;
|
||
time_range: string;
|
||
max_results: number;
|
||
auth_key: string;
|
||
auth_type: string;
|
||
format: string;
|
||
fetch_count: number;
|
||
fetch_mode: string;
|
||
}
|
||
|
||
// ===== 引擎定义 =====
|
||
|
||
interface EngineDef {
|
||
name: string;
|
||
weight: number;
|
||
searchUrl: (query: string, timeRange?: string) => string;
|
||
parse: (html: string) => SearchResult[];
|
||
}
|
||
|
||
const ENGINES: EngineDef[] = [
|
||
{
|
||
name: 'bing',
|
||
weight: 90,
|
||
searchUrl: (q, tr) => {
|
||
const freshness = tr ? `&filters=ex1:"ez${tr === 'day' ? '1' : tr === 'week' ? '2' : tr === 'month' ? '3' : '4'}"` : '';
|
||
return `https://www.bing.com/search?q=${encodeURIComponent(q)}${freshness}&count=20`;
|
||
},
|
||
parse: parseBing,
|
||
},
|
||
{
|
||
name: '百度',
|
||
weight: 80,
|
||
searchUrl: (q) => `https://www.baidu.com/s?wd=${encodeURIComponent(q)}&rn=20`,
|
||
parse: parseBaidu,
|
||
},
|
||
{
|
||
name: '搜狗',
|
||
weight: 75,
|
||
searchUrl: (q) => `https://www.sogou.com/web?query=${encodeURIComponent(q)}&num=20`,
|
||
parse: parseSogou,
|
||
},
|
||
{
|
||
name: '360搜索',
|
||
weight: 75,
|
||
searchUrl: (q) => `https://www.so.com/s?q=${encodeURIComponent(q)}&pn=20`,
|
||
parse: parse360,
|
||
},
|
||
];
|
||
|
||
// ===== HTML 解析器（正则实现，后续可迁移至 cheerio） =====
|
||
|
||
function parseBing(html: string): SearchResult[] {
|
||
const results: SearchResult[] = [];
|
||
const blocks = html.split(/<li[^>]*class="b_algo"/i).slice(1);
|
||
for (const block of blocks) {
|
||
const titleMatch = block.match(/<a[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
|
||
if (!titleMatch) continue;
|
||
const url = titleMatch[1];
|
||
const title = titleMatch[2].replace(/<[^>]+>/g, '').trim();
|
||
const snippetMatch = block.match(/<p[^>]*>([\s\S]*?)<\/p>/i) || block.match(/class="b_caption"[^>]*>([\s\S]*?)<\/div>/i);
|
||
const snippet = snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '').trim() : '';
|
||
if (title && url && !url.includes('bing.com')) {
|
||
results.push({ title, url, snippet, engine: 'bing', weight: 90 });
|
||
}
|
||
}
|
||
return results;
|
||
}
|
||
|
||
function parseBaidu(html: string): SearchResult[] {
|
||
const results: SearchResult[] = [];
|
||
const blocks = html.split(/<div[^>]*class="result[^"]*"/i).slice(1);
|
||
for (const block of blocks) {
|
||
const titleMatch = block.match(/<a[^>]*data-url="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i)
|
||
|| block.match(/<a[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
|
||
if (!titleMatch) continue;
|
||
const url = titleMatch[1].startsWith('http') ? titleMatch[1] : `https://${titleMatch[1]}`;
|
||
const title = titleMatch[2].replace(/<[^>]+>/g, '').trim();
|
||
const snippetMatch = block.match(/class="c-abstract[^"]*"[^>]*>([\s\S]*?)<\/span>/i)
|
||
|| block.match(/class="content-right[^"]*"[^>]*>([\s\S]*?)<\/div>/i);
|
||
const snippet = snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '').trim() : '';
|
||
if (title && url && !url.includes('baidu.com/link')) {
|
||
results.push({ title, url, snippet, engine: '百度', weight: 80 });
|
||
}
|
||
}
|
||
return results;
|
||
}
|
||
|
||
function parseSogou(html: string): SearchResult[] {
|
||
const results: SearchResult[] = [];
|
||
const blocks = html.split(/<div[^>]*class="vrwrap"/i).slice(1)
|
||
.concat(html.split(/<div[^>]*class="rb"/i).slice(1));
|
||
for (const block of blocks) {
|
||
const titleMatch = block.match(/<a[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
|
||
if (!titleMatch) continue;
|
||
const url = titleMatch[1].startsWith('http') ? titleMatch[1] : `https://www.sogou.com${titleMatch[1]}`;
|
||
const title = titleMatch[2].replace(/<[^>]+>/g, '').trim();
|
||
const snippetMatch = block.match(/class="star-wiki[^"]*"[^>]*>([\s\S]*?)<\/div>/i)
|
||
|| block.match(/class="space-txt[^"]*"[^>]*>([\s\S]*?)<\/p>/i)
|
||
|| block.match(/class="str_info[^"]*"[^>]*>([\s\S]*?)<\/p>/i);
|
||
const snippet = snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '').trim() : '';
|
||
if (title && url && !url.includes('sogou.com')) {
|
||
results.push({ title, url, snippet, engine: '搜狗', weight: 75 });
|
||
}
|
||
}
|
||
return results;
|
||
}
|
||
|
||
function parse360(html: string): SearchResult[] {
|
||
const results: SearchResult[] = [];
|
||
const blocks = html.split(/<li[^>]*class="res-list"/i).slice(1)
|
||
.concat(html.split(/<div[^>]*class="result"/i).slice(1));
|
||
for (const block of blocks) {
|
||
const titleMatch = block.match(/<a[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
|
||
if (!titleMatch) continue;
|
||
const url = titleMatch[1];
|
||
const title = titleMatch[2].replace(/<[^>]+>/g, '').trim();
|
||
const snippetMatch = block.match(/class="res-desc[^"]*"[^>]*>([\s\S]*?)<\/p>/i)
|
||
|| block.match(/class="res-rich[^"]*"[^>]*>([\s\S]*?)<\/div>/i)
|
||
|| block.match(/class="res-summary[^"]*"[^>]*>([\s\S]*?)<\/p>/i)
|
||
|| block.match(/<dd[^>]*>([\s\S]*?)<\/dd>/i);
|
||
const snippet = snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '').trim() : '';
|
||
if (title && url && !url.includes('so.com')) {
|
||
results.push({ title, url, snippet, engine: '360搜索', weight: 75 });
|
||
}
|
||
}
|
||
return results;
|
||
}
|
||
|
||
// ===== 可达性预检 =====
|
||
|
||
async function checkReachability(urls: string[], concurrency = 5): Promise<Map<string, boolean>> {
|
||
const result = new Map<string, boolean>();
|
||
for (let i = 0; i < urls.length; i += concurrency) {
|
||
const batch = urls.slice(i, i + concurrency);
|
||
const checks = batch.map(async (url) => {
|
||
try {
|
||
const resp = await fetchWithTimeout(url, { method: 'HEAD', redirect: 'follow' }, 3_000);
|
||
result.set(url, resp.ok);
|
||
} catch {
|
||
result.set(url, false);
|
||
}
|
||
});
|
||
await Promise.allSettled(checks);
|
||
}
|
||
return result;
|
||
}
|
||
|
||
// ===== 智能排序 =====
|
||
|
||
function smartSort(results: SearchResult[], reachabilityMap: Map<string, boolean>): SearchResult[] {
|
||
for (const r of results) {
|
||
const reachability = r.reachable ? 30 : -20;
|
||
const snippetQuality = Math.min(r.snippet.length, 100) / 100 * 20;
|
||
const weightScore = (r.weight / 100) * 50;
|
||
r._score = weightScore + reachability + snippetQuality;
|
||
}
|
||
return results.sort((a, b) => (b._score ?? 0) - (a._score ?? 0));
|
||
}
|
||
|
||
// ===== 相关性过滤（用于自动抓取） =====
|
||
|
||
function computeRelevance(query: string, result: SearchResult): number {
|
||
const terms: string[] = [];
|
||
// 中文双字/三字片段
|
||
const chinese = query.match(/[\u4e00-\u9fa5]{2,3}/g);
|
||
if (chinese) terms.push(...chinese);
|
||
// 英文单词
|
||
const english = query.match(/[a-zA-Z]{2,}/g);
|
||
if (english) terms.push(...english);
|
||
|
||
let score = 0;
|
||
const text = `${result.title} ${result.snippet}`.toLowerCase();
|
||
for (const term of terms) {
|
||
if (text.includes(term.toLowerCase())) score += 10;
|
||
}
|
||
return Math.min(score, 100);
|
||
}
|
||
|
||
// ===== WebSearchTool =====
|
||
|
||
export class WebSearchTool implements IMetonaTool {
|
||
readonly definition: MetonaToolDef = {
|
||
name: 'web_search',
|
||
description: 'Search the web for information. Returns titles, snippets, and URLs. When SearXNG is enabled, uses the configured SearXNG instance; otherwise uses built-in engines (Bing, Baidu, Sogou, 360). Automatically fetches full content for top results.',
|
||
parameters: {
|
||
type: 'object',
|
||
properties: {
|
||
query: { type: 'string', description: 'Search query keywords' },
|
||
time_range: { type: 'string', description: 'Time filter: day, week, month, year (optional)' },
|
||
enhance_snippets: { type: 'boolean', description: 'Auto-enhance short snippets (default true)' },
|
||
},
|
||
required: ['query'],
|
||
},
|
||
category: MetonaToolCategory.NETWORK,
|
||
riskLevel: MetonaRiskLevel.LOW,
|
||
requiresPermission: false,
|
||
timeoutMs: 300_000,
|
||
};
|
||
|
||
constructor(
|
||
private configService: ConfigService,
|
||
private webFetchTool: WebFetchTool,
|
||
) {}
|
||
|
||
async execute(args: Record<string, unknown>, _context: ToolExecutionContext): Promise<unknown> {
|
||
const query = args.query as string;
|
||
|
||
// 读取 SearXNG 配置
|
||
const searxngConfig = this.readSearXNGConfig();
|
||
const useSearXNG = searxngConfig.enabled && !!searxngConfig.url;
|
||
|
||
// 两个模式参数完全独立：
|
||
// - 内置模式：固定默认值 maxResults=30, fetchTop=5
|
||
// - SearXNG 模式：由配置面板控制
|
||
const maxResults = useSearXNG
|
||
? Math.min(50, Math.max(1, searxngConfig.max_results > 0 ? searxngConfig.max_results : 30))
|
||
: 30;
|
||
|
||
const timeRange = (args.time_range as string) ?? '';
|
||
const enhanceSnippets = (args.enhance_snippets as boolean) ?? true;
|
||
|
||
const fetchTop = useSearXNG
|
||
? Math.min(8, Math.max(3, searxngConfig.fetch_count > 0 ? searxngConfig.fetch_count : 5))
|
||
: 5;
|
||
|
||
logTool('web_search', `Mode=${useSearXNG ? 'searxng' : 'builtin'}, maxResults=${maxResults}, fetchTop=${fetchTop}`);
|
||
|
||
// 缓存检查（key 含模式 + maxResults + fetchTop，避免配置变更后返回旧缓存）
|
||
const cacheKey = `${searxngConfig.enabled ? 'searxng' : 'builtin'}:${maxResults}:${fetchTop}:${normalizeUrl(query).toLowerCase()}`;
|
||
const cached = searchCache.get(cacheKey);
|
||
if (cached) {
|
||
logTool('web_search', `Cache hit: "${query}"`);
|
||
return { ...(cached as Record<string, unknown>), from_cache: true };
|
||
}
|
||
|
||
let results: SearchResult[];
|
||
let mode: string;
|
||
let engineStats: Record<string, string>;
|
||
|
||
if (useSearXNG) {
|
||
const searxResult = await this.searchSearXNG(query, searxngConfig, maxResults, timeRange);
|
||
results = searxResult.results;
|
||
mode = 'searxng';
|
||
engineStats = searxResult.engineStats;
|
||
} else {
|
||
const builtinResult = await this.searchBuiltinEngines(query, maxResults, timeRange);
|
||
results = builtinResult.results;
|
||
mode = 'builtin';
|
||
engineStats = builtinResult.engineStats;
|
||
}
|
||
|
||
// 合并去重
|
||
const deduped = this.deduplicate(results);
|
||
|
||
// 可达性预检
|
||
const topUrls = deduped.slice(0, Math.min(20, deduped.length)).map((r) => r.url);
|
||
const reachabilityMap = await checkReachability(topUrls);
|
||
for (const r of deduped) {
|
||
r.reachable = reachabilityMap.get(r.url) ?? false;
|
||
}
|
||
|
||
// 智能排序
|
||
const sorted = smartSort(deduped, reachabilityMap).slice(0, maxResults);
|
||
|
||
// 摘要增强
|
||
if (enhanceSnippets) {
|
||
await this.enhanceSnippets(sorted, 3);
|
||
}
|
||
|
||
// 自动抓取完整内容
|
||
const fetchedContent = await this.autoFetch(query, sorted, searxngConfig.fetch_mode, fetchTop);
|
||
|
||
// 格式化输出
|
||
const formatted = this.formatResults(query, sorted);
|
||
|
||
const output = {
|
||
success: true,
|
||
query,
|
||
results: sorted.map((r) => ({
|
||
title: r.title,
|
||
url: r.url,
|
||
snippet: r.snippet,
|
||
engine: r.engine,
|
||
reachable: r.reachable,
|
||
_score: r._score ? Math.round(r._score * 10) / 10 : undefined,
|
||
_enhanced: r._enhanced,
|
||
})),
|
||
total: sorted.length,
|
||
formatted,
|
||
from_cache: false,
|
||
engine_stats: engineStats,
|
||
_mode: mode,
|
||
_fetched: fetchedContent.map((f) => ({ url: f.url, title: f.title, content: f.content })),
|
||
_fetched_count: fetchedContent.length,
|
||
};
|
||
|
||
// 写入缓存
|
||
searchCache.set(cacheKey, output);
|
||
logTool('web_search', `Completed: ${sorted.length} results, ${fetchedContent.length} fetched, mode=${mode}`);
|
||
|
||
return output;
|
||
}
|
||
|
||
// ===== SearXNG 搜索 =====
|
||
|
||
private async searchSearXNG(
|
||
query: string,
|
||
config: SearXNGConfig,
|
||
maxResults: number,
|
||
timeRange: string,
|
||
): Promise<{ results: SearchResult[]; engineStats: Record<string, string> }> {
|
||
const results: SearchResult[] = [];
|
||
const engineStats: Record<string, string> = {};
|
||
const engineCounts = new Map<string, number>();
|
||
const baseUrl = config.url.replace(/\/$/, '');
|
||
const headers = buildSearXNGAuthHeaders(config.auth_key, config.auth_type);
|
||
const tr = timeRange || config.time_range;
|
||
|
||
// SearXNG 标准分页：每页由实例配置决定（通常 10 条），用 pageno 翻页直到达到 maxResults
|
||
const maxPages = Math.ceil(maxResults / 5) + 1; // 保守估计，每页至少 5 条
|
||
let page = 1;
|
||
|
||
while (results.length < maxResults && page <= maxPages) {
|
||
const params = new URLSearchParams();
|
||
params.set('q', query);
|
||
params.set('format', config.format || 'json');
|
||
params.set('pageno', String(page));
|
||
if (config.engines) params.set('engines', config.engines);
|
||
if (config.language && config.language !== 'auto') params.set('language', config.language);
|
||
params.set('safesearch', String(config.safesearch));
|
||
if (tr) params.set('time_range', tr);
|
||
|
||
const searchUrl = `${baseUrl}/search?${params.toString()}`;
|
||
logTool('web_search', `[SearXNG] Fetching page ${page}: ${searchUrl}`);
|
||
|
||
let pageResults: SearchResult[] = [];
|
||
|
||
try {
|
||
const response = await fetchWithTimeout(searchUrl, { headers }, 15_000);
|
||
if (!response.ok) {
|
||
throw new Error(`SearXNG API error: ${response.status} ${response.statusText}`);
|
||
}
|
||
|
||
if (config.format === 'html') {
|
||
const html = await response.text();
|
||
const matches = html.matchAll(/<a[^>]*href="(https?:\/\/[^"]+)"[^>]*>([\s\S]*?)<\/a>/gi);
|
||
for (const m of matches) {
|
||
const url = m[1];
|
||
const title = m[2].replace(/<[^>]+>/g, '').trim();
|
||
if (title && url && !url.includes(config.url)) {
|
||
pageResults.push({ title, url, snippet: '', engine: 'searxng', weight: 85 });
|
||
}
|
||
}
|
||
} else {
|
||
const data = await response.json() as { results?: Array<Record<string, unknown>> };
|
||
for (const item of data.results ?? []) {
|
||
const url = item.url as string;
|
||
const title = item.title as string;
|
||
const snippet = item.content as string;
|
||
const engine = (item.engine as string) ?? 'searxng';
|
||
pageResults.push({ title, url, snippet: snippet ?? '', engine, weight: 85 });
|
||
engineCounts.set(engine, (engineCounts.get(engine) ?? 0) + 1);
|
||
}
|
||
}
|
||
} catch (err) {
|
||
logTool('web_search', `[SearXNG] Page ${page} error: ${(err as Error).message}`);
|
||
// 第一页就失败则抛出，后续页失败则用已获取的结果
|
||
if (page === 1) throw err;
|
||
break;
|
||
}
|
||
|
||
// 本页无新结果 → 已到末尾
|
||
if (pageResults.length === 0) {
|
||
logTool('web_search', `[SearXNG] Page ${page} returned 0 results, stopping pagination`);
|
||
break;
|
||
}
|
||
|
||
results.push(...pageResults);
|
||
logTool('web_search', `[SearXNG] Page ${page}: +${pageResults.length} (total ${results.length})`);
|
||
page++;
|
||
}
|
||
|
||
for (const [engine, count] of engineCounts) {
|
||
engineStats[engine] = `${count} 条`;
|
||
}
|
||
|
||
logTool('web_search', `[SearXNG] Total: ${results.length} raw results (target ${maxResults})`);
|
||
return { results, engineStats };
|
||
}
|
||
|
||
// ===== 内置四引擎并行搜索 =====
|
||
|
||
private async searchBuiltinEngines(
|
||
query: string,
|
||
maxResults: number,
|
||
timeRange: string,
|
||
): Promise<{ results: SearchResult[]; engineStats: Record<string, string> }> {
|
||
const searchPromises = ENGINES.map(async (engine) => {
|
||
try {
|
||
const url = engine.searchUrl(query, timeRange);
|
||
const response = await fetchWithTimeout(url, {
|
||
headers: {
|
||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36',
|
||
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
|
||
},
|
||
}, 8_000);
|
||
|
||
if (!response.ok) {
|
||
logTool('web_search', `[内置] ${engine.name} HTTP ${response.status}`);
|
||
return [];
|
||
}
|
||
|
||
const html = await response.text();
|
||
const parsed = engine.parse(html);
|
||
logTool('web_search', `[内置] ${engine.name}: ${parsed.length} results`);
|
||
return parsed;
|
||
} catch (err) {
|
||
logTool('web_search', `[内置] ${engine.name} error: ${(err as Error).message}`);
|
||
return [];
|
||
}
|
||
});
|
||
|
||
const settled = await Promise.allSettled(searchPromises);
|
||
const allResults: SearchResult[] = [];
|
||
const engineStats: Record<string, string> = {};
|
||
|
||
for (let i = 0; i < ENGINES.length; i++) {
|
||
const result = settled[i];
|
||
const engineName = ENGINES[i].name;
|
||
if (result.status === 'fulfilled') {
|
||
allResults.push(...result.value);
|
||
engineStats[engineName] = `${result.value.length} 条`;
|
||
} else {
|
||
engineStats[engineName] = '失败';
|
||
}
|
||
}
|
||
|
||
return { results: allResults, engineStats };
|
||
}
|
||
|
||
// ===== 合并去重 =====
|
||
|
||
private deduplicate(results: SearchResult[]): SearchResult[] {
|
||
const seen = new Map<string, SearchResult>();
|
||
for (const r of results) {
|
||
const normalized = normalizeUrl(r.url);
|
||
if (!seen.has(normalized)) {
|
||
seen.set(normalized, r);
|
||
}
|
||
}
|
||
return Array.from(seen.values());
|
||
}
|
||
|
||
// ===== 摘要增强（委托给 WebFetchTool） =====
|
||
|
||
private async enhanceSnippets(results: SearchResult[], maxEnhance: number): Promise<void> {
|
||
let enhanced = 0;
|
||
for (const r of results) {
|
||
if (enhanced >= maxEnhance) break;
|
||
if (r.snippet.length < 30 && r.reachable) {
|
||
try {
|
||
const fetchResult = await this.webFetchTool.execute(
|
||
{ url: r.url },
|
||
{ sessionId: '', workspacePath: '', iteration: 0, requestId: '' },
|
||
) as { success: boolean; content?: string };
|
||
|
||
if (fetchResult.success && fetchResult.content) {
|
||
const text = fetchResult.content.slice(0, 200);
|
||
if (text.length > r.snippet.length) {
|
||
r.snippet = text;
|
||
r._enhanced = true;
|
||
enhanced++;
|
||
}
|
||
}
|
||
} catch {
|
||
// 忽略增强失败
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// ===== 自动抓取完整内容（委托给 WebFetchTool） =====
|
||
|
||
private async autoFetch(
|
||
query: string,
|
||
results: SearchResult[],
|
||
fetchMode: string,
|
||
fetchTop: number,
|
||
): Promise<Array<{ url: string; title: string; content: string }>> {
|
||
// 相关性评分（不过滤，relevance=0 的结果也参与抓取候选）
|
||
const withRelevance = results.map((r) => ({ result: r, relevance: computeRelevance(query, r) }));
|
||
const filtered = withRelevance.length > 0 ? withRelevance : [];
|
||
|
||
// 确定抓取数量：fetchTop 已在 execute() 中综合了配置面板和工具参数
|
||
const topN = Math.min(fetchTop, 8, filtered.length);
|
||
|
||
// 构建抓取列表
|
||
let toFetch: typeof filtered;
|
||
if (fetchMode === 'random') {
|
||
// Fisher-Yates 洗牌
|
||
const shuffled = [...filtered];
|
||
for (let i = shuffled.length - 1; i > 0; i--) {
|
||
const j = Math.floor(Math.random() * (i + 1));
|
||
[shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
|
||
}
|
||
toFetch = shuffled.slice(0, topN);
|
||
} else {
|
||
toFetch = filtered
|
||
.sort((a, b) => b.relevance - a.relevance)
|
||
.slice(0, topN);
|
||
}
|
||
|
||
const fetched: Array<{ url: string; title: string; content: string }> = [];
|
||
|
||
for (const item of toFetch) {
|
||
try {
|
||
// 委托给 WebFetchTool — 享受三阶段回退策略（HTTP + 反爬 + 浏览器渲染）
|
||
const fetchResult = await this.webFetchTool.execute(
|
||
{ url: item.result.url },
|
||
{ sessionId: '', workspacePath: '', iteration: 0, requestId: '' },
|
||
) as { success: boolean; content?: string; method?: string };
|
||
|
||
if (fetchResult.success && fetchResult.content) {
|
||
fetched.push({ url: item.result.url, title: item.result.title, content: fetchResult.content });
|
||
}
|
||
} catch (err) {
|
||
logTool('web_search', `Auto-fetch failed for ${item.result.url}: ${(err as Error).message}`);
|
||
// 从剩余结果中随机补充
|
||
const remaining = withRelevance.filter((x) => !toFetch.includes(x) && !fetched.some((f) => f.url === x.result.url));
|
||
if (remaining.length > 0) {
|
||
const randomPick = remaining[Math.floor(Math.random() * remaining.length)];
|
||
try {
|
||
const fetchResult2 = await this.webFetchTool.execute(
|
||
{ url: randomPick.result.url },
|
||
{ sessionId: '', workspacePath: '', iteration: 0, requestId: '' },
|
||
) as { success: boolean; content?: string };
|
||
|
||
if (fetchResult2.success && fetchResult2.content) {
|
||
fetched.push({ url: randomPick.result.url, title: randomPick.result.title, content: fetchResult2.content });
|
||
}
|
||
} catch {
|
||
// 忽略补充失败
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
return fetched;
|
||
}
|
||
|
||
// ===== 格式化人类可读输出 =====
|
||
|
||
private formatResults(query: string, results: SearchResult[]): string {
|
||
if (results.length === 0) return `搜索 "${query}" 无结果。`;
|
||
const lines = [`搜索 "${query}" — ${results.length} 条结果：\n`];
|
||
results.forEach((r, i) => {
|
||
lines.push(`${i + 1}. ${r.title}`);
|
||
lines.push(` URL: ${r.url}`);
|
||
if (r.snippet) lines.push(` 摘要: ${r.snippet.slice(0, 150)}`);
|
||
lines.push(` 来源: ${r.engine}${r.reachable === false ? ' (不可达)' : ''}${r._enhanced ? ' [已增强]' : ''}\n`);
|
||
});
|
||
return lines.join('\n');
|
||
}
|
||
|
||
// ===== 读取 SearXNG 配置 =====
|
||
|
||
private readSearXNGConfig(): SearXNGConfig {
|
||
const cs = this.configService;
|
||
return {
|
||
enabled: cs.get<boolean>('searxng.enabled') ?? false,
|
||
url: cs.get<string>('searxng.url') ?? '',
|
||
engines: cs.get<string>('searxng.engines') ?? '',
|
||
language: cs.get<string>('searxng.language') ?? 'zh-CN',
|
||
safesearch: cs.get<number>('searxng.safesearch') ?? 1,
|
||
time_range: cs.get<string>('searxng.time_range') ?? '',
|
||
max_results: cs.get<number>('searxng.max_results') ?? 0,
|
||
auth_key: cs.get<string>('searxng.auth_key') ?? '',
|
||
auth_type: cs.get<string>('searxng.auth_type') ?? 'bearer',
|
||
format: cs.get<string>('searxng.format') ?? 'json',
|
||
fetch_count: cs.get<number>('searxng.fetch_count') ?? 0,
|
||
fetch_mode: cs.get<string>('searxng.fetch_mode') ?? 'sequential',
|
||
};
|
||
}
|
||
}
|