thzxx
8cdb93f8bf
MEMORY.md 保护: 新建 file-guard.ts 共享守卫模块; read_file/write_file/search_files/run_command 禁止访问根目录 MEMORY.md; permissions.ts 增加 deniedPatterns 深度防御。格式简化: 元数据从 # 注释改为 > 引用语法; 校验规则从 6 条简化为 3 条; context-builder extractContent 适配。预存问题: 修复路径遍历前缀碰撞漏洞; 移除 browser_extract 内容截断; SearXNG 配置实时读取; web_search/web_fetch 移除截断; 侧边栏动态获取工具列表; 版本号 0.1.1
601 lines
22 KiB
TypeScript
601 lines
22 KiB
TypeScript
/**
|
|
* web_search — 网络搜索工具
|
|
*
|
|
* 双模式搜索：SearXNG 元搜索 / 内置四引擎并行搜索
|
|
* 内置引擎：Bing + 百度 + 搜狗 + 360 搜索（Promise.allSettled 容错并发）
|
|
* 智能排序：引擎权重(50%) + 可达性(30%) + 摘要质量(20%)
|
|
* 自动抓取：对前 N 条结果调用 web_fetch 获取完整正文
|
|
*
|
|
* @see docs/Agent网络工具通用设计-v2.md — 第 2 章 web_search 搜索设计
|
|
*/
|
|
|
|
import type { IMetonaTool, ToolExecutionContext } from '../../types/metona-tool';
|
|
import type { MetonaToolDef } from '../../../harness/types';
|
|
import { MetonaToolCategory, MetonaRiskLevel } from '../../../harness/types';
|
|
import type { ConfigService } from '../../../services/config.service';
|
|
import {
|
|
searchCache,
|
|
normalizeUrl,
|
|
fetchWithTimeout,
|
|
buildSearXNGAuthHeaders,
|
|
logTool,
|
|
} from './network-utils';
|
|
|
|
// ===== 类型定义 =====
|
|
|
|
interface SearchResult {
|
|
title: string;
|
|
url: string;
|
|
snippet: string;
|
|
engine: string;
|
|
weight: number;
|
|
reachable?: boolean;
|
|
_score?: number;
|
|
_enhanced?: boolean;
|
|
}
|
|
|
|
interface SearXNGConfig {
|
|
enabled: boolean;
|
|
url: string;
|
|
engines: string;
|
|
language: string;
|
|
safesearch: number;
|
|
time_range: string;
|
|
max_results: number;
|
|
auth_key: string;
|
|
auth_type: string;
|
|
format: string;
|
|
fetch_count: number;
|
|
fetch_mode: string;
|
|
}
|
|
|
|
// ===== 引擎定义 =====
|
|
|
|
interface EngineDef {
|
|
name: string;
|
|
weight: number;
|
|
searchUrl: (query: string, timeRange?: string) => string;
|
|
parse: (html: string) => SearchResult[];
|
|
}
|
|
|
|
const ENGINES: EngineDef[] = [
|
|
{
|
|
name: 'bing',
|
|
weight: 90,
|
|
searchUrl: (q, tr) => {
|
|
const freshness = tr ? `&filters=ex1:"ez${tr === 'day' ? '1' : tr === 'week' ? '2' : tr === 'month' ? '3' : '4'}"` : '';
|
|
return `https://www.bing.com/search?q=${encodeURIComponent(q)}${freshness}&count=20`;
|
|
},
|
|
parse: parseBing,
|
|
},
|
|
{
|
|
name: '百度',
|
|
weight: 80,
|
|
searchUrl: (q) => `https://www.baidu.com/s?wd=${encodeURIComponent(q)}&rn=20`,
|
|
parse: parseBaidu,
|
|
},
|
|
{
|
|
name: '搜狗',
|
|
weight: 75,
|
|
searchUrl: (q) => `https://www.sogou.com/web?query=${encodeURIComponent(q)}&num=20`,
|
|
parse: parseSogou,
|
|
},
|
|
{
|
|
name: '360搜索',
|
|
weight: 75,
|
|
searchUrl: (q) => `https://www.so.com/s?q=${encodeURIComponent(q)}&pn=20`,
|
|
parse: parse360,
|
|
},
|
|
];
|
|
|
|
// ===== HTML 解析器（正则实现，后续可迁移至 cheerio） =====
|
|
|
|
function parseBing(html: string): SearchResult[] {
|
|
const results: SearchResult[] = [];
|
|
const blocks = html.split(/<li[^>]*class="b_algo"/i).slice(1);
|
|
for (const block of blocks) {
|
|
const titleMatch = block.match(/<a[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
|
|
if (!titleMatch) continue;
|
|
const url = titleMatch[1];
|
|
const title = titleMatch[2].replace(/<[^>]+>/g, '').trim();
|
|
const snippetMatch = block.match(/<p[^>]*>([\s\S]*?)<\/p>/i) || block.match(/class="b_caption"[^>]*>([\s\S]*?)<\/div>/i);
|
|
const snippet = snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '').trim() : '';
|
|
if (title && url && !url.includes('bing.com')) {
|
|
results.push({ title, url, snippet, engine: 'bing', weight: 90 });
|
|
}
|
|
}
|
|
return results;
|
|
}
|
|
|
|
function parseBaidu(html: string): SearchResult[] {
|
|
const results: SearchResult[] = [];
|
|
const blocks = html.split(/<div[^>]*class="result[^"]*"/i).slice(1);
|
|
for (const block of blocks) {
|
|
const titleMatch = block.match(/<a[^>]*data-url="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i)
|
|
|| block.match(/<a[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
|
|
if (!titleMatch) continue;
|
|
const url = titleMatch[1].startsWith('http') ? titleMatch[1] : `https://${titleMatch[1]}`;
|
|
const title = titleMatch[2].replace(/<[^>]+>/g, '').trim();
|
|
const snippetMatch = block.match(/class="c-abstract[^"]*"[^>]*>([\s\S]*?)<\/span>/i)
|
|
|| block.match(/class="content-right[^"]*"[^>]*>([\s\S]*?)<\/div>/i);
|
|
const snippet = snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '').trim() : '';
|
|
if (title && url && !url.includes('baidu.com/link')) {
|
|
results.push({ title, url, snippet, engine: '百度', weight: 80 });
|
|
}
|
|
}
|
|
return results;
|
|
}
|
|
|
|
function parseSogou(html: string): SearchResult[] {
|
|
const results: SearchResult[] = [];
|
|
const blocks = html.split(/<div[^>]*class="vrwrap"/i).slice(1)
|
|
.concat(html.split(/<div[^>]*class="rb"/i).slice(1));
|
|
for (const block of blocks) {
|
|
const titleMatch = block.match(/<a[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
|
|
if (!titleMatch) continue;
|
|
const url = titleMatch[1].startsWith('http') ? titleMatch[1] : `https://www.sogou.com${titleMatch[1]}`;
|
|
const title = titleMatch[2].replace(/<[^>]+>/g, '').trim();
|
|
const snippetMatch = block.match(/class="star-wiki[^"]*"[^>]*>([\s\S]*?)<\/div>/i)
|
|
|| block.match(/class="space-txt[^"]*"[^>]*>([\s\S]*?)<\/p>/i)
|
|
|| block.match(/class="str_info[^"]*"[^>]*>([\s\S]*?)<\/p>/i);
|
|
const snippet = snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '').trim() : '';
|
|
if (title && url && !url.includes('sogou.com')) {
|
|
results.push({ title, url, snippet, engine: '搜狗', weight: 75 });
|
|
}
|
|
}
|
|
return results;
|
|
}
|
|
|
|
function parse360(html: string): SearchResult[] {
|
|
const results: SearchResult[] = [];
|
|
const blocks = html.split(/<li[^>]*class="res-list"/i).slice(1)
|
|
.concat(html.split(/<div[^>]*class="result"/i).slice(1));
|
|
for (const block of blocks) {
|
|
const titleMatch = block.match(/<a[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
|
|
if (!titleMatch) continue;
|
|
const url = titleMatch[1];
|
|
const title = titleMatch[2].replace(/<[^>]+>/g, '').trim();
|
|
const snippetMatch = block.match(/class="res-desc[^"]*"[^>]*>([\s\S]*?)<\/p>/i)
|
|
|| block.match(/class="res-rich[^"]*"[^>]*>([\s\S]*?)<\/div>/i)
|
|
|| block.match(/class="res-summary[^"]*"[^>]*>([\s\S]*?)<\/p>/i)
|
|
|| block.match(/<dd[^>]*>([\s\S]*?)<\/dd>/i);
|
|
const snippet = snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '').trim() : '';
|
|
if (title && url && !url.includes('so.com')) {
|
|
results.push({ title, url, snippet, engine: '360搜索', weight: 75 });
|
|
}
|
|
}
|
|
return results;
|
|
}
|
|
|
|
// ===== 可达性预检 =====
|
|
|
|
async function checkReachability(urls: string[], concurrency = 5): Promise<Map<string, boolean>> {
|
|
const result = new Map<string, boolean>();
|
|
for (let i = 0; i < urls.length; i += concurrency) {
|
|
const batch = urls.slice(i, i + concurrency);
|
|
const checks = batch.map(async (url) => {
|
|
try {
|
|
const resp = await fetchWithTimeout(url, { method: 'HEAD', redirect: 'follow' }, 3_000);
|
|
result.set(url, resp.ok);
|
|
} catch {
|
|
result.set(url, false);
|
|
}
|
|
});
|
|
await Promise.allSettled(checks);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
// ===== 智能排序 =====
|
|
|
|
function smartSort(results: SearchResult[], reachabilityMap: Map<string, boolean>): SearchResult[] {
|
|
for (const r of results) {
|
|
const reachability = r.reachable ? 30 : -20;
|
|
const snippetQuality = Math.min(r.snippet.length, 100) / 100 * 20;
|
|
const weightScore = (r.weight / 100) * 50;
|
|
r._score = weightScore + reachability + snippetQuality;
|
|
}
|
|
return results.sort((a, b) => (b._score ?? 0) - (a._score ?? 0));
|
|
}
|
|
|
|
// ===== 相关性过滤（用于自动抓取） =====
|
|
|
|
function computeRelevance(query: string, result: SearchResult): number {
|
|
const terms: string[] = [];
|
|
// 中文双字/三字片段
|
|
const chinese = query.match(/[\u4e00-\u9fa5]{2,3}/g);
|
|
if (chinese) terms.push(...chinese);
|
|
// 英文单词
|
|
const english = query.match(/[a-zA-Z]{2,}/g);
|
|
if (english) terms.push(...english);
|
|
|
|
let score = 0;
|
|
const text = `${result.title} ${result.snippet}`.toLowerCase();
|
|
for (const term of terms) {
|
|
if (text.includes(term.toLowerCase())) score += 10;
|
|
}
|
|
return Math.min(score, 100);
|
|
}
|
|
|
|
// ===== WebSearchTool =====
|
|
|
|
export class WebSearchTool implements IMetonaTool {
|
|
readonly definition: MetonaToolDef = {
|
|
name: 'web_search',
|
|
description: 'Search the web for information. Returns titles, snippets, and URLs. When SearXNG is enabled, uses the configured SearXNG instance; otherwise uses built-in engines (Bing, Baidu, Sogou, 360). Automatically fetches full content for top results.',
|
|
parameters: {
|
|
type: 'object',
|
|
properties: {
|
|
query: { type: 'string', description: 'Search query keywords' },
|
|
max_results: { type: 'number', description: 'Maximum results (default 30, max 30)' },
|
|
time_range: { type: 'string', description: 'Time filter: day, week, month, year (optional)' },
|
|
enhance_snippets: { type: 'boolean', description: 'Auto-enhance short snippets (default true)' },
|
|
fetch_top: { type: 'number', description: 'Auto-fetch full content for top N results (3-8, default 5)' },
|
|
},
|
|
required: ['query'],
|
|
},
|
|
category: MetonaToolCategory.NETWORK,
|
|
riskLevel: MetonaRiskLevel.LOW,
|
|
requiresPermission: false,
|
|
timeoutMs: 300_000,
|
|
};
|
|
|
|
constructor(private configService: ConfigService) {}
|
|
|
|
async execute(args: Record<string, unknown>, _context: ToolExecutionContext): Promise<unknown> {
|
|
const query = args.query as string;
|
|
|
|
// 读取 SearXNG 配置（提前读取，使工具参数默认值与配置一致）
|
|
const searxngConfig = this.readSearXNGConfig();
|
|
|
|
// max_results: 工具参数 > 配置面板 > 默认 30。取较大值保证配置面板的下限约束
|
|
const argMaxResults = (args.max_results as number) ?? undefined;
|
|
const cfgMaxResults = searxngConfig.max_results > 0 ? searxngConfig.max_results : undefined;
|
|
const maxResults = Math.min(30, Math.max(1, argMaxResults ?? cfgMaxResults ?? 30));
|
|
|
|
const timeRange = (args.time_range as string) ?? '';
|
|
const enhanceSnippets = (args.enhance_snippets as boolean) ?? true;
|
|
|
|
// fetch_top: 工具参数 > 配置面板 > 默认 5。取较大值保证配置面板的下限约束
|
|
const argFetchTop = (args.fetch_top as number) ?? undefined;
|
|
const cfgFetchCount = searxngConfig.fetch_count > 0 ? searxngConfig.fetch_count : undefined;
|
|
const fetchTop = Math.min(8, Math.max(3, argFetchTop ?? cfgFetchCount ?? 5));
|
|
|
|
// 缓存检查（缓存 key 含 SearXNG 模式，避免切换后返回旧缓存）
|
|
const cacheKey = `${searxngConfig.enabled ? 'searxng' : 'builtin'}:${normalizeUrl(query).toLowerCase()}`;
|
|
const cached = searchCache.get(cacheKey);
|
|
if (cached) {
|
|
logTool('web_search', `Cache hit: "${query}"`);
|
|
return { ...(cached as Record<string, unknown>), from_cache: true };
|
|
}
|
|
|
|
let results: SearchResult[];
|
|
let mode: string;
|
|
let engineStats: Record<string, string>;
|
|
|
|
if (searxngConfig.enabled && searxngConfig.url) {
|
|
const searxResult = await this.searchSearXNG(query, searxngConfig, maxResults, timeRange);
|
|
results = searxResult.results;
|
|
mode = 'searxng';
|
|
engineStats = searxResult.engineStats;
|
|
} else {
|
|
const builtinResult = await this.searchBuiltinEngines(query, maxResults, timeRange);
|
|
results = builtinResult.results;
|
|
mode = 'builtin';
|
|
engineStats = builtinResult.engineStats;
|
|
}
|
|
|
|
// 合并去重
|
|
const deduped = this.deduplicate(results);
|
|
|
|
// 可达性预检
|
|
const topUrls = deduped.slice(0, Math.min(20, deduped.length)).map((r) => r.url);
|
|
const reachabilityMap = await checkReachability(topUrls);
|
|
for (const r of deduped) {
|
|
r.reachable = reachabilityMap.get(r.url) ?? false;
|
|
}
|
|
|
|
// 智能排序
|
|
const sorted = smartSort(deduped, reachabilityMap).slice(0, maxResults);
|
|
|
|
// 摘要增强
|
|
if (enhanceSnippets) {
|
|
await this.enhanceSnippets(sorted, 3);
|
|
}
|
|
|
|
// 自动抓取完整内容
|
|
const fetchedContent = await this.autoFetch(query, sorted, searxngConfig.fetch_mode, fetchTop);
|
|
|
|
// 格式化输出
|
|
const formatted = this.formatResults(query, sorted);
|
|
|
|
const output = {
|
|
success: true,
|
|
query,
|
|
results: sorted.map((r) => ({
|
|
title: r.title,
|
|
url: r.url,
|
|
snippet: r.snippet,
|
|
engine: r.engine,
|
|
reachable: r.reachable,
|
|
_score: r._score ? Math.round(r._score * 10) / 10 : undefined,
|
|
_enhanced: r._enhanced,
|
|
})),
|
|
total: sorted.length,
|
|
formatted,
|
|
from_cache: false,
|
|
engine_stats: engineStats,
|
|
_mode: mode,
|
|
_fetched: fetchedContent.map((f) => ({ url: f.url, title: f.title, content: f.content })),
|
|
_fetched_count: fetchedContent.length,
|
|
};
|
|
|
|
// 写入缓存
|
|
searchCache.set(cacheKey, output);
|
|
logTool('web_search', `Completed: ${sorted.length} results, ${fetchedContent.length} fetched, mode=${mode}`);
|
|
|
|
return output;
|
|
}
|
|
|
|
// ===== SearXNG 搜索 =====
|
|
|
|
private async searchSearXNG(
|
|
query: string,
|
|
config: SearXNGConfig,
|
|
maxResults: number,
|
|
timeRange: string,
|
|
): Promise<{ results: SearchResult[]; engineStats: Record<string, string> }> {
|
|
const params = new URLSearchParams();
|
|
params.set('q', query);
|
|
params.set('format', config.format || 'json');
|
|
if (config.engines) params.set('engines', config.engines);
|
|
if (config.language && config.language !== 'auto') params.set('language', config.language);
|
|
params.set('safesearch', String(config.safesearch));
|
|
const tr = timeRange || config.time_range;
|
|
if (tr) params.set('time_range', tr);
|
|
const limit = Math.max(maxResults, config.max_results);
|
|
if (limit > 0) params.set('results_count', String(limit));
|
|
|
|
const searchUrl = `${config.url.replace(/\/$/, '')}/search?${params.toString()}`;
|
|
const headers = buildSearXNGAuthHeaders(config.auth_key, config.auth_type);
|
|
|
|
logTool('web_search', `[SearXNG] Fetching: ${searchUrl}`);
|
|
|
|
const response = await fetchWithTimeout(searchUrl, { headers }, 15_000);
|
|
if (!response.ok) {
|
|
throw new Error(`SearXNG API error: ${response.status} ${response.statusText}`);
|
|
}
|
|
|
|
const results: SearchResult[] = [];
|
|
const engineStats: Record<string, string> = {};
|
|
const engineCounts = new Map<string, number>();
|
|
|
|
if (config.format === 'html') {
|
|
const html = await response.text();
|
|
// HTML 模式：简单提取链接和标题
|
|
const matches = html.matchAll(/<a[^>]*href="(https?:\/\/[^"]+)"[^>]*>([\s\S]*?)<\/a>/gi);
|
|
for (const m of matches) {
|
|
const url = m[1];
|
|
const title = m[2].replace(/<[^>]+>/g, '').trim();
|
|
if (title && url && !url.includes(config.url)) {
|
|
results.push({ title, url, snippet: '', engine: 'searxng', weight: 85 });
|
|
}
|
|
}
|
|
} else {
|
|
const data = await response.json() as { results?: Array<Record<string, unknown>> };
|
|
for (const item of data.results ?? []) {
|
|
const url = item.url as string;
|
|
const title = item.title as string;
|
|
const snippet = item.content as string;
|
|
const engine = (item.engine as string) ?? 'searxng';
|
|
results.push({ title, url, snippet: snippet ?? '', engine, weight: 85 });
|
|
engineCounts.set(engine, (engineCounts.get(engine) ?? 0) + 1);
|
|
}
|
|
}
|
|
|
|
for (const [engine, count] of engineCounts) {
|
|
engineStats[engine] = `${count} 条`;
|
|
}
|
|
|
|
return { results, engineStats };
|
|
}
|
|
|
|
// ===== 内置四引擎并行搜索 =====
|
|
|
|
private async searchBuiltinEngines(
|
|
query: string,
|
|
maxResults: number,
|
|
timeRange: string,
|
|
): Promise<{ results: SearchResult[]; engineStats: Record<string, string> }> {
|
|
const searchPromises = ENGINES.map(async (engine) => {
|
|
try {
|
|
const url = engine.searchUrl(query, timeRange);
|
|
const response = await fetchWithTimeout(url, {
|
|
headers: {
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36',
|
|
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
|
|
},
|
|
}, 8_000);
|
|
|
|
if (!response.ok) {
|
|
logTool('web_search', `[内置] ${engine.name} HTTP ${response.status}`);
|
|
return [];
|
|
}
|
|
|
|
const html = await response.text();
|
|
const parsed = engine.parse(html);
|
|
logTool('web_search', `[内置] ${engine.name}: ${parsed.length} results`);
|
|
return parsed;
|
|
} catch (err) {
|
|
logTool('web_search', `[内置] ${engine.name} error: ${(err as Error).message}`);
|
|
return [];
|
|
}
|
|
});
|
|
|
|
const settled = await Promise.allSettled(searchPromises);
|
|
const allResults: SearchResult[] = [];
|
|
const engineStats: Record<string, string> = {};
|
|
|
|
for (let i = 0; i < ENGINES.length; i++) {
|
|
const result = settled[i];
|
|
const engineName = ENGINES[i].name;
|
|
if (result.status === 'fulfilled') {
|
|
allResults.push(...result.value);
|
|
engineStats[engineName] = `${result.value.length} 条`;
|
|
} else {
|
|
engineStats[engineName] = '失败';
|
|
}
|
|
}
|
|
|
|
return { results: allResults, engineStats };
|
|
}
|
|
|
|
// ===== 合并去重 =====
|
|
|
|
private deduplicate(results: SearchResult[]): SearchResult[] {
|
|
const seen = new Map<string, SearchResult>();
|
|
for (const r of results) {
|
|
const normalized = normalizeUrl(r.url);
|
|
if (!seen.has(normalized)) {
|
|
seen.set(normalized, r);
|
|
}
|
|
}
|
|
return Array.from(seen.values());
|
|
}
|
|
|
|
// ===== 摘要增强 =====
|
|
|
|
private async enhanceSnippets(results: SearchResult[], maxEnhance: number): Promise<void> {
|
|
let enhanced = 0;
|
|
for (const r of results) {
|
|
if (enhanced >= maxEnhance) break;
|
|
if (r.snippet.length < 30 && r.reachable) {
|
|
try {
|
|
const resp = await fetchWithTimeout(r.url, {
|
|
headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36' },
|
|
}, 5_000);
|
|
if (resp.ok) {
|
|
const html = await resp.text();
|
|
// 延迟导入 htmlToText 以避免循环依赖
|
|
const { htmlToText } = await import('./network-utils');
|
|
const text = htmlToText(html).slice(0, 200);
|
|
if (text.length > r.snippet.length) {
|
|
r.snippet = text;
|
|
r._enhanced = true;
|
|
enhanced++;
|
|
}
|
|
}
|
|
} catch {
|
|
// 忽略增强失败
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// ===== 自动抓取完整内容 =====
|
|
|
|
private async autoFetch(
|
|
query: string,
|
|
results: SearchResult[],
|
|
fetchMode: string,
|
|
fetchTop: number,
|
|
): Promise<Array<{ url: string; title: string; content: string }>> {
|
|
// 相关性评分（不过滤，relevance=0 的结果也参与抓取候选）
|
|
const withRelevance = results.map((r) => ({ result: r, relevance: computeRelevance(query, r) }));
|
|
const filtered = withRelevance.length > 0 ? withRelevance : [];
|
|
|
|
// 确定抓取数量：fetchTop 已在 execute() 中综合了配置面板和工具参数
|
|
const topN = Math.min(fetchTop, 8, filtered.length);
|
|
|
|
// 构建抓取列表
|
|
let toFetch: typeof filtered;
|
|
if (fetchMode === 'random') {
|
|
// Fisher-Yates 洗牌
|
|
const shuffled = [...filtered];
|
|
for (let i = shuffled.length - 1; i > 0; i--) {
|
|
const j = Math.floor(Math.random() * (i + 1));
|
|
[shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
|
|
}
|
|
toFetch = shuffled.slice(0, topN);
|
|
} else {
|
|
toFetch = filtered
|
|
.sort((a, b) => b.relevance - a.relevance)
|
|
.slice(0, topN);
|
|
}
|
|
|
|
const fetched: Array<{ url: string; title: string; content: string }> = [];
|
|
const { htmlToText } = await import('./network-utils');
|
|
|
|
for (const item of toFetch) {
|
|
try {
|
|
const resp = await fetchWithTimeout(item.result.url, {
|
|
headers: {
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36',
|
|
'Accept-Language': 'zh-CN,zh;q=0.9',
|
|
},
|
|
}, 15_000);
|
|
|
|
if (resp.ok) {
|
|
const html = await resp.text();
|
|
const content = htmlToText(html);
|
|
fetched.push({ url: item.result.url, title: item.result.title, content });
|
|
}
|
|
} catch (err) {
|
|
logTool('web_search', `Auto-fetch failed for ${item.result.url}: ${(err as Error).message}`);
|
|
// 从剩余结果中随机补充
|
|
const remaining = withRelevance.filter((x) => !toFetch.includes(x) && !fetched.some((f) => f.url === x.result.url));
|
|
if (remaining.length > 0) {
|
|
const randomPick = remaining[Math.floor(Math.random() * remaining.length)];
|
|
try {
|
|
const resp2 = await fetchWithTimeout(randomPick.result.url, {
|
|
headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36' },
|
|
}, 10_000);
|
|
if (resp2.ok) {
|
|
const html2 = await resp2.text();
|
|
fetched.push({ url: randomPick.result.url, title: randomPick.result.title, content: htmlToText(html2) });
|
|
}
|
|
} catch {
|
|
// 忽略补充失败
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return fetched;
|
|
}
|
|
|
|
// ===== 格式化人类可读输出 =====
|
|
|
|
private formatResults(query: string, results: SearchResult[]): string {
|
|
if (results.length === 0) return `搜索 "${query}" 无结果。`;
|
|
const lines = [`搜索 "${query}" — ${results.length} 条结果：\n`];
|
|
results.forEach((r, i) => {
|
|
lines.push(`${i + 1}. ${r.title}`);
|
|
lines.push(` URL: ${r.url}`);
|
|
if (r.snippet) lines.push(` 摘要: ${r.snippet.slice(0, 150)}`);
|
|
lines.push(` 来源: ${r.engine}${r.reachable === false ? ' (不可达)' : ''}${r._enhanced ? ' [已增强]' : ''}\n`);
|
|
});
|
|
return lines.join('\n');
|
|
}
|
|
|
|
// ===== 读取 SearXNG 配置 =====
|
|
|
|
private readSearXNGConfig(): SearXNGConfig {
|
|
const cs = this.configService;
|
|
return {
|
|
enabled: cs.get<boolean>('searxng.enabled') ?? false,
|
|
url: cs.get<string>('searxng.url') ?? '',
|
|
engines: cs.get<string>('searxng.engines') ?? '',
|
|
language: cs.get<string>('searxng.language') ?? 'zh-CN',
|
|
safesearch: cs.get<number>('searxng.safesearch') ?? 1,
|
|
time_range: cs.get<string>('searxng.time_range') ?? '',
|
|
max_results: cs.get<number>('searxng.max_results') ?? 0,
|
|
auth_key: cs.get<string>('searxng.auth_key') ?? '',
|
|
auth_type: cs.get<string>('searxng.auth_type') ?? 'bearer',
|
|
format: cs.get<string>('searxng.format') ?? 'json',
|
|
fetch_count: cs.get<number>('searxng.fetch_count') ?? 0,
|
|
fetch_mode: cs.get<string>('searxng.fetch_mode') ?? 'sequential',
|
|
};
|
|
}
|
|
}
|