feat: MEMORY.md 访问保护、格式简化及预存问题修复
MEMORY.md 保护: 新建 file-guard.ts 共享守卫模块; read_file/write_file/search_files/run_command 禁止访问根目录 MEMORY.md; permissions.ts 增加 deniedPatterns 深度防御。格式简化: 元数据从 # 注释改为 > 引用语法; 校验规则从 6 条简化为 3 条; context-builder extractContent 适配。预存问题: 修复路径遍历前缀碰撞漏洞; 移除 browser_extract 内容截断; SearXNG 配置实时读取; web_search/web_fetch 移除截断; 侧边栏动态获取工具列表; 版本号 0.1.1
This commit is contained in:
thzxx
@@ -0,0 +1,377 @@
|
||||
/**
|
||||
* BrowserWindowManager — 单例浏览器窗口管理器
|
||||
*
|
||||
* 提供网页加载、截图、JS 执行、内容提取、交互操作能力。
|
||||
* 单例懒加载，首次调用时创建，browserClose() 显式销毁。
|
||||
*
|
||||
* @see docs/Agent网络工具通用设计-v2.md — 第 4 章 browser 浏览器设计
|
||||
*/
|
||||
|
||||
import { BrowserWindow } from 'electron';
|
||||
import log from 'electron-log';
|
||||
|
||||
export interface BrowserOpenOptions {
|
||||
url: string;
|
||||
waitSelector?: string;
|
||||
}
|
||||
|
||||
export interface BrowserOpenResult {
|
||||
title: string;
|
||||
url: string;
|
||||
}
|
||||
|
||||
export interface ScreenshotOptions {
|
||||
fullPage?: boolean;
|
||||
selector?: string;
|
||||
}
|
||||
|
||||
export interface ScreenshotResult {
|
||||
data: string; // Base64 PNG
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export interface ExtractResult {
|
||||
text: string;
|
||||
links: Array<{ text: string; url: string }>;
|
||||
}
|
||||
|
||||
export interface ScrollOptions {
|
||||
direction?: 'down' | 'up' | 'top' | 'bottom';
|
||||
selector?: string;
|
||||
}
|
||||
|
||||
export interface WaitOptions {
|
||||
selector?: string;
|
||||
timeMs?: number;
|
||||
}
|
||||
|
||||
export class BrowserWindowManager {
|
||||
private win: BrowserWindow | null = null;
|
||||
private currentUrl: string | null = null;
|
||||
|
||||
/** 窗口是否就绪 */
|
||||
get ready(): boolean {
|
||||
return this.win !== null && !this.win.isDestroyed();
|
||||
}
|
||||
|
||||
// ===== browserOpen =====
|
||||
|
||||
async open(options: BrowserOpenOptions): Promise<BrowserOpenResult> {
|
||||
// 若已有窗口加载了不同 URL → 先关闭重建
|
||||
if (this.win && this.currentUrl !== options.url) {
|
||||
this.destroy();
|
||||
}
|
||||
|
||||
if (!this.win) {
|
||||
this.win = new BrowserWindow({
|
||||
width: 1280,
|
||||
height: 800,
|
||||
show: false,
|
||||
webPreferences: {
|
||||
nodeIntegration: false,
|
||||
contextIsolation: true,
|
||||
sandbox: true,
|
||||
webSecurity: false,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
await this.loadURLWithTimeout(this.win, options.url, 30_000);
|
||||
this.currentUrl = options.url;
|
||||
|
||||
// 可选等待选择器
|
||||
if (options.waitSelector) {
|
||||
await this.waitForSelector(options.waitSelector, 10_000);
|
||||
}
|
||||
|
||||
const title = await this.evaluate('document.title');
|
||||
return { title: String(title ?? ''), url: options.url };
|
||||
}
|
||||
|
||||
// ===== browserScreenshot =====
|
||||
|
||||
async screenshot(options: ScreenshotOptions = {}): Promise<ScreenshotResult> {
|
||||
this.ensureReady();
|
||||
const win = this.win!;
|
||||
|
||||
// 元素截图
|
||||
if (options.selector) {
|
||||
const rect = await win.webContents.executeJavaScript(
|
||||
`(() => {
|
||||
const el = document.querySelector(${JSON.stringify(options.selector)});
|
||||
if (!el) return null;
|
||||
const r = el.getBoundingClientRect();
|
||||
return { x: r.x, y: r.y, width: r.width, height: r.height };
|
||||
})()`,
|
||||
true,
|
||||
) as { x: number; y: number; width: number; height: number } | null;
|
||||
|
||||
if (!rect) throw new Error(`Element not found: ${options.selector}`);
|
||||
|
||||
const image = await win.webContents.capturePage({
|
||||
x: Math.round(rect.x),
|
||||
y: Math.round(rect.y),
|
||||
width: Math.round(rect.width),
|
||||
height: Math.round(rect.height),
|
||||
});
|
||||
return { data: image.toPNG().toString('base64'), width: rect.width, height: rect.height };
|
||||
}
|
||||
|
||||
// 全页截图
|
||||
if (options.fullPage) {
|
||||
const dims = await win.webContents.executeJavaScript(
|
||||
`({ width: document.documentElement.scrollWidth, height: document.documentElement.scrollHeight })`,
|
||||
true,
|
||||
) as { width: number; height: number };
|
||||
|
||||
// 先滚动到底部触发懒加载
|
||||
await win.webContents.executeJavaScript(
|
||||
`window.scrollTo(0, document.body.scrollHeight)`,
|
||||
true,
|
||||
);
|
||||
await this.sleep(500);
|
||||
await win.webContents.executeJavaScript(
|
||||
`window.scrollTo(0, 0)`,
|
||||
true,
|
||||
);
|
||||
await this.sleep(300);
|
||||
|
||||
const image = await win.webContents.capturePage({
|
||||
x: 0, y: 0,
|
||||
width: dims.width,
|
||||
height: dims.height,
|
||||
});
|
||||
return { data: image.toPNG().toString('base64'), width: dims.width, height: dims.height };
|
||||
}
|
||||
|
||||
// 视口截图
|
||||
const image = await win.webContents.capturePage();
|
||||
const size = win.getContentSize();
|
||||
return { data: image.toPNG().toString('base64'), width: size[0], height: size[1] };
|
||||
}
|
||||
|
||||
// ===== browserEvaluate =====
|
||||
|
||||
async evaluate(js: string): Promise<unknown> {
|
||||
this.ensureReady();
|
||||
const result = await this.win!.webContents.executeJavaScript(js, true);
|
||||
if (typeof result === 'string') return result;
|
||||
return result;
|
||||
}
|
||||
|
||||
// ===== browserExtract =====
|
||||
|
||||
async extract(selector?: string): Promise<ExtractResult> {
|
||||
this.ensureReady();
|
||||
|
||||
const result = await this.win!.webContents.executeJavaScript(
|
||||
`(() => {
|
||||
const root = ${selector ? `document.querySelector(${JSON.stringify(selector)})` : 'document.body'};
|
||||
if (!root) return null;
|
||||
const clone = root.cloneNode(true);
|
||||
clone.querySelectorAll('script, style, noscript, iframe, svg, nav, header, footer, aside').forEach(el => el.remove());
|
||||
const text = (clone.innerText || '').trim();
|
||||
|
||||
const links = [];
|
||||
const anchors = (selector ? root : document).querySelectorAll('a[href^="http"]');
|
||||
for (const a of anchors) {
|
||||
if (links.length >= 50) break;
|
||||
const text = (a.textContent || '').trim();
|
||||
if (text && a.href) links.push({ text, url: a.href });
|
||||
}
|
||||
return { text, links };
|
||||
})()`,
|
||||
true,
|
||||
) as { text: string; links: Array<{ text: string; url: string }> } | null;
|
||||
|
||||
if (!result) throw new Error(selector ? `Element not found: ${selector}` : 'No body content');
|
||||
|
||||
return { text: result.text, links: result.links };
|
||||
}
|
||||
|
||||
// ===== browserClick =====
|
||||
|
||||
async click(selector: string, wait = false): Promise<void> {
|
||||
this.ensureReady();
|
||||
|
||||
if (wait) {
|
||||
await this.waitForSelector(selector, 10_000);
|
||||
}
|
||||
|
||||
const found = await this.win!.webContents.executeJavaScript(
|
||||
`(() => {
|
||||
const el = document.querySelector(${JSON.stringify(selector)});
|
||||
if (!el) return false;
|
||||
el.scrollIntoView({ behavior: 'smooth', block: 'center' });
|
||||
return true;
|
||||
})()`,
|
||||
true,
|
||||
);
|
||||
|
||||
if (!found) throw new Error(`Element not found: ${selector}`);
|
||||
|
||||
await this.sleep(300);
|
||||
await this.win!.webContents.executeJavaScript(
|
||||
`(() => {
|
||||
const el = document.querySelector(${JSON.stringify(selector)});
|
||||
if (!el) return;
|
||||
el.click();
|
||||
})()`,
|
||||
true,
|
||||
);
|
||||
await this.sleep(500);
|
||||
}
|
||||
|
||||
// ===== browserType =====
|
||||
|
||||
async type(selector: string, text: string, options: { clear?: boolean; submit?: boolean } = {}): Promise<void> {
|
||||
this.ensureReady();
|
||||
|
||||
const found = await this.win!.webContents.executeJavaScript(
|
||||
`(() => {
|
||||
const el = document.querySelector(${JSON.stringify(selector)});
|
||||
if (!el) return false;
|
||||
el.scrollIntoView({ behavior: 'smooth', block: 'center' });
|
||||
el.focus();
|
||||
return true;
|
||||
})()`,
|
||||
true,
|
||||
);
|
||||
|
||||
if (!found) throw new Error(`Element not found: ${selector}`);
|
||||
await this.sleep(300);
|
||||
|
||||
// 清空 + 赋值 + 触发事件（兼容 React/Vue）
|
||||
await this.win!.webContents.executeJavaScript(
|
||||
`(() => {
|
||||
const el = document.querySelector(${JSON.stringify(selector)});
|
||||
if (!el) return;
|
||||
${options.clear ? 'el.value = "";' : ''}
|
||||
el.value += ${JSON.stringify(text)};
|
||||
el.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
el.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
})()`,
|
||||
true,
|
||||
);
|
||||
|
||||
if (options.submit) {
|
||||
await this.win!.webContents.executeJavaScript(
|
||||
`(() => {
|
||||
const el = document.querySelector(${JSON.stringify(selector)});
|
||||
if (!el) return;
|
||||
if (el.form) el.form.submit();
|
||||
else el.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true }));
|
||||
})()`,
|
||||
true,
|
||||
);
|
||||
await this.sleep(1_000);
|
||||
} else {
|
||||
await this.sleep(300);
|
||||
}
|
||||
}
|
||||
|
||||
// ===== browserScroll =====
|
||||
|
||||
async scroll(options: ScrollOptions = {}): Promise<void> {
|
||||
this.ensureReady();
|
||||
|
||||
if (options.selector) {
|
||||
await this.win!.webContents.executeJavaScript(
|
||||
`(() => {
|
||||
const el = document.querySelector(${JSON.stringify(options.selector)});
|
||||
if (el) el.scrollIntoView({ behavior: 'smooth', block: 'center' });
|
||||
})()`,
|
||||
true,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const direction = options.direction ?? 'down';
|
||||
const js = {
|
||||
down: 'window.scrollBy(0, 500)',
|
||||
up: 'window.scrollBy(0, -500)',
|
||||
top: 'window.scrollTo(0, 0)',
|
||||
bottom: 'window.scrollTo(0, document.body.scrollHeight)',
|
||||
}[direction] ?? 'window.scrollBy(0, 500)';
|
||||
|
||||
await this.win!.webContents.executeJavaScript(js, true);
|
||||
await this.sleep(300);
|
||||
}
|
||||
|
||||
// ===== browserWait =====
|
||||
|
||||
async wait(options: WaitOptions = {}): Promise<void> {
|
||||
if (options.selector) {
|
||||
await this.waitForSelector(options.selector, options.timeMs ?? 10_000);
|
||||
} else {
|
||||
await this.sleep(options.timeMs ?? 1_000);
|
||||
}
|
||||
}
|
||||
|
||||
// ===== browserClose =====
|
||||
|
||||
close(): void {
|
||||
this.destroy();
|
||||
}
|
||||
|
||||
// ===== 内部辅助 =====
|
||||
|
||||
private ensureReady(): void {
|
||||
if (!this.ready) {
|
||||
throw new Error('Browser window not ready. Call browser_open first.');
|
||||
}
|
||||
}
|
||||
|
||||
/** 带超时的 loadURL（Electron 原生不支持 timeout 选项） */
|
||||
private async loadURLWithTimeout(win: BrowserWindow, url: string, timeoutMs: number): Promise<void> {
|
||||
let timer: NodeJS.Timeout | null = null;
|
||||
const timeoutPromise = new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(() => reject(new Error(`Page load timeout after ${timeoutMs}ms: ${url}`)), timeoutMs);
|
||||
});
|
||||
|
||||
try {
|
||||
await Promise.race([
|
||||
win.loadURL(url),
|
||||
timeoutPromise,
|
||||
]);
|
||||
} finally {
|
||||
if (timer) clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
private async waitForSelector(selector: string, timeoutMs: number): Promise<void> {
|
||||
const startTime = Date.now();
|
||||
while (Date.now() - startTime < timeoutMs) {
|
||||
const found = await this.win!.webContents.executeJavaScript(
|
||||
`!!document.querySelector(${JSON.stringify(selector)})`,
|
||||
true,
|
||||
);
|
||||
if (found) return;
|
||||
await this.sleep(300);
|
||||
}
|
||||
throw new Error(`Timeout waiting for selector: ${selector}`);
|
||||
}
|
||||
|
||||
private destroy(): void {
|
||||
if (this.win && !this.win.isDestroyed()) {
|
||||
try {
|
||||
this.win.close();
|
||||
} catch {
|
||||
// 忽略关闭错误
|
||||
}
|
||||
}
|
||||
this.win = null;
|
||||
this.currentUrl = null;
|
||||
}
|
||||
|
||||
private sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/** 应用退出时清理 */
|
||||
static cleanup(manager: BrowserWindowManager | null): void {
|
||||
if (manager) manager.close();
|
||||
log.info('[BrowserWindowManager] Cleaned up');
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,388 @@
|
||||
/**
|
||||
* browser — 浏览器交互工具集（9 个函数）
|
||||
*
|
||||
* 基于单例 BrowserWindowManager 实现网页加载、截图、JS 执行、
|
||||
* 内容提取、点击、输入、滚动、等待、关闭等操作。
|
||||
*
|
||||
* 工具列表：
|
||||
* 1. browser_open — 打开 URL
|
||||
* 2. browser_screenshot — 截图（视口/元素/全页三模式）
|
||||
* 3. browser_evaluate — 执行 JavaScript
|
||||
* 4. browser_extract — 提取页面文本与链接
|
||||
* 5. browser_click — 点击元素
|
||||
* 6. browser_type — 输入文本
|
||||
* 7. browser_scroll — 滚动页面
|
||||
* 8. browser_wait — 等待条件
|
||||
* 9. browser_close — 关闭窗口
|
||||
*
|
||||
* @see docs/Agent网络工具通用设计-v2.md — 第 4 章 browser 浏览器设计
|
||||
*/
|
||||
|
||||
import type { IMetonaTool, ToolExecutionContext } from '../../types/metona-tool';
|
||||
import type { MetonaToolDef } from '../../../harness/types';
|
||||
import { MetonaToolCategory, MetonaRiskLevel } from '../../../harness/types';
|
||||
import { BrowserWindowManager } from './browser-window-manager';
|
||||
import { logTool } from './network-utils';
|
||||
|
||||
// ===== 单例 Manager =====
|
||||
|
||||
let managerInstance: BrowserWindowManager | null = null;
|
||||
|
||||
function getManager(): BrowserWindowManager {
|
||||
if (!managerInstance) {
|
||||
managerInstance = new BrowserWindowManager();
|
||||
}
|
||||
return managerInstance;
|
||||
}
|
||||
|
||||
/** 应用退出时清理（由 main.ts 调用） */
|
||||
export function cleanupBrowser(): void {
|
||||
BrowserWindowManager.cleanup(managerInstance);
|
||||
managerInstance = null;
|
||||
}
|
||||
|
||||
// ===== 1. browser_open =====
|
||||
|
||||
export class BrowserOpenTool implements IMetonaTool {
|
||||
readonly definition: MetonaToolDef = {
|
||||
name: 'browser_open',
|
||||
description: 'Open a URL in a hidden browser window. Loads the page and optionally waits for a selector to appear. Returns the page title and URL.',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
url: { type: 'string', description: 'Target URL (http/https only)' },
|
||||
wait_selector: { type: 'string', description: 'CSS selector to wait for before returning (optional)' },
|
||||
},
|
||||
required: ['url'],
|
||||
},
|
||||
category: MetonaToolCategory.NETWORK,
|
||||
riskLevel: MetonaRiskLevel.MEDIUM,
|
||||
requiresPermission: true,
|
||||
timeoutMs: 60_000,
|
||||
};
|
||||
|
||||
async execute(args: Record<string, unknown>, _context: ToolExecutionContext): Promise<unknown> {
|
||||
const url = args.url as string;
|
||||
if (!url || !/^https?:\/\//i.test(url)) {
|
||||
return { success: false, error: 'URL must start with http:// or https://' };
|
||||
}
|
||||
|
||||
const waitSelector = args.wait_selector as string | undefined;
|
||||
logTool('browser_open', `Opening: ${url}`);
|
||||
|
||||
try {
|
||||
const result = await getManager().open({ url, waitSelector });
|
||||
return { success: true, ...result };
|
||||
} catch (err) {
|
||||
return { success: false, error: (err as Error).message };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ===== 2. browser_screenshot =====
|
||||
|
||||
export class BrowserScreenshotTool implements IMetonaTool {
|
||||
readonly definition: MetonaToolDef = {
|
||||
name: 'browser_screenshot',
|
||||
description: 'Capture a screenshot of the current browser window. Three modes: viewport (default), full page, or specific element by CSS selector. Returns base64-encoded PNG image.',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
full_page: { type: 'boolean', description: 'Capture entire scrollable page (default false)' },
|
||||
selector: { type: 'string', description: 'CSS selector to capture a specific element (overrides full_page)' },
|
||||
},
|
||||
required: [],
|
||||
},
|
||||
category: MetonaToolCategory.NETWORK,
|
||||
riskLevel: MetonaRiskLevel.LOW,
|
||||
requiresPermission: false,
|
||||
timeoutMs: 30_000,
|
||||
};
|
||||
|
||||
async execute(args: Record<string, unknown>, _context: ToolExecutionContext): Promise<unknown> {
|
||||
const fullPage = (args.full_page as boolean) ?? false;
|
||||
const selector = args.selector as string | undefined;
|
||||
|
||||
logTool('browser_screenshot', `full_page=${fullPage}, selector=${selector ?? 'none'}`);
|
||||
|
||||
try {
|
||||
const result = await getManager().screenshot({ fullPage, selector });
|
||||
return {
|
||||
success: true,
|
||||
image: result.data,
|
||||
width: result.width,
|
||||
height: result.height,
|
||||
mime_type: 'image/png',
|
||||
};
|
||||
} catch (err) {
|
||||
return { success: false, error: (err as Error).message };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ===== 3. browser_evaluate =====
|
||||
|
||||
export class BrowserEvaluateTool implements IMetonaTool {
|
||||
readonly definition: MetonaToolDef = {
|
||||
name: 'browser_evaluate',
|
||||
description: 'Execute JavaScript code in the current browser page context. Returns the result of the last expression. Use for data extraction, DOM queries, or triggering page actions.',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
script: { type: 'string', description: 'JavaScript code to execute (must return a value)' },
|
||||
},
|
||||
required: ['script'],
|
||||
},
|
||||
category: MetonaToolCategory.NETWORK,
|
||||
riskLevel: MetonaRiskLevel.HIGH,
|
||||
requiresPermission: true,
|
||||
timeoutMs: 30_000,
|
||||
};
|
||||
|
||||
async execute(args: Record<string, unknown>, _context: ToolExecutionContext): Promise<unknown> {
|
||||
const script = args.script as string;
|
||||
if (!script) {
|
||||
return { success: false, error: 'No script provided' };
|
||||
}
|
||||
|
||||
logTool('browser_evaluate', `Executing ${script.length} chars of JS`);
|
||||
|
||||
try {
|
||||
const result = await getManager().evaluate(script);
|
||||
return { success: true, result };
|
||||
} catch (err) {
|
||||
return { success: false, error: (err as Error).message };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ===== 4. browser_extract =====
|
||||
|
||||
export class BrowserExtractTool implements IMetonaTool {
|
||||
readonly definition: MetonaToolDef = {
|
||||
name: 'browser_extract',
|
||||
description: 'Extract text content and links from the current browser page. Optionally target a specific element by CSS selector. Strips scripts, styles, and navigation elements. Returns clean text and up to 50 links.',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
selector: { type: 'string', description: 'CSS selector to extract from (default: entire body)' },
|
||||
},
|
||||
required: [],
|
||||
},
|
||||
category: MetonaToolCategory.NETWORK,
|
||||
riskLevel: MetonaRiskLevel.LOW,
|
||||
requiresPermission: false,
|
||||
timeoutMs: 30_000,
|
||||
};
|
||||
|
||||
async execute(args: Record<string, unknown>, _context: ToolExecutionContext): Promise<unknown> {
|
||||
const selector = args.selector as string | undefined;
|
||||
|
||||
logTool('browser_extract', `selector=${selector ?? 'body'}`);
|
||||
|
||||
try {
|
||||
const result = await getManager().extract(selector);
|
||||
return {
|
||||
success: true,
|
||||
text: result.text,
|
||||
links: result.links,
|
||||
link_count: result.links.length,
|
||||
};
|
||||
} catch (err) {
|
||||
return { success: false, error: (err as Error).message };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ===== 5. browser_click =====
|
||||
|
||||
export class BrowserClickTool implements IMetonaTool {
|
||||
readonly definition: MetonaToolDef = {
|
||||
name: 'browser_click',
|
||||
description: 'Click an element on the page by CSS selector. Scrolls the element into view before clicking. Optionally wait for the selector to appear first.',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
selector: { type: 'string', description: 'CSS selector of the element to click' },
|
||||
wait: { type: 'boolean', description: 'Wait for selector to appear before clicking (default false)' },
|
||||
},
|
||||
required: ['selector'],
|
||||
},
|
||||
category: MetonaToolCategory.NETWORK,
|
||||
riskLevel: MetonaRiskLevel.MEDIUM,
|
||||
requiresPermission: true,
|
||||
timeoutMs: 30_000,
|
||||
};
|
||||
|
||||
async execute(args: Record<string, unknown>, _context: ToolExecutionContext): Promise<unknown> {
|
||||
const selector = args.selector as string;
|
||||
const wait = (args.wait as boolean) ?? false;
|
||||
|
||||
if (!selector) {
|
||||
return { success: false, error: 'No selector provided' };
|
||||
}
|
||||
|
||||
logTool('browser_click', `selector=${selector}, wait=${wait}`);
|
||||
|
||||
try {
|
||||
await getManager().click(selector, wait);
|
||||
return { success: true, selector, clicked: true };
|
||||
} catch (err) {
|
||||
return { success: false, error: (err as Error).message };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ===== 6. browser_type =====
|
||||
|
||||
export class BrowserTypeTool implements IMetonaTool {
|
||||
readonly definition: MetonaToolDef = {
|
||||
name: 'browser_type',
|
||||
description: 'Type text into an input element by CSS selector. Focuses the element, optionally clears it first, and dispatches input/change events for React/Vue compatibility. Can optionally submit the form.',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
selector: { type: 'string', description: 'CSS selector of the input element' },
|
||||
text: { type: 'string', description: 'Text to type into the element' },
|
||||
clear: { type: 'boolean', description: 'Clear the field before typing (default true)' },
|
||||
submit: { type: 'boolean', description: 'Submit the form after typing (default false)' },
|
||||
},
|
||||
required: ['selector', 'text'],
|
||||
},
|
||||
category: MetonaToolCategory.NETWORK,
|
||||
riskLevel: MetonaRiskLevel.MEDIUM,
|
||||
requiresPermission: true,
|
||||
timeoutMs: 30_000,
|
||||
};
|
||||
|
||||
async execute(args: Record<string, unknown>, _context: ToolExecutionContext): Promise<unknown> {
|
||||
const selector = args.selector as string;
|
||||
const text = args.text as string;
|
||||
const clear = (args.clear as boolean) ?? true;
|
||||
const submit = (args.submit as boolean) ?? false;
|
||||
|
||||
if (!selector) {
|
||||
return { success: false, error: 'No selector provided' };
|
||||
}
|
||||
if (text === undefined || text === null) {
|
||||
return { success: false, error: 'No text provided' };
|
||||
}
|
||||
|
||||
logTool('browser_type', `selector=${selector}, len=${text.length}, submit=${submit}`);
|
||||
|
||||
try {
|
||||
await getManager().type(selector, text, { clear, submit });
|
||||
return { success: true, selector, typed: text.length };
|
||||
} catch (err) {
|
||||
return { success: false, error: (err as Error).message };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ===== 7. browser_scroll =====
|
||||
|
||||
export class BrowserScrollTool implements IMetonaTool {
|
||||
readonly definition: MetonaToolDef = {
|
||||
name: 'browser_scroll',
|
||||
description: 'Scroll the browser page. Modes: down/up (500px increment), top/bottom (absolute), or scroll to a specific element by CSS selector.',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
direction: { type: 'string', description: 'Scroll direction: down, up, top, bottom (default down)' },
|
||||
selector: { type: 'string', description: 'CSS selector to scroll to (overrides direction)' },
|
||||
},
|
||||
required: [],
|
||||
},
|
||||
category: MetonaToolCategory.NETWORK,
|
||||
riskLevel: MetonaRiskLevel.LOW,
|
||||
requiresPermission: false,
|
||||
timeoutMs: 15_000,
|
||||
};
|
||||
|
||||
async execute(args: Record<string, unknown>, _context: ToolExecutionContext): Promise<unknown> {
|
||||
const direction = (args.direction as string) ?? 'down';
|
||||
const selector = args.selector as string | undefined;
|
||||
|
||||
logTool('browser_scroll', `direction=${direction}, selector=${selector ?? 'none'}`);
|
||||
|
||||
try {
|
||||
await getManager().scroll({ direction: direction as 'down' | 'up' | 'top' | 'bottom', selector });
|
||||
return { success: true, direction, selector };
|
||||
} catch (err) {
|
||||
return { success: false, error: (err as Error).message };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ===== 8. browser_wait =====
|
||||
|
||||
export class BrowserWaitTool implements IMetonaTool {
|
||||
readonly definition: MetonaToolDef = {
|
||||
name: 'browser_wait',
|
||||
description: 'Wait for a condition on the browser page. Either wait for a CSS selector to appear (with timeout) or wait for a fixed duration.',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
selector: { type: 'string', description: 'CSS selector to wait for (mutually exclusive with time_ms)' },
|
||||
time_ms: { type: 'number', description: 'Fixed wait time in milliseconds (default 1000)' },
|
||||
},
|
||||
required: [],
|
||||
},
|
||||
category: MetonaToolCategory.NETWORK,
|
||||
riskLevel: MetonaRiskLevel.LOW,
|
||||
requiresPermission: false,
|
||||
timeoutMs: 30_000,
|
||||
};
|
||||
|
||||
async execute(args: Record<string, unknown>, _context: ToolExecutionContext): Promise<unknown> {
|
||||
const selector = args.selector as string | undefined;
|
||||
const timeMs = (args.time_ms as number) ?? 1_000;
|
||||
|
||||
logTool('browser_wait', `selector=${selector ?? 'none'}, time_ms=${timeMs}`);
|
||||
|
||||
try {
|
||||
await getManager().wait({ selector, timeMs });
|
||||
return { success: true, waited_for: selector ?? `${timeMs}ms` };
|
||||
} catch (err) {
|
||||
return { success: false, error: (err as Error).message };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ===== 9. browser_close =====
|
||||
|
||||
export class BrowserCloseTool implements IMetonaTool {
|
||||
readonly definition: MetonaToolDef = {
|
||||
name: 'browser_close',
|
||||
description: 'Close the current browser window and release resources. Call this when browser interaction is complete to free memory.',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {},
|
||||
required: [],
|
||||
},
|
||||
category: MetonaToolCategory.NETWORK,
|
||||
riskLevel: MetonaRiskLevel.LOW,
|
||||
requiresPermission: false,
|
||||
timeoutMs: 10_000,
|
||||
};
|
||||
|
||||
async execute(_args: Record<string, unknown>, _context: ToolExecutionContext): Promise<unknown> {
|
||||
logTool('browser_close', 'Closing browser window');
|
||||
getManager().close();
|
||||
return { success: true, closed: true };
|
||||
}
|
||||
}
|
||||
|
||||
// ===== 导出所有 Browser 工具 =====
|
||||
|
||||
export const browserTools: IMetonaTool[] = [
|
||||
new BrowserOpenTool(),
|
||||
new BrowserScreenshotTool(),
|
||||
new BrowserEvaluateTool(),
|
||||
new BrowserExtractTool(),
|
||||
new BrowserClickTool(),
|
||||
new BrowserTypeTool(),
|
||||
new BrowserScrollTool(),
|
||||
new BrowserWaitTool(),
|
||||
new BrowserCloseTool(),
|
||||
];
|
||||
@@ -12,6 +12,7 @@ import { promisify } from 'util';
|
||||
import type { IMetonaTool, ToolExecutionContext } from '../../types/metona-tool';
|
||||
import type { MetonaToolDef } from '../../../harness/types';
|
||||
import { MetonaToolCategory, MetonaRiskLevel } from '../../../harness/types';
|
||||
import { commandTouchesProtectedFile } from './file-guard';
|
||||
|
||||
const execAsync = promisify(exec);
|
||||
|
||||
@@ -83,6 +84,11 @@ export class RunCommandTool implements IMetonaTool {
|
||||
private validateCommand(command: string): { allowed: boolean; reason?: string } {
|
||||
const cmd = command.trim().toLowerCase();
|
||||
|
||||
// 受保护文件检查：禁止通过命令行读写工作空间根目录的 MEMORY.md
|
||||
if (commandTouchesProtectedFile(command)) {
|
||||
return { allowed: false, reason: 'Access denied: MEMORY.md is managed by the memory system and cannot be accessed via command execution' };
|
||||
}
|
||||
|
||||
// 硬阻止列表（绝对禁止执行）
|
||||
const hardBlocks = [
|
||||
{ pattern: /\brm\b.*\//, reason: 'rm with absolute path is forbidden' },
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
/**
|
||||
* File Guard — 受保护文件守卫
|
||||
*
|
||||
* 确保工作空间根目录的 MEMORY.md 只能由系统内部（WorkspaceService）管理，
|
||||
* 任何工具（read_file / write_file / search_files / run_command 等）均禁止直接读写。
|
||||
*
|
||||
* 注意：仅保护工作空间根目录的 MEMORY.md，
|
||||
* 子目录或其他位置的同名文件不受限制。
|
||||
*/
|
||||
|
||||
import { resolve, sep } from 'path';
|
||||
|
||||
/**
|
||||
* 受保护文件名列表（工作空间根目录）
|
||||
*/
|
||||
const PROTECTED_FILES = ['MEMORY.md'];
|
||||
|
||||
/**
|
||||
* 检查目标路径是否为工作空间根目录的受保护文件
|
||||
*
|
||||
* @param filePath 用户传入的文件路径（绝对或相对）
|
||||
* @param workspacePath 当前工作空间根路径
|
||||
* @returns true 如果路径指向受保护文件
|
||||
*/
|
||||
export function isProtectedWorkspaceFile(
|
||||
filePath: string,
|
||||
workspacePath: string,
|
||||
): boolean {
|
||||
const resolved = resolve(workspacePath, filePath);
|
||||
const workspaceRoot = resolve(workspacePath);
|
||||
|
||||
for (const protectedName of PROTECTED_FILES) {
|
||||
const protectedPath = resolve(workspaceRoot, protectedName);
|
||||
if (resolved === protectedPath) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查路径是否在工作空间内（防止路径遍历攻击）
|
||||
*
|
||||
* 修复前缀碰撞漏洞：`/home/user/app-evil` 不应被误判为在 `/home/user/app` 内。
|
||||
*
|
||||
* @param filePath 用户传入的文件路径
|
||||
* @param workspacePath 当前工作空间根路径
|
||||
* @returns true 如果路径在工作空间内
|
||||
*/
|
||||
export function isPathWithinWorkspace(
|
||||
filePath: string,
|
||||
workspacePath: string,
|
||||
): boolean {
|
||||
const resolved = resolve(workspacePath, filePath);
|
||||
const workspaceRoot = resolve(workspacePath);
|
||||
return resolved === workspaceRoot || resolved.startsWith(workspaceRoot + sep);
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查命令字符串是否尝试访问工作空间根目录的受保护文件
|
||||
*
|
||||
* 用于 run_command 工具的命令校验。
|
||||
* 采用大小写不敏感匹配，覆盖 cat/type/Get-Content 等常见读取命令。
|
||||
*
|
||||
* @param command Shell 命令字符串
|
||||
* @returns true 如果命令包含受保护文件名
|
||||
*/
|
||||
export function commandTouchesProtectedFile(command: string): boolean {
|
||||
const lowerCmd = command.toLowerCase();
|
||||
for (const protectedName of PROTECTED_FILES) {
|
||||
if (lowerCmd.includes(protectedName.toLowerCase())) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
@@ -3,6 +3,11 @@
|
||||
*
|
||||
* read_file, write_file, list_directory, search_files
|
||||
*
|
||||
* 安全策略：
|
||||
* 1. 路径遍历防护 — 所有路径必须解析到工作空间内（修复前缀碰撞漏洞）
|
||||
* 2. 受保护文件拦截 — MEMORY.md 仅由系统内部 WorkspaceService 管理，
|
||||
* 禁止任何文件工具直接读写（仅限工作空间根目录，子目录不受限）
|
||||
*
|
||||
* @see docs/MetonaAI-Desktop 架构与交互设计.html — 9 个基础工具
|
||||
* @see standard/开发规范.md — 使用 fs/path 内置模块
|
||||
*/
|
||||
@@ -13,6 +18,21 @@ import { existsSync } from 'fs';
|
||||
import type { IMetonaTool, ToolExecutionContext } from '../../types/metona-tool';
|
||||
import type { MetonaToolDef } from '../../../harness/types';
|
||||
import { MetonaToolCategory, MetonaRiskLevel } from '../../../harness/types';
|
||||
import { isProtectedWorkspaceFile, isPathWithinWorkspace } from './file-guard';
|
||||
|
||||
/** 共享路径解析 + 安全校验 */
|
||||
function safeResolvePath(filePath: string, workspacePath: string): string {
|
||||
const resolved = resolve(workspacePath, filePath);
|
||||
// 安全检查：路径遍历防护（修复前缀碰撞漏洞）
|
||||
if (!isPathWithinWorkspace(filePath, workspacePath)) {
|
||||
throw new Error(`Path traversal detected: ${filePath}`);
|
||||
}
|
||||
// 受保护文件检查：MEMORY.md 仅由系统内部管理
|
||||
if (isProtectedWorkspaceFile(filePath, workspacePath)) {
|
||||
throw new Error('Access denied: MEMORY.md is managed by the memory system and cannot be accessed via file tools');
|
||||
}
|
||||
return resolved;
|
||||
}
|
||||
|
||||
// ===== 1. read_file =====
|
||||
|
||||
@@ -36,7 +56,7 @@ export class ReadFileTool implements IMetonaTool {
|
||||
};
|
||||
|
||||
async execute(args: Record<string, unknown>, context: ToolExecutionContext): Promise<unknown> {
|
||||
const filePath = this.resolvePath(args.file_path as string, context.workspacePath);
|
||||
const filePath = safeResolvePath(args.file_path as string, context.workspacePath);
|
||||
const offset = Math.max(1, (args.offset as number) ?? 1);
|
||||
const limit = Math.min(2000, Math.max(1, (args.limit as number) ?? 500));
|
||||
|
||||
@@ -52,15 +72,6 @@ export class ReadFileTool implements IMetonaTool {
|
||||
file_size: content.length,
|
||||
};
|
||||
}
|
||||
|
||||
private resolvePath(filePath: string, workspacePath: string): string {
|
||||
const resolved = resolve(workspacePath, filePath);
|
||||
// 安全检查：路径遍历防护
|
||||
if (!resolved.startsWith(resolve(workspacePath))) {
|
||||
throw new Error(`Path traversal detected: ${filePath}`);
|
||||
}
|
||||
return resolved;
|
||||
}
|
||||
}
|
||||
|
||||
// ===== 2. write_file =====
|
||||
@@ -85,7 +96,7 @@ export class WriteFileTool implements IMetonaTool {
|
||||
};
|
||||
|
||||
async execute(args: Record<string, unknown>, context: ToolExecutionContext): Promise<unknown> {
|
||||
const filePath = this.resolvePath(args.file_path as string, context.workspacePath);
|
||||
const filePath = safeResolvePath(args.file_path as string, context.workspacePath);
|
||||
const content = args.content as string;
|
||||
const mode = (args.mode as string) ?? 'overwrite';
|
||||
|
||||
@@ -97,14 +108,6 @@ export class WriteFileTool implements IMetonaTool {
|
||||
|
||||
return { bytes_written: Buffer.byteLength(content, 'utf-8'), path: filePath };
|
||||
}
|
||||
|
||||
private resolvePath(filePath: string, workspacePath: string): string {
|
||||
const resolved = resolve(workspacePath, filePath);
|
||||
if (!resolved.startsWith(resolve(workspacePath))) {
|
||||
throw new Error(`Path traversal detected: ${filePath}`);
|
||||
}
|
||||
return resolved;
|
||||
}
|
||||
}
|
||||
|
||||
// ===== 3. list_directory =====
|
||||
@@ -129,7 +132,7 @@ export class ListDirectoryTool implements IMetonaTool {
|
||||
|
||||
async execute(args: Record<string, unknown>, context: ToolExecutionContext): Promise<unknown> {
|
||||
const dirPath = args.dir_path
|
||||
? this.resolvePath(args.dir_path as string, context.workspacePath)
|
||||
? safeResolvePath(args.dir_path as string, context.workspacePath)
|
||||
: context.workspacePath;
|
||||
const depth = Math.min(5, Math.max(1, (args.depth as number) ?? 1));
|
||||
const glob = args.glob as string | undefined;
|
||||
@@ -176,14 +179,6 @@ export class ListDirectoryTool implements IMetonaTool {
|
||||
const pattern = glob.replace(/\*/g, '.*').replace(/\?/g, '.');
|
||||
return new RegExp(`^${pattern}$`, 'i').test(name);
|
||||
}
|
||||
|
||||
private resolvePath(filePath: string, workspacePath: string): string {
|
||||
const resolved = resolve(workspacePath, filePath);
|
||||
if (!resolved.startsWith(resolve(workspacePath))) {
|
||||
throw new Error(`Path traversal detected: ${filePath}`);
|
||||
}
|
||||
return resolved;
|
||||
}
|
||||
}
|
||||
|
||||
// ===== 4. search_files =====
|
||||
@@ -213,7 +208,7 @@ export class SearchFilesTool implements IMetonaTool {
|
||||
const pattern = args.pattern as string;
|
||||
const target = (args.target as string) ?? 'content';
|
||||
const searchPath = args.path
|
||||
? this.resolvePath(args.path as string, context.workspacePath)
|
||||
? this.resolveSearchPath(args.path as string, context.workspacePath)
|
||||
: context.workspacePath;
|
||||
const fileGlob = args.file_glob as string | undefined;
|
||||
const limit = Math.min(200, Math.max(1, (args.limit as number) ?? 50));
|
||||
@@ -221,7 +216,7 @@ export class SearchFilesTool implements IMetonaTool {
|
||||
if (target === 'files') {
|
||||
return this.searchByFilename(searchPath, pattern, fileGlob, limit);
|
||||
}
|
||||
return this.searchByContent(searchPath, pattern, fileGlob, limit);
|
||||
return this.searchByContent(searchPath, pattern, fileGlob, limit, context.workspacePath);
|
||||
}
|
||||
|
||||
private async searchByFilename(dirPath: string, pattern: string, fileGlob: string | undefined, limit: number): Promise<unknown> {
|
||||
@@ -239,12 +234,16 @@ export class SearchFilesTool implements IMetonaTool {
|
||||
return { results, count: results.length };
|
||||
}
|
||||
|
||||
private async searchByContent(dirPath: string, pattern: string, fileGlob: string | undefined, limit: number): Promise<unknown> {
|
||||
private async searchByContent(dirPath: string, pattern: string, fileGlob: string | undefined, limit: number, workspacePath: string): Promise<unknown> {
|
||||
const results: Array<{ path: string; line: number; match: string }> = [];
|
||||
const regex = new RegExp(pattern, 'gi');
|
||||
|
||||
await this.walkDir(dirPath, async (filePath) => {
|
||||
if (results.length >= limit) return;
|
||||
// 跳过工作空间根目录的 MEMORY.md（受保护文件，使用完整路径精确匹配）
|
||||
if (isProtectedWorkspaceFile(filePath, workspacePath)) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const content = await readFile(filePath, 'utf-8');
|
||||
const lines = content.split('\n');
|
||||
@@ -295,9 +294,10 @@ export class SearchFilesTool implements IMetonaTool {
|
||||
return new RegExp(`^${pattern}$`, 'i').test(name);
|
||||
}
|
||||
|
||||
private resolvePath(filePath: string, workspacePath: string): string {
|
||||
/** search_files 的路径解析：仅需遍历防护，不需要 MEMORY.md 拦截（搜索时单独跳过） */
|
||||
private resolveSearchPath(filePath: string, workspacePath: string): string {
|
||||
const resolved = resolve(workspacePath, filePath);
|
||||
if (!resolved.startsWith(resolve(workspacePath))) {
|
||||
if (!isPathWithinWorkspace(filePath, workspacePath)) {
|
||||
throw new Error(`Path traversal detected: ${filePath}`);
|
||||
}
|
||||
return resolved;
|
||||
|
||||
@@ -5,6 +5,19 @@
|
||||
*/
|
||||
|
||||
export { ReadFileTool, WriteFileTool, ListDirectoryTool, SearchFilesTool } from './filesystem';
|
||||
export { WebSearchTool, WebExtractTool } from './network';
|
||||
export { WebSearchTool, WebFetchTool } from './network';
|
||||
export { MemoryStoreTool, MemorySearchTool } from './memory';
|
||||
export { RunCommandTool } from './command';
|
||||
export {
|
||||
BrowserOpenTool,
|
||||
BrowserScreenshotTool,
|
||||
BrowserEvaluateTool,
|
||||
BrowserExtractTool,
|
||||
BrowserClickTool,
|
||||
BrowserTypeTool,
|
||||
BrowserScrollTool,
|
||||
BrowserWaitTool,
|
||||
BrowserCloseTool,
|
||||
browserTools,
|
||||
cleanupBrowser,
|
||||
} from './browser';
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
/**
|
||||
* 网络工具共享模块
|
||||
*
|
||||
* 提供 UA 轮换池、反爬请求头、HTML 转文本、拦截检测、超时 fetch、LRU 缓存等通用能力。
|
||||
* web_search 和 web_fetch 共享此模块。
|
||||
*
|
||||
* @see docs/Agent网络工具通用设计-v2.md — 第 3 章 web_fetch 抓取设计
|
||||
*/
|
||||
|
||||
import { LRUCache } from 'lru-cache';
|
||||
import log from 'electron-log';
|
||||
|
||||
// ===== LRU 缓存 =====
|
||||
|
||||
/** 搜索结果缓存（200 条，5 分钟 TTL） */
|
||||
export const searchCache = new LRUCache<string, Record<string, unknown>>({
|
||||
max: 200,
|
||||
ttl: 300_000,
|
||||
});
|
||||
|
||||
/** 浏览器回退缓存（100 条，10 分钟 TTL） */
|
||||
export const fetchCache = new LRUCache<string, string>({
|
||||
max: 100,
|
||||
ttl: 600_000,
|
||||
});
|
||||
|
||||
// ===== UA 轮换池 =====
|
||||
|
||||
export const UA_POOL = [
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Edg/131.0.0.0 Safari/537.36',
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.1 Safari/605.1.15',
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0',
|
||||
];
|
||||
|
||||
export const MOBILE_UA = 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1';
|
||||
|
||||
export const ACCEPT_LANGUAGE_POOL = [
|
||||
'zh-CN,zh;q=0.9,en;q=0.8',
|
||||
'zh-CN,zh;q=0.9',
|
||||
'en-US,en;q=0.9,zh-CN;q=0.8',
|
||||
];
|
||||
|
||||
// ===== 反爬请求头构建 =====
|
||||
|
||||
export function buildAntiCrawlHeaders(
|
||||
url: string,
|
||||
attempt: number,
|
||||
mobileUA = false,
|
||||
): Record<string, string> {
|
||||
const uaIdx = attempt % UA_POOL.length;
|
||||
const langIdx = attempt % ACCEPT_LANGUAGE_POOL.length;
|
||||
const userAgent = mobileUA ? MOBILE_UA : UA_POOL[uaIdx];
|
||||
|
||||
let origin = '';
|
||||
try { origin = new URL(url).origin; } catch { /* ignore */ }
|
||||
|
||||
return {
|
||||
'User-Agent': userAgent,
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
|
||||
'Accept-Language': ACCEPT_LANGUAGE_POOL[langIdx],
|
||||
'Accept-Encoding': 'gzip, deflate, br',
|
||||
'Cache-Control': 'no-cache',
|
||||
'DNT': '1',
|
||||
'Referer': origin || '',
|
||||
'Sec-Fetch-Dest': 'document',
|
||||
'Sec-Fetch-Mode': 'navigate',
|
||||
'Sec-Fetch-Site': 'none',
|
||||
'Sec-Fetch-User': '?1',
|
||||
'Pragma': 'no-cache',
|
||||
};
|
||||
}
|
||||
|
||||
// ===== 超时 fetch =====
|
||||
|
||||
export async function fetchWithTimeout(
|
||||
url: string,
|
||||
options: RequestInit = {},
|
||||
timeoutMs = 20_000,
|
||||
): Promise<Response> {
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||
try {
|
||||
return await fetch(url, { ...options, signal: controller.signal });
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
// ===== URL 标准化（去重用） =====
|
||||
|
||||
export function normalizeUrl(url: string): string {
|
||||
try {
|
||||
const u = new URL(url);
|
||||
// 去除追踪参数
|
||||
const trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'gclid', 'fbclid'];
|
||||
for (const p of trackingParams) u.searchParams.delete(p);
|
||||
// 去除尾部斜杠（根路径除外）
|
||||
let path = u.pathname;
|
||||
if (path.length > 1 && path.endsWith('/')) path = path.slice(0, -1);
|
||||
// 强制小写 host
|
||||
return `${u.protocol}//${u.host.toLowerCase()}${path}${u.search}${u.hash}`;
|
||||
} catch {
|
||||
return url;
|
||||
}
|
||||
}
|
||||
|
||||
// ===== 拦截页面检测 =====
|
||||
|
||||
const INTERCEPTION_PATTERNS = [
|
||||
/just a moment/i,
|
||||
/attention required/i,
|
||||
/challenge-platform/i,
|
||||
/\.cf-challenge-/i,
|
||||
/access denied/i,
|
||||
/403 forbidden/i,
|
||||
/请启用\s*javascript/i,
|
||||
/please enable javascript/i,
|
||||
/checking your browser/i,
|
||||
/ddos protection/i,
|
||||
];
|
||||
|
||||
export function isInterceptedPage(html: string): boolean {
|
||||
if (html.length < 80) return true;
|
||||
const lower = html.toLowerCase();
|
||||
return INTERCEPTION_PATTERNS.some((p) => p.test(lower));
|
||||
}
|
||||
|
||||
// ===== HTML → 纯文本转换 =====
|
||||
|
||||
const HTML_ENTITY_MAP: Record<string, string> = {
|
||||
' ': ' ', '<': '<', '>': '>', '&': '&', '"': '"',
|
||||
''': "'", '…': '…', '—': '—', '–': '–',
|
||||
'«': '«', '»': '»', '×': '×', '÷': '÷',
|
||||
'©': '©', '®': '®', '™': '™', '€': '€',
|
||||
'£': '£', '¥': '¥', '¢': '¢', '°': '°',
|
||||
};
|
||||
|
||||
export function htmlToText(html: string): string {
|
||||
return html
|
||||
// 移除噪声标签及内容
|
||||
.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
|
||||
.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
|
||||
.replace(/<noscript[^>]*>[\s\S]*?<\/noscript>/gi, '')
|
||||
.replace(/<nav[^>]*>[\s\S]*?<\/nav>/gi, '')
|
||||
.replace(/<header[^>]*>[\s\S]*?<\/header>/gi, '')
|
||||
.replace(/<footer[^>]*>[\s\S]*?<\/footer>/gi, '')
|
||||
.replace(/<aside[^>]*>[\s\S]*?<\/aside>/gi, '')
|
||||
.replace(/<iframe[^>]*>[\s\S]*?<\/iframe>/gi, '')
|
||||
.replace(/<svg[^>]*>[\s\S]*?<\/svg>/gi, '')
|
||||
// 移除 HTML 注释
|
||||
.replace(/<!--[\s\S]*?-->/g, '')
|
||||
// 块级标签转换行
|
||||
.replace(/<\/?(p|div|h[1-6]|li|tr|blockquote|section|article|pre|br|hr)[^>]*>/gi, '\n')
|
||||
// 表格单元格转制表符
|
||||
.replace(/<\/?(td|th)[^>]*>/gi, '\t')
|
||||
// 移除剩余标签
|
||||
.replace(/<[^>]+>/g, '')
|
||||
// 解码 HTML 实体
|
||||
.replace(/&#(\d+);/g, (_, n) => String.fromCharCode(Number(n)))
|
||||
.replace(/&#x([0-9a-f]+);/gi, (_, h) => String.fromCharCode(parseInt(h, 16)))
|
||||
.replace(/&[a-z]+;/gi, (m) => HTML_ENTITY_MAP[m.toLowerCase()] ?? m)
|
||||
// 清理空白
|
||||
.replace(/\n{3,}/g, '\n\n')
|
||||
.replace(/[ \t]+/g, ' ')
|
||||
.replace(/^[ \t]+/gm, '')
|
||||
.trim();
|
||||
}
|
||||
|
||||
// ===== 流式读取（大文件保护，10MB 上限） =====
|
||||
|
||||
export async function readBodyWithLimit(response: Response, maxBytes = 10 * 1024 * 1024): Promise<string> {
|
||||
const contentLength = response.headers.get('content-length');
|
||||
if (contentLength && parseInt(contentLength) > maxBytes) {
|
||||
throw new Error(`Response too large: ${contentLength} bytes (limit: ${maxBytes})`);
|
||||
}
|
||||
|
||||
const reader = response.body?.getReader();
|
||||
if (!reader) return '';
|
||||
|
||||
const chunks: Uint8Array[] = [];
|
||||
let totalBytes = 0;
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
if (value) {
|
||||
totalBytes += value.byteLength;
|
||||
if (totalBytes > maxBytes) {
|
||||
throw new Error(`Response exceeded ${maxBytes} bytes limit`);
|
||||
}
|
||||
chunks.push(value);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
|
||||
const decoder = new TextDecoder('utf-8', { fatal: false });
|
||||
return decoder.decode(Buffer.concat(chunks));
|
||||
}
|
||||
|
||||
// ===== SearXNG 认证头构建 =====
|
||||
|
||||
export function buildSearXNGAuthHeaders(authKey: string, authType: string): Record<string, string> {
|
||||
if (!authKey) return {};
|
||||
if (authType === 'bearer') {
|
||||
return { Authorization: `Bearer ${authKey}` };
|
||||
}
|
||||
if (authType === 'basic') {
|
||||
return { Authorization: `Basic ${Buffer.from(authKey).toString('base64')}` };
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
// ===== 日志辅助 =====
|
||||
|
||||
export function logTool(toolName: string, message: string): void {
|
||||
log.info(`[Tool:${toolName}] ${message}`);
|
||||
}
|
||||
@@ -1,151 +1,11 @@
|
||||
/**
|
||||
* 网络工具（2 个）
|
||||
* 网络工具导出
|
||||
*
|
||||
* web_search, web_extract
|
||||
* web_search — 双模式搜索（SearXNG / 内置四引擎）
|
||||
* web_fetch — 三阶段回退抓取（HTTP / SPA 升级 / 浏览器渲染）
|
||||
*
|
||||
* @see docs/MetonaAI-Desktop 架构与交互设计.html — 9 个基础工具
|
||||
* @see standard/开发规范.md — 使用原生 fetch（允许在工具层使用）
|
||||
* @see docs/Agent网络工具通用设计-v2.md
|
||||
*/
|
||||
|
||||
import type { IMetonaTool, ToolExecutionContext } from '../../types/metona-tool';
|
||||
import type { MetonaToolDef } from '../../../harness/types';
|
||||
import { MetonaToolCategory, MetonaRiskLevel } from '../../../harness/types';
|
||||
|
||||
// ===== 5. web_search =====
|
||||
|
||||
export class WebSearchTool implements IMetonaTool {
|
||||
readonly definition: MetonaToolDef = {
|
||||
name: 'web_search',
|
||||
description: 'Search the web for information. Returns titles, snippets, and URLs. Supports search operators (site:, filetype:, etc.).',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
query: { type: 'string', description: 'Search query. Supports operators like site:domain filetype:pdf' },
|
||||
limit: { type: 'number', description: 'Number of results (default 5, max 100)' },
|
||||
},
|
||||
required: ['query'],
|
||||
},
|
||||
category: MetonaToolCategory.NETWORK,
|
||||
riskLevel: MetonaRiskLevel.LOW,
|
||||
requiresPermission: false,
|
||||
timeoutMs: 30_000,
|
||||
};
|
||||
|
||||
async execute(args: Record<string, unknown>, _context: ToolExecutionContext): Promise<unknown> {
|
||||
const query = args.query as string;
|
||||
const limit = Math.min(100, Math.max(1, (args.limit as number) ?? 5));
|
||||
|
||||
// 使用 DuckDuckGo Instant Answer API（免费，无需 API Key）
|
||||
try {
|
||||
const url = `https://api.duckduckgo.com/?q=${encodeURIComponent(query)}&format=json&no_html=1&skip_disambig=1`;
|
||||
const response = await fetch(url, {
|
||||
signal: AbortSignal.timeout(15_000),
|
||||
headers: { 'User-Agent': 'MetonaAI/1.0' },
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Search API error: ${response.status}`);
|
||||
}
|
||||
|
||||
const data = await response.json() as Record<string, unknown>;
|
||||
const results: Array<{ title: string; snippet: string; url: string }> = [];
|
||||
|
||||
// 提取 AbstractText
|
||||
if (data.AbstractText) {
|
||||
results.push({
|
||||
title: (data.Heading as string) ?? query,
|
||||
snippet: (data.AbstractText as string).slice(0, 300),
|
||||
url: (data.AbstractURL as string) ?? '',
|
||||
});
|
||||
}
|
||||
|
||||
// 提取 RelatedTopics
|
||||
const related = data.RelatedTopics as Array<Record<string, unknown>> | undefined;
|
||||
if (related) {
|
||||
for (const topic of related.slice(0, limit - results.length)) {
|
||||
if (topic.Text && topic.FirstURL) {
|
||||
results.push({
|
||||
title: ((topic.Text as string) ?? '').slice(0, 100),
|
||||
snippet: (topic.Text as string) ?? '',
|
||||
url: (topic.FirstURL as string) ?? '',
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { query, results, count: results.length };
|
||||
} catch (error) {
|
||||
return { query, results: [], count: 0, error: (error as Error).message };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ===== 6. web_extract =====
|
||||
|
||||
export class WebExtractTool implements IMetonaTool {
|
||||
readonly definition: MetonaToolDef = {
|
||||
name: 'web_extract',
|
||||
description: 'Fetch web page content and convert to plain text. Supports HTML pages. Content over 5000 chars is auto-summarized.',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
urls: { type: 'array', description: 'URL list to fetch (max 5)', items: { type: 'string', description: 'URL to fetch' } },
|
||||
},
|
||||
required: ['urls'],
|
||||
},
|
||||
category: MetonaToolCategory.NETWORK,
|
||||
riskLevel: MetonaRiskLevel.LOW,
|
||||
requiresPermission: false,
|
||||
timeoutMs: 30_000,
|
||||
};
|
||||
|
||||
async execute(args: Record<string, unknown>, _context: ToolExecutionContext): Promise<unknown> {
|
||||
const urls = (args.urls as string[]).slice(0, 5);
|
||||
const results: Array<{ url: string; content: string; success: boolean; error?: string }> = [];
|
||||
|
||||
for (const url of urls) {
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
signal: AbortSignal.timeout(15_000),
|
||||
headers: {
|
||||
'User-Agent': 'MetonaAI/1.0',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
results.push({ url, content: '', success: false, error: `HTTP ${response.status}` });
|
||||
continue;
|
||||
}
|
||||
|
||||
const html = await response.text();
|
||||
const text = this.htmlToText(html);
|
||||
const truncated = text.length > 5000 ? text.slice(0, 5000) + '\n\n[Content truncated at 5000 characters]' : text;
|
||||
|
||||
results.push({ url, content: truncated, success: true });
|
||||
} catch (error) {
|
||||
results.push({ url, content: '', success: false, error: (error as Error).message });
|
||||
}
|
||||
}
|
||||
|
||||
return { results, count: results.length };
|
||||
}
|
||||
|
||||
/**
|
||||
* 简单 HTML → 文本转换
|
||||
* @see standard/开发规范.md — < 20 行纯函数，允许自写
|
||||
*/
|
||||
private htmlToText(html: string): string {
|
||||
return html
|
||||
.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
|
||||
.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
|
||||
.replace(/<[^>]+>/g, ' ')
|
||||
.replace(/ /g, ' ')
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
}
|
||||
export { WebSearchTool } from './web-search';
|
||||
export { WebFetchTool } from './web-fetch';
|
||||
|
||||
@@ -0,0 +1,240 @@
|
||||
/**
|
||||
* web_fetch — 网页抓取工具
|
||||
*
|
||||
* 三阶段回退策略：
|
||||
* Phase 1: HTTP 抓取（UA 轮换 + 反爬请求头 + 指数退避重试 + 拦截检测）
|
||||
* Phase 2: 内容过短自动升级（< 200 字符 → 浏览器渲染）
|
||||
* Phase 3: 浏览器回退（隐藏 BrowserWindow + JS 渲染 + 内容提取）
|
||||
*
|
||||
* @see docs/Agent网络工具通用设计-v2.md — 第 3 章 web_fetch 抓取设计
|
||||
*/
|
||||
|
||||
import { BrowserWindow } from 'electron';
|
||||
import type { IMetonaTool, ToolExecutionContext } from '../../types/metona-tool';
|
||||
import type { MetonaToolDef } from '../../../harness/types';
|
||||
import { MetonaToolCategory, MetonaRiskLevel } from '../../../harness/types';
|
||||
import {
|
||||
fetchCache,
|
||||
buildAntiCrawlHeaders,
|
||||
fetchWithTimeout,
|
||||
htmlToText,
|
||||
isInterceptedPage,
|
||||
readBodyWithLimit,
|
||||
logTool,
|
||||
} from './network-utils';
|
||||
|
||||
// ===== 跳过重试的状态码 =====
|
||||
|
||||
const SKIP_RETRY_STATUS = new Set([403, 429, 502, 503]);
|
||||
|
||||
// ===== WebFetchTool =====
|
||||
|
||||
export class WebFetchTool implements IMetonaTool {
|
||||
readonly definition: MetonaToolDef = {
|
||||
name: 'web_fetch',
|
||||
description: 'Fetch a web page and convert to plain text. Uses a three-phase fallback strategy: HTTP fetch with anti-crawl headers → SPA auto-upgrade → browser rendering. Handles Cloudflare interception, JavaScript-rendered pages, and large files (10MB limit).',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
url: { type: 'string', description: 'Target URL (http/https only)' },
|
||||
mobile_ua: { type: 'boolean', description: 'Use mobile User-Agent (default false)' },
|
||||
retry: { type: 'boolean', description: 'Enable retry with exponential backoff (default true)' },
|
||||
},
|
||||
required: ['url'],
|
||||
},
|
||||
category: MetonaToolCategory.NETWORK,
|
||||
riskLevel: MetonaRiskLevel.LOW,
|
||||
requiresPermission: false,
|
||||
timeoutMs: 120_000,
|
||||
};
|
||||
|
||||
async execute(args: Record<string, unknown>, _context: ToolExecutionContext): Promise<unknown> {
|
||||
const url = args.url as string;
|
||||
const mobileUA = (args.mobile_ua as boolean) ?? false;
|
||||
const enableRetry = (args.retry as boolean) ?? true;
|
||||
|
||||
if (!url || !/^https?:\/\//i.test(url)) {
|
||||
return { url, content: '', success: false, error: 'URL must start with http:// or https://' };
|
||||
}
|
||||
|
||||
logTool('web_fetch', `Fetching: ${url}`);
|
||||
|
||||
// ===== Phase 1: HTTP 抓取 =====
|
||||
const phase1Result = await this.httpFetch(url, mobileUA, enableRetry);
|
||||
|
||||
if (phase1Result.success && !phase1Result.intercepted) {
|
||||
// 内容过短检测 → Phase 2 升级
|
||||
if (phase1Result.text.length < 200) {
|
||||
logTool('web_fetch', `Phase 2: Content too short (${phase1Result.text.length} chars), upgrading to browser`);
|
||||
const browserResult = await this.browserFetch(url);
|
||||
if (browserResult) {
|
||||
return this.buildSuccess(url, browserResult, 'browser');
|
||||
}
|
||||
}
|
||||
return this.buildSuccess(url, phase1Result.text, 'http');
|
||||
}
|
||||
|
||||
// ===== Phase 3: 浏览器回退 =====
|
||||
logTool('web_fetch', `Phase 3: Falling back to browser (${phase1Result.reason})`);
|
||||
const browserResult = await this.browserFetch(url);
|
||||
if (browserResult) {
|
||||
return this.buildSuccess(url, browserResult, 'browser');
|
||||
}
|
||||
|
||||
// 全部失败
|
||||
return {
|
||||
url,
|
||||
content: '',
|
||||
success: false,
|
||||
error: `All phases failed. HTTP: ${phase1Result.reason}. Browser fallback also failed.`,
|
||||
};
|
||||
}
|
||||
|
||||
// ===== Phase 1: HTTP 抓取 =====
|
||||
|
||||
private async httpFetch(
|
||||
url: string,
|
||||
mobileUA: boolean,
|
||||
enableRetry: boolean,
|
||||
): Promise<{ success: boolean; text: string; intercepted: boolean; reason: string }> {
|
||||
const maxRetries = enableRetry ? 3 : 1;
|
||||
const backoffBase = 2_000;
|
||||
|
||||
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
||||
try {
|
||||
const headers = buildAntiCrawlHeaders(url, attempt, mobileUA);
|
||||
const response = await fetchWithTimeout(url, { headers, redirect: 'follow' }, 20_000);
|
||||
|
||||
// 跳过重试的状态码 → 直接进入浏览器回退
|
||||
if (SKIP_RETRY_STATUS.has(response.status)) {
|
||||
return { success: false, text: '', intercepted: true, reason: `HTTP ${response.status}` };
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
// 5xx 可重试
|
||||
if (response.status >= 500 && attempt < maxRetries - 1) {
|
||||
await this.sleep(backoffBase * Math.pow(2, attempt) + Math.random() * backoffBase * 0.6);
|
||||
continue;
|
||||
}
|
||||
return { success: false, text: '', intercepted: false, reason: `HTTP ${response.status} ${response.statusText}` };
|
||||
}
|
||||
|
||||
// 读取正文（10MB 限制）
|
||||
const html = await readBodyWithLimit(response, 10 * 1024 * 1024);
|
||||
|
||||
// 拦截检测
|
||||
if (isInterceptedPage(html)) {
|
||||
return { success: false, text: '', intercepted: true, reason: 'Intercepted page detected' };
|
||||
}
|
||||
|
||||
// HTML → 纯文本
|
||||
const text = htmlToText(html);
|
||||
return { success: true, text, intercepted: false, reason: '' };
|
||||
} catch (err) {
|
||||
const errorMsg = (err as Error).message;
|
||||
if (attempt < maxRetries - 1) {
|
||||
logTool('web_fetch', `Attempt ${attempt + 1} failed: ${errorMsg}, retrying...`);
|
||||
await this.sleep(backoffBase * Math.pow(2, attempt) + Math.random() * backoffBase * 0.6);
|
||||
continue;
|
||||
}
|
||||
return { success: false, text: '', intercepted: false, reason: errorMsg };
|
||||
}
|
||||
}
|
||||
|
||||
return { success: false, text: '', intercepted: false, reason: 'All retries exhausted' };
|
||||
}
|
||||
|
||||
// ===== Phase 2/3: 浏览器回退 =====
|
||||
|
||||
private async browserFetch(url: string): Promise<string | null> {
|
||||
// 查缓存
|
||||
const cached = fetchCache.get(url);
|
||||
if (cached) {
|
||||
logTool('web_fetch', 'Browser cache hit');
|
||||
return cached;
|
||||
}
|
||||
|
||||
let win: BrowserWindow | null = null;
|
||||
let cleanup: (() => void) | null = null;
|
||||
|
||||
try {
|
||||
win = new BrowserWindow({
|
||||
width: 1280,
|
||||
height: 800,
|
||||
show: false,
|
||||
webPreferences: {
|
||||
nodeIntegration: false,
|
||||
contextIsolation: true,
|
||||
sandbox: true,
|
||||
webSecurity: false, // 允许跨域（截图需要）
|
||||
},
|
||||
});
|
||||
|
||||
cleanup = () => {
|
||||
try {
|
||||
if (win && !win.isDestroyed()) win.close();
|
||||
} catch {
|
||||
// 忽略关闭错误
|
||||
}
|
||||
};
|
||||
|
||||
await this.loadURLWithTimeout(win, url, 30_000);
|
||||
|
||||
// 等待 JS 渲染
|
||||
await this.sleep(2_500);
|
||||
|
||||
// 提取页面正文
|
||||
const text = await win.webContents.executeJavaScript(`
|
||||
(function() {
|
||||
var clone = document.body.cloneNode(true);
|
||||
var noise = clone.querySelectorAll('script, style, noscript, nav, header, footer, aside, iframe, svg');
|
||||
noise.forEach(function(el) { el.remove(); });
|
||||
return clone.innerText || '';
|
||||
})();
|
||||
`, true);
|
||||
|
||||
if (text && text.trim().length >= 80) {
|
||||
// 写缓存
|
||||
fetchCache.set(url, text);
|
||||
logTool('web_fetch', `Browser fetch success: ${text.length} chars`);
|
||||
return text;
|
||||
}
|
||||
|
||||
return null;
|
||||
} catch (err) {
|
||||
logTool('web_fetch', `Browser fetch failed: ${(err as Error).message}`);
|
||||
return null;
|
||||
} finally {
|
||||
if (cleanup) cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
// ===== 辅助方法 =====
|
||||
|
||||
private buildSuccess(url: string, text: string, method: string): unknown {
|
||||
return {
|
||||
url,
|
||||
content: text,
|
||||
success: true,
|
||||
method,
|
||||
length: text.length,
|
||||
};
|
||||
}
|
||||
|
||||
private sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/** 带超时的 loadURL（Electron 原生不支持 timeout 选项） */
|
||||
private async loadURLWithTimeout(win: BrowserWindow, url: string, timeoutMs: number): Promise<void> {
|
||||
let timer: NodeJS.Timeout | null = null;
|
||||
const timeoutPromise = new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(() => reject(new Error(`Page load timeout after ${timeoutMs}ms: ${url}`)), timeoutMs);
|
||||
});
|
||||
try {
|
||||
await Promise.race([win.loadURL(url), timeoutPromise]);
|
||||
} finally {
|
||||
if (timer) clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,600 @@
|
||||
/**
|
||||
* web_search — 网络搜索工具
|
||||
*
|
||||
* 双模式搜索：SearXNG 元搜索 / 内置四引擎并行搜索
|
||||
* 内置引擎：Bing + 百度 + 搜狗 + 360 搜索（Promise.allSettled 容错并发）
|
||||
* 智能排序：引擎权重(50%) + 可达性(30%) + 摘要质量(20%)
|
||||
* 自动抓取：对前 N 条结果调用 web_fetch 获取完整正文
|
||||
*
|
||||
* @see docs/Agent网络工具通用设计-v2.md — 第 2 章 web_search 搜索设计
|
||||
*/
|
||||
|
||||
import type { IMetonaTool, ToolExecutionContext } from '../../types/metona-tool';
|
||||
import type { MetonaToolDef } from '../../../harness/types';
|
||||
import { MetonaToolCategory, MetonaRiskLevel } from '../../../harness/types';
|
||||
import type { ConfigService } from '../../../services/config.service';
|
||||
import {
|
||||
searchCache,
|
||||
normalizeUrl,
|
||||
fetchWithTimeout,
|
||||
buildSearXNGAuthHeaders,
|
||||
logTool,
|
||||
} from './network-utils';
|
||||
|
||||
// ===== 类型定义 =====
|
||||
|
||||
interface SearchResult {
|
||||
title: string;
|
||||
url: string;
|
||||
snippet: string;
|
||||
engine: string;
|
||||
weight: number;
|
||||
reachable?: boolean;
|
||||
_score?: number;
|
||||
_enhanced?: boolean;
|
||||
}
|
||||
|
||||
interface SearXNGConfig {
|
||||
enabled: boolean;
|
||||
url: string;
|
||||
engines: string;
|
||||
language: string;
|
||||
safesearch: number;
|
||||
time_range: string;
|
||||
max_results: number;
|
||||
auth_key: string;
|
||||
auth_type: string;
|
||||
format: string;
|
||||
fetch_count: number;
|
||||
fetch_mode: string;
|
||||
}
|
||||
|
||||
// ===== 引擎定义 =====
|
||||
|
||||
interface EngineDef {
|
||||
name: string;
|
||||
weight: number;
|
||||
searchUrl: (query: string, timeRange?: string) => string;
|
||||
parse: (html: string) => SearchResult[];
|
||||
}
|
||||
|
||||
const ENGINES: EngineDef[] = [
|
||||
{
|
||||
name: 'bing',
|
||||
weight: 90,
|
||||
searchUrl: (q, tr) => {
|
||||
const freshness = tr ? `&filters=ex1:"ez${tr === 'day' ? '1' : tr === 'week' ? '2' : tr === 'month' ? '3' : '4'}"` : '';
|
||||
return `https://www.bing.com/search?q=${encodeURIComponent(q)}${freshness}&count=20`;
|
||||
},
|
||||
parse: parseBing,
|
||||
},
|
||||
{
|
||||
name: '百度',
|
||||
weight: 80,
|
||||
searchUrl: (q) => `https://www.baidu.com/s?wd=${encodeURIComponent(q)}&rn=20`,
|
||||
parse: parseBaidu,
|
||||
},
|
||||
{
|
||||
name: '搜狗',
|
||||
weight: 75,
|
||||
searchUrl: (q) => `https://www.sogou.com/web?query=${encodeURIComponent(q)}&num=20`,
|
||||
parse: parseSogou,
|
||||
},
|
||||
{
|
||||
name: '360搜索',
|
||||
weight: 75,
|
||||
searchUrl: (q) => `https://www.so.com/s?q=${encodeURIComponent(q)}&pn=20`,
|
||||
parse: parse360,
|
||||
},
|
||||
];
|
||||
|
||||
// ===== HTML 解析器（正则实现，后续可迁移至 cheerio） =====
|
||||
|
||||
function parseBing(html: string): SearchResult[] {
|
||||
const results: SearchResult[] = [];
|
||||
const blocks = html.split(/<li[^>]*class="b_algo"/i).slice(1);
|
||||
for (const block of blocks) {
|
||||
const titleMatch = block.match(/<a[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
|
||||
if (!titleMatch) continue;
|
||||
const url = titleMatch[1];
|
||||
const title = titleMatch[2].replace(/<[^>]+>/g, '').trim();
|
||||
const snippetMatch = block.match(/<p[^>]*>([\s\S]*?)<\/p>/i) || block.match(/class="b_caption"[^>]*>([\s\S]*?)<\/div>/i);
|
||||
const snippet = snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '').trim() : '';
|
||||
if (title && url && !url.includes('bing.com')) {
|
||||
results.push({ title, url, snippet, engine: 'bing', weight: 90 });
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
function parseBaidu(html: string): SearchResult[] {
|
||||
const results: SearchResult[] = [];
|
||||
const blocks = html.split(/<div[^>]*class="result[^"]*"/i).slice(1);
|
||||
for (const block of blocks) {
|
||||
const titleMatch = block.match(/<a[^>]*data-url="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i)
|
||||
|| block.match(/<a[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
|
||||
if (!titleMatch) continue;
|
||||
const url = titleMatch[1].startsWith('http') ? titleMatch[1] : `https://${titleMatch[1]}`;
|
||||
const title = titleMatch[2].replace(/<[^>]+>/g, '').trim();
|
||||
const snippetMatch = block.match(/class="c-abstract[^"]*"[^>]*>([\s\S]*?)<\/span>/i)
|
||||
|| block.match(/class="content-right[^"]*"[^>]*>([\s\S]*?)<\/div>/i);
|
||||
const snippet = snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '').trim() : '';
|
||||
if (title && url && !url.includes('baidu.com/link')) {
|
||||
results.push({ title, url, snippet, engine: '百度', weight: 80 });
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
function parseSogou(html: string): SearchResult[] {
|
||||
const results: SearchResult[] = [];
|
||||
const blocks = html.split(/<div[^>]*class="vrwrap"/i).slice(1)
|
||||
.concat(html.split(/<div[^>]*class="rb"/i).slice(1));
|
||||
for (const block of blocks) {
|
||||
const titleMatch = block.match(/<a[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
|
||||
if (!titleMatch) continue;
|
||||
const url = titleMatch[1].startsWith('http') ? titleMatch[1] : `https://www.sogou.com${titleMatch[1]}`;
|
||||
const title = titleMatch[2].replace(/<[^>]+>/g, '').trim();
|
||||
const snippetMatch = block.match(/class="star-wiki[^"]*"[^>]*>([\s\S]*?)<\/div>/i)
|
||||
|| block.match(/class="space-txt[^"]*"[^>]*>([\s\S]*?)<\/p>/i)
|
||||
|| block.match(/class="str_info[^"]*"[^>]*>([\s\S]*?)<\/p>/i);
|
||||
const snippet = snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '').trim() : '';
|
||||
if (title && url && !url.includes('sogou.com')) {
|
||||
results.push({ title, url, snippet, engine: '搜狗', weight: 75 });
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
function parse360(html: string): SearchResult[] {
|
||||
const results: SearchResult[] = [];
|
||||
const blocks = html.split(/<li[^>]*class="res-list"/i).slice(1)
|
||||
.concat(html.split(/<div[^>]*class="result"/i).slice(1));
|
||||
for (const block of blocks) {
|
||||
const titleMatch = block.match(/<a[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i);
|
||||
if (!titleMatch) continue;
|
||||
const url = titleMatch[1];
|
||||
const title = titleMatch[2].replace(/<[^>]+>/g, '').trim();
|
||||
const snippetMatch = block.match(/class="res-desc[^"]*"[^>]*>([\s\S]*?)<\/p>/i)
|
||||
|| block.match(/class="res-rich[^"]*"[^>]*>([\s\S]*?)<\/div>/i)
|
||||
|| block.match(/class="res-summary[^"]*"[^>]*>([\s\S]*?)<\/p>/i)
|
||||
|| block.match(/<dd[^>]*>([\s\S]*?)<\/dd>/i);
|
||||
const snippet = snippetMatch ? snippetMatch[1].replace(/<[^>]+>/g, '').trim() : '';
|
||||
if (title && url && !url.includes('so.com')) {
|
||||
results.push({ title, url, snippet, engine: '360搜索', weight: 75 });
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
// ===== 可达性预检 =====
|
||||
|
||||
async function checkReachability(urls: string[], concurrency = 5): Promise<Map<string, boolean>> {
|
||||
const result = new Map<string, boolean>();
|
||||
for (let i = 0; i < urls.length; i += concurrency) {
|
||||
const batch = urls.slice(i, i + concurrency);
|
||||
const checks = batch.map(async (url) => {
|
||||
try {
|
||||
const resp = await fetchWithTimeout(url, { method: 'HEAD', redirect: 'follow' }, 3_000);
|
||||
result.set(url, resp.ok);
|
||||
} catch {
|
||||
result.set(url, false);
|
||||
}
|
||||
});
|
||||
await Promise.allSettled(checks);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// ===== 智能排序 =====
|
||||
|
||||
function smartSort(results: SearchResult[], reachabilityMap: Map<string, boolean>): SearchResult[] {
|
||||
for (const r of results) {
|
||||
const reachability = r.reachable ? 30 : -20;
|
||||
const snippetQuality = Math.min(r.snippet.length, 100) / 100 * 20;
|
||||
const weightScore = (r.weight / 100) * 50;
|
||||
r._score = weightScore + reachability + snippetQuality;
|
||||
}
|
||||
return results.sort((a, b) => (b._score ?? 0) - (a._score ?? 0));
|
||||
}
|
||||
|
||||
// ===== 相关性过滤（用于自动抓取） =====
|
||||
|
||||
function computeRelevance(query: string, result: SearchResult): number {
|
||||
const terms: string[] = [];
|
||||
// 中文双字/三字片段
|
||||
const chinese = query.match(/[\u4e00-\u9fa5]{2,3}/g);
|
||||
if (chinese) terms.push(...chinese);
|
||||
// 英文单词
|
||||
const english = query.match(/[a-zA-Z]{2,}/g);
|
||||
if (english) terms.push(...english);
|
||||
|
||||
let score = 0;
|
||||
const text = `${result.title} ${result.snippet}`.toLowerCase();
|
||||
for (const term of terms) {
|
||||
if (text.includes(term.toLowerCase())) score += 10;
|
||||
}
|
||||
return Math.min(score, 100);
|
||||
}
|
||||
|
||||
// ===== WebSearchTool =====
|
||||
|
||||
export class WebSearchTool implements IMetonaTool {
|
||||
readonly definition: MetonaToolDef = {
|
||||
name: 'web_search',
|
||||
description: 'Search the web for information. Returns titles, snippets, and URLs. When SearXNG is enabled, uses the configured SearXNG instance; otherwise uses built-in engines (Bing, Baidu, Sogou, 360). Automatically fetches full content for top results.',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
query: { type: 'string', description: 'Search query keywords' },
|
||||
max_results: { type: 'number', description: 'Maximum results (default 30, max 30)' },
|
||||
time_range: { type: 'string', description: 'Time filter: day, week, month, year (optional)' },
|
||||
enhance_snippets: { type: 'boolean', description: 'Auto-enhance short snippets (default true)' },
|
||||
fetch_top: { type: 'number', description: 'Auto-fetch full content for top N results (3-8, default 5)' },
|
||||
},
|
||||
required: ['query'],
|
||||
},
|
||||
category: MetonaToolCategory.NETWORK,
|
||||
riskLevel: MetonaRiskLevel.LOW,
|
||||
requiresPermission: false,
|
||||
timeoutMs: 300_000,
|
||||
};
|
||||
|
||||
constructor(private configService: ConfigService) {}
|
||||
|
||||
async execute(args: Record<string, unknown>, _context: ToolExecutionContext): Promise<unknown> {
|
||||
const query = args.query as string;
|
||||
|
||||
// 读取 SearXNG 配置（提前读取，使工具参数默认值与配置一致）
|
||||
const searxngConfig = this.readSearXNGConfig();
|
||||
|
||||
// max_results: 工具参数 > 配置面板 > 默认 30。取较大值保证配置面板的下限约束
|
||||
const argMaxResults = (args.max_results as number) ?? undefined;
|
||||
const cfgMaxResults = searxngConfig.max_results > 0 ? searxngConfig.max_results : undefined;
|
||||
const maxResults = Math.min(30, Math.max(1, argMaxResults ?? cfgMaxResults ?? 30));
|
||||
|
||||
const timeRange = (args.time_range as string) ?? '';
|
||||
const enhanceSnippets = (args.enhance_snippets as boolean) ?? true;
|
||||
|
||||
// fetch_top: 工具参数 > 配置面板 > 默认 5。取较大值保证配置面板的下限约束
|
||||
const argFetchTop = (args.fetch_top as number) ?? undefined;
|
||||
const cfgFetchCount = searxngConfig.fetch_count > 0 ? searxngConfig.fetch_count : undefined;
|
||||
const fetchTop = Math.min(8, Math.max(3, argFetchTop ?? cfgFetchCount ?? 5));
|
||||
|
||||
// 缓存检查（缓存 key 含 SearXNG 模式，避免切换后返回旧缓存）
|
||||
const cacheKey = `${searxngConfig.enabled ? 'searxng' : 'builtin'}:${normalizeUrl(query).toLowerCase()}`;
|
||||
const cached = searchCache.get(cacheKey);
|
||||
if (cached) {
|
||||
logTool('web_search', `Cache hit: "${query}"`);
|
||||
return { ...(cached as Record<string, unknown>), from_cache: true };
|
||||
}
|
||||
|
||||
let results: SearchResult[];
|
||||
let mode: string;
|
||||
let engineStats: Record<string, string>;
|
||||
|
||||
if (searxngConfig.enabled && searxngConfig.url) {
|
||||
const searxResult = await this.searchSearXNG(query, searxngConfig, maxResults, timeRange);
|
||||
results = searxResult.results;
|
||||
mode = 'searxng';
|
||||
engineStats = searxResult.engineStats;
|
||||
} else {
|
||||
const builtinResult = await this.searchBuiltinEngines(query, maxResults, timeRange);
|
||||
results = builtinResult.results;
|
||||
mode = 'builtin';
|
||||
engineStats = builtinResult.engineStats;
|
||||
}
|
||||
|
||||
// 合并去重
|
||||
const deduped = this.deduplicate(results);
|
||||
|
||||
// 可达性预检
|
||||
const topUrls = deduped.slice(0, Math.min(20, deduped.length)).map((r) => r.url);
|
||||
const reachabilityMap = await checkReachability(topUrls);
|
||||
for (const r of deduped) {
|
||||
r.reachable = reachabilityMap.get(r.url) ?? false;
|
||||
}
|
||||
|
||||
// 智能排序
|
||||
const sorted = smartSort(deduped, reachabilityMap).slice(0, maxResults);
|
||||
|
||||
// 摘要增强
|
||||
if (enhanceSnippets) {
|
||||
await this.enhanceSnippets(sorted, 3);
|
||||
}
|
||||
|
||||
// 自动抓取完整内容
|
||||
const fetchedContent = await this.autoFetch(query, sorted, searxngConfig.fetch_mode, fetchTop);
|
||||
|
||||
// 格式化输出
|
||||
const formatted = this.formatResults(query, sorted);
|
||||
|
||||
const output = {
|
||||
success: true,
|
||||
query,
|
||||
results: sorted.map((r) => ({
|
||||
title: r.title,
|
||||
url: r.url,
|
||||
snippet: r.snippet,
|
||||
engine: r.engine,
|
||||
reachable: r.reachable,
|
||||
_score: r._score ? Math.round(r._score * 10) / 10 : undefined,
|
||||
_enhanced: r._enhanced,
|
||||
})),
|
||||
total: sorted.length,
|
||||
formatted,
|
||||
from_cache: false,
|
||||
engine_stats: engineStats,
|
||||
_mode: mode,
|
||||
_fetched: fetchedContent.map((f) => ({ url: f.url, title: f.title, content: f.content })),
|
||||
_fetched_count: fetchedContent.length,
|
||||
};
|
||||
|
||||
// 写入缓存
|
||||
searchCache.set(cacheKey, output);
|
||||
logTool('web_search', `Completed: ${sorted.length} results, ${fetchedContent.length} fetched, mode=${mode}`);
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
// ===== SearXNG 搜索 =====
|
||||
|
||||
private async searchSearXNG(
|
||||
query: string,
|
||||
config: SearXNGConfig,
|
||||
maxResults: number,
|
||||
timeRange: string,
|
||||
): Promise<{ results: SearchResult[]; engineStats: Record<string, string> }> {
|
||||
const params = new URLSearchParams();
|
||||
params.set('q', query);
|
||||
params.set('format', config.format || 'json');
|
||||
if (config.engines) params.set('engines', config.engines);
|
||||
if (config.language && config.language !== 'auto') params.set('language', config.language);
|
||||
params.set('safesearch', String(config.safesearch));
|
||||
const tr = timeRange || config.time_range;
|
||||
if (tr) params.set('time_range', tr);
|
||||
const limit = Math.max(maxResults, config.max_results);
|
||||
if (limit > 0) params.set('results_count', String(limit));
|
||||
|
||||
const searchUrl = `${config.url.replace(/\/$/, '')}/search?${params.toString()}`;
|
||||
const headers = buildSearXNGAuthHeaders(config.auth_key, config.auth_type);
|
||||
|
||||
logTool('web_search', `[SearXNG] Fetching: ${searchUrl}`);
|
||||
|
||||
const response = await fetchWithTimeout(searchUrl, { headers }, 15_000);
|
||||
if (!response.ok) {
|
||||
throw new Error(`SearXNG API error: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
const results: SearchResult[] = [];
|
||||
const engineStats: Record<string, string> = {};
|
||||
const engineCounts = new Map<string, number>();
|
||||
|
||||
if (config.format === 'html') {
|
||||
const html = await response.text();
|
||||
// HTML 模式：简单提取链接和标题
|
||||
const matches = html.matchAll(/<a[^>]*href="(https?:\/\/[^"]+)"[^>]*>([\s\S]*?)<\/a>/gi);
|
||||
for (const m of matches) {
|
||||
const url = m[1];
|
||||
const title = m[2].replace(/<[^>]+>/g, '').trim();
|
||||
if (title && url && !url.includes(config.url)) {
|
||||
results.push({ title, url, snippet: '', engine: 'searxng', weight: 85 });
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const data = await response.json() as { results?: Array<Record<string, unknown>> };
|
||||
for (const item of data.results ?? []) {
|
||||
const url = item.url as string;
|
||||
const title = item.title as string;
|
||||
const snippet = item.content as string;
|
||||
const engine = (item.engine as string) ?? 'searxng';
|
||||
results.push({ title, url, snippet: snippet ?? '', engine, weight: 85 });
|
||||
engineCounts.set(engine, (engineCounts.get(engine) ?? 0) + 1);
|
||||
}
|
||||
}
|
||||
|
||||
for (const [engine, count] of engineCounts) {
|
||||
engineStats[engine] = `${count} 条`;
|
||||
}
|
||||
|
||||
return { results, engineStats };
|
||||
}
|
||||
|
||||
// ===== 内置四引擎并行搜索 =====
|
||||
|
||||
private async searchBuiltinEngines(
|
||||
query: string,
|
||||
maxResults: number,
|
||||
timeRange: string,
|
||||
): Promise<{ results: SearchResult[]; engineStats: Record<string, string> }> {
|
||||
const searchPromises = ENGINES.map(async (engine) => {
|
||||
try {
|
||||
const url = engine.searchUrl(query, timeRange);
|
||||
const response = await fetchWithTimeout(url, {
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36',
|
||||
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
|
||||
},
|
||||
}, 8_000);
|
||||
|
||||
if (!response.ok) {
|
||||
logTool('web_search', `[内置] ${engine.name} HTTP ${response.status}`);
|
||||
return [];
|
||||
}
|
||||
|
||||
const html = await response.text();
|
||||
const parsed = engine.parse(html);
|
||||
logTool('web_search', `[内置] ${engine.name}: ${parsed.length} results`);
|
||||
return parsed;
|
||||
} catch (err) {
|
||||
logTool('web_search', `[内置] ${engine.name} error: ${(err as Error).message}`);
|
||||
return [];
|
||||
}
|
||||
});
|
||||
|
||||
const settled = await Promise.allSettled(searchPromises);
|
||||
const allResults: SearchResult[] = [];
|
||||
const engineStats: Record<string, string> = {};
|
||||
|
||||
for (let i = 0; i < ENGINES.length; i++) {
|
||||
const result = settled[i];
|
||||
const engineName = ENGINES[i].name;
|
||||
if (result.status === 'fulfilled') {
|
||||
allResults.push(...result.value);
|
||||
engineStats[engineName] = `${result.value.length} 条`;
|
||||
} else {
|
||||
engineStats[engineName] = '失败';
|
||||
}
|
||||
}
|
||||
|
||||
return { results: allResults, engineStats };
|
||||
}
|
||||
|
||||
// ===== 合并去重 =====
|
||||
|
||||
private deduplicate(results: SearchResult[]): SearchResult[] {
|
||||
const seen = new Map<string, SearchResult>();
|
||||
for (const r of results) {
|
||||
const normalized = normalizeUrl(r.url);
|
||||
if (!seen.has(normalized)) {
|
||||
seen.set(normalized, r);
|
||||
}
|
||||
}
|
||||
return Array.from(seen.values());
|
||||
}
|
||||
|
||||
// ===== 摘要增强 =====
|
||||
|
||||
private async enhanceSnippets(results: SearchResult[], maxEnhance: number): Promise<void> {
|
||||
let enhanced = 0;
|
||||
for (const r of results) {
|
||||
if (enhanced >= maxEnhance) break;
|
||||
if (r.snippet.length < 30 && r.reachable) {
|
||||
try {
|
||||
const resp = await fetchWithTimeout(r.url, {
|
||||
headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36' },
|
||||
}, 5_000);
|
||||
if (resp.ok) {
|
||||
const html = await resp.text();
|
||||
// 延迟导入 htmlToText 以避免循环依赖
|
||||
const { htmlToText } = await import('./network-utils');
|
||||
const text = htmlToText(html).slice(0, 200);
|
||||
if (text.length > r.snippet.length) {
|
||||
r.snippet = text;
|
||||
r._enhanced = true;
|
||||
enhanced++;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// 忽略增强失败
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ===== 自动抓取完整内容 =====
|
||||
|
||||
private async autoFetch(
|
||||
query: string,
|
||||
results: SearchResult[],
|
||||
fetchMode: string,
|
||||
fetchTop: number,
|
||||
): Promise<Array<{ url: string; title: string; content: string }>> {
|
||||
// 相关性评分（不过滤，relevance=0 的结果也参与抓取候选）
|
||||
const withRelevance = results.map((r) => ({ result: r, relevance: computeRelevance(query, r) }));
|
||||
const filtered = withRelevance.length > 0 ? withRelevance : [];
|
||||
|
||||
// 确定抓取数量：fetchTop 已在 execute() 中综合了配置面板和工具参数
|
||||
const topN = Math.min(fetchTop, 8, filtered.length);
|
||||
|
||||
// 构建抓取列表
|
||||
let toFetch: typeof filtered;
|
||||
if (fetchMode === 'random') {
|
||||
// Fisher-Yates 洗牌
|
||||
const shuffled = [...filtered];
|
||||
for (let i = shuffled.length - 1; i > 0; i--) {
|
||||
const j = Math.floor(Math.random() * (i + 1));
|
||||
[shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
|
||||
}
|
||||
toFetch = shuffled.slice(0, topN);
|
||||
} else {
|
||||
toFetch = filtered
|
||||
.sort((a, b) => b.relevance - a.relevance)
|
||||
.slice(0, topN);
|
||||
}
|
||||
|
||||
const fetched: Array<{ url: string; title: string; content: string }> = [];
|
||||
const { htmlToText } = await import('./network-utils');
|
||||
|
||||
for (const item of toFetch) {
|
||||
try {
|
||||
const resp = await fetchWithTimeout(item.result.url, {
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36',
|
||||
'Accept-Language': 'zh-CN,zh;q=0.9',
|
||||
},
|
||||
}, 15_000);
|
||||
|
||||
if (resp.ok) {
|
||||
const html = await resp.text();
|
||||
const content = htmlToText(html);
|
||||
fetched.push({ url: item.result.url, title: item.result.title, content });
|
||||
}
|
||||
} catch (err) {
|
||||
logTool('web_search', `Auto-fetch failed for ${item.result.url}: ${(err as Error).message}`);
|
||||
// 从剩余结果中随机补充
|
||||
const remaining = withRelevance.filter((x) => !toFetch.includes(x) && !fetched.some((f) => f.url === x.result.url));
|
||||
if (remaining.length > 0) {
|
||||
const randomPick = remaining[Math.floor(Math.random() * remaining.length)];
|
||||
try {
|
||||
const resp2 = await fetchWithTimeout(randomPick.result.url, {
|
||||
headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36' },
|
||||
}, 10_000);
|
||||
if (resp2.ok) {
|
||||
const html2 = await resp2.text();
|
||||
fetched.push({ url: randomPick.result.url, title: randomPick.result.title, content: htmlToText(html2) });
|
||||
}
|
||||
} catch {
|
||||
// 忽略补充失败
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return fetched;
|
||||
}
|
||||
|
||||
// ===== 格式化人类可读输出 =====
|
||||
|
||||
private formatResults(query: string, results: SearchResult[]): string {
|
||||
if (results.length === 0) return `搜索 "${query}" 无结果。`;
|
||||
const lines = [`搜索 "${query}" — ${results.length} 条结果：\n`];
|
||||
results.forEach((r, i) => {
|
||||
lines.push(`${i + 1}. ${r.title}`);
|
||||
lines.push(` URL: ${r.url}`);
|
||||
if (r.snippet) lines.push(` 摘要: ${r.snippet.slice(0, 150)}`);
|
||||
lines.push(` 来源: ${r.engine}${r.reachable === false ? ' (不可达)' : ''}${r._enhanced ? ' [已增强]' : ''}\n`);
|
||||
});
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
// ===== 读取 SearXNG 配置 =====
|
||||
|
||||
private readSearXNGConfig(): SearXNGConfig {
|
||||
const cs = this.configService;
|
||||
return {
|
||||
enabled: cs.get<boolean>('searxng.enabled') ?? false,
|
||||
url: cs.get<string>('searxng.url') ?? '',
|
||||
engines: cs.get<string>('searxng.engines') ?? '',
|
||||
language: cs.get<string>('searxng.language') ?? 'zh-CN',
|
||||
safesearch: cs.get<number>('searxng.safesearch') ?? 1,
|
||||
time_range: cs.get<string>('searxng.time_range') ?? '',
|
||||
max_results: cs.get<number>('searxng.max_results') ?? 0,
|
||||
auth_key: cs.get<string>('searxng.auth_key') ?? '',
|
||||
auth_type: cs.get<string>('searxng.auth_type') ?? 'bearer',
|
||||
format: cs.get<string>('searxng.format') ?? 'json',
|
||||
fetch_count: cs.get<number>('searxng.fetch_count') ?? 0,
|
||||
fetch_mode: cs.get<string>('searxng.fetch_mode') ?? 'sequential',
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -58,6 +58,14 @@ export class ToolRegistry {
|
||||
.map((e) => e.tool.definition);
|
||||
}
|
||||
|
||||
/** 列出所有工具定义（含已禁用的，供设置 UI 使用） */
|
||||
listAllTools(): Array<MetonaToolDef & { enabled: boolean }> {
|
||||
return Array.from(this.tools.values()).map((e) => ({
|
||||
...e.tool.definition,
|
||||
enabled: e.enabled && !this.disabledTools.has(e.tool.definition.name),
|
||||
}));
|
||||
}
|
||||
|
||||
/** 设置工具启用/禁用状态（供 IPC tools:toggle 调用） */
|
||||
setToolEnabled(name: string, enabled: boolean): void {
|
||||
if (enabled) {
|
||||
|
||||
Reference in New Issue
Block a user