feat: MEMORY.md 访问保护、格式简化及预存问题修复
MEMORY.md 保护: 新建 file-guard.ts 共享守卫模块; read_file/write_file/search_files/run_command 禁止访问根目录 MEMORY.md; permissions.ts 增加 deniedPatterns 深度防御。格式简化: 元数据从 # 注释改为 > 引用语法; 校验规则从 6 条简化为 3 条; context-builder extractContent 适配。预存问题: 修复路径遍历前缀碰撞漏洞; 移除 browser_extract 内容截断; SearXNG 配置实时读取; web_search/web_fetch 移除截断; 侧边栏动态获取工具列表; 版本号 0.1.1
This commit is contained in:
thzxx
@@ -0,0 +1,240 @@
|
||||
/**
|
||||
* web_fetch — 网页抓取工具
|
||||
*
|
||||
* 三阶段回退策略：
|
||||
* Phase 1: HTTP 抓取（UA 轮换 + 反爬请求头 + 指数退避重试 + 拦截检测）
|
||||
* Phase 2: 内容过短自动升级（< 200 字符 → 浏览器渲染）
|
||||
* Phase 3: 浏览器回退（隐藏 BrowserWindow + JS 渲染 + 内容提取）
|
||||
*
|
||||
* @see docs/Agent网络工具通用设计-v2.md — 第 3 章 web_fetch 抓取设计
|
||||
*/
|
||||
|
||||
import { BrowserWindow } from 'electron';
|
||||
import type { IMetonaTool, ToolExecutionContext } from '../../types/metona-tool';
|
||||
import type { MetonaToolDef } from '../../../harness/types';
|
||||
import { MetonaToolCategory, MetonaRiskLevel } from '../../../harness/types';
|
||||
import {
|
||||
fetchCache,
|
||||
buildAntiCrawlHeaders,
|
||||
fetchWithTimeout,
|
||||
htmlToText,
|
||||
isInterceptedPage,
|
||||
readBodyWithLimit,
|
||||
logTool,
|
||||
} from './network-utils';
|
||||
|
||||
// ===== 跳过重试的状态码 =====
|
||||
|
||||
const SKIP_RETRY_STATUS = new Set([403, 429, 502, 503]);
|
||||
|
||||
// ===== WebFetchTool =====
|
||||
|
||||
export class WebFetchTool implements IMetonaTool {
|
||||
readonly definition: MetonaToolDef = {
|
||||
name: 'web_fetch',
|
||||
description: 'Fetch a web page and convert to plain text. Uses a three-phase fallback strategy: HTTP fetch with anti-crawl headers → SPA auto-upgrade → browser rendering. Handles Cloudflare interception, JavaScript-rendered pages, and large files (10MB limit).',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
url: { type: 'string', description: 'Target URL (http/https only)' },
|
||||
mobile_ua: { type: 'boolean', description: 'Use mobile User-Agent (default false)' },
|
||||
retry: { type: 'boolean', description: 'Enable retry with exponential backoff (default true)' },
|
||||
},
|
||||
required: ['url'],
|
||||
},
|
||||
category: MetonaToolCategory.NETWORK,
|
||||
riskLevel: MetonaRiskLevel.LOW,
|
||||
requiresPermission: false,
|
||||
timeoutMs: 120_000,
|
||||
};
|
||||
|
||||
async execute(args: Record<string, unknown>, _context: ToolExecutionContext): Promise<unknown> {
|
||||
const url = args.url as string;
|
||||
const mobileUA = (args.mobile_ua as boolean) ?? false;
|
||||
const enableRetry = (args.retry as boolean) ?? true;
|
||||
|
||||
if (!url || !/^https?:\/\//i.test(url)) {
|
||||
return { url, content: '', success: false, error: 'URL must start with http:// or https://' };
|
||||
}
|
||||
|
||||
logTool('web_fetch', `Fetching: ${url}`);
|
||||
|
||||
// ===== Phase 1: HTTP 抓取 =====
|
||||
const phase1Result = await this.httpFetch(url, mobileUA, enableRetry);
|
||||
|
||||
if (phase1Result.success && !phase1Result.intercepted) {
|
||||
// 内容过短检测 → Phase 2 升级
|
||||
if (phase1Result.text.length < 200) {
|
||||
logTool('web_fetch', `Phase 2: Content too short (${phase1Result.text.length} chars), upgrading to browser`);
|
||||
const browserResult = await this.browserFetch(url);
|
||||
if (browserResult) {
|
||||
return this.buildSuccess(url, browserResult, 'browser');
|
||||
}
|
||||
}
|
||||
return this.buildSuccess(url, phase1Result.text, 'http');
|
||||
}
|
||||
|
||||
// ===== Phase 3: 浏览器回退 =====
|
||||
logTool('web_fetch', `Phase 3: Falling back to browser (${phase1Result.reason})`);
|
||||
const browserResult = await this.browserFetch(url);
|
||||
if (browserResult) {
|
||||
return this.buildSuccess(url, browserResult, 'browser');
|
||||
}
|
||||
|
||||
// 全部失败
|
||||
return {
|
||||
url,
|
||||
content: '',
|
||||
success: false,
|
||||
error: `All phases failed. HTTP: ${phase1Result.reason}. Browser fallback also failed.`,
|
||||
};
|
||||
}
|
||||
|
||||
// ===== Phase 1: HTTP 抓取 =====
|
||||
|
||||
private async httpFetch(
|
||||
url: string,
|
||||
mobileUA: boolean,
|
||||
enableRetry: boolean,
|
||||
): Promise<{ success: boolean; text: string; intercepted: boolean; reason: string }> {
|
||||
const maxRetries = enableRetry ? 3 : 1;
|
||||
const backoffBase = 2_000;
|
||||
|
||||
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
||||
try {
|
||||
const headers = buildAntiCrawlHeaders(url, attempt, mobileUA);
|
||||
const response = await fetchWithTimeout(url, { headers, redirect: 'follow' }, 20_000);
|
||||
|
||||
// 跳过重试的状态码 → 直接进入浏览器回退
|
||||
if (SKIP_RETRY_STATUS.has(response.status)) {
|
||||
return { success: false, text: '', intercepted: true, reason: `HTTP ${response.status}` };
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
// 5xx 可重试
|
||||
if (response.status >= 500 && attempt < maxRetries - 1) {
|
||||
await this.sleep(backoffBase * Math.pow(2, attempt) + Math.random() * backoffBase * 0.6);
|
||||
continue;
|
||||
}
|
||||
return { success: false, text: '', intercepted: false, reason: `HTTP ${response.status} ${response.statusText}` };
|
||||
}
|
||||
|
||||
// 读取正文（10MB 限制）
|
||||
const html = await readBodyWithLimit(response, 10 * 1024 * 1024);
|
||||
|
||||
// 拦截检测
|
||||
if (isInterceptedPage(html)) {
|
||||
return { success: false, text: '', intercepted: true, reason: 'Intercepted page detected' };
|
||||
}
|
||||
|
||||
// HTML → 纯文本
|
||||
const text = htmlToText(html);
|
||||
return { success: true, text, intercepted: false, reason: '' };
|
||||
} catch (err) {
|
||||
const errorMsg = (err as Error).message;
|
||||
if (attempt < maxRetries - 1) {
|
||||
logTool('web_fetch', `Attempt ${attempt + 1} failed: ${errorMsg}, retrying...`);
|
||||
await this.sleep(backoffBase * Math.pow(2, attempt) + Math.random() * backoffBase * 0.6);
|
||||
continue;
|
||||
}
|
||||
return { success: false, text: '', intercepted: false, reason: errorMsg };
|
||||
}
|
||||
}
|
||||
|
||||
return { success: false, text: '', intercepted: false, reason: 'All retries exhausted' };
|
||||
}
|
||||
|
||||
// ===== Phase 2/3: 浏览器回退 =====
|
||||
|
||||
private async browserFetch(url: string): Promise<string | null> {
|
||||
// 查缓存
|
||||
const cached = fetchCache.get(url);
|
||||
if (cached) {
|
||||
logTool('web_fetch', 'Browser cache hit');
|
||||
return cached;
|
||||
}
|
||||
|
||||
let win: BrowserWindow | null = null;
|
||||
let cleanup: (() => void) | null = null;
|
||||
|
||||
try {
|
||||
win = new BrowserWindow({
|
||||
width: 1280,
|
||||
height: 800,
|
||||
show: false,
|
||||
webPreferences: {
|
||||
nodeIntegration: false,
|
||||
contextIsolation: true,
|
||||
sandbox: true,
|
||||
webSecurity: false, // 允许跨域（截图需要）
|
||||
},
|
||||
});
|
||||
|
||||
cleanup = () => {
|
||||
try {
|
||||
if (win && !win.isDestroyed()) win.close();
|
||||
} catch {
|
||||
// 忽略关闭错误
|
||||
}
|
||||
};
|
||||
|
||||
await this.loadURLWithTimeout(win, url, 30_000);
|
||||
|
||||
// 等待 JS 渲染
|
||||
await this.sleep(2_500);
|
||||
|
||||
// 提取页面正文
|
||||
const text = await win.webContents.executeJavaScript(`
|
||||
(function() {
|
||||
var clone = document.body.cloneNode(true);
|
||||
var noise = clone.querySelectorAll('script, style, noscript, nav, header, footer, aside, iframe, svg');
|
||||
noise.forEach(function(el) { el.remove(); });
|
||||
return clone.innerText || '';
|
||||
})();
|
||||
`, true);
|
||||
|
||||
if (text && text.trim().length >= 80) {
|
||||
// 写缓存
|
||||
fetchCache.set(url, text);
|
||||
logTool('web_fetch', `Browser fetch success: ${text.length} chars`);
|
||||
return text;
|
||||
}
|
||||
|
||||
return null;
|
||||
} catch (err) {
|
||||
logTool('web_fetch', `Browser fetch failed: ${(err as Error).message}`);
|
||||
return null;
|
||||
} finally {
|
||||
if (cleanup) cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
// ===== 辅助方法 =====
|
||||
|
||||
private buildSuccess(url: string, text: string, method: string): unknown {
|
||||
return {
|
||||
url,
|
||||
content: text,
|
||||
success: true,
|
||||
method,
|
||||
length: text.length,
|
||||
};
|
||||
}
|
||||
|
||||
private sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/** 带超时的 loadURL（Electron 原生不支持 timeout 选项） */
|
||||
private async loadURLWithTimeout(win: BrowserWindow, url: string, timeoutMs: number): Promise<void> {
|
||||
let timer: NodeJS.Timeout | null = null;
|
||||
const timeoutPromise = new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(() => reject(new Error(`Page load timeout after ${timeoutMs}ms: ${url}`)), timeoutMs);
|
||||
});
|
||||
try {
|
||||
await Promise.race([win.loadURL(url), timeoutPromise]);
|
||||
} finally {
|
||||
if (timer) clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user