feat: MEMORY.md 访问保护、格式简化及预存问题修复
MEMORY.md 保护: 新建 file-guard.ts 共享守卫模块; read_file/write_file/search_files/run_command 禁止访问根目录 MEMORY.md; permissions.ts 增加 deniedPatterns 深度防御。格式简化: 元数据从 # 注释改为 > 引用语法; 校验规则从 6 条简化为 3 条; context-builder extractContent 适配。预存问题: 修复路径遍历前缀碰撞漏洞; 移除 browser_extract 内容截断; SearXNG 配置实时读取; web_search/web_fetch 移除截断; 侧边栏动态获取工具列表; 版本号 0.1.1
This commit is contained in:
thzxx
@@ -0,0 +1,222 @@
|
||||
/**
|
||||
* 网络工具共享模块
|
||||
*
|
||||
* 提供 UA 轮换池、反爬请求头、HTML 转文本、拦截检测、超时 fetch、LRU 缓存等通用能力。
|
||||
* web_search 和 web_fetch 共享此模块。
|
||||
*
|
||||
* @see docs/Agent网络工具通用设计-v2.md — 第 3 章 web_fetch 抓取设计
|
||||
*/
|
||||
|
||||
import { LRUCache } from 'lru-cache';
|
||||
import log from 'electron-log';
|
||||
|
||||
// ===== LRU 缓存 =====
|
||||
|
||||
/** 搜索结果缓存（200 条，5 分钟 TTL） */
|
||||
export const searchCache = new LRUCache<string, Record<string, unknown>>({
|
||||
max: 200,
|
||||
ttl: 300_000,
|
||||
});
|
||||
|
||||
/** 浏览器回退缓存（100 条，10 分钟 TTL） */
|
||||
export const fetchCache = new LRUCache<string, string>({
|
||||
max: 100,
|
||||
ttl: 600_000,
|
||||
});
|
||||
|
||||
// ===== UA 轮换池 =====
|
||||
|
||||
export const UA_POOL = [
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Edg/131.0.0.0 Safari/537.36',
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.1 Safari/605.1.15',
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0',
|
||||
];
|
||||
|
||||
export const MOBILE_UA = 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1';
|
||||
|
||||
export const ACCEPT_LANGUAGE_POOL = [
|
||||
'zh-CN,zh;q=0.9,en;q=0.8',
|
||||
'zh-CN,zh;q=0.9',
|
||||
'en-US,en;q=0.9,zh-CN;q=0.8',
|
||||
];
|
||||
|
||||
// ===== 反爬请求头构建 =====
|
||||
|
||||
export function buildAntiCrawlHeaders(
|
||||
url: string,
|
||||
attempt: number,
|
||||
mobileUA = false,
|
||||
): Record<string, string> {
|
||||
const uaIdx = attempt % UA_POOL.length;
|
||||
const langIdx = attempt % ACCEPT_LANGUAGE_POOL.length;
|
||||
const userAgent = mobileUA ? MOBILE_UA : UA_POOL[uaIdx];
|
||||
|
||||
let origin = '';
|
||||
try { origin = new URL(url).origin; } catch { /* ignore */ }
|
||||
|
||||
return {
|
||||
'User-Agent': userAgent,
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
|
||||
'Accept-Language': ACCEPT_LANGUAGE_POOL[langIdx],
|
||||
'Accept-Encoding': 'gzip, deflate, br',
|
||||
'Cache-Control': 'no-cache',
|
||||
'DNT': '1',
|
||||
'Referer': origin || '',
|
||||
'Sec-Fetch-Dest': 'document',
|
||||
'Sec-Fetch-Mode': 'navigate',
|
||||
'Sec-Fetch-Site': 'none',
|
||||
'Sec-Fetch-User': '?1',
|
||||
'Pragma': 'no-cache',
|
||||
};
|
||||
}
|
||||
|
||||
// ===== 超时 fetch =====
|
||||
|
||||
export async function fetchWithTimeout(
|
||||
url: string,
|
||||
options: RequestInit = {},
|
||||
timeoutMs = 20_000,
|
||||
): Promise<Response> {
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||
try {
|
||||
return await fetch(url, { ...options, signal: controller.signal });
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
// ===== URL 标准化（去重用） =====
|
||||
|
||||
export function normalizeUrl(url: string): string {
|
||||
try {
|
||||
const u = new URL(url);
|
||||
// 去除追踪参数
|
||||
const trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'gclid', 'fbclid'];
|
||||
for (const p of trackingParams) u.searchParams.delete(p);
|
||||
// 去除尾部斜杠（根路径除外）
|
||||
let path = u.pathname;
|
||||
if (path.length > 1 && path.endsWith('/')) path = path.slice(0, -1);
|
||||
// 强制小写 host
|
||||
return `${u.protocol}//${u.host.toLowerCase()}${path}${u.search}${u.hash}`;
|
||||
} catch {
|
||||
return url;
|
||||
}
|
||||
}
|
||||
|
||||
// ===== 拦截页面检测 =====
|
||||
|
||||
const INTERCEPTION_PATTERNS = [
|
||||
/just a moment/i,
|
||||
/attention required/i,
|
||||
/challenge-platform/i,
|
||||
/\.cf-challenge-/i,
|
||||
/access denied/i,
|
||||
/403 forbidden/i,
|
||||
/请启用\s*javascript/i,
|
||||
/please enable javascript/i,
|
||||
/checking your browser/i,
|
||||
/ddos protection/i,
|
||||
];
|
||||
|
||||
export function isInterceptedPage(html: string): boolean {
|
||||
if (html.length < 80) return true;
|
||||
const lower = html.toLowerCase();
|
||||
return INTERCEPTION_PATTERNS.some((p) => p.test(lower));
|
||||
}
|
||||
|
||||
// ===== HTML → 纯文本转换 =====
|
||||
|
||||
const HTML_ENTITY_MAP: Record<string, string> = {
|
||||
' ': ' ', '<': '<', '>': '>', '&': '&', '"': '"',
|
||||
''': "'", '…': '…', '—': '—', '–': '–',
|
||||
'«': '«', '»': '»', '×': '×', '÷': '÷',
|
||||
'©': '©', '®': '®', '™': '™', '€': '€',
|
||||
'£': '£', '¥': '¥', '¢': '¢', '°': '°',
|
||||
};
|
||||
|
||||
export function htmlToText(html: string): string {
|
||||
return html
|
||||
// 移除噪声标签及内容
|
||||
.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
|
||||
.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
|
||||
.replace(/<noscript[^>]*>[\s\S]*?<\/noscript>/gi, '')
|
||||
.replace(/<nav[^>]*>[\s\S]*?<\/nav>/gi, '')
|
||||
.replace(/<header[^>]*>[\s\S]*?<\/header>/gi, '')
|
||||
.replace(/<footer[^>]*>[\s\S]*?<\/footer>/gi, '')
|
||||
.replace(/<aside[^>]*>[\s\S]*?<\/aside>/gi, '')
|
||||
.replace(/<iframe[^>]*>[\s\S]*?<\/iframe>/gi, '')
|
||||
.replace(/<svg[^>]*>[\s\S]*?<\/svg>/gi, '')
|
||||
// 移除 HTML 注释
|
||||
.replace(/<!--[\s\S]*?-->/g, '')
|
||||
// 块级标签转换行
|
||||
.replace(/<\/?(p|div|h[1-6]|li|tr|blockquote|section|article|pre|br|hr)[^>]*>/gi, '\n')
|
||||
// 表格单元格转制表符
|
||||
.replace(/<\/?(td|th)[^>]*>/gi, '\t')
|
||||
// 移除剩余标签
|
||||
.replace(/<[^>]+>/g, '')
|
||||
// 解码 HTML 实体
|
||||
.replace(/&#(\d+);/g, (_, n) => String.fromCharCode(Number(n)))
|
||||
.replace(/&#x([0-9a-f]+);/gi, (_, h) => String.fromCharCode(parseInt(h, 16)))
|
||||
.replace(/&[a-z]+;/gi, (m) => HTML_ENTITY_MAP[m.toLowerCase()] ?? m)
|
||||
// 清理空白
|
||||
.replace(/\n{3,}/g, '\n\n')
|
||||
.replace(/[ \t]+/g, ' ')
|
||||
.replace(/^[ \t]+/gm, '')
|
||||
.trim();
|
||||
}
|
||||
|
||||
// ===== 流式读取（大文件保护，10MB 上限） =====
|
||||
|
||||
export async function readBodyWithLimit(response: Response, maxBytes = 10 * 1024 * 1024): Promise<string> {
|
||||
const contentLength = response.headers.get('content-length');
|
||||
if (contentLength && parseInt(contentLength) > maxBytes) {
|
||||
throw new Error(`Response too large: ${contentLength} bytes (limit: ${maxBytes})`);
|
||||
}
|
||||
|
||||
const reader = response.body?.getReader();
|
||||
if (!reader) return '';
|
||||
|
||||
const chunks: Uint8Array[] = [];
|
||||
let totalBytes = 0;
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
if (value) {
|
||||
totalBytes += value.byteLength;
|
||||
if (totalBytes > maxBytes) {
|
||||
throw new Error(`Response exceeded ${maxBytes} bytes limit`);
|
||||
}
|
||||
chunks.push(value);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
|
||||
const decoder = new TextDecoder('utf-8', { fatal: false });
|
||||
return decoder.decode(Buffer.concat(chunks));
|
||||
}
|
||||
|
||||
// ===== SearXNG 认证头构建 =====
|
||||
|
||||
export function buildSearXNGAuthHeaders(authKey: string, authType: string): Record<string, string> {
|
||||
if (!authKey) return {};
|
||||
if (authType === 'bearer') {
|
||||
return { Authorization: `Bearer ${authKey}` };
|
||||
}
|
||||
if (authType === 'basic') {
|
||||
return { Authorization: `Basic ${Buffer.from(authKey).toString('base64')}` };
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
// ===== 日志辅助 =====
|
||||
|
||||
export function logTool(toolName: string, message: string): void {
|
||||
log.info(`[Tool:${toolName}] ${message}`);
|
||||
}
|
||||
Reference in New Issue
Block a user