/** * 网络工具共享模块 * * 提供 UA 轮换池、反爬请求头、HTML 转文本、拦截检测、超时 fetch、LRU 缓存等通用能力。 * web_search 和 web_fetch 共享此模块。 * * @see docs/Agent网络工具通用设计-v2.md — 第 3 章 web_fetch 抓取设计 */ import { LRUCache } from 'lru-cache'; import log from 'electron-log'; // ===== LRU 缓存 ===== /** 搜索结果缓存(200 条,5 分钟 TTL) */ export const searchCache = new LRUCache>({ max: 200, ttl: 300_000, }); /** 浏览器回退缓存(100 条,10 分钟 TTL) */ export const fetchCache = new LRUCache({ max: 100, ttl: 600_000, }); // ===== UA 轮换池 ===== export const UA_POOL = [ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Edg/131.0.0.0 Safari/537.36', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.1 Safari/605.1.15', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0', ]; export const MOBILE_UA = 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1'; export const ACCEPT_LANGUAGE_POOL = [ 'zh-CN,zh;q=0.9,en;q=0.8', 'zh-CN,zh;q=0.9', 'en-US,en;q=0.9,zh-CN;q=0.8', ]; // ===== 反爬请求头构建 ===== export function buildAntiCrawlHeaders( url: string, attempt: number, mobileUA = false, ): Record { const uaIdx = attempt % UA_POOL.length; const langIdx = attempt % ACCEPT_LANGUAGE_POOL.length; const userAgent = mobileUA ? MOBILE_UA : UA_POOL[uaIdx]; let origin = ''; try { origin = new URL(url).origin; } catch { /* ignore */ } return { 'User-Agent': userAgent, 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', 'Accept-Language': ACCEPT_LANGUAGE_POOL[langIdx], 'Accept-Encoding': 'gzip, deflate, br', 'Cache-Control': 'no-cache', 'DNT': '1', 'Referer': origin || '', 'Sec-Fetch-Dest': 'document', 'Sec-Fetch-Mode': 'navigate', 'Sec-Fetch-Site': 'none', 'Sec-Fetch-User': '?1', 'Pragma': 'no-cache', }; } // ===== 超时 fetch ===== export async function fetchWithTimeout( url: string, options: RequestInit = {}, timeoutMs = 20_000, ): Promise { const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), timeoutMs); try { return await fetch(url, { ...options, signal: controller.signal }); } finally { clearTimeout(timer); } } // ===== URL 标准化(去重用) ===== /** * H-6 增强: URL 标准化用于搜索结果去重 * * 规范推荐使用 normalize-url 库(@see docs/Agent网络工具通用设计-v2.md §6.1.3), * 但当前实现已覆盖核心场景,且避免 ESM-only 依赖兼容性风险, * 故在现有基础上增强以下能力(对标 normalize-url 默认行为): * 1. 去除追踪参数(utm_*, gclid, fbclid) * 2. 强制小写 host * 3. 去除尾部斜杠(根路径除外) * 4. H-6 新增: 去除默认端口(http→:80, https→:443) * 5. H-6 新增: 排序查询参数(避免 ?a=1&b=2 vs ?b=2&a=1 被视为不同 URL) */ export function normalizeUrl(url: string): string { try { const u = new URL(url); // 去除追踪参数 const trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'gclid', 'fbclid']; for (const p of trackingParams) u.searchParams.delete(p); // H-6 增强: 排序查询参数(确保参数顺序一致,便于去重) // searchParams.sort() 原地排序,URLSearchParams 按码点顺序 u.searchParams.sort(); // 去除尾部斜杠(根路径除外) let path = u.pathname; if (path.length > 1 && path.endsWith('/')) path = path.slice(0, -1); // H-6 增强: 去除默认端口 // http 默认 :80, https 默认 :443, ws 默认 :80, wss 默认 :443 const isDefaultPort = (u.protocol === 'http:' && u.port === '80') || (u.protocol === 'https:' && u.port === '443') || (u.protocol === 'ws:' && u.port === '80') || (u.protocol === 'wss:' && u.port === '443'); const portSuffix = isDefaultPort ? '' : (u.port ? `:${u.port}` : ''); // 强制小写 host return `${u.protocol}//${u.hostname.toLowerCase()}${portSuffix}${path}${u.search}${u.hash}`; } catch { return url; } } // ===== 拦截页面检测 ===== const INTERCEPTION_PATTERNS = [ /just a moment/i, /attention required/i, /challenge-platform/i, /\.cf-challenge-/i, /access denied/i, /403 forbidden/i, /请启用\s*javascript/i, /please enable javascript/i, /checking your browser/i, /ddos protection/i, ]; export function isInterceptedPage(html: string): boolean { if (html.length < 80) return true; const lower = html.toLowerCase(); return INTERCEPTION_PATTERNS.some((p) => p.test(lower)); } // ===== HTML → 纯文本转换 ===== const HTML_ENTITY_MAP: Record = { ' ': ' ', '<': '<', '>': '>', '&': '&', '"': '"', ''': "'", '…': '…', '—': '—', '–': '–', '«': '«', '»': '»', '×': '×', '÷': '÷', '©': '©', '®': '®', '™': '™', '€': '€', '£': '£', '¥': '¥', '¢': '¢', '°': '°', }; export function htmlToText(html: string): string { return html // 移除噪声标签及内容 .replace(/]*>[\s\S]*?<\/script>/gi, '') .replace(/]*>[\s\S]*?<\/style>/gi, '') .replace(/]*>[\s\S]*?<\/noscript>/gi, '') .replace(/]*>[\s\S]*?<\/nav>/gi, '') .replace(/]*>[\s\S]*?<\/header>/gi, '') .replace(/]*>[\s\S]*?<\/footer>/gi, '') .replace(/]*>[\s\S]*?<\/aside>/gi, '') .replace(/]*>[\s\S]*?<\/iframe>/gi, '') .replace(/]*>[\s\S]*?<\/svg>/gi, '') // 移除 HTML 注释 .replace(//g, '') // 块级标签转换行 .replace(/<\/?(p|div|h[1-6]|li|tr|blockquote|section|article|pre|br|hr)[^>]*>/gi, '\n') // 表格单元格转制表符 .replace(/<\/?(td|th)[^>]*>/gi, '\t') // 移除剩余标签 .replace(/<[^>]+>/g, '') // 解码 HTML 实体 .replace(/&#(\d+);/g, (_, n) => String.fromCharCode(Number(n))) .replace(/&#x([0-9a-f]+);/gi, (_, h) => String.fromCharCode(parseInt(h, 16))) .replace(/&[a-z]+;/gi, (m) => HTML_ENTITY_MAP[m.toLowerCase()] ?? m) // 清理空白 .replace(/\n{3,}/g, '\n\n') .replace(/[ \t]+/g, ' ') .replace(/^[ \t]+/gm, '') .trim(); } // ===== 流式读取(大文件保护,10MB 上限) ===== export async function readBodyWithLimit(response: Response, maxBytes = 10 * 1024 * 1024): Promise { const contentLength = response.headers.get('content-length'); if (contentLength && parseInt(contentLength) > maxBytes) { throw new Error(`Response too large: ${contentLength} bytes (limit: ${maxBytes})`); } const reader = response.body?.getReader(); if (!reader) return ''; const chunks: Uint8Array[] = []; let totalBytes = 0; try { while (true) { const { done, value } = await reader.read(); if (done) break; if (value) { totalBytes += value.byteLength; if (totalBytes > maxBytes) { throw new Error(`Response exceeded ${maxBytes} bytes limit`); } chunks.push(value); } } } finally { reader.releaseLock(); } const decoder = new TextDecoder('utf-8', { fatal: false }); return decoder.decode(Buffer.concat(chunks)); } // ===== SearXNG 认证头构建 ===== export function buildSearXNGAuthHeaders(authKey: string, authType: string): Record { if (!authKey) return {}; if (authType === 'bearer') { return { Authorization: `Bearer ${authKey}` }; } if (authType === 'basic') { return { Authorization: `Basic ${Buffer.from(authKey).toString('base64')}` }; } return {}; } // ===== 日志辅助 ===== export function logTool(toolName: string, message: string): void { log.info(`[Tool:${toolName}] ${message}`); }