/**
* 网络工具共享模块
*
* 提供 UA 轮换池、反爬请求头、HTML 转文本、拦截检测、超时 fetch、LRU 缓存等通用能力。
* web_search 和 web_fetch 共享此模块。
*
* @see docs/Agent网络工具通用设计-v2.md — 第 3 章 web_fetch 抓取设计
*/
import { LRUCache } from 'lru-cache';
import log from 'electron-log';
// ===== LRU 缓存 =====
/** 搜索结果缓存(200 条,5 分钟 TTL) */
export const searchCache = new LRUCache>({
max: 200,
ttl: 300_000,
});
/** 浏览器回退缓存(100 条,10 分钟 TTL) */
export const fetchCache = new LRUCache({
max: 100,
ttl: 600_000,
});
// ===== UA 轮换池 =====
export const UA_POOL = [
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Edg/131.0.0.0 Safari/537.36',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.1 Safari/605.1.15',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0',
];
export const MOBILE_UA = 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1';
export const ACCEPT_LANGUAGE_POOL = [
'zh-CN,zh;q=0.9,en;q=0.8',
'zh-CN,zh;q=0.9',
'en-US,en;q=0.9,zh-CN;q=0.8',
];
// ===== 反爬请求头构建 =====
export function buildAntiCrawlHeaders(
url: string,
attempt: number,
mobileUA = false,
): Record {
const uaIdx = attempt % UA_POOL.length;
const langIdx = attempt % ACCEPT_LANGUAGE_POOL.length;
const userAgent = mobileUA ? MOBILE_UA : UA_POOL[uaIdx];
let origin = '';
try { origin = new URL(url).origin; } catch { /* ignore */ }
return {
'User-Agent': userAgent,
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
'Accept-Language': ACCEPT_LANGUAGE_POOL[langIdx],
'Accept-Encoding': 'gzip, deflate, br',
'Cache-Control': 'no-cache',
'DNT': '1',
'Referer': origin || '',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'none',
'Sec-Fetch-User': '?1',
'Pragma': 'no-cache',
};
}
// ===== 超时 fetch =====
export async function fetchWithTimeout(
url: string,
options: RequestInit = {},
timeoutMs = 20_000,
): Promise {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), timeoutMs);
try {
return await fetch(url, { ...options, signal: controller.signal });
} finally {
clearTimeout(timer);
}
}
// ===== URL 标准化(去重用) =====
/**
* H-6 增强: URL 标准化用于搜索结果去重
*
* 规范推荐使用 normalize-url 库(@see docs/Agent网络工具通用设计-v2.md §6.1.3),
* 但当前实现已覆盖核心场景,且避免 ESM-only 依赖兼容性风险,
* 故在现有基础上增强以下能力(对标 normalize-url 默认行为):
* 1. 去除追踪参数(utm_*, gclid, fbclid)
* 2. 强制小写 host
* 3. 去除尾部斜杠(根路径除外)
* 4. H-6 新增: 去除默认端口(http→:80, https→:443)
* 5. H-6 新增: 排序查询参数(避免 ?a=1&b=2 vs ?b=2&a=1 被视为不同 URL)
*/
export function normalizeUrl(url: string): string {
try {
const u = new URL(url);
// 去除追踪参数
const trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'gclid', 'fbclid'];
for (const p of trackingParams) u.searchParams.delete(p);
// H-6 增强: 排序查询参数(确保参数顺序一致,便于去重)
// searchParams.sort() 原地排序,URLSearchParams 按码点顺序
u.searchParams.sort();
// 去除尾部斜杠(根路径除外)
let path = u.pathname;
if (path.length > 1 && path.endsWith('/')) path = path.slice(0, -1);
// H-6 增强: 去除默认端口
// http 默认 :80, https 默认 :443, ws 默认 :80, wss 默认 :443
const isDefaultPort =
(u.protocol === 'http:' && u.port === '80') ||
(u.protocol === 'https:' && u.port === '443') ||
(u.protocol === 'ws:' && u.port === '80') ||
(u.protocol === 'wss:' && u.port === '443');
const portSuffix = isDefaultPort ? '' : (u.port ? `:${u.port}` : '');
// 强制小写 host
return `${u.protocol}//${u.hostname.toLowerCase()}${portSuffix}${path}${u.search}${u.hash}`;
} catch {
return url;
}
}
// ===== 拦截页面检测 =====
const INTERCEPTION_PATTERNS = [
/just a moment/i,
/attention required/i,
/challenge-platform/i,
/\.cf-challenge-/i,
/access denied/i,
/403 forbidden/i,
/请启用\s*javascript/i,
/please enable javascript/i,
/checking your browser/i,
/ddos protection/i,
];
export function isInterceptedPage(html: string): boolean {
if (html.length < 80) return true;
const lower = html.toLowerCase();
return INTERCEPTION_PATTERNS.some((p) => p.test(lower));
}
// ===== HTML → 纯文本转换 =====
const HTML_ENTITY_MAP: Record = {
' ': ' ', '<': '<', '>': '>', '&': '&', '"': '"',
''': "'", '…': '…', '—': '—', '–': '–',
'«': '«', '»': '»', '×': '×', '÷': '÷',
'©': '©', '®': '®', '™': '™', '€': '€',
'£': '£', '¥': '¥', '¢': '¢', '°': '°',
};
export function htmlToText(html: string): string {
return html
// 移除噪声标签及内容
.replace(/