metona-ai-desktop/electron/harness/tools/built-in/network-utils.ts

/**
 * 网络工具共享模块
 *
 * 提供 UA 轮换池、反爬请求头、HTML 转文本、拦截检测、超时 fetch、LRU 缓存等通用能力。
 * web_search 和 web_fetch 共享此模块。
 *
 * @see docs/Agent网络工具通用设计-v2.md — 第 3 章 web_fetch 抓取设计
 */

import { LRUCache } from 'lru-cache';
import log from 'electron-log';

// ===== LRU 缓存 =====

/** 搜索结果缓存（200 条，5 分钟 TTL） */
export const searchCache = new LRUCache<string, Record<string, unknown>>({
  max: 200,
  ttl: 300_000,
});

/** 浏览器回退缓存（100 条，10 分钟 TTL） */
export const fetchCache = new LRUCache<string, string>({
  max: 100,
  ttl: 600_000,
});

// ===== UA 轮换池 =====

export const UA_POOL = [
  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Edg/131.0.0.0 Safari/537.36',
  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.1 Safari/605.1.15',
  'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0',
];

export const MOBILE_UA = 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1';

export const ACCEPT_LANGUAGE_POOL = [
  'zh-CN,zh;q=0.9,en;q=0.8',
  'zh-CN,zh;q=0.9',
  'en-US,en;q=0.9,zh-CN;q=0.8',
];

// ===== 反爬请求头构建 =====

export function buildAntiCrawlHeaders(
  url: string,
  attempt: number,
  mobileUA = false,
): Record<string, string> {
  const uaIdx = attempt % UA_POOL.length;
  const langIdx = attempt % ACCEPT_LANGUAGE_POOL.length;
  const userAgent = mobileUA ? MOBILE_UA : UA_POOL[uaIdx];

  let origin = '';
  try { origin = new URL(url).origin; } catch { /* ignore */ }

  return {
    'User-Agent': userAgent,
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
    'Accept-Language': ACCEPT_LANGUAGE_POOL[langIdx],
    'Accept-Encoding': 'gzip, deflate, br',
    'Cache-Control': 'no-cache',
    'DNT': '1',
    'Referer': origin || '',
    'Sec-Fetch-Dest': 'document',
    'Sec-Fetch-Mode': 'navigate',
    'Sec-Fetch-Site': 'none',
    'Sec-Fetch-User': '?1',
    'Pragma': 'no-cache',
  };
}

// ===== 超时 fetch =====

export async function fetchWithTimeout(
  url: string,
  options: RequestInit = {},
  timeoutMs = 20_000,
): Promise<Response> {
  const controller = new AbortController();
  const timer = setTimeout(() => controller.abort(), timeoutMs);
  try {
    return await fetch(url, { ...options, signal: controller.signal });
  } finally {
    clearTimeout(timer);
  }
}

// ===== URL 标准化（去重用） =====

/**
 * H-6 增强: URL 标准化用于搜索结果去重
 *
 * 规范推荐使用 normalize-url 库（@see docs/Agent网络工具通用设计-v2.md §6.1.3），
 * 但当前实现已覆盖核心场景，且避免 ESM-only 依赖兼容性风险，
 * 故在现有基础上增强以下能力（对标 normalize-url 默认行为）：
 * 1. 去除追踪参数（utm_*, gclid, fbclid）
 * 2. 强制小写 host
 * 3. 去除尾部斜杠（根路径除外）
 * 4. H-6 新增: 去除默认端口（http→:80, https→:443）
 * 5. H-6 新增: 排序查询参数（避免 ?a=1&b=2 vs ?b=2&a=1 被视为不同 URL）
 */
export function normalizeUrl(url: string): string {
  try {
    const u = new URL(url);

    // 去除追踪参数
    const trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'gclid', 'fbclid'];
    for (const p of trackingParams) u.searchParams.delete(p);

    // H-6 增强: 排序查询参数（确保参数顺序一致，便于去重）
    // searchParams.sort() 原地排序，URLSearchParams 按码点顺序
    u.searchParams.sort();

    // 去除尾部斜杠（根路径除外）
    let path = u.pathname;
    if (path.length > 1 && path.endsWith('/')) path = path.slice(0, -1);

    // H-6 增强: 去除默认端口
    // http 默认 :80, https 默认 :443, ws 默认 :80, wss 默认 :443
    const isDefaultPort =
      (u.protocol === 'http:' && u.port === '80') ||
      (u.protocol === 'https:' && u.port === '443') ||
      (u.protocol === 'ws:' && u.port === '80') ||
      (u.protocol === 'wss:' && u.port === '443');
    const portSuffix = isDefaultPort ? '' : (u.port ? `:${u.port}` : '');

    // 强制小写 host
    return `${u.protocol}//${u.hostname.toLowerCase()}${portSuffix}${path}${u.search}${u.hash}`;
  } catch {
    return url;
  }
}

// ===== 拦截页面检测 =====

const INTERCEPTION_PATTERNS = [
  /just a moment/i,
  /attention required/i,
  /challenge-platform/i,
  /\.cf-challenge-/i,
  /access denied/i,
  /403 forbidden/i,
  /请启用\s*javascript/i,
  /please enable javascript/i,
  /checking your browser/i,
  /ddos protection/i,
];

export function isInterceptedPage(html: string): boolean {
  if (html.length < 80) return true;
  const lower = html.toLowerCase();
  return INTERCEPTION_PATTERNS.some((p) => p.test(lower));
}

// ===== HTML → 纯文本转换 =====

const HTML_ENTITY_MAP: Record<string, string> = {
  '&nbsp;': ' ', '&lt;': '<', '&gt;': '>', '&amp;': '&', '&quot;': '"',
  '&apos;': "'", '&hellip;': '…', '&mdash;': '—', '&ndash;': '–',
  '&laquo;': '«', '&raquo;': '»', '&times;': '×', '&divide;': '÷',
  '&copy;': '©', '&reg;': '®', '&trade;': '™', '&euro;': '€',
  '&pound;': '£', '&yen;': '¥', '&cent;': '¢', '&deg;': '°',
};

export function htmlToText(html: string): string {
  return html
    // 移除噪声标签及内容
    .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
    .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
    .replace(/<noscript[^>]*>[\s\S]*?<\/noscript>/gi, '')
    .replace(/<nav[^>]*>[\s\S]*?<\/nav>/gi, '')
    .replace(/<header[^>]*>[\s\S]*?<\/header>/gi, '')
    .replace(/<footer[^>]*>[\s\S]*?<\/footer>/gi, '')
    .replace(/<aside[^>]*>[\s\S]*?<\/aside>/gi, '')
    .replace(/<iframe[^>]*>[\s\S]*?<\/iframe>/gi, '')
    .replace(/<svg[^>]*>[\s\S]*?<\/svg>/gi, '')
    // 移除 HTML 注释
    .replace(/<!--[\s\S]*?-->/g, '')
    // 块级标签转换行
    .replace(/<\/?(p|div|h[1-6]|li|tr|blockquote|section|article|pre|br|hr)[^>]*>/gi, '\n')
    // 表格单元格转制表符
    .replace(/<\/?(td|th)[^>]*>/gi, '\t')
    // 移除剩余标签
    .replace(/<[^>]+>/g, '')
    // 解码 HTML 实体
    .replace(/&#(\d+);/g, (_, n) => String.fromCharCode(Number(n)))
    .replace(/&#x([0-9a-f]+);/gi, (_, h) => String.fromCharCode(parseInt(h, 16)))
    .replace(/&[a-z]+;/gi, (m) => HTML_ENTITY_MAP[m.toLowerCase()] ?? m)
    // 清理空白
    .replace(/\n{3,}/g, '\n\n')
    .replace(/[ \t]+/g, ' ')
    .replace(/^[ \t]+/gm, '')
    .trim();
}

// ===== 流式读取（大文件保护，10MB 上限） =====

export async function readBodyWithLimit(response: Response, maxBytes = 10 * 1024 * 1024): Promise<string> {
  const contentLength = response.headers.get('content-length');
  if (contentLength && parseInt(contentLength) > maxBytes) {
    throw new Error(`Response too large: ${contentLength} bytes (limit: ${maxBytes})`);
  }

  const reader = response.body?.getReader();
  if (!reader) return '';

  const chunks: Uint8Array[] = [];
  let totalBytes = 0;

  try {
    while (true) {
      const { done, value } = await reader.read();
      if (done) break;
      if (value) {
        totalBytes += value.byteLength;
        if (totalBytes > maxBytes) {
          throw new Error(`Response exceeded ${maxBytes} bytes limit`);
        }
        chunks.push(value);
      }
    }
  } finally {
    reader.releaseLock();
  }

  const decoder = new TextDecoder('utf-8', { fatal: false });
  return decoder.decode(Buffer.concat(chunks));
}

// ===== SearXNG 认证头构建 =====

export function buildSearXNGAuthHeaders(authKey: string, authType: string): Record<string, string> {
  if (!authKey) return {};
  if (authType === 'bearer') {
    return { Authorization: `Bearer ${authKey}` };
  }
  if (authType === 'basic') {
    return { Authorization: `Basic ${Buffer.from(authKey).toString('base64')}` };
  }
  return {};
}

// ===== 日志辅助 =====

export function logTool(toolName: string, message: string): void {
  log.info(`[Tool:${toolName}] ${message}`);
}