metona-ai-desktop/electron/harness/tools/built-in/web-fetch.ts

/**
 * web_fetch — 网页抓取工具
 *
 * 三阶段回退策略：
 *   Phase 1: HTTP 抓取（UA 轮换 + 反爬请求头 + 指数退避重试 + 拦截检测）
 *   Phase 2: 内容过短自动升级（< 200 字符 → 浏览器渲染）
 *   Phase 3: 浏览器回退（隐藏 BrowserWindow + JS 渲染 + 内容提取）
 *
 * @see docs/Agent网络工具通用设计-v2.md — 第 3 章 web_fetch 抓取设计
 */

import { BrowserWindow } from 'electron';
import type { IMetonaTool, ToolExecutionContext } from '../../types/metona-tool';
import type { MetonaToolDef } from '../../../harness/types';
import { MetonaToolCategory, MetonaRiskLevel } from '../../../harness/types';
import {
  fetchCache,
  buildAntiCrawlHeaders,
  fetchWithTimeout,
  htmlToText,
  isInterceptedPage,
  readBodyWithLimit,
  logTool,
} from './network-utils';

// ===== 跳过重试的状态码 =====

const SKIP_RETRY_STATUS = new Set([403, 429, 502, 503]);

// ===== WebFetchTool =====

export class WebFetchTool implements IMetonaTool {
  readonly definition: MetonaToolDef = {
    name: 'web_fetch',
    description: 'Fetch a web page and convert to plain text. Uses a three-phase fallback strategy: HTTP fetch with anti-crawl headers → SPA auto-upgrade → browser rendering. Handles Cloudflare interception, JavaScript-rendered pages, and large files (10MB limit).',
    parameters: {
      type: 'object',
      properties: {
        url: { type: 'string', description: 'Target URL (http/https only)' },
        mobile_ua: { type: 'boolean', description: 'Use mobile User-Agent (default false)' },
        retry: { type: 'boolean', description: 'Enable retry with exponential backoff (default true)' },
      },
      required: ['url'],
    },
    category: MetonaToolCategory.NETWORK,
    riskLevel: MetonaRiskLevel.LOW,
    requiresPermission: false,
    timeoutMs: 120_000,
  };

  async execute(args: Record<string, unknown>, _context: ToolExecutionContext): Promise<unknown> {
    const url = args.url as string;
    const mobileUA = (args.mobile_ua as boolean) ?? false;
    const enableRetry = (args.retry as boolean) ?? true;

    if (!url || !/^https?:\/\//i.test(url)) {
      return { url, content: '', success: false, error: 'URL must start with http:// or https://' };
    }

    logTool('web_fetch', `Fetching: ${url}`);

    // ===== Phase 1: HTTP 抓取 =====
    const phase1Result = await this.httpFetch(url, mobileUA, enableRetry);

    if (phase1Result.success && !phase1Result.intercepted) {
      // 内容过短检测 → Phase 2 升级
      if (phase1Result.text.length < 200) {
        logTool('web_fetch', `Phase 2: Content too short (${phase1Result.text.length} chars), upgrading to browser`);
        const browserResult = await this.browserFetch(url);
        if (browserResult) {
          return this.buildSuccess(url, browserResult, 'browser');
        }
      }
      return this.buildSuccess(url, phase1Result.text, 'http');
    }

    // ===== Phase 3: 浏览器回退 =====
    logTool('web_fetch', `Phase 3: Falling back to browser (${phase1Result.reason})`);
    const browserResult = await this.browserFetch(url);
    if (browserResult) {
      return this.buildSuccess(url, browserResult, 'browser');
    }

    // 全部失败
    return {
      url,
      content: '',
      success: false,
      error: `All phases failed. HTTP: ${phase1Result.reason}. Browser fallback also failed.`,
    };
  }

  // ===== Phase 1: HTTP 抓取 =====

  private async httpFetch(
    url: string,
    mobileUA: boolean,
    enableRetry: boolean,
  ): Promise<{ success: boolean; text: string; intercepted: boolean; reason: string }> {
    const maxRetries = enableRetry ? 3 : 1;
    const backoffBase = 2_000;

    for (let attempt = 0; attempt < maxRetries; attempt++) {
      try {
        const headers = buildAntiCrawlHeaders(url, attempt, mobileUA);
        const response = await fetchWithTimeout(url, { headers, redirect: 'follow' }, 20_000);

        // 跳过重试的状态码 → 直接进入浏览器回退
        if (SKIP_RETRY_STATUS.has(response.status)) {
          return { success: false, text: '', intercepted: true, reason: `HTTP ${response.status}` };
        }

        if (!response.ok) {
          // 5xx 可重试
          if (response.status >= 500 && attempt < maxRetries - 1) {
            await this.sleep(backoffBase * Math.pow(2, attempt) + Math.random() * backoffBase * 0.6);
            continue;
          }
          return { success: false, text: '', intercepted: false, reason: `HTTP ${response.status} ${response.statusText}` };
        }

        // 读取正文（10MB 限制）
        const html = await readBodyWithLimit(response, 10 * 1024 * 1024);

        // 拦截检测
        if (isInterceptedPage(html)) {
          return { success: false, text: '', intercepted: true, reason: 'Intercepted page detected' };
        }

        // HTML → 纯文本
        const text = htmlToText(html);
        return { success: true, text, intercepted: false, reason: '' };
      } catch (err) {
        const errorMsg = (err as Error).message;
        if (attempt < maxRetries - 1) {
          logTool('web_fetch', `Attempt ${attempt + 1} failed: ${errorMsg}, retrying...`);
          await this.sleep(backoffBase * Math.pow(2, attempt) + Math.random() * backoffBase * 0.6);
          continue;
        }
        return { success: false, text: '', intercepted: false, reason: errorMsg };
      }
    }

    return { success: false, text: '', intercepted: false, reason: 'All retries exhausted' };
  }

  // ===== Phase 2/3: 浏览器回退 =====

  private async browserFetch(url: string): Promise<string | null> {
    // 查缓存
    const cached = fetchCache.get(url);
    if (cached) {
      logTool('web_fetch', 'Browser cache hit');
      return cached;
    }

    let win: BrowserWindow | null = null;
    let cleanup: (() => void) | null = null;

    try {
      win = new BrowserWindow({
        width: 1280,
        height: 800,
        show: false,
        webPreferences: {
          nodeIntegration: false,
          contextIsolation: true,
          sandbox: true,
          webSecurity: false, // 允许跨域（截图需要）
        },
      });

      cleanup = () => {
        try {
          if (win && !win.isDestroyed()) win.close();
        } catch {
          // 忽略关闭错误
        }
      };

      await this.loadURLWithTimeout(win, url, 30_000);

      // 等待 JS 渲染
      await this.sleep(2_500);

      // 提取页面正文
      const text = await win.webContents.executeJavaScript(`
        (function() {
          var clone = document.body.cloneNode(true);
          var noise = clone.querySelectorAll('script, style, noscript, nav, header, footer, aside, iframe, svg');
          noise.forEach(function(el) { el.remove(); });
          return clone.innerText || '';
        })();
      `, true);

      if (text && text.trim().length >= 80) {
        // 写缓存
        fetchCache.set(url, text);
        logTool('web_fetch', `Browser fetch success: ${text.length} chars`);
        return text;
      }

      return null;
    } catch (err) {
      logTool('web_fetch', `Browser fetch failed: ${(err as Error).message}`);
      return null;
    } finally {
      if (cleanup) cleanup();
    }
  }

  // ===== 辅助方法 =====

  private buildSuccess(url: string, text: string, method: string): unknown {
    return {
      url,
      content: text,
      success: true,
      method,
      length: text.length,
    };
  }

  private sleep(ms: number): Promise<void> {
    return new Promise((resolve) => setTimeout(resolve, ms));
  }

  /** 带超时的 loadURL（Electron 原生不支持 timeout 选项） */
  private async loadURLWithTimeout(win: BrowserWindow, url: string, timeoutMs: number): Promise<void> {
    let timer: NodeJS.Timeout | null = null;
    const timeoutPromise = new Promise<never>((_, reject) => {
      timer = setTimeout(() => reject(new Error(`Page load timeout after ${timeoutMs}ms: ${url}`)), timeoutMs);
    });
    try {
      await Promise.race([win.loadURL(url), timeoutPromise]);
    } finally {
      if (timer) clearTimeout(timer);
    }
  }
}