/**
* web_fetch 三阶段回退工具测试（v0.7.5 新建覆盖）
*
* 通过 mock ssrf-dispatcher（ssrfPinnedFetch/resolveRedirectTarget）、
* ssrf-guard（validateSSRF）与 browser（getBrowserManager）锁定：
* - Phase1 HTTP 成功（text/html/markdown 三种 extract_mode）
* - Phase2 内容过短自动升级浏览器
* - Phase3 浏览器回退 / 拦截页检测 / 重定向逐跳 / blocked 不进回退
* - 缓存命中 / max_chars 截断 / 10MB 闸门 / retry 语义
*
* 注意：HTML 夹具必须 > 80 字符（绕过 isInterceptedPage 空壳判定），
* text 模式正文必须 >= 200 字符（避免 Phase2 自动升级）。
*/
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
vi.mock('electron-log', () => ({
default: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() },
}));
const ssrfMock = vi.hoisted(() => ({
validateSSRF: vi.fn(async () => undefined),
}));
vi.mock('../ssrf-guard', () => ({
validateSSRF: ssrfMock.validateSSRF,
}));
const dispatcherMock = vi.hoisted(() => ({
ssrfPinnedFetch: vi.fn(),
// 类型签名与 ssrf-dispatcher.ts 的 resolveRedirectTarget 一致（string | null），
// 使测试可通过 mockReturnValueOnce 注入下一跳 URL。
resolveRedirectTarget: vi.fn((_response: unknown, _url: string): string | null => null),
}));
vi.mock('../ssrf-dispatcher', () => ({
ssrfPinnedFetch: dispatcherMock.ssrfPinnedFetch,
resolveRedirectTarget: dispatcherMock.resolveRedirectTarget,
}));
const browserMock = vi.hoisted(() => ({
getBrowserManager: vi.fn(),
}));
vi.mock('../browser', () => ({
getBrowserManager: browserMock.getBrowserManager,
}));
import { WebFetchTool } from '../web-fetch';
import { fetchCache } from '../network-utils';
import type { ToolExecutionContext } from '../../../types/metona-tool';
const context: ToolExecutionContext = {
sessionId: 't',
workspacePath: process.cwd(),
iteration: 1,
requestId: 'r',
};
/** 生成足够长（> 80 字符）避免空壳拦截判定的 HTML 页面 */
function page(body: string, status = 200): Response {
const html = `${body}Padding text to exceed the minimum shell detection threshold of eighty characters in total length.
`;
return new Response(html, { status, headers: { 'Content-Type': 'text/html' } });
}
/** 生成文本内容 >= 200 字符的页面（避免 Phase2 升级） */
function longTextPage(text: string, status = 200): Response {
const body = `${text}
${'padding-'.repeat(30)}
`;
return page(body, status);
}
/** 生成带 Location 头的重定向响应 */
function redirectResponse(location: string, status = 302): Response {
return new Response('', { status, headers: { Location: location } });
}
/** 每次调用生成全新 Response（避免 body 消费后复用报错） */
function mockFetchWith(factory: () => Response) {
dispatcherMock.ssrfPinnedFetch.mockImplementation(async () => factory());
}
interface FetchResult {
success: boolean;
error?: string;
content?: string;
method?: string;
truncated?: boolean;
original_length?: number;
extract_mode?: string;
}
function resetAllMocks(): void {
vi.clearAllMocks();
ssrfMock.validateSSRF.mockReset().mockImplementation(async () => undefined);
dispatcherMock.ssrfPinnedFetch.mockReset();
dispatcherMock.resolveRedirectTarget.mockReset().mockReturnValue(null);
browserMock.getBrowserManager.mockReset().mockReturnValue({
fetchPageText: vi.fn(async () => 'browser rendered content '.repeat(30)),
});
}
describe('web_fetch — URL 与 SSRF 入口', () => {
let tool: WebFetchTool;
beforeEach(() => {
tool = new WebFetchTool();
fetchCache.clear();
resetAllMocks();
});
afterEach(() => vi.clearAllMocks());
it('非 http/https URL → 拒绝', async () => {
const r = (await tool.execute({ url: 'file:///etc/passwd' }, context)) as FetchResult;
expect(r.success).toBe(false);
expect(String(r.error)).toContain('URL must start with');
});
it('缺 url → 拒绝', async () => {
const r = (await tool.execute({}, context)) as FetchResult;
expect(r.success).toBe(false);
});
it('SSRF 校验失败 → 拒绝且不进入浏览器回退', async () => {
ssrfMock.validateSSRF.mockRejectedValueOnce(
new Error('Blocked SSRF: private/loopback address'),
);
const r = (await tool.execute({ url: 'http://127.0.0.1:1/x' }, context)) as FetchResult;
expect(r.success).toBe(false);
expect(String(r.error)).toContain('Blocked SSRF');
expect(dispatcherMock.ssrfPinnedFetch).not.toHaveBeenCalled();
expect(browserMock.getBrowserManager).not.toHaveBeenCalled();
});
it('SSRF 校验异常信息直接回传', async () => {
ssrfMock.validateSSRF.mockRejectedValueOnce(new Error('Blocked SSRF: no DNS records'));
const r = (await tool.execute({ url: 'http://nx.test/' }, context)) as FetchResult;
expect(r.success).toBe(false);
expect(String(r.error)).toContain('no DNS records');
});
});
describe('web_fetch — Phase1 HTTP 抓取', () => {
let tool: WebFetchTool;
beforeEach(() => {
tool = new WebFetchTool();
fetchCache.clear();
resetAllMocks();
});
afterEach(() => vi.clearAllMocks());
it('text 模式：成功解析 HTML 为纯文本', async () => {
mockFetchWith(() => longTextPage('标题段落与正文内容 ABC'));
const r = (await tool.execute({ url: 'https://ok.test/page' }, context)) as FetchResult;
expect(r.success).toBe(true);
expect(r.method).toBe('http');
expect(String(r.content)).toContain('标题段落与正文内容 ABC');
expect(r.extract_mode).toBe('text');
});
it('html 模式：返回原始 HTML 原文（实况契约：原文透传不做清理）', async () => {
mockFetchWith(() => page('keep
'));
const r = (await tool.execute(
{ url: 'https://ok.test/page', extract_mode: 'html' },
context,
)) as FetchResult;
expect(r.success).toBe(true);
expect(r.method).toBe('http');
// 实况契约：html 模式直接返回 readBodyWithLimit 原文
expect(String(r.content)).toContain('keep
');
expect(String(r.content)).toContain('');
expect(r.extract_mode).toBe('html');
});
it('markdown 模式：结构化转换（标题 ATX + 链接）', async () => {
mockFetchWith(() => page('MD 标题
链接正文
'));
const r = (await tool.execute(
{ url: 'https://ok.test/md', extract_mode: 'markdown' },
context,
)) as FetchResult;
expect(r.success).toBe(true);
expect(r.method).toBe('http');
expect(String(r.content)).toContain('# MD 标题');
expect(String(r.content)).toContain('[链接](https://x.test/)');
expect(r.extract_mode).toBe('markdown');
});
it('拦截页检测（Cloudflare）→ 触发浏览器回退', async () => {
mockFetchWith(
() => new Response('Attention Required! | Cloudflare', { status: 200 }),
);
browserMock.getBrowserManager.mockReturnValue({
fetchPageText: vi.fn(async () => 'browser ok content '.repeat(30)),
});
const r = (await tool.execute({ url: 'https://cf.test/' }, context)) as FetchResult;
expect(r.success).toBe(true);
expect(r.method).toBe('browser');
expect(String(r.content)).toContain('browser ok content');
});
it('重定向逐跳跟随（resolveRedirectTarget 返回下一跳 → 再次请求）', async () => {
dispatcherMock.resolveRedirectTarget
.mockReturnValueOnce('https://redirected.test/final')
.mockReturnValue(null);
dispatcherMock.ssrfPinnedFetch
.mockResolvedValueOnce(redirectResponse('https://redirected.test/final'))
.mockImplementation(async () => longTextPage('最终页内容足够长以绕过拦截与升级判定'));
const r = (await tool.execute({ url: 'https://start.test/old' }, context)) as FetchResult;
expect(r.success).toBe(true);
expect(r.method).toBe('http');
expect(dispatcherMock.ssrfPinnedFetch).toHaveBeenCalledTimes(2);
expect(String(r.content)).toContain('最终页内容');
});
it('重定向目标 SSRF 被拦 → blocked 且不进浏览器回退', async () => {
dispatcherMock.resolveRedirectTarget.mockReturnValue('http://169.254.169.254/latest/meta-data');
mockFetchWith(() => redirectResponse('http://169.254.169.254/latest/meta-data'));
ssrfMock.validateSSRF
.mockResolvedValueOnce(undefined)
.mockRejectedValueOnce(new Error('Blocked SSRF: link-local'));
const r = (await tool.execute({ url: 'https://evil.test/redirect' }, context)) as FetchResult;
expect(r.success).toBe(false);
expect(String(r.error)).toContain('Redirect target blocked by SSRF guard');
expect(browserMock.getBrowserManager).not.toHaveBeenCalled();
});
it('超过 5 跳重定向 → Phase1 失败（Too many redirects），浏览器也失败时整体失败', async () => {
dispatcherMock.resolveRedirectTarget.mockReturnValue('https://loop.test/next');
mockFetchWith(() => redirectResponse('https://loop.test/next'));
browserMock.getBrowserManager.mockReturnValue({ fetchPageText: vi.fn(async () => null) });
const r = (await tool.execute({ url: 'https://loop.test/start' }, context)) as FetchResult;
expect(r.success).toBe(false);
expect(String(r.error)).toContain('Too many redirects');
});
it('HTTP 404 → Phase3 浏览器回退', async () => {
mockFetchWith(() => page('nope', 404));
browserMock.getBrowserManager.mockReturnValue({
fetchPageText: vi.fn(async () => 'fallback text '.repeat(30)),
});
const r = (await tool.execute({ url: 'https://miss.test/' }, context)) as FetchResult;
expect(r.success).toBe(true);
expect(r.method).toBe('browser');
});
it('HTTP 500 重试后仍失败 → 浏览器回退', async () => {
// v0.7.4 回归修复: fake timers 推进指数退避（原实现真实等待 6s+，CI 脆弱）。
// 不调用 restoreAllMocks（会清掉 beforeEach 的 mock），手动还原 Math.random。
mockFetchWith(() => page('err', 500));
browserMock.getBrowserManager.mockReturnValue({
fetchPageText: vi.fn(async () => 'browser rescue '.repeat(30)),
});
const randomSpy = vi.spyOn(Math, 'random').mockReturnValue(0.5);
vi.useFakeTimers();
try {
const promise = tool.execute({ url: 'https://fail.test/' }, context);
// 推进退避总时长（2s+1.2s + 4s+2.4s = 9.6s）
await vi.advanceTimersByTimeAsync(12_000);
const r = (await promise) as FetchResult;
expect(r.success).toBe(true);
expect(r.method).toBe('browser');
} finally {
vi.useRealTimers();
randomSpy.mockRestore();
}
});
it('retry=false 时单次尝试（5xx 不重试直接失败→回退）', async () => {
mockFetchWith(() => page('err', 500));
browserMock.getBrowserManager.mockReturnValue({ fetchPageText: vi.fn(async () => null) });
const r = (await tool.execute(
{ url: 'https://noretry.test/', retry: false },
context,
)) as FetchResult;
expect(r.success).toBe(false);
expect(dispatcherMock.ssrfPinnedFetch).toHaveBeenCalledTimes(1);
expect(String(r.error)).toContain('All phases failed');
});
it('10MB 响应体超限 → 失败（readBodyWithLimit 闸门，retry=false 快路径）', async () => {
mockFetchWith(
() =>
new Response('x'.repeat(50), {
status: 200,
headers: { 'Content-Length': String(20 * 1024 * 1024) },
}),
);
browserMock.getBrowserManager.mockReturnValue({ fetchPageText: vi.fn(async () => null) });
const r = (await tool.execute(
{ url: 'https://big.test/', retry: false },
context,
)) as FetchResult;
expect(r.success).toBe(false);
expect(String(r.error)).toContain('All phases failed');
});
it('网络异常（fetch reject）→ 重试后浏览器回退', async () => {
// v0.7.4 回归修复: fake timers 推进指数退避（原实现真实等待 6s+）
dispatcherMock.ssrfPinnedFetch.mockRejectedValue(new Error('ECONNREFUSED'));
browserMock.getBrowserManager.mockReturnValue({
fetchPageText: vi.fn(async () => 'net rescue '.repeat(30)),
});
const randomSpy = vi.spyOn(Math, 'random').mockReturnValue(0.5);
vi.useFakeTimers();
try {
const promise = tool.execute({ url: 'https://net.test/' }, context);
await vi.advanceTimersByTimeAsync(12_000);
const r = (await promise) as FetchResult;
expect(r.success).toBe(true);
expect(r.method).toBe('browser');
} finally {
vi.useRealTimers();
randomSpy.mockRestore();
}
});
it('HTTP 429 → 直接进入浏览器回退（SKIP_RETRY）', async () => {
mockFetchWith(() => page('rate', 429));
browserMock.getBrowserManager.mockReturnValue({
fetchPageText: vi.fn(async () => 'rate rescue '.repeat(30)),
});
const r = (await tool.execute({ url: 'https://rate.test/' }, context)) as FetchResult;
expect(r.success).toBe(true);
expect(r.method).toBe('browser');
});
});
describe('web_fetch — Phase2 内容升级 / 缓存 / 截断', () => {
let tool: WebFetchTool;
beforeEach(() => {
tool = new WebFetchTool();
fetchCache.clear();
resetAllMocks();
});
afterEach(() => vi.clearAllMocks());
it('Phase2：text 内容 < 200 字符 → 升级浏览器渲染', async () => {
mockFetchWith(() => page('短内容
'));
browserMock.getBrowserManager.mockReturnValue({
fetchPageText: vi.fn(async () => 'browser rich content '.repeat(30)),
});
const r = (await tool.execute({ url: 'https://spa.test/' }, context)) as FetchResult;
expect(r.success).toBe(true);
expect(r.method).toBe('browser');
expect(String(r.content)).toContain('browser rich content');
});
it('Phase2 升级但浏览器也失败 → 返回 Phase2 的短内容（http 方法）', async () => {
mockFetchWith(() => page('短内容保留
'));
browserMock.getBrowserManager.mockReturnValue({ fetchPageText: vi.fn(async () => null) });
const r = (await tool.execute({ url: 'https://spa2.test/' }, context)) as FetchResult;
expect(r.success).toBe(true);
expect(r.method).toBe('http');
expect(String(r.content)).toContain('短内容保留');
});
it('html 模式不触发 Phase2 升级', async () => {
mockFetchWith(() => page('tiny
'));
const r = (await tool.execute(
{ url: 'https://html-tiny.test/', extract_mode: 'html' },
context,
)) as FetchResult;
expect(r.success).toBe(true);
expect(r.method).toBe('http');
expect(browserMock.getBrowserManager).not.toHaveBeenCalled();
});
it('缓存命中（第二次请求走 fetchCache，不发网络请求）', async () => {
mockFetchWith(() => longTextPage('缓存内容正文'));
await tool.execute({ url: 'https://cache.test/page' }, context);
const r = (await tool.execute({ url: 'https://cache.test/page' }, context)) as FetchResult;
expect(r.success).toBe(true);
expect(r.method).toBe('cache');
expect(dispatcherMock.ssrfPinnedFetch).toHaveBeenCalledTimes(1);
});
it('缓存键含 extract_mode：text 与 html 不同键', async () => {
mockFetchWith(() => page('键隔离正文内容足够长
'));
await tool.execute({ url: 'https://key.test/', extract_mode: 'text' }, context);
await tool.execute({ url: 'https://key.test/', extract_mode: 'html' }, context);
expect(dispatcherMock.ssrfPinnedFetch).toHaveBeenCalledTimes(2);
});
it('html 模式不写缓存（避免模式混淆）', async () => {
mockFetchWith(() => page('不缓存正文
'));
await tool.execute({ url: 'https://nocache.test/', extract_mode: 'html' }, context);
expect(fetchCache.has('html:https://nocache.test/')).toBe(false);
});
it('max_chars 截断内容并标记 truncated', async () => {
mockFetchWith(() => page(`${'x'.repeat(5000)}
`));
const r = (await tool.execute(
{ url: 'https://max.test/', max_chars: 100 },
context,
)) as FetchResult;
expect(r.success).toBe(true);
expect(r.truncated).toBe(true);
expect(String(r.content)).toContain('content truncated at 100 chars');
expect(Number(r.original_length)).toBeGreaterThan(100);
});
it('浏览器回退文本 > 500K 被截断', async () => {
mockFetchWith(() => page('', 404));
browserMock.getBrowserManager.mockReturnValue({
fetchPageText: vi.fn(async () => 'B'.repeat(600_000)),
});
const r = (await tool.execute({ url: 'https://bigbrowser.test/' }, context)) as FetchResult;
expect(r.success).toBe(true);
expect(String(r.content)).toContain('content truncated');
});
it('浏览器回退产出的拦截页 → 判为失败', async () => {
mockFetchWith(() => page('', 403));
browserMock.getBrowserManager.mockReturnValue({
fetchPageText: vi.fn(async () => 'Just a moment... verifying you are human'),
});
const r = (await tool.execute(
{ url: 'https://browser-intercepted.test/' },
context,
)) as FetchResult;
expect(r.success).toBe(false);
expect(String(r.error)).toContain('All phases failed');
});
it('浏览器回退文本 < 80 字符 → 判为失败', async () => {
mockFetchWith(() => page('', 404));
browserMock.getBrowserManager.mockReturnValue({
fetchPageText: vi.fn(async () => 'too short'),
});
const r = (await tool.execute({ url: 'https://short.test/' }, context)) as FetchResult;
expect(r.success).toBe(false);
});
it('浏览器回退文本写入缓存 → 第二次请求直接命中顶层缓存（method=cache）', async () => {
mockFetchWith(() => page('', 404));
browserMock.getBrowserManager.mockReturnValue({
fetchPageText: vi.fn(async () => 'browser cacheable content '.repeat(30)),
});
await tool.execute({ url: 'https://browser-cache.test/' }, context);
// 第二次：顶层 fetchCache（text:url 键，浏览器阶段写入）直接命中
const r = (await tool.execute({ url: 'https://browser-cache.test/' }, context)) as FetchResult;
expect(r.success).toBe(true);
expect(r.method).toBe('cache');
expect(String(r.content)).toContain('browser cacheable content');
// 浏览器不再被调用（缓存短路）
expect(browserMock.getBrowserManager).toHaveBeenCalledTimes(1);
});
});