Files
metona-ai-desktop/electron/harness/tools/built-in/__tests__/web-fetch.test.ts
T
thzxx 99d0c54129
CI / 类型检查 + Lint + 单元测试 (push) Failing after 6m27s
CI / 产物编译验证 (push) Successful in 9m57s
CI / 全量测试 (Electron ABI) (push) Failing after 5m19s
feat: v0.7.4 时序语义修正 · 防线实效补漏 · 全量测试翻倍 — 2406 用例 + jsdom 组件测试全量回归
P1 修复面收口:
- 超时三态区分(aborted→USER_INTERRUPT / ETIMEDOUT→TIMEOUT / 其余→ERROR),
  根治"真实网络超时被误报为用户中断"
- 流空闲超时统一(SSE/Ollama/Anthropic 读循环 60s 无数据抛 504 进重试通道)
- 同会话并发 sendMessage 防重入(isRunning 守卫)+ 会话存在性预检 +
  前置调用移入 try(ERROR+DONE 双事件保证,根治 isStreaming 假死)
- 清空审计后 resetChainCache(根治 verifyChain 误报 TAMPERED)
- DONE 不再提前清理 TRACE(TERMINATED 统一收尾,补全最终迭代录制)
- IME 合成回车不发送(普通 Enter + Cmd/Ctrl+Enter 双分支)+ handleSend 闭包修复

P2 安全纵深:
- preload 移除原始 electronAPI 暴露(渲染层零使用,关掉 XSS invoke 任意通道单点风险)
- CORS 同源回显根治(仅当前浏览页面 Origin,did-navigate 同步)
- MEMORY.md 命令保护正则扩展(括号/$/反引号/< 重定向边界 + 前导路径)
- write_file append TOCTOU 统一(open 后 realpath 校验,新文件分支补漏)
- 敏感键归一化(authKey 驼峰/连字符命中)+ MCP headers 鉴权值加密落库
- ReDoS 检测共享化(search_files/file_editor 统一拦截)
- run_tests/lint_code 升风险 + 需确认 + npx --no-install(执行边界对齐 run_command)
- MCP/SearXNG/llm.baseURL/updateFeedUrl 配置类 URL 高危目标校验(IPv6 去括号 +
  十六进制映射解析 + 尾点剥离)

P3 架构还债:
- temperature/maxTokens 热生效(引擎/编排器/SubAgent 三处接线)+ setBatch 单事务落盘
- SessionRecorder flush 竞态根治(flushPromise 等待 + 超限内联落盘 + stopRecording async)
- 内存收口(lastConsolidationBySession LRU / subTraces 清理 / 会话删除 disposeEngine)
- i18n 全量收口(28 组件 + 353 key 双字典,状态标签改渲染时函数)
- 死代码清理(updateTraceStep/HEADER_HEIGHT/void preA/失实注释)
- 斜杠菜单 MUI 化 + 删除逻辑收敛 resetSessionState + Blob URL 统一释放 +
  用户消息"仅保存"落库(saveMessage 透传前端 id 修复 id 错位)

P4 能力演进:
- 死循环检测拆分(驻留前置 + 乒乓后置带进度信号,合法交替不误报)
- run-lock 30s 超时强制 abort(旧 run 卡死不无限排队)
- RETRY 双通道 stream_reset(前端按 run 归属精确清空,根治重试文本重复)
- FTS5 trigram 中文子串搜索(迁移 9 版本化 SCHEMA_VERSION=2,≤2 字符 LIKE 回退)
- getContextWindow 兜底 1M→128K(未知模型防 413)

测试:
- 855 → 2406 用例(+1551,2.8 倍):服务层 +325(含 MemoryManager 51 新用例)、
  工具实体 +483、IPC/适配器 +390(含 OpenAI/Anthropic/Ollama 独立套件)、
  纯函数表格化 +330;引入 jsdom + @testing-library(14 组件测试文件 249 用例)
- 修复 R1(saveMessage id 透传)/ R2(stream_reset 精确归属)两个回归缺陷
- 遗留低危项清零:git-tools 顺序耦合 / web-fetch 真实时间退避 / slo 内存断言 /
  mcp-security 多余 skipIf / deepseek-balance 命名误导 / 组件 mock 注入脆弱性

版本: 0.7.4; README 同步(工具风险表/版本徽章); 依赖: 移除 @electron-toolkit/preload,
新增 jsdom/@testing-library(devDependencies 不打包)

回归: typecheck 双端 0 错误; ESLint 0/0; Electron ABI 全量 2406/2406 零跳过;
系统 Node 2110 通过 296 跳过(better-sqlite3 ABI)
2026-08-30 19:19:07 +08:00

444 lines
18 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* web_fetch 三阶段回退工具测试(v0.7.5 新建覆盖)
*
* 通过 mock ssrf-dispatcherssrfPinnedFetch/resolveRedirectTarget)、
* ssrf-guardvalidateSSRF)与 browsergetBrowserManager)锁定:
* - Phase1 HTTP 成功(text/html/markdown 三种 extract_mode
* - Phase2 内容过短自动升级浏览器
* - Phase3 浏览器回退 / 拦截页检测 / 重定向逐跳 / blocked 不进回退
* - 缓存命中 / max_chars 截断 / 10MB 闸门 / retry 语义
*
* 注意:HTML 夹具必须 > 80 字符(绕过 isInterceptedPage 空壳判定),
* text 模式正文必须 >= 200 字符(避免 Phase2 自动升级)。
*/
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
vi.mock('electron-log', () => ({
default: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() },
}));
const ssrfMock = vi.hoisted(() => ({
validateSSRF: vi.fn(async () => undefined),
}));
vi.mock('../ssrf-guard', () => ({
validateSSRF: ssrfMock.validateSSRF,
}));
const dispatcherMock = vi.hoisted(() => ({
ssrfPinnedFetch: vi.fn(),
// 类型签名与 ssrf-dispatcher.ts 的 resolveRedirectTarget 一致(string | null),
// 使测试可通过 mockReturnValueOnce 注入下一跳 URL。
resolveRedirectTarget: vi.fn((_response: unknown, _url: string): string | null => null),
}));
vi.mock('../ssrf-dispatcher', () => ({
ssrfPinnedFetch: dispatcherMock.ssrfPinnedFetch,
resolveRedirectTarget: dispatcherMock.resolveRedirectTarget,
}));
const browserMock = vi.hoisted(() => ({
getBrowserManager: vi.fn(),
}));
vi.mock('../browser', () => ({
getBrowserManager: browserMock.getBrowserManager,
}));
import { WebFetchTool } from '../web-fetch';
import { fetchCache } from '../network-utils';
import type { ToolExecutionContext } from '../../../types/metona-tool';
const context: ToolExecutionContext = {
sessionId: 't',
workspacePath: process.cwd(),
iteration: 1,
requestId: 'r',
};
/** 生成足够长(> 80 字符)避免空壳拦截判定的 HTML 页面 */
function page(body: string, status = 200): Response {
const html = `<html><body>${body}<p>Padding text to exceed the minimum shell detection threshold of eighty characters in total length.</p></body></html>`;
return new Response(html, { status, headers: { 'Content-Type': 'text/html' } });
}
/** 生成文本内容 >= 200 字符的页面(避免 Phase2 升级) */
function longTextPage(text: string, status = 200): Response {
const body = `<p>${text}</p><p>${'padding-'.repeat(30)}</p>`;
return page(body, status);
}
/** 生成带 Location 头的重定向响应 */
function redirectResponse(location: string, status = 302): Response {
return new Response('', { status, headers: { Location: location } });
}
/** 每次调用生成全新 Response(避免 body 消费后复用报错) */
function mockFetchWith(factory: () => Response) {
dispatcherMock.ssrfPinnedFetch.mockImplementation(async () => factory());
}
interface FetchResult {
success: boolean;
error?: string;
content?: string;
method?: string;
truncated?: boolean;
original_length?: number;
extract_mode?: string;
}
function resetAllMocks(): void {
vi.clearAllMocks();
ssrfMock.validateSSRF.mockReset().mockImplementation(async () => undefined);
dispatcherMock.ssrfPinnedFetch.mockReset();
dispatcherMock.resolveRedirectTarget.mockReset().mockReturnValue(null);
browserMock.getBrowserManager.mockReset().mockReturnValue({
fetchPageText: vi.fn(async () => 'browser rendered content '.repeat(30)),
});
}
describe('web_fetch — URL 与 SSRF 入口', () => {
let tool: WebFetchTool;
beforeEach(() => {
tool = new WebFetchTool();
fetchCache.clear();
resetAllMocks();
});
afterEach(() => vi.clearAllMocks());
it('非 http/https URL → 拒绝', async () => {
const r = (await tool.execute({ url: 'file:///etc/passwd' }, context)) as FetchResult;
expect(r.success).toBe(false);
expect(String(r.error)).toContain('URL must start with');
});
it('缺 url → 拒绝', async () => {
const r = (await tool.execute({}, context)) as FetchResult;
expect(r.success).toBe(false);
});
it('SSRF 校验失败 → 拒绝且不进入浏览器回退', async () => {
ssrfMock.validateSSRF.mockRejectedValueOnce(
new Error('Blocked SSRF: private/loopback address'),
);
const r = (await tool.execute({ url: 'http://127.0.0.1:1/x' }, context)) as FetchResult;
expect(r.success).toBe(false);
expect(String(r.error)).toContain('Blocked SSRF');
expect(dispatcherMock.ssrfPinnedFetch).not.toHaveBeenCalled();
expect(browserMock.getBrowserManager).not.toHaveBeenCalled();
});
it('SSRF 校验异常信息直接回传', async () => {
ssrfMock.validateSSRF.mockRejectedValueOnce(new Error('Blocked SSRF: no DNS records'));
const r = (await tool.execute({ url: 'http://nx.test/' }, context)) as FetchResult;
expect(r.success).toBe(false);
expect(String(r.error)).toContain('no DNS records');
});
});
describe('web_fetch — Phase1 HTTP 抓取', () => {
let tool: WebFetchTool;
beforeEach(() => {
tool = new WebFetchTool();
fetchCache.clear();
resetAllMocks();
});
afterEach(() => vi.clearAllMocks());
it('text 模式:成功解析 HTML 为纯文本', async () => {
mockFetchWith(() => longTextPage('标题段落与正文内容 ABC'));
const r = (await tool.execute({ url: 'https://ok.test/page' }, context)) as FetchResult;
expect(r.success).toBe(true);
expect(r.method).toBe('http');
expect(String(r.content)).toContain('标题段落与正文内容 ABC');
expect(r.extract_mode).toBe('text');
});
it('html 模式:返回原始 HTML 原文(实况契约:原文透传不做清理)', async () => {
mockFetchWith(() => page('<div>keep</div><script>evil()</script>'));
const r = (await tool.execute(
{ url: 'https://ok.test/page', extract_mode: 'html' },
context,
)) as FetchResult;
expect(r.success).toBe(true);
expect(r.method).toBe('http');
// 实况契约:html 模式直接返回 readBodyWithLimit 原文
expect(String(r.content)).toContain('<div>keep</div>');
expect(String(r.content)).toContain('<script>evil()</script>');
expect(r.extract_mode).toBe('html');
});
it('markdown 模式:结构化转换(标题 ATX + 链接)', async () => {
mockFetchWith(() => page('<h1>MD 标题</h1><a href="https://x.test/">链接</a><p>正文</p>'));
const r = (await tool.execute(
{ url: 'https://ok.test/md', extract_mode: 'markdown' },
context,
)) as FetchResult;
expect(r.success).toBe(true);
expect(r.method).toBe('http');
expect(String(r.content)).toContain('# MD 标题');
expect(String(r.content)).toContain('[链接](https://x.test/)');
expect(r.extract_mode).toBe('markdown');
});
it('拦截页检测(Cloudflare)→ 触发浏览器回退', async () => {
mockFetchWith(
() => new Response('<title>Attention Required! | Cloudflare</title>', { status: 200 }),
);
browserMock.getBrowserManager.mockReturnValue({
fetchPageText: vi.fn(async () => 'browser ok content '.repeat(30)),
});
const r = (await tool.execute({ url: 'https://cf.test/' }, context)) as FetchResult;
expect(r.success).toBe(true);
expect(r.method).toBe('browser');
expect(String(r.content)).toContain('browser ok content');
});
it('重定向逐跳跟随(resolveRedirectTarget 返回下一跳 → 再次请求)', async () => {
dispatcherMock.resolveRedirectTarget
.mockReturnValueOnce('https://redirected.test/final')
.mockReturnValue(null);
dispatcherMock.ssrfPinnedFetch
.mockResolvedValueOnce(redirectResponse('https://redirected.test/final'))
.mockImplementation(async () => longTextPage('最终页内容足够长以绕过拦截与升级判定'));
const r = (await tool.execute({ url: 'https://start.test/old' }, context)) as FetchResult;
expect(r.success).toBe(true);
expect(r.method).toBe('http');
expect(dispatcherMock.ssrfPinnedFetch).toHaveBeenCalledTimes(2);
expect(String(r.content)).toContain('最终页内容');
});
it('重定向目标 SSRF 被拦 → blocked 且不进浏览器回退', async () => {
dispatcherMock.resolveRedirectTarget.mockReturnValue('http://169.254.169.254/latest/meta-data');
mockFetchWith(() => redirectResponse('http://169.254.169.254/latest/meta-data'));
ssrfMock.validateSSRF
.mockResolvedValueOnce(undefined)
.mockRejectedValueOnce(new Error('Blocked SSRF: link-local'));
const r = (await tool.execute({ url: 'https://evil.test/redirect' }, context)) as FetchResult;
expect(r.success).toBe(false);
expect(String(r.error)).toContain('Redirect target blocked by SSRF guard');
expect(browserMock.getBrowserManager).not.toHaveBeenCalled();
});
it('超过 5 跳重定向 → Phase1 失败(Too many redirects),浏览器也失败时整体失败', async () => {
dispatcherMock.resolveRedirectTarget.mockReturnValue('https://loop.test/next');
mockFetchWith(() => redirectResponse('https://loop.test/next'));
browserMock.getBrowserManager.mockReturnValue({ fetchPageText: vi.fn(async () => null) });
const r = (await tool.execute({ url: 'https://loop.test/start' }, context)) as FetchResult;
expect(r.success).toBe(false);
expect(String(r.error)).toContain('Too many redirects');
});
it('HTTP 404 → Phase3 浏览器回退', async () => {
mockFetchWith(() => page('nope', 404));
browserMock.getBrowserManager.mockReturnValue({
fetchPageText: vi.fn(async () => 'fallback text '.repeat(30)),
});
const r = (await tool.execute({ url: 'https://miss.test/' }, context)) as FetchResult;
expect(r.success).toBe(true);
expect(r.method).toBe('browser');
});
it('HTTP 500 重试后仍失败 → 浏览器回退', async () => {
// v0.7.4 回归修复: fake timers 推进指数退避(原实现真实等待 6s+,CI 脆弱)。
// 不调用 restoreAllMocks(会清掉 beforeEach 的 mock),手动还原 Math.random。
mockFetchWith(() => page('err', 500));
browserMock.getBrowserManager.mockReturnValue({
fetchPageText: vi.fn(async () => 'browser rescue '.repeat(30)),
});
const randomSpy = vi.spyOn(Math, 'random').mockReturnValue(0.5);
vi.useFakeTimers();
try {
const promise = tool.execute({ url: 'https://fail.test/' }, context);
// 推进退避总时长(2s+1.2s + 4s+2.4s = 9.6s
await vi.advanceTimersByTimeAsync(12_000);
const r = (await promise) as FetchResult;
expect(r.success).toBe(true);
expect(r.method).toBe('browser');
} finally {
vi.useRealTimers();
randomSpy.mockRestore();
}
});
it('retry=false 时单次尝试(5xx 不重试直接失败→回退)', async () => {
mockFetchWith(() => page('err', 500));
browserMock.getBrowserManager.mockReturnValue({ fetchPageText: vi.fn(async () => null) });
const r = (await tool.execute(
{ url: 'https://noretry.test/', retry: false },
context,
)) as FetchResult;
expect(r.success).toBe(false);
expect(dispatcherMock.ssrfPinnedFetch).toHaveBeenCalledTimes(1);
expect(String(r.error)).toContain('All phases failed');
});
it('10MB 响应体超限 → 失败(readBodyWithLimit 闸门,retry=false 快路径)', async () => {
mockFetchWith(
() =>
new Response('x'.repeat(50), {
status: 200,
headers: { 'Content-Length': String(20 * 1024 * 1024) },
}),
);
browserMock.getBrowserManager.mockReturnValue({ fetchPageText: vi.fn(async () => null) });
const r = (await tool.execute(
{ url: 'https://big.test/', retry: false },
context,
)) as FetchResult;
expect(r.success).toBe(false);
expect(String(r.error)).toContain('All phases failed');
});
it('网络异常(fetch reject)→ 重试后浏览器回退', async () => {
// v0.7.4 回归修复: fake timers 推进指数退避(原实现真实等待 6s+)
dispatcherMock.ssrfPinnedFetch.mockRejectedValue(new Error('ECONNREFUSED'));
browserMock.getBrowserManager.mockReturnValue({
fetchPageText: vi.fn(async () => 'net rescue '.repeat(30)),
});
const randomSpy = vi.spyOn(Math, 'random').mockReturnValue(0.5);
vi.useFakeTimers();
try {
const promise = tool.execute({ url: 'https://net.test/' }, context);
await vi.advanceTimersByTimeAsync(12_000);
const r = (await promise) as FetchResult;
expect(r.success).toBe(true);
expect(r.method).toBe('browser');
} finally {
vi.useRealTimers();
randomSpy.mockRestore();
}
});
it('HTTP 429 → 直接进入浏览器回退(SKIP_RETRY', async () => {
mockFetchWith(() => page('rate', 429));
browserMock.getBrowserManager.mockReturnValue({
fetchPageText: vi.fn(async () => 'rate rescue '.repeat(30)),
});
const r = (await tool.execute({ url: 'https://rate.test/' }, context)) as FetchResult;
expect(r.success).toBe(true);
expect(r.method).toBe('browser');
});
});
describe('web_fetch — Phase2 内容升级 / 缓存 / 截断', () => {
let tool: WebFetchTool;
beforeEach(() => {
tool = new WebFetchTool();
fetchCache.clear();
resetAllMocks();
});
afterEach(() => vi.clearAllMocks());
it('Phase2text 内容 < 200 字符 → 升级浏览器渲染', async () => {
mockFetchWith(() => page('<p>短内容</p>'));
browserMock.getBrowserManager.mockReturnValue({
fetchPageText: vi.fn(async () => 'browser rich content '.repeat(30)),
});
const r = (await tool.execute({ url: 'https://spa.test/' }, context)) as FetchResult;
expect(r.success).toBe(true);
expect(r.method).toBe('browser');
expect(String(r.content)).toContain('browser rich content');
});
it('Phase2 升级但浏览器也失败 → 返回 Phase2 的短内容(http 方法)', async () => {
mockFetchWith(() => page('<p>短内容保留</p>'));
browserMock.getBrowserManager.mockReturnValue({ fetchPageText: vi.fn(async () => null) });
const r = (await tool.execute({ url: 'https://spa2.test/' }, context)) as FetchResult;
expect(r.success).toBe(true);
expect(r.method).toBe('http');
expect(String(r.content)).toContain('短内容保留');
});
it('html 模式不触发 Phase2 升级', async () => {
mockFetchWith(() => page('<div>tiny</div>'));
const r = (await tool.execute(
{ url: 'https://html-tiny.test/', extract_mode: 'html' },
context,
)) as FetchResult;
expect(r.success).toBe(true);
expect(r.method).toBe('http');
expect(browserMock.getBrowserManager).not.toHaveBeenCalled();
});
it('缓存命中(第二次请求走 fetchCache,不发网络请求)', async () => {
mockFetchWith(() => longTextPage('缓存内容正文'));
await tool.execute({ url: 'https://cache.test/page' }, context);
const r = (await tool.execute({ url: 'https://cache.test/page' }, context)) as FetchResult;
expect(r.success).toBe(true);
expect(r.method).toBe('cache');
expect(dispatcherMock.ssrfPinnedFetch).toHaveBeenCalledTimes(1);
});
it('缓存键含 extract_modetext 与 html 不同键', async () => {
mockFetchWith(() => page('<p>键隔离正文内容足够长</p>'));
await tool.execute({ url: 'https://key.test/', extract_mode: 'text' }, context);
await tool.execute({ url: 'https://key.test/', extract_mode: 'html' }, context);
expect(dispatcherMock.ssrfPinnedFetch).toHaveBeenCalledTimes(2);
});
it('html 模式不写缓存(避免模式混淆)', async () => {
mockFetchWith(() => page('<p>不缓存正文</p>'));
await tool.execute({ url: 'https://nocache.test/', extract_mode: 'html' }, context);
expect(fetchCache.has('html:https://nocache.test/')).toBe(false);
});
it('max_chars 截断内容并标记 truncated', async () => {
mockFetchWith(() => page(`<p>${'x'.repeat(5000)}</p>`));
const r = (await tool.execute(
{ url: 'https://max.test/', max_chars: 100 },
context,
)) as FetchResult;
expect(r.success).toBe(true);
expect(r.truncated).toBe(true);
expect(String(r.content)).toContain('content truncated at 100 chars');
expect(Number(r.original_length)).toBeGreaterThan(100);
});
it('浏览器回退文本 > 500K 被截断', async () => {
mockFetchWith(() => page('', 404));
browserMock.getBrowserManager.mockReturnValue({
fetchPageText: vi.fn(async () => 'B'.repeat(600_000)),
});
const r = (await tool.execute({ url: 'https://bigbrowser.test/' }, context)) as FetchResult;
expect(r.success).toBe(true);
expect(String(r.content)).toContain('content truncated');
});
it('浏览器回退产出的拦截页 → 判为失败', async () => {
mockFetchWith(() => page('', 403));
browserMock.getBrowserManager.mockReturnValue({
fetchPageText: vi.fn(async () => 'Just a moment... verifying you are human'),
});
const r = (await tool.execute(
{ url: 'https://browser-intercepted.test/' },
context,
)) as FetchResult;
expect(r.success).toBe(false);
expect(String(r.error)).toContain('All phases failed');
});
it('浏览器回退文本 < 80 字符 → 判为失败', async () => {
mockFetchWith(() => page('', 404));
browserMock.getBrowserManager.mockReturnValue({
fetchPageText: vi.fn(async () => 'too short'),
});
const r = (await tool.execute({ url: 'https://short.test/' }, context)) as FetchResult;
expect(r.success).toBe(false);
});
it('浏览器回退文本写入缓存 → 第二次请求直接命中顶层缓存(method=cache', async () => {
mockFetchWith(() => page('', 404));
browserMock.getBrowserManager.mockReturnValue({
fetchPageText: vi.fn(async () => 'browser cacheable content '.repeat(30)),
});
await tool.execute({ url: 'https://browser-cache.test/' }, context);
// 第二次:顶层 fetchCachetext:url 键,浏览器阶段写入)直接命中
const r = (await tool.execute({ url: 'https://browser-cache.test/' }, context)) as FetchResult;
expect(r.success).toBe(true);
expect(r.method).toBe('cache');
expect(String(r.content)).toContain('browser cacheable content');
// 浏览器不再被调用(缓存短路)
expect(browserMock.getBrowserManager).toHaveBeenCalledTimes(1);
});
});