thzxx
3940716dc2
P1 修复面收口: v0.6.3 截断自愈推全量(Anthropic/Ollama/非流式/引擎兜底); SSE 上游错误帧检测进重试通道; clearMessages 摘要游标根治; truncateResult 内联图片白名单统一; 前端四 bug(确认弹窗锁死/MemoryViewer/ Virtuoso Footer/abort 尾部过滤) + reasoning 缓冲跨迭代污染; 托盘通知过滤与新建会话死链接线 P2 安全纵深: MCP 审批闭环(ConfirmationHook×PolicyEngine 联动+重名拒注册); SSRF 收敛 ssrf-guard 共享模块 (web_fetch 双通道校验+重定向终态复检); Electron 加固(preload CJS 化→sandbox:true/CSP/权限白名单/will-navigate); run_command cmd.exe 白名单通道元字符守门; diff_viewer 10MB 预检; Anthropic thinking 预算下限; Agnes 思考显式关闭 P3 架构还债: OpenAICompatibleAdapter 中间基类收敛四家样板; 错误分类单轨化(删 mapError/getFetchSignal, 超时显式 ETIMEDOUT); PRAGMA user_version 迁移版本化; 死代码清理专项(cn.ts/SHORTCUTS/ContextMenu 分支/ getWindowState/modifiedArgs/sandbox 空壳); i18next 引入; a11y 第一轮; SearXNG 页批量草稿模型统一 P4 能力演进: Ollama pull 可取消/capabilities 探测/num_ctx 实测缓存; UpdateService feed 比对式自动更新 (app:updateCheck IPC + StatusBar 入口); MiMo providerOptions(web_search 服务端工具/strict JSON); web_fetch extract_mode=markdown(turndown); network.proxyUrl 全局代理(Chromium sessions+undici dispatcher) 测试: 264 → 507 用例(Electron ABI 全绿零跳过), 覆盖引擎压缩管线/重试竞速/MEMORY.md 闸门/file_editor 五操作/ filesystem 七工具实体夹具/git 真实仓库/SSE 错误帧/全线截断自愈/Provider 请求形态矩阵/SSRF 表测/钩子分级矩阵/ OutputValidator 全量/SLO 指标/MCP 安全纯函数/task_manager 链路/渲染层纯域/i18n 桥契约
178 lines
6.8 KiB
TypeScript
178 lines
6.8 KiB
TypeScript
/**
|
||
* network-utils 纯函数层契约测试（v0.7.0 覆盖补齐）
|
||
*
|
||
* 此前该共享模块（UA 轮换 / 反爬头 / URL 归一化 / 拦截页特征 / 正文提取 /
|
||
* 流式限读 / SearXNG 认证头 / 双 LRU 缓存）只有 web_fetch/web_search 间接触达，
|
||
* 直接行为契约零锁定。本文件逐一钉死。
|
||
*/
|
||
|
||
import { describe, it, expect, vi, afterEach } from 'vitest';
|
||
|
||
vi.mock('electron-log', () => ({
|
||
default: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() },
|
||
}));
|
||
|
||
import {
|
||
searchCache,
|
||
fetchCache,
|
||
UA_POOL,
|
||
MOBILE_UA,
|
||
buildAntiCrawlHeaders,
|
||
normalizeUrl,
|
||
isInterceptedPage,
|
||
htmlToText,
|
||
readBodyWithLimit,
|
||
buildSearXNGAuthHeaders,
|
||
} from '../network-utils';
|
||
|
||
describe('normalizeUrl — 去重键归一化', () => {
|
||
it.each([
|
||
// 大小写 host 归一
|
||
['HTTPS://EXAMPLE.COM/A', 'https://example.com/A'],
|
||
// 默认端口剥离（根路径保留单斜杠形态）
|
||
['http://example.com:80/a', 'http://example.com/a'],
|
||
['https://example.com:443/', 'https://example.com/'],
|
||
// 尾斜杠剥离仅作用于非空路径
|
||
['https://example.com/path/', 'https://example.com/path'],
|
||
// UTM / 追踪参数剔除
|
||
['https://a.com/p?utm_source=x&id=3', 'https://a.com/p?id=3'],
|
||
['https://a.com/p?gclid=xyz&q=1&fbclid=abc', 'https://a.com/p?q=1'],
|
||
// 参数按字典序稳定排序（去重键的关键）；根路径 query 以 '?' 形态保留
|
||
['https://a.com/?z=1&a=2&m=3', 'https://a.com/?a=2&m=3&z=1'],
|
||
// 全部参数被清洗后保留根路径形态
|
||
['https://a.com/?utm_medium=y', 'https://a.com/'],
|
||
])('%s → %s', (input, expected) => {
|
||
expect(normalizeUrl(input)).toBe(expected);
|
||
});
|
||
});
|
||
|
||
describe('isInterceptedPage — 反爬/验证码拦截特征', () => {
|
||
it('Cloudflare 挑战页被识别', () => {
|
||
expect(isInterceptedPage('<title>Attention Required! | Cloudflare</title>')).toBe(true);
|
||
expect(isInterceptedPage('<div>Checking your browser before accessing.</div>')).toBe(true);
|
||
});
|
||
|
||
it('JS-required 空壳页（中英文）与 403 页识别', () => {
|
||
expect(isInterceptedPage('<noscript>请启用 JavaScript</noscript><body></body>')).toBe(true);
|
||
expect(isInterceptedPage('<h1>Access Denied</h1>')).toBe(true);
|
||
expect(isInterceptedPage('<title>403 Forbidden</title>')).toBe(true);
|
||
});
|
||
|
||
it('正常正文不误报；超短正文触发空壳判定', () => {
|
||
const normal = '<html><body>' + '<p>'.repeat(0) + '<article>' + 'x'.repeat(2000) + '</article></body></html>';
|
||
expect(isInterceptedPage(normal)).toBe(false);
|
||
expect(isInterceptedPage('<html><body>hi</body></html>')).toBe(true); // <80 字符空壳
|
||
});
|
||
});
|
||
|
||
describe('htmlToText — HTML→纯文本管线', () => {
|
||
it('噪声标签剔除 + 块级换行 + 实体解码', () => {
|
||
const text = htmlToText(
|
||
`<script>alert(1)</script><style>.x{}</style>
|
||
<h2>标题</h2><p>第一段 & 符号</p><p>第二段 不间断</p>
|
||
<table><tr><td>a</td><td>b</td></tr></table>`,
|
||
);
|
||
expect(text).not.toContain('alert');
|
||
expect(text).not.toContain('.x');
|
||
expect(text).toContain('标题');
|
||
expect(text).toContain('第一段 & 符号');
|
||
expect(text).toContain('\n'); // 块级元素产生换行
|
||
});
|
||
});
|
||
|
||
describe('readBodyWithLimit — 流式硬上限', () => {
|
||
afterEach(() => vi.unstubAllGlobals());
|
||
|
||
function streamOf(chunks: string[]): ReadableStream<Uint8Array> {
|
||
const enc = new TextEncoder();
|
||
return new ReadableStream<Uint8Array>({
|
||
start(c) {
|
||
for (const ch of chunks) c.enqueue(enc.encode(ch));
|
||
c.close();
|
||
},
|
||
});
|
||
}
|
||
|
||
it('正常读取全文并正确拼接跨 chunk 内容', async () => {
|
||
const response = new Response(streamOf(['你好,', '世界!']));
|
||
const body = await readBodyWithLimit(response as unknown as Response, 1024);
|
||
expect(body).toBe('你好,世界!');
|
||
});
|
||
|
||
it('超过 maxBytes 时硬性抛错（fail-fast 防线语义：调用方据此转入失败/回退路径）', async () => {
|
||
const big = 'z'.repeat(5000);
|
||
const response = new Response(streamOf([big]));
|
||
await expect(readBodyWithLimit(response as unknown as Response, 1000)).rejects.toThrow(
|
||
/bytes limit/,
|
||
);
|
||
});
|
||
|
||
it('content-length 超限时短路抛错（不发完整读取）', async () => {
|
||
const response = new Response(streamOf(['x'.repeat(50)]), {
|
||
headers: { 'Content-Length': String(20 * 1024 * 1024) },
|
||
});
|
||
await expect(readBodyWithLimit(response as unknown as Response)).rejects.toThrow(
|
||
/Response too large/,
|
||
);
|
||
});
|
||
});
|
||
|
||
describe('buildAntiCrawlHeaders — UA 轮换与移动端分支', () => {
|
||
it('attempt 序号驱动桌面 UA 池轮换（确定性取模）', () => {
|
||
for (let attempt = 0; attempt < UA_POOL.length * 2; attempt++) {
|
||
const h = buildAntiCrawlHeaders('https://t.test/x', attempt, false);
|
||
const ua = String(h['User-Agent'] ?? h['user-agent'] ?? '');
|
||
expect(UA_POOL).toContain(ua);
|
||
// 非 mobile 分支绝不产生移动 UA
|
||
expect(ua).not.toBe(MOBILE_UA);
|
||
}
|
||
});
|
||
|
||
it('mobile_ua=true 时固定使用移动 UA，并携带 Sec-Fetch/语言族反爬头', () => {
|
||
const h = buildAntiCrawlHeaders('https://t.test/x?lang=zh', 0, true);
|
||
const entries = Object.entries(h).map(([k, v]) => [k.toLowerCase(), v] as const);
|
||
const map = new Map(entries);
|
||
expect(map.get('user-agent')).toBe(MOBILE_UA);
|
||
expect(map.has('sec-fetch-site')).toBe(true);
|
||
expect(String(map.get('referer'))).toContain('https://t.test');
|
||
});
|
||
});
|
||
|
||
describe('buildSearXNGAuthHeaders — 认证注入规则', () => {
|
||
it('bearer：原样透传到 Authorization', () => {
|
||
const h = buildSearXNGAuthHeaders('tok-123', 'bearer');
|
||
expect(h.Authorization).toBe('Bearer tok-123');
|
||
});
|
||
|
||
it('basic：username:password 整体 Base64（文档口径）', () => {
|
||
const key = 'admin:s3cret';
|
||
const h = buildSearXNGAuthHeaders(key, 'basic');
|
||
expect(h.Authorization).toBe(`Basic ${Buffer.from(key, 'utf-8').toString('base64')}`);
|
||
});
|
||
|
||
it('auth_key 为空时不注入任何认证头（文档边界：空值零注入）', () => {
|
||
expect(buildSearXNGAuthHeaders('', 'bearer')).toEqual({});
|
||
expect(buildSearXNGAuthHeaders('', 'basic')).toEqual({});
|
||
});
|
||
|
||
it('未知 authType 不注入', () => {
|
||
expect(buildSearXNGAuthHeaders('k', 'digest')).toEqual({});
|
||
});
|
||
});
|
||
|
||
describe('searchCache / fetchCache — LRU 行为', () => {
|
||
afterEach(() => vi.restoreAllMocks());
|
||
|
||
it('写入后在 TTL 内命中', () => {
|
||
searchCache.set('s:k1', { v: 1 } as unknown as Record<string, unknown>);
|
||
fetchCache.set('text:f:k1', 'hello');
|
||
expect(searchCache.get('s:k1')).toEqual({ v: 1 });
|
||
expect(fetchCache.get('text:f:k1')).toBe('hello');
|
||
});
|
||
|
||
it('未命中返回 undefined/falsy（不存在键）', () => {
|
||
expect(searchCache.get('never:/x')).toBeUndefined();
|
||
expect(fetchCache.get('never:/x')).toBeUndefined();
|
||
});
|
||
});
|