/** * network-utils 纯函数层契约测试（v0.7.0 覆盖补齐 → v0.7.5 大幅扩充） * * 共享模块（UA 轮换 / 语言轮换 / 反爬头 / URL 归一化 / 拦截页特征 / 正文提取 / * 流式限读 / SearXNG 认证头 / CORS / Origin 提取 / HTML→Markdown / 双 LRU 缓存） * 全部行为契约逐一钉死。 */ import { describe, it, expect, vi, afterEach } from 'vitest'; vi.mock('electron-log', () => ({ default: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, })); import { searchCache, fetchCache, UA_POOL, MOBILE_UA, ACCEPT_LANGUAGE_POOL, buildAntiCrawlHeaders, normalizeUrl, isInterceptedPage, htmlToText, readBodyWithLimit, buildSearXNGAuthHeaders, htmlToMarkdown, extractOriginHeader, corsAllowOrigin, fetchWithTimeout, } from '../network-utils'; describe('normalizeUrl — 去重键归一化', () => { it.each([ // 大小写 host 归一 ['HTTPS://EXAMPLE.COM/A', 'https://example.com/A'], // 默认端口剥离（根路径保留单斜杠形态） ['http://example.com:80/a', 'http://example.com/a'], ['https://example.com:443/', 'https://example.com/'], // 尾斜杠剥离仅作用于非空路径 ['https://example.com/path/', 'https://example.com/path'], // UTM / 追踪参数剔除 ['https://a.com/p?utm_source=x&id=3', 'https://a.com/p?id=3'], ['https://a.com/p?gclid=xyz&q=1&fbclid=abc', 'https://a.com/p?q=1'], // 参数按字典序稳定排序（去重键的关键）；根路径 query 以 '?' 形态保留 ['https://a.com/?z=1&a=2&m=3', 'https://a.com/?a=2&m=3&z=1'], // 全部参数被清洗后保留根路径形态 ['https://a.com/?utm_medium=y', 'https://a.com/'], ])('%s → %s', (input, expected) => { expect(normalizeUrl(input)).toBe(expected); }); it('非默认端口保留', () => { expect(normalizeUrl('http://a.com:8080/x')).toBe('http://a.com:8080/x'); expect(normalizeUrl('https://a.com:8443/x')).toBe('https://a.com:8443/x'); }); it('ws/wss 默认端口剥离', () => { expect(normalizeUrl('ws://a.com:80/socket')).toBe('ws://a.com/socket'); expect(normalizeUrl('wss://a.com:443/socket')).toBe('wss://a.com/socket'); }); it('fragment 保留', () => { expect(normalizeUrl('https://a.com/x?q=1#sec')).toBe('https://a.com/x?q=1#sec'); }); it('非法 URL 原样返回', () => { expect(normalizeUrl('not-a-url')).toBe('not-a-url'); expect(normalizeUrl('')).toBe(''); }); it('根路径（pathname=/）保留尾斜杠', () => { expect(normalizeUrl('https://a.com/')).toBe('https://a.com/'); }); it('端口大小写 host 归一同时生效', () => { expect(normalizeUrl('HTTP://A.COM:80/X')).toBe('http://a.com/X'); }); }); describe('isInterceptedPage — 反爬/验证码拦截特征', () => { it('Cloudflare 挑战页被识别', () => { expect(isInterceptedPage('Attention Required! | Cloudflare')).toBe(true); expect(isInterceptedPage('
Checking your browser before accessing.
')).toBe(true); }); it('JS-required 空壳页（中英文）与 403 页识别', () => { expect(isInterceptedPage('')).toBe(true); expect(isInterceptedPage('

Access Denied

')).toBe(true); expect(isInterceptedPage('403 Forbidden')).toBe(true); }); it('正常正文不误报；超短正文触发空壳判定', () => { const normal = '' + '

'.repeat(0) + '

' + 'x'.repeat(2000) + '
'; expect(isInterceptedPage(normal)).toBe(false); expect(isInterceptedPage('hi')).toBe(true); // <80 字符空壳 }); it('just a moment / DDoS protection 特征', () => { expect(isInterceptedPage('
Just a moment...
')).toBe(true); expect(isInterceptedPage('DDoS protection by Cloudflare')).toBe(true); }); it('challenge-platform / cf-challenge 特征', () => { expect( isInterceptedPage(''), ).toBe(true); }); it('恰好 80 字符的非拦截正文不误报', () => { const exact80 = 'x'.repeat(80); expect(isInterceptedPage(exact80)).toBe(false); expect(isInterceptedPage('x'.repeat(79))).toBe(true); // <80 空壳 }); }); describe('htmlToText — HTML→纯文本管线', () => { it('噪声标签剔除 + 块级换行 + 实体解码', () => { const text = htmlToText( `

标题

第一段 & 符号

第二段 不间断

ab
`, ); expect(text).not.toContain('alert'); expect(text).not.toContain('.x'); expect(text).toContain('标题'); expect(text).toContain('第一段 & 符号'); expect(text).toContain('\n'); // 块级元素产生换行 }); it('nav/header/footer/aside/iframe/svg 整体剔除', () => { const text = htmlToText( '
页头
' + 'svg 文本正文', ); expect(text).not.toContain('导航'); expect(text).not.toContain('页头'); expect(text).not.toContain('页脚'); expect(text).not.toContain('侧栏'); expect(text).not.toContain('iframe'); expect(text).not.toContain('svg 文本'); expect(text).toContain('正文'); }); it('HTML 注释被剔除', () => { const text = htmlToText('可见'); expect(text).not.toContain('隐藏注释'); expect(text).toContain('可见'); }); it('表格单元格转制表符后由空白折叠为单空格（td/th → tab → 空格）', () => { const text = htmlToText( '
头A头B
v1v2
', ); // 实况契约：td/th 先转 \t，末尾 [ \t]+ 折叠为单空格 expect(text).toContain('头A 头B'); expect(text).toContain('v1 v2'); }); it('数字/十六进制实体解码', () => { const text = htmlToText('

AB

'); expect(text).toContain('AB'); }); it('符号实体解码（nbsp/lt/gt/quot/apos/hellip 等）', () => { const text = htmlToText('

a b <c> "q" 'x' …

'); expect(text).toContain('a b "q"'); expect(text).toContain('…'); }); it('连续换行折叠（3+ → 2）', () => { const text = htmlToText('

a

b

c

'); expect(text).not.toContain('\n\n\n'); }); it('br/hr 也产生换行', () => { const text = htmlToText('a
b
c'); expect(text.split('\n').length).toBeGreaterThanOrEqual(2); }); it('空输入与纯标签输入', () => { expect(htmlToText('')).toBe(''); expect(htmlToText('
')).toBe(''); }); }); describe('htmlToMarkdown — HTML→Markdown 结构化转换（v0.6.4 P4-4）', () => { it('h1-h6 输出 ATX 标题', () => { expect(htmlToMarkdown('

一级

')).toContain('# 一级'); expect(htmlToMarkdown('

二级

')).toContain('## 二级'); expect(htmlToMarkdown('
六级
')).toContain('###### 六级'); }); it('段落 / 链接 / 强调 / 行内代码', () => { const md = htmlToMarkdown( '

看 链接 和 粗 code

', ); expect(md).toContain('[链接](https://x.test)'); expect(md).toContain('**粗**'); expect(md).toContain('`code`'); }); it('pre 围栏代码块与 ul/ol 列表', () => { const md = htmlToMarkdown( '
const x = 1;
', ); expect(md).toContain('```'); expect(md).toContain('- 甲'); expect(md).toContain('- 乙'); }); it('blockquote 与 hr', () => { const md = htmlToMarkdown('
引用
'); expect(md).toContain('> 引用'); expect(md).toContain('---'); }); it('script/style/svg/noscript/iframe 整体剔除', () => { const md = htmlToMarkdown( 't正文', ); expect(md).not.toContain('evil'); expect(md).not.toContain('.x'); expect(md).not.toContain('t'); expect(md).toContain('正文'); }); it('空输入返回空串', () => { expect(htmlToMarkdown('')).toBe(''); expect(htmlToMarkdown(' ')).toBe(''); }); }); describe('readBodyWithLimit — 流式硬上限', () => { afterEach(() => vi.unstubAllGlobals()); function streamOf(chunks: string[]): ReadableStream { const enc = new TextEncoder(); return new ReadableStream({ start(c) { for (const ch of chunks) c.enqueue(enc.encode(ch)); c.close(); }, }); } it('正常读取全文并正确拼接跨 chunk 内容', async () => { const response = new Response(streamOf(['你好,', '世界!'])); const body = await readBodyWithLimit(response as unknown as Response, 1024); expect(body).toBe('你好,世界!'); }); it('超过 maxBytes 时硬性抛错（fail-fast 防线语义：调用方据此转入失败/回退路径）', async () => { const big = 'z'.repeat(5000); const response = new Response(streamOf([big])); await expect(readBodyWithLimit(response as unknown as Response, 1000)).rejects.toThrow( /bytes limit/, ); }); it('content-length 超限时短路抛错（不发完整读取）', async () => { const response = new Response(streamOf(['x'.repeat(50)]), { headers: { 'Content-Length': String(20 * 1024 * 1024) }, }); await expect(readBodyWithLimit(response as unknown as Response)).rejects.toThrow( /Response too large/, ); }); it('content-length 虚报偏小（真实流量超限）→ 流式累计超限抛错', async () => { const response = new Response(streamOf(['y'.repeat(900), 'z'.repeat(900)]), { headers: { 'Content-Length': String(500) }, // 虚报：预检通过，流式读取时超限 }); await expect(readBodyWithLimit(response as unknown as Response, 1000)).rejects.toThrow( /bytes limit/, ); }); it('无 body 的响应（null body）→ 返回空串', async () => { const response = new Response(null); const body = await readBodyWithLimit(response as unknown as Response); expect(body).toBe(''); }); it('非 UTF-8 字节以替换字符容错解码（fatal:false）', async () => { const enc = new TextEncoder(); const bad = new Uint8Array([0x48, 0x69, 0xff, 0xfe, 0x21]); // Hi + 非法字节 + ! const response = new Response( new ReadableStream({ start(c) { c.enqueue(enc.encode('')); c.enqueue(bad); c.close(); }, }), ); const body = await readBodyWithLimit(response as unknown as Response); expect(body).toContain('Hi'); }); }); describe('buildAntiCrawlHeaders — UA 轮换与移动端分支', () => { it('attempt 序号驱动桌面 UA 池轮换（确定性取模）', () => { for (let attempt = 0; attempt < UA_POOL.length * 2; attempt++) { const h = buildAntiCrawlHeaders('https://t.test/x', attempt, false); const ua = String(h['User-Agent'] ?? h['user-agent'] ?? ''); expect(UA_POOL).toContain(ua); // 非 mobile 分支绝不产生移动 UA expect(ua).not.toBe(MOBILE_UA); } }); it('UA 轮换取模：attempt=UA_POOL.length 回到首个 UA', () => { const h0 = buildAntiCrawlHeaders('https://t.test/', 0, false); const hN = buildAntiCrawlHeaders('https://t.test/', UA_POOL.length, false); expect(hN['User-Agent']).toBe(h0['User-Agent']); }); it('语言头随 attempt 轮换（Accept-Language 池）', () => { const h0 = buildAntiCrawlHeaders('https://t.test/', 0, false); const h1 = buildAntiCrawlHeaders('https://t.test/', 1, false); expect(ACCEPT_LANGUAGE_POOL).toContain(h0['Accept-Language']); expect(ACCEPT_LANGUAGE_POOL).toContain(h1['Accept-Language']); expect(h0['Accept-Language']).not.toBe(h1['Accept-Language']); }); it('mobile_ua=true 时固定使用移动 UA，并携带 Sec-Fetch/语言族反爬头', () => { const h = buildAntiCrawlHeaders('https://t.test/x?lang=zh', 0, true); const entries = Object.entries(h).map(([k, v]) => [k.toLowerCase(), v] as const); const map = new Map(entries); expect(map.get('user-agent')).toBe(MOBILE_UA); expect(map.has('sec-fetch-site')).toBe(true); expect(String(map.get('referer'))).toContain('https://t.test'); }); it('构建完整 12 头反爬签名', () => { const h = buildAntiCrawlHeaders('https://cdn.test/path', 0, false); expect(h['Accept']).toContain('text/html'); expect(h['Accept-Encoding']).toBe('gzip, deflate, br'); expect(h['Cache-Control']).toBe('no-cache'); expect(h['DNT']).toBe('1'); expect(h['Sec-Fetch-Dest']).toBe('document'); expect(h['Sec-Fetch-Mode']).toBe('navigate'); expect(h['Sec-Fetch-Site']).toBe('none'); expect(h['Sec-Fetch-User']).toBe('?1'); expect(h['Pragma']).toBe('no-cache'); }); it('Referer 使用 URL origin（含路径时只取源）', () => { const h = buildAntiCrawlHeaders('https://sub.example.com/a/b?x=1', 0, false); expect(h['Referer']).toBe('https://sub.example.com'); }); it('非法 URL → Referer 为空串（不抛错）', () => { const h = buildAntiCrawlHeaders('not a url', 0, false); expect(h['Referer']).toBe(''); }); }); describe('buildSearXNGAuthHeaders — 认证注入规则', () => { it('bearer：原样透传到 Authorization', () => { const h = buildSearXNGAuthHeaders('tok-123', 'bearer'); expect(h.Authorization).toBe('Bearer tok-123'); }); it('basic：username:password 整体 Base64（文档口径）', () => { const key = 'admin:s3cret'; const h = buildSearXNGAuthHeaders(key, 'basic'); expect(h.Authorization).toBe(`Basic ${Buffer.from(key, 'utf-8').toString('base64')}`); }); it('auth_key 为空时不注入任何认证头（文档边界：空值零注入）', () => { expect(buildSearXNGAuthHeaders('', 'bearer')).toEqual({}); expect(buildSearXNGAuthHeaders('', 'basic')).toEqual({}); }); it('未知 authType 不注入', () => { expect(buildSearXNGAuthHeaders('k', 'digest')).toEqual({}); expect(buildSearXNGAuthHeaders('k', '')).toEqual({}); }); }); describe('corsAllowOrigin — 仅回显当前浏览页面同源（P2-2 根治）', () => { it('请求 Origin 与当前页面同源 → 回显该 Origin', async () => { const result = corsAllowOrigin('https://example.com', 'https://example.com'); expect(result).toEqual(['https://example.com']); }); it('请求 Origin 与当前页面跨域 → 返回 null（不加 ACAO，保持默认同源策略）', async () => { expect(corsAllowOrigin('https://evil.com', 'https://example.com')).toBeNull(); }); it('无 Origin / 无当前页面 → 返回 null（不回退 *）', async () => { expect(corsAllowOrigin(undefined, 'https://example.com')).toBeNull(); expect(corsAllowOrigin('https://example.com', null)).toBeNull(); expect(corsAllowOrigin(undefined, null)).toBeNull(); }); it('大小写/尾斜杠差异不误判（同源归一化）', async () => { expect(corsAllowOrigin('HTTPS://EXAMPLE.COM/', 'https://example.com')).toEqual([ 'HTTPS://EXAMPLE.COM/', ]); }); it('同源请求回显原始 Origin（含端口差异保留）', () => { expect(corsAllowOrigin('https://a.com:8443', 'https://a.com:8443')).toEqual([ 'https://a.com:8443', ]); }); it('空白 Origin 视为无 → null', () => { expect(corsAllowOrigin(' ', 'https://a.com')).toBeNull(); }); }); describe('extractOriginHeader — 请求头 Origin 提取', () => { it('大小写不敏感提取单值 Origin', () => { expect(extractOriginHeader({ ORIGIN: 'https://x.com' })).toBe('https://x.com'); expect(extractOriginHeader({ Origin: 'https://x.com' })).toBe('https://x.com'); expect(extractOriginHeader({ origin: 'https://x.com' })).toBe('https://x.com'); }); it('数组值取第一个', () => { expect(extractOriginHeader({ Origin: ['https://a.com', 'https://b.com'] })).toBe( 'https://a.com', ); }); it('无 Origin 头 / 无头对象 → undefined', () => { expect(extractOriginHeader(undefined)).toBeUndefined(); expect(extractOriginHeader({ Referer: 'x' })).toBeUndefined(); }); }); describe('searchCache / fetchCache — LRU 行为', () => { afterEach(() => vi.restoreAllMocks()); it('写入后在 TTL 内命中', () => { searchCache.set('s:k1', { v: 1 } as unknown as Record); fetchCache.set('text:f:k1', 'hello'); expect(searchCache.get('s:k1')).toEqual({ v: 1 }); expect(fetchCache.get('text:f:k1')).toBe('hello'); }); it('未命中返回 undefined/falsy（不存在键）', () => { expect(searchCache.get('never:/x')).toBeUndefined(); expect(fetchCache.get('never:/x')).toBeUndefined(); }); it('超容量淘汰最旧条目（LRU max 语义）', () => { searchCache.clear(); for (let i = 0; i < 210; i++) searchCache.set(`s:evict-${i}`, { i }); expect(searchCache.get('s:evict-0')).toBeUndefined(); // 最早写入被淘汰 expect(searchCache.get('s:evict-209')).toEqual({ i: 209 }); }); }); describe('fetchWithTimeout — 超时中止', () => { afterEach(() => vi.unstubAllGlobals()); it('正常响应透传返回', async () => { const stub = vi.fn(async () => new Response('ok', { status: 200 })); vi.stubGlobal('fetch', stub); const resp = await fetchWithTimeout('https://x.test/', {}, 1000); expect(resp.status).toBe(200); expect(await resp.text()).toBe('ok'); expect(stub).toHaveBeenCalledTimes(1); }); it('超时触发 AbortError（fetch 收到 abort signal）', async () => { vi.stubGlobal( 'fetch', vi.fn((_url: string, init: RequestInit) => { const signal = init.signal as AbortSignal; return new Promise((_resolve, reject) => { signal.addEventListener('abort', () => { const err = new Error('Aborted'); err.name = 'AbortError'; reject(err); }); }); }), ); await expect(fetchWithTimeout('https://slow.test/', {}, 30)).rejects.toMatchObject({ name: 'AbortError', }); }); it('fetch 拒绝原样向上传播', async () => { vi.stubGlobal( 'fetch', vi.fn(async () => { throw new Error('network down'); }), ); await expect(fetchWithTimeout('https://x.test/', {}, 100)).rejects.toThrow('network down'); }); }); // ===== assertSafeConfigTarget 补充（配置类 URL 高危目标校验）===== describe('assertSafeConfigTarget — 配置 URL 校验（P2-9）', () => { it('拦截云元数据地址', async () => { const { assertSafeConfigTarget } = await import('../ssrf-guard'); expect(() => assertSafeConfigTarget('http://169.254.169.254/latest/meta-data/')).toThrow(); expect(() => assertSafeConfigTarget('http://169.254.169.254')).toThrow(); expect(() => assertSafeConfigTarget('http://metadata.google.internal/')).toThrow(); }); it('拦截链路本地/组播/保留段与 0.0.0.0', async () => { const { assertSafeConfigTarget } = await import('../ssrf-guard'); expect(() => assertSafeConfigTarget('http://0.0.0.0:8080')).toThrow(); expect(() => assertSafeConfigTarget('http://224.0.0.1/')).toThrow(); expect(() => assertSafeConfigTarget('http://240.0.0.1/')).toThrow(); }); it('放行本地回环/私网实例（合法 MCP/SearXNG）', async () => { const { assertSafeConfigTarget } = await import('../ssrf-guard'); expect(() => assertSafeConfigTarget('http://127.0.0.1:3000')).not.toThrow(); expect(() => assertSafeConfigTarget('http://192.168.1.10:8080')).not.toThrow(); expect(() => assertSafeConfigTarget('http://10.0.0.5:8888')).not.toThrow(); expect(() => assertSafeConfigTarget('https://searxng.example.com')).not.toThrow(); }); it('拦截非 http/https 协议', async () => { const { assertSafeConfigTarget } = await import('../ssrf-guard'); expect(() => assertSafeConfigTarget('file:///etc/passwd')).toThrow(); expect(() => assertSafeConfigTarget('ftp://example.com')).toThrow(); }); it('拦截 IPv6 高危地址（去括号后判定，P2-9-A 修正）', async () => { const { assertSafeConfigTarget } = await import('../ssrf-guard'); expect(() => assertSafeConfigTarget('http://[::ffff:169.254.169.254]/')).toThrow(); expect(() => assertSafeConfigTarget('http://[fe80::1]/')).toThrow(); expect(() => assertSafeConfigTarget('http://[ff02::1]/')).toThrow(); expect(() => assertSafeConfigTarget('http://[::]/')).toThrow(); expect(() => assertSafeConfigTarget('http://[::1]:11434/')).not.toThrow(); }); it('拦截域名尾点绕过（P2-9-B 修正）', async () => { const { assertSafeConfigTarget } = await import('../ssrf-guard'); expect(() => assertSafeConfigTarget('http://metadata.google.internal./')).toThrow(); expect(() => assertSafeConfigTarget('http://169.254.169.254./latest/meta-data/')).toThrow(); }); it('拦截 IPv4-mapped 十六进制云元数据（::ffff:a9fe:a9fe）', async () => { const { assertSafeConfigTarget } = await import('../ssrf-guard'); // 169.254 = 0xa9fe expect(() => assertSafeConfigTarget('http://[::ffff:a9fe:a9fe]/')).toThrow(); }); it('放行 IPv4-mapped 公网（::ffff:0808:0808 = 8.8.8.8）', async () => { const { assertSafeConfigTarget } = await import('../ssrf-guard'); expect(() => assertSafeConfigTarget('http://[::ffff:0808:0808]/')).not.toThrow(); }); it('拦截 169.254 链路本地变体（169.254.0.1）', async () => { const { assertSafeConfigTarget } = await import('../ssrf-guard'); expect(() => assertSafeConfigTarget('http://169.254.0.1/')).toThrow(); }); it('非法 URL → Invalid URL', async () => { const { assertSafeConfigTarget } = await import('../ssrf-guard'); expect(() => assertSafeConfigTarget('not a url')).toThrow(/Invalid URL/); }); });