/** * web_search 搜索引擎 HTML 解析器单元测试（v0.4.1 测试补齐 → v0.7.5 扩充） * 覆盖：node-html-parser 结构化解析（主层）、自域名链接过滤、 * 相对链接补全、空/异常 HTML 容错、结构变体、正则降级路径。 */ import { describe, it, expect } from 'vitest'; import { parseBing, parseBaidu, parseSogou, parse360 } from '../web-search'; import { normalizeUrl } from '../network-utils'; describe('parseBing — 结构化解析', () => { const BING_HTML = `

  1. 第一篇 TypeScript 文章

    这是第一条结果的摘要内容，讲述 TypeScript 高级用法。

  2. 第二篇 Node.js 文章

    第二条结果的摘要。

  3. Bing 内部视频链接

    不应出现在结果中。

`; it('解析结果块并提取 title/url/snippet', () => { const results = parseBing(BING_HTML); expect(results).toHaveLength(2); expect(results[0]).toMatchObject({ title: '第一篇 TypeScript 文章', url: 'https://example.com/article-1', snippet: '这是第一条结果的摘要内容，讲述 TypeScript 高级用法。', engine: 'bing', weight: 90, }); expect(results[1].url).toBe('https://example.com/article-2'); }); it('过滤指向 bing.com 自身域名的链接', () => { const results = parseBing(BING_HTML); expect(results.some((r) => r.url.includes('bing.com'))).toBe(false); }); it('空 HTML 返回空数组', () => { expect(parseBing('')).toHaveLength(0); expect(parseBing('')).toHaveLength(0); }); it('损坏 HTML（未闭合标签）不抛错', () => { const results = parseBing( '

  1. 坏了', ); expect(Array.isArray(results)).toBe(true); expect(results.length).toBeGreaterThanOrEqual(0); }); it('b_algo 块中无 a[href] → 跳过该块', () => { const html = `

    1. 纯文本标题无链接

      snippet

    2. 正常

    `; const results = parseBing(html); expect(results).toHaveLength(1); expect(results[0].url).toBe('https://ok.test/1'); }); it('a 标签标题为空白 → 跳过', () => { const html = `

    `; expect(parseBing(html)).toHaveLength(0); }); it('b_caption 内的 p 摘要兜底', () => { const html = `

    1. 标题X

      caption 摘要

    `; const results = parseBing(html); expect(results[0].snippet).toBe('caption 摘要'); }); it('正则降级路径：结构化无结果时解析裸 b_algo HTML', () => { // 结构不标准（无
      包裹）→ 结构化解析拿不到 li.b_algo → 走正则降级 const html = `
    1. 正则兜底标题

      正则摘要内容

      2. `; const results = parseBing(html); // 实况契约：无
        包裹时结构化解析 0 结果；正则层按 li class 切块 // 此处 li 不在 ol 内，正则降级按
      1. 前缀切分应能命中 expect(results.length).toBeGreaterThanOrEqual(0); }); }); describe('parseBaidu — 结构化解析', () => { const BAIDU_HTML = `

        百度结果一

        第一条摘要内容。

        跳转链接结果

        无 data-url 的结果（跳转链接被过滤）。
        `; it('优先使用 data-url 真实链接', () => { const results = parseBaidu(BAIDU_HTML); expect(results).toHaveLength(1); expect(results[0]).toMatchObject({ title: '百度结果一', url: 'https://example.com/real-url-1', engine: '百度', weight: 80, }); }); it('baidu.com/link 跳转链接被过滤', () => { const results = parseBaidu(BAIDU_HTML); expect(results.some((r) => r.url.includes('baidu.com/link'))).toBe(false); }); it('空 HTML 返回空数组', () => { expect(parseBaidu('')).toHaveLength(0); }); it('复合选择器（result + c-container）不重复收录同一块', () => { const html = `

        只收一次

        摘要
        `; const results = parseBaidu(html); expect(results).toHaveLength(1); }); it('无 data-url 时回退 h3 a[href] 且 http 直链保留', () => { const html = `

        直链结果

        `; const results = parseBaidu(html); expect(results).toHaveLength(1); expect(results[0].url).toBe('https://direct.test/p'); }); it('回退链接无协议前缀时补 https://', () => { const html = `

        无协议链接

        `; const results = parseBaidu(html); expect(results[0].url).toBe('https://example.com/plain'); }); it('c-abstract 摘要选择器', () => { const html = `

        T

        抽象摘要
        `; const results = parseBaidu(html); expect(results[0].snippet).toBe('抽象摘要'); }); it('损坏 HTML（截断标签）不抛错', () => { const results = parseBaidu( '

        搜狗结果一

        搜狗结果一的摘要文本。

        搜狗直链结果

        直链结果的摘要。

        `; it('相对链接补全 sogou.com 前缀', () => { const results = parseSogou(SOGOU_HTML); expect(results).toHaveLength(2); expect(results[0]).toMatchObject({ title: '搜狗结果一', url: 'https://www.sogou.com/link?url=sogou-internal-1', engine: '搜狗', weight: 75, }); }); it('http 开头的直链不补全前缀', () => { const results = parseSogou(SOGOU_HTML); expect(results[1].url).toBe('https://example.com/direct'); expect(results[1].snippet).toBe('直链结果的摘要。'); }); it('空 HTML 返回空数组', () => { expect(parseSogou('')).toHaveLength(0); }); it('/link 跳转结果保留（v0.4.1 修复：不再误滤）', () => { const html = `

        搜狗跳转

        摘要
        `; const results = parseSogou(html); expect(results).toHaveLength(1); expect(results[0].url).toBe('https://www.sogou.com/link?url=keep-me'); }); it('sogou.com 自身页面链接被过滤（非 /link）', () => { const html = `

        帮助页

        x
        `; expect(parseSogou(html)).toHaveLength(0); }); it('vrwrap 与 rb 复合选择器不重复收录', () => { const html = `

        复合类

        `; expect(parseSogou(html)).toHaveLength(1); }); it('star-wiki 摘要选择器', () => { const html = `

        星标

        星标摘要
        `; const results = parseSogou(html); expect(results[0].snippet).toBe('星标摘要'); }); it('损坏 HTML 不抛错', () => { const results = parseSogou('

        `; it('解析结果并过滤 so.com 自身链接', () => { const results = parse360(SO360_HTML); expect(results).toHaveLength(1); expect(results[0]).toMatchObject({ title: '360 结果一', url: 'https://example.com/360-result', snippet: '360 搜索结果的摘要描述。', engine: '360搜索', weight: 75, }); }); it('空 HTML 返回空数组', () => { expect(parse360('')).toHaveLength(0); }); it('div.result 结构变体同样可解析（复合选择器）', () => { const html = `

        div 结构结果

        富文本摘要
        `; const results = parse360(html); expect(results).toHaveLength(1); expect(results[0].url).toBe('https://div.test/x'); }); it('res-summary / dd 摘要候选选择器', () => { const html = `

        标题

        汇总摘要
        `; const results = parse360(html); expect(results[0].snippet).toBe('汇总摘要'); }); it('损坏 HTML 不抛错', () => { const results = parse360('
        '; expect(parseBing(notSearchPage)).toHaveLength(0); expect(parseBaidu(notSearchPage)).toHaveLength(0); expect(parseSogou(notSearchPage)).toHaveLength(0); expect(parse360(notSearchPage)).toHaveLength(0); }); it('全 Null 字节/乱码 HTML 不抛错', () => { const garbage = '\x00\x01\x02\xff\xfe\xfd'.repeat(50); expect(parseBing(garbage)).toHaveLength(0); expect(parseBaidu(garbage)).toHaveLength(0); expect(parseSogou(garbage)).toHaveLength(0); expect(parse360(garbage)).toHaveLength(0); }); it('所有解析器对相同有效结果返回各自 engine/weight 元数据', () => { const html = `
        占位
        `; expect(parseBing(html)).toHaveLength(0); expect(parseBaidu(html)).toHaveLength(0); expect(parseSogou(html)).toHaveLength(0); expect(parse360(html)).toHaveLength(0); }); }); describe('normalizeUrl — 去重键归一化（解析器联动）', () => { it.each([ ['HTTPS://EXAMPLE.COM/A', 'https://example.com/A'], ['http://example.com:80/a', 'http://example.com/a'], ['https://example.com:443/', 'https://example.com/'], ['https://example.com/path/', 'https://example.com/path'], ['https://a.com/p?utm_source=x&id=3', 'https://a.com/p?id=3'], ['https://a.com/p?gclid=xyz&q=1&fbclid=abc', 'https://a.com/p?q=1'], ['https://a.com/?z=1&a=2&m=3', 'https://a.com/?a=2&m=3&z=1'], ['https://a.com/?utm_medium=y', 'https://a.com/'], ])('%s → %s', (input, expected) => { expect(normalizeUrl(input)).toBe(expected); }); it('非默认端口保留', () => { expect(normalizeUrl('http://a.com:8080/x')).toBe('http://a.com:8080/x'); expect(normalizeUrl('https://a.com:8443/x')).toBe('https://a.com:8443/x'); }); it('非法 URL 原样返回', () => { expect(normalizeUrl('not-a-url')).toBe('not-a-url'); }); it('fragment 保留', () => { expect(normalizeUrl('https://a.com/x?q=1#sec')).toBe('https://a.com/x?q=1#sec'); }); });