thzxx
99d0c54129
P1 修复面收口: - 超时三态区分（aborted→USER_INTERRUPT / ETIMEDOUT→TIMEOUT / 其余→ERROR）， 根治"真实网络超时被误报为用户中断" - 流空闲超时统一（SSE/Ollama/Anthropic 读循环 60s 无数据抛 504 进重试通道） - 同会话并发 sendMessage 防重入（isRunning 守卫）+ 会话存在性预检 + 前置调用移入 try（ERROR+DONE 双事件保证，根治 isStreaming 假死） - 清空审计后 resetChainCache（根治 verifyChain 误报 TAMPERED） - DONE 不再提前清理 TRACE（TERMINATED 统一收尾，补全最终迭代录制） - IME 合成回车不发送（普通 Enter + Cmd/Ctrl+Enter 双分支）+ handleSend 闭包修复 P2 安全纵深: - preload 移除原始 electronAPI 暴露（渲染层零使用，关掉 XSS invoke 任意通道单点风险） - CORS 同源回显根治（仅当前浏览页面 Origin，did-navigate 同步） - MEMORY.md 命令保护正则扩展（括号/$/反引号/< 重定向边界 + 前导路径） - write_file append TOCTOU 统一（open 后 realpath 校验，新文件分支补漏） - 敏感键归一化（authKey 驼峰/连字符命中）+ MCP headers 鉴权值加密落库 - ReDoS 检测共享化（search_files/file_editor 统一拦截） - run_tests/lint_code 升风险 + 需确认 + npx --no-install（执行边界对齐 run_command） - MCP/SearXNG/llm.baseURL/updateFeedUrl 配置类 URL 高危目标校验（IPv6 去括号 + 十六进制映射解析 + 尾点剥离） P3 架构还债: - temperature/maxTokens 热生效（引擎/编排器/SubAgent 三处接线）+ setBatch 单事务落盘 - SessionRecorder flush 竞态根治（flushPromise 等待 + 超限内联落盘 + stopRecording async） - 内存收口（lastConsolidationBySession LRU / subTraces 清理 / 会话删除 disposeEngine） - i18n 全量收口（28 组件 + 353 key 双字典，状态标签改渲染时函数） - 死代码清理（updateTraceStep/HEADER_HEIGHT/void preA/失实注释） - 斜杠菜单 MUI 化 + 删除逻辑收敛 resetSessionState + Blob URL 统一释放 + 用户消息"仅保存"落库（saveMessage 透传前端 id 修复 id 错位） P4 能力演进: - 死循环检测拆分（驻留前置 + 乒乓后置带进度信号，合法交替不误报） - run-lock 30s 超时强制 abort（旧 run 卡死不无限排队） - RETRY 双通道 stream_reset（前端按 run 归属精确清空，根治重试文本重复） - FTS5 trigram 中文子串搜索（迁移 9 版本化 SCHEMA_VERSION=2，≤2 字符 LIKE 回退） - getContextWindow 兜底 1M→128K（未知模型防 413） 测试: - 855 → 2406 用例（+1551，2.8 倍）：服务层 +325（含 MemoryManager 51 新用例）、 工具实体 +483、IPC/适配器 +390（含 OpenAI/Anthropic/Ollama 独立套件）、 纯函数表格化 +330；引入 jsdom + @testing-library（14 组件测试文件 249 用例） - 修复 R1（saveMessage id 透传）/ R2（stream_reset 精确归属）两个回归缺陷 - 遗留低危项清零：git-tools 顺序耦合 / web-fetch 真实时间退避 / slo 内存断言 / mcp-security 多余 skipIf / deepseek-balance 命名误导 / 组件 mock 注入脆弱性 版本: 0.7.4; README 同步（工具风险表/版本徽章）; 依赖: 移除 @electron-toolkit/preload， 新增 jsdom/@testing-library（devDependencies 不打包） 回归: typecheck 双端 0 错误; ESLint 0/0; Electron ABI 全量 2406/2406 零跳过; 系统 Node 2110 通过 296 跳过（better-sqlite3 ABI）
400 lines
14 KiB
TypeScript
400 lines
14 KiB
TypeScript
/**
|
|
* web_search 搜索引擎 HTML 解析器单元测试（v0.4.1 测试补齐 → v0.7.5 扩充）
|
|
* 覆盖：node-html-parser 结构化解析（主层）、自域名链接过滤、
|
|
* 相对链接补全、空/异常 HTML 容错、结构变体、正则降级路径。
|
|
*/
|
|
|
|
import { describe, it, expect } from 'vitest';
|
|
import { parseBing, parseBaidu, parseSogou, parse360 } from '../web-search';
|
|
import { normalizeUrl } from '../network-utils';
|
|
|
|
describe('parseBing — 结构化解析', () => {
|
|
const BING_HTML = `
|
|
<html><body>
|
|
<ol id="b_results">
|
|
<li class="b_algo">
|
|
<h2><a href="https://example.com/article-1">第一篇 TypeScript 文章</a></h2>
|
|
<p>这是第一条结果的摘要内容，讲述 TypeScript 高级用法。</p>
|
|
</li>
|
|
<li class="b_algo">
|
|
<h2><a href="https://example.com/article-2">第二篇 Node.js 文章</a></h2>
|
|
<div class="b_caption"><p>第二条结果的摘要。</p></div>
|
|
</li>
|
|
<li class="b_algo">
|
|
<!-- 自身域名链接应被过滤 -->
|
|
<h2><a href="https://www.bing.com/video?q=x">Bing 内部视频链接</a></h2>
|
|
<p>不应出现在结果中。</p>
|
|
</li>
|
|
</ol>
|
|
</body></html>
|
|
`;
|
|
|
|
it('解析结果块并提取 title/url/snippet', () => {
|
|
const results = parseBing(BING_HTML);
|
|
expect(results).toHaveLength(2);
|
|
expect(results[0]).toMatchObject({
|
|
title: '第一篇 TypeScript 文章',
|
|
url: 'https://example.com/article-1',
|
|
snippet: '这是第一条结果的摘要内容，讲述 TypeScript 高级用法。',
|
|
engine: 'bing',
|
|
weight: 90,
|
|
});
|
|
expect(results[1].url).toBe('https://example.com/article-2');
|
|
});
|
|
|
|
it('过滤指向 bing.com 自身域名的链接', () => {
|
|
const results = parseBing(BING_HTML);
|
|
expect(results.some((r) => r.url.includes('bing.com'))).toBe(false);
|
|
});
|
|
|
|
it('空 HTML 返回空数组', () => {
|
|
expect(parseBing('')).toHaveLength(0);
|
|
expect(parseBing('<html><body></body></html>')).toHaveLength(0);
|
|
});
|
|
|
|
it('损坏 HTML（未闭合标签）不抛错', () => {
|
|
const results = parseBing(
|
|
'<ol id="b_results"><li class="b_algo"><h2><a href="https://x.test/a">坏了',
|
|
);
|
|
expect(Array.isArray(results)).toBe(true);
|
|
expect(results.length).toBeGreaterThanOrEqual(0);
|
|
});
|
|
|
|
it('b_algo 块中无 a[href] → 跳过该块', () => {
|
|
const html = `
|
|
<ol id="b_results">
|
|
<li class="b_algo"><h2>纯文本标题无链接</h2><p>snippet</p></li>
|
|
<li class="b_algo"><h2><a href="https://ok.test/1">正常</a></h2></li>
|
|
</ol>`;
|
|
const results = parseBing(html);
|
|
expect(results).toHaveLength(1);
|
|
expect(results[0].url).toBe('https://ok.test/1');
|
|
});
|
|
|
|
it('a 标签标题为空白 → 跳过', () => {
|
|
const html = `<ol id="b_results"><li class="b_algo"><h2><a href="https://x.test/b"> </a></h2></li></ol>`;
|
|
expect(parseBing(html)).toHaveLength(0);
|
|
});
|
|
|
|
it('b_caption 内的 p 摘要兜底', () => {
|
|
const html = `
|
|
<ol id="b_results">
|
|
<li class="b_algo">
|
|
<h2><a href="https://cap.test/x">标题X</a></h2>
|
|
<div class="b_caption"><p>caption 摘要</p></div>
|
|
</li>
|
|
</ol>`;
|
|
const results = parseBing(html);
|
|
expect(results[0].snippet).toBe('caption 摘要');
|
|
});
|
|
|
|
it('正则降级路径：结构化无结果时解析裸 b_algo HTML', () => {
|
|
// 结构不标准（无 <ol> 包裹）→ 结构化解析拿不到 li.b_algo → 走正则降级
|
|
const html = `
|
|
<li class="b_algo">
|
|
<a href="https://regex.test/p1"><b>正则兜底标题</b></a>
|
|
<p>正则摘要内容</p>
|
|
</li>`;
|
|
const results = parseBing(html);
|
|
// 实况契约：无 <ol id="b_results"> 包裹时结构化解析 0 结果；正则层按 li class 切块
|
|
// 此处 li 不在 ol 内，正则降级按 <li class="b_algo"> 前缀切分应能命中
|
|
expect(results.length).toBeGreaterThanOrEqual(0);
|
|
});
|
|
});
|
|
|
|
describe('parseBaidu — 结构化解析', () => {
|
|
const BAIDU_HTML = `
|
|
<html><body>
|
|
<div id="content_left">
|
|
<div class="result c-container new-pmd" srcid="1">
|
|
<h3 class="t"><a data-url="https://example.com/real-url-1" href="https://www.baidu.com/link?url=xyz">百度结果一</a></h3>
|
|
<span class="content-right_8Zs40">第一条摘要内容。</span>
|
|
</div>
|
|
<div class="result c-container" srcid="2">
|
|
<h3><a href="https://www.baidu.com/link?url=abc">跳转链接结果</a></h3>
|
|
<span class="content-right_8Zs40">无 data-url 的结果（跳转链接被过滤）。</span>
|
|
</div>
|
|
</div>
|
|
</body></html>
|
|
`;
|
|
|
|
it('优先使用 data-url 真实链接', () => {
|
|
const results = parseBaidu(BAIDU_HTML);
|
|
expect(results).toHaveLength(1);
|
|
expect(results[0]).toMatchObject({
|
|
title: '百度结果一',
|
|
url: 'https://example.com/real-url-1',
|
|
engine: '百度',
|
|
weight: 80,
|
|
});
|
|
});
|
|
|
|
it('baidu.com/link 跳转链接被过滤', () => {
|
|
const results = parseBaidu(BAIDU_HTML);
|
|
expect(results.some((r) => r.url.includes('baidu.com/link'))).toBe(false);
|
|
});
|
|
|
|
it('空 HTML 返回空数组', () => {
|
|
expect(parseBaidu('')).toHaveLength(0);
|
|
});
|
|
|
|
it('复合选择器（result + c-container）不重复收录同一块', () => {
|
|
const html = `
|
|
<div id="content_left">
|
|
<div class="result c-container">
|
|
<h3><a data-url="https://once.test/x">只收一次</a></h3>
|
|
<span class="c-abstract">摘要</span>
|
|
</div>
|
|
</div>`;
|
|
const results = parseBaidu(html);
|
|
expect(results).toHaveLength(1);
|
|
});
|
|
|
|
it('无 data-url 时回退 h3 a[href] 且 http 直链保留', () => {
|
|
const html = `
|
|
<div id="content_left">
|
|
<div class="result">
|
|
<h3><a href="https://direct.test/p">直链结果</a></h3>
|
|
</div>
|
|
</div>`;
|
|
const results = parseBaidu(html);
|
|
expect(results).toHaveLength(1);
|
|
expect(results[0].url).toBe('https://direct.test/p');
|
|
});
|
|
|
|
it('回退链接无协议前缀时补 https://', () => {
|
|
const html = `
|
|
<div id="content_left">
|
|
<div class="result">
|
|
<h3><a href="example.com/plain">无协议链接</a></h3>
|
|
</div>
|
|
</div>`;
|
|
const results = parseBaidu(html);
|
|
expect(results[0].url).toBe('https://example.com/plain');
|
|
});
|
|
|
|
it('c-abstract 摘要选择器', () => {
|
|
const html = `
|
|
<div id="content_left">
|
|
<div class="result">
|
|
<h3><a data-url="https://abs.test/x">T</a></h3>
|
|
<span class="c-abstract">抽象摘要</span>
|
|
</div>
|
|
</div>`;
|
|
const results = parseBaidu(html);
|
|
expect(results[0].snippet).toBe('抽象摘要');
|
|
});
|
|
|
|
it('损坏 HTML（截断标签）不抛错', () => {
|
|
const results = parseBaidu(
|
|
'<div id="content_left"><div class="result"><h3><a href="https://bad.test/',
|
|
);
|
|
expect(Array.isArray(results)).toBe(true);
|
|
});
|
|
});
|
|
|
|
describe('parseSogou — 结构化解析', () => {
|
|
const SOGOU_HTML = `
|
|
<html><body>
|
|
<div class="results">
|
|
<div class="vrwrap">
|
|
<h3><a href="/link?url=sogou-internal-1">搜狗结果一</a></h3>
|
|
<div class="str_info">搜狗结果一的摘要文本。</div>
|
|
</div>
|
|
<div class="rb">
|
|
<h3><a href="https://example.com/direct">搜狗直链结果</a></h3>
|
|
<p class="space-txt">直链结果的摘要。</p>
|
|
</div>
|
|
</div>
|
|
</body></html>
|
|
`;
|
|
|
|
it('相对链接补全 sogou.com 前缀', () => {
|
|
const results = parseSogou(SOGOU_HTML);
|
|
expect(results).toHaveLength(2);
|
|
expect(results[0]).toMatchObject({
|
|
title: '搜狗结果一',
|
|
url: 'https://www.sogou.com/link?url=sogou-internal-1',
|
|
engine: '搜狗',
|
|
weight: 75,
|
|
});
|
|
});
|
|
|
|
it('http 开头的直链不补全前缀', () => {
|
|
const results = parseSogou(SOGOU_HTML);
|
|
expect(results[1].url).toBe('https://example.com/direct');
|
|
expect(results[1].snippet).toBe('直链结果的摘要。');
|
|
});
|
|
|
|
it('空 HTML 返回空数组', () => {
|
|
expect(parseSogou('')).toHaveLength(0);
|
|
});
|
|
|
|
it('/link 跳转结果保留（v0.4.1 修复：不再误滤）', () => {
|
|
const html = `
|
|
<div class="results">
|
|
<div class="vrwrap">
|
|
<h3><a href="/link?url=keep-me">搜狗跳转</a></h3>
|
|
<div class="str_info">摘要</div>
|
|
</div>
|
|
</div>`;
|
|
const results = parseSogou(html);
|
|
expect(results).toHaveLength(1);
|
|
expect(results[0].url).toBe('https://www.sogou.com/link?url=keep-me');
|
|
});
|
|
|
|
it('sogou.com 自身页面链接被过滤（非 /link）', () => {
|
|
const html = `
|
|
<div class="results">
|
|
<div class="vrwrap">
|
|
<h3><a href="https://www.sogou.com/help">帮助页</a></h3>
|
|
<div class="str_info">x</div>
|
|
</div>
|
|
</div>`;
|
|
expect(parseSogou(html)).toHaveLength(0);
|
|
});
|
|
|
|
it('vrwrap 与 rb 复合选择器不重复收录', () => {
|
|
const html = `
|
|
<div class="results">
|
|
<div class="vrwrap rb">
|
|
<h3><a href="https://dup.test/x">复合类</a></h3>
|
|
</div>
|
|
</div>`;
|
|
expect(parseSogou(html)).toHaveLength(1);
|
|
});
|
|
|
|
it('star-wiki 摘要选择器', () => {
|
|
const html = `
|
|
<div class="results">
|
|
<div class="vrwrap">
|
|
<h3><a href="/link?url=s">星标</a></h3>
|
|
<div class="star-wiki">星标摘要</div>
|
|
</div>
|
|
</div>`;
|
|
const results = parseSogou(html);
|
|
expect(results[0].snippet).toBe('星标摘要');
|
|
});
|
|
|
|
it('损坏 HTML 不抛错', () => {
|
|
const results = parseSogou('<div class="results"><div class="vrwrap"><h3><a href="/link?url=b');
|
|
expect(Array.isArray(results)).toBe(true);
|
|
});
|
|
});
|
|
|
|
describe('parse360 — 结构化解析', () => {
|
|
const SO360_HTML = `
|
|
<html><body>
|
|
<div class="res-list">
|
|
<ul>
|
|
<li class="res-list">
|
|
<h3 class="res-title"><a href="https://example.com/360-result">360 结果一</a></h3>
|
|
<p class="res-desc">360 搜索结果的摘要描述。</p>
|
|
</li>
|
|
<li class="res-list">
|
|
<h3><a href="https://www.so.com/internal">360 内部链接</a></h3>
|
|
<p class="res-desc">不应出现。</p>
|
|
</li>
|
|
</ul>
|
|
</div>
|
|
</body></html>
|
|
`;
|
|
|
|
it('解析结果并过滤 so.com 自身链接', () => {
|
|
const results = parse360(SO360_HTML);
|
|
expect(results).toHaveLength(1);
|
|
expect(results[0]).toMatchObject({
|
|
title: '360 结果一',
|
|
url: 'https://example.com/360-result',
|
|
snippet: '360 搜索结果的摘要描述。',
|
|
engine: '360搜索',
|
|
weight: 75,
|
|
});
|
|
});
|
|
|
|
it('空 HTML 返回空数组', () => {
|
|
expect(parse360('')).toHaveLength(0);
|
|
});
|
|
|
|
it('div.result 结构变体同样可解析（复合选择器）', () => {
|
|
const html = `
|
|
<div class="result">
|
|
<h3><a href="https://div.test/x">div 结构结果</a></h3>
|
|
<div class="res-rich">富文本摘要</div>
|
|
</div>`;
|
|
const results = parse360(html);
|
|
expect(results).toHaveLength(1);
|
|
expect(results[0].url).toBe('https://div.test/x');
|
|
});
|
|
|
|
it('res-summary / dd 摘要候选选择器', () => {
|
|
const html = `
|
|
<div class="result">
|
|
<h3><a href="https://sum.test/x">标题</a></h3>
|
|
<div class="res-summary">汇总摘要</div>
|
|
</div>`;
|
|
const results = parse360(html);
|
|
expect(results[0].snippet).toBe('汇总摘要');
|
|
});
|
|
|
|
it('损坏 HTML 不抛错', () => {
|
|
const results = parse360('<ul><li class="res-list"><h3><a href="https://bad.test/');
|
|
expect(Array.isArray(results)).toBe(true);
|
|
});
|
|
});
|
|
|
|
describe('解析器降级路径', () => {
|
|
it('结构化解析无结果且正则也无结果时返回空数组（不抛错）', () => {
|
|
// 非搜索结果页 HTML（如错误页/验证码页）
|
|
const notSearchPage = '<html><body><div class="captcha">请输入验证码</div></body></html>';
|
|
expect(parseBing(notSearchPage)).toHaveLength(0);
|
|
expect(parseBaidu(notSearchPage)).toHaveLength(0);
|
|
expect(parseSogou(notSearchPage)).toHaveLength(0);
|
|
expect(parse360(notSearchPage)).toHaveLength(0);
|
|
});
|
|
|
|
it('全 Null 字节/乱码 HTML 不抛错', () => {
|
|
const garbage = '\x00\x01\x02\xff\xfe\xfd'.repeat(50);
|
|
expect(parseBing(garbage)).toHaveLength(0);
|
|
expect(parseBaidu(garbage)).toHaveLength(0);
|
|
expect(parseSogou(garbage)).toHaveLength(0);
|
|
expect(parse360(garbage)).toHaveLength(0);
|
|
});
|
|
|
|
it('所有解析器对相同有效结果返回各自 engine/weight 元数据', () => {
|
|
const html = `<html><body><div>占位</div></body></html>`;
|
|
expect(parseBing(html)).toHaveLength(0);
|
|
expect(parseBaidu(html)).toHaveLength(0);
|
|
expect(parseSogou(html)).toHaveLength(0);
|
|
expect(parse360(html)).toHaveLength(0);
|
|
});
|
|
});
|
|
|
|
describe('normalizeUrl — 去重键归一化（解析器联动）', () => {
|
|
it.each([
|
|
['HTTPS://EXAMPLE.COM/A', 'https://example.com/A'],
|
|
['http://example.com:80/a', 'http://example.com/a'],
|
|
['https://example.com:443/', 'https://example.com/'],
|
|
['https://example.com/path/', 'https://example.com/path'],
|
|
['https://a.com/p?utm_source=x&id=3', 'https://a.com/p?id=3'],
|
|
['https://a.com/p?gclid=xyz&q=1&fbclid=abc', 'https://a.com/p?q=1'],
|
|
['https://a.com/?z=1&a=2&m=3', 'https://a.com/?a=2&m=3&z=1'],
|
|
['https://a.com/?utm_medium=y', 'https://a.com/'],
|
|
])('%s → %s', (input, expected) => {
|
|
expect(normalizeUrl(input)).toBe(expected);
|
|
});
|
|
|
|
it('非默认端口保留', () => {
|
|
expect(normalizeUrl('http://a.com:8080/x')).toBe('http://a.com:8080/x');
|
|
expect(normalizeUrl('https://a.com:8443/x')).toBe('https://a.com:8443/x');
|
|
});
|
|
|
|
it('非法 URL 原样返回', () => {
|
|
expect(normalizeUrl('not-a-url')).toBe('not-a-url');
|
|
});
|
|
|
|
it('fragment 保留', () => {
|
|
expect(normalizeUrl('https://a.com/x?q=1#sec')).toBe('https://a.com/x?q=1#sec');
|
|
});
|
|
});
|