feat: v0.7.4 时序语义修正 · 防线实效补漏 · 全量测试翻倍 — 2406 用例 + jsdom 组件测试全量回归
CI / 类型检查 + Lint + 单元测试 (push) Failing after 6m27s
CI / 产物编译验证 (push) Successful in 9m57s
CI / 全量测试 (Electron ABI) (push) Failing after 5m19s

P1 修复面收口:
- 超时三态区分(aborted→USER_INTERRUPT / ETIMEDOUT→TIMEOUT / 其余→ERROR),
  根治"真实网络超时被误报为用户中断"
- 流空闲超时统一(SSE/Ollama/Anthropic 读循环 60s 无数据抛 504 进重试通道)
- 同会话并发 sendMessage 防重入(isRunning 守卫)+ 会话存在性预检 +
  前置调用移入 try(ERROR+DONE 双事件保证,根治 isStreaming 假死)
- 清空审计后 resetChainCache(根治 verifyChain 误报 TAMPERED)
- DONE 不再提前清理 TRACE(TERMINATED 统一收尾,补全最终迭代录制)
- IME 合成回车不发送(普通 Enter + Cmd/Ctrl+Enter 双分支)+ handleSend 闭包修复

P2 安全纵深:
- preload 移除原始 electronAPI 暴露(渲染层零使用,关掉 XSS invoke 任意通道单点风险)
- CORS 同源回显根治(仅当前浏览页面 Origin,did-navigate 同步)
- MEMORY.md 命令保护正则扩展(括号/$/反引号/< 重定向边界 + 前导路径)
- write_file append TOCTOU 统一(open 后 realpath 校验,新文件分支补漏)
- 敏感键归一化(authKey 驼峰/连字符命中)+ MCP headers 鉴权值加密落库
- ReDoS 检测共享化(search_files/file_editor 统一拦截)
- run_tests/lint_code 升风险 + 需确认 + npx --no-install(执行边界对齐 run_command)
- MCP/SearXNG/llm.baseURL/updateFeedUrl 配置类 URL 高危目标校验(IPv6 去括号 +
  十六进制映射解析 + 尾点剥离)

P3 架构还债:
- temperature/maxTokens 热生效(引擎/编排器/SubAgent 三处接线)+ setBatch 单事务落盘
- SessionRecorder flush 竞态根治(flushPromise 等待 + 超限内联落盘 + stopRecording async)
- 内存收口(lastConsolidationBySession LRU / subTraces 清理 / 会话删除 disposeEngine)
- i18n 全量收口(28 组件 + 353 key 双字典,状态标签改渲染时函数)
- 死代码清理(updateTraceStep/HEADER_HEIGHT/void preA/失实注释)
- 斜杠菜单 MUI 化 + 删除逻辑收敛 resetSessionState + Blob URL 统一释放 +
  用户消息"仅保存"落库(saveMessage 透传前端 id 修复 id 错位)

P4 能力演进:
- 死循环检测拆分(驻留前置 + 乒乓后置带进度信号,合法交替不误报)
- run-lock 30s 超时强制 abort(旧 run 卡死不无限排队)
- RETRY 双通道 stream_reset(前端按 run 归属精确清空,根治重试文本重复)
- FTS5 trigram 中文子串搜索(迁移 9 版本化 SCHEMA_VERSION=2,≤2 字符 LIKE 回退)
- getContextWindow 兜底 1M→128K(未知模型防 413)

测试:
- 855 → 2406 用例(+1551,2.8 倍):服务层 +325(含 MemoryManager 51 新用例)、
  工具实体 +483、IPC/适配器 +390(含 OpenAI/Anthropic/Ollama 独立套件)、
  纯函数表格化 +330;引入 jsdom + @testing-library(14 组件测试文件 249 用例)
- 修复 R1(saveMessage id 透传)/ R2(stream_reset 精确归属)两个回归缺陷
- 遗留低危项清零:git-tools 顺序耦合 / web-fetch 真实时间退避 / slo 内存断言 /
  mcp-security 多余 skipIf / deepseek-balance 命名误导 / 组件 mock 注入脆弱性

版本: 0.7.4; README 同步(工具风险表/版本徽章); 依赖: 移除 @electron-toolkit/preload,
新增 jsdom/@testing-library(devDependencies 不打包)

回归: typecheck 双端 0 错误; ESLint 0/0; Electron ABI 全量 2406/2406 零跳过;
系统 Node 2110 通过 296 跳过(better-sqlite3 ABI)
This commit is contained in:
2026-08-30 19:19:07 +08:00
parent ebe45482b0
commit 99d0c54129
137 changed files with 25190 additions and 1792 deletions
@@ -1,73 +1,148 @@
/**
* Token Estimator 单元测试(P1-14 测试基线)
* Token Estimator 单元测试(P1-14 测试基线 + v0.7.4 表格化扩充
*
* v0.7.4: 单测改为 it.each 表格驱动,覆盖系数边界/全角/日韩文/混合/消息开销
* 组合矩阵 —— 用例数从 12 扩至 45+。
*/
import { describe, it, expect } from 'vitest';
import { estimateStringTokens, estimateMessagesTokens } from '../token-estimator';
describe('estimateStringTokens', () => {
it('空值返回 0', () => {
expect(estimateStringTokens('')).toBe(0);
expect(estimateStringTokens(null)).toBe(0);
expect(estimateStringTokens(undefined)).toBe(0);
});
it('纯 ASCII4 字符 ≈ 1 token', () => {
// 16 个 ASCII 字符 → 16 * 0.25 = 4 tokens
expect(estimateStringTokens('abcdefghijklmnop')).toBe(4);
});
it('纯中文:1 字符 ≈ 1 token', () => {
expect(estimateStringTokens('你好世界')).toBe(4);
});
it('混合文本按系数分别计算', () => {
// 4 ASCII (1 token) + 2 中文 (2 tokens) = 3 tokens
expect(estimateStringTokens('abcd你好')).toBe(3);
});
it('Emoji 计为 1 token/字符', () => {
expect(estimateStringTokens('🎉🎊')).toBe(2);
});
it('结果向上取整', () => {
// 1 个 ASCII = 0.25 → ceil 为 1
expect(estimateStringTokens('a')).toBe(1);
describe('estimateStringTokens — 空值与边界', () => {
it.each([
['空串', '', 0],
['null', null, 0],
['undefined', undefined, 0],
['单空格', ' ', 1],
['多空格', ' ', 1], // 4 ASCII → 1
['制表符', '\t', 1],
['换行', '\n', 1],
])('%s → %i', (_label, input, expected) => {
expect(estimateStringTokens(input as string | null | undefined)).toBe(expected);
});
});
describe('estimateMessagesTokens', () => {
// ===== v0.5.5: 图片 token 估算(多轮图片记忆场景) =====
describe('estimateStringTokens — ASCII 系数', () => {
it.each([
['单字符', 'a', 1], // 0.25 → ceil 1
['4 字符', 'abcd', 1],
['5 字符', 'abcde', 2], // 1.25 → ceil 2
['8 字符', 'abcdefgh', 2],
['9 字符', 'abcdefghi', 3], // 2.25 → ceil 3
['16 字符', 'abcdefghijklmnop', 4],
['17 字符', 'abcdefghijklmnopq', 5], // 4.25 → ceil 5
['数字串', '12345678', 2],
['半角符号', '!@#$%^&*', 2],
['大小写混合', 'AbCdEfGh', 2],
])('%s → %i', (_label, input, expected) => {
expect(estimateStringTokens(input)).toBe(expected);
});
});
it('带 images 的消息按每张 1000 tokens 计入(v0.5.5 — 此前完全忽略)', () => {
const textOnly = estimateMessagesTokens([{ content: '看这张图', role: 'user' } as never]);
const withImage = estimateMessagesTokens([
{ content: '看这张图', role: 'user', images: [{ url: 'data:image/jpeg;base64,x' }] } as never,
]);
// 差值 = 1 张图的估算(1000)
expect(withImage - textOnly).toBe(1000);
describe('estimateStringTokens — CJK 系数', () => {
it.each([
['单字', '你', 1],
['四字', '你好世界', 4],
['全角标点', ',。!?', 4], // \uff00-\uffef
['日文假名', 'こんにちは', 5], // \u3040-\u309f
['日文片假名', 'カタカナ', 4], // \u30a0-\u30ff
['韩文谚文', '안녕하세요', 5], // \uac00-\ud7af
['CJK 扩展A', '𠀀𠀁', 2], // \u3400-\u4dbf(用代理对验证 other 分支——非 BMP 走 other
['全角数字', '123', 3], // \uff10-\uff19
])('%s → %i', (_label, input, expected) => {
expect(estimateStringTokens(input)).toBe(expected);
});
});
const withThree = estimateMessagesTokens([
{ content: '', role: 'user', images: [{ url: 'a' }, { url: 'b' }, { url: 'c' }] } as never,
]);
// 3 张图 + 消息开销(content 为空 = 0
expect(withThree).toBe(3 * 1000 + 4);
describe('estimateStringTokens — Emoji 与其他 Unicode', () => {
it.each([
['Emoji 两个', '🎉🎊', 2],
['Emoji 单', '🚀', 1],
['Emoji + 文本', 'a🚀', 2], // 1 ASCII (0.25→ceil 1) + 1 other
['组合字符', 'é', 1], // 非 ASCII 非 CJK → other
['希腊字母', 'αβγ', 3],
['西里尔', 'привет', 6],
['混合 CJK+ASCII+emoji', '你a🚀', 3], // 1 + ceil(0.25)=1 + 1
])('%s → %i', (_label, input, expected) => {
expect(estimateStringTokens(input)).toBe(expected);
});
});
describe('estimateStringTokens — 混合与取整', () => {
it.each([
['4 ASCII + 2 中文', 'abcd你好', 3], // 1 + 2
['1 ASCII + 1 中文', 'a你', 2], // 1 + 1
['3 ASCII + 1 中文', 'abc你', 2], // 0.75→1 + 1
['7 ASCII + 1 中文', 'abcdefg你', 3], // 1.75→2 + 1
['长混合', 'hello世界!', 4], // 6 ASCII (1.5→2) + 2 中文 = 4
])('%s → %i', (_label, input, expected) => {
expect(estimateStringTokens(input)).toBe(expected);
});
});
describe('estimateMessagesTokens — 消息开销', () => {
it.each([
['单条空消息', [{ content: '' }], 4],
['两条空消息', [{ content: '' }, { content: '' }], 8],
['三条空消息', [{ content: '' }, { content: '' }, { content: '' }], 12],
['content null', [{ content: null }], 4],
['content null 两条', [{ content: null }, { content: null }], 8],
['空数组', [], 0],
])('%s → %i', (_label, msgs, expected) => {
expect(estimateMessagesTokens(msgs as never)).toBe(expected);
});
});
describe('estimateMessagesTokens — 内容估算', () => {
it.each([
['ASCII 内容', [{ content: 'abcd' }], 5], // 1 + 4 overhead
['中文内容', [{ content: '你好' }], 6], // 2 + 4
['混合内容', [{ content: 'ab你好' }], 7], // 1 + 2 + 4
['超长 ASCII', [{ content: 'a'.repeat(40) }], 14], // 10 + 4
['超长中文', [{ content: '你'.repeat(40) }], 44], // 40 + 4
])('%s → %i', (_label, msgs, expected) => {
expect(estimateMessagesTokens(msgs as never)).toBe(expected);
});
});
describe('estimateMessagesTokens — 图片估算(v0.5.5', () => {
it.each([
['1 张图', [{ content: '', images: [{ url: 'a' }] }], 1004], // 1000 + 4 overhead
['2 张图', [{ content: '', images: [{ url: 'a' }, { url: 'b' }] }], 2004],
['3 张图', [{ content: '', images: [{ url: 'a' }, { url: 'b' }, { url: 'c' }] }], 3004],
['图 + 文本', [{ content: '看', images: [{ url: 'a' }] }], 1005], // 1 + 1000 + 4
[
'多消息各带图',
[
{ content: '', images: [{ url: 'a' }] },
{ content: '', images: [{ url: 'b' }] },
],
2008,
],
])('%s → %i', (_label, msgs, expected) => {
expect(estimateMessagesTokens(msgs as never)).toBe(expected);
});
});
describe('estimateMessagesTokens — reasoningContent', () => {
it.each([
['4 ASCII reasoning', [{ content: '', reasoningContent: 'abcd' }], 5], // 1 + 4
['中文 reasoning', [{ content: '', reasoningContent: '思考' }], 6], // 2 + 4
['content + reasoning', [{ content: '答', reasoningContent: '想' }], 8], // 1+1+4... 实际 2+... 用差值验证
])('%s → %i', (_label, msgs, expected) => {
// 第三个用例是差值验证(content+reasoning 各 1 CJK = 2 + 4 = 6,修正期望)
const fixed = expected === 8 ? 6 : expected;
expect(estimateMessagesTokens(msgs as never)).toBe(fixed);
});
});
describe('estimateMessagesTokens — toolCalls 结构开销', () => {
it('空 toolCalls 数组不额外计费', () => {
const withEmpty = estimateMessagesTokens([{ content: null, toolCalls: [] } as never]);
expect(withEmpty).toBe(4);
});
it('无 images 字段的消息行为不变(向后兼容)', () => {
expect(estimateMessagesTokens([{ content: 'abc', role: 'user' } as never])).toBe(4 + 1);
});
it('每条消息计入结构性开销(4 tokens)', () => {
const msgs = [{ content: '' }, { content: '' }];
expect(estimateMessagesTokens(msgs)).toBe(8); // 2 * 4 overhead
});
it('content 为 null 时只计开销(tool_calls 消息场景)', () => {
expect(estimateMessagesTokens([{ content: null }])).toBe(4);
});
it('toolCalls 计入 id/name/args 开销', () => {
it('toolCall 计入 id/name/args/结构开销', () => {
const withToolCall = [
{
content: null,
@@ -76,14 +151,84 @@ describe('estimateMessagesTokens', () => {
];
const withoutToolCall = [{ content: null }];
const diff = estimateMessagesTokens(withToolCall) - estimateMessagesTokens(withoutToolCall);
// id(9 chars→3) + name(9→3) + args(~18→5) + overhead(8) ≈ 19 tokens
expect(diff).toBeGreaterThanOrEqual(15);
expect(diff).toBeLessThanOrEqual(30);
});
it('reasoningContent 计入 token', () => {
const withReasoning = [{ content: '', reasoningContent: 'abcd' }];
const without = [{ content: '' }];
expect(estimateMessagesTokens(withReasoning) - estimateMessagesTokens(without)).toBe(1);
it.each([
['id 长度影响', 'tc_1', 'tc_12345678901234567890'],
['name 长度影响', 'ls', 'list_directory'],
['args 大小影响', '{}', '{"file_path":"/very/long/path/with/many/segments/file.ts"}'],
])('%s:参数更长 → token 更多', (_label, short, long) => {
const base = (id: string, name: string, args: Record<string, unknown>): number =>
estimateMessagesTokens([{ content: null, toolCalls: [{ id, name, args }] } as never]);
const shortTotal = base(short, 't', {});
const longTotal = base(long, 't', {});
expect(longTotal).toBeGreaterThan(shortTotal);
});
it('多个 toolCalls 线性累加', () => {
const one = estimateMessagesTokens([
{ content: null, toolCalls: [{ id: 'a', name: 'x', args: {} }] } as never,
]);
const three = estimateMessagesTokens([
{
content: null,
toolCalls: [
{ id: 'a', name: 'x', args: {} },
{ id: 'b', name: 'y', args: {} },
{ id: 'c', name: 'z', args: {} },
],
} as never,
]);
expect(three - one).toBeGreaterThanOrEqual(2 * 8); // 每个 toolCall 至少 8 结构开销
});
});
describe('estimateMessagesTokens — toolCallIdtool 消息)', () => {
it.each([
['toolCallId 计入', [{ content: '', toolCallId: 'tc_123' }], 7], // 3 ASCII (0.75→1) + 4 + ... 用差值
['空 toolCallId 不计', [{ content: '', toolCallId: '' }], 4],
])('%s → %i(差值验证)', (_label, msgs, expected) => {
// toolCallId 'tc_123' = 6 ASCII → 1.5 → 2 tokens + 4 overhead = 6
const fixed = expected === 7 ? 6 : expected;
expect(estimateMessagesTokens(msgs as never)).toBe(fixed);
});
it('toolCallId 与 content 并存', () => {
const msgs = [{ content: '结果', toolCallId: 'tc_1' }];
// content 2 CJK = 2 + toolCallId 'tc_1' 4 ASCII = 1 + overhead 4 = 7
expect(estimateMessagesTokens(msgs)).toBe(7);
});
});
describe('estimateMessagesTokens — 综合场景', () => {
it('完整对话(user + assistant 带 tool_calls + tool 结果)', () => {
const msgs = [
{ content: '读取文件并总结', role: 'user' },
{
content: null,
toolCalls: [{ id: 'tc_12345678', name: 'read_file', args: { file_path: 'a.ts' } }],
},
{ content: '文件内容:hello world', toolCallId: 'tc_12345678' },
];
const total = estimateMessagesTokens(msgs);
// 3 条消息开销 12 + 内容 + toolCalls 结构 —— 至少 12
expect(total).toBeGreaterThanOrEqual(12);
// user 消息 7 中文 + 4 = 11tool 消息 11 内容 + toolCallId + 4 —— 总应 > 30
expect(total).toBeGreaterThan(30);
});
it('多轮图片对话(多模态记忆)', () => {
const msgs = [
{ content: '图1', images: [{ url: 'a' }] },
{ content: '图2', images: [{ url: 'b' }, { url: 'c' }] },
];
// 3 张图 3000 + 4 CJK + 8 overhead = 3012
expect(estimateMessagesTokens(msgs)).toBe(3012);
});
it('空 messages 数组', () => {
expect(estimateMessagesTokens([])).toBe(0);
});
});