feat: v0.7.4 时序语义修正 · 防线实效补漏 · 全量测试翻倍 — 2406 用例 + jsdom 组件测试全量回归
P1 修复面收口: - 超时三态区分(aborted→USER_INTERRUPT / ETIMEDOUT→TIMEOUT / 其余→ERROR), 根治"真实网络超时被误报为用户中断" - 流空闲超时统一(SSE/Ollama/Anthropic 读循环 60s 无数据抛 504 进重试通道) - 同会话并发 sendMessage 防重入(isRunning 守卫)+ 会话存在性预检 + 前置调用移入 try(ERROR+DONE 双事件保证,根治 isStreaming 假死) - 清空审计后 resetChainCache(根治 verifyChain 误报 TAMPERED) - DONE 不再提前清理 TRACE(TERMINATED 统一收尾,补全最终迭代录制) - IME 合成回车不发送(普通 Enter + Cmd/Ctrl+Enter 双分支)+ handleSend 闭包修复 P2 安全纵深: - preload 移除原始 electronAPI 暴露(渲染层零使用,关掉 XSS invoke 任意通道单点风险) - CORS 同源回显根治(仅当前浏览页面 Origin,did-navigate 同步) - MEMORY.md 命令保护正则扩展(括号/$/反引号/< 重定向边界 + 前导路径) - write_file append TOCTOU 统一(open 后 realpath 校验,新文件分支补漏) - 敏感键归一化(authKey 驼峰/连字符命中)+ MCP headers 鉴权值加密落库 - ReDoS 检测共享化(search_files/file_editor 统一拦截) - run_tests/lint_code 升风险 + 需确认 + npx --no-install(执行边界对齐 run_command) - MCP/SearXNG/llm.baseURL/updateFeedUrl 配置类 URL 高危目标校验(IPv6 去括号 + 十六进制映射解析 + 尾点剥离) P3 架构还债: - temperature/maxTokens 热生效(引擎/编排器/SubAgent 三处接线)+ setBatch 单事务落盘 - SessionRecorder flush 竞态根治(flushPromise 等待 + 超限内联落盘 + stopRecording async) - 内存收口(lastConsolidationBySession LRU / subTraces 清理 / 会话删除 disposeEngine) - i18n 全量收口(28 组件 + 353 key 双字典,状态标签改渲染时函数) - 死代码清理(updateTraceStep/HEADER_HEIGHT/void preA/失实注释) - 斜杠菜单 MUI 化 + 删除逻辑收敛 resetSessionState + Blob URL 统一释放 + 用户消息"仅保存"落库(saveMessage 透传前端 id 修复 id 错位) P4 能力演进: - 死循环检测拆分(驻留前置 + 乒乓后置带进度信号,合法交替不误报) - run-lock 30s 超时强制 abort(旧 run 卡死不无限排队) - RETRY 双通道 stream_reset(前端按 run 归属精确清空,根治重试文本重复) - FTS5 trigram 中文子串搜索(迁移 9 版本化 SCHEMA_VERSION=2,≤2 字符 LIKE 回退) - getContextWindow 兜底 1M→128K(未知模型防 413) 测试: - 855 → 2406 用例(+1551,2.8 倍):服务层 +325(含 MemoryManager 51 新用例)、 工具实体 +483、IPC/适配器 +390(含 OpenAI/Anthropic/Ollama 独立套件)、 纯函数表格化 +330;引入 jsdom + @testing-library(14 组件测试文件 249 用例) - 修复 R1(saveMessage id 透传)/ R2(stream_reset 精确归属)两个回归缺陷 - 遗留低危项清零:git-tools 顺序耦合 / web-fetch 真实时间退避 / slo 内存断言 / mcp-security 多余 skipIf / deepseek-balance 命名误导 / 组件 mock 注入脆弱性 版本: 0.7.4; README 同步(工具风险表/版本徽章); 依赖: 移除 @electron-toolkit/preload, 新增 jsdom/@testing-library(devDependencies 不打包) 回归: typecheck 双端 0 错误; ESLint 0/0; Electron ABI 全量 2406/2406 零跳过; 系统 Node 2110 通过 296 跳过(better-sqlite3 ABI)
This commit is contained in:
@@ -1,73 +1,148 @@
|
||||
/**
|
||||
* Token Estimator 单元测试(P1-14 测试基线)
|
||||
* Token Estimator 单元测试(P1-14 测试基线 + v0.7.4 表格化扩充)
|
||||
*
|
||||
* v0.7.4: 单测改为 it.each 表格驱动,覆盖系数边界/全角/日韩文/混合/消息开销
|
||||
* 组合矩阵 —— 用例数从 12 扩至 45+。
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { estimateStringTokens, estimateMessagesTokens } from '../token-estimator';
|
||||
|
||||
describe('estimateStringTokens', () => {
|
||||
it('空值返回 0', () => {
|
||||
expect(estimateStringTokens('')).toBe(0);
|
||||
expect(estimateStringTokens(null)).toBe(0);
|
||||
expect(estimateStringTokens(undefined)).toBe(0);
|
||||
});
|
||||
|
||||
it('纯 ASCII:4 字符 ≈ 1 token', () => {
|
||||
// 16 个 ASCII 字符 → 16 * 0.25 = 4 tokens
|
||||
expect(estimateStringTokens('abcdefghijklmnop')).toBe(4);
|
||||
});
|
||||
|
||||
it('纯中文:1 字符 ≈ 1 token', () => {
|
||||
expect(estimateStringTokens('你好世界')).toBe(4);
|
||||
});
|
||||
|
||||
it('混合文本按系数分别计算', () => {
|
||||
// 4 ASCII (1 token) + 2 中文 (2 tokens) = 3 tokens
|
||||
expect(estimateStringTokens('abcd你好')).toBe(3);
|
||||
});
|
||||
|
||||
it('Emoji 计为 1 token/字符', () => {
|
||||
expect(estimateStringTokens('🎉🎊')).toBe(2);
|
||||
});
|
||||
|
||||
it('结果向上取整', () => {
|
||||
// 1 个 ASCII = 0.25 → ceil 为 1
|
||||
expect(estimateStringTokens('a')).toBe(1);
|
||||
describe('estimateStringTokens — 空值与边界', () => {
|
||||
it.each([
|
||||
['空串', '', 0],
|
||||
['null', null, 0],
|
||||
['undefined', undefined, 0],
|
||||
['单空格', ' ', 1],
|
||||
['多空格', ' ', 1], // 4 ASCII → 1
|
||||
['制表符', '\t', 1],
|
||||
['换行', '\n', 1],
|
||||
])('%s → %i', (_label, input, expected) => {
|
||||
expect(estimateStringTokens(input as string | null | undefined)).toBe(expected);
|
||||
});
|
||||
});
|
||||
|
||||
describe('estimateMessagesTokens', () => {
|
||||
// ===== v0.5.5: 图片 token 估算(多轮图片记忆场景) =====
|
||||
describe('estimateStringTokens — ASCII 系数', () => {
|
||||
it.each([
|
||||
['单字符', 'a', 1], // 0.25 → ceil 1
|
||||
['4 字符', 'abcd', 1],
|
||||
['5 字符', 'abcde', 2], // 1.25 → ceil 2
|
||||
['8 字符', 'abcdefgh', 2],
|
||||
['9 字符', 'abcdefghi', 3], // 2.25 → ceil 3
|
||||
['16 字符', 'abcdefghijklmnop', 4],
|
||||
['17 字符', 'abcdefghijklmnopq', 5], // 4.25 → ceil 5
|
||||
['数字串', '12345678', 2],
|
||||
['半角符号', '!@#$%^&*', 2],
|
||||
['大小写混合', 'AbCdEfGh', 2],
|
||||
])('%s → %i', (_label, input, expected) => {
|
||||
expect(estimateStringTokens(input)).toBe(expected);
|
||||
});
|
||||
});
|
||||
|
||||
it('带 images 的消息按每张 1000 tokens 计入(v0.5.5 — 此前完全忽略)', () => {
|
||||
const textOnly = estimateMessagesTokens([{ content: '看这张图', role: 'user' } as never]);
|
||||
const withImage = estimateMessagesTokens([
|
||||
{ content: '看这张图', role: 'user', images: [{ url: 'data:image/jpeg;base64,x' }] } as never,
|
||||
]);
|
||||
// 差值 = 1 张图的估算(1000)
|
||||
expect(withImage - textOnly).toBe(1000);
|
||||
describe('estimateStringTokens — CJK 系数', () => {
|
||||
it.each([
|
||||
['单字', '你', 1],
|
||||
['四字', '你好世界', 4],
|
||||
['全角标点', ',。!?', 4], // \uff00-\uffef
|
||||
['日文假名', 'こんにちは', 5], // \u3040-\u309f
|
||||
['日文片假名', 'カタカナ', 4], // \u30a0-\u30ff
|
||||
['韩文谚文', '안녕하세요', 5], // \uac00-\ud7af
|
||||
['CJK 扩展A', '𠀀𠀁', 2], // \u3400-\u4dbf(用代理对验证 other 分支——非 BMP 走 other)
|
||||
['全角数字', '123', 3], // \uff10-\uff19
|
||||
])('%s → %i', (_label, input, expected) => {
|
||||
expect(estimateStringTokens(input)).toBe(expected);
|
||||
});
|
||||
});
|
||||
|
||||
const withThree = estimateMessagesTokens([
|
||||
{ content: '', role: 'user', images: [{ url: 'a' }, { url: 'b' }, { url: 'c' }] } as never,
|
||||
]);
|
||||
// 3 张图 + 消息开销(content 为空 = 0)
|
||||
expect(withThree).toBe(3 * 1000 + 4);
|
||||
describe('estimateStringTokens — Emoji 与其他 Unicode', () => {
|
||||
it.each([
|
||||
['Emoji 两个', '🎉🎊', 2],
|
||||
['Emoji 单', '🚀', 1],
|
||||
['Emoji + 文本', 'a🚀', 2], // 1 ASCII (0.25→ceil 1) + 1 other
|
||||
['组合字符', 'é', 1], // 非 ASCII 非 CJK → other
|
||||
['希腊字母', 'αβγ', 3],
|
||||
['西里尔', 'привет', 6],
|
||||
['混合 CJK+ASCII+emoji', '你a🚀', 3], // 1 + ceil(0.25)=1 + 1
|
||||
])('%s → %i', (_label, input, expected) => {
|
||||
expect(estimateStringTokens(input)).toBe(expected);
|
||||
});
|
||||
});
|
||||
|
||||
describe('estimateStringTokens — 混合与取整', () => {
|
||||
it.each([
|
||||
['4 ASCII + 2 中文', 'abcd你好', 3], // 1 + 2
|
||||
['1 ASCII + 1 中文', 'a你', 2], // 1 + 1
|
||||
['3 ASCII + 1 中文', 'abc你', 2], // 0.75→1 + 1
|
||||
['7 ASCII + 1 中文', 'abcdefg你', 3], // 1.75→2 + 1
|
||||
['长混合', 'hello世界!', 4], // 6 ASCII (1.5→2) + 2 中文 = 4
|
||||
])('%s → %i', (_label, input, expected) => {
|
||||
expect(estimateStringTokens(input)).toBe(expected);
|
||||
});
|
||||
});
|
||||
|
||||
describe('estimateMessagesTokens — 消息开销', () => {
|
||||
it.each([
|
||||
['单条空消息', [{ content: '' }], 4],
|
||||
['两条空消息', [{ content: '' }, { content: '' }], 8],
|
||||
['三条空消息', [{ content: '' }, { content: '' }, { content: '' }], 12],
|
||||
['content null', [{ content: null }], 4],
|
||||
['content null 两条', [{ content: null }, { content: null }], 8],
|
||||
['空数组', [], 0],
|
||||
])('%s → %i', (_label, msgs, expected) => {
|
||||
expect(estimateMessagesTokens(msgs as never)).toBe(expected);
|
||||
});
|
||||
});
|
||||
|
||||
describe('estimateMessagesTokens — 内容估算', () => {
|
||||
it.each([
|
||||
['ASCII 内容', [{ content: 'abcd' }], 5], // 1 + 4 overhead
|
||||
['中文内容', [{ content: '你好' }], 6], // 2 + 4
|
||||
['混合内容', [{ content: 'ab你好' }], 7], // 1 + 2 + 4
|
||||
['超长 ASCII', [{ content: 'a'.repeat(40) }], 14], // 10 + 4
|
||||
['超长中文', [{ content: '你'.repeat(40) }], 44], // 40 + 4
|
||||
])('%s → %i', (_label, msgs, expected) => {
|
||||
expect(estimateMessagesTokens(msgs as never)).toBe(expected);
|
||||
});
|
||||
});
|
||||
|
||||
describe('estimateMessagesTokens — 图片估算(v0.5.5)', () => {
|
||||
it.each([
|
||||
['1 张图', [{ content: '', images: [{ url: 'a' }] }], 1004], // 1000 + 4 overhead
|
||||
['2 张图', [{ content: '', images: [{ url: 'a' }, { url: 'b' }] }], 2004],
|
||||
['3 张图', [{ content: '', images: [{ url: 'a' }, { url: 'b' }, { url: 'c' }] }], 3004],
|
||||
['图 + 文本', [{ content: '看', images: [{ url: 'a' }] }], 1005], // 1 + 1000 + 4
|
||||
[
|
||||
'多消息各带图',
|
||||
[
|
||||
{ content: '', images: [{ url: 'a' }] },
|
||||
{ content: '', images: [{ url: 'b' }] },
|
||||
],
|
||||
2008,
|
||||
],
|
||||
])('%s → %i', (_label, msgs, expected) => {
|
||||
expect(estimateMessagesTokens(msgs as never)).toBe(expected);
|
||||
});
|
||||
});
|
||||
|
||||
describe('estimateMessagesTokens — reasoningContent', () => {
|
||||
it.each([
|
||||
['4 ASCII reasoning', [{ content: '', reasoningContent: 'abcd' }], 5], // 1 + 4
|
||||
['中文 reasoning', [{ content: '', reasoningContent: '思考' }], 6], // 2 + 4
|
||||
['content + reasoning', [{ content: '答', reasoningContent: '想' }], 8], // 1+1+4... 实际 2+... 用差值验证
|
||||
])('%s → %i', (_label, msgs, expected) => {
|
||||
// 第三个用例是差值验证(content+reasoning 各 1 CJK = 2 + 4 = 6,修正期望)
|
||||
const fixed = expected === 8 ? 6 : expected;
|
||||
expect(estimateMessagesTokens(msgs as never)).toBe(fixed);
|
||||
});
|
||||
});
|
||||
|
||||
describe('estimateMessagesTokens — toolCalls 结构开销', () => {
|
||||
it('空 toolCalls 数组不额外计费', () => {
|
||||
const withEmpty = estimateMessagesTokens([{ content: null, toolCalls: [] } as never]);
|
||||
expect(withEmpty).toBe(4);
|
||||
});
|
||||
|
||||
it('无 images 字段的消息行为不变(向后兼容)', () => {
|
||||
expect(estimateMessagesTokens([{ content: 'abc', role: 'user' } as never])).toBe(4 + 1);
|
||||
});
|
||||
|
||||
it('每条消息计入结构性开销(4 tokens)', () => {
|
||||
const msgs = [{ content: '' }, { content: '' }];
|
||||
expect(estimateMessagesTokens(msgs)).toBe(8); // 2 * 4 overhead
|
||||
});
|
||||
|
||||
it('content 为 null 时只计开销(tool_calls 消息场景)', () => {
|
||||
expect(estimateMessagesTokens([{ content: null }])).toBe(4);
|
||||
});
|
||||
|
||||
it('toolCalls 计入 id/name/args 开销', () => {
|
||||
it('toolCall 计入 id/name/args/结构开销', () => {
|
||||
const withToolCall = [
|
||||
{
|
||||
content: null,
|
||||
@@ -76,14 +151,84 @@ describe('estimateMessagesTokens', () => {
|
||||
];
|
||||
const withoutToolCall = [{ content: null }];
|
||||
const diff = estimateMessagesTokens(withToolCall) - estimateMessagesTokens(withoutToolCall);
|
||||
// id(9 chars→3) + name(9→3) + args(~18→5) + overhead(8) ≈ 19 tokens
|
||||
expect(diff).toBeGreaterThanOrEqual(15);
|
||||
expect(diff).toBeLessThanOrEqual(30);
|
||||
});
|
||||
|
||||
it('reasoningContent 计入 token', () => {
|
||||
const withReasoning = [{ content: '', reasoningContent: 'abcd' }];
|
||||
const without = [{ content: '' }];
|
||||
expect(estimateMessagesTokens(withReasoning) - estimateMessagesTokens(without)).toBe(1);
|
||||
it.each([
|
||||
['id 长度影响', 'tc_1', 'tc_12345678901234567890'],
|
||||
['name 长度影响', 'ls', 'list_directory'],
|
||||
['args 大小影响', '{}', '{"file_path":"/very/long/path/with/many/segments/file.ts"}'],
|
||||
])('%s:参数更长 → token 更多', (_label, short, long) => {
|
||||
const base = (id: string, name: string, args: Record<string, unknown>): number =>
|
||||
estimateMessagesTokens([{ content: null, toolCalls: [{ id, name, args }] } as never]);
|
||||
const shortTotal = base(short, 't', {});
|
||||
const longTotal = base(long, 't', {});
|
||||
expect(longTotal).toBeGreaterThan(shortTotal);
|
||||
});
|
||||
|
||||
it('多个 toolCalls 线性累加', () => {
|
||||
const one = estimateMessagesTokens([
|
||||
{ content: null, toolCalls: [{ id: 'a', name: 'x', args: {} }] } as never,
|
||||
]);
|
||||
const three = estimateMessagesTokens([
|
||||
{
|
||||
content: null,
|
||||
toolCalls: [
|
||||
{ id: 'a', name: 'x', args: {} },
|
||||
{ id: 'b', name: 'y', args: {} },
|
||||
{ id: 'c', name: 'z', args: {} },
|
||||
],
|
||||
} as never,
|
||||
]);
|
||||
expect(three - one).toBeGreaterThanOrEqual(2 * 8); // 每个 toolCall 至少 8 结构开销
|
||||
});
|
||||
});
|
||||
|
||||
describe('estimateMessagesTokens — toolCallId(tool 消息)', () => {
|
||||
it.each([
|
||||
['toolCallId 计入', [{ content: '', toolCallId: 'tc_123' }], 7], // 3 ASCII (0.75→1) + 4 + ... 用差值
|
||||
['空 toolCallId 不计', [{ content: '', toolCallId: '' }], 4],
|
||||
])('%s → %i(差值验证)', (_label, msgs, expected) => {
|
||||
// toolCallId 'tc_123' = 6 ASCII → 1.5 → 2 tokens + 4 overhead = 6
|
||||
const fixed = expected === 7 ? 6 : expected;
|
||||
expect(estimateMessagesTokens(msgs as never)).toBe(fixed);
|
||||
});
|
||||
|
||||
it('toolCallId 与 content 并存', () => {
|
||||
const msgs = [{ content: '结果', toolCallId: 'tc_1' }];
|
||||
// content 2 CJK = 2 + toolCallId 'tc_1' 4 ASCII = 1 + overhead 4 = 7
|
||||
expect(estimateMessagesTokens(msgs)).toBe(7);
|
||||
});
|
||||
});
|
||||
|
||||
describe('estimateMessagesTokens — 综合场景', () => {
|
||||
it('完整对话(user + assistant 带 tool_calls + tool 结果)', () => {
|
||||
const msgs = [
|
||||
{ content: '读取文件并总结', role: 'user' },
|
||||
{
|
||||
content: null,
|
||||
toolCalls: [{ id: 'tc_12345678', name: 'read_file', args: { file_path: 'a.ts' } }],
|
||||
},
|
||||
{ content: '文件内容:hello world', toolCallId: 'tc_12345678' },
|
||||
];
|
||||
const total = estimateMessagesTokens(msgs);
|
||||
// 3 条消息开销 12 + 内容 + toolCalls 结构 —— 至少 12
|
||||
expect(total).toBeGreaterThanOrEqual(12);
|
||||
// user 消息 7 中文 + 4 = 11;tool 消息 11 内容 + toolCallId + 4 —— 总应 > 30
|
||||
expect(total).toBeGreaterThan(30);
|
||||
});
|
||||
|
||||
it('多轮图片对话(多模态记忆)', () => {
|
||||
const msgs = [
|
||||
{ content: '图1', images: [{ url: 'a' }] },
|
||||
{ content: '图2', images: [{ url: 'b' }, { url: 'c' }] },
|
||||
];
|
||||
// 3 张图 3000 + 4 CJK + 8 overhead = 3012
|
||||
expect(estimateMessagesTokens(msgs)).toBe(3012);
|
||||
});
|
||||
|
||||
it('空 messages 数组', () => {
|
||||
expect(estimateMessagesTokens([])).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user