feat: v0.4.0 四阶段迭代 — 安全加固 + 工程基线 + 架构重构 + 双 Provider 扩展
P0 安全修复: - API Key 加密存储(safeStorage 密钥链,版本化前缀,历史明文平滑兼容) - 间接提示注入防护(SecurityScanHook 工具结果深扫描,网络工具脱敏/本地工具警示分级) - error:report IPC 断链修复(渲染进程错误上报落 electron-log + 审计) - abort 信号贯通工具层(run_command/dev-tools 子进程随会话中断终止) - run_command 沙箱加固(cd 系统目录/敏感文件读取拦截 + chcp 前缀剥离防解析退化) - .env 真实生效(dotenv 回退加载,应用内配置优先) P1 工程基础: - ESLint 9 flat config + 全部 34 条存量 warnings 清零(零容忍基线) - 测试基线 118 用例 11 文件(token/文件防护/权限/沙箱/注入/命令/引擎/注册表/审计链/摘要分层) - test:electron 双模式(ELECTRON_RUN_AS_NODE 跑 Electron ABI,SQLite 套件全执行) - SessionRecorder 多会话隔离 + 9 种 TRACE 事件补全(含最终轮 iteration_end) - Provider 故障转移(重试耗尽/不可重试一次性切换 fallback + 前端通知) - MCP 真就绪(等待全部连接完成再广播 tools:ready) - SLO/HealthChecker 真实接入(60s 巡检 + 托盘状态) - CONFIG_DEFAULTS 单一来源(消除 SEED 双源漂移) P2 架构升级: - handlers.ts 1940 行拆分为 13 个 IPC 域模块(防重入注册 + 多窗口广播) - AgentEngineManager 每会话独立引擎(LRU 30 + adapter 工厂隔离 abort 信号) - TaskOrchestrator EngineProvider 改造 + abortByParent 联动中断 SubAgent - 会话摘要分层上下文(session_summaries 滚动摘要 + 截断游标清理防因果污染) - 消息编辑重发/重新生成(truncateAfter IPC + store 动作 + UI) - Markdown 导出 / WebSearch 并行抓取(并发 3)/ 记忆 TF 缓存 / 版本构建期注入 P3 能力扩展: - OpenAI Adapter(o 系列推理模型 reasoning_effort/max_completion_tokens) - Anthropic Adapter(原生 Messages API:tool_use 块/角色合并/thinking budget/图片 base64/SSE 事件机) - 设置页/Onboarding 六 Provider 全链路接入
This commit is contained in:
@@ -0,0 +1,68 @@
|
||||
/**
|
||||
* Token Estimator 单元测试(P1-14 测试基线)
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { estimateStringTokens, estimateMessagesTokens } from '../token-estimator';
|
||||
|
||||
describe('estimateStringTokens', () => {
|
||||
it('空值返回 0', () => {
|
||||
expect(estimateStringTokens('')).toBe(0);
|
||||
expect(estimateStringTokens(null)).toBe(0);
|
||||
expect(estimateStringTokens(undefined)).toBe(0);
|
||||
});
|
||||
|
||||
it('纯 ASCII:4 字符 ≈ 1 token', () => {
|
||||
// 16 个 ASCII 字符 → 16 * 0.25 = 4 tokens
|
||||
expect(estimateStringTokens('abcdefghijklmnop')).toBe(4);
|
||||
});
|
||||
|
||||
it('纯中文:1 字符 ≈ 1 token', () => {
|
||||
expect(estimateStringTokens('你好世界')).toBe(4);
|
||||
});
|
||||
|
||||
it('混合文本按系数分别计算', () => {
|
||||
// 4 ASCII (1 token) + 2 中文 (2 tokens) = 3 tokens
|
||||
expect(estimateStringTokens('abcd你好')).toBe(3);
|
||||
});
|
||||
|
||||
it('Emoji 计为 1 token/字符', () => {
|
||||
expect(estimateStringTokens('🎉🎊')).toBe(2);
|
||||
});
|
||||
|
||||
it('结果向上取整', () => {
|
||||
// 1 个 ASCII = 0.25 → ceil 为 1
|
||||
expect(estimateStringTokens('a')).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('estimateMessagesTokens', () => {
|
||||
it('每条消息计入结构性开销(4 tokens)', () => {
|
||||
const msgs = [{ content: '' }, { content: '' }];
|
||||
expect(estimateMessagesTokens(msgs)).toBe(8); // 2 * 4 overhead
|
||||
});
|
||||
|
||||
it('content 为 null 时只计开销(tool_calls 消息场景)', () => {
|
||||
expect(estimateMessagesTokens([{ content: null }])).toBe(4);
|
||||
});
|
||||
|
||||
it('toolCalls 计入 id/name/args 开销', () => {
|
||||
const withToolCall = [
|
||||
{
|
||||
content: null,
|
||||
toolCalls: [{ id: 'tc_12345678', name: 'read_file', args: { file_path: '/a/b.ts' } }],
|
||||
},
|
||||
];
|
||||
const withoutToolCall = [{ content: null }];
|
||||
const diff = estimateMessagesTokens(withToolCall) - estimateMessagesTokens(withoutToolCall);
|
||||
// id(9 chars→3) + name(9→3) + args(~18→5) + overhead(8) ≈ 19 tokens
|
||||
expect(diff).toBeGreaterThanOrEqual(15);
|
||||
expect(diff).toBeLessThanOrEqual(30);
|
||||
});
|
||||
|
||||
it('reasoningContent 计入 token', () => {
|
||||
const withReasoning = [{ content: '', reasoningContent: 'abcd' }];
|
||||
const without = [{ content: '' }];
|
||||
expect(estimateMessagesTokens(withReasoning) - estimateMessagesTokens(without)).toBe(1);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user