/** * Token Estimator 单元测试(P1-14 测试基线) */ import { describe, it, expect } from 'vitest'; import { estimateStringTokens, estimateMessagesTokens } from '../token-estimator'; describe('estimateStringTokens', () => { it('空值返回 0', () => { expect(estimateStringTokens('')).toBe(0); expect(estimateStringTokens(null)).toBe(0); expect(estimateStringTokens(undefined)).toBe(0); }); it('纯 ASCII:4 字符 ≈ 1 token', () => { // 16 个 ASCII 字符 → 16 * 0.25 = 4 tokens expect(estimateStringTokens('abcdefghijklmnop')).toBe(4); }); it('纯中文:1 字符 ≈ 1 token', () => { expect(estimateStringTokens('你好世界')).toBe(4); }); it('混合文本按系数分别计算', () => { // 4 ASCII (1 token) + 2 中文 (2 tokens) = 3 tokens expect(estimateStringTokens('abcd你好')).toBe(3); }); it('Emoji 计为 1 token/字符', () => { expect(estimateStringTokens('🎉🎊')).toBe(2); }); it('结果向上取整', () => { // 1 个 ASCII = 0.25 → ceil 为 1 expect(estimateStringTokens('a')).toBe(1); }); }); describe('estimateMessagesTokens', () => { // ===== v0.5.5: 图片 token 估算(多轮图片记忆场景) ===== it('带 images 的消息按每张 1000 tokens 计入(v0.5.5 — 此前完全忽略)', () => { const textOnly = estimateMessagesTokens([{ content: '看这张图', role: 'user' } as never]); const withImage = estimateMessagesTokens([ { content: '看这张图', role: 'user', images: [{ url: 'data:image/jpeg;base64,x' }] } as never, ]); // 差值 = 1 张图的估算(1000) expect(withImage - textOnly).toBe(1000); const withThree = estimateMessagesTokens([ { content: '', role: 'user', images: [{ url: 'a' }, { url: 'b' }, { url: 'c' }] } as never, ]); // 3 张图 + 消息开销(content 为空 = 0) expect(withThree).toBe(3 * 1000 + 4); }); it('无 images 字段的消息行为不变(向后兼容)', () => { expect(estimateMessagesTokens([{ content: 'abc', role: 'user' } as never])).toBe(4 + 1); }); it('每条消息计入结构性开销(4 tokens)', () => { const msgs = [{ content: '' }, { content: '' }]; expect(estimateMessagesTokens(msgs)).toBe(8); // 2 * 4 overhead }); it('content 为 null 时只计开销(tool_calls 消息场景)', () => { expect(estimateMessagesTokens([{ content: null }])).toBe(4); }); it('toolCalls 计入 id/name/args 开销', () => { const withToolCall = [ { content: null, toolCalls: [{ id: 'tc_12345678', name: 'read_file', args: { file_path: '/a/b.ts' } }], }, ]; const withoutToolCall = [{ content: null }]; const diff = estimateMessagesTokens(withToolCall) - estimateMessagesTokens(withoutToolCall); // id(9 chars→3) + name(9→3) + args(~18→5) + overhead(8) ≈ 19 tokens expect(diff).toBeGreaterThanOrEqual(15); expect(diff).toBeLessThanOrEqual(30); }); it('reasoningContent 计入 token', () => { const withReasoning = [{ content: '', reasoningContent: 'abcd' }]; const without = [{ content: '' }]; expect(estimateMessagesTokens(withReasoning) - estimateMessagesTokens(without)).toBe(1); }); });