/** * Token Estimator 单元测试(P1-14 测试基线) */ import { describe, it, expect } from 'vitest'; import { estimateStringTokens, estimateMessagesTokens } from '../token-estimator'; describe('estimateStringTokens', () => { it('空值返回 0', () => { expect(estimateStringTokens('')).toBe(0); expect(estimateStringTokens(null)).toBe(0); expect(estimateStringTokens(undefined)).toBe(0); }); it('纯 ASCII:4 字符 ≈ 1 token', () => { // 16 个 ASCII 字符 → 16 * 0.25 = 4 tokens expect(estimateStringTokens('abcdefghijklmnop')).toBe(4); }); it('纯中文:1 字符 ≈ 1 token', () => { expect(estimateStringTokens('你好世界')).toBe(4); }); it('混合文本按系数分别计算', () => { // 4 ASCII (1 token) + 2 中文 (2 tokens) = 3 tokens expect(estimateStringTokens('abcd你好')).toBe(3); }); it('Emoji 计为 1 token/字符', () => { expect(estimateStringTokens('🎉🎊')).toBe(2); }); it('结果向上取整', () => { // 1 个 ASCII = 0.25 → ceil 为 1 expect(estimateStringTokens('a')).toBe(1); }); }); describe('estimateMessagesTokens', () => { it('每条消息计入结构性开销(4 tokens)', () => { const msgs = [{ content: '' }, { content: '' }]; expect(estimateMessagesTokens(msgs)).toBe(8); // 2 * 4 overhead }); it('content 为 null 时只计开销(tool_calls 消息场景)', () => { expect(estimateMessagesTokens([{ content: null }])).toBe(4); }); it('toolCalls 计入 id/name/args 开销', () => { const withToolCall = [ { content: null, toolCalls: [{ id: 'tc_12345678', name: 'read_file', args: { file_path: '/a/b.ts' } }], }, ]; const withoutToolCall = [{ content: null }]; const diff = estimateMessagesTokens(withToolCall) - estimateMessagesTokens(withoutToolCall); // id(9 chars→3) + name(9→3) + args(~18→5) + overhead(8) ≈ 19 tokens expect(diff).toBeGreaterThanOrEqual(15); expect(diff).toBeLessThanOrEqual(30); }); it('reasoningContent 计入 token', () => { const withReasoning = [{ content: '', reasoningContent: 'abcd' }]; const without = [{ content: '' }]; expect(estimateMessagesTokens(withReasoning) - estimateMessagesTokens(without)).toBe(1); }); });