feat: 升级至 v0.3.16 — 工单修复 51 项 + 审查修复 30 项(安全加固/适配器/工具系统/数据层/前端)
This commit is contained in:
@@ -52,6 +52,11 @@ export function estimateStringTokens(text: string | null | undefined): number {
|
||||
return Math.ceil(cjkCount * CJK_TOKEN_RATIO + asciiCount * ASCII_TOKEN_RATIO + otherCount * OTHER_TOKEN_RATIO);
|
||||
}
|
||||
|
||||
/**
|
||||
* #50 修复: tool_call 结构开销({"id":"","name":"","arguments":""} 等结构字符,参考 OpenAI 规范)
|
||||
*/
|
||||
const TOOL_CALL_OVERHEAD_TOKENS = 8;
|
||||
|
||||
/**
|
||||
* 估算多条消息的总 token 数
|
||||
*
|
||||
@@ -63,7 +68,8 @@ export function estimateStringTokens(text: string | null | undefined): number {
|
||||
export function estimateMessagesTokens(messages: Array<{
|
||||
content: string | null;
|
||||
reasoningContent?: string;
|
||||
toolCalls?: Array<{ args: Record<string, unknown> }>;
|
||||
toolCalls?: Array<{ id?: string; name?: string; args: Record<string, unknown> }>;
|
||||
toolCallId?: string;
|
||||
}>): number {
|
||||
let total = 0;
|
||||
for (const msg of messages) {
|
||||
@@ -71,9 +77,19 @@ export function estimateMessagesTokens(messages: Array<{
|
||||
if (msg.reasoningContent) total += estimateStringTokens(msg.reasoningContent);
|
||||
if (msg.toolCalls) {
|
||||
for (const tc of msg.toolCalls) {
|
||||
total += estimateStringTokens(JSON.stringify(tc.args));
|
||||
// #50 修复: OpenAI tokenizer 会将 tool_call 的完整结构(id、name、args)都计入 token
|
||||
// 之前仅估算 args,忽略 id(通常 24 字符 call_xxx)和 name(通常 5-20 字符),导致每个 tool_call 少算 5-10 tokens
|
||||
total += estimateStringTokens(tc.id ?? '');
|
||||
total += estimateStringTokens(tc.name ?? '');
|
||||
total += estimateStringTokens(JSON.stringify(tc.args ?? {}));
|
||||
// 结构开销({"id":"","name":"","arguments":""} 等结构字符)
|
||||
total += TOOL_CALL_OVERHEAD_TOKENS;
|
||||
}
|
||||
}
|
||||
// #50 修复: tool 消息的 tool_call_id 字段也计入 token
|
||||
if (msg.toolCallId) {
|
||||
total += estimateStringTokens(msg.toolCallId);
|
||||
}
|
||||
// L-17 修复: 使用命名常量替代魔法数字
|
||||
total += MSG_OVERHEAD_TOKENS;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user