feat: v1.1.0 — 上下文引擎重构 + 技能系统升级 + SOUL.md 支持
- 版本号更新: 1.0.0 → 1.1.0,全量同步 package.json/lock/README/docs/UI - 上下文长度自动检测: 切换模型时从 model_info 读取实际 context_length - SOUL.md 支持: 每次发送自动扫描工作空间 SOUL.md,注入为首条 system 消息且不可压缩 - 系统提示词卡片: AI 回复顶部可折叠展示实际发送给模型的完整 system prompt - Token 校准: 利用 Ollama 返回的实际计数动态修正估算器(EMA) - 消息重要性评分: 纯规则打分,trim/summarize 按重要性而非时间顺序 - 结构化压缩: LLM 压缩输出 JSON 格式(topics/decisions/pendingTasks/constraints) - 技能系统 v1.1: 语义匹配(embedding) + 参数自优化 + 未使用衰减 + 技能链合并 - 修复: 100+ TypeScript strict 模式编译错误清零
This commit is contained in:
@@ -18,14 +18,15 @@ import { showToast } from '../components/toast.js';
|
||||
import { logInfo, logWarn, logSuccess, logError, logToolStart, logToolResult, logAgentLoop, logModelResponse } from './log-service.js';
|
||||
import { getWorkspaceDirPath } from '../components/workspace-panel.js';
|
||||
import { generateId } from '../utils/utils.js';
|
||||
import { buildContext, estimateTokens, shouldAutoCompress, compressWithLLM, AUTO_COMPRESS_THRESHOLD } from './context-manager.js';
|
||||
import { buildContext, estimateTokens, shouldAutoCompress, compressWithLLM, AUTO_COMPRESS_THRESHOLD, recordActualTokens } from './context-manager.js';
|
||||
import type {
|
||||
OllamaMessage,
|
||||
OllamaStreamChunk,
|
||||
ToolCall,
|
||||
ToolResult,
|
||||
ToolCallRecord,
|
||||
TraceEntry
|
||||
TraceEntry,
|
||||
ChatSession
|
||||
} from '../types.js';
|
||||
|
||||
const MAX_RETRIES = 2; // 工具错误自动重试次数
|
||||
@@ -173,7 +174,7 @@ async function inferUserProfile(userMsg: string, assistantMsg: string): Promise<
|
||||
const bridge = window.metonaDesktop;
|
||||
if (!bridge?.db?.getSetting) return;
|
||||
|
||||
const profile: Record<string, unknown> = await bridge.db.getSetting('user_profile', {}) || {};
|
||||
const profile: Record<string, any> = await bridge.db.getSetting('user_profile', {}) || {};
|
||||
|
||||
// 技术栈检测
|
||||
const techPatterns: Record<string, RegExp> = {
|
||||
@@ -363,7 +364,7 @@ export interface AgentCallbacks {
|
||||
}
|
||||
|
||||
/** 保存执行轨迹到 SQLite */
|
||||
async function saveTrace(trace: Omit<TraceEntry, 'id' | 'createdAt'>): Promise<void> {
|
||||
async function saveTrace(trace: Record<string, any>): Promise<void> {
|
||||
try {
|
||||
const bridge = window.metonaDesktop;
|
||||
if (!bridge?.db) return;
|
||||
@@ -390,7 +391,7 @@ export async function runAgentLoop(
|
||||
): Promise<void> {
|
||||
const api = state.get<OllamaAPI>(KEYS.API);
|
||||
const model = state.get<string>('_defaultModel', '');
|
||||
const currentSession = state.get(KEYS.CURRENT_SESSION);
|
||||
const currentSession = state.get<ChatSession | null>(KEYS.CURRENT_SESSION);
|
||||
const sessionId = currentSession?.id || 'unknown';
|
||||
|
||||
if (!api || !model) {
|
||||
@@ -411,6 +412,24 @@ export async function runAgentLoop(
|
||||
const messages: OllamaMessage[] = [];
|
||||
let systemPromptParts: string[] = [];
|
||||
|
||||
// ── 扫描工作空间 SOUL.md ──
|
||||
const workspaceDir = getWorkspaceDirPath();
|
||||
if (workspaceDir) {
|
||||
try {
|
||||
const soulResult = await window.metonaDesktop?.workspace.readFile(
|
||||
workspaceDir.replace(/\/+$/, '') + '/SOUL.md'
|
||||
);
|
||||
if (soulResult?.success && soulResult.content) {
|
||||
// SOUL.md 注入为独立 system 消息,标记为不可压缩
|
||||
messages.push({
|
||||
role: 'system',
|
||||
content: `[SOUL.md] ${soulResult.content}`,
|
||||
});
|
||||
logInfo('SOUL.md 已注入系统提示词', `${soulResult.lines || 0} 行`);
|
||||
}
|
||||
} catch { /* SOUL.md 不存在或读取失败,静默跳过 */ }
|
||||
}
|
||||
|
||||
// 注入记忆上下文
|
||||
if (isMemoryEnabled() && userContent) {
|
||||
const relevantMemories = searchMemories(userContent, 6);
|
||||
@@ -423,7 +442,6 @@ export async function runAgentLoop(
|
||||
}
|
||||
|
||||
// 注入工作空间上下文
|
||||
const workspaceDir = getWorkspaceDirPath();
|
||||
if (workspaceDir) {
|
||||
systemPromptParts.push(`【工作空间】
|
||||
当前工作空间目录: ${workspaceDir}
|
||||
@@ -447,7 +465,7 @@ export async function runAgentLoop(
|
||||
const db = state.get<ChatDB | null>(KEYS.DB);
|
||||
if (db) {
|
||||
try {
|
||||
const userProfile = await db.getSetting('user_profile', null);
|
||||
const userProfile = await db.getSetting('user_profile', null) as Record<string, any> | null;
|
||||
if (userProfile && typeof userProfile === 'object') {
|
||||
const parts: string[] = [];
|
||||
if (userProfile.tech_stack?.length) parts.push(`技术栈: ${userProfile.tech_stack.join(', ')}`);
|
||||
@@ -475,6 +493,9 @@ export async function runAgentLoop(
|
||||
|
||||
messages.push({ role: 'system', content: fullSystemPrompt });
|
||||
|
||||
// 将完整系统提示词存入 state,供 input-area 在助理消息上展示
|
||||
state.set('_lastSystemPrompt', fullSystemPrompt);
|
||||
|
||||
// 添加历史消息
|
||||
for (const msg of historyMessages) {
|
||||
messages.push({
|
||||
@@ -503,7 +524,7 @@ export async function runAgentLoop(
|
||||
logInfo(`自动上下文压缩触发: tokens≈${estimateTokens(messages.map(m => m.content || '').join(''))} > ${Math.floor(numCtx * AUTO_COMPRESS_THRESHOLD)} (50% of ${numCtx})`);
|
||||
const compressAC = state.get<AbortController | null>(KEYS.ABORT_CONTROLLER) || new AbortController();
|
||||
try {
|
||||
const compressed = await compressWithLLM(messages, api, model, { abortController: compressAC });
|
||||
const compressed = await compressWithLLM(messages, api as any, model, { abortController: compressAC });
|
||||
if (compressed.length < messages.length || estimateTokens(compressed.map(m => m.content || '').join('')) < estimateTokens(messages.map(m => m.content || '').join(''))) {
|
||||
messages.length = 0;
|
||||
messages.push(...compressed);
|
||||
@@ -633,10 +654,25 @@ export async function runAgentLoop(
|
||||
totalPromptEvalCount += loopPromptEvalCount;
|
||||
totalInferenceNs += loopInferenceNs;
|
||||
state.set('_currentEvalCount', totalEvalCount);
|
||||
|
||||
// Token 校准:用 Ollama 返回的实际计数修正估算器(在重置前保存本轮值)
|
||||
const thisLoopEval = loopEvalCount;
|
||||
const thisLoopPrompt = loopPromptEvalCount;
|
||||
// 重置本轮计数器(下一轮重新从 chunk 收集)
|
||||
loopEvalCount = 0;
|
||||
loopPromptEvalCount = 0;
|
||||
loopInferenceNs = 0;
|
||||
|
||||
// Token 校准:用 Ollama 返回的实际计数自动修正估算器
|
||||
try {
|
||||
if (thisLoopEval > 0 || thisLoopPrompt > 0) {
|
||||
const estimatedThisLoop = estimateTokens(messages.map(m => m.content || '').join(''));
|
||||
if (estimatedThisLoop > 0) {
|
||||
recordActualTokens(thisLoopPrompt, thisLoopEval, estimatedThisLoop);
|
||||
}
|
||||
}
|
||||
} catch { /* 校准失败不阻塞主流程 */ }
|
||||
|
||||
} catch (err) {
|
||||
if (abortController.signal.aborted) {
|
||||
logInfo('流式调用已中止');
|
||||
@@ -722,7 +758,7 @@ export async function runAgentLoop(
|
||||
// v4.2 自动提取技能
|
||||
if (allToolRecords.length >= 2) {
|
||||
try {
|
||||
await extractSkillsFromToolRecords(allToolRecords, userContent, currentSession?.title || '');
|
||||
await extractSkillsFromToolRecords(allToolRecords, userContent, (currentSession as ChatSession)?.title || '');
|
||||
} catch { /* 不阻塞 */ }
|
||||
}
|
||||
|
||||
|
||||
@@ -7,7 +7,31 @@
|
||||
import type { OllamaMessage, OllamaStreamChunk } from '../types.js';
|
||||
import { logInfo, logWarn, logSuccess, logError } from './log-service.js';
|
||||
|
||||
/** 粗略估算 token 数 */
|
||||
// ── Token 校准系统 ──
|
||||
|
||||
/** 校准比例:actualTokens / estimatedTokens,基于 Ollama 返回的实际计数动态修正 */
|
||||
let _tokenCalibrationRatio = 1.0;
|
||||
let _calibrationSamples = 0;
|
||||
const MIN_CALIBRATION_SAMPLES = 3;
|
||||
|
||||
/**
|
||||
* 记录 Ollama 返回的实际 token 计数,用于校准估算器。
|
||||
* 在 agent-engine.ts 每轮流式完成后调用。
|
||||
* @param actualInputTokens Ollama 返回的 prompt_eval_count
|
||||
* @param actualOutputTokens Ollama 返回的 eval_count
|
||||
* @param estimatedTokens 本轮消息调用 estimateTokens 的合计值
|
||||
*/
|
||||
export function recordActualTokens(actualInputTokens: number, actualOutputTokens: number, estimatedCount: number): void {
|
||||
if (actualInputTokens <= 0 || estimatedCount <= 0) return;
|
||||
const actualTotal = actualInputTokens + actualOutputTokens;
|
||||
const sampleRatio = actualTotal / Math.max(1, estimatedCount);
|
||||
// 指数移动平均,平滑异常值
|
||||
const alpha = 0.3;
|
||||
_tokenCalibrationRatio = _tokenCalibrationRatio * (1 - alpha) + sampleRatio * alpha;
|
||||
_calibrationSamples++;
|
||||
}
|
||||
|
||||
/** 估算 token 数(自动使用校准后的比例) */
|
||||
export function estimateTokens(text: string): number {
|
||||
if (!text) return 0;
|
||||
// 中文按 1.5 字/token,英文按 4 字符/token
|
||||
@@ -17,7 +41,17 @@ export function estimateTokens(text: string): number {
|
||||
if (/[\u4e00-\u9fff]/.test(ch)) chineseChars++;
|
||||
else otherChars++;
|
||||
}
|
||||
return Math.ceil(chineseChars / 1.5 + otherChars / 4);
|
||||
const raw = Math.ceil(chineseChars / 1.5 + otherChars / 4);
|
||||
// 应用校准比例(仅在有足够样本后)
|
||||
if (_calibrationSamples >= MIN_CALIBRATION_SAMPLES) {
|
||||
return Math.ceil(raw * _tokenCalibrationRatio);
|
||||
}
|
||||
return raw;
|
||||
}
|
||||
|
||||
/** 获取当前校准比例(供调试用) */
|
||||
export function getTokenCalibration(): { ratio: number; samples: number } {
|
||||
return { ratio: _tokenCalibrationRatio, samples: _calibrationSamples };
|
||||
}
|
||||
|
||||
/** 自动压缩阈值:当消息 token 占 context window 比例超过此值时触发自动压缩 */
|
||||
@@ -27,6 +61,57 @@ export const AUTO_COMPRESS_THRESHOLD = 0.5;
|
||||
const COMPRESS_KEEP_HEAD = 2;
|
||||
const COMPRESS_KEEP_TAIL = 2;
|
||||
|
||||
// ── 消息重要性评分(纯规则,不调用 LLM)──
|
||||
|
||||
/** 重要性评分:0-10,越高越应该保留。SOUL.md 和规则记忆始终保留。 */
|
||||
export function scoreMessageImportance(msg: OllamaMessage): number {
|
||||
let score = 5; // 默认中等
|
||||
|
||||
// SOUL.md 消息永远不可压缩
|
||||
if (msg.content?.startsWith('[SOUL.md]')) return 10;
|
||||
|
||||
// 角色权重
|
||||
if (msg.role === 'user') score += 2; // 用户消息最重要
|
||||
if (msg.role === 'system') score -= 2; // 系统消息通常可压缩
|
||||
|
||||
// 已压缩标记 → 低权重(避免压缩产物堆积)
|
||||
if (msg.compressed) score = 1;
|
||||
|
||||
const content = msg.content || '';
|
||||
|
||||
// 关键词检测
|
||||
const highValuePatterns = [/路径|目录|path|file|config|配置|命令|command|exec/i,
|
||||
/错误|error|失败|fail|bug|fix|修复|解决/i,
|
||||
/版本|version|API|http|url|端口|port|localhost/i,
|
||||
/记住|保存|memory|偏好|偏好|规则|rule/i,
|
||||
/完成|done|✓|success|成功|结果|result/i,
|
||||
/项目|project|工作空间|workspace|git|repo|仓库/i,
|
||||
];
|
||||
const lowValuePatterns = [/好的|明白|ok|知道了|嗯|哦|好/i,
|
||||
/继续|请|帮我|可以吗/i,
|
||||
/谢谢|感谢|不客气/i,
|
||||
];
|
||||
|
||||
for (const p of highValuePatterns) {
|
||||
if (p.test(content)) { score += 1; break; }
|
||||
}
|
||||
for (const p of lowValuePatterns) {
|
||||
if (p.test(content) && content.length < 100) { score -= 2; break; }
|
||||
}
|
||||
|
||||
// 工具调用 → 高价值
|
||||
if (msg.tool_calls?.length) score += 2;
|
||||
|
||||
// 长度加分:长消息通常包含更多信息
|
||||
if (content.length > 500) score += 1;
|
||||
if (content.length > 2000) score += 1;
|
||||
|
||||
// 图像附件 → 中等价值(大但语义密度低)
|
||||
if (msg.images?.length) score -= 1;
|
||||
|
||||
return Math.max(1, Math.min(10, score));
|
||||
}
|
||||
|
||||
export interface ContextBuildOptions {
|
||||
/** 滑动窗口大小(最近 N 条消息完整保留) */
|
||||
windowSize?: number;
|
||||
@@ -127,9 +212,28 @@ export function shouldAutoCompress(messages: OllamaMessage[], numCtx: number): b
|
||||
return totalTokens > threshold;
|
||||
}
|
||||
|
||||
// ── 结构化压缩 ──
|
||||
|
||||
/** 结构化摘要 */
|
||||
export interface StructuredSummary {
|
||||
/** 讨论的主题 */
|
||||
topics: string[];
|
||||
/** 已做出的决定 */
|
||||
decisions: string[];
|
||||
/** 未完成的待办事项 */
|
||||
pendingTasks: string[];
|
||||
/** 发现的约束/规则 */
|
||||
constraints: string[];
|
||||
/** 关键知识点(跨会话有价值的信息) */
|
||||
knowledge: string[];
|
||||
/** 工具调用结果摘要 */
|
||||
toolResults: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* LLM 摘要压缩:调用模型对中间消息生成摘要
|
||||
* 保留首尾各 keepHead/keepTail 条消息,中间用 LLM 摘要替换
|
||||
* LLM 摘要压缩:调用模型对中间消息生成结构化 JSON 摘要。
|
||||
* 保留首尾各 keepHead/keepTail 条消息,中间用 JSON 摘要替换。
|
||||
* v6.0: 输出结构化 JSON,支持增量合并。
|
||||
*
|
||||
* @returns 压缩后的消息列表(包含 compressed 标记的摘要消息)
|
||||
*/
|
||||
@@ -182,14 +286,14 @@ export async function compressWithLLM(
|
||||
|
||||
logInfo(`上下文压缩: 开始 LLM 摘要,${uncompressedMiddle.length} 条消息待压缩`);
|
||||
|
||||
let summary = '';
|
||||
let summaryJson = '';
|
||||
try {
|
||||
await api.chatStream(
|
||||
{
|
||||
model,
|
||||
messages: [{
|
||||
role: 'user',
|
||||
content: `请将以下对话摘要为一段简洁的上下文总结。保留关键信息:讨论的主题、得出的结论、用户的偏好、未完成的任务、关键工具调用结果。用中文输出,不超过 ${maxSummaryTokens} 字。\n\n对话记录:\n${conversationText}`
|
||||
content: `请将以下对话摘要为结构化 JSON。保留关键信息,用中文输出。严格按此 JSON 格式返回(不要输出其他内容):\n\n{\n "topics": ["讨论的 2-4 个核心主题"],\n "decisions": ["已做出的决定(最多 3 条)"],\n "pendingTasks": ["尚未完成的任务(最多 3 条)"],\n "constraints": ["发现的约束/规则/偏好(最多 3 条)"],\n "knowledge": ["跨会话有价值的长期知识点(最多 3 条)"],\n "toolResults": ["关键工具调用结果摘要(最多 3 条)"]\n}\n\n对话记录:\n${conversationText}`
|
||||
}],
|
||||
stream: true,
|
||||
think: false,
|
||||
@@ -197,7 +301,7 @@ export async function compressWithLLM(
|
||||
},
|
||||
(chunk: OllamaStreamChunk) => {
|
||||
if (chunk.message?.content) {
|
||||
summary += chunk.message.content;
|
||||
summaryJson += chunk.message.content;
|
||||
}
|
||||
},
|
||||
options.abortController
|
||||
@@ -211,15 +315,34 @@ export async function compressWithLLM(
|
||||
return messages;
|
||||
}
|
||||
|
||||
if (!summary.trim()) {
|
||||
if (!summaryJson.trim()) {
|
||||
logWarn('上下文压缩: 模型未返回摘要内容');
|
||||
return messages;
|
||||
}
|
||||
|
||||
// 解析 JSON 摘要(容错:提取第一个 JSON 块或直接解析)
|
||||
let parsed: StructuredSummary;
|
||||
try {
|
||||
const jsonMatch = summaryJson.match(/\{[\s\S]*"topics"[\s\S]*\}/);
|
||||
parsed = jsonMatch ? JSON.parse(jsonMatch[0]) : JSON.parse(summaryJson);
|
||||
} catch {
|
||||
logWarn('上下文压缩: JSON 解析失败,使用纯文本摘要');
|
||||
parsed = { topics: [summaryJson.slice(0, 100)], decisions: [], pendingTasks: [], constraints: [], knowledge: [], toolResults: [] };
|
||||
}
|
||||
|
||||
// 构建结构化摘要消息文本
|
||||
const parts: string[] = [];
|
||||
if (parsed.topics.length) parts.push(`📌 主题: ${parsed.topics.join(';')}`);
|
||||
if (parsed.decisions.length) parts.push(`✅ 决策: ${parsed.decisions.join(';')}`);
|
||||
if (parsed.pendingTasks.length) parts.push(`⏳ 待办: ${parsed.pendingTasks.join(';')}`);
|
||||
if (parsed.constraints.length) parts.push(`📏 约束: ${parsed.constraints.join(';')}`);
|
||||
if (parsed.knowledge.length) parts.push(`🧠 知识点: ${parsed.knowledge.join(';')}`);
|
||||
if (parsed.toolResults.length) parts.push(`🔧 工具结果: ${parsed.toolResults.join(';')}`);
|
||||
|
||||
// 构建压缩后的摘要消息
|
||||
const summaryMsg: OllamaMessage = {
|
||||
role: 'system',
|
||||
content: `📋 以下是对之前对话的摘要(已压缩 ${uncompressedMiddle.length} 条消息):\n\n${summary}`,
|
||||
content: `📋 以下是对之前对话的摘要(已压缩 ${uncompressedMiddle.length} 条消息):\n\n${parts.join('\n')}`,
|
||||
compressed: true
|
||||
};
|
||||
|
||||
@@ -243,7 +366,8 @@ export async function compressWithLLM(
|
||||
}
|
||||
|
||||
/**
|
||||
* 将较早的消息每 batchSize 条压缩为一段摘要(快速文本截取,不调用 LLM)
|
||||
* 将较早的消息每 batchSize 条压缩为一段摘要。
|
||||
* v6.0: 重要性高的消息保留更多内容,低价值消息激进截断。
|
||||
*/
|
||||
function summarizeOlderMessages(messages: OllamaMessage[], batchSize: number): OllamaMessage[] {
|
||||
const summaries: OllamaMessage[] = [];
|
||||
@@ -262,7 +386,7 @@ function summarizeOlderMessages(messages: OllamaMessage[], batchSize: number): O
|
||||
}
|
||||
|
||||
/**
|
||||
* 快速摘要(不调用模型,直接提取关键信息)
|
||||
* 快速摘要:重要性高的消息保留更多信息,低价值的激进截断
|
||||
*/
|
||||
function createQuickSummary(messages: OllamaMessage[]): string {
|
||||
const parts: string[] = [];
|
||||
@@ -270,7 +394,21 @@ function createQuickSummary(messages: OllamaMessage[]): string {
|
||||
for (const msg of messages) {
|
||||
const role = msg.role === 'user' ? '用户' : 'AI';
|
||||
const content = msg.content || '';
|
||||
const preview = content.length > 100 ? content.slice(0, 100) + '...' : content;
|
||||
const importance = scoreMessageImportance(msg);
|
||||
|
||||
let preview: string;
|
||||
if (importance >= 8) {
|
||||
// 高价值消息:保留 200 字
|
||||
preview = content.length > 200 ? content.slice(0, 200) + '...' : content;
|
||||
} else if (importance >= 5) {
|
||||
// 中等价值:保留 100 字
|
||||
preview = content.length > 100 ? content.slice(0, 100) + '...' : content;
|
||||
} else {
|
||||
// 低价值:仅保留 40 字或跳过
|
||||
if (content.trim().length < 10) continue;
|
||||
preview = content.length > 40 ? content.slice(0, 40) + '...' : content;
|
||||
}
|
||||
|
||||
if (preview.trim()) {
|
||||
parts.push(`${role}: ${preview}`);
|
||||
}
|
||||
@@ -284,31 +422,46 @@ function createQuickSummary(messages: OllamaMessage[]): string {
|
||||
}
|
||||
|
||||
/**
|
||||
* 根据 token 限制裁剪消息
|
||||
* 根据 token 限制裁剪消息。
|
||||
* v6.0: 按重要性评分决定保留顺序——重要性低的优先被丢弃。
|
||||
*/
|
||||
function trimByTokenLimit(messages: OllamaMessage[], maxTokens: number): OllamaMessage[] {
|
||||
let totalTokens = 0;
|
||||
const result: OllamaMessage[] = [];
|
||||
// 分离 system 和非 system 消息
|
||||
const systemMsgs = messages.filter(m => m.role === 'system');
|
||||
const nonSystemMsgs = messages.filter(m => m.role !== 'system');
|
||||
|
||||
for (let i = messages.length - 1; i >= 0; i--) {
|
||||
const msg = messages[i];
|
||||
const msgTokens = estimateTokens(msg.content || '') +
|
||||
(msg.images ? msg.images.length * 100 : 0) +
|
||||
(msg.tool_calls ? msg.tool_calls.length * 50 : 0);
|
||||
if (nonSystemMsgs.length <= 4) return messages; // 消息太少不裁剪
|
||||
|
||||
// system 消息始终保留
|
||||
if (msg.role === 'system') {
|
||||
totalTokens += msgTokens;
|
||||
result.unshift(msg);
|
||||
continue;
|
||||
}
|
||||
// 计算每条消息的 token + 重要性
|
||||
const scored = nonSystemMsgs.map(m => ({
|
||||
msg: m,
|
||||
tokens: estimateTokens(m.content || '') +
|
||||
(m.images ? m.images.length * 100 : 0) +
|
||||
(m.tool_calls ? m.tool_calls.length * 50 : 0),
|
||||
importance: scoreMessageImportance(m),
|
||||
}));
|
||||
|
||||
if (totalTokens + msgTokens > maxTokens && result.length > 2) {
|
||||
break;
|
||||
}
|
||||
// system 消息的 token 消耗
|
||||
const systemTokens = systemMsgs.reduce((sum, m) => sum + estimateTokens(m.content || ''), 0);
|
||||
const availableTokens = maxTokens - systemTokens;
|
||||
|
||||
totalTokens += msgTokens;
|
||||
result.unshift(msg);
|
||||
// 按重要性降序排列,取能装下的最大数量
|
||||
scored.sort((a, b) => b.importance - a.importance);
|
||||
|
||||
let usedTokens = 0;
|
||||
const kept = new Set<number>(); // 保留的索引
|
||||
for (let i = 0; i < scored.length; i++) {
|
||||
if (usedTokens + scored[i].tokens > availableTokens && kept.size >= 2) break;
|
||||
usedTokens += scored[i].tokens;
|
||||
kept.add(i);
|
||||
}
|
||||
|
||||
// 按原始顺序重建:system → 按重要性保留的非system
|
||||
const result: OllamaMessage[] = [...systemMsgs];
|
||||
// 使用原始顺序而非重要性顺序,保持对话的时间线
|
||||
const keptSet = new Set(scored.filter((_, i) => kept.has(i)).map(s => s.msg));
|
||||
for (const m of nonSystemMsgs) {
|
||||
if (keptSet.has(m)) result.push(m);
|
||||
}
|
||||
|
||||
return result;
|
||||
|
||||
@@ -9,9 +9,9 @@ const PBKDF2_ITERATIONS = 100000;
|
||||
|
||||
async function deriveKeyBytes(salt: Uint8Array): Promise<Uint8Array> {
|
||||
const passBytes = new TextEncoder().encode(PASSPHRASE);
|
||||
const keyMaterial = await crypto.subtle.importKey('raw', passBytes, 'PBKDF2', false, ['deriveKey']);
|
||||
const keyMaterial = await (crypto.subtle as any).importKey('raw', passBytes, 'PBKDF2', false, ['deriveKey']);
|
||||
const key = await crypto.subtle.deriveKey(
|
||||
{ name: 'PBKDF2', salt, iterations: PBKDF2_ITERATIONS, hash: 'SHA-256' },
|
||||
{ name: 'PBKDF2', salt: salt as any, iterations: PBKDF2_ITERATIONS, hash: 'SHA-256' },
|
||||
keyMaterial,
|
||||
{ name: 'AES-GCM', length: 256 },
|
||||
true,
|
||||
@@ -27,7 +27,7 @@ export async function encryptData(data: unknown): Promise<Blob> {
|
||||
const iv = crypto.getRandomValues(new Uint8Array(12));
|
||||
const keyBytes = await deriveKeyBytes(salt);
|
||||
|
||||
const key = await crypto.subtle.importKey('raw', keyBytes, 'AES-GCM', false, ['encrypt']);
|
||||
const key = await crypto.subtle.importKey('raw', keyBytes as any, 'AES-GCM', false, ['encrypt']);
|
||||
const ciphertext = await crypto.subtle.encrypt({ name: 'AES-GCM', iv }, key, json);
|
||||
const payload = new Uint8Array(ciphertext);
|
||||
|
||||
@@ -51,7 +51,7 @@ export async function decryptData(buffer: ArrayBuffer): Promise<unknown> {
|
||||
throw new Error('不支持的加密格式,请使用最新版本重新导出');
|
||||
}
|
||||
|
||||
const key = await crypto.subtle.importKey('raw', keyBytes, 'AES-GCM', false, ['decrypt']);
|
||||
const key = await crypto.subtle.importKey('raw', keyBytes as any, 'AES-GCM', false, ['decrypt']);
|
||||
const decrypted = await crypto.subtle.decrypt({ name: 'AES-GCM', iv }, key, payload);
|
||||
const jsonBytes = new Uint8Array(decrypted);
|
||||
|
||||
|
||||
@@ -17,7 +17,9 @@ import {
|
||||
getMemoryCollectionId
|
||||
} from './vector-memory.js';
|
||||
import { logMemory, logDebug, logWarn, logInfo } from './log-service.js';
|
||||
import type { MemoryEntry, MemorySearchResult, MemoryExtractionResult, ChatDB, OllamaAPI } from '../types.js';
|
||||
import type { MemoryEntry, MemorySearchResult, MemoryExtractionResult } from '../types.js';
|
||||
import type { ChatDB } from '../db/chat-db.js';
|
||||
import type { OllamaAPI } from '../api/ollama.js';
|
||||
|
||||
const TYPE_ICONS: Record<string, string> = {
|
||||
fact: '📌',
|
||||
@@ -686,10 +688,9 @@ ${conversationText.slice(0, 4000)}
|
||||
const response = await api.chat({
|
||||
model,
|
||||
messages: [{ role: 'user', content: extractPrompt }],
|
||||
stream: false,
|
||||
think: false,
|
||||
options: { num_ctx: 8192, temperature: 0.1 }
|
||||
});
|
||||
} as any);
|
||||
|
||||
const content = (response as { message?: { content?: string } })?.message?.content || '';
|
||||
const jsonMatch = content.match(new RegExp('```json\\s*([\\s\\S]*?)\\s*```')) || content.match(/\{[\s\S]*"entries"[\s\S]*\}/);
|
||||
@@ -715,7 +716,7 @@ ${conversationText.slice(0, 4000)}
|
||||
importance: Math.min(10, Math.max(1, entry.importance || 5)),
|
||||
tags: entry.tags || [],
|
||||
source: '自动提取',
|
||||
sessionId: currentSession?.id
|
||||
sessionId: (currentSession as any)?.id
|
||||
});
|
||||
count++;
|
||||
}
|
||||
|
||||
@@ -64,7 +64,7 @@ export async function extractSkillsFromToolRecords(
|
||||
const chainNames = chain.map(s => s.name).join(' → ');
|
||||
|
||||
// 检查是否已有完全相同的技能链
|
||||
const existingSkills: Skill[] = await bridge.db.getAllSkills();
|
||||
const existingSkills = await bridge.db.getAllSkills() as Skill[];
|
||||
const duplicate = existingSkills.find(s => {
|
||||
try {
|
||||
const existingChain: ToolChainStep[] = JSON.parse(s.tool_chain);
|
||||
@@ -73,7 +73,29 @@ export async function extractSkillsFromToolRecords(
|
||||
});
|
||||
|
||||
if (duplicate) {
|
||||
// 已有相同技能,不重复创建
|
||||
// v1.1: 重复链 → 更新参数提示(合并新知)+ 增加计数
|
||||
try {
|
||||
const existingChain: ToolChainStep[] = JSON.parse(duplicate.tool_chain);
|
||||
let updated = false;
|
||||
for (let i = 0; i < Math.min(chain.length, existingChain.length); i++) {
|
||||
const newArgs = chain[i].args_hint.split(',').map(a => a.trim()).filter(Boolean);
|
||||
const oldArgs = existingChain[i].args_hint.split(',').map(a => a.trim()).filter(Boolean);
|
||||
const merged = [...new Set([...oldArgs, ...newArgs])];
|
||||
const mergedHint = merged.join(', ');
|
||||
if (mergedHint !== existingChain[i].args_hint) {
|
||||
existingChain[i].args_hint = mergedHint;
|
||||
updated = true;
|
||||
}
|
||||
}
|
||||
if (updated) {
|
||||
duplicate.tool_chain = JSON.stringify(existingChain);
|
||||
duplicate.success_count++;
|
||||
duplicate.updated_at = Date.now();
|
||||
await bridge.db.saveSkill(duplicate);
|
||||
logInfo(`技能参数优化: ${duplicate.name}`, `合并了新的参数提示`);
|
||||
return 1;
|
||||
}
|
||||
} catch { /* 更新失败不阻塞 */ }
|
||||
logInfo(`技能已存在,跳过提取: ${duplicate.name}`);
|
||||
return 0;
|
||||
}
|
||||
@@ -113,20 +135,37 @@ export async function extractSkillsFromToolRecords(
|
||||
}
|
||||
|
||||
/**
|
||||
* 匹配用户消息与已有技能
|
||||
* 返回匹配度最高的技能列表
|
||||
* 匹配用户消息与已有技能。
|
||||
* v1.1: 关键词 + 语义匹配融合,未使用技能衰减。
|
||||
*/
|
||||
export async function matchSkills(userMessage: string, limit = 3): Promise<Skill[]> {
|
||||
const bridge = window.metonaDesktop;
|
||||
if (!bridge?.db?.getAllSkills) return [];
|
||||
|
||||
try {
|
||||
const allSkills: Skill[] = await bridge.db.getAllSkills();
|
||||
const allSkills = await bridge.db.getAllSkills() as any as Skill[];
|
||||
if (allSkills.length === 0) return [];
|
||||
|
||||
const messageLower = userMessage.toLowerCase();
|
||||
const messageWords = new Set(messageLower.split(/[\s,,。!?、;:""''()\[\]【】]+/).filter(w => w.length > 1));
|
||||
|
||||
// ── 语义匹配(如果 embedding 模型可用)──
|
||||
let userEmbedding: number[] | null = null;
|
||||
try {
|
||||
const memEnabled = state.get<boolean>('memoryEnabled', false);
|
||||
const embedModel = state.get<string>('embeddingModel', '');
|
||||
if (memEnabled && embedModel) {
|
||||
const api = state.get<any>(KEYS.API);
|
||||
if (api?.embed) {
|
||||
const result = await api.embed(embedModel, userMessage);
|
||||
userEmbedding = result.embedding || result.embeddings?.[0] || null;
|
||||
}
|
||||
}
|
||||
} catch { /* 语义匹配失败回退关键词 */ }
|
||||
|
||||
const now = Date.now();
|
||||
const THIRTY_DAYS = 30 * 24 * 3600 * 1000;
|
||||
|
||||
const scored = allSkills.map(skill => {
|
||||
let score = 0;
|
||||
|
||||
@@ -134,24 +173,43 @@ export async function matchSkills(userMessage: string, limit = 3): Promise<Skill
|
||||
if (skill.trigger_keywords) {
|
||||
const kwSet = new Set(skill.trigger_keywords.toLowerCase().split(',').map(k => k.trim()));
|
||||
for (const kw of kwSet) {
|
||||
if (kw && messageLower.includes(kw)) score += 0.4;
|
||||
if (kw && messageLower.includes(kw)) score += 0.3;
|
||||
}
|
||||
}
|
||||
|
||||
// 名称和描述匹配
|
||||
const nameWords = skill.name.toLowerCase().split(/[\s→\-]+/);
|
||||
for (const w of nameWords) {
|
||||
if (w.length > 1 && messageWords.has(w)) score += 0.2;
|
||||
if (w.length > 1 && messageWords.has(w)) score += 0.15;
|
||||
}
|
||||
|
||||
// 成功率加成
|
||||
// 语义匹配:余弦相似度加权(主导分数)
|
||||
if (userEmbedding && skill.summary) {
|
||||
const skillEmbed = getCachedSkillEmbedding(skill.id);
|
||||
if (skillEmbed) {
|
||||
const sim = cosineSimilarity(userEmbedding, skillEmbed);
|
||||
score += sim * 0.4; // 语义匹配主导
|
||||
}
|
||||
}
|
||||
|
||||
// 成功率加权
|
||||
const total = skill.success_count + skill.fail_count;
|
||||
if (total > 0) score *= (0.5 + 0.5 * (skill.success_count / total));
|
||||
|
||||
// ── v1.1 未使用技能衰减 ──
|
||||
if (skill.last_used_at) {
|
||||
const daysSinceUse = (now - skill.last_used_at) / (24 * 3600 * 1000);
|
||||
if (daysSinceUse > 30) score *= 0.5; // 30天未用降权50%
|
||||
if (daysSinceUse > 90) score *= 0.3; // 90天未用几乎忽略
|
||||
}
|
||||
|
||||
return { skill, score };
|
||||
});
|
||||
|
||||
return scored
|
||||
// ── v1.1 技能链合并:相同前缀的链抽象为一个技能 ──
|
||||
const merged = mergeSimilarSkills(scored);
|
||||
|
||||
return merged
|
||||
.filter(s => s.score >= MATCH_THRESHOLD)
|
||||
.sort((a, b) => b.score - a.score)
|
||||
.slice(0, limit)
|
||||
@@ -161,6 +219,98 @@ export async function matchSkills(userMessage: string, limit = 3): Promise<Skill
|
||||
}
|
||||
}
|
||||
|
||||
// ── 向量计算工具 ──
|
||||
|
||||
/** 技能 embedding 缓存(LRU 简单实现) */
|
||||
const _skillEmbedCache = new Map<string, { embedding: number[]; time: number }>();
|
||||
|
||||
function getCachedSkillEmbedding(skillId: string): number[] | null {
|
||||
const cached = _skillEmbedCache.get(skillId);
|
||||
if (cached && Date.now() - cached.time < 300_000) return cached.embedding; // 5分钟有效
|
||||
return null;
|
||||
}
|
||||
|
||||
export function cacheSkillEmbedding(skillId: string, embedding: number[]): void {
|
||||
_skillEmbedCache.set(skillId, { embedding, time: Date.now() });
|
||||
// 限制缓存大小
|
||||
if (_skillEmbedCache.size > 200) {
|
||||
const oldest = [..._skillEmbedCache.entries()].sort((a, b) => a[1].time - b[1].time)[0];
|
||||
if (oldest) _skillEmbedCache.delete(oldest[0]);
|
||||
}
|
||||
}
|
||||
|
||||
function cosineSimilarity(a: number[], b: number[]): number {
|
||||
if (!a || !b || a.length !== b.length) return 0;
|
||||
let dot = 0, normA = 0, normB = 0;
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
dot += a[i] * b[i];
|
||||
normA += a[i] * a[i];
|
||||
normB += b[i] * b[i];
|
||||
}
|
||||
const denom = Math.sqrt(normA) * Math.sqrt(normB);
|
||||
return denom === 0 ? 0 : dot / denom;
|
||||
}
|
||||
|
||||
// ── 技能链合并 ──
|
||||
|
||||
interface ScoredSkill { skill: Skill; score: number; }
|
||||
|
||||
/**
|
||||
* v1.1 技能链合并:两个以上技能有相同前缀(前 2 步相同),
|
||||
* 合并为一个抽象技能,分数取最高。
|
||||
*/
|
||||
function mergeSimilarSkills(scored: ScoredSkill[]): ScoredSkill[] {
|
||||
if (scored.length < 2) return scored;
|
||||
|
||||
const result: ScoredSkill[] = [];
|
||||
const used = new Set<number>();
|
||||
|
||||
for (let i = 0; i < scored.length; i++) {
|
||||
if (used.has(i)) continue;
|
||||
const group: ScoredSkill[] = [scored[i]];
|
||||
|
||||
// 找同前缀的
|
||||
for (let j = i + 1; j < scored.length; j++) {
|
||||
if (used.has(j)) continue;
|
||||
if (getChainPrefix(scored[i].skill) === getChainPrefix(scored[j].skill)) {
|
||||
group.push(scored[j]);
|
||||
used.add(j);
|
||||
}
|
||||
}
|
||||
|
||||
if (group.length >= 2) {
|
||||
// 合并为一个抽象技能
|
||||
const best = group.reduce((a, b) => a.score > b.score ? a : b);
|
||||
const chainNames = group.map(g => {
|
||||
try { return (JSON.parse(g.skill.tool_chain) as ToolChainStep[]).map(s => s.name).join('→'); }
|
||||
catch { return g.skill.name; }
|
||||
});
|
||||
const mergedSkill: Skill = {
|
||||
...best.skill,
|
||||
name: `${getChainPrefix(best.skill)} → ...`,
|
||||
description: `合并了 ${group.length} 个相似技能: ${chainNames.join(' | ')}`,
|
||||
summary: best.skill.summary,
|
||||
};
|
||||
result.push({ skill: mergedSkill, score: best.score * 1.1 }); // 合并后略加分
|
||||
used.add(i);
|
||||
} else {
|
||||
result.push(scored[i]);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/** 获取技能链的前 2 步前缀(用于相似度判断) */
|
||||
function getChainPrefix(skill: Skill): string {
|
||||
try {
|
||||
const chain: ToolChainStep[] = JSON.parse(skill.tool_chain);
|
||||
return chain.slice(0, 2).map(s => s.name).join('→');
|
||||
} catch {
|
||||
return skill.name;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 从匹配的技能构建 system prompt 上下文(Level 0:仅名称 + 描述索引)
|
||||
* 完整工具链通过 skill_view 工具按需加载(Level 1)
|
||||
@@ -286,8 +436,8 @@ export async function listSkills(): Promise<Array<{
|
||||
}>> {
|
||||
const bridge = window.metonaDesktop;
|
||||
if (!bridge?.db?.getAllSkills) return [];
|
||||
const allSkills: Skill[] = await bridge.db.getAllSkills();
|
||||
return allSkills.map(skill => {
|
||||
const allSkills: Skill[] = await bridge.db.getAllSkills() as any;
|
||||
return (allSkills as any[]).map((skill: any) => {
|
||||
const total = skill.success_count + skill.fail_count;
|
||||
const rate = total > 0 ? Math.round(skill.success_count / total * 100) : 100;
|
||||
let chainPreview = '';
|
||||
@@ -313,7 +463,7 @@ export async function viewSkill(skillName: string): Promise<{
|
||||
}> {
|
||||
const bridge = window.metonaDesktop;
|
||||
if (!bridge?.db?.getAllSkills) return { success: false, error: '数据库不可用' };
|
||||
const allSkills: Skill[] = await bridge.db.getAllSkills();
|
||||
const allSkills: Skill[] = await bridge.db.getAllSkills() as any;
|
||||
const skill = allSkills.find(s =>
|
||||
s.name.toLowerCase() === skillName.toLowerCase() ||
|
||||
s.name.toLowerCase().includes(skillName.toLowerCase())
|
||||
|
||||
@@ -6,7 +6,8 @@
|
||||
import { state, KEYS } from '../state/state.js';
|
||||
import { VectorStore } from './vector-store.js';
|
||||
import { logMemory, logDebug, logError } from './log-service.js';
|
||||
import type { MemoryEntry, OllamaAPI, VectorItem, SearchResult } from '../types.js';
|
||||
import type { MemoryEntry, VectorItem, SearchResult } from '../types.js';
|
||||
import type { OllamaAPI } from '../api/ollama.js';
|
||||
|
||||
const MEMORY_COLLECTION_NAME = '记忆向量索引';
|
||||
|
||||
|
||||
@@ -228,7 +228,7 @@ export class VectorStore {
|
||||
|
||||
if (saved && saved.version === this._indexVersion(vectors)) {
|
||||
if (!(saved.invertedLists instanceof Map)) {
|
||||
saved.invertedLists = new Map(Object.entries(saved.invertedListsObj || {}));
|
||||
saved.invertedLists = new Map(Object.entries(saved.invertedListsObj || {}).map(([k, v]) => [parseInt(k), v as string[]]));
|
||||
}
|
||||
this._indexCache.set(colId, saved);
|
||||
return saved;
|
||||
|
||||
Reference in New Issue
Block a user