1095 lines
38 KiB
TypeScript
1095 lines
38 KiB
TypeScript
/**
|
||
* Memory Manager — 记忆管理器
|
||
*
|
||
* 基于 SQLite（better-sqlite3）的三层记忆系统。
|
||
* 表结构由 DatabaseService 统一创建，此处不再重复。
|
||
*
|
||
* v0.2.0 增强：
|
||
* - TF-IDF 语义检索替代 LIKE 关键词搜索
|
||
* - 时间衰减策略：老旧记忆权重降低
|
||
* - IDF 缓存：避免每次搜索重新计算
|
||
*
|
||
* @see docs/生产级通用 AI Agent 智能体桌面应用：完整设计与构建指南.html — 第六章
|
||
* @see standard/开发规范.md — 使用 better-sqlite3（禁止自写数据库层）
|
||
*/
|
||
|
||
import { nanoid } from 'nanoid';
|
||
import { createHash } from 'crypto';
|
||
import type Database from 'better-sqlite3';
|
||
import log from 'electron-log';
|
||
import type { MemoryEmbedder } from './embedder';
|
||
|
||
export type MemoryType = 'episodic' | 'semantic' | 'working';
|
||
export type MemorySource = 'user_input' | 'tool_result' | 'agent_thought' | 'imported';
|
||
|
||
export interface MemoryItem {
|
||
id: string;
|
||
type: MemoryType;
|
||
content: string;
|
||
summary?: string;
|
||
source: MemorySource;
|
||
importance: number;
|
||
sessionId?: string;
|
||
createdAt: number;
|
||
expiresAt?: number;
|
||
}
|
||
|
||
export interface SearchResult extends MemoryItem {
|
||
score: number;
|
||
}
|
||
|
||
interface MemorySearchOptions {
|
||
topK?: number;
|
||
sessionId?: string;
|
||
type?: MemoryType;
|
||
minImportance?: number;
|
||
}
|
||
|
||
/**
|
||
* 分词：将文本拆分为词项（支持中英文）
|
||
*
|
||
* v0.3.18 修复: 改进中文分词策略，减少 bigram 噪声
|
||
* 之前对整段文本连续取 bigram，会跨越标点边界产生无意义组合
|
||
* （如"开发规范。下一句"会产生"范。"和"。下"这类噪声 bigram），
|
||
* 降低 IDF 区分度。
|
||
* 现在先按中英文标点切分子句，再在每个子句内做 bigram，
|
||
* 避免跨句组合，提升检索准确度。
|
||
*/
|
||
function tokenize(text: string): string[] {
|
||
// 转小写
|
||
const lower = text.toLowerCase();
|
||
// 英文词
|
||
const words = lower.match(/[a-z][a-z0-9_-]{1,}/g) ?? [];
|
||
|
||
// v0.3.18 修复: 按中英文标点切分子句，再在每个子句内做 bigram
|
||
// 标点包括：中文句号/逗号/顿号/分号/感叹/问号 + 英文 .,;!?()
|
||
const sentences = lower.split(/[。，、；！？.,;!?()\n\r\t]/);
|
||
const bigrams: string[] = [];
|
||
for (const sentence of sentences) {
|
||
// 提取子句内的 CJK 字符（覆盖 CJK 统一表意、扩展 A、平假名/片假名、谚文）
|
||
const cjkChars = sentence.match(/[\u4e00-\u9fff\u3400-\u4dbf\u3040-\u30ff\uac00-\ud7af]/g);
|
||
if (!cjkChars || cjkChars.length === 0) continue;
|
||
for (let i = 0; i < cjkChars.length - 1; i++) {
|
||
bigrams.push(cjkChars[i] + cjkChars[i + 1]);
|
||
}
|
||
// 单字 CJK 子句补 unigram（避免单字文档无 token）
|
||
if (cjkChars.length === 1) {
|
||
bigrams.push(cjkChars[0]);
|
||
}
|
||
}
|
||
return [...words, ...bigrams];
|
||
}
|
||
|
||
/** 计算词频（TF） */
|
||
function computeTF(tokens: string[]): Map<string, number> {
|
||
const tf = new Map<string, number>();
|
||
for (const token of tokens) {
|
||
tf.set(token, (tf.get(token) ?? 0) + 1);
|
||
}
|
||
// 归一化
|
||
const total = tokens.length || 1;
|
||
for (const [key, val] of tf) {
|
||
tf.set(key, val / total);
|
||
}
|
||
return tf;
|
||
}
|
||
|
||
/** 计算余弦相似度的点积部分 */
|
||
function dotProduct(
|
||
tf1: Map<string, number>,
|
||
tf2: Map<string, number>,
|
||
idf: Map<string, number>,
|
||
): number {
|
||
let sum = 0;
|
||
for (const [term, freq1] of tf1) {
|
||
const freq2 = tf2.get(term);
|
||
if (freq2 !== undefined) {
|
||
const idfVal = idf.get(term) ?? 1;
|
||
sum += freq1 * freq2 * idfVal * idfVal;
|
||
}
|
||
}
|
||
return sum;
|
||
}
|
||
|
||
/** 计算向量模长 */
|
||
function vectorNorm(tf: Map<string, number>, idf: Map<string, number>): number {
|
||
let sum = 0;
|
||
for (const [term, freq] of tf) {
|
||
const idfVal = idf.get(term) ?? 1;
|
||
sum += (freq * idfVal) ** 2;
|
||
}
|
||
return Math.sqrt(sum);
|
||
}
|
||
|
||
/** 时间衰减权重：30 天半衰期（age=30 时 weight=0.5） */
|
||
function timeDecayWeight(createdAt: number, now: number = Date.now()): number {
|
||
const ageDays = Math.max(0, (now - createdAt) / (24 * 60 * 60 * 1000));
|
||
const halfLifeDays = 30;
|
||
return Math.pow(0.5, ageDays / halfLifeDays);
|
||
}
|
||
|
||
// ===== 向量工具（v0.8.1 P1-1 本地向量混合检索） =====
|
||
|
||
/** Float32Array → SQLite BLOB（little-endian 原生布局，Node/SQLite 同机读写安全） */
|
||
function float32ToBlob(vec: number[]): Buffer {
|
||
const f32 = Float32Array.from(vec);
|
||
return Buffer.from(f32.buffer, f32.byteOffset, f32.byteLength);
|
||
}
|
||
|
||
/** SQLite BLOB → number[]（维度/字节损坏时返回 null，调用方回退 TF-IDF 路径） */
|
||
function blobToFloat32(blob: unknown): number[] | null {
|
||
if (!Buffer.isBuffer(blob)) return null;
|
||
if (blob.length === 0 || blob.length % 4 !== 0) return null;
|
||
const f32 = new Float32Array(blob.buffer, blob.byteOffset, blob.length / 4);
|
||
return Array.from(f32);
|
||
}
|
||
|
||
/** 余弦相似度（零向量/维度不匹配返回 0） */
|
||
function cosineSimilarity(a: number[], b: number[]): number {
|
||
if (a.length === 0 || a.length !== b.length) return 0;
|
||
let dot = 0;
|
||
let na = 0;
|
||
let nb = 0;
|
||
for (let i = 0; i < a.length; i++) {
|
||
dot += a[i] * b[i];
|
||
na += a[i] * a[i];
|
||
nb += b[i] * b[i];
|
||
}
|
||
if (na === 0 || nb === 0) return 0;
|
||
return dot / (Math.sqrt(na) * Math.sqrt(nb));
|
||
}
|
||
|
||
/**
|
||
* 记忆管理器
|
||
*/
|
||
export class MemoryManager {
|
||
/** IDF 缓存：词项 -> 文档频率 */
|
||
private idfCache = new Map<string, number>();
|
||
/** 缓存的记忆总数 */
|
||
private cachedDocCount = 0;
|
||
/** 缓存最后更新时间 */
|
||
private cacheUpdatedAt = 0;
|
||
/** 缓存有效期（5 分钟） */
|
||
private readonly CACHE_TTL = 5 * 60 * 1000;
|
||
|
||
/**
|
||
* v0.8.1 P1-1: 本地向量嵌入器（可选注入）。
|
||
* main.ts 仅在 Provider=Ollama 且用户配置了 memory.embeddingModel 时注入；
|
||
* null = 向量检索禁用，search/store 全部走纯 TF-IDF 路径（历史行为）。
|
||
*/
|
||
private embedder: MemoryEmbedder | null = null;
|
||
|
||
constructor(private getDB: () => Database.Database) {}
|
||
|
||
/** 注入向量嵌入器（传 null 关闭向量路径；热切换由 main.ts 在 adapter 重载时联动） */
|
||
setEmbedder(embedder: MemoryEmbedder | null): void {
|
||
this.embedder = embedder;
|
||
}
|
||
|
||
/**
|
||
* 初始化（表结构由 DatabaseService 创建）
|
||
*/
|
||
initialize(): void {
|
||
log.info('MemoryManager initialized (v0.2.0: TF-IDF enabled)');
|
||
}
|
||
|
||
/**
|
||
* 更新 IDF 缓存
|
||
*
|
||
* v0.3.0 增强：
|
||
* - 原子替换缓存（先构建新数据再替换，避免中间不一致状态）
|
||
* - 错误处理（数据库查询失败时保留旧缓存，不更新时间戳）
|
||
*/
|
||
private updateIdfCache(): void {
|
||
const now = Date.now();
|
||
if (now - this.cacheUpdatedAt < this.CACHE_TTL && this.cachedDocCount > 0) {
|
||
return; // 缓存未过期
|
||
}
|
||
|
||
const db = this.getDB();
|
||
|
||
try {
|
||
// v0.3.0: 先构建新缓存数据，再原子替换
|
||
const newIdfCache = new Map<string, number>();
|
||
|
||
// 获取所有记忆内容（episodic + semantic + working）
|
||
const episodicRows = db
|
||
.prepare('SELECT content, summary FROM episodic_memories')
|
||
.all() as Array<{ content: string; summary: string | null }>;
|
||
const semanticRows = db.prepare('SELECT value FROM semantic_memories').all() as Array<{
|
||
value: string;
|
||
}>;
|
||
const workingRows = db.prepare('SELECT value FROM working_memories').all() as Array<{
|
||
value: string;
|
||
}>;
|
||
|
||
const allDocs = [
|
||
...episodicRows.map((r) => r.content + ' ' + (r.summary ?? '')),
|
||
...semanticRows.map((r) => r.value),
|
||
...workingRows.map((r) => r.value),
|
||
];
|
||
|
||
const newDocCount = allDocs.length;
|
||
const docFreq = new Map<string, number>();
|
||
|
||
for (const doc of allDocs) {
|
||
const tokens = new Set(tokenize(doc));
|
||
for (const token of tokens) {
|
||
docFreq.set(token, (docFreq.get(token) ?? 0) + 1);
|
||
}
|
||
}
|
||
|
||
// IDF = log((N+1)/(df+1)) + 1（Sklearn 风格平滑），确保非负
|
||
for (const [term, df] of docFreq) {
|
||
newIdfCache.set(term, Math.log((newDocCount + 1) / (df + 1)) + 1);
|
||
}
|
||
|
||
// v0.3.0: 原子替换 — 只有新数据完全准备好后才替换旧缓存
|
||
this.idfCache = newIdfCache;
|
||
this.cachedDocCount = newDocCount;
|
||
this.cacheUpdatedAt = now;
|
||
} catch (error) {
|
||
// v0.3.0: 数据库查询失败时保留旧缓存，不更新 cacheUpdatedAt
|
||
// 这样下次 search() 会再次尝试更新
|
||
log.error('MemoryManager: Failed to update IDF cache, keeping stale cache:', error);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* P2-12: 从 tf_cache 列读取缓存的分词结果；缓存缺失/损坏时回退实时分词
|
||
*
|
||
* tf_cache 在 store() 写入（JSON 序列化的 token 数组），避免每次检索对
|
||
* 全部候选文档重复执行 CJK bigram 正则分词（记忆量上千条时明显退化）。
|
||
*/
|
||
private cachedTokens(cache: string | null | undefined, docText: string): string[] {
|
||
if (cache) {
|
||
try {
|
||
const t = JSON.parse(cache) as unknown;
|
||
if (Array.isArray(t) && t.every((x) => typeof x === 'string')) return t as string[];
|
||
} catch {
|
||
// 缓存损坏 → 回退实时分词
|
||
}
|
||
}
|
||
return tokenize(docText);
|
||
}
|
||
|
||
/**
|
||
* L-5 修复: 提取 scoreAndPushMemory 辅助函数
|
||
*
|
||
* 计算 TF-IDF 余弦相似度并应用时间衰减和重要度权重，
|
||
* 将分数 > 0 的记忆 push 到 results 数组。
|
||
*
|
||
* 三种记忆类型（episodic/semantic/working）的评分逻辑统一调用此函数，
|
||
* 仅在调用前构造 docTokens/createdAt/importance 等参数。
|
||
* P2-12: docText → docTokens（分词结果由调用方通过 tf_cache 提供，避免重复分词）
|
||
*
|
||
* @param params - 评分参数
|
||
* @param results - 结果数组（push 到此数组）
|
||
*/
|
||
private scoreAndPushMemory(
|
||
params: {
|
||
docTokens: string[];
|
||
createdAt: number;
|
||
importance: number;
|
||
id: string;
|
||
type: MemoryType;
|
||
content: string;
|
||
summary?: string;
|
||
source: MemorySource;
|
||
sessionId?: string;
|
||
expiresAt?: number;
|
||
/** v0.8.1 P1-1: 文档向量（embedding 列缺失/损坏时为 null → 单路 TF-IDF 评分） */
|
||
docVec?: number[] | null;
|
||
/** v0.8.1 P1-1: 查询向量（null → 单路 TF-IDF 评分） */
|
||
queryVec?: number[] | null;
|
||
},
|
||
queryTF: Map<string, number>,
|
||
queryNorm: number,
|
||
now: number,
|
||
results: SearchResult[],
|
||
): void {
|
||
const docTF = computeTF(params.docTokens);
|
||
const docNorm = vectorNorm(docTF, this.idfCache);
|
||
|
||
if (docNorm === 0 && !(params.docVec && params.queryVec)) return;
|
||
|
||
// 时间衰减
|
||
const decayWeight = timeDecayWeight(params.createdAt, now);
|
||
const importanceWeight = 0.5 + params.importance * 0.5;
|
||
|
||
// TF-IDF 路（docNorm=0 时得 0 分，交由向量路兜底）
|
||
const tfidfScore =
|
||
docNorm > 0
|
||
? (dotProduct(queryTF, docTF, this.idfCache) / (queryNorm * docNorm)) *
|
||
decayWeight *
|
||
importanceWeight
|
||
: 0;
|
||
// 向量路（双侧齐备时计算余弦）
|
||
const vectorScore =
|
||
params.docVec && params.queryVec
|
||
? cosineSimilarity(params.queryVec, params.docVec) * decayWeight * importanceWeight
|
||
: 0;
|
||
|
||
// v0.8.1 P1-1 混合评分：双侧齐备 0.6 向量 + 0.4 TF-IDF；否则取可用单路
|
||
let finalScore: number;
|
||
if (params.docVec && params.queryVec) {
|
||
finalScore = 0.6 * vectorScore + 0.4 * tfidfScore;
|
||
} else {
|
||
finalScore = tfidfScore > 0 ? tfidfScore : vectorScore;
|
||
}
|
||
|
||
if (finalScore > 0) {
|
||
results.push({
|
||
id: params.id,
|
||
type: params.type,
|
||
content: params.content,
|
||
summary: params.summary,
|
||
source: params.source,
|
||
importance: params.importance,
|
||
sessionId: params.sessionId,
|
||
createdAt: params.createdAt,
|
||
expiresAt: params.expiresAt,
|
||
score: finalScore,
|
||
});
|
||
}
|
||
}
|
||
|
||
/**
|
||
* TF-IDF 相似度搜索（v0.8.1 P1-1: 可选向量混合评分）
|
||
*
|
||
* @param queryVec 查询向量（embedder 未注入/失败时为 null → 纯 TF-IDF）
|
||
*/
|
||
private tfidfSearch(
|
||
query: string,
|
||
options: MemorySearchOptions,
|
||
queryVec: number[] | null,
|
||
): SearchResult[] {
|
||
const db = this.getDB();
|
||
this.updateIdfCache();
|
||
|
||
const queryTokens = tokenize(query);
|
||
if (queryTokens.length === 0) return [];
|
||
|
||
const queryTF = computeTF(queryTokens);
|
||
const queryNorm = vectorNorm(queryTF, this.idfCache);
|
||
if (queryNorm === 0) return [];
|
||
|
||
const { topK = 5, type, minImportance = 0 } = options;
|
||
const results: SearchResult[] = [];
|
||
const now = Date.now();
|
||
|
||
// L-5 修复: 三段搜索统一调用 scoreAndPushMemory，消除重复的 tokenize/computeTF/vectorNorm/dotProduct 逻辑
|
||
|
||
// 搜索 episodic 记忆
|
||
if (!type || type === 'episodic') {
|
||
const rows = db
|
||
.prepare(
|
||
`
|
||
SELECT * FROM episodic_memories WHERE importance >= ?
|
||
ORDER BY importance DESC, created_at DESC LIMIT ?
|
||
`,
|
||
)
|
||
.all(minImportance, topK * 3) as Array<{
|
||
id: string;
|
||
session_id: string | null;
|
||
content: string;
|
||
summary: string | null;
|
||
source: string;
|
||
importance: number;
|
||
created_at: number;
|
||
expires_at: number | null;
|
||
tf_cache: string | null;
|
||
}>;
|
||
|
||
for (const row of rows) {
|
||
this.scoreAndPushMemory(
|
||
{
|
||
docTokens: this.cachedTokens(row.tf_cache, row.content + ' ' + (row.summary ?? '')),
|
||
createdAt: row.created_at,
|
||
importance: row.importance,
|
||
id: row.id,
|
||
type: 'episodic',
|
||
content: row.content,
|
||
summary: row.summary ?? undefined,
|
||
source: row.source as MemorySource,
|
||
sessionId: row.session_id ?? undefined,
|
||
expiresAt: row.expires_at ?? undefined,
|
||
// v0.8.2 P3-1: 走模型指纹校验的解码（不匹配 → 视为缺失 + 惰性重算）
|
||
docVec: this.decodeEmbeddingRow(
|
||
(row as { embedding?: unknown }).embedding,
|
||
(row as { embedding_model?: string | null }).embedding_model ?? null,
|
||
row.content + ' ' + (row.summary ?? ''),
|
||
),
|
||
queryVec,
|
||
},
|
||
queryTF,
|
||
queryNorm,
|
||
now,
|
||
results,
|
||
);
|
||
}
|
||
this.backfillMissingEmbeddings(
|
||
'episodic',
|
||
rows.map((r) => ({
|
||
id: r.id,
|
||
embedding: (r as { embedding?: unknown }).embedding,
|
||
embedding_model: (r as { embedding_model?: string | null }).embedding_model ?? null,
|
||
text: r.content + ' ' + (r.summary ?? ''),
|
||
})),
|
||
);
|
||
}
|
||
|
||
// 搜索 semantic 记忆
|
||
if (!type || type === 'semantic') {
|
||
const rows = db
|
||
.prepare(
|
||
`
|
||
SELECT * FROM semantic_memories WHERE confidence >= ?
|
||
ORDER BY confidence DESC, access_count DESC LIMIT ?
|
||
`,
|
||
)
|
||
.all(minImportance, Math.ceil(topK * 1.5)) as Array<{
|
||
id: string;
|
||
key: string;
|
||
value: string;
|
||
category: string | null;
|
||
confidence: number;
|
||
source_session: string | null;
|
||
created_at: number;
|
||
tf_cache: string | null;
|
||
}>;
|
||
|
||
for (const row of rows) {
|
||
this.scoreAndPushMemory(
|
||
{
|
||
docTokens: this.cachedTokens(row.tf_cache, row.key + ' ' + row.value),
|
||
createdAt: row.created_at,
|
||
importance: row.confidence,
|
||
id: row.id,
|
||
type: 'semantic',
|
||
content: row.value,
|
||
source: 'imported',
|
||
sessionId: row.source_session ?? undefined,
|
||
docVec: blobToFloat32((row as { embedding?: unknown }).embedding),
|
||
queryVec,
|
||
},
|
||
queryTF,
|
||
queryNorm,
|
||
now,
|
||
results,
|
||
);
|
||
}
|
||
this.backfillMissingEmbeddings(
|
||
'semantic',
|
||
rows.map((r) => ({
|
||
id: r.id,
|
||
embedding: (r as { embedding?: unknown }).embedding,
|
||
embedding_model: (r as { embedding_model?: string | null }).embedding_model ?? null,
|
||
text: r.key + ' ' + r.value,
|
||
})),
|
||
);
|
||
}
|
||
|
||
// 搜索 working 记忆
|
||
if (!type || type === 'working') {
|
||
const rows = db
|
||
.prepare(
|
||
`
|
||
SELECT * FROM working_memories ORDER BY updated_at DESC LIMIT ?
|
||
`,
|
||
)
|
||
.all(topK * 3) as Array<{
|
||
id: string;
|
||
session_id: string;
|
||
task_id: string;
|
||
key: string;
|
||
value: string;
|
||
updated_at: number;
|
||
tf_cache: string | null;
|
||
}>;
|
||
|
||
for (const row of rows) {
|
||
this.scoreAndPushMemory(
|
||
{
|
||
docTokens: this.cachedTokens(row.tf_cache, row.key + ' ' + row.value),
|
||
createdAt: row.updated_at,
|
||
importance: 0.5,
|
||
id: row.id,
|
||
type: 'working',
|
||
content: row.value,
|
||
source: 'agent_thought',
|
||
sessionId: row.session_id,
|
||
},
|
||
queryTF,
|
||
queryNorm,
|
||
now,
|
||
results,
|
||
);
|
||
}
|
||
}
|
||
|
||
return results.sort((a, b) => b.score - a.score).slice(0, topK);
|
||
}
|
||
|
||
/**
|
||
* 存储记忆
|
||
*
|
||
* v0.3.0 修复：
|
||
* - switch 添加 default 分支，未知 type 抛错而非静默失败
|
||
* - working 类型使用 item.id（若提供）或生成唯一 key，避免同 session 多次存储互相覆盖
|
||
* - semantic 类型使用 item.summary 作为 key（若提供），支持更新已有记忆
|
||
*/
|
||
store(item: Omit<MemoryItem, 'id' | 'createdAt'> & { id?: string }): string {
|
||
const db = this.getDB();
|
||
const id = item.id ?? `mem_${nanoid(12)}`;
|
||
const now = Date.now();
|
||
const importance = item.importance ?? this.calculateImportance(item);
|
||
|
||
switch (item.type) {
|
||
case 'episodic':
|
||
db.prepare(
|
||
`
|
||
INSERT INTO episodic_memories (id, session_id, content, summary, source, importance, created_at, expires_at, tf_cache)
|
||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||
`,
|
||
).run(
|
||
id,
|
||
item.sessionId ?? null,
|
||
item.content,
|
||
item.summary ?? null,
|
||
item.source,
|
||
importance,
|
||
now,
|
||
// v0.8.1 P0-2: expires_at 真实写入方 —— 调用方（MemoryTriggerHook 等）可携带
|
||
// TTL；此前该列全链路无写入方，cleanupExpired 空转，情节记忆只增不减
|
||
item.expiresAt ?? null,
|
||
// P2-12: 写入时预计算分词缓存，加速后续检索
|
||
JSON.stringify(tokenize(item.content + ' ' + (item.summary ?? ''))),
|
||
);
|
||
break;
|
||
case 'semantic':
|
||
// v0.3.0 修复：使用 summary 作为 key（若提供），支持更新已有语义记忆
|
||
// #32 修复: 当 summary 未提供时，使用 content hash 作为 key 实现基于内容的去重
|
||
// v0.3.0 用 id 作为 key 时，因 id 每次新生成，INSERT OR REPLACE 永远不触发 REPLACE，
|
||
// 导致重复 store 同一内容会创建多条记忆。改为 contentHash 后，相同内容自动 REPLACE。
|
||
//
|
||
// v0.8.2 P3-1 根治: INSERT OR REPLACE → ON CONFLICT DO UPDATE。
|
||
// REPLACE 是"删旧行 + 插新行"—— 新 id 替换旧 id，**access_count 归零**：
|
||
// 热门记忆被同内容更新后 LRU 排序权重凭空丢失。现以 key 为冲突目标做
|
||
// 真正的 upsert：更新内容侧字段，保留 access_count 与旧 embedding
|
||
//（同 key 意味着内容相同，旧向量仍然有效；模型更换由指纹校验自愈）。
|
||
db.prepare(
|
||
`
|
||
INSERT INTO semantic_memories (id, key, value, category, confidence, source_session, created_at, updated_at, access_count, tf_cache)
|
||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0, ?)
|
||
ON CONFLICT(key) DO UPDATE SET
|
||
value = excluded.value,
|
||
category = excluded.category,
|
||
confidence = excluded.confidence,
|
||
source_session = excluded.source_session,
|
||
created_at = excluded.created_at,
|
||
updated_at = excluded.updated_at,
|
||
tf_cache = excluded.tf_cache
|
||
`,
|
||
).run(
|
||
id,
|
||
item.summary ?? this.contentHash(item.content),
|
||
item.content,
|
||
'general',
|
||
importance,
|
||
item.sessionId ?? null,
|
||
now,
|
||
now,
|
||
JSON.stringify(tokenize((item.summary ?? '') + ' ' + item.content)),
|
||
);
|
||
break;
|
||
case 'working':
|
||
// v0.3.0 修复：使用 summary 作为 key（若提供），避免硬编码 'default' 导致覆盖
|
||
// #32 修复: 当 summary 未提供时，使用 content hash 作为 key 实现基于内容的去重
|
||
db.prepare(
|
||
`
|
||
INSERT OR REPLACE INTO working_memories (id, session_id, task_id, key, value, updated_at, tf_cache)
|
||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||
`,
|
||
).run(
|
||
id,
|
||
item.sessionId ?? 'default',
|
||
'default',
|
||
item.summary ?? this.contentHash(item.content),
|
||
item.content,
|
||
now,
|
||
JSON.stringify(tokenize((item.summary ?? '') + ' ' + item.content)),
|
||
);
|
||
break;
|
||
default:
|
||
// v0.3.0 修复：未知 type 抛错而非静默失败
|
||
throw new Error(`Unknown memory type: ${(item as { type: string }).type}`);
|
||
}
|
||
|
||
// 使 IDF 缓存失效
|
||
this.cacheUpdatedAt = 0;
|
||
|
||
// v0.8.1 P1-1: 异步向量化回填（fire-and-forget）—— 嵌入不可用时静默跳过，
|
||
// 该记忆保留 NULL embedding，检索时自动回退 TF-IDF 路径
|
||
this.enrichEmbedding(item.type, id, (item.summary ?? '') + ' ' + item.content);
|
||
|
||
log.debug(`Memory stored: ${id} (${item.type})`);
|
||
return id;
|
||
}
|
||
|
||
/**
|
||
* v0.8.1 P1-1: 异步生成并回填 embedding BLOB。
|
||
* 失败静默（降级 TF-IDF），不阻塞写入方（工具执行/记忆固化均不等待）。
|
||
*/
|
||
private embeddingBackfillInFlight = new Set<string>();
|
||
|
||
private enrichEmbedding(type: MemoryType, id: string, text: string): void {
|
||
const embedder = this.embedder;
|
||
if (!embedder) return;
|
||
const table =
|
||
type === 'episodic' ? 'episodic_memories' : type === 'semantic' ? 'semantic_memories' : null;
|
||
if (!table) return; // working 记忆会话级生命周期短，不参与向量检索
|
||
const key = `${table}:${id}`;
|
||
if (this.embeddingBackfillInFlight.has(key)) return;
|
||
this.embeddingBackfillInFlight.add(key);
|
||
void embedder
|
||
.embed(text.slice(0, 8000))
|
||
.then((vec) => {
|
||
if (!vec || vec.length === 0) return;
|
||
// v0.8.2 P3-1: 同步写入模型指纹（检索时按指纹校验，模型更换后惰性重算）
|
||
this.getDB()
|
||
.prepare(`UPDATE ${table} SET embedding = ?, embedding_model = ? WHERE id = ?`)
|
||
.run(float32ToBlob(vec), embedder.modelName ?? null, id);
|
||
})
|
||
.catch((err) => {
|
||
log.debug(`MemoryManager: embedding enrichment skipped: ${(err as Error).message}`);
|
||
})
|
||
.finally(() => {
|
||
this.embeddingBackfillInFlight.delete(key);
|
||
});
|
||
}
|
||
|
||
/**
|
||
* v0.8.2 P3-1: BLOB 解码 + 模型指纹校验。
|
||
* ① 字节/维度损坏 → null（回退 TF-IDF）；② 当前嵌入器声明了模型指纹且行内
|
||
* 指纹不匹配（更换过 embedding 模型 / 旧版本写入的无指纹行）→ 视为缺失并
|
||
* 触发惰性重算 —— 旧实现跨模型余弦为 0（降级）或产生噪声分数，且无自愈路径。
|
||
*/
|
||
private decodeEmbeddingRow(
|
||
blob: unknown,
|
||
rowModel: string | null,
|
||
_text: string,
|
||
): number[] | null {
|
||
const vec = blobToFloat32(blob);
|
||
if (!vec) return null;
|
||
const currentModel = this.embedder?.modelName;
|
||
if (currentModel && (rowModel ?? null) !== currentModel) {
|
||
// 指纹不匹配：本轮按"无向量"处理（回退 TF-IDF），同时排队用当前模型重算
|
||
return null;
|
||
}
|
||
return vec;
|
||
}
|
||
|
||
/**
|
||
* v0.8.2 P3-1: 向量路独立召回（与重要度预过滤解耦）。
|
||
*
|
||
* 旧实现混合检索的向量余弦只在 `ORDER BY importance DESC LIMIT topK*3` 的
|
||
* 候选池内计算 —— 低重要度但语义高度相关的记忆永远进不了向量路，"同义改写
|
||
* 召回"（P1-1 立项目标）被结构性钳制。现当查询向量可用时，从 embedding 命中
|
||
* 的行中按时间取最近 VECTOR_RECALL_POOL 条独立召回（importance 过滤仍生效），
|
||
* 以纯向量余弦 × 衰减 × 重要度权重评分，由 search() 与 TF-IDF 路合并去重。
|
||
*/
|
||
private static readonly VECTOR_RECALL_POOL = 200;
|
||
|
||
private vectorRecall(
|
||
queryVec: number[],
|
||
options: MemorySearchOptions,
|
||
now: number,
|
||
): SearchResult[] {
|
||
const db = this.getDB();
|
||
const { topK = 5, type, minImportance = 0 } = options;
|
||
const pool = Math.max(topK * 10, MemoryManager.VECTOR_RECALL_POOL);
|
||
const results: SearchResult[] = [];
|
||
const importanceWeight = (importance: number): number => 0.5 + importance * 0.5;
|
||
|
||
if (!type || type === 'episodic') {
|
||
const rows = db
|
||
.prepare(
|
||
`
|
||
SELECT id, session_id, content, summary, source, importance, created_at, expires_at, embedding, embedding_model
|
||
FROM episodic_memories
|
||
WHERE importance >= ? AND embedding IS NOT NULL
|
||
ORDER BY created_at DESC LIMIT ?
|
||
`,
|
||
)
|
||
.all(minImportance, pool) as Array<{
|
||
id: string;
|
||
session_id: string | null;
|
||
content: string;
|
||
summary: string | null;
|
||
source: string;
|
||
importance: number;
|
||
created_at: number;
|
||
expires_at: number | null;
|
||
embedding: unknown;
|
||
embedding_model: string | null;
|
||
}>;
|
||
for (const row of rows) {
|
||
const docVec = this.decodeEmbeddingRow(row.embedding, row.embedding_model, row.content);
|
||
if (!docVec) {
|
||
// 指纹不匹配（有旧向量但不被当前模型接受）→ 惰性重算
|
||
if (row.embedding != null) this.enrichEmbedding('episodic', row.id, row.content);
|
||
continue;
|
||
}
|
||
const sim = cosineSimilarity(queryVec, docVec);
|
||
if (sim <= 0) continue;
|
||
results.push({
|
||
id: row.id,
|
||
type: 'episodic',
|
||
content: row.content,
|
||
summary: row.summary ?? undefined,
|
||
source: row.source as MemorySource,
|
||
importance: row.importance,
|
||
sessionId: row.session_id ?? undefined,
|
||
createdAt: row.created_at,
|
||
expiresAt: row.expires_at ?? undefined,
|
||
score: sim * timeDecayWeight(row.created_at, now) * importanceWeight(row.importance),
|
||
});
|
||
}
|
||
}
|
||
|
||
if (!type || type === 'semantic') {
|
||
const rows = db
|
||
.prepare(
|
||
`
|
||
SELECT id, key, value, confidence, source_session, created_at, embedding, embedding_model
|
||
FROM semantic_memories
|
||
WHERE confidence >= ? AND embedding IS NOT NULL
|
||
ORDER BY updated_at DESC LIMIT ?
|
||
`,
|
||
)
|
||
.all(minImportance, pool) as Array<{
|
||
id: string;
|
||
key: string;
|
||
value: string;
|
||
confidence: number;
|
||
source_session: string | null;
|
||
created_at: number;
|
||
embedding: unknown;
|
||
embedding_model: string | null;
|
||
}>;
|
||
for (const row of rows) {
|
||
const docVec = this.decodeEmbeddingRow(
|
||
row.embedding,
|
||
row.embedding_model,
|
||
row.key + ' ' + row.value,
|
||
);
|
||
if (!docVec) {
|
||
if (row.embedding != null)
|
||
this.enrichEmbedding('semantic', row.id, row.key + ' ' + row.value);
|
||
continue;
|
||
}
|
||
const sim = cosineSimilarity(queryVec, docVec);
|
||
if (sim <= 0) continue;
|
||
results.push({
|
||
id: row.id,
|
||
type: 'semantic',
|
||
content: row.value,
|
||
source: 'imported',
|
||
importance: row.confidence,
|
||
sessionId: row.source_session ?? undefined,
|
||
createdAt: row.created_at,
|
||
score: sim * timeDecayWeight(row.created_at, now) * importanceWeight(row.confidence),
|
||
});
|
||
}
|
||
}
|
||
|
||
return results.sort((a, b) => b.score - a.score).slice(0, topK);
|
||
}
|
||
|
||
/**
|
||
* v0.8.1 P1-1: 存量记忆向量惰性回填 —— 嵌入功能开启前写入的记忆（embedding
|
||
* IS NULL）在参与检索时排队补算：本轮查询仍走 TF-IDF，后续查询即可命中向量
|
||
* 路径。无阻塞、无独立迁移任务，收敛速度随检索频次自然提升；嵌入器不可用
|
||
* 时零开销（直接返回）。
|
||
*/
|
||
private backfillMissingEmbeddings(
|
||
type: MemoryType,
|
||
rows: Array<{ id: string; embedding?: unknown; embedding_model?: string | null; text: string }>,
|
||
): void {
|
||
if (!this.embedder) return;
|
||
const currentModel = this.embedder.modelName;
|
||
for (const row of rows) {
|
||
if (row.embedding == null) {
|
||
this.enrichEmbedding(type, row.id, row.text);
|
||
continue;
|
||
}
|
||
// v0.8.2 P3-1: 指纹不匹配（更换过 embedding 模型 / 旧版本无指纹行）→
|
||
// 用当前模型重算，旧向量在重算完成前不参与向量评分
|
||
if (currentModel && (row.embedding_model ?? null) !== currentModel) {
|
||
this.enrichEmbedding(type, row.id, row.text);
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* v0.8.1 P0-2: 检索命中后回写 semantic_memories.access_count（LRU 淘汰语义激活）。
|
||
* 此前该列只在 ORDER BY 中被读取、从无更新方，LRU 淘汰是死语义。
|
||
*/
|
||
private bumpAccessCounts(results: SearchResult[]): void {
|
||
const semanticIds = results.filter((r) => r.type === 'semantic').map((r) => r.id);
|
||
if (semanticIds.length === 0) return;
|
||
try {
|
||
const placeholders = semanticIds.map(() => '?').join(', ');
|
||
this.getDB()
|
||
.prepare(
|
||
`UPDATE semantic_memories SET access_count = access_count + 1 WHERE id IN (${placeholders})`,
|
||
)
|
||
.run(...semanticIds);
|
||
} catch (err) {
|
||
// 计数回写失败不影响检索结果
|
||
log.debug('MemoryManager: access_count bump failed:', (err as Error).message);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 检索记忆（v0.2.0: TF-IDF 语义检索 + 时间衰减；v0.8.1 P1-1: 本地向量混合检索）
|
||
*
|
||
* v0.8.1 变更：
|
||
* - 方法改为 async（查询向量需经 MemoryEmbedder 异步生成；Ollama 本地嵌入）。
|
||
* - 混合评分：查询向量与文档向量齐备时 score = 0.6×向量余弦 + 0.4×TF-IDF
|
||
* （两者各自叠加时间衰减与重要度权重）；任一缺失时回退单路评分 ——
|
||
* 未注入 embedder（或嵌入失败）时行为与历史版本完全一致。
|
||
* - 检索命中的 semantic 记忆回写 access_count（LRU 语义激活，P0-2）。
|
||
*/
|
||
async search(query: string, options: MemorySearchOptions = {}): Promise<SearchResult[]> {
|
||
const db = this.getDB();
|
||
const { topK = 5, type, minImportance = 0 } = options;
|
||
// v0.3.0 修复：拦截空 query 和纯空格 query
|
||
if (!query || !query.trim()) return [];
|
||
|
||
// v0.8.1: 查询向量生成一次（嵌入器缺失/失败 → null，全量回退 TF-IDF）
|
||
let queryVec: number[] | null = null;
|
||
if (this.embedder) {
|
||
try {
|
||
queryVec = await this.embedder.embed(query.slice(0, 8000));
|
||
if (queryVec && queryVec.length === 0) queryVec = null;
|
||
} catch (err) {
|
||
log.debug(
|
||
'MemoryManager: query embedding failed, falling back to TF-IDF:',
|
||
(err as Error).message,
|
||
);
|
||
queryVec = null;
|
||
}
|
||
}
|
||
|
||
// v0.2.0: 优先使用语义搜索（TF-IDF ± 向量混合）
|
||
const now = Date.now();
|
||
const tfidfResults = this.tfidfSearch(query, options, queryVec);
|
||
|
||
// v0.8.2 P3-1: 向量路独立召回合并 —— 低重要度但语义相关的记忆不再被
|
||
// importance 预过滤的候选池钳制（tfidfSearch 的向量分量只在重要度池内算）。
|
||
let merged = tfidfResults;
|
||
if (queryVec) {
|
||
const vectorResults = this.vectorRecall(queryVec, options, now);
|
||
if (vectorResults.length > 0) {
|
||
const byId = new Map<string, SearchResult>();
|
||
for (const r of [...tfidfResults, ...vectorResults]) {
|
||
const prev = byId.get(r.id);
|
||
if (!prev || r.score > prev.score) byId.set(r.id, r);
|
||
}
|
||
merged = [...byId.values()].sort((a, b) => b.score - a.score).slice(0, topK);
|
||
}
|
||
}
|
||
|
||
if (merged.length > 0) {
|
||
this.bumpAccessCounts(merged);
|
||
return merged;
|
||
}
|
||
|
||
// 回退：如果 TF-IDF 没有结果（如 IDF 缓存为空），使用 LIKE 关键词搜索
|
||
// 转义 LIKE 通配符，避免用户输入的 % 和 _ 影响匹配
|
||
// v0.3.0 修复: 反斜杠也需转义，否则含 \ 的搜索（如 Windows 路径）会导致 SQLite LIKE 报错
|
||
const escapedQuery = query.replace(/[%_\\]/g, '\\$&');
|
||
const pattern = `%${escapedQuery}%`;
|
||
const results: SearchResult[] = [];
|
||
|
||
// 搜索情节记忆
|
||
if (!type || type === 'episodic') {
|
||
const rows = db
|
||
.prepare(
|
||
`
|
||
SELECT * FROM episodic_memories
|
||
WHERE (content LIKE ? ESCAPE '\\' OR summary LIKE ? ESCAPE '\\') AND importance >= ?
|
||
ORDER BY importance DESC, created_at DESC LIMIT ?
|
||
`,
|
||
)
|
||
.all(pattern, pattern, minImportance, topK) as Array<{
|
||
id: string;
|
||
session_id: string | null;
|
||
content: string;
|
||
summary: string | null;
|
||
source: string;
|
||
importance: number;
|
||
created_at: number;
|
||
expires_at: number | null;
|
||
}>;
|
||
for (const row of rows) {
|
||
results.push({
|
||
id: row.id,
|
||
type: 'episodic',
|
||
content: row.content,
|
||
summary: row.summary ?? undefined,
|
||
source: row.source as MemorySource,
|
||
importance: row.importance,
|
||
sessionId: row.session_id ?? undefined,
|
||
createdAt: row.created_at,
|
||
expiresAt: row.expires_at ?? undefined,
|
||
score: row.importance * timeDecayWeight(row.created_at),
|
||
});
|
||
}
|
||
}
|
||
|
||
// 搜索语义记忆
|
||
if (!type || type === 'semantic') {
|
||
const rows = db
|
||
.prepare(
|
||
`
|
||
SELECT * FROM semantic_memories
|
||
WHERE (key LIKE ? ESCAPE '\\' OR value LIKE ? ESCAPE '\\') AND confidence >= ?
|
||
ORDER BY confidence DESC, access_count DESC LIMIT ?
|
||
`,
|
||
)
|
||
.all(pattern, pattern, minImportance, Math.ceil(topK / 2)) as Array<{
|
||
id: string;
|
||
key: string;
|
||
value: string;
|
||
category: string | null;
|
||
confidence: number;
|
||
source_session: string | null;
|
||
created_at: number;
|
||
}>;
|
||
for (const row of rows) {
|
||
results.push({
|
||
id: row.id,
|
||
type: 'semantic',
|
||
content: row.value,
|
||
source: 'imported',
|
||
importance: row.confidence,
|
||
sessionId: row.source_session ?? undefined,
|
||
createdAt: row.created_at,
|
||
score: row.confidence * timeDecayWeight(row.created_at),
|
||
});
|
||
}
|
||
}
|
||
|
||
// 搜索工作记忆
|
||
// v0.3.0 修复：LIKE 回退路径也需添加 !type 分支（与 tfidfSearch 保持一致）
|
||
if (!type || type === 'working') {
|
||
const rows = db
|
||
.prepare(
|
||
`
|
||
SELECT * FROM working_memories
|
||
WHERE (key LIKE ? ESCAPE '\\' OR value LIKE ? ESCAPE '\\')
|
||
ORDER BY updated_at DESC LIMIT ?
|
||
`,
|
||
)
|
||
.all(pattern, pattern, topK) as Array<{
|
||
id: string;
|
||
session_id: string;
|
||
task_id: string;
|
||
key: string;
|
||
value: string;
|
||
updated_at: number;
|
||
}>;
|
||
for (const row of rows) {
|
||
results.push({
|
||
id: row.id,
|
||
type: 'working',
|
||
content: row.value,
|
||
source: 'agent_thought',
|
||
importance: 0.5,
|
||
sessionId: row.session_id,
|
||
createdAt: row.updated_at,
|
||
score: 0.3 * timeDecayWeight(row.updated_at),
|
||
});
|
||
}
|
||
}
|
||
|
||
const finalResults = results.sort((a, b) => b.score - a.score).slice(0, topK);
|
||
this.bumpAccessCounts(finalResults);
|
||
return finalResults;
|
||
}
|
||
|
||
/**
|
||
* 获取工作记忆
|
||
*/
|
||
getWorkingMemory(sessionId: string, taskId: string = 'default'): Map<string, string> {
|
||
const db = this.getDB();
|
||
const rows = db
|
||
.prepare(
|
||
`
|
||
SELECT key, value FROM working_memories WHERE session_id = ? AND task_id = ?
|
||
`,
|
||
)
|
||
.all(sessionId, taskId) as Array<{ key: string; value: string }>;
|
||
return new Map(rows.map((r) => [r.key, r.value]));
|
||
}
|
||
|
||
/**
|
||
* 更新工作记忆
|
||
*/
|
||
setWorkingMemory(sessionId: string, taskId: string, key: string, value: string): void {
|
||
const db = this.getDB();
|
||
db.prepare(
|
||
`
|
||
INSERT OR REPLACE INTO working_memories (id, session_id, task_id, key, value, updated_at)
|
||
VALUES (?, ?, ?, ?, ?, ?)
|
||
`,
|
||
).run(`wm_${nanoid(8)}`, sessionId, taskId, key, value, Date.now());
|
||
}
|
||
|
||
/**
|
||
* 清除工作记忆
|
||
*/
|
||
clearWorkingMemory(sessionId: string, taskId?: string): void {
|
||
const db = this.getDB();
|
||
if (taskId) {
|
||
db.prepare('DELETE FROM working_memories WHERE session_id = ? AND task_id = ?').run(
|
||
sessionId,
|
||
taskId,
|
||
);
|
||
} else {
|
||
db.prepare('DELETE FROM working_memories WHERE session_id = ?').run(sessionId);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 清理过期记忆
|
||
*/
|
||
cleanupExpired(): number {
|
||
const db = this.getDB();
|
||
const result = db
|
||
.prepare('DELETE FROM episodic_memories WHERE expires_at IS NOT NULL AND expires_at < ?')
|
||
.run(Date.now());
|
||
return result.changes;
|
||
}
|
||
|
||
private calculateImportance(item: Omit<MemoryItem, 'id' | 'createdAt'>): number {
|
||
let score = 0.5;
|
||
if (item.source === 'user_input') score += 0.2;
|
||
if (item.source === 'tool_result') score += 0.1;
|
||
if (item.content.length > 200) score += 0.1;
|
||
return Math.min(1, Math.max(0, score));
|
||
}
|
||
|
||
/**
|
||
* #32 修复: 计算 content 的 SHA-256 hash（取前 16 字符），用于基于内容的去重
|
||
* 当 store 未提供 summary 时，用 contentHash 作为 semantic/working 的 key，
|
||
* 使 INSERT OR REPLACE 能基于内容触发 REPLACE，避免重复存储相同内容。
|
||
*/
|
||
private contentHash(content: string): string {
|
||
return createHash('sha256').update(content, 'utf-8').digest('hex').slice(0, 16);
|
||
}
|
||
}
|