feat: v0.8.2 安全纵深补全 · 协议保真 · 断链修复 — 图片SSRF/根MEMORY.md保护根治 · Anthropic thinking回传+pause_turn续传 · 2523 用例全量回归 + E2E 扩充
This commit is contained in:
@@ -414,7 +414,12 @@ export class MemoryManager {
|
||||
source: row.source as MemorySource,
|
||||
sessionId: row.session_id ?? undefined,
|
||||
expiresAt: row.expires_at ?? undefined,
|
||||
docVec: blobToFloat32((row as { embedding?: unknown }).embedding),
|
||||
// v0.8.2 P3-1: 走模型指纹校验的解码（不匹配 → 视为缺失 + 惰性重算）
|
||||
docVec: this.decodeEmbeddingRow(
|
||||
(row as { embedding?: unknown }).embedding,
|
||||
(row as { embedding_model?: string | null }).embedding_model ?? null,
|
||||
row.content + ' ' + (row.summary ?? ''),
|
||||
),
|
||||
queryVec,
|
||||
},
|
||||
queryTF,
|
||||
@@ -428,6 +433,7 @@ export class MemoryManager {
|
||||
rows.map((r) => ({
|
||||
id: r.id,
|
||||
embedding: (r as { embedding?: unknown }).embedding,
|
||||
embedding_model: (r as { embedding_model?: string | null }).embedding_model ?? null,
|
||||
text: r.content + ' ' + (r.summary ?? ''),
|
||||
})),
|
||||
);
|
||||
@@ -478,6 +484,7 @@ export class MemoryManager {
|
||||
rows.map((r) => ({
|
||||
id: r.id,
|
||||
embedding: (r as { embedding?: unknown }).embedding,
|
||||
embedding_model: (r as { embedding_model?: string | null }).embedding_model ?? null,
|
||||
text: r.key + ' ' + r.value,
|
||||
})),
|
||||
);
|
||||
@@ -565,10 +572,24 @@ export class MemoryManager {
|
||||
// #32 修复: 当 summary 未提供时，使用 content hash 作为 key 实现基于内容的去重
|
||||
// v0.3.0 用 id 作为 key 时，因 id 每次新生成，INSERT OR REPLACE 永远不触发 REPLACE，
|
||||
// 导致重复 store 同一内容会创建多条记忆。改为 contentHash 后，相同内容自动 REPLACE。
|
||||
//
|
||||
// v0.8.2 P3-1 根治: INSERT OR REPLACE → ON CONFLICT DO UPDATE。
|
||||
// REPLACE 是"删旧行 + 插新行"—— 新 id 替换旧 id，**access_count 归零**：
|
||||
// 热门记忆被同内容更新后 LRU 排序权重凭空丢失。现以 key 为冲突目标做
|
||||
// 真正的 upsert：更新内容侧字段，保留 access_count 与旧 embedding
|
||||
//（同 key 意味着内容相同，旧向量仍然有效；模型更换由指纹校验自愈）。
|
||||
db.prepare(
|
||||
`
|
||||
INSERT OR REPLACE INTO semantic_memories (id, key, value, category, confidence, source_session, created_at, updated_at, access_count, tf_cache)
|
||||
INSERT INTO semantic_memories (id, key, value, category, confidence, source_session, created_at, updated_at, access_count, tf_cache)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0, ?)
|
||||
ON CONFLICT(key) DO UPDATE SET
|
||||
value = excluded.value,
|
||||
category = excluded.category,
|
||||
confidence = excluded.confidence,
|
||||
source_session = excluded.source_session,
|
||||
created_at = excluded.created_at,
|
||||
updated_at = excluded.updated_at,
|
||||
tf_cache = excluded.tf_cache
|
||||
`,
|
||||
).run(
|
||||
id,
|
||||
@@ -635,9 +656,10 @@ export class MemoryManager {
|
||||
.embed(text.slice(0, 8000))
|
||||
.then((vec) => {
|
||||
if (!vec || vec.length === 0) return;
|
||||
// v0.8.2 P3-1: 同步写入模型指纹（检索时按指纹校验，模型更换后惰性重算）
|
||||
this.getDB()
|
||||
.prepare(`UPDATE ${table} SET embedding = ? WHERE id = ?`)
|
||||
.run(float32ToBlob(vec), id);
|
||||
.prepare(`UPDATE ${table} SET embedding = ?, embedding_model = ? WHERE id = ?`)
|
||||
.run(float32ToBlob(vec), embedder.modelName ?? null, id);
|
||||
})
|
||||
.catch((err) => {
|
||||
log.debug(`MemoryManager: embedding enrichment skipped: ${(err as Error).message}`);
|
||||
@@ -647,6 +669,144 @@ export class MemoryManager {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.8.2 P3-1: BLOB 解码 + 模型指纹校验。
|
||||
* ① 字节/维度损坏 → null（回退 TF-IDF）；② 当前嵌入器声明了模型指纹且行内
|
||||
* 指纹不匹配（更换过 embedding 模型 / 旧版本写入的无指纹行）→ 视为缺失并
|
||||
* 触发惰性重算 —— 旧实现跨模型余弦为 0（降级）或产生噪声分数，且无自愈路径。
|
||||
*/
|
||||
private decodeEmbeddingRow(
|
||||
blob: unknown,
|
||||
rowModel: string | null,
|
||||
_text: string,
|
||||
): number[] | null {
|
||||
const vec = blobToFloat32(blob);
|
||||
if (!vec) return null;
|
||||
const currentModel = this.embedder?.modelName;
|
||||
if (currentModel && (rowModel ?? null) !== currentModel) {
|
||||
// 指纹不匹配：本轮按"无向量"处理（回退 TF-IDF），同时排队用当前模型重算
|
||||
return null;
|
||||
}
|
||||
return vec;
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.8.2 P3-1: 向量路独立召回（与重要度预过滤解耦）。
|
||||
*
|
||||
* 旧实现混合检索的向量余弦只在 `ORDER BY importance DESC LIMIT topK*3` 的
|
||||
* 候选池内计算 —— 低重要度但语义高度相关的记忆永远进不了向量路，"同义改写
|
||||
* 召回"（P1-1 立项目标）被结构性钳制。现当查询向量可用时，从 embedding 命中
|
||||
* 的行中按时间取最近 VECTOR_RECALL_POOL 条独立召回（importance 过滤仍生效），
|
||||
* 以纯向量余弦 × 衰减 × 重要度权重评分，由 search() 与 TF-IDF 路合并去重。
|
||||
*/
|
||||
private static readonly VECTOR_RECALL_POOL = 200;
|
||||
|
||||
private vectorRecall(
|
||||
queryVec: number[],
|
||||
options: MemorySearchOptions,
|
||||
now: number,
|
||||
): SearchResult[] {
|
||||
const db = this.getDB();
|
||||
const { topK = 5, type, minImportance = 0 } = options;
|
||||
const pool = Math.max(topK * 10, MemoryManager.VECTOR_RECALL_POOL);
|
||||
const results: SearchResult[] = [];
|
||||
const importanceWeight = (importance: number): number => 0.5 + importance * 0.5;
|
||||
|
||||
if (!type || type === 'episodic') {
|
||||
const rows = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT id, session_id, content, summary, source, importance, created_at, expires_at, embedding, embedding_model
|
||||
FROM episodic_memories
|
||||
WHERE importance >= ? AND embedding IS NOT NULL
|
||||
ORDER BY created_at DESC LIMIT ?
|
||||
`,
|
||||
)
|
||||
.all(minImportance, pool) as Array<{
|
||||
id: string;
|
||||
session_id: string | null;
|
||||
content: string;
|
||||
summary: string | null;
|
||||
source: string;
|
||||
importance: number;
|
||||
created_at: number;
|
||||
expires_at: number | null;
|
||||
embedding: unknown;
|
||||
embedding_model: string | null;
|
||||
}>;
|
||||
for (const row of rows) {
|
||||
const docVec = this.decodeEmbeddingRow(row.embedding, row.embedding_model, row.content);
|
||||
if (!docVec) {
|
||||
// 指纹不匹配（有旧向量但不被当前模型接受）→ 惰性重算
|
||||
if (row.embedding != null) this.enrichEmbedding('episodic', row.id, row.content);
|
||||
continue;
|
||||
}
|
||||
const sim = cosineSimilarity(queryVec, docVec);
|
||||
if (sim <= 0) continue;
|
||||
results.push({
|
||||
id: row.id,
|
||||
type: 'episodic',
|
||||
content: row.content,
|
||||
summary: row.summary ?? undefined,
|
||||
source: row.source as MemorySource,
|
||||
importance: row.importance,
|
||||
sessionId: row.session_id ?? undefined,
|
||||
createdAt: row.created_at,
|
||||
expiresAt: row.expires_at ?? undefined,
|
||||
score: sim * timeDecayWeight(row.created_at, now) * importanceWeight(row.importance),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (!type || type === 'semantic') {
|
||||
const rows = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT id, key, value, confidence, source_session, created_at, embedding, embedding_model
|
||||
FROM semantic_memories
|
||||
WHERE confidence >= ? AND embedding IS NOT NULL
|
||||
ORDER BY updated_at DESC LIMIT ?
|
||||
`,
|
||||
)
|
||||
.all(minImportance, pool) as Array<{
|
||||
id: string;
|
||||
key: string;
|
||||
value: string;
|
||||
confidence: number;
|
||||
source_session: string | null;
|
||||
created_at: number;
|
||||
embedding: unknown;
|
||||
embedding_model: string | null;
|
||||
}>;
|
||||
for (const row of rows) {
|
||||
const docVec = this.decodeEmbeddingRow(
|
||||
row.embedding,
|
||||
row.embedding_model,
|
||||
row.key + ' ' + row.value,
|
||||
);
|
||||
if (!docVec) {
|
||||
if (row.embedding != null)
|
||||
this.enrichEmbedding('semantic', row.id, row.key + ' ' + row.value);
|
||||
continue;
|
||||
}
|
||||
const sim = cosineSimilarity(queryVec, docVec);
|
||||
if (sim <= 0) continue;
|
||||
results.push({
|
||||
id: row.id,
|
||||
type: 'semantic',
|
||||
content: row.value,
|
||||
source: 'imported',
|
||||
importance: row.confidence,
|
||||
sessionId: row.source_session ?? undefined,
|
||||
createdAt: row.created_at,
|
||||
score: sim * timeDecayWeight(row.created_at, now) * importanceWeight(row.confidence),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return results.sort((a, b) => b.score - a.score).slice(0, topK);
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.8.1 P1-1: 存量记忆向量惰性回填 —— 嵌入功能开启前写入的记忆（embedding
|
||||
* IS NULL）在参与检索时排队补算：本轮查询仍走 TF-IDF，后续查询即可命中向量
|
||||
@@ -655,12 +815,20 @@ export class MemoryManager {
|
||||
*/
|
||||
private backfillMissingEmbeddings(
|
||||
type: MemoryType,
|
||||
rows: Array<{ id: string; embedding?: unknown; text: string }>,
|
||||
rows: Array<{ id: string; embedding?: unknown; embedding_model?: string | null; text: string }>,
|
||||
): void {
|
||||
if (!this.embedder) return;
|
||||
const currentModel = this.embedder.modelName;
|
||||
for (const row of rows) {
|
||||
if (row.embedding != null) continue;
|
||||
this.enrichEmbedding(type, row.id, row.text);
|
||||
if (row.embedding == null) {
|
||||
this.enrichEmbedding(type, row.id, row.text);
|
||||
continue;
|
||||
}
|
||||
// v0.8.2 P3-1: 指纹不匹配（更换过 embedding 模型 / 旧版本无指纹行）→
|
||||
// 用当前模型重算，旧向量在重算完成前不参与向量评分
|
||||
if (currentModel && (row.embedding_model ?? null) !== currentModel) {
|
||||
this.enrichEmbedding(type, row.id, row.text);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -716,10 +884,27 @@ export class MemoryManager {
|
||||
}
|
||||
|
||||
// v0.2.0: 优先使用语义搜索（TF-IDF ± 向量混合）
|
||||
const now = Date.now();
|
||||
const tfidfResults = this.tfidfSearch(query, options, queryVec);
|
||||
if (tfidfResults.length > 0) {
|
||||
this.bumpAccessCounts(tfidfResults);
|
||||
return tfidfResults;
|
||||
|
||||
// v0.8.2 P3-1: 向量路独立召回合并 —— 低重要度但语义相关的记忆不再被
|
||||
// importance 预过滤的候选池钳制（tfidfSearch 的向量分量只在重要度池内算）。
|
||||
let merged = tfidfResults;
|
||||
if (queryVec) {
|
||||
const vectorResults = this.vectorRecall(queryVec, options, now);
|
||||
if (vectorResults.length > 0) {
|
||||
const byId = new Map<string, SearchResult>();
|
||||
for (const r of [...tfidfResults, ...vectorResults]) {
|
||||
const prev = byId.get(r.id);
|
||||
if (!prev || r.score > prev.score) byId.set(r.id, r);
|
||||
}
|
||||
merged = [...byId.values()].sort((a, b) => b.score - a.score).slice(0, topK);
|
||||
}
|
||||
}
|
||||
|
||||
if (merged.length > 0) {
|
||||
this.bumpAccessCounts(merged);
|
||||
return merged;
|
||||
}
|
||||
|
||||
// 回退：如果 TF-IDF 没有结果（如 IDF 缓存为空），使用 LIKE 关键词搜索
|
||||
|
||||
Reference in New Issue
Block a user