feat: v0.8.2 安全纵深补全 · 协议保真 · 断链修复 — 图片SSRF/根MEMORY.md保护根治 · Anthropic thinking回传+pause_turn续传 · 2523 用例全量回归 + E2E 扩充
This commit is contained in:
@@ -31,7 +31,8 @@ try {
|
||||
|
||||
import { MemoryManager } from '../manager';
|
||||
|
||||
// 与 DatabaseService.createTables 一致的记忆三表 schema(含 tf_cache 列)
|
||||
// 与 DatabaseService.createTables 一致的记忆三表 schema(含 tf_cache / embedding /
|
||||
// embedding_model 列 —— v0.8.2 P3-1 迁移 14 对齐)
|
||||
function createMemorySchema(db: any): void {
|
||||
db.exec(`
|
||||
CREATE TABLE episodic_memories (
|
||||
@@ -44,7 +45,8 @@ function createMemorySchema(db: any): void {
|
||||
created_at INTEGER NOT NULL DEFAULT 0,
|
||||
expires_at INTEGER,
|
||||
tf_cache TEXT,
|
||||
embedding BLOB
|
||||
embedding BLOB,
|
||||
embedding_model TEXT
|
||||
);
|
||||
CREATE TABLE semantic_memories (
|
||||
id TEXT PRIMARY KEY,
|
||||
@@ -57,7 +59,8 @@ function createMemorySchema(db: any): void {
|
||||
updated_at INTEGER NOT NULL DEFAULT 0,
|
||||
access_count INTEGER DEFAULT 0,
|
||||
tf_cache TEXT,
|
||||
embedding BLOB
|
||||
embedding BLOB,
|
||||
embedding_model TEXT
|
||||
);
|
||||
CREATE TABLE working_memories (
|
||||
id TEXT PRIMARY KEY,
|
||||
@@ -783,9 +786,9 @@ describe.skipIf(!dbAvailable)('MemoryManager — cleanupExpired', () => {
|
||||
}
|
||||
});
|
||||
|
||||
// 注意:manager.store() 的 episodic INSERT 不含 expires_at 列(源码已知缺口,
|
||||
// main.ts 注释亦确认"expires_at 无写入方")—— 本组用例直接经 SQL 写入
|
||||
// expires_at 模拟真实过期行,锁定 cleanupExpired 自身的删除契约。
|
||||
// v0.8.2 P3-6 注释修正:v0.8.1 P0-2 已让 store() 真实写入 expires_at(本组
|
||||
// 早期版本注释"INSERT 不含 expires_at 列"已过时)。此处直接经 SQL 写入仅为
|
||||
// 测试夹具便利(绕过 store 的哈希/分词管线),锁定 cleanupExpired 的删除契约。
|
||||
const insertExpiring = (id: string, expiresAt: number, content = '带过期记忆'): void => {
|
||||
db.prepare(
|
||||
`INSERT INTO episodic_memories (id, content, source, importance, created_at, expires_at)
|
||||
@@ -984,3 +987,73 @@ describe.skipIf(!dbAvailable)('MemoryManager — v0.8.1 生命周期与混合检
|
||||
expect(row.embedding!.length % 4).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ===== v0.8.2 P3-1: access_count 保留 + 模型指纹 =====
|
||||
|
||||
describe.skipIf(!dbAvailable)('MemoryManager — v0.8.2 P3-1', () => {
|
||||
let db: InstanceType<typeof Database>;
|
||||
let mgr: MemoryManager;
|
||||
|
||||
beforeEach(() => {
|
||||
db = new Database(':memory:');
|
||||
createMemorySchema(db);
|
||||
mgr = new MemoryManager(() => db);
|
||||
});
|
||||
|
||||
it('semantic 同内容重复 store → ON CONFLICT upsert,access_count 不归零', () => {
|
||||
mgr.store({
|
||||
type: 'semantic',
|
||||
content: '用户偏好深色主题',
|
||||
source: 'user_input',
|
||||
importance: 0.9,
|
||||
});
|
||||
db.prepare('UPDATE semantic_memories SET access_count = 7').run();
|
||||
// 同内容重复写入(key = contentHash,冲突命中同一行)
|
||||
mgr.store({
|
||||
type: 'semantic',
|
||||
content: '用户偏好深色主题',
|
||||
source: 'user_input',
|
||||
importance: 0.9,
|
||||
});
|
||||
const rows = db.prepare('SELECT access_count, value FROM semantic_memories').all() as Array<{
|
||||
access_count: number;
|
||||
value: string;
|
||||
}>;
|
||||
// 旧 REPLACE 语义会删除重插 → 2 行且 access_count 归零
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0].access_count).toBe(7);
|
||||
});
|
||||
|
||||
it('embedding_model 指纹:模型不匹配的向量按缺失处理并触发重算', async () => {
|
||||
mgr.store({
|
||||
type: 'semantic',
|
||||
content: '指纹校验内容',
|
||||
summary: 'fp-check',
|
||||
source: 'agent_thought',
|
||||
importance: 0.8,
|
||||
});
|
||||
await new Promise((r) => setTimeout(r, 10));
|
||||
// 模拟"用户更换了 embedding 模型":旧向量标记为旧模型
|
||||
db.prepare(
|
||||
"UPDATE semantic_memories SET embedding_model = 'old-model' WHERE key = 'fp-check'",
|
||||
).run();
|
||||
let reembedCalls = 0;
|
||||
mgr.setEmbedder({
|
||||
modelName: 'new-model',
|
||||
embed: async () => {
|
||||
reembedCalls += 1;
|
||||
return [0.1, 0.2, 0.3];
|
||||
},
|
||||
});
|
||||
const results = await mgr.search('指纹校验内容');
|
||||
// 检索本身可用(回退 TF-IDF);旧向量被排队用新模型重算
|
||||
expect(results.length).toBeGreaterThan(0);
|
||||
await new Promise((r) => setTimeout(r, 10));
|
||||
expect(reembedCalls).toBeGreaterThanOrEqual(1);
|
||||
const row = db
|
||||
.prepare("SELECT embedding, embedding_model FROM semantic_memories WHERE key = 'fp-check'")
|
||||
.get() as { embedding: Buffer | null; embedding_model: string | null };
|
||||
expect(row.embedding_model).toBe('new-model');
|
||||
expect(row.embedding).not.toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -16,4 +16,12 @@ export type MemoryEmbedFn = (text: string) => Promise<number[] | null>;
|
||||
|
||||
export interface MemoryEmbedder {
|
||||
embed: MemoryEmbedFn;
|
||||
/**
|
||||
* v0.8.2 P3-1: 嵌入模型标识(指纹)。
|
||||
* 用户更换 embedding 模型后,旧向量与新查询向量维度/语义空间不匹配 ——
|
||||
* 维度不同余弦静默为 0,同维不同模型产生噪声分数。Manager 以该标识标注
|
||||
* 每条向量(embedding_model 列),检索时模型不匹配的向量视为缺失并惰性重算。
|
||||
* 未提供时无法做指纹校验(旧向量一律接受 —— 保持旧行为兼容)。
|
||||
*/
|
||||
readonly modelName?: string;
|
||||
}
|
||||
|
||||
@@ -414,7 +414,12 @@ export class MemoryManager {
|
||||
source: row.source as MemorySource,
|
||||
sessionId: row.session_id ?? undefined,
|
||||
expiresAt: row.expires_at ?? undefined,
|
||||
docVec: blobToFloat32((row as { embedding?: unknown }).embedding),
|
||||
// v0.8.2 P3-1: 走模型指纹校验的解码(不匹配 → 视为缺失 + 惰性重算)
|
||||
docVec: this.decodeEmbeddingRow(
|
||||
(row as { embedding?: unknown }).embedding,
|
||||
(row as { embedding_model?: string | null }).embedding_model ?? null,
|
||||
row.content + ' ' + (row.summary ?? ''),
|
||||
),
|
||||
queryVec,
|
||||
},
|
||||
queryTF,
|
||||
@@ -428,6 +433,7 @@ export class MemoryManager {
|
||||
rows.map((r) => ({
|
||||
id: r.id,
|
||||
embedding: (r as { embedding?: unknown }).embedding,
|
||||
embedding_model: (r as { embedding_model?: string | null }).embedding_model ?? null,
|
||||
text: r.content + ' ' + (r.summary ?? ''),
|
||||
})),
|
||||
);
|
||||
@@ -478,6 +484,7 @@ export class MemoryManager {
|
||||
rows.map((r) => ({
|
||||
id: r.id,
|
||||
embedding: (r as { embedding?: unknown }).embedding,
|
||||
embedding_model: (r as { embedding_model?: string | null }).embedding_model ?? null,
|
||||
text: r.key + ' ' + r.value,
|
||||
})),
|
||||
);
|
||||
@@ -565,10 +572,24 @@ export class MemoryManager {
|
||||
// #32 修复: 当 summary 未提供时,使用 content hash 作为 key 实现基于内容的去重
|
||||
// v0.3.0 用 id 作为 key 时,因 id 每次新生成,INSERT OR REPLACE 永远不触发 REPLACE,
|
||||
// 导致重复 store 同一内容会创建多条记忆。改为 contentHash 后,相同内容自动 REPLACE。
|
||||
//
|
||||
// v0.8.2 P3-1 根治: INSERT OR REPLACE → ON CONFLICT DO UPDATE。
|
||||
// REPLACE 是"删旧行 + 插新行"—— 新 id 替换旧 id,**access_count 归零**:
|
||||
// 热门记忆被同内容更新后 LRU 排序权重凭空丢失。现以 key 为冲突目标做
|
||||
// 真正的 upsert:更新内容侧字段,保留 access_count 与旧 embedding
|
||||
//(同 key 意味着内容相同,旧向量仍然有效;模型更换由指纹校验自愈)。
|
||||
db.prepare(
|
||||
`
|
||||
INSERT OR REPLACE INTO semantic_memories (id, key, value, category, confidence, source_session, created_at, updated_at, access_count, tf_cache)
|
||||
INSERT INTO semantic_memories (id, key, value, category, confidence, source_session, created_at, updated_at, access_count, tf_cache)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0, ?)
|
||||
ON CONFLICT(key) DO UPDATE SET
|
||||
value = excluded.value,
|
||||
category = excluded.category,
|
||||
confidence = excluded.confidence,
|
||||
source_session = excluded.source_session,
|
||||
created_at = excluded.created_at,
|
||||
updated_at = excluded.updated_at,
|
||||
tf_cache = excluded.tf_cache
|
||||
`,
|
||||
).run(
|
||||
id,
|
||||
@@ -635,9 +656,10 @@ export class MemoryManager {
|
||||
.embed(text.slice(0, 8000))
|
||||
.then((vec) => {
|
||||
if (!vec || vec.length === 0) return;
|
||||
// v0.8.2 P3-1: 同步写入模型指纹(检索时按指纹校验,模型更换后惰性重算)
|
||||
this.getDB()
|
||||
.prepare(`UPDATE ${table} SET embedding = ? WHERE id = ?`)
|
||||
.run(float32ToBlob(vec), id);
|
||||
.prepare(`UPDATE ${table} SET embedding = ?, embedding_model = ? WHERE id = ?`)
|
||||
.run(float32ToBlob(vec), embedder.modelName ?? null, id);
|
||||
})
|
||||
.catch((err) => {
|
||||
log.debug(`MemoryManager: embedding enrichment skipped: ${(err as Error).message}`);
|
||||
@@ -647,6 +669,144 @@ export class MemoryManager {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.8.2 P3-1: BLOB 解码 + 模型指纹校验。
|
||||
* ① 字节/维度损坏 → null(回退 TF-IDF);② 当前嵌入器声明了模型指纹且行内
|
||||
* 指纹不匹配(更换过 embedding 模型 / 旧版本写入的无指纹行)→ 视为缺失并
|
||||
* 触发惰性重算 —— 旧实现跨模型余弦为 0(降级)或产生噪声分数,且无自愈路径。
|
||||
*/
|
||||
private decodeEmbeddingRow(
|
||||
blob: unknown,
|
||||
rowModel: string | null,
|
||||
_text: string,
|
||||
): number[] | null {
|
||||
const vec = blobToFloat32(blob);
|
||||
if (!vec) return null;
|
||||
const currentModel = this.embedder?.modelName;
|
||||
if (currentModel && (rowModel ?? null) !== currentModel) {
|
||||
// 指纹不匹配:本轮按"无向量"处理(回退 TF-IDF),同时排队用当前模型重算
|
||||
return null;
|
||||
}
|
||||
return vec;
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.8.2 P3-1: 向量路独立召回(与重要度预过滤解耦)。
|
||||
*
|
||||
* 旧实现混合检索的向量余弦只在 `ORDER BY importance DESC LIMIT topK*3` 的
|
||||
* 候选池内计算 —— 低重要度但语义高度相关的记忆永远进不了向量路,"同义改写
|
||||
* 召回"(P1-1 立项目标)被结构性钳制。现当查询向量可用时,从 embedding 命中
|
||||
* 的行中按时间取最近 VECTOR_RECALL_POOL 条独立召回(importance 过滤仍生效),
|
||||
* 以纯向量余弦 × 衰减 × 重要度权重评分,由 search() 与 TF-IDF 路合并去重。
|
||||
*/
|
||||
private static readonly VECTOR_RECALL_POOL = 200;
|
||||
|
||||
private vectorRecall(
|
||||
queryVec: number[],
|
||||
options: MemorySearchOptions,
|
||||
now: number,
|
||||
): SearchResult[] {
|
||||
const db = this.getDB();
|
||||
const { topK = 5, type, minImportance = 0 } = options;
|
||||
const pool = Math.max(topK * 10, MemoryManager.VECTOR_RECALL_POOL);
|
||||
const results: SearchResult[] = [];
|
||||
const importanceWeight = (importance: number): number => 0.5 + importance * 0.5;
|
||||
|
||||
if (!type || type === 'episodic') {
|
||||
const rows = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT id, session_id, content, summary, source, importance, created_at, expires_at, embedding, embedding_model
|
||||
FROM episodic_memories
|
||||
WHERE importance >= ? AND embedding IS NOT NULL
|
||||
ORDER BY created_at DESC LIMIT ?
|
||||
`,
|
||||
)
|
||||
.all(minImportance, pool) as Array<{
|
||||
id: string;
|
||||
session_id: string | null;
|
||||
content: string;
|
||||
summary: string | null;
|
||||
source: string;
|
||||
importance: number;
|
||||
created_at: number;
|
||||
expires_at: number | null;
|
||||
embedding: unknown;
|
||||
embedding_model: string | null;
|
||||
}>;
|
||||
for (const row of rows) {
|
||||
const docVec = this.decodeEmbeddingRow(row.embedding, row.embedding_model, row.content);
|
||||
if (!docVec) {
|
||||
// 指纹不匹配(有旧向量但不被当前模型接受)→ 惰性重算
|
||||
if (row.embedding != null) this.enrichEmbedding('episodic', row.id, row.content);
|
||||
continue;
|
||||
}
|
||||
const sim = cosineSimilarity(queryVec, docVec);
|
||||
if (sim <= 0) continue;
|
||||
results.push({
|
||||
id: row.id,
|
||||
type: 'episodic',
|
||||
content: row.content,
|
||||
summary: row.summary ?? undefined,
|
||||
source: row.source as MemorySource,
|
||||
importance: row.importance,
|
||||
sessionId: row.session_id ?? undefined,
|
||||
createdAt: row.created_at,
|
||||
expiresAt: row.expires_at ?? undefined,
|
||||
score: sim * timeDecayWeight(row.created_at, now) * importanceWeight(row.importance),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (!type || type === 'semantic') {
|
||||
const rows = db
|
||||
.prepare(
|
||||
`
|
||||
SELECT id, key, value, confidence, source_session, created_at, embedding, embedding_model
|
||||
FROM semantic_memories
|
||||
WHERE confidence >= ? AND embedding IS NOT NULL
|
||||
ORDER BY updated_at DESC LIMIT ?
|
||||
`,
|
||||
)
|
||||
.all(minImportance, pool) as Array<{
|
||||
id: string;
|
||||
key: string;
|
||||
value: string;
|
||||
confidence: number;
|
||||
source_session: string | null;
|
||||
created_at: number;
|
||||
embedding: unknown;
|
||||
embedding_model: string | null;
|
||||
}>;
|
||||
for (const row of rows) {
|
||||
const docVec = this.decodeEmbeddingRow(
|
||||
row.embedding,
|
||||
row.embedding_model,
|
||||
row.key + ' ' + row.value,
|
||||
);
|
||||
if (!docVec) {
|
||||
if (row.embedding != null)
|
||||
this.enrichEmbedding('semantic', row.id, row.key + ' ' + row.value);
|
||||
continue;
|
||||
}
|
||||
const sim = cosineSimilarity(queryVec, docVec);
|
||||
if (sim <= 0) continue;
|
||||
results.push({
|
||||
id: row.id,
|
||||
type: 'semantic',
|
||||
content: row.value,
|
||||
source: 'imported',
|
||||
importance: row.confidence,
|
||||
sessionId: row.source_session ?? undefined,
|
||||
createdAt: row.created_at,
|
||||
score: sim * timeDecayWeight(row.created_at, now) * importanceWeight(row.confidence),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return results.sort((a, b) => b.score - a.score).slice(0, topK);
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.8.1 P1-1: 存量记忆向量惰性回填 —— 嵌入功能开启前写入的记忆(embedding
|
||||
* IS NULL)在参与检索时排队补算:本轮查询仍走 TF-IDF,后续查询即可命中向量
|
||||
@@ -655,12 +815,20 @@ export class MemoryManager {
|
||||
*/
|
||||
private backfillMissingEmbeddings(
|
||||
type: MemoryType,
|
||||
rows: Array<{ id: string; embedding?: unknown; text: string }>,
|
||||
rows: Array<{ id: string; embedding?: unknown; embedding_model?: string | null; text: string }>,
|
||||
): void {
|
||||
if (!this.embedder) return;
|
||||
const currentModel = this.embedder.modelName;
|
||||
for (const row of rows) {
|
||||
if (row.embedding != null) continue;
|
||||
this.enrichEmbedding(type, row.id, row.text);
|
||||
if (row.embedding == null) {
|
||||
this.enrichEmbedding(type, row.id, row.text);
|
||||
continue;
|
||||
}
|
||||
// v0.8.2 P3-1: 指纹不匹配(更换过 embedding 模型 / 旧版本无指纹行)→
|
||||
// 用当前模型重算,旧向量在重算完成前不参与向量评分
|
||||
if (currentModel && (row.embedding_model ?? null) !== currentModel) {
|
||||
this.enrichEmbedding(type, row.id, row.text);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -716,10 +884,27 @@ export class MemoryManager {
|
||||
}
|
||||
|
||||
// v0.2.0: 优先使用语义搜索(TF-IDF ± 向量混合)
|
||||
const now = Date.now();
|
||||
const tfidfResults = this.tfidfSearch(query, options, queryVec);
|
||||
if (tfidfResults.length > 0) {
|
||||
this.bumpAccessCounts(tfidfResults);
|
||||
return tfidfResults;
|
||||
|
||||
// v0.8.2 P3-1: 向量路独立召回合并 —— 低重要度但语义相关的记忆不再被
|
||||
// importance 预过滤的候选池钳制(tfidfSearch 的向量分量只在重要度池内算)。
|
||||
let merged = tfidfResults;
|
||||
if (queryVec) {
|
||||
const vectorResults = this.vectorRecall(queryVec, options, now);
|
||||
if (vectorResults.length > 0) {
|
||||
const byId = new Map<string, SearchResult>();
|
||||
for (const r of [...tfidfResults, ...vectorResults]) {
|
||||
const prev = byId.get(r.id);
|
||||
if (!prev || r.score > prev.score) byId.set(r.id, r);
|
||||
}
|
||||
merged = [...byId.values()].sort((a, b) => b.score - a.score).slice(0, topK);
|
||||
}
|
||||
}
|
||||
|
||||
if (merged.length > 0) {
|
||||
this.bumpAccessCounts(merged);
|
||||
return merged;
|
||||
}
|
||||
|
||||
// 回退:如果 TF-IDF 没有结果(如 IDF 缓存为空),使用 LIKE 关键词搜索
|
||||
|
||||
Reference in New Issue
Block a user