feat: v0.8.2 安全纵深补全 · 协议保真 · 断链修复 — 图片SSRF/根MEMORY.md保护根治 · Anthropic thinking回传+pause_turn续传 · 2523 用例全量回归 + E2E 扩充
CI / 类型检查 + Lint + 单元测试 (push) Failing after 9m45s
CI / 全量测试 (Electron ABI) (push) Failing after 6m28s
CI / 产物编译验证 (push) Successful in 11m18s

This commit is contained in:
2026-09-08 14:30:27 +08:00
parent 69776e447f
commit 4cd6e997b5
86 changed files with 4303 additions and 956 deletions
@@ -31,7 +31,8 @@ try {
import { MemoryManager } from '../manager';
// 与 DatabaseService.createTables 一致的记忆三表 schema(含 tf_cache 列)
// 与 DatabaseService.createTables 一致的记忆三表 schema(含 tf_cache / embedding /
// embedding_model 列 —— v0.8.2 P3-1 迁移 14 对齐)
function createMemorySchema(db: any): void {
db.exec(`
CREATE TABLE episodic_memories (
@@ -44,7 +45,8 @@ function createMemorySchema(db: any): void {
created_at INTEGER NOT NULL DEFAULT 0,
expires_at INTEGER,
tf_cache TEXT,
embedding BLOB
embedding BLOB,
embedding_model TEXT
);
CREATE TABLE semantic_memories (
id TEXT PRIMARY KEY,
@@ -57,7 +59,8 @@ function createMemorySchema(db: any): void {
updated_at INTEGER NOT NULL DEFAULT 0,
access_count INTEGER DEFAULT 0,
tf_cache TEXT,
embedding BLOB
embedding BLOB,
embedding_model TEXT
);
CREATE TABLE working_memories (
id TEXT PRIMARY KEY,
@@ -783,9 +786,9 @@ describe.skipIf(!dbAvailable)('MemoryManager — cleanupExpired', () => {
}
});
// 注意:manager.store() 的 episodic INSERT 不含 expires_at 列(源码已知缺口,
// main.ts 注释亦确认"expires_at 无写入方")—— 本组用例直接经 SQL 写入
// expires_at 模拟真实过期行,锁定 cleanupExpired 自身的删除契约。
// v0.8.2 P3-6 注释修正:v0.8.1 P0-2 已让 store() 真实写入 expires_at(本组
// 早期版本注释"INSERT 不含 expires_at 列"已过时)。此处直接经 SQL 写入仅为
// 测试夹具便利(绕过 store 的哈希/分词管线),锁定 cleanupExpired 的删除契约。
const insertExpiring = (id: string, expiresAt: number, content = '带过期记忆'): void => {
db.prepare(
`INSERT INTO episodic_memories (id, content, source, importance, created_at, expires_at)
@@ -984,3 +987,73 @@ describe.skipIf(!dbAvailable)('MemoryManager — v0.8.1 生命周期与混合检
expect(row.embedding!.length % 4).toBe(0);
});
});
// ===== v0.8.2 P3-1: access_count 保留 + 模型指纹 =====
describe.skipIf(!dbAvailable)('MemoryManager — v0.8.2 P3-1', () => {
let db: InstanceType<typeof Database>;
let mgr: MemoryManager;
beforeEach(() => {
db = new Database(':memory:');
createMemorySchema(db);
mgr = new MemoryManager(() => db);
});
it('semantic 同内容重复 store → ON CONFLICT upsertaccess_count 不归零', () => {
mgr.store({
type: 'semantic',
content: '用户偏好深色主题',
source: 'user_input',
importance: 0.9,
});
db.prepare('UPDATE semantic_memories SET access_count = 7').run();
// 同内容重复写入(key = contentHash,冲突命中同一行)
mgr.store({
type: 'semantic',
content: '用户偏好深色主题',
source: 'user_input',
importance: 0.9,
});
const rows = db.prepare('SELECT access_count, value FROM semantic_memories').all() as Array<{
access_count: number;
value: string;
}>;
// 旧 REPLACE 语义会删除重插 → 2 行且 access_count 归零
expect(rows).toHaveLength(1);
expect(rows[0].access_count).toBe(7);
});
it('embedding_model 指纹:模型不匹配的向量按缺失处理并触发重算', async () => {
mgr.store({
type: 'semantic',
content: '指纹校验内容',
summary: 'fp-check',
source: 'agent_thought',
importance: 0.8,
});
await new Promise((r) => setTimeout(r, 10));
// 模拟"用户更换了 embedding 模型":旧向量标记为旧模型
db.prepare(
"UPDATE semantic_memories SET embedding_model = 'old-model' WHERE key = 'fp-check'",
).run();
let reembedCalls = 0;
mgr.setEmbedder({
modelName: 'new-model',
embed: async () => {
reembedCalls += 1;
return [0.1, 0.2, 0.3];
},
});
const results = await mgr.search('指纹校验内容');
// 检索本身可用(回退 TF-IDF);旧向量被排队用新模型重算
expect(results.length).toBeGreaterThan(0);
await new Promise((r) => setTimeout(r, 10));
expect(reembedCalls).toBeGreaterThanOrEqual(1);
const row = db
.prepare("SELECT embedding, embedding_model FROM semantic_memories WHERE key = 'fp-check'")
.get() as { embedding: Buffer | null; embedding_model: string | null };
expect(row.embedding_model).toBe('new-model');
expect(row.embedding).not.toBeNull();
});
});
+8
View File
@@ -16,4 +16,12 @@ export type MemoryEmbedFn = (text: string) => Promise<number[] | null>;
export interface MemoryEmbedder {
embed: MemoryEmbedFn;
/**
* v0.8.2 P3-1: 嵌入模型标识(指纹)。
* 用户更换 embedding 模型后,旧向量与新查询向量维度/语义空间不匹配 ——
* 维度不同余弦静默为 0,同维不同模型产生噪声分数。Manager 以该标识标注
* 每条向量(embedding_model 列),检索时模型不匹配的向量视为缺失并惰性重算。
* 未提供时无法做指纹校验(旧向量一律接受 —— 保持旧行为兼容)。
*/
readonly modelName?: string;
}
+195 -10
View File
@@ -414,7 +414,12 @@ export class MemoryManager {
source: row.source as MemorySource,
sessionId: row.session_id ?? undefined,
expiresAt: row.expires_at ?? undefined,
docVec: blobToFloat32((row as { embedding?: unknown }).embedding),
// v0.8.2 P3-1: 走模型指纹校验的解码(不匹配 → 视为缺失 + 惰性重算)
docVec: this.decodeEmbeddingRow(
(row as { embedding?: unknown }).embedding,
(row as { embedding_model?: string | null }).embedding_model ?? null,
row.content + ' ' + (row.summary ?? ''),
),
queryVec,
},
queryTF,
@@ -428,6 +433,7 @@ export class MemoryManager {
rows.map((r) => ({
id: r.id,
embedding: (r as { embedding?: unknown }).embedding,
embedding_model: (r as { embedding_model?: string | null }).embedding_model ?? null,
text: r.content + ' ' + (r.summary ?? ''),
})),
);
@@ -478,6 +484,7 @@ export class MemoryManager {
rows.map((r) => ({
id: r.id,
embedding: (r as { embedding?: unknown }).embedding,
embedding_model: (r as { embedding_model?: string | null }).embedding_model ?? null,
text: r.key + ' ' + r.value,
})),
);
@@ -565,10 +572,24 @@ export class MemoryManager {
// #32 修复: 当 summary 未提供时,使用 content hash 作为 key 实现基于内容的去重
// v0.3.0 用 id 作为 key 时,因 id 每次新生成,INSERT OR REPLACE 永远不触发 REPLACE
// 导致重复 store 同一内容会创建多条记忆。改为 contentHash 后,相同内容自动 REPLACE。
//
// v0.8.2 P3-1 根治: INSERT OR REPLACE → ON CONFLICT DO UPDATE。
// REPLACE 是"删旧行 + 插新行"—— 新 id 替换旧 id**access_count 归零**
// 热门记忆被同内容更新后 LRU 排序权重凭空丢失。现以 key 为冲突目标做
// 真正的 upsert:更新内容侧字段,保留 access_count 与旧 embedding
//(同 key 意味着内容相同,旧向量仍然有效;模型更换由指纹校验自愈)。
db.prepare(
`
INSERT OR REPLACE INTO semantic_memories (id, key, value, category, confidence, source_session, created_at, updated_at, access_count, tf_cache)
INSERT INTO semantic_memories (id, key, value, category, confidence, source_session, created_at, updated_at, access_count, tf_cache)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0, ?)
ON CONFLICT(key) DO UPDATE SET
value = excluded.value,
category = excluded.category,
confidence = excluded.confidence,
source_session = excluded.source_session,
created_at = excluded.created_at,
updated_at = excluded.updated_at,
tf_cache = excluded.tf_cache
`,
).run(
id,
@@ -635,9 +656,10 @@ export class MemoryManager {
.embed(text.slice(0, 8000))
.then((vec) => {
if (!vec || vec.length === 0) return;
// v0.8.2 P3-1: 同步写入模型指纹(检索时按指纹校验,模型更换后惰性重算)
this.getDB()
.prepare(`UPDATE ${table} SET embedding = ? WHERE id = ?`)
.run(float32ToBlob(vec), id);
.prepare(`UPDATE ${table} SET embedding = ?, embedding_model = ? WHERE id = ?`)
.run(float32ToBlob(vec), embedder.modelName ?? null, id);
})
.catch((err) => {
log.debug(`MemoryManager: embedding enrichment skipped: ${(err as Error).message}`);
@@ -647,6 +669,144 @@ export class MemoryManager {
});
}
/**
* v0.8.2 P3-1: BLOB 解码 + 模型指纹校验。
* ① 字节/维度损坏 → null(回退 TF-IDF);② 当前嵌入器声明了模型指纹且行内
* 指纹不匹配(更换过 embedding 模型 / 旧版本写入的无指纹行)→ 视为缺失并
* 触发惰性重算 —— 旧实现跨模型余弦为 0(降级)或产生噪声分数,且无自愈路径。
*/
private decodeEmbeddingRow(
blob: unknown,
rowModel: string | null,
_text: string,
): number[] | null {
const vec = blobToFloat32(blob);
if (!vec) return null;
const currentModel = this.embedder?.modelName;
if (currentModel && (rowModel ?? null) !== currentModel) {
// 指纹不匹配:本轮按"无向量"处理(回退 TF-IDF),同时排队用当前模型重算
return null;
}
return vec;
}
/**
* v0.8.2 P3-1: 向量路独立召回(与重要度预过滤解耦)。
*
* 旧实现混合检索的向量余弦只在 `ORDER BY importance DESC LIMIT topK*3` 的
* 候选池内计算 —— 低重要度但语义高度相关的记忆永远进不了向量路,"同义改写
* 召回"(P1-1 立项目标)被结构性钳制。现当查询向量可用时,从 embedding 命中
* 的行中按时间取最近 VECTOR_RECALL_POOL 条独立召回(importance 过滤仍生效),
* 以纯向量余弦 × 衰减 × 重要度权重评分,由 search() 与 TF-IDF 路合并去重。
*/
private static readonly VECTOR_RECALL_POOL = 200;
private vectorRecall(
queryVec: number[],
options: MemorySearchOptions,
now: number,
): SearchResult[] {
const db = this.getDB();
const { topK = 5, type, minImportance = 0 } = options;
const pool = Math.max(topK * 10, MemoryManager.VECTOR_RECALL_POOL);
const results: SearchResult[] = [];
const importanceWeight = (importance: number): number => 0.5 + importance * 0.5;
if (!type || type === 'episodic') {
const rows = db
.prepare(
`
SELECT id, session_id, content, summary, source, importance, created_at, expires_at, embedding, embedding_model
FROM episodic_memories
WHERE importance >= ? AND embedding IS NOT NULL
ORDER BY created_at DESC LIMIT ?
`,
)
.all(minImportance, pool) as Array<{
id: string;
session_id: string | null;
content: string;
summary: string | null;
source: string;
importance: number;
created_at: number;
expires_at: number | null;
embedding: unknown;
embedding_model: string | null;
}>;
for (const row of rows) {
const docVec = this.decodeEmbeddingRow(row.embedding, row.embedding_model, row.content);
if (!docVec) {
// 指纹不匹配(有旧向量但不被当前模型接受)→ 惰性重算
if (row.embedding != null) this.enrichEmbedding('episodic', row.id, row.content);
continue;
}
const sim = cosineSimilarity(queryVec, docVec);
if (sim <= 0) continue;
results.push({
id: row.id,
type: 'episodic',
content: row.content,
summary: row.summary ?? undefined,
source: row.source as MemorySource,
importance: row.importance,
sessionId: row.session_id ?? undefined,
createdAt: row.created_at,
expiresAt: row.expires_at ?? undefined,
score: sim * timeDecayWeight(row.created_at, now) * importanceWeight(row.importance),
});
}
}
if (!type || type === 'semantic') {
const rows = db
.prepare(
`
SELECT id, key, value, confidence, source_session, created_at, embedding, embedding_model
FROM semantic_memories
WHERE confidence >= ? AND embedding IS NOT NULL
ORDER BY updated_at DESC LIMIT ?
`,
)
.all(minImportance, pool) as Array<{
id: string;
key: string;
value: string;
confidence: number;
source_session: string | null;
created_at: number;
embedding: unknown;
embedding_model: string | null;
}>;
for (const row of rows) {
const docVec = this.decodeEmbeddingRow(
row.embedding,
row.embedding_model,
row.key + ' ' + row.value,
);
if (!docVec) {
if (row.embedding != null)
this.enrichEmbedding('semantic', row.id, row.key + ' ' + row.value);
continue;
}
const sim = cosineSimilarity(queryVec, docVec);
if (sim <= 0) continue;
results.push({
id: row.id,
type: 'semantic',
content: row.value,
source: 'imported',
importance: row.confidence,
sessionId: row.source_session ?? undefined,
createdAt: row.created_at,
score: sim * timeDecayWeight(row.created_at, now) * importanceWeight(row.confidence),
});
}
}
return results.sort((a, b) => b.score - a.score).slice(0, topK);
}
/**
* v0.8.1 P1-1: 存量记忆向量惰性回填 —— 嵌入功能开启前写入的记忆(embedding
* IS NULL)在参与检索时排队补算:本轮查询仍走 TF-IDF,后续查询即可命中向量
@@ -655,12 +815,20 @@ export class MemoryManager {
*/
private backfillMissingEmbeddings(
type: MemoryType,
rows: Array<{ id: string; embedding?: unknown; text: string }>,
rows: Array<{ id: string; embedding?: unknown; embedding_model?: string | null; text: string }>,
): void {
if (!this.embedder) return;
const currentModel = this.embedder.modelName;
for (const row of rows) {
if (row.embedding != null) continue;
this.enrichEmbedding(type, row.id, row.text);
if (row.embedding == null) {
this.enrichEmbedding(type, row.id, row.text);
continue;
}
// v0.8.2 P3-1: 指纹不匹配(更换过 embedding 模型 / 旧版本无指纹行)→
// 用当前模型重算,旧向量在重算完成前不参与向量评分
if (currentModel && (row.embedding_model ?? null) !== currentModel) {
this.enrichEmbedding(type, row.id, row.text);
}
}
}
@@ -716,10 +884,27 @@ export class MemoryManager {
}
// v0.2.0: 优先使用语义搜索(TF-IDF ± 向量混合)
const now = Date.now();
const tfidfResults = this.tfidfSearch(query, options, queryVec);
if (tfidfResults.length > 0) {
this.bumpAccessCounts(tfidfResults);
return tfidfResults;
// v0.8.2 P3-1: 向量路独立召回合并 —— 低重要度但语义相关的记忆不再被
// importance 预过滤的候选池钳制(tfidfSearch 的向量分量只在重要度池内算)。
let merged = tfidfResults;
if (queryVec) {
const vectorResults = this.vectorRecall(queryVec, options, now);
if (vectorResults.length > 0) {
const byId = new Map<string, SearchResult>();
for (const r of [...tfidfResults, ...vectorResults]) {
const prev = byId.get(r.id);
if (!prev || r.score > prev.score) byId.set(r.id, r);
}
merged = [...byId.values()].sort((a, b) => b.score - a.score).slice(0, topK);
}
}
if (merged.length > 0) {
this.bumpAccessCounts(merged);
return merged;
}
// 回退:如果 TF-IDF 没有结果(如 IDF 缓存为空),使用 LIKE 关键词搜索