fix: 去重文本规范化 — 全角标点→半角后比较,消除 AI 输出的标点差异

问题: AI 多次调用 memory add 创建了 5 条语义相同的 Gitee 信息
根因: 全角标点(:,)与半角(:,)被去重逻辑视为不同文本

修复:
- 新增 normalizeForDedup(): 全角→半角标点、统一空白、小写
- addEntry 去重: 规范化后前缀匹配(30字) + 短文本子串包含 + 词相似度
- extractAndSaveMemories 去重: 同步使用相同的规范化逻辑
This commit is contained in:
紫影233
2026-06-24 15:53:33 +08:00
parent 3564087fb4
commit 5a6a689382
+47 -8
View File
@@ -440,12 +440,19 @@ export async function addEntry(
entries = autoCleanEntries(entries); entries = autoCleanEntries(entries);
} }
// 去重检查(前 50 字符相同视为重复 // 去重检查(规范化后比较:全角→半角、统一空白、小写
const normalizedNew = normalizeForDedup(content);
const existing = entries.find(e => { const existing = entries.find(e => {
if (e.type !== type) return false; if (e.type !== type) return false;
const prefixLen = Math.min(50, content.length, e.content.length); const normalizedExisting = normalizeForDedup(e.content);
if (prefixLen > 20 && content.slice(0, prefixLen) === e.content.slice(0, prefixLen)) return true; // 1) 规范化前缀匹配(30 字符)
return simpleSimilarity(e.content, content) > 0.8; const prefixLen = Math.min(30, normalizedNew.length, normalizedExisting.length);
if (prefixLen > 10 && normalizedNew.slice(0, prefixLen) === normalizedExisting.slice(0, prefixLen)) return true;
// 2) 如果短文本,检查子串包含
if (normalizedNew.length < 40 && normalizedExisting.includes(normalizedNew)) return true;
if (normalizedExisting.length < 40 && normalizedNew.includes(normalizedExisting)) return true;
// 3) 词级相似度
return simpleSimilarity(normalizedExisting, normalizedNew) > 0.8;
}); });
if (existing) { if (existing) {
logDebug('记忆去重: 已有相同条目', `${type}: ${content.slice(0, 40)}`); logDebug('记忆去重: 已有相同条目', `${type}: ${content.slice(0, 40)}`);
@@ -574,6 +581,34 @@ function simpleSimilarity(a: string, b: string): number {
return union === 0 ? 0 : intersection / union; return union === 0 ? 0 : intersection / union;
} }
/**
* 规范化文本用于去重比较:全角→半角标点、统一空白
* 消除 AI 输出中常见的全角逗号/冒号/括号等差异
*/
function normalizeForDedup(text: string): string {
const FULLWIDTH_MAP: Record<string, string> = {
'\uff0c': ',', //
'\uff1a': ':', //
'\uff1b': ';', //
'\uff08': '(', //
'\uff09': ')', //
'\u300a': '<', // 《
'\u300b': '>', // 》
'\u201c': '"', // "
'\u201d': '"', // "
'\u2018': "'", // '
'\u2019': "'", // '
'\uff01': '!', //
'\uff0f': '/', //
'\u3001': ',', // 、
};
let result = '';
for (const ch of text) {
result += FULLWIDTH_MAP[ch] || ch;
}
return result.replace(/\s+/g, ' ').trim().toLowerCase();
}
/** /**
* 自动清理低价值条目(容量超限时) * 自动清理低价值条目(容量超限时)
* rule 类型受保护不被清理 * rule 类型受保护不被清理
@@ -736,12 +771,16 @@ ${conversationText.slice(0, 5000)}
continue; continue;
} }
// ── 过滤 5: 去重(与已有记忆比对)── // ── 过滤 5: 去重(规范化后比较)──
const normalizedNew = normalizeForDedup(entry.content);
const isDuplicate = existingEntries.some(e => { const isDuplicate = existingEntries.some(e => {
if (e.type !== entry.type) return false; if (e.type !== entry.type) return false;
const prefixLen = Math.min(40, entry.content.length, e.content.length); const normalizedExisting = normalizeForDedup(e.content);
if (prefixLen > 15 && entry.content.slice(0, prefixLen) === e.content.slice(0, prefixLen)) return true; const prefixLen = Math.min(30, normalizedNew.length, normalizedExisting.length);
return simpleSimilarity(e.content, entry.content) > 0.75; if (prefixLen > 10 && normalizedNew.slice(0, prefixLen) === normalizedExisting.slice(0, prefixLen)) return true;
if (normalizedNew.length < 40 && normalizedExisting.includes(normalizedNew)) return true;
if (normalizedExisting.length < 40 && normalizedNew.includes(normalizedExisting)) return true;
return simpleSimilarity(normalizedExisting, normalizedNew) > 0.75;
}); });
if (isDuplicate) { if (isDuplicate) {
logDebug('自动记忆过滤: 重复', entry.content.slice(0, 40)); logDebug('自动记忆过滤: 重复', entry.content.slice(0, 40));