fix: 去重文本规范化 — 全角标点→半角后比较,消除 AI 输出的标点差异
问题: AI 多次调用 memory add 创建了 5 条语义相同的 Gitee 信息 根因: 全角标点(:,)与半角(:,)被去重逻辑视为不同文本 修复: - 新增 normalizeForDedup(): 全角→半角标点、统一空白、小写 - addEntry 去重: 规范化后前缀匹配(30字) + 短文本子串包含 + 词相似度 - extractAndSaveMemories 去重: 同步使用相同的规范化逻辑
This commit is contained in:
@@ -440,12 +440,19 @@ export async function addEntry(
|
||||
entries = autoCleanEntries(entries);
|
||||
}
|
||||
|
||||
// 去重检查(前 50 字符相同视为重复)
|
||||
// 去重检查(规范化后比较:全角→半角、统一空白、小写)
|
||||
const normalizedNew = normalizeForDedup(content);
|
||||
const existing = entries.find(e => {
|
||||
if (e.type !== type) return false;
|
||||
const prefixLen = Math.min(50, content.length, e.content.length);
|
||||
if (prefixLen > 20 && content.slice(0, prefixLen) === e.content.slice(0, prefixLen)) return true;
|
||||
return simpleSimilarity(e.content, content) > 0.8;
|
||||
const normalizedExisting = normalizeForDedup(e.content);
|
||||
// 1) 规范化前缀匹配(30 字符)
|
||||
const prefixLen = Math.min(30, normalizedNew.length, normalizedExisting.length);
|
||||
if (prefixLen > 10 && normalizedNew.slice(0, prefixLen) === normalizedExisting.slice(0, prefixLen)) return true;
|
||||
// 2) 如果短文本,检查子串包含
|
||||
if (normalizedNew.length < 40 && normalizedExisting.includes(normalizedNew)) return true;
|
||||
if (normalizedExisting.length < 40 && normalizedNew.includes(normalizedExisting)) return true;
|
||||
// 3) 词级相似度
|
||||
return simpleSimilarity(normalizedExisting, normalizedNew) > 0.8;
|
||||
});
|
||||
if (existing) {
|
||||
logDebug('记忆去重: 已有相同条目', `${type}: ${content.slice(0, 40)}`);
|
||||
@@ -574,6 +581,34 @@ function simpleSimilarity(a: string, b: string): number {
|
||||
return union === 0 ? 0 : intersection / union;
|
||||
}
|
||||
|
||||
/**
|
||||
* 规范化文本用于去重比较:全角→半角标点、统一空白
|
||||
* 消除 AI 输出中常见的全角逗号/冒号/括号等差异
|
||||
*/
|
||||
function normalizeForDedup(text: string): string {
|
||||
const FULLWIDTH_MAP: Record<string, string> = {
|
||||
'\uff0c': ',', // ,
|
||||
'\uff1a': ':', // :
|
||||
'\uff1b': ';', // ;
|
||||
'\uff08': '(', // (
|
||||
'\uff09': ')', // )
|
||||
'\u300a': '<', // 《
|
||||
'\u300b': '>', // 》
|
||||
'\u201c': '"', // "
|
||||
'\u201d': '"', // "
|
||||
'\u2018': "'", // '
|
||||
'\u2019': "'", // '
|
||||
'\uff01': '!', // !
|
||||
'\uff0f': '/', // /
|
||||
'\u3001': ',', // 、
|
||||
};
|
||||
let result = '';
|
||||
for (const ch of text) {
|
||||
result += FULLWIDTH_MAP[ch] || ch;
|
||||
}
|
||||
return result.replace(/\s+/g, ' ').trim().toLowerCase();
|
||||
}
|
||||
|
||||
/**
|
||||
* 自动清理低价值条目(容量超限时)
|
||||
* rule 类型受保护不被清理
|
||||
@@ -736,12 +771,16 @@ ${conversationText.slice(0, 5000)}
|
||||
continue;
|
||||
}
|
||||
|
||||
// ── 过滤 5: 去重(与已有记忆比对)──
|
||||
// ── 过滤 5: 去重(规范化后比较)──
|
||||
const normalizedNew = normalizeForDedup(entry.content);
|
||||
const isDuplicate = existingEntries.some(e => {
|
||||
if (e.type !== entry.type) return false;
|
||||
const prefixLen = Math.min(40, entry.content.length, e.content.length);
|
||||
if (prefixLen > 15 && entry.content.slice(0, prefixLen) === e.content.slice(0, prefixLen)) return true;
|
||||
return simpleSimilarity(e.content, entry.content) > 0.75;
|
||||
const normalizedExisting = normalizeForDedup(e.content);
|
||||
const prefixLen = Math.min(30, normalizedNew.length, normalizedExisting.length);
|
||||
if (prefixLen > 10 && normalizedNew.slice(0, prefixLen) === normalizedExisting.slice(0, prefixLen)) return true;
|
||||
if (normalizedNew.length < 40 && normalizedExisting.includes(normalizedNew)) return true;
|
||||
if (normalizedExisting.length < 40 && normalizedNew.includes(normalizedExisting)) return true;
|
||||
return simpleSimilarity(normalizedExisting, normalizedNew) > 0.75;
|
||||
});
|
||||
if (isDuplicate) {
|
||||
logDebug('自动记忆过滤: 重复', entry.content.slice(0, 40));
|
||||
|
||||
Reference in New Issue
Block a user