fix: 去重文本规范化 — 全角标点→半角后比较,消除 AI 输出的标点差异
问题: AI 多次调用 memory add 创建了 5 条语义相同的 Gitee 信息 根因: 全角标点(:,)与半角(:,)被去重逻辑视为不同文本 修复: - 新增 normalizeForDedup(): 全角→半角标点、统一空白、小写 - addEntry 去重: 规范化后前缀匹配(30字) + 短文本子串包含 + 词相似度 - extractAndSaveMemories 去重: 同步使用相同的规范化逻辑
This commit is contained in:
@@ -440,12 +440,19 @@ export async function addEntry(
|
|||||||
entries = autoCleanEntries(entries);
|
entries = autoCleanEntries(entries);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 去重检查(前 50 字符相同视为重复)
|
// 去重检查(规范化后比较:全角→半角、统一空白、小写)
|
||||||
|
const normalizedNew = normalizeForDedup(content);
|
||||||
const existing = entries.find(e => {
|
const existing = entries.find(e => {
|
||||||
if (e.type !== type) return false;
|
if (e.type !== type) return false;
|
||||||
const prefixLen = Math.min(50, content.length, e.content.length);
|
const normalizedExisting = normalizeForDedup(e.content);
|
||||||
if (prefixLen > 20 && content.slice(0, prefixLen) === e.content.slice(0, prefixLen)) return true;
|
// 1) 规范化前缀匹配(30 字符)
|
||||||
return simpleSimilarity(e.content, content) > 0.8;
|
const prefixLen = Math.min(30, normalizedNew.length, normalizedExisting.length);
|
||||||
|
if (prefixLen > 10 && normalizedNew.slice(0, prefixLen) === normalizedExisting.slice(0, prefixLen)) return true;
|
||||||
|
// 2) 如果短文本,检查子串包含
|
||||||
|
if (normalizedNew.length < 40 && normalizedExisting.includes(normalizedNew)) return true;
|
||||||
|
if (normalizedExisting.length < 40 && normalizedNew.includes(normalizedExisting)) return true;
|
||||||
|
// 3) 词级相似度
|
||||||
|
return simpleSimilarity(normalizedExisting, normalizedNew) > 0.8;
|
||||||
});
|
});
|
||||||
if (existing) {
|
if (existing) {
|
||||||
logDebug('记忆去重: 已有相同条目', `${type}: ${content.slice(0, 40)}`);
|
logDebug('记忆去重: 已有相同条目', `${type}: ${content.slice(0, 40)}`);
|
||||||
@@ -574,6 +581,34 @@ function simpleSimilarity(a: string, b: string): number {
|
|||||||
return union === 0 ? 0 : intersection / union;
|
return union === 0 ? 0 : intersection / union;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 规范化文本用于去重比较:全角→半角标点、统一空白
|
||||||
|
* 消除 AI 输出中常见的全角逗号/冒号/括号等差异
|
||||||
|
*/
|
||||||
|
function normalizeForDedup(text: string): string {
|
||||||
|
const FULLWIDTH_MAP: Record<string, string> = {
|
||||||
|
'\uff0c': ',', // ,
|
||||||
|
'\uff1a': ':', // :
|
||||||
|
'\uff1b': ';', // ;
|
||||||
|
'\uff08': '(', // (
|
||||||
|
'\uff09': ')', // )
|
||||||
|
'\u300a': '<', // 《
|
||||||
|
'\u300b': '>', // 》
|
||||||
|
'\u201c': '"', // "
|
||||||
|
'\u201d': '"', // "
|
||||||
|
'\u2018': "'", // '
|
||||||
|
'\u2019': "'", // '
|
||||||
|
'\uff01': '!', // !
|
||||||
|
'\uff0f': '/', // /
|
||||||
|
'\u3001': ',', // 、
|
||||||
|
};
|
||||||
|
let result = '';
|
||||||
|
for (const ch of text) {
|
||||||
|
result += FULLWIDTH_MAP[ch] || ch;
|
||||||
|
}
|
||||||
|
return result.replace(/\s+/g, ' ').trim().toLowerCase();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 自动清理低价值条目(容量超限时)
|
* 自动清理低价值条目(容量超限时)
|
||||||
* rule 类型受保护不被清理
|
* rule 类型受保护不被清理
|
||||||
@@ -736,12 +771,16 @@ ${conversationText.slice(0, 5000)}
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── 过滤 5: 去重(与已有记忆比对)──
|
// ── 过滤 5: 去重(规范化后比较)──
|
||||||
|
const normalizedNew = normalizeForDedup(entry.content);
|
||||||
const isDuplicate = existingEntries.some(e => {
|
const isDuplicate = existingEntries.some(e => {
|
||||||
if (e.type !== entry.type) return false;
|
if (e.type !== entry.type) return false;
|
||||||
const prefixLen = Math.min(40, entry.content.length, e.content.length);
|
const normalizedExisting = normalizeForDedup(e.content);
|
||||||
if (prefixLen > 15 && entry.content.slice(0, prefixLen) === e.content.slice(0, prefixLen)) return true;
|
const prefixLen = Math.min(30, normalizedNew.length, normalizedExisting.length);
|
||||||
return simpleSimilarity(e.content, entry.content) > 0.75;
|
if (prefixLen > 10 && normalizedNew.slice(0, prefixLen) === normalizedExisting.slice(0, prefixLen)) return true;
|
||||||
|
if (normalizedNew.length < 40 && normalizedExisting.includes(normalizedNew)) return true;
|
||||||
|
if (normalizedExisting.length < 40 && normalizedNew.includes(normalizedExisting)) return true;
|
||||||
|
return simpleSimilarity(normalizedExisting, normalizedNew) > 0.75;
|
||||||
});
|
});
|
||||||
if (isDuplicate) {
|
if (isDuplicate) {
|
||||||
logDebug('自动记忆过滤: 重复', entry.content.slice(0, 40));
|
logDebug('自动记忆过滤: 重复', entry.content.slice(0, 40));
|
||||||
|
|||||||
Reference in New Issue
Block a user