feat: TypeScript + Electron v2 重构 - 纯桌面版
- 全面迁移到 TypeScript,严格类型定义 - 放弃 Web 版,专注 Electron 桌面应用 - 主进程模块化:main.ts, preload.ts, menu.ts, tray.ts, ipc.ts, utils.ts - 渲染进程完整迁移所有功能组件 - 删除 PWA 相关文件 (sw.js, manifest.json) - 删除 Web 版降级逻辑 - 保留所有核心功能:流式对话、多模型、Think推理、多模态、RAG知识库、Agent预设、历史管理 - 保留 Windows 11 Fluent Design 暗色主题样式
This commit is contained in:
@@ -0,0 +1,74 @@
|
||||
/**
|
||||
* DocumentProcessor - 文档分块处理
|
||||
*/
|
||||
|
||||
export function chunkText(text: string, { maxChunkSize = 1500, overlap = 200 } = {}): string[] {
|
||||
if (!text || text.trim().length === 0) return [];
|
||||
|
||||
const paragraphs = text.split(/\n\s*\n/).filter(p => p.trim());
|
||||
const chunks: string[] = [];
|
||||
let current = '';
|
||||
|
||||
for (const para of paragraphs) {
|
||||
const trimmed = para.trim();
|
||||
if (!trimmed) continue;
|
||||
|
||||
if (trimmed.length > maxChunkSize) {
|
||||
if (current) {
|
||||
chunks.push(current.trim());
|
||||
current = '';
|
||||
}
|
||||
const sentences = splitSentences(trimmed);
|
||||
let sentenceBuf = '';
|
||||
for (const sent of sentences) {
|
||||
if ((sentenceBuf + sent).length > maxChunkSize && sentenceBuf) {
|
||||
chunks.push(sentenceBuf.trim());
|
||||
sentenceBuf = overlap > 0 ? sentenceBuf.slice(-overlap) + sent : sent;
|
||||
} else {
|
||||
sentenceBuf += sent;
|
||||
}
|
||||
}
|
||||
if (sentenceBuf.trim()) {
|
||||
current = sentenceBuf;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if ((current + '\n\n' + trimmed).length > maxChunkSize && current) {
|
||||
chunks.push(current.trim());
|
||||
if (overlap > 0 && current.length > overlap) {
|
||||
current = current.slice(-overlap) + '\n\n' + trimmed;
|
||||
} else {
|
||||
current = trimmed;
|
||||
}
|
||||
} else {
|
||||
current = current ? current + '\n\n' + trimmed : trimmed;
|
||||
}
|
||||
}
|
||||
|
||||
if (current.trim()) {
|
||||
chunks.push(current.trim());
|
||||
}
|
||||
|
||||
return chunks;
|
||||
}
|
||||
|
||||
function splitSentences(text: string): string[] {
|
||||
const parts = text.split(/(?<=[。!?.!?])\s*/);
|
||||
return parts.filter(p => p.trim()).map(p => p.trim() + ' ');
|
||||
}
|
||||
|
||||
export function extractDocument(content: string, filename: string): { text: string; filename: string } {
|
||||
return { text: content, filename };
|
||||
}
|
||||
|
||||
export function createChunkMetadata(chunk: string, index: number, docId: string, filename: string) {
|
||||
return {
|
||||
id: `${docId}_chunk_${index}`,
|
||||
docId,
|
||||
filename,
|
||||
chunkIndex: index,
|
||||
text: chunk,
|
||||
charCount: chunk.length
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user