feat: v0.5.4 多模态增强 — 多轮图片记忆 + 多模态总开关 + DeepSeek vision 模型支持
多轮图片记忆:
- 此前历史轮次的图片不回传 LLM(attachments 仅存压缩 preview,历史组装
时被丢弃)— 跨轮对话中模型对图片内容"失忆"
- 修复:SessionSummaryService.buildHistoryMessages 从持久化的 attachments
恢复 images(type=image 的 preview base64),历史图片随上下文回传
- Token 控制:最多注入最近 10 张(MAX_HISTORY_IMAGES,从最新消息向前
收集)— 每张 1024px 压缩图约数百至千余 token,无上限会吃满上下文
- 摘要区间(summarizedUntilRowid 之前)的图片不恢复,符合滚动摘要语义
多模态总开关 llm.multimodalEnabled(默认关闭):
- 新配置项:CONFIG_DEFAULTS 种子 + 设置弹框 LLM 配置 Switch +
首次引导向导 LLM 步骤 Switch(含说明文案)
- 上传入口双重判断:总开关 × 模型能力 — 未开启时即使模型支持多模态
也不能上传图片(ChatInput 的选择/拖拽/粘贴统一拦截,Toast 区分
"开关未开启"与"当前模型不支持"两种原因)
- 保存成功后同步 Agent Store 立即生效;App 启动时随 setProvider 加载
DeepSeek vision 模型支持:
- 新增 deepseek-v4-flash-vision-exp(OpenAI image_url content parts 格式,
128K 上下文 / 8K 输出)
- adapter 按 isVisionModel() 判断:vision 模型将带 images 的消息转换为
[{type:'text'},{type:'image_url'}] parts;非 vision 模型保持 images
静默丢弃(防 API 400)
测试(236 → 243 用例):
- 多轮图片记忆 ×3(session-summary.test.ts):历史 attachments 恢复
images / 上限 10 张从最新向前 / 摘要区间图片不恢复
- DeepSeek vision 请求格式 ×4(deepseek-vision.test.ts,契约级 mock
fetch 断言请求体):image_url parts 转换 / 非 vision 模型丢弃 /
max_tokens 钳制 8192 / 无图不转换
- 测试顺序修正:多轮图片用例置于 describe 末尾(插入新行消耗全局自增
rowid,插在中间会破坏既有用例对 rowid 数值的断言)
文档: README 同步(DeepSeek 模型表 + vision 多模态列、llm.multimodalEnabled
配置项、多轮图片记忆特性行、243 用例数)
验证: lint 0 / typecheck 双工程 0 / test:electron 243 全过 / build 成功
This commit is contained in:
@@ -35,6 +35,8 @@ const SUMMARY_TIMEOUT_MS = 30_000;
|
||||
const PER_MESSAGE_TRUNCATE = 600;
|
||||
/** 传给 LLM 的总字符上限 */
|
||||
const MAX_DIGEST_CHARS = 24_000;
|
||||
/** v0.5.4: 多轮图片记忆 — 历史上下文注入的最大图片数(从最新向前收集,防 token 爆炸) */
|
||||
const MAX_HISTORY_IMAGES = 10;
|
||||
|
||||
export class SessionSummaryService {
|
||||
constructor(
|
||||
@@ -62,10 +64,18 @@ export class SessionSummaryService {
|
||||
reasoningContent: m.reasoningContent,
|
||||
toolCalls: m.toolCalls as MetonaMessage['toolCalls'],
|
||||
toolResult: m.toolResult as MetonaMessage['toolResult'],
|
||||
// v0.5.4: 保留 attachments 供 restoreHistoryImages 恢复图片(多轮图片记忆)
|
||||
attachments: (m as { attachments?: unknown[] }).attachments,
|
||||
timestamp: m.timestamp,
|
||||
iteration: m.iteration,
|
||||
}));
|
||||
|
||||
// v0.5.4: 多轮图片记忆 — 从持久化的 attachments(压缩 base64 preview)恢复 images,
|
||||
// 历史轮次的图片重新注入 LLM 上下文(此前仅发送当轮可见,跨轮即"失忆")。
|
||||
// 数量上限防 token 爆炸:只取最近 MAX_HISTORY_IMAGES 张(从最新消息向前收集)。
|
||||
// 摘要区间(summarizedUntilRowid 之前)的图片无法恢复 — 符合滚动摘要的语义。
|
||||
this.restoreHistoryImages(messages);
|
||||
|
||||
if (existing && messages.length > 0) {
|
||||
// 摘要以 assistant 角色注入(与 engine 运行时压缩的注入策略一致)
|
||||
const summaryMessage: MetonaMessage = {
|
||||
@@ -78,6 +88,38 @@ export class SessionSummaryService {
|
||||
return messages;
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.5.4: 恢复历史消息的 images(多轮图片记忆)
|
||||
*
|
||||
* attachments 中 type=image 的 preview(1024px JPEG 压缩 base64)在消息
|
||||
* 持久化时已保存(与编辑重发的恢复逻辑同源)。此处将其映射回
|
||||
* MetonaMessage.images,让历史轮次图片随上下文回传 LLM。
|
||||
*
|
||||
* 上限策略:从最新消息向前收集,最多 MAX_HISTORY_IMAGES 张 —
|
||||
* 每张 1024px 图约数百至千余 token,无上限的长会话会迅速吃满上下文。
|
||||
*/
|
||||
private restoreHistoryImages(messages: MetonaMessage[]): void {
|
||||
let remaining = MAX_HISTORY_IMAGES;
|
||||
for (let i = messages.length - 1; i >= 0 && remaining > 0; i--) {
|
||||
const raw = messages[i] as MetonaMessage & {
|
||||
attachments?: Array<{ type?: string; preview?: string }>;
|
||||
};
|
||||
const imageAttachments = (raw.attachments ?? []).filter(
|
||||
(a) => a.type === 'image' && typeof a.preview === 'string' && a.preview.length > 0,
|
||||
);
|
||||
if (imageAttachments.length === 0) continue;
|
||||
|
||||
const take = Math.min(imageAttachments.length, remaining);
|
||||
// 优先保留靠后的图片(时间更近)
|
||||
const picked = imageAttachments.slice(-take);
|
||||
messages[i].images = picked.map((a) => ({
|
||||
url: a.preview as string,
|
||||
detail: 'auto' as const,
|
||||
}));
|
||||
remaining -= take;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 会话结束后评估并生成滚动摘要(fire-and-forget 调用,失败仅记录日志)
|
||||
*/
|
||||
@@ -117,14 +159,16 @@ export class SessionSummaryService {
|
||||
|
||||
saveSummary(sessionId: string, summary: string, untilRowid: number): void {
|
||||
const db = this.getDB();
|
||||
db.prepare(`
|
||||
db.prepare(
|
||||
`
|
||||
INSERT INTO session_summaries (session_id, summary, summarized_until_rowid, updated_at)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(session_id) DO UPDATE SET
|
||||
summary = excluded.summary,
|
||||
summarized_until_rowid = excluded.summarized_until_rowid,
|
||||
updated_at = excluded.updated_at
|
||||
`).run(sessionId, summary, untilRowid, Date.now());
|
||||
`,
|
||||
).run(sessionId, summary, untilRowid, Date.now());
|
||||
}
|
||||
|
||||
// ===== LLM 摘要 =====
|
||||
@@ -167,7 +211,8 @@ export class SessionSummaryService {
|
||||
'If a prior summary exists, merge it with the new content into one updated summary. ' +
|
||||
'Preserve key facts, decisions, tool outcomes, file paths, and open questions needed for future reasoning. ' +
|
||||
'Output in the same language as the conversation. Maximum 400 words. Output ONLY the summary text.',
|
||||
safetyGuidelines: 'Do not include sensitive data like passwords or API keys in the summary.',
|
||||
safetyGuidelines:
|
||||
'Do not include sensitive data like passwords or API keys in the summary.',
|
||||
},
|
||||
messages: [
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user