feat: v0.5.4 多模态增强 — 多轮图片记忆 + 多模态总开关 + DeepSeek vision 模型支持
CI / 类型检查 + Lint + 单元测试 (push) Failing after 5m47s
CI / 全量测试 (Electron ABI) (push) Failing after 5m25s
CI / 产物编译验证 (push) Successful in 10m4s

多轮图片记忆:
- 此前历史轮次的图片不回传 LLM(attachments 仅存压缩 preview,历史组装
  时被丢弃)— 跨轮对话中模型对图片内容"失忆"
- 修复:SessionSummaryService.buildHistoryMessages 从持久化的 attachments
  恢复 images(type=image 的 preview base64),历史图片随上下文回传
- Token 控制:最多注入最近 10 张(MAX_HISTORY_IMAGES,从最新消息向前
  收集)— 每张 1024px 压缩图约数百至千余 token,无上限会吃满上下文
- 摘要区间(summarizedUntilRowid 之前)的图片不恢复,符合滚动摘要语义

多模态总开关 llm.multimodalEnabled(默认关闭):
- 新配置项:CONFIG_DEFAULTS 种子 + 设置弹框 LLM 配置 Switch +
  首次引导向导 LLM 步骤 Switch(含说明文案)
- 上传入口双重判断:总开关 × 模型能力 — 未开启时即使模型支持多模态
  也不能上传图片(ChatInput 的选择/拖拽/粘贴统一拦截,Toast 区分
  "开关未开启"与"当前模型不支持"两种原因)
- 保存成功后同步 Agent Store 立即生效;App 启动时随 setProvider 加载

DeepSeek vision 模型支持:
- 新增 deepseek-v4-flash-vision-exp(OpenAI image_url content parts 格式,
  128K 上下文 / 8K 输出)
- adapter 按 isVisionModel() 判断:vision 模型将带 images 的消息转换为
  [{type:'text'},{type:'image_url'}] parts;非 vision 模型保持 images
  静默丢弃(防 API 400)

测试(236 → 243 用例):
- 多轮图片记忆 ×3(session-summary.test.ts):历史 attachments 恢复
  images / 上限 10 张从最新向前 / 摘要区间图片不恢复
- DeepSeek vision 请求格式 ×4(deepseek-vision.test.ts,契约级 mock
  fetch 断言请求体):image_url parts 转换 / 非 vision 模型丢弃 /
  max_tokens 钳制 8192 / 无图不转换
- 测试顺序修正:多轮图片用例置于 describe 末尾(插入新行消耗全局自增
  rowid,插在中间会破坏既有用例对 rowid 数值的断言)

文档: README 同步(DeepSeek 模型表 + vision 多模态列、llm.multimodalEnabled
配置项、多轮图片记忆特性行、243 用例数)

验证: lint 0 / typecheck 双工程 0 / test:electron 243 全过 / build 成功
This commit is contained in:
2026-08-21 23:00:20 +08:00
parent 3a30e8f5b4
commit 4ee5100661
12 changed files with 1363 additions and 353 deletions
@@ -0,0 +1,125 @@
/**
* DeepSeekAdapter 多模态(vision 模型)请求格式测试(v0.5.4)
*
* 背景:DeepSeek 新增 vision 实验模型 deepseek-v4-flash-vision-exp
* OpenAI image_url content parts 格式)。适配器行为:
* - vision 模型:带 images 的消息 content 转换为 [{type:'text'},{type:'image_url'}] parts
* - 非 vision 模型:images 静默丢弃(共享层行为,防 API 400)
*
* 测试策略(契约级):mock fetch 记录真实请求体断言。
*/
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
vi.mock('electron-log', () => ({
default: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() },
}));
import { DeepSeekAdapter } from '../deepseek.adapter';
import type { MetonaRequest } from '../../types';
const mockFetch = vi.fn();
vi.stubGlobal('fetch', mockFetch);
beforeEach(() => {
mockFetch.mockReset();
});
afterEach(() => {
mockFetch.mockReset();
});
function makeAdapter(model: string): DeepSeekAdapter {
return new DeepSeekAdapter({
provider: 'deepseek',
baseURL: 'https://api.deepseek.com',
apiKey: 'sk-test',
defaultModel: model,
});
}
function makeRequest(images?: Array<{ url: string }>): MetonaRequest {
return {
meta: {
sessionId: 's',
iteration: 1,
requestId: 'r',
timestamp: Date.now(),
agentVersion: '1',
},
systemPrompt: { roleDefinition: 'sys', outputConstraints: '', safetyGuidelines: '' },
messages: [{ role: 'user', content: '这张图片里有什么?', images, timestamp: Date.now() }],
params: { temperature: 0, stream: false, thinkingEnabled: false },
};
}
function okResponse(): Response {
return {
ok: true,
status: 200,
json: async () => ({ choices: [{ message: { content: 'ok' } }], usage: {} }),
} as unknown as Response;
}
function requestBody(): { model: string; messages: Array<Record<string, unknown>> } {
const [, init] = mockFetch.mock.calls[0] as [string, RequestInit];
return JSON.parse(init.body as string);
}
describe('DeepSeek vision 模型多模态请求格式(v0.5.4', () => {
it('vision 模型:带图片的消息转换为 image_url content parts', async () => {
mockFetch.mockResolvedValue(okResponse());
const adapter = makeAdapter('deepseek-v4-flash-vision-exp');
await adapter.send(makeRequest([{ url: 'data:image/jpeg;base64,TESTPIC' }]));
const body = requestBody();
expect(body.model).toBe('deepseek-v4-flash-vision-exp');
// system 消息 + user 消息(含 content parts
expect(body.messages).toHaveLength(2);
const userMsg = body.messages[1];
expect(userMsg.role).toBe('user');
expect(Array.isArray(userMsg.content)).toBe(true);
const parts = userMsg.content as Array<Record<string, unknown>>;
expect(parts[0]).toEqual({ type: 'text', text: '这张图片里有什么?' });
expect(parts[1]).toEqual({
type: 'image_url',
image_url: { url: 'data:image/jpeg;base64,TESTPIC' },
});
});
it('非 vision 模型:images 被静默丢弃(content 保持纯文本)', async () => {
mockFetch.mockResolvedValue(okResponse());
const adapter = makeAdapter('deepseek-v4-pro');
await adapter.send(makeRequest([{ url: 'data:image/jpeg;base64,TESTPIC' }]));
const body = requestBody();
const userMsg = body.messages[1];
// 非 vision 模型 content 保持字符串(不转 parts,不发图片 → 不会 400)
expect(userMsg.content).toBe('这张图片里有什么?');
});
it('vision 模型 max_tokens 钳制到 8192MODEL_INFO 上限)', async () => {
mockFetch.mockResolvedValue(okResponse());
const adapter = makeAdapter('deepseek-v4-flash-vision-exp');
await adapter.send({
...makeRequest(),
params: { maxTokens: 63_488, temperature: 0, stream: false },
} as MetonaRequest);
const body = JSON.parse((mockFetch.mock.calls[0] as [string, RequestInit])[1].body as string);
expect(body.max_tokens).toBe(8_192);
});
it('vision 模型无图片时不转换(content 保持纯文本)', async () => {
mockFetch.mockResolvedValue(okResponse());
const adapter = makeAdapter('deepseek-v4-flash-vision-exp');
await adapter.send(makeRequest(undefined));
const body = requestBody();
const userMsg = body.messages[1];
expect(userMsg.content).toBe('这张图片里有什么?');
});
});
+55 -1
View File
@@ -10,6 +10,7 @@
* @see apis/deepseek-api-docs-20260518.html
*/
import log from 'electron-log';
import { BaseAdapter } from './base-adapter';
import type { MetonaRequest, MetonaResponse, MetonaStreamEvent } from '../types';
import { MetonaFinishReason } from '../types';
@@ -20,7 +21,11 @@ import { parseSSEStream, parseOpenAICompatibleResponse } from './shared/sse-stre
export class DeepSeekAdapter extends BaseAdapter {
// H-2 修复: provider → providerId(规范要求)
override readonly providerId: string = 'deepseek';
readonly supportedModels = ['deepseek-v4-pro', 'deepseek-v4-flash'];
readonly supportedModels = [
'deepseek-v4-pro',
'deepseek-v4-flash',
'deepseek-v4-flash-vision-exp',
];
readonly supportsToolCalling = true;
readonly supportsThinking = true;
@@ -44,8 +49,29 @@ export class DeepSeekAdapter extends BaseAdapter {
supportsThinking: true,
description: 'DeepSeek 快速版,1M 上下文,低延迟推理',
},
// v0.5.4: DeepSeek 多模态实验模型(OpenAI image_url content parts 格式)
'deepseek-v4-flash-vision-exp': {
id: 'deepseek-v4-flash-vision-exp',
name: 'DeepSeek V4 Flash Vision (Exp)',
contextWindow: 128_000,
maxOutputTokens: 8_192,
supportsToolCalling: true,
supportsThinking: false,
description: 'DeepSeek 多模态实验模型,支持图片输入(image_url content parts',
},
};
/**
* v0.5.4: 当前模型是否支持多模态图片输入
*
* DeepSeek 仅 vision 系列模型支持图片(命名含 'vision');
* 非 vision 模型收到 images 时静默丢弃(避免 API 400)。
* 前端上传入口由 llm.multimodalEnabled 配置总开关控制,此处是 adapter 侧的模型级防线。
*/
private isVisionModel(): boolean {
return this.config.defaultModel.includes('vision');
}
// ===== POST /chat/completions (非流式) =====
// H-2 修复: chat → send(规范要求)
@@ -252,6 +278,34 @@ export class DeepSeekAdapter extends BaseAdapter {
stream,
};
// v0.5.4: vision 模型的图片处理(OpenAI image_url content parts 格式)
// 非 vision 模型保持 images 静默丢弃(共享层行为,避免 API 400)
if (this.isVisionModel()) {
const nonSystemMsgs = request.messages.filter((m) => m.role !== 'system');
let imageCount = 0;
// messages[0] 是 system,非 system 消息从 messages[1] 开始(与 nonSystemMsgs 对齐)
for (let i = 1; i < messages.length; i++) {
const origMsg = nonSystemMsgs[i - 1];
if (!origMsg?.images?.length) continue;
imageCount += origMsg.images.length;
const contentParts: Array<Record<string, unknown>> = [];
if (origMsg.content) {
contentParts.push({ type: 'text', text: origMsg.content });
}
for (const img of origMsg.images) {
contentParts.push({
type: 'image_url',
image_url: { url: img.url },
});
}
messages[i].content = contentParts;
}
if (imageCount > 0) {
log.info(`[DeepSeek] Vision model processing ${imageCount} image(s)`);
}
}
if (stream) {
body.stream_options = { include_usage: true };
}
@@ -91,7 +91,11 @@ describe.skipIf(!dbAvailable)('SessionSummary 分层上下文 × 截断交互',
});
afterAll(() => {
try { db?.close(); } catch { /* ignore */ }
try {
db?.close();
} catch {
/* ignore */
}
rmSync(dir, { recursive: true, force: true });
});
@@ -118,6 +122,9 @@ describe.skipIf(!dbAvailable)('SessionSummary 分层上下文 × 截断交互',
expect(history.length).toBe(3);
});
// ===== v0.5.4: 多轮图片记忆用例置于 describe 末尾(插入新行消耗全局自增 rowid,
// 若插在中间会破坏 inclusive=false 用例对 rowid 数值的断言) =====
it('截断清理摘要(缺陷 #1 回归):游标落在删除范围内时摘要被删', () => {
// 编辑重发第 2 条消息(rowid 2inclusive)→ 删除 rowid>=2 的 4 条
// 摘要游标为 3,落在 [2, ∞) 内 → 摘要必须被清理
@@ -159,4 +166,92 @@ describe.skipIf(!dbAvailable)('SessionSummary 分层上下文 × 截断交互',
expect(sessionService.truncateMessagesAfter('s_test', 'not_exist', true)).toBe(false);
expect(sessionService.getMessages('s_test').length).toBe(before);
});
/** 插入带 attachments 的用户消息(模拟 ChatInput 持久化形态) */
const insertMessageWithAttachments = (
sessionId: string,
id: string,
content: string,
attachments: Array<{ type: string; preview?: string; textContent?: string }>,
): void => {
db.prepare(
`INSERT INTO messages (id, session_id, role, content, attachments, created_at) VALUES (?, ?, 'user', ?, ?, ?)`,
).run(id, sessionId, content, JSON.stringify(attachments), Date.now());
};
it('多轮图片记忆:历史消息的图片 attachments 恢复为 images 回传', () => {
db.prepare(`DELETE FROM messages WHERE session_id = 's_img'`).run();
db.prepare(
`INSERT INTO sessions (id, created_at, updated_at, message_count) VALUES ('s_img', ?, ?, 0)`,
).run(Date.now(), Date.now());
insertMessageWithAttachments('s_img', 'img_m1', '第一轮带图', [
{ type: 'image', preview: 'data:image/jpeg;base64,ROUND1PIC' },
]);
insertMessageWithAttachments('s_img', 'img_m2', '第二轮无图', []);
insertMessageWithAttachments('s_img', 'img_m3', '第三轮带图', [
{ type: 'image', preview: 'data:image/jpeg;base64,ROUND3PIC' },
{ type: 'text', textContent: '文本附件不恢复为图片' },
]);
const history = summaryService.buildHistoryMessages('s_img');
// 第一轮的图片仍被恢复(多轮记忆修复点:此前历史消息 images 恒为 undefined
const round1 = history.find((m: any) => m.content === '第一轮带图');
expect(round1?.images).toEqual([{ url: 'data:image/jpeg;base64,ROUND1PIC', detail: 'auto' }]);
// 第三轮两张附件只恢复 image 类型
const round3 = history.find((m: any) => m.content === '第三轮带图');
expect(round3?.images).toEqual([{ url: 'data:image/jpeg;base64,ROUND3PIC', detail: 'auto' }]);
// 无图消息不带 images 字段
const round2 = history.find((m: any) => m.content === '第二轮无图');
expect(round2?.images).toBeUndefined();
});
it('多轮图片记忆:注入上限 10 张(从最新向前收集,更早的丢弃)', () => {
db.prepare(`DELETE FROM messages WHERE session_id = 's_cap'`).run();
db.prepare(
`INSERT INTO sessions (id, created_at, updated_at, message_count) VALUES ('s_cap', ?, ?, 0)`,
).run(Date.now(), Date.now());
// 12 条带图消息(每条 1 张)— 上限 10 张,最早的 2 条不注入
for (let i = 1; i <= 12; i++) {
insertMessageWithAttachments('s_cap', `cap_m${i}`, `${i}`, [
{ type: 'image', preview: `data:image/jpeg;base64,PIC${i}` },
]);
}
const history = summaryService.buildHistoryMessages('s_cap');
const withImages = history.filter((m: any) => m.images?.length > 0);
expect(withImages).toHaveLength(10);
// 保留的是第 3~12 轮(最新的 10 张)
const contents = withImages.map((m: any) => m.content);
expect(contents).not.toContain('第 1 轮');
expect(contents).not.toContain('第 2 轮');
expect(contents).toContain('第 12 轮');
});
it('多轮图片记忆:摘要区间内的图片不恢复(符合滚动摘要语义)', () => {
db.prepare(`DELETE FROM messages WHERE session_id = 's_sum_img'`).run();
db.prepare(
`INSERT INTO sessions (id, created_at, updated_at, message_count) VALUES ('s_sum_img', ?, ?, 0)`,
).run(Date.now(), Date.now());
for (let i = 1; i <= 3; i++) {
insertMessageWithAttachments('s_sum_img', `sm${i}`, `摘要区间第 ${i}`, [
{ type: 'image', preview: `data:image/jpeg;base64,OLDPIC${i}` },
]);
}
// 摘要覆盖前 3 条(含全部图片消息)
const cursor = (
db.prepare(`SELECT MAX(rowid) AS r FROM messages WHERE session_id = 's_sum_img'`).get() as {
r: number;
}
).r;
summaryService.saveSummary('s_sum_img', '早期含图对话的摘要', cursor);
insertMessageWithAttachments('s_sum_img', 'sm4', '摘要后的新消息(无图)', []);
const history = summaryService.buildHistoryMessages('s_sum_img');
// 摘要消息 + 1 条原文,均无图片(游标前的图片消息不在 tail 中)
expect(history).toHaveLength(2);
expect(history.every((m: any) => !m.images?.length)).toBe(true);
});
});
+3
View File
@@ -42,6 +42,9 @@ export const CONFIG_DEFAULTS: ConfigDefaultEntry[] = [
{ key: 'llm.baseURL', value: '', category: 'llm' },
{ key: 'llm.temperature', value: 0, category: 'llm' },
{ key: 'llm.maxTokens', value: 63488, category: 'llm' },
// v0.5.4: 多模态总开关 — 即使模型支持多模态,未开启也不能上传图片(默认关闭,
// 用户在设置/引导向导显式开启;上传入口 = 开关 × 模型能力双重判断)
{ key: 'llm.multimodalEnabled', value: false, category: 'llm' },
// P1: Provider 故障转移配置
{ key: 'llm.fallbackProvider', value: '', category: 'llm' },
{ key: 'llm.fallbackModel', value: '', category: 'llm' },
+48 -3
View File
@@ -35,6 +35,8 @@ const SUMMARY_TIMEOUT_MS = 30_000;
const PER_MESSAGE_TRUNCATE = 600;
/** 传给 LLM 的总字符上限 */
const MAX_DIGEST_CHARS = 24_000;
/** v0.5.4: 多轮图片记忆 — 历史上下文注入的最大图片数(从最新向前收集,防 token 爆炸) */
const MAX_HISTORY_IMAGES = 10;
export class SessionSummaryService {
constructor(
@@ -62,10 +64,18 @@ export class SessionSummaryService {
reasoningContent: m.reasoningContent,
toolCalls: m.toolCalls as MetonaMessage['toolCalls'],
toolResult: m.toolResult as MetonaMessage['toolResult'],
// v0.5.4: 保留 attachments 供 restoreHistoryImages 恢复图片(多轮图片记忆)
attachments: (m as { attachments?: unknown[] }).attachments,
timestamp: m.timestamp,
iteration: m.iteration,
}));
// v0.5.4: 多轮图片记忆 — 从持久化的 attachments(压缩 base64 preview)恢复 images
// 历史轮次的图片重新注入 LLM 上下文(此前仅发送当轮可见,跨轮即"失忆")。
// 数量上限防 token 爆炸:只取最近 MAX_HISTORY_IMAGES 张(从最新消息向前收集)。
// 摘要区间(summarizedUntilRowid 之前)的图片无法恢复 — 符合滚动摘要的语义。
this.restoreHistoryImages(messages);
if (existing && messages.length > 0) {
// 摘要以 assistant 角色注入(与 engine 运行时压缩的注入策略一致)
const summaryMessage: MetonaMessage = {
@@ -78,6 +88,38 @@ export class SessionSummaryService {
return messages;
}
/**
* v0.5.4: 恢复历史消息的 images(多轮图片记忆)
*
* attachments 中 type=image 的 preview1024px JPEG 压缩 base64)在消息
* 持久化时已保存(与编辑重发的恢复逻辑同源)。此处将其映射回
* MetonaMessage.images,让历史轮次图片随上下文回传 LLM。
*
* 上限策略:从最新消息向前收集,最多 MAX_HISTORY_IMAGES 张 —
* 每张 1024px 图约数百至千余 token,无上限的长会话会迅速吃满上下文。
*/
private restoreHistoryImages(messages: MetonaMessage[]): void {
let remaining = MAX_HISTORY_IMAGES;
for (let i = messages.length - 1; i >= 0 && remaining > 0; i--) {
const raw = messages[i] as MetonaMessage & {
attachments?: Array<{ type?: string; preview?: string }>;
};
const imageAttachments = (raw.attachments ?? []).filter(
(a) => a.type === 'image' && typeof a.preview === 'string' && a.preview.length > 0,
);
if (imageAttachments.length === 0) continue;
const take = Math.min(imageAttachments.length, remaining);
// 优先保留靠后的图片(时间更近)
const picked = imageAttachments.slice(-take);
messages[i].images = picked.map((a) => ({
url: a.preview as string,
detail: 'auto' as const,
}));
remaining -= take;
}
}
/**
* 会话结束后评估并生成滚动摘要(fire-and-forget 调用,失败仅记录日志)
*/
@@ -117,14 +159,16 @@ export class SessionSummaryService {
saveSummary(sessionId: string, summary: string, untilRowid: number): void {
const db = this.getDB();
db.prepare(`
db.prepare(
`
INSERT INTO session_summaries (session_id, summary, summarized_until_rowid, updated_at)
VALUES (?, ?, ?, ?)
ON CONFLICT(session_id) DO UPDATE SET
summary = excluded.summary,
summarized_until_rowid = excluded.summarized_until_rowid,
updated_at = excluded.updated_at
`).run(sessionId, summary, untilRowid, Date.now());
`,
).run(sessionId, summary, untilRowid, Date.now());
}
// ===== LLM 摘要 =====
@@ -167,7 +211,8 @@ export class SessionSummaryService {
'If a prior summary exists, merge it with the new content into one updated summary. ' +
'Preserve key facts, decisions, tool outcomes, file paths, and open questions needed for future reasoning. ' +
'Output in the same language as the conversation. Maximum 400 words. Output ONLY the summary text.',
safetyGuidelines: 'Do not include sensitive data like passwords or API keys in the summary.',
safetyGuidelines:
'Do not include sensitive data like passwords or API keys in the summary.',
},
messages: [
{