feat: v0.5.4 多模态增强 — 多轮图片记忆 + 多模态总开关 + DeepSeek vision 模型支持
多轮图片记忆:
- 此前历史轮次的图片不回传 LLM(attachments 仅存压缩 preview,历史组装
时被丢弃)— 跨轮对话中模型对图片内容"失忆"
- 修复:SessionSummaryService.buildHistoryMessages 从持久化的 attachments
恢复 images(type=image 的 preview base64),历史图片随上下文回传
- Token 控制:最多注入最近 10 张(MAX_HISTORY_IMAGES,从最新消息向前
收集)— 每张 1024px 压缩图约数百至千余 token,无上限会吃满上下文
- 摘要区间(summarizedUntilRowid 之前)的图片不恢复,符合滚动摘要语义
多模态总开关 llm.multimodalEnabled(默认关闭):
- 新配置项:CONFIG_DEFAULTS 种子 + 设置弹框 LLM 配置 Switch +
首次引导向导 LLM 步骤 Switch(含说明文案)
- 上传入口双重判断:总开关 × 模型能力 — 未开启时即使模型支持多模态
也不能上传图片(ChatInput 的选择/拖拽/粘贴统一拦截,Toast 区分
"开关未开启"与"当前模型不支持"两种原因)
- 保存成功后同步 Agent Store 立即生效;App 启动时随 setProvider 加载
DeepSeek vision 模型支持:
- 新增 deepseek-v4-flash-vision-exp(OpenAI image_url content parts 格式,
128K 上下文 / 8K 输出)
- adapter 按 isVisionModel() 判断:vision 模型将带 images 的消息转换为
[{type:'text'},{type:'image_url'}] parts;非 vision 模型保持 images
静默丢弃(防 API 400)
测试(236 → 243 用例):
- 多轮图片记忆 ×3(session-summary.test.ts):历史 attachments 恢复
images / 上限 10 张从最新向前 / 摘要区间图片不恢复
- DeepSeek vision 请求格式 ×4(deepseek-vision.test.ts,契约级 mock
fetch 断言请求体):image_url parts 转换 / 非 vision 模型丢弃 /
max_tokens 钳制 8192 / 无图不转换
- 测试顺序修正:多轮图片用例置于 describe 末尾(插入新行消耗全局自增
rowid,插在中间会破坏既有用例对 rowid 数值的断言)
文档: README 同步(DeepSeek 模型表 + vision 多模态列、llm.multimodalEnabled
配置项、多轮图片记忆特性行、243 用例数)
验证: lint 0 / typecheck 双工程 0 / test:electron 243 全过 / build 成功
This commit is contained in:
@@ -0,0 +1,125 @@
|
||||
/**
|
||||
* DeepSeekAdapter 多模态(vision 模型)请求格式测试(v0.5.4)
|
||||
*
|
||||
* 背景:DeepSeek 新增 vision 实验模型 deepseek-v4-flash-vision-exp
|
||||
* (OpenAI image_url content parts 格式)。适配器行为:
|
||||
* - vision 模型:带 images 的消息 content 转换为 [{type:'text'},{type:'image_url'}] parts
|
||||
* - 非 vision 模型:images 静默丢弃(共享层行为,防 API 400)
|
||||
*
|
||||
* 测试策略(契约级):mock fetch 记录真实请求体断言。
|
||||
*/
|
||||
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
||||
|
||||
vi.mock('electron-log', () => ({
|
||||
default: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() },
|
||||
}));
|
||||
|
||||
import { DeepSeekAdapter } from '../deepseek.adapter';
|
||||
import type { MetonaRequest } from '../../types';
|
||||
|
||||
const mockFetch = vi.fn();
|
||||
vi.stubGlobal('fetch', mockFetch);
|
||||
|
||||
beforeEach(() => {
|
||||
mockFetch.mockReset();
|
||||
});
|
||||
afterEach(() => {
|
||||
mockFetch.mockReset();
|
||||
});
|
||||
|
||||
function makeAdapter(model: string): DeepSeekAdapter {
|
||||
return new DeepSeekAdapter({
|
||||
provider: 'deepseek',
|
||||
baseURL: 'https://api.deepseek.com',
|
||||
apiKey: 'sk-test',
|
||||
defaultModel: model,
|
||||
});
|
||||
}
|
||||
|
||||
function makeRequest(images?: Array<{ url: string }>): MetonaRequest {
|
||||
return {
|
||||
meta: {
|
||||
sessionId: 's',
|
||||
iteration: 1,
|
||||
requestId: 'r',
|
||||
timestamp: Date.now(),
|
||||
agentVersion: '1',
|
||||
},
|
||||
systemPrompt: { roleDefinition: 'sys', outputConstraints: '', safetyGuidelines: '' },
|
||||
messages: [{ role: 'user', content: '这张图片里有什么?', images, timestamp: Date.now() }],
|
||||
params: { temperature: 0, stream: false, thinkingEnabled: false },
|
||||
};
|
||||
}
|
||||
|
||||
function okResponse(): Response {
|
||||
return {
|
||||
ok: true,
|
||||
status: 200,
|
||||
json: async () => ({ choices: [{ message: { content: 'ok' } }], usage: {} }),
|
||||
} as unknown as Response;
|
||||
}
|
||||
|
||||
function requestBody(): { model: string; messages: Array<Record<string, unknown>> } {
|
||||
const [, init] = mockFetch.mock.calls[0] as [string, RequestInit];
|
||||
return JSON.parse(init.body as string);
|
||||
}
|
||||
|
||||
describe('DeepSeek vision 模型多模态请求格式(v0.5.4)', () => {
|
||||
it('vision 模型:带图片的消息转换为 image_url content parts', async () => {
|
||||
mockFetch.mockResolvedValue(okResponse());
|
||||
const adapter = makeAdapter('deepseek-v4-flash-vision-exp');
|
||||
|
||||
await adapter.send(makeRequest([{ url: 'data:image/jpeg;base64,TESTPIC' }]));
|
||||
|
||||
const body = requestBody();
|
||||
expect(body.model).toBe('deepseek-v4-flash-vision-exp');
|
||||
// system 消息 + user 消息(含 content parts)
|
||||
expect(body.messages).toHaveLength(2);
|
||||
const userMsg = body.messages[1];
|
||||
expect(userMsg.role).toBe('user');
|
||||
expect(Array.isArray(userMsg.content)).toBe(true);
|
||||
const parts = userMsg.content as Array<Record<string, unknown>>;
|
||||
expect(parts[0]).toEqual({ type: 'text', text: '这张图片里有什么?' });
|
||||
expect(parts[1]).toEqual({
|
||||
type: 'image_url',
|
||||
image_url: { url: 'data:image/jpeg;base64,TESTPIC' },
|
||||
});
|
||||
});
|
||||
|
||||
it('非 vision 模型:images 被静默丢弃(content 保持纯文本)', async () => {
|
||||
mockFetch.mockResolvedValue(okResponse());
|
||||
const adapter = makeAdapter('deepseek-v4-pro');
|
||||
|
||||
await adapter.send(makeRequest([{ url: 'data:image/jpeg;base64,TESTPIC' }]));
|
||||
|
||||
const body = requestBody();
|
||||
const userMsg = body.messages[1];
|
||||
// 非 vision 模型 content 保持字符串(不转 parts,不发图片 → 不会 400)
|
||||
expect(userMsg.content).toBe('这张图片里有什么?');
|
||||
});
|
||||
|
||||
it('vision 模型 max_tokens 钳制到 8192(MODEL_INFO 上限)', async () => {
|
||||
mockFetch.mockResolvedValue(okResponse());
|
||||
const adapter = makeAdapter('deepseek-v4-flash-vision-exp');
|
||||
|
||||
await adapter.send({
|
||||
...makeRequest(),
|
||||
params: { maxTokens: 63_488, temperature: 0, stream: false },
|
||||
} as MetonaRequest);
|
||||
|
||||
const body = JSON.parse((mockFetch.mock.calls[0] as [string, RequestInit])[1].body as string);
|
||||
expect(body.max_tokens).toBe(8_192);
|
||||
});
|
||||
|
||||
it('vision 模型无图片时不转换(content 保持纯文本)', async () => {
|
||||
mockFetch.mockResolvedValue(okResponse());
|
||||
const adapter = makeAdapter('deepseek-v4-flash-vision-exp');
|
||||
|
||||
await adapter.send(makeRequest(undefined));
|
||||
|
||||
const body = requestBody();
|
||||
const userMsg = body.messages[1];
|
||||
expect(userMsg.content).toBe('这张图片里有什么?');
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user