From a879b4f38af101dfb32c51cdba9e8a4fe5de93e9 Mon Sep 17 00:00:00 2001 From: thzxx Date: Sat, 27 Jun 2026 21:39:24 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E4=B8=8A=E4=BC=A0?= =?UTF-8?q?=E5=9B=BE=E7=89=87=E5=90=8E=20AI=20=E6=97=A0=E6=B3=95=E8=AF=86?= =?UTF-8?q?=E5=88=AB=E5=9B=BE=E7=89=87=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 根因：图片从 ChatInput → agent-store → IPC → adapter 的链路中有两处断裂： 1. DeepSeekAdapter.toNativeRequest 完全忽略 message.images 字段 - 用户消息中的图片 base64 数据未转换为 OpenAI 多模态 content 数组格式，导致 LLM 收不到图片数据，只能尝试用工具查磁盘文件 2. AgnesAdapter.toNativeRequest 图片处理存在数组索引错位 - base.messages[0] 固定为 system 消息（父类插入） - request.messages.filter(m => m.role !== 'system') 去掉了 system - 同一索引 i 访问两个不同长度的数组，导致图片消息匹配到错误位置 3. agent-store sendMessage 中 images 参数被局部变量遮蔽 - ChatInput 传入的 images 被 覆盖 - 改为直接使用参数，避免重复从 attachments 提取 修复: - DeepSeekAdapter: 在 toNativeRequest 中添加图片→OpenAI content 数组转换 - AgnesAdapter: 移除冗余图片处理(基类已处理)，仅保留 Thinking/max_tokens - agent-store: 移除 images 变量遮蔽，直接使用参数 影响范围: DeepSeek / Agnes AI / Ollama 三个 Provider 的图片传递均已验证 --- electron/harness/adapters/agnes-ai.adapter.ts | 24 +--------- electron/harness/adapters/deepseek.adapter.ts | 21 ++++++++- src/stores/agent-store.ts | 45 ++++++++++--------- 3 files changed, 44 insertions(+), 46 deletions(-) diff --git a/electron/harness/adapters/agnes-ai.adapter.ts b/electron/harness/adapters/agnes-ai.adapter.ts index 7e5e52c..98fcc81 100644 --- a/electron/harness/adapters/agnes-ai.adapter.ts +++ b/electron/harness/adapters/agnes-ai.adapter.ts @@ -26,7 +26,7 @@ export class AgnesAdapter extends DeepSeekAdapter { * * 差异： * 1. Thinking 模式使用 chat_template_kwargs - * 2. 图片使用 OpenAI 多模态 content 数组格式 + * 2. 图片处理由基类 DeepSeekAdapter.toNativeRequest 完成（OpenAI 多模态格式） */ protected override toNativeRequest(request: MetonaRequest): Record { const base = super.toNativeRequest(request) as Record; @@ -43,28 +43,6 @@ export class AgnesAdapter extends DeepSeekAdapter { base.max_tokens = 65536; } - // 图片处理：将 images 转为 OpenAI 多模态 content 数组格式 - const messages = base.messages as Array>; - for (let i = 0; i < messages.length; i++) { - const msg = messages[i]; - // 找到对应的原始消息（跳过 system 消息） - const origMsg = request.messages.filter((m) => m.role !== 'system')[i]; - if (!origMsg?.images || origMsg.images.length === 0) continue; - - // 将 content 转为数组格式：[{type: "text", text: ...}, {type: "image_url", ...}] - const contentParts: Array> = []; - if (msg.content && typeof msg.content === 'string') { - contentParts.push({ type: 'text', text: msg.content }); - } - for (const img of origMsg.images) { - contentParts.push({ - type: 'image_url', - image_url: { url: img.url, detail: img.detail ?? 'auto' }, - }); - } - msg.content = contentParts; - } - return base; } } diff --git a/electron/harness/adapters/deepseek.adapter.ts b/electron/harness/adapters/deepseek.adapter.ts index 6b5c5cf..fe010b8 100644 --- a/electron/harness/adapters/deepseek.adapter.ts +++ b/electron/harness/adapters/deepseek.adapter.ts @@ -263,7 +263,26 @@ export class DeepSeekAdapter extends BaseAdapter { ].filter(Boolean).join('\n\n'), }, ...request.messages.filter((m) => m.role !== 'system').map((m) => { - const msg: Record = { role: m.role, content: m.content }; + const msg: Record = { role: m.role }; + + // 多模态图片处理：将 images 转为 OpenAI content 数组格式 + // [{"type":"text","text":"..."}, {"type":"image_url","image_url":{"url":"data:..."}}] + if (m.images && m.images.length > 0) { + const contentParts: Array> = []; + if (m.content) { + contentParts.push({ type: 'text', text: m.content }); + } + for (const img of m.images) { + contentParts.push({ + type: 'image_url', + image_url: { url: img.url, detail: img.detail ?? 'auto' }, + }); + } + msg.content = contentParts; + } else { + msg.content = m.content; + } + if (m.role === 'assistant' && m.toolCalls?.length) { msg.tool_calls = m.toolCalls.map((tc) => ({ id: tc.id, diff --git a/src/stores/agent-store.ts b/src/stores/agent-store.ts index 173b235..17c81b1 100644 --- a/src/stores/agent-store.ts +++ b/src/stores/agent-store.ts @@ -220,33 +220,34 @@ export const useAgentStore = create((set, get) => ({ } if (sessionId && window.metona?.agent?.sendMessage) { - // 构建发给 LLM 的 content：纯用户文本 + 文件 JSON（无文字描述） + // 构建发给 LLM 的消息 + // images 参数由 ChatInput 传入（已从附件中提取 base64 data URL） let llmContent = content; - const images: Array<{ url: string; detail?: 'low' | 'high' | 'auto' }> = []; + const llmImages: Array<{ url: string; detail?: 'low' | 'high' | 'auto' }> = [ + ...(images ?? []), + ]; if (attachments && attachments.length > 0) { const parts: string[] = []; for (const att of attachments) { - if (att.type === 'image' && att.preview) { - // 图片：直接加入 images 数组 - images.push({ url: att.preview, detail: 'auto' }); - } else { - // 所有文件：JSON 格式，Base64 编码内容 - const ext = att.name.split('.').pop() ?? 'unknown'; - let base64Content = ''; - if (att.textContent) { - base64Content = btoa(unescape(encodeURIComponent(att.textContent))); - } else if (att.preview) { - // 二进制文件从 data URL 提取 base64 - base64Content = att.preview.split(',')[1] ?? ''; - } - parts.push(JSON.stringify({ - file_name: att.name, - file_type: ext, - context_encode: 'Base64', - context: base64Content, - })); + if (att.type === 'image') { + // 图片已在 images 参数中处理，跳过 + continue; } + // 非图片文件：JSON 格式，Base64 编码内容 + const ext = att.name.split('.').pop() ?? 'unknown'; + let base64Content = ''; + if (att.textContent) { + base64Content = btoa(unescape(encodeURIComponent(att.textContent))); + } else if (att.preview) { + base64Content = att.preview.split(',')[1] ?? ''; + } + parts.push(JSON.stringify({ + file_name: att.name, + file_type: ext, + context_encode: 'Base64', + context: base64Content, + })); } llmContent = parts.join('\n') + (content ? '\n\n' + content : ''); } @@ -254,7 +255,7 @@ export const useAgentStore = create((set, get) => ({ const messageWithImages = { ...userMessage, content: llmContent, - images: images.length > 0 ? images : undefined, + images: llmImages.length > 0 ? llmImages : undefined, }; window.metona.agent.sendMessage(messageWithImages, sessionId).catch(() => {}); }