fix: 修复上传图片后 AI 无法识别图片的问题
根因:图片从 ChatInput → agent-store → IPC → adapter 的链路中有两处断裂:
1. DeepSeekAdapter.toNativeRequest 完全忽略 message.images 字段
- 用户消息中的图片 base64 数据未转换为 OpenAI 多模态 content
数组格式,导致 LLM 收不到图片数据,只能尝试用工具查磁盘文件
2. AgnesAdapter.toNativeRequest 图片处理存在数组索引错位
- base.messages[0] 固定为 system 消息(父类插入)
- request.messages.filter(m => m.role !== 'system') 去掉了 system
- 同一索引 i 访问两个不同长度的数组,导致图片消息匹配到错误位置
3. agent-store sendMessage 中 images 参数被局部变量遮蔽
- ChatInput 传入的 images 被 覆盖
- 改为直接使用参数,避免重复从 attachments 提取
修复:
- DeepSeekAdapter: 在 toNativeRequest 中添加图片→OpenAI content 数组转换
- AgnesAdapter: 移除冗余图片处理(基类已处理),仅保留 Thinking/max_tokens
- agent-store: 移除 images 变量遮蔽,直接使用参数
影响范围: DeepSeek / Agnes AI / Ollama 三个 Provider 的图片传递均已验证
This commit is contained in:
@@ -26,7 +26,7 @@ export class AgnesAdapter extends DeepSeekAdapter {
|
|||||||
*
|
*
|
||||||
* 差异:
|
* 差异:
|
||||||
* 1. Thinking 模式使用 chat_template_kwargs
|
* 1. Thinking 模式使用 chat_template_kwargs
|
||||||
* 2. 图片使用 OpenAI 多模态 content 数组格式
|
* 2. 图片处理由基类 DeepSeekAdapter.toNativeRequest 完成(OpenAI 多模态格式)
|
||||||
*/
|
*/
|
||||||
protected override toNativeRequest(request: MetonaRequest): Record<string, unknown> {
|
protected override toNativeRequest(request: MetonaRequest): Record<string, unknown> {
|
||||||
const base = super.toNativeRequest(request) as Record<string, unknown>;
|
const base = super.toNativeRequest(request) as Record<string, unknown>;
|
||||||
@@ -43,28 +43,6 @@ export class AgnesAdapter extends DeepSeekAdapter {
|
|||||||
base.max_tokens = 65536;
|
base.max_tokens = 65536;
|
||||||
}
|
}
|
||||||
|
|
||||||
// 图片处理:将 images 转为 OpenAI 多模态 content 数组格式
|
|
||||||
const messages = base.messages as Array<Record<string, unknown>>;
|
|
||||||
for (let i = 0; i < messages.length; i++) {
|
|
||||||
const msg = messages[i];
|
|
||||||
// 找到对应的原始消息(跳过 system 消息)
|
|
||||||
const origMsg = request.messages.filter((m) => m.role !== 'system')[i];
|
|
||||||
if (!origMsg?.images || origMsg.images.length === 0) continue;
|
|
||||||
|
|
||||||
// 将 content 转为数组格式:[{type: "text", text: ...}, {type: "image_url", ...}]
|
|
||||||
const contentParts: Array<Record<string, unknown>> = [];
|
|
||||||
if (msg.content && typeof msg.content === 'string') {
|
|
||||||
contentParts.push({ type: 'text', text: msg.content });
|
|
||||||
}
|
|
||||||
for (const img of origMsg.images) {
|
|
||||||
contentParts.push({
|
|
||||||
type: 'image_url',
|
|
||||||
image_url: { url: img.url, detail: img.detail ?? 'auto' },
|
|
||||||
});
|
|
||||||
}
|
|
||||||
msg.content = contentParts;
|
|
||||||
}
|
|
||||||
|
|
||||||
return base;
|
return base;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -263,7 +263,26 @@ export class DeepSeekAdapter extends BaseAdapter {
|
|||||||
].filter(Boolean).join('\n\n'),
|
].filter(Boolean).join('\n\n'),
|
||||||
},
|
},
|
||||||
...request.messages.filter((m) => m.role !== 'system').map((m) => {
|
...request.messages.filter((m) => m.role !== 'system').map((m) => {
|
||||||
const msg: Record<string, unknown> = { role: m.role, content: m.content };
|
const msg: Record<string, unknown> = { role: m.role };
|
||||||
|
|
||||||
|
// 多模态图片处理:将 images 转为 OpenAI content 数组格式
|
||||||
|
// [{"type":"text","text":"..."}, {"type":"image_url","image_url":{"url":"data:..."}}]
|
||||||
|
if (m.images && m.images.length > 0) {
|
||||||
|
const contentParts: Array<Record<string, unknown>> = [];
|
||||||
|
if (m.content) {
|
||||||
|
contentParts.push({ type: 'text', text: m.content });
|
||||||
|
}
|
||||||
|
for (const img of m.images) {
|
||||||
|
contentParts.push({
|
||||||
|
type: 'image_url',
|
||||||
|
image_url: { url: img.url, detail: img.detail ?? 'auto' },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
msg.content = contentParts;
|
||||||
|
} else {
|
||||||
|
msg.content = m.content;
|
||||||
|
}
|
||||||
|
|
||||||
if (m.role === 'assistant' && m.toolCalls?.length) {
|
if (m.role === 'assistant' && m.toolCalls?.length) {
|
||||||
msg.tool_calls = m.toolCalls.map((tc) => ({
|
msg.tool_calls = m.toolCalls.map((tc) => ({
|
||||||
id: tc.id,
|
id: tc.id,
|
||||||
|
|||||||
+23
-22
@@ -220,33 +220,34 @@ export const useAgentStore = create<AgentState>((set, get) => ({
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (sessionId && window.metona?.agent?.sendMessage) {
|
if (sessionId && window.metona?.agent?.sendMessage) {
|
||||||
// 构建发给 LLM 的 content:纯用户文本 + 文件 JSON(无文字描述)
|
// 构建发给 LLM 的消息
|
||||||
|
// images 参数由 ChatInput 传入(已从附件中提取 base64 data URL)
|
||||||
let llmContent = content;
|
let llmContent = content;
|
||||||
const images: Array<{ url: string; detail?: 'low' | 'high' | 'auto' }> = [];
|
const llmImages: Array<{ url: string; detail?: 'low' | 'high' | 'auto' }> = [
|
||||||
|
...(images ?? []),
|
||||||
|
];
|
||||||
|
|
||||||
if (attachments && attachments.length > 0) {
|
if (attachments && attachments.length > 0) {
|
||||||
const parts: string[] = [];
|
const parts: string[] = [];
|
||||||
for (const att of attachments) {
|
for (const att of attachments) {
|
||||||
if (att.type === 'image' && att.preview) {
|
if (att.type === 'image') {
|
||||||
// 图片:直接加入 images 数组
|
// 图片已在 images 参数中处理,跳过
|
||||||
images.push({ url: att.preview, detail: 'auto' });
|
continue;
|
||||||
} else {
|
|
||||||
// 所有文件:JSON 格式,Base64 编码内容
|
|
||||||
const ext = att.name.split('.').pop() ?? 'unknown';
|
|
||||||
let base64Content = '';
|
|
||||||
if (att.textContent) {
|
|
||||||
base64Content = btoa(unescape(encodeURIComponent(att.textContent)));
|
|
||||||
} else if (att.preview) {
|
|
||||||
// 二进制文件从 data URL 提取 base64
|
|
||||||
base64Content = att.preview.split(',')[1] ?? '';
|
|
||||||
}
|
|
||||||
parts.push(JSON.stringify({
|
|
||||||
file_name: att.name,
|
|
||||||
file_type: ext,
|
|
||||||
context_encode: 'Base64',
|
|
||||||
context: base64Content,
|
|
||||||
}));
|
|
||||||
}
|
}
|
||||||
|
// 非图片文件:JSON 格式,Base64 编码内容
|
||||||
|
const ext = att.name.split('.').pop() ?? 'unknown';
|
||||||
|
let base64Content = '';
|
||||||
|
if (att.textContent) {
|
||||||
|
base64Content = btoa(unescape(encodeURIComponent(att.textContent)));
|
||||||
|
} else if (att.preview) {
|
||||||
|
base64Content = att.preview.split(',')[1] ?? '';
|
||||||
|
}
|
||||||
|
parts.push(JSON.stringify({
|
||||||
|
file_name: att.name,
|
||||||
|
file_type: ext,
|
||||||
|
context_encode: 'Base64',
|
||||||
|
context: base64Content,
|
||||||
|
}));
|
||||||
}
|
}
|
||||||
llmContent = parts.join('\n') + (content ? '\n\n' + content : '');
|
llmContent = parts.join('\n') + (content ? '\n\n' + content : '');
|
||||||
}
|
}
|
||||||
@@ -254,7 +255,7 @@ export const useAgentStore = create<AgentState>((set, get) => ({
|
|||||||
const messageWithImages = {
|
const messageWithImages = {
|
||||||
...userMessage,
|
...userMessage,
|
||||||
content: llmContent,
|
content: llmContent,
|
||||||
images: images.length > 0 ? images : undefined,
|
images: llmImages.length > 0 ? llmImages : undefined,
|
||||||
};
|
};
|
||||||
window.metona.agent.sendMessage(messageWithImages, sessionId).catch(() => {});
|
window.metona.agent.sendMessage(messageWithImages, sessionId).catch(() => {});
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user