From fed1e6d1c2ba762946d2087a3bedc03573ab2145 Mon Sep 17 00:00:00 2001 From: thzxx Date: Thu, 18 Jun 2026 12:45:06 +0800 Subject: [PATCH] =?UTF-8?q?refactor:=20=E8=A7=86=E9=A2=91/=E5=9B=BE?= =?UTF-8?q?=E7=89=87/=E6=96=87=E4=BB=B6=E7=BB=84=E5=90=88=E5=8F=91?= =?UTF-8?q?=E9=80=81=EF=BC=8C=E5=90=84=E8=B5=B0=E5=90=84=E7=9A=84=E6=B8=B2?= =?UTF-8?q?=E6=9F=93=E9=80=9A=E9=81=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ChatMessage 新增 _videoFrames/_videoMeta 独立字段 - msg.images 只放图片，视频帧走 _videoFrames - 聊天卡片：图片缩略图 + 视频指示牌 + 文件 chips 可同时显示 - Ollama API 仍合并全部帧到 images 数组 --- src/renderer/components/chat-area.ts | 27 +++++-------- src/renderer/components/input-area.ts | 57 +++++++++++++++------------ src/renderer/types.d.ts | 3 ++ 3 files changed, 44 insertions(+), 43 deletions(-) diff --git a/src/renderer/components/chat-area.ts b/src/renderer/components/chat-area.ts index 1cc36cb..60222d6 100644 --- a/src/renderer/components/chat-area.ts +++ b/src/renderer/components/chat-area.ts @@ -149,13 +149,6 @@ export function appendMessageDOM(msg: ChatMessage, index: number): void { let contentHtml = ''; let safeContent = (msg.content != null) ? String(msg.content) : ''; - // 视频消息：折叠帧列表，只显示摘要 - const isVideoMsg = safeContent.includes('[视频帧序列'); - if (isVideoMsg) { - const idx = safeContent.indexOf('[视频帧序列'); - safeContent = safeContent.slice(0, idx) + safeContent.slice(idx).split('\n')[0]; - } - if (msg.role === 'assistant') { // ── 系统提示词折叠卡片（每条助理消息顶部展示）── const sysPrompt = state.get('_lastSystemPrompt', ''); @@ -183,18 +176,16 @@ export function appendMessageDOM(msg: ChatMessage, index: number): void { } if (msg.images && msg.images.length > 0) { - // 视频帧序列：不逐帧渲染缩略图，只显示视频指示 - const isVideo = msg.content?.includes('[视频帧序列'); - if (isVideo) { - const frameCount = msg.images.length; - contentHtml += `
🎬 视频帧序列 · ${frameCount} 帧 · 1fps
`; - } else { - contentHtml += '
'; - for (const img of msg.images) { - contentHtml += `用户图片`; - } - contentHtml += '
'; + contentHtml += '
'; + for (const img of msg.images) { + contentHtml += `用户图片`; } + contentHtml += '
'; + } + + if ((msg as any)._videoFrames && (msg as any)._videoFrames.length > 0) { + const n = (msg as any)._videoFrames.length; + contentHtml += `
🎬 视频帧序列 · ${n} 帧 · 1fps
`; } if (msg.files && msg.files.length > 0) { diff --git a/src/renderer/components/input-area.ts b/src/renderer/components/input-area.ts index e4e5b14..a56dc5d 100644 --- a/src/renderer/components/input-area.ts +++ b/src/renderer/components/input-area.ts @@ -953,40 +953,46 @@ async function sendMessageWithAgentLoop(text: string, currentSession: ChatSessio ...pendingImages.map(img => img.base64), ...pendingVideoFrames.map(f => f.base64) ]; - const totalAttachments = pendingImages.length + pendingVideoFrames.length; + let apiContentForModel = text || ''; + const hasAttachments = pendingImages.length > 0 || pendingVideoFrames.length > 0 || userFiles.length > 0; - if (totalAttachments > 0) { - let imgDesc: string; + if (hasAttachments) { + // 聊天卡片显示用的简短描述 + const displayParts: string[] = []; + if (text) displayParts.push(text); + if (pendingVideoMeta && pendingVideoFrames.length > 0) { + displayParts.push(`[视频: ${pendingVideoMeta.fileName}, ${pendingVideoFrames.length}帧, 1fps]`); + } + if (pendingImages.length === 1) { + displayParts.push(`[图片: ${pendingImages[0].name}]`); + } else if (pendingImages.length > 1) { + displayParts.push(`[${pendingImages.length} 张图片]`); + } + const displayContent = displayParts.join('\n'); + + // Ollama API 用的完整描述（视频带帧序列） if (pendingVideoMeta && pendingVideoFrames.length > 0) { - // 有视频：构建帧序列描述 const frameList = pendingVideoFrames .map(f => ` ${String(f.timestampSeconds).padStart(4, ' ')}s — ${f.name}`) .join('\n'); - imgDesc = `[视频帧序列 · ${pendingVideoFrames.length}帧 · 1fps · ${pendingVideoMeta.duration.toFixed(0)}s]\n${frameList}\n\n上述帧按时间顺序排列，帧间存在时序关系。请分析视频内容。`; - } else if (pendingImages.length === 1) { - imgDesc = `[上传了图片: ${pendingImages[0].name}]`; - } else if (pendingImages.length > 0) { - imgDesc = `[上传了 ${pendingImages.length} 张图片]`; - } else { - imgDesc = ''; + apiContentForModel = (text || '') + `\n\n[视频帧序列 · ${pendingVideoFrames.length}帧 · 1fps · ${pendingVideoMeta.duration.toFixed(0)}s]\n${frameList}\n\n上述帧按时间顺序排列，帧间存在时序关系。请分析视频内容。`; + if (pendingImages.length > 0) { + apiContentForModel += `\n\n[同时上传了 ${pendingImages.length} 张图片]`; + } + } else if (displayContent) { + apiContentForModel = displayContent; } + const msg: ChatMessage = { role: 'user', - content: text ? `${text}\n\n${imgDesc}` : imgDesc, - images, + content: displayContent, + images: [...pendingImages.map(img => img.base64)], timestamp: now }; - if (userFiles.length > 0) { - msg.files = userFiles; - msg._fileContents = pendingFiles.map(f => ({ language: f.language, content: f.content })); + if (pendingVideoFrames.length > 0 && pendingVideoMeta) { + msg._videoFrames = [...pendingVideoFrames]; + msg._videoMeta = { ...pendingVideoMeta, frameCount: pendingVideoFrames.length }; } - msgsToAdd.push(msg); - } else if (text || pendingFiles.length > 0) { - const msg: ChatMessage = { - role: 'user', - content: text || '', - timestamp: now - }; if (userFiles.length > 0) { msg.files = userFiles; msg._fileContents = pendingFiles.map(f => ({ language: f.language, content: f.content })); @@ -1046,7 +1052,8 @@ async function sendMessageWithAgentLoop(text: string, currentSession: ChatSessio return { role: m.role, content, - ...(m.images?.length && { images: m.images }) + ...(m.images?.length && { images: m.images }), + ...((m as any)._videoFrames?.length && { images: [...(m.images || []), ...(m as any)._videoFrames.map((f: any) => f.base64)] }) }; }); @@ -1066,7 +1073,7 @@ async function sendMessageWithAgentLoop(text: string, currentSession: ChatSessio }, 2000); try { - await runAgentLoop(text || (pendingFiles.length > 0 ? `请分析 ${pendingFiles.map(f => f.name).join(', ')}` : ''), images, historyMessages, { + await runAgentLoop(apiContentForModel || (pendingFiles.length > 0 ? `请分析 ${pendingFiles.map(f => f.name).join(', ')}` : ''), images, historyMessages, { /** 模型正在生成 tool_call 参数中 — 提前在工作空间显示"准备中"卡片 */ onToolCallPrepare: (call) => { addToolCard({ diff --git a/src/renderer/types.d.ts b/src/renderer/types.d.ts index 2b2061b..4b7aa76 100644 --- a/src/renderer/types.d.ts +++ b/src/renderer/types.d.ts @@ -111,6 +111,9 @@ export interface ChatMessage { images?: string[]; files?: ChatFile[]; _fileContents?: FileContent[]; + /** 视频帧数据（用于消息卡片渲染，不走 images 缩略图） */ + _videoFrames?: Array<{ name: string; base64: string; timestampSeconds: number }>; + _videoMeta?: { fileName: string; duration: number; frameCount: number }; stopped?: boolean; toolCalls?: ToolCallRecord[]; /** 标记此消息为 LLM 压缩摘要生成 */