refactor: 视频/图片/文件组合发送，各走各的渲染通道
- ChatMessage 新增 _videoFrames/_videoMeta 独立字段 - msg.images 只放图片，视频帧走 _videoFrames - 聊天卡片：图片缩略图 + 视频指示牌 + 文件 chips 可同时显示 - Ollama API 仍合并全部帧到 images 数组
This commit is contained in:
thzxx
@@ -149,13 +149,6 @@ export function appendMessageDOM(msg: ChatMessage, index: number): void {
|
||||
let contentHtml = '';
|
||||
let safeContent = (msg.content != null) ? String(msg.content) : '';
|
||||
|
||||
// 视频消息：折叠帧列表，只显示摘要
|
||||
const isVideoMsg = safeContent.includes('[视频帧序列');
|
||||
if (isVideoMsg) {
|
||||
const idx = safeContent.indexOf('[视频帧序列');
|
||||
safeContent = safeContent.slice(0, idx) + safeContent.slice(idx).split('\n')[0];
|
||||
}
|
||||
|
||||
if (msg.role === 'assistant') {
|
||||
// ── 系统提示词折叠卡片（每条助理消息顶部展示）──
|
||||
const sysPrompt = state.get<string>('_lastSystemPrompt', '');
|
||||
@@ -183,18 +176,16 @@ export function appendMessageDOM(msg: ChatMessage, index: number): void {
|
||||
}
|
||||
|
||||
if (msg.images && msg.images.length > 0) {
|
||||
// 视频帧序列：不逐帧渲染缩略图，只显示视频指示
|
||||
const isVideo = msg.content?.includes('[视频帧序列');
|
||||
if (isVideo) {
|
||||
const frameCount = msg.images.length;
|
||||
contentHtml += `<div class="msg-video-indicator">🎬 视频帧序列 · ${frameCount} 帧 · 1fps</div>`;
|
||||
} else {
|
||||
contentHtml += '<div class="msg-images">';
|
||||
for (const img of msg.images) {
|
||||
contentHtml += `<img class="msg-img" src="data:image/png;base64,${img}" alt="用户图片" data-lightbox="true">`;
|
||||
}
|
||||
contentHtml += '</div>';
|
||||
}
|
||||
|
||||
if ((msg as any)._videoFrames && (msg as any)._videoFrames.length > 0) {
|
||||
const n = (msg as any)._videoFrames.length;
|
||||
contentHtml += `<div class="msg-video-indicator">🎬 视频帧序列 · ${n} 帧 · 1fps</div>`;
|
||||
}
|
||||
|
||||
if (msg.files && msg.files.length > 0) {
|
||||
|
||||
@@ -953,40 +953,46 @@ async function sendMessageWithAgentLoop(text: string, currentSession: ChatSessio
|
||||
...pendingImages.map(img => img.base64),
|
||||
...pendingVideoFrames.map(f => f.base64)
|
||||
];
|
||||
const totalAttachments = pendingImages.length + pendingVideoFrames.length;
|
||||
let apiContentForModel = text || '';
|
||||
const hasAttachments = pendingImages.length > 0 || pendingVideoFrames.length > 0 || userFiles.length > 0;
|
||||
|
||||
if (totalAttachments > 0) {
|
||||
let imgDesc: string;
|
||||
if (hasAttachments) {
|
||||
// 聊天卡片显示用的简短描述
|
||||
const displayParts: string[] = [];
|
||||
if (text) displayParts.push(text);
|
||||
if (pendingVideoMeta && pendingVideoFrames.length > 0) {
|
||||
displayParts.push(`[视频: ${pendingVideoMeta.fileName}, ${pendingVideoFrames.length}帧, 1fps]`);
|
||||
}
|
||||
if (pendingImages.length === 1) {
|
||||
displayParts.push(`[图片: ${pendingImages[0].name}]`);
|
||||
} else if (pendingImages.length > 1) {
|
||||
displayParts.push(`[${pendingImages.length} 张图片]`);
|
||||
}
|
||||
const displayContent = displayParts.join('\n');
|
||||
|
||||
// Ollama API 用的完整描述（视频带帧序列）
|
||||
if (pendingVideoMeta && pendingVideoFrames.length > 0) {
|
||||
// 有视频：构建帧序列描述
|
||||
const frameList = pendingVideoFrames
|
||||
.map(f => ` ${String(f.timestampSeconds).padStart(4, ' ')}s — ${f.name}`)
|
||||
.join('\n');
|
||||
imgDesc = `[视频帧序列 · ${pendingVideoFrames.length}帧 · 1fps · ${pendingVideoMeta.duration.toFixed(0)}s]\n${frameList}\n\n上述帧按时间顺序排列，帧间存在时序关系。请分析视频内容。`;
|
||||
} else if (pendingImages.length === 1) {
|
||||
imgDesc = `[上传了图片: ${pendingImages[0].name}]`;
|
||||
} else if (pendingImages.length > 0) {
|
||||
imgDesc = `[上传了 ${pendingImages.length} 张图片]`;
|
||||
} else {
|
||||
imgDesc = '';
|
||||
apiContentForModel = (text || '') + `\n\n[视频帧序列 · ${pendingVideoFrames.length}帧 · 1fps · ${pendingVideoMeta.duration.toFixed(0)}s]\n${frameList}\n\n上述帧按时间顺序排列，帧间存在时序关系。请分析视频内容。`;
|
||||
if (pendingImages.length > 0) {
|
||||
apiContentForModel += `\n\n[同时上传了 ${pendingImages.length} 张图片]`;
|
||||
}
|
||||
} else if (displayContent) {
|
||||
apiContentForModel = displayContent;
|
||||
}
|
||||
|
||||
const msg: ChatMessage = {
|
||||
role: 'user',
|
||||
content: text ? `${text}\n\n${imgDesc}` : imgDesc,
|
||||
images,
|
||||
content: displayContent,
|
||||
images: [...pendingImages.map(img => img.base64)],
|
||||
timestamp: now
|
||||
};
|
||||
if (userFiles.length > 0) {
|
||||
msg.files = userFiles;
|
||||
msg._fileContents = pendingFiles.map(f => ({ language: f.language, content: f.content }));
|
||||
if (pendingVideoFrames.length > 0 && pendingVideoMeta) {
|
||||
msg._videoFrames = [...pendingVideoFrames];
|
||||
msg._videoMeta = { ...pendingVideoMeta, frameCount: pendingVideoFrames.length };
|
||||
}
|
||||
msgsToAdd.push(msg);
|
||||
} else if (text || pendingFiles.length > 0) {
|
||||
const msg: ChatMessage = {
|
||||
role: 'user',
|
||||
content: text || '',
|
||||
timestamp: now
|
||||
};
|
||||
if (userFiles.length > 0) {
|
||||
msg.files = userFiles;
|
||||
msg._fileContents = pendingFiles.map(f => ({ language: f.language, content: f.content }));
|
||||
@@ -1046,7 +1052,8 @@ async function sendMessageWithAgentLoop(text: string, currentSession: ChatSessio
|
||||
return {
|
||||
role: m.role,
|
||||
content,
|
||||
...(m.images?.length && { images: m.images })
|
||||
...(m.images?.length && { images: m.images }),
|
||||
...((m as any)._videoFrames?.length && { images: [...(m.images || []), ...(m as any)._videoFrames.map((f: any) => f.base64)] })
|
||||
};
|
||||
});
|
||||
|
||||
@@ -1066,7 +1073,7 @@ async function sendMessageWithAgentLoop(text: string, currentSession: ChatSessio
|
||||
}, 2000);
|
||||
|
||||
try {
|
||||
await runAgentLoop(text || (pendingFiles.length > 0 ? `请分析 ${pendingFiles.map(f => f.name).join(', ')}` : ''), images, historyMessages, {
|
||||
await runAgentLoop(apiContentForModel || (pendingFiles.length > 0 ? `请分析 ${pendingFiles.map(f => f.name).join(', ')}` : ''), images, historyMessages, {
|
||||
/** 模型正在生成 tool_call 参数中 — 提前在工作空间显示"准备中"卡片 */
|
||||
onToolCallPrepare: (call) => {
|
||||
addToolCard({
|
||||
|
||||
Vendored
+3
@@ -111,6 +111,9 @@ export interface ChatMessage {
|
||||
images?: string[];
|
||||
files?: ChatFile[];
|
||||
_fileContents?: FileContent[];
|
||||
/** 视频帧数据（用于消息卡片渲染，不走 images 缩略图） */
|
||||
_videoFrames?: Array<{ name: string; base64: string; timestampSeconds: number }>;
|
||||
_videoMeta?: { fileName: string; duration: number; frameCount: number };
|
||||
stopped?: boolean;
|
||||
toolCalls?: ToolCallRecord[];
|
||||
/** 标记此消息为 LLM 压缩摘要生成 */
|
||||
|
||||
Reference in New Issue
Block a user