feat: v0.5.4 多模态增强 — 多轮图片记忆 + 多模态总开关 + DeepSeek vision 模型支持
CI / 类型检查 + Lint + 单元测试 (push) Failing after 5m47s
CI / 全量测试 (Electron ABI) (push) Failing after 5m25s
CI / 产物编译验证 (push) Successful in 10m4s

多轮图片记忆:
- 此前历史轮次的图片不回传 LLM(attachments 仅存压缩 preview,历史组装
  时被丢弃)— 跨轮对话中模型对图片内容"失忆"
- 修复:SessionSummaryService.buildHistoryMessages 从持久化的 attachments
  恢复 images(type=image 的 preview base64),历史图片随上下文回传
- Token 控制:最多注入最近 10 张(MAX_HISTORY_IMAGES,从最新消息向前
  收集)— 每张 1024px 压缩图约数百至千余 token,无上限会吃满上下文
- 摘要区间(summarizedUntilRowid 之前)的图片不恢复,符合滚动摘要语义

多模态总开关 llm.multimodalEnabled(默认关闭):
- 新配置项:CONFIG_DEFAULTS 种子 + 设置弹框 LLM 配置 Switch +
  首次引导向导 LLM 步骤 Switch(含说明文案)
- 上传入口双重判断:总开关 × 模型能力 — 未开启时即使模型支持多模态
  也不能上传图片(ChatInput 的选择/拖拽/粘贴统一拦截,Toast 区分
  "开关未开启"与"当前模型不支持"两种原因)
- 保存成功后同步 Agent Store 立即生效;App 启动时随 setProvider 加载

DeepSeek vision 模型支持:
- 新增 deepseek-v4-flash-vision-exp(OpenAI image_url content parts 格式,
  128K 上下文 / 8K 输出)
- adapter 按 isVisionModel() 判断:vision 模型将带 images 的消息转换为
  [{type:'text'},{type:'image_url'}] parts;非 vision 模型保持 images
  静默丢弃(防 API 400)

测试(236 → 243 用例):
- 多轮图片记忆 ×3(session-summary.test.ts):历史 attachments 恢复
  images / 上限 10 张从最新向前 / 摘要区间图片不恢复
- DeepSeek vision 请求格式 ×4(deepseek-vision.test.ts,契约级 mock
  fetch 断言请求体):image_url parts 转换 / 非 vision 模型丢弃 /
  max_tokens 钳制 8192 / 无图不转换
- 测试顺序修正:多轮图片用例置于 describe 末尾(插入新行消耗全局自增
  rowid,插在中间会破坏既有用例对 rowid 数值的断言)

文档: README 同步(DeepSeek 模型表 + vision 多模态列、llm.multimodalEnabled
配置项、多轮图片记忆特性行、243 用例数)

验证: lint 0 / typecheck 双工程 0 / test:electron 243 全过 / build 成功
This commit is contained in:
2026-08-21 23:00:20 +08:00
parent 3a30e8f5b4
commit 4ee5100661
12 changed files with 1363 additions and 353 deletions
+269 -58
View File
@@ -3,7 +3,26 @@
*/
import { useState, useEffect } from 'react';
import { Dialog, DialogContent, Button, TextField, Select, MenuItem, Stepper, Step, StepLabel, Box, Typography, Stack, FormControl, InputLabel, IconButton, InputAdornment } from '@mui/material';
import {
Dialog,
DialogContent,
Button,
TextField,
Select,
MenuItem,
Stepper,
Step,
StepLabel,
Box,
Typography,
Stack,
FormControl,
InputLabel,
IconButton,
InputAdornment,
FormControlLabel,
Switch,
} from '@mui/material';
import { ArrowRight, ArrowLeft, CheckCircle, Eye, EyeOff } from 'lucide-react';
import { useUIStore } from '@renderer/stores/ui-store';
import { useAgentStore } from '@renderer/stores/agent-store';
@@ -18,6 +37,8 @@ export function OnboardingWizard(): React.JSX.Element | null {
const setOnboardingCompleted = useUIStore((s) => s.setOnboardingCompleted);
const [step, setStep] = useState(0);
const [provider, setProvider] = useState('');
// v0.5.4: 多模态总开关(保存到 llm.multimodalEnabled,控制图片上传入口)
const [multimodalEnabled, setMultimodalEnabled] = useState(false);
const [baseURL, setBaseURL] = useState('');
const [model, setModel] = useState('');
const [apiKey, setApiKey] = useState('');
@@ -33,8 +54,11 @@ export function OnboardingWizard(): React.JSX.Element | null {
const saved = localStorage.getItem(ONBOARDING_PROGRESS_KEY);
if (!saved) return;
const p = JSON.parse(saved) as {
step?: number; provider?: string; baseURL?: string;
model?: string; workspacePath?: string;
step?: number;
provider?: string;
baseURL?: string;
model?: string;
workspacePath?: string;
contextWindow?: number | null;
};
if (typeof p.step === 'number' && p.step >= 0 && p.step < STEPS.length) setStep(p.step);
@@ -58,9 +82,17 @@ export function OnboardingWizard(): React.JSX.Element | null {
// 注意: 不保存 apiKey(敏感信息不写入 localStorage
useEffect(() => {
try {
localStorage.setItem(ONBOARDING_PROGRESS_KEY, JSON.stringify({
step, provider, baseURL, model, workspacePath, contextWindow,
}));
localStorage.setItem(
ONBOARDING_PROGRESS_KEY,
JSON.stringify({
step,
provider,
baseURL,
model,
workspacePath,
contextWindow,
}),
);
} catch {
// 写入失败(如隐私模式)忽略
}
@@ -84,7 +116,10 @@ export function OnboardingWizard(): React.JSX.Element | null {
contextWindow != null && (!Number.isFinite(contextWindow) || contextWindow < ctxMin);
const handleNext = async () => {
if (step < STEPS.length - 1) { setStep(step + 1); return; }
if (step < STEPS.length - 1) {
setStep(step + 1);
return;
}
try {
if (window.metona?.config?.setBatch) {
// v0.3.9: 改用批量保存,避免并行 config.set 中间态触发 reloadAdapter 失败
@@ -98,7 +133,10 @@ export function OnboardingWizard(): React.JSX.Element | null {
if (baseURL.trim()) entries.push({ key: 'llm.baseURL', value: baseURL.trim() });
if (model.trim()) entries.push({ key: 'llm.model', value: model.trim() });
if (apiKey.trim()) entries.push({ key: 'llm.apiKey', value: apiKey.trim() });
if (workspacePath.trim()) entries.push({ key: 'workspace.path', value: workspacePath.trim() });
// v0.5.4: 多模态总开关
entries.push({ key: 'llm.multimodalEnabled', value: multimodalEnabled });
if (workspacePath.trim())
entries.push({ key: 'workspace.path', value: workspacePath.trim() });
// 上下文窗口:根据 Provider 落库到对应 key
// - ollama: ollama.numCtx(允许 null=由模型决定)
// - deepseek/agnes/mimo: {provider}.contextWindow(必须有值且 >= 4096
@@ -112,10 +150,14 @@ export function OnboardingWizard(): React.JSX.Element | null {
const r = await window.metona.config.setBatch(entries);
if (r && !r.success) {
console.error('[OnboardingWizard]', 'Batch config save failed:', r.error);
import('@metona-team/metona-toast').then((mod) => mod.default.error(r.error ?? '配置保存失败,请重试')).catch(() => {});
import('@metona-team/metona-toast')
.then((mod) => mod.default.error(r.error ?? '配置保存失败,请重试'))
.catch(() => {});
return;
}
useAgentStore.getState().setProvider(provider.trim() || 'deepseek', model.trim() || '');
// v0.5.4: 多模态开关立即同步(setProvider 内部会从配置异步加载,此处确保即时生效)
useAgentStore.getState().setMultimodalEnabled(multimodalEnabled);
// 同步 contextWindow 到 Agent Store(与 SettingsModal 行为一致)
if (contextWindow != null && contextWindow >= ctxMin) {
useAgentStore.setState({ contextWindow });
@@ -125,42 +167,86 @@ export function OnboardingWizard(): React.JSX.Element | null {
}
setOnboardingCompleted(true);
// #48 修复: 引导完成后清理 localStorage 进度,下次启动不再恢复
try { localStorage.removeItem(ONBOARDING_PROGRESS_KEY); } catch { /* ignore */ }
try {
localStorage.removeItem(ONBOARDING_PROGRESS_KEY);
} catch {
/* ignore */
}
} catch (err) {
console.error('[OnboardingWizard]', 'Failed to save configuration:', err);
// 用户主动操作失败必须有反馈,否则向导不关闭、用户卡死
import('@metona-team/metona-toast').then((mod) => mod.default.error(`保存配置失败:${(err as Error).message}`)).catch(() => {});
import('@metona-team/metona-toast')
.then((mod) => mod.default.error(`保存配置失败:${(err as Error).message}`))
.catch(() => {});
}
};
return (
<Dialog open maxWidth="sm" slotProps={{ paper: { sx: { borderRadius: 3, overflow: 'hidden' } } }}>
<Dialog
open
maxWidth="sm"
slotProps={{ paper: { sx: { borderRadius: 3, overflow: 'hidden' } } }}
>
<Box sx={{ px: 3, pt: 2, pb: 0 }}>
<Stepper activeStep={step} alternativeLabel sx={{ '& .MuiStepLabel-label': { fontSize: 11 } }}>
{STEPS.map((s) => <Step key={s}><StepLabel>{s}</StepLabel></Step>)}
<Stepper
activeStep={step}
alternativeLabel
sx={{ '& .MuiStepLabel-label': { fontSize: 11 } }}
>
{STEPS.map((s) => (
<Step key={s}>
<StepLabel>{s}</StepLabel>
</Step>
))}
</Stepper>
</Box>
<DialogContent sx={{ minHeight: 280, display: 'flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center' }}>
<DialogContent
sx={{
minHeight: 280,
display: 'flex',
flexDirection: 'column',
alignItems: 'center',
justifyContent: 'center',
}}
>
{step === 0 && (
<Box sx={{ textAlign: 'center' }}>
<Box component="img" src="./logo.png" alt="Metona" sx={{ width: 64, height: 64, mx: 'auto', mb: 2, borderRadius: 2 }} />
<Typography variant="h6" sx={{ mb: 1 }}>使 MetonaAI Desktop</Typography>
<Typography variant="body2" sx={{ color: 'text.secondary', mb: 2 }}> AI Agent MCP </Typography>
<Box
component="img"
src="./logo.png"
alt="Metona"
sx={{ width: 64, height: 64, mx: 'auto', mb: 2, borderRadius: 2 }}
/>
<Typography variant="h6" sx={{ mb: 1 }}>
使 MetonaAI Desktop
</Typography>
<Typography variant="body2" sx={{ color: 'text.secondary', mb: 2 }}>
AI Agent MCP
</Typography>
<Typography variant="caption"> 1 </Typography>
</Box>
)}
{step === 1 && (
<Box sx={{ width: '100%' }}>
<Typography variant="h6" sx={{ mb: 2 }}> LLM Provider</Typography>
<Typography variant="body2" sx={{ color: 'text.secondary', mb: 2 }}> Provider API Base URL </Typography>
<Typography variant="h6" sx={{ mb: 2 }}>
LLM Provider
</Typography>
<Typography variant="body2" sx={{ color: 'text.secondary', mb: 2 }}>
Provider API Base URL
</Typography>
<Stack spacing={2}>
<FormControl size="small"><InputLabel>Provider</InputLabel>
<Select value={provider} label="Provider" onChange={(e) => {
const v = e.target.value;
setProvider(v);
// 联动默认上下文窗口(与 SettingsModal 默认值一致)
setContextWindow(DEFAULT_CTX[v] ?? null);
}}>
<FormControl size="small">
<InputLabel>Provider</InputLabel>
<Select
value={provider}
label="Provider"
onChange={(e) => {
const v = e.target.value;
setProvider(v);
// 联动默认上下文窗口(与 SettingsModal 默认值一致)
setContextWindow(DEFAULT_CTX[v] ?? null);
}}
>
<MenuItem value="deepseek">DeepSeek</MenuItem>
<MenuItem value="agnes">Agnes AI</MenuItem>
<MenuItem value="mimo">MiMo ()</MenuItem>
@@ -169,71 +255,196 @@ export function OnboardingWizard(): React.JSX.Element | null {
<MenuItem value="anthropic">Anthropic</MenuItem>
</Select>
</FormControl>
<TextField size="small" label="API Base URL" value={baseURL} onChange={(e) => setBaseURL(e.target.value)} placeholder="如 https://api.deepseek.com" />
<TextField size="small" label="模型名称" value={model} onChange={(e) => setModel(e.target.value)} placeholder="如 deepseek-v4-pro、qwen3:latest" />
<TextField size="small" label="API Key" type={showKey ? 'text' : 'password'} value={apiKey} onChange={(e) => setApiKey(e.target.value)} placeholder="sk-...(本地模型可留空)"
slotProps={{ input: { endAdornment: <InputAdornment position="end"><IconButton size="small" onClick={() => setShowKey(!showKey)}>{showKey ? <EyeOff size={14} /> : <Eye size={14} />}</IconButton></InputAdornment> } }}
<TextField
size="small"
label="API Base URL"
value={baseURL}
onChange={(e) => setBaseURL(e.target.value)}
placeholder="如 https://api.deepseek.com"
/>
<TextField
size="small"
label={provider === 'ollama' ? '上下文长度 (num_ctx)' : '上下文窗口 (contextWindow)'}
label="模型名称"
value={model}
onChange={(e) => setModel(e.target.value)}
placeholder="如 deepseek-v4-pro、qwen3:latest"
/>
<TextField
size="small"
label="API Key"
type={showKey ? 'text' : 'password'}
value={apiKey}
onChange={(e) => setApiKey(e.target.value)}
placeholder="sk-...(本地模型可留空)"
slotProps={{
input: {
endAdornment: (
<InputAdornment position="end">
<IconButton size="small" onClick={() => setShowKey(!showKey)}>
{showKey ? <EyeOff size={14} /> : <Eye size={14} />}
</IconButton>
</InputAdornment>
),
},
}}
/>
<TextField
size="small"
label={
provider === 'ollama' ? '上下文长度 (num_ctx)' : '上下文窗口 (contextWindow)'
}
type="number"
value={contextWindow ?? ''}
onChange={(e) => {
const v = e.target.value;
setContextWindow(v === '' ? null : Number(v));
}}
placeholder={provider === 'ollama' ? '默认由模型决定(如 2048、4096、128000' : '如 64000、128000、1000000'}
placeholder={
provider === 'ollama'
? '默认由模型决定(如 2048、4096、128000'
: '如 64000、128000、1000000'
}
slotProps={{ htmlInput: { min: ctxMin, step: ctxMin } }}
error={ctxError}
helperText={ctxError ? `最小值为 ${ctxMin}` : (provider === 'ollama' ? ' ' : '用于上下文压缩判断,不传给 API')}
helperText={
ctxError
? `最小值为 ${ctxMin}`
: provider === 'ollama'
? ' '
: '用于上下文压缩判断,不传给 API'
}
/>
{/* v0.5.4: 多模态总开关 — 未开启时即使模型支持也不能上传图片 */}
<FormControlLabel
control={
<Switch
size="small"
checked={multimodalEnabled}
onChange={(e) => setMultimodalEnabled(e.target.checked)}
/>
}
label={
<Stack>
<Typography variant="body2"></Typography>
<Typography variant="caption" sx={{ color: 'text.disabled' }}>
DeepSeek vision
</Typography>
</Stack>
}
sx={{ alignItems: 'flex-start', m: 0 }}
/>
</Stack>
</Box>
)}
{step === 2 && (
<Box sx={{ width: '100%' }}>
<Typography variant="h6" sx={{ mb: 2 }}> Agent</Typography>
<Typography variant="body2" sx={{ color: 'text.secondary', mb: 2 }}> SOUL.md Agent </Typography>
<Box sx={{ px: 2, py: 1.5, borderRadius: 2, bgcolor: 'action.hover', border: '1px solid', borderColor: 'divider', fontSize: 12, color: 'text.secondary' }}>
<div><strong style={{ color: '#e1e4ed' }}>SOUL.md</strong> Agent </div>
<div style={{ marginTop: 8, fontSize: 11, opacity: 0.7 }}></div>
<Typography variant="h6" sx={{ mb: 2 }}>
Agent
</Typography>
<Typography variant="body2" sx={{ color: 'text.secondary', mb: 2 }}>
SOUL.md Agent
</Typography>
<Box
sx={{
px: 2,
py: 1.5,
borderRadius: 2,
bgcolor: 'action.hover',
border: '1px solid',
borderColor: 'divider',
fontSize: 12,
color: 'text.secondary',
}}
>
<div>
<strong style={{ color: '#e1e4ed' }}>SOUL.md</strong> Agent
</div>
<div style={{ marginTop: 8, fontSize: 11, opacity: 0.7 }}>
</div>
</Box>
</Box>
)}
{step === 3 && (
<Box sx={{ width: '100%' }}>
<Typography variant="h6" sx={{ mb: 2 }}></Typography>
<Typography variant="body2" sx={{ color: 'text.secondary', mb: 2 }}>使</Typography>
<Typography variant="h6" sx={{ mb: 2 }}>
</Typography>
<Typography variant="body2" sx={{ color: 'text.secondary', mb: 2 }}>
使
</Typography>
<Stack direction="row" spacing={1} sx={{ mb: 2, alignItems: 'center' }}>
<TextField size="small" value={workspacePath} onChange={(e) => setWorkspacePath(e.target.value)} placeholder="~/MetonaWorkspaces/default/" sx={{ flex: 1 }} />
<Button variant="outlined" size="small" onClick={async () => {
if (window.metona?.app?.selectFolder) {
try {
const r = await window.metona.app.selectFolder(workspacePath || undefined);
if (!r.canceled && r.path) setWorkspacePath(r.path);
} catch (err) {
console.error('[OnboardingWizard]', err);
import('@metona-team/metona-toast').then((mod) => mod.default.error('选择文件夹失败')).catch(() => {});
<TextField
size="small"
value={workspacePath}
onChange={(e) => setWorkspacePath(e.target.value)}
placeholder="~/MetonaWorkspaces/default/"
sx={{ flex: 1 }}
/>
<Button
variant="outlined"
size="small"
onClick={async () => {
if (window.metona?.app?.selectFolder) {
try {
const r = await window.metona.app.selectFolder(workspacePath || undefined);
if (!r.canceled && r.path) setWorkspacePath(r.path);
} catch (err) {
console.error('[OnboardingWizard]', err);
import('@metona-team/metona-toast')
.then((mod) => mod.default.error('选择文件夹失败'))
.catch(() => {});
}
}
}
}}></Button>
}}
>
</Button>
</Stack>
<Typography variant="caption" sx={{ color: 'text.secondary' }}> SOUL.mdMEMORY.md </Typography>
<Typography variant="caption" sx={{ color: 'text.secondary' }}>
SOUL.mdMEMORY.md
</Typography>
</Box>
)}
{step === 4 && (
<Box sx={{ textAlign: 'center' }}>
<CheckCircle size={48} style={{ color: '#34d399', margin: '0 auto 16px' }} />
<Typography variant="h6" sx={{ mb: 1 }}></Typography>
<Typography variant="body2" sx={{ color: 'text.secondary', mb: 1 }}>MetonaAI Desktop AI Agent </Typography>
<Typography variant="h6" sx={{ mb: 1 }}>
</Typography>
<Typography variant="body2" sx={{ color: 'text.secondary', mb: 1 }}>
MetonaAI Desktop AI Agent
</Typography>
<Typography variant="caption"> Ctrl+Enter / </Typography>
</Box>
)}
</DialogContent>
<Box sx={{ display: 'flex', justifyContent: 'space-between', px: 3, py: 1.5, borderTop: 1, borderColor: 'divider' }}>
<Button startIcon={<ArrowLeft size={12} />} onClick={() => setStep(step - 1)} disabled={step === 0} size="small" sx={{ color: 'text.secondary' }}></Button>
<Button variant="contained" endIcon={step < STEPS.length - 1 ? <ArrowRight size={12} /> : undefined} onClick={handleNext} size="small">
<Box
sx={{
display: 'flex',
justifyContent: 'space-between',
px: 3,
py: 1.5,
borderTop: 1,
borderColor: 'divider',
}}
>
<Button
startIcon={<ArrowLeft size={12} />}
onClick={() => setStep(step - 1)}
disabled={step === 0}
size="small"
sx={{ color: 'text.secondary' }}
>
</Button>
<Button
variant="contained"
endIcon={step < STEPS.length - 1 ? <ArrowRight size={12} /> : undefined}
onClick={handleNext}
size="small"
>
{step === STEPS.length - 1 ? '开始使用' : '下一步'}
</Button>
</Box>