From 516d8a728fe3b0900f316716a027d779ac52c719 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=B4=AB=E5=BD=B1233?= <1440196015@qq.com> Date: Tue, 21 Jul 2026 21:05:50 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=8D=87=E7=BA=A7=E8=87=B3=20v0.3.15?= =?UTF-8?q?=20=E2=80=94=20MiMo=20=E5=A4=9A=E6=A8=A1=E6=80=81=20+=20?= =?UTF-8?q?=E5=B7=A5=E4=BD=9C=E7=A9=BA=E9=97=B4=E6=95=B0=E6=8D=AE=E5=BA=93?= =?UTF-8?q?=E7=BB=A7=E6=89=BF=20+=20=E5=BC=95=E5=AF=BC=E7=AA=97=E5=8F=A3?= =?UTF-8?q?=E4=B8=8A=E4=B8=8B=E6=96=87=E9=85=8D=E7=BD=AE=20+=20TS=20?= =?UTF-8?q?=E9=94=99=E8=AF=AF=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - feat(mimo): 适配器支持图片输入，将 images 转为 OpenAI 兼容 content parts 数组 - feat(workspace): 切换工作空间时支持继承数据库 agent.db（SQLite backup API 原子导出，自动 checkpoint WAL） - feat(onboarding): 引导弹窗 LLM 配置加上下文长度字段（DeepSeek/Agnes=1000000、MiMo=131072、Ollama=null） - fix(types): 修复 16 个 TypeScript 编译错误（setConfigLoaded 接口声明、10 个 IPC 返回类型加 error 字段、MUI TextField readOnly slot 迁移） - chore(workspace): 移除死代码 traces 目录（trace 数据实际存储于数据库 sessions.metadata 字段，traces 目录从未被任何代码读写） - docs(mimo): API 文档补充多模态输入章节和 curl/Python 代码示例 - chore: 版本号 0.3.14 → 0.3.15 --- README.md | 2 +- apis/mimo-api-docs-20260715.html | 145 ++++++++++++++++++ electron/harness/adapters/mimo.adapter.ts | 27 ++++ electron/ipc/handlers.ts | 71 +++++++-- electron/services/workspace.service.ts | 4 +- package-lock.json | 4 +- package.json | 2 +- .../onboarding/OnboardingWizard.tsx | 48 +++++- src/components/settings/SettingsModal.tsx | 22 ++- src/components/workspace/WorkspaceViewer.tsx | 4 +- src/stores/agent-store.ts | 2 + src/types/global.d.ts | 20 +-- 12 files changed, 318 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 90e9489..9da9074 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ > 生产级通用 AI Agent 智能体桌面应用 -[![Version](https://img.shields.io/badge/version-0.3.14-blue)](./package.json) +[![Version](https://img.shields.io/badge/version-0.3.15-blue)](./package.json) [![License](https://img.shields.io/badge/license-MIT-green)](./LICENSE) [![Electron](https://img.shields.io/badge/Electron-35-47848F)](https://www.electronjs.org/) [![React](https://img.shields.io/badge/React-19-61DAFB)](https://react.dev/) diff --git a/apis/mimo-api-docs-20260715.html b/apis/mimo-api-docs-20260715.html index 14b5f3a..55d883d 100644 --- a/apis/mimo-api-docs-20260715.html +++ b/apis/mimo-api-docs-20260715.html @@ -132,6 +132,7 @@ 🔧 工具调用 (Function Calling) 🌐 联网搜索 🎙️ 语音合成 (TTS) + 🖼️ 多模态输入 💻 代码示例 ⚠️ 错误码说明 @@ -331,6 +332,56 @@ Content-Type: application/json + +
+

🖼️ 多模态输入

+

当 messages[].content 为数组时，可混合传入文本、图像内容。采用 OpenAI 兼容的 content parts 数组格式。

+ +

content parts 字段

+ + + + + + + + +
字段名类型描述
contentstring | array纯文本字符串，或 content parts 数组
content[].typestringpart 类型："text" / "image_url"
content[].textstring文本内容（type 为 "text" 时必填）
content[].image_urlobject图像对象（type 为 "image_url" 时必填）
content[].image_url.urlstring图像公网 URL 或 base64 data URI（格式：data:image/png;base64,...）
content[].image_url.detailstring可选，清晰度："low" / "high" / "auto"（默认 "auto"）
+ +
+ 📌 支持的图像格式： + +
+ +

多模态消息示例

+ 
json{
+  "role": "user",
+  "content": [
+    { "type": "text", "text": "请描述这张图片的内容" },
+    {
+      "type": "image_url",
+      "image_url": {
+        "url": "https://example.com/image.jpg",
+        "detail": "auto"
+      }
+    }
+  ]
+}
+ +
+ ⚠️ 多模态限制： + +
+
+

📤 响应对象（非流式输出）

@@ -746,6 +797,100 @@ audio_bytes = base64.b64decode(audio_data) with output("output.wav", "wb") as f: f.write(audio_bytes) print("音频已保存到 output.wav") + +

多模态图像输入

+
+ + + +
+ +
+
bashcurl --location --request POST 'https://api.xiaomimimo.com/v1/chat/completions' \
+--header "api-key: $MIMO_API_KEY" \
+--header "Content-Type: application/json" \
+--data-raw '{
+    "model": "mimo-v2.5-pro",
+    "messages": [
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "请描述这张图片的内容"},
+                {
+                    "type": "image_url",
+                    "image_url": {
+                        "url": "https://example.com/image.jpg",
+                        "detail": "auto"
+                    }
+                }
+            ]
+        }
+    ],
+    "max_completion_tokens": 1024,
+    "thinking": {"type": "disabled"}
+}'
+
+ +
+
pythonfrom openai import OpenAI
+
+client = OpenAI(api_key="$MIMO_API_KEY", base_url="https://api.xiaomimimo.com/v1")
+
+response = client.chat.completions.create(
+    model="mimo-v2.5-pro",
+    messages=[
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "请描述这张图片的内容"},
+                {
+                    "type": "image_url",
+                    "image_url": {
+                        "url": "https://example.com/image.jpg",
+                        "detail": "auto"
+                    }
+                }
+            ]
+        }
+    ],
+    max_completion_tokens=1024,
+    thinking={"type": "disabled"}
+)
+
+print(response.choices[0].message.content)
+print(f"图像 token: {response.usage.prompt_tokens_details.image_tokens}")
+
+ +
+
pythonimport base64
+from openai import OpenAI
+
+client = OpenAI(api_key="$MIMO_API_KEY", base_url="https://api.xiaomimimo.com/v1")
+
+# 读取本地图片并转为 base64 data URI
+with open("local_image.jpg", "rb") as f:
+    image_data = base64.b64encode(f.read()).decode("utf-8")
+image_data_uri = f"data:image/jpeg;base64,{image_data}"
+
+response = client.chat.completions.create(
+    model="mimo-v2.5-pro",
+    messages=[
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "这张图里有什么？"},
+                {
+                    "type": "image_url",
+                    "image_url": {"url": image_data_uri}
+                }
+            ]
+        }
+    ],
+    max_completion_tokens=1024
+)
+
+print(response.choices[0].message.content)
+
diff --git a/electron/harness/adapters/mimo.adapter.ts b/electron/harness/adapters/mimo.adapter.ts index d7089de..83734a3 100644 --- a/electron/harness/adapters/mimo.adapter.ts +++ b/electron/harness/adapters/mimo.adapter.ts @@ -152,6 +152,7 @@ export class MimoAdapter extends BaseAdapter { * 构建 MiMo 原生请求体 * * MiMo 特有参数： + * - 多模态图片：user 消息的 images[] → OpenAI content 数组 [{type:"text"}, {type:"image_url"}] * - thinking: { type: "enabled" / "disabled" } — 与 DeepSeek 一致 * - max_completion_tokens — 非 max_tokens（MiMo 使用新字段名） * - stream_options: { include_usage: true } — 流式返回 usage @@ -163,6 +164,32 @@ export class MimoAdapter extends BaseAdapter { const messages = buildOpenAICompatibleMessages(request); const tools = buildOpenAICompatibleTools(request.tools); + // === MiMo 多模态：将 images 转为 OpenAI content 数组 === + // buildOpenAICompatibleMessages 不处理图片（各 Provider 自行处理） + // MiMo 是 OpenAI 兼容 API，多模态格式与 Agnes AI 一致 + const nonSystemMsgs = request.messages.filter((m) => m.role !== 'system'); + let imageCount = 0; + for (let i = 0; i < messages.length; i++) { + // messages[0] 是 system，非 system 消息从 messages[1] 开始 + if (i === 0) continue; + const origMsg = nonSystemMsgs[i - 1]; + if (!origMsg?.images?.length) continue; + + imageCount += origMsg.images.length; + + const contentParts: Array> = []; + if (origMsg.content) { + contentParts.push({ type: 'text', text: origMsg.content }); + } + for (const img of origMsg.images) { + contentParts.push({ + type: 'image_url', + image_url: { url: img.url }, + }); + } + messages[i].content = contentParts; + } + const body: Record = { model: this.config.defaultModel, messages, diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 67c88fa..1d76039 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -1132,25 +1132,74 @@ export function registerAllIPCHandlers( mkdirSync(resolvedTarget, { recursive: true }); } + // 继承白名单：键为前端传入的标识，值为实际相对路径分段 + // - SOUL.md: 工作空间根目录的 Agent 身份定义 + // - .metona/agent.db: 工作空间内的 SQLite 数据库（会话/消息/记忆/Trace） + // 白名单映射防止路径遍历攻击（不直接使用用户传入的路径拼接到 fs 调用） + const INHERIT_WHITELIST: Record = { + 'SOUL.md': ['SOUL.md'], + '.metona/agent.db': ['.metona', 'agent.db'], + }; + const inherited: string[] = []; const failed: Array<{ file: string; error: string }> = []; for (const fileName of files) { - // 仅允许继承白名单文件（防止路径遍历）— v0.3.14: 仅 SOUL.md - if (!['SOUL.md'].includes(fileName)) { + const pathSegments = INHERIT_WHITELIST[fileName]; + // 不在白名单中：拒绝（防止路径遍历） + if (!pathSegments) { failed.push({ file: fileName, error: '不在继承白名单中' }); continue; } - const srcFile = join(resolvedSource, fileName); - const dstFile = join(resolvedTarget, fileName); + const srcFile = join(resolvedSource, ...pathSegments); + const dstFile = join(resolvedTarget, ...pathSegments); try { - if (existsSync(srcFile)) { - copyFileSync(srcFile, dstFile); - inherited.push(fileName); - log.info(`[WORKSPACE] Inherited ${fileName}: ${resolvedSource} → ${resolvedTarget}`); - } else { - failed.push({ file: fileName, error: '源文件不存在' }); + // 确保目标文件的父目录存在（如 .metona/） + const dstDir = join(resolvedTarget, ...pathSegments.slice(0, -1)); + if (!existsSync(dstDir)) { + mkdirSync(dstDir, { recursive: true }); } + if (!existsSync(srcFile)) { + failed.push({ file: fileName, error: '源文件不存在' }); + continue; + } + + // === 数据库文件特殊处理 === + // agent.db 启用了 WAL 模式（database.service.ts:53），直接 copyFileSync 存在两个问题： + // 1. WAL 中未 checkpoint 的事务会丢失（最近会话/消息可能没了） + // 2. 主进程可能仍持有 db 句柄，Windows 上文件锁定会导致复制失败或不一致 + // 解决方案：用 better-sqlite3 的 backup API（自带 checkpoint + 一致性快照） + if (fileName === '.metona/agent.db') { + try { + const Database = (await import('better-sqlite3')).default; + // 以只读方式打开源库（如果主进程已持有句柄，better-sqlite3 会共享锁） + const srcDb = new Database(srcFile, { readonly: true, fileMustExist: true }); + try { + // backup API 原子性导出完整快照，自动合并 WAL 内容 + srcDb.backup(dstFile); + inherited.push(fileName); + log.info(`[WORKSPACE] Inherited ${fileName} (via SQLite backup): ${resolvedSource} → ${resolvedTarget}`); + } finally { + srcDb.close(); + } + } catch (err) { + // backup 失败时回退到 copyFileSync（至少保证基本可用） + log.warn(`[WORKSPACE] SQLite backup failed, fallback to copyFileSync: ${(err as Error).message}`); + try { + copyFileSync(srcFile, dstFile); + inherited.push(fileName); + log.info(`[WORKSPACE] Inherited ${fileName} (fallback copyFileSync): ${resolvedSource} → ${resolvedTarget}`); + } catch (err2) { + failed.push({ file: fileName, error: `backup 和 fallback 均失败: ${(err2 as Error).message}` }); + } + } + continue; + } + + // === 普通文件直接复制 === + copyFileSync(srcFile, dstFile); + inherited.push(fileName); + log.info(`[WORKSPACE] Inherited ${fileName}: ${resolvedSource} → ${resolvedTarget}`); } catch (err) { failed.push({ file: fileName, error: (err as Error).message }); } @@ -1166,7 +1215,7 @@ export function registerAllIPCHandlers( const files = workspaceService.reload(); // 同步外部可能的手动修改 const REQUIRED = ['SOUL.md', 'MEMORY.md'] as const; - const AUTO_DIRS = ['logs', 'traces', '.metona'] as const; + const AUTO_DIRS = ['logs', '.metona'] as const; const fileInfos = REQUIRED.map((name) => { const filePath = join(workspacePath, name); diff --git a/electron/services/workspace.service.ts b/electron/services/workspace.service.ts index a05b9af..7dc8b76 100644 --- a/electron/services/workspace.service.ts +++ b/electron/services/workspace.service.ts @@ -35,7 +35,7 @@ export interface WorkspaceInfo { const REQUIRED_FILES = ['SOUL.md', 'MEMORY.md'] as const; -const AUTO_CREATED_DIRS = ['logs', 'traces', '.metona'] as const; +const AUTO_CREATED_DIRS = ['logs', '.metona'] as const; const MEMORY_TEMPLATE = `# MEMORY.md — AI 持久记忆 @@ -77,7 +77,7 @@ export class WorkspaceService { // 创建工作空间目录 this.ensureDirectory(this.workspacePath); - // 创建自动目录（logs/, traces/, .metona/） + // 创建自动目录（logs/, .metona/） for (const dir of AUTO_CREATED_DIRS) { this.ensureDirectory(join(this.workspacePath, dir)); } diff --git a/package-lock.json b/package-lock.json index 43b8af0..7f378a5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "metona-ai-desktop", - "version": "0.3.14", + "version": "0.3.15", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "metona-ai-desktop", - "version": "0.3.14", + "version": "0.3.15", "license": "MIT", "dependencies": { "@emotion/react": "^11.14.0", diff --git a/package.json b/package.json index 8afa0e1..37bf3c9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "metona-ai-desktop", - "version": "0.3.14", + "version": "0.3.15", "description": "MetonaAI Desktop — 生产级通用 AI Agent 智能体桌面应用", "main": "dist-electron/main/main.js", "author": "Metona Team", diff --git a/src/components/onboarding/OnboardingWizard.tsx b/src/components/onboarding/OnboardingWizard.tsx index bb05a66..41c8c8d 100644 --- a/src/components/onboarding/OnboardingWizard.tsx +++ b/src/components/onboarding/OnboardingWizard.tsx @@ -20,9 +20,24 @@ export function OnboardingWizard(): React.JSX.Element | null { const [apiKey, setApiKey] = useState(''); const [showKey, setShowKey] = useState(false); const [workspacePath, setWorkspacePath] = useState(''); + // 上下文窗口（联动 Provider：ollama 可空=由模型决定；其他 provider 默认值见 DEFAULT_CTX） + const [contextWindow, setContextWindow] = useState(null); if (onboardingCompleted) return null; + // 各 Provider 上下文窗口默认值（与 SettingsModal 保持一致） + // ollama 返回 null（由模型决定），其他 provider 返回正整数 + const DEFAULT_CTX: Record = { + deepseek: 1_000_000, + agnes: 1_000_000, + mimo: 131_072, + ollama: null, + }; + // 上下文窗口校验：ollama 允许空，最小 512；其他 provider 最小 4096 + const ctxMin = provider === 'ollama' ? 512 : 4096; + const ctxError = + contextWindow != null && (!Number.isFinite(contextWindow) || contextWindow < ctxMin); + const handleNext = async () => { if (step < STEPS.length - 1) { setStep(step + 1); return; } try { @@ -39,6 +54,14 @@ export function OnboardingWizard(): React.JSX.Element | null { if (model.trim()) entries.push({ key: 'llm.model', value: model.trim() }); if (apiKey.trim()) entries.push({ key: 'llm.apiKey', value: apiKey.trim() }); if (workspacePath.trim()) entries.push({ key: 'workspace.path', value: workspacePath.trim() }); + // 上下文窗口：根据 Provider 落库到对应 key + // - ollama: ollama.numCtx（允许 null=由模型决定） + // - deepseek/agnes/mimo: {provider}.contextWindow（必须有值且 >= 4096） + if (provider === 'ollama') { + entries.push({ key: 'ollama.numCtx', value: contextWindow }); + } else if (provider && contextWindow != null && contextWindow >= 4096) { + entries.push({ key: `${provider}.contextWindow`, value: contextWindow }); + } entries.push({ key: 'onboarding.completed', value: true }); const r = await window.metona.config.setBatch(entries); @@ -48,6 +71,10 @@ export function OnboardingWizard(): React.JSX.Element | null { return; } useAgentStore.getState().setProvider(provider.trim() || 'deepseek', model.trim() || ''); + // 同步 contextWindow 到 Agent Store（与 SettingsModal 行为一致） + if (contextWindow != null && contextWindow >= ctxMin) { + useAgentStore.setState({ contextWindow }); + } // P1-6 修复: Onboarding 完成后标记配置已加载 useAgentStore.getState().setConfigLoaded(true); } @@ -81,7 +108,12 @@ export function OnboardingWizard(): React.JSX.Element | null { 选择 Provider 并填写 API 信息。Base URL 和模型名称支持任意输入。 Provider - { + const v = e.target.value; + setProvider(v); + // 联动默认上下文窗口（与 SettingsModal 默认值一致） + setContextWindow(DEFAULT_CTX[v] ?? null); + }}> DeepSeek Agnes AI MiMo (小米) @@ -93,6 +125,20 @@ export function OnboardingWizard(): React.JSX.Element | null { setApiKey(e.target.value)} placeholder="sk-...（本地模型可留空）" slotProps={{ input: { endAdornment: setShowKey(!showKey)}>{showKey ? : } } }} /> + { + const v = e.target.value; + setContextWindow(v === '' ? null : Number(v)); + }} + placeholder={provider === 'ollama' ? '默认由模型决定（如 2048、4096、128000）' : '如 64000、128000、1000000'} + slotProps={{ htmlInput: { min: ctxMin, step: ctxMin } }} + error={ctxError} + helperText={ctxError ? `最小值为 ${ctxMin}` : (provider === 'ollama' ? ' ' : '用于上下文压缩判断，不传给 API')} + /> )} diff --git a/src/components/settings/SettingsModal.tsx b/src/components/settings/SettingsModal.tsx index 47c3b23..8041034 100644 --- a/src/components/settings/SettingsModal.tsx +++ b/src/components/settings/SettingsModal.tsx @@ -116,6 +116,8 @@ function WorkspaceSettings() { const [checkResult, setCheckResult] = useState(null); const [checking, setChecking] = useState(false); const [inheritSoul, setInheritSoul] = useState(true); + // 数据库继承：默认勾选（历史会话/消息/记忆/Trace 丢失不可逆，默认带过去更安全） + const [inheritDatabase, setInheritDatabase] = useState(true); const [applying, setApplying] = useState(false); const [showRestartDialog, setShowRestartDialog] = useState(false); @@ -145,9 +147,12 @@ function WorkspaceSettings() { if (!pendingPath || !checkResult?.valid) return; setApplying(true); try { - // 如果勾选了继承文件，从旧工作空间复制到新工作空间（v0.3.14: 仅 SOUL.md） + // 如果勾选了继承文件，从旧工作空间复制到新工作空间 + // - SOUL.md: Agent 身份定义 + // - .metona/agent.db: 数据库（会话/消息/记忆/Trace），后端白名单校验 const filesToInherit: string[] = []; if (inheritSoul) filesToInherit.push('SOUL.md'); + if (inheritDatabase) filesToInherit.push('.metona/agent.db'); if (filesToInherit.length > 0 && currentPath) { try { @@ -260,6 +265,18 @@ function WorkspaceSettings() { control={ setInheritSoul(e.target.checked)} />} label={SOUL.md（身份与角色定义）} /> + {/* 数据库继承：仅在新工作空间（空目录）时显示，避免覆盖已有工作空间的数据 */} + {checkResult.isNewWorkspace && ( + <> + setInheritDatabase(e.target.checked)} />} + label={数据库 agent.db（会话/消息/记忆/Trace 历史记录）} + /> + + 勾选后将复制当前工作空间的全部历史数据到新工作空间（通过 SQLite backup API 原子性导出） + + + )} MEMORY.md 不继承（记忆与工作空间项目上下文绑定） @@ -1288,8 +1305,7 @@ function LogsSettings() { fullWidth value={logFilePath} placeholder={logPathLoading ? '正在获取路径...' : '路径不可用'} - readOnly - slotProps={{ input: { sx: { fontSize: 11, fontFamily: 'monospace' } } }} + slotProps={{ input: { readOnly: true, sx: { fontSize: 11, fontFamily: 'monospace' } } }} />