feat: 视频上传支持 (v0.11.7)
- 新增视频上传按钮,支持 mp4/avi/mov/mkv/webm (≤10MB) - ffmpeg 1fps 提取帧序列,自动缩放至512px,带时序标注 - 消息中注入帧序列时间线,多模态模型原生理解视频时序 - 视频按钮与 vision 模型能力自动联动 - 版本号 0.11.6 → 0.11.7,同步更新 README/帮助面板/菜单
This commit is contained in:
+131
@@ -5,6 +5,7 @@
|
||||
import { ipcMain, dialog, shell } from 'electron';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
import { mainWindow } from './main.js';
|
||||
import { showNotification } from './utils.js';
|
||||
import {
|
||||
@@ -58,6 +59,8 @@ import { startServer, stopServer, stopAllServers, callTool, getAllTools, getServ
|
||||
import { getAllowedDirs, getBlockedDirs, setAllowedDirs } from './tool-security.js';
|
||||
import { startProcess, killProcess, getWorkspaceDir, setWorkspaceDir, listWorkspaceDir } from './workspace.js';
|
||||
import { setHTTPTimeout } from './tool-handlers-system.js';
|
||||
import { execFile } from 'child_process';
|
||||
import * as crypto from 'crypto';
|
||||
|
||||
/** 工具结果摘要(用于日志面板) */
|
||||
function summarizeResult(toolName: string, result: Record<string, unknown>): string {
|
||||
@@ -479,4 +482,132 @@ export async function setupIPC(): Promise<void> {
|
||||
ipcMain.handle('mcp:refreshTools', async (_, name: string) => {
|
||||
return refreshTools(name);
|
||||
});
|
||||
|
||||
// ── 视频帧提取 ──
|
||||
ipcMain.handle('video:extractFrames', async (_, filePath: string, options?: { maxFrames?: number; maxWidth?: number; maxSize?: number }) => {
|
||||
const maxFrames = options?.maxFrames || 600;
|
||||
const maxWidth = options?.maxWidth || 512;
|
||||
const maxSize = options?.maxSize || 10 * 1024 * 1024; // 默认 10MB
|
||||
|
||||
// 文件大小检查
|
||||
try {
|
||||
const stat = await fs.promises.stat(filePath);
|
||||
if (stat.size > maxSize) {
|
||||
return { success: false, error: `视频文件过大: ${(stat.size / 1024 / 1024).toFixed(1)}MB,限制 ${maxSize / 1024 / 1024}MB` };
|
||||
}
|
||||
} catch {
|
||||
return { success: false, error: '无法读取视频文件' };
|
||||
}
|
||||
|
||||
return extractVideoFrames(filePath, maxFrames, maxWidth);
|
||||
});
|
||||
}
|
||||
|
||||
// ── 视频帧提取 (ffmpeg) ──
|
||||
|
||||
interface ExtractedFrame {
|
||||
name: string; // 如 "frame_01_0.0s.jpg"
|
||||
base64: string; // JPEG base64
|
||||
timestampSeconds: number; // 帧在视频中的时间位置(秒)
|
||||
}
|
||||
|
||||
interface VideoInfo {
|
||||
duration: number; // 秒
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
function ffprobeVideo(filePath: string): Promise<VideoInfo> {
|
||||
return new Promise((resolve, reject) => {
|
||||
execFile('ffprobe', [
|
||||
'-v', 'quiet', '-print_format', 'json', '-show_format', '-show_streams',
|
||||
'-select_streams', 'v:0', filePath
|
||||
], { timeout: 15_000 }, (err, stdout) => {
|
||||
if (err) { reject(new Error(`ffprobe 失败: ${err.message}`)); return; }
|
||||
try {
|
||||
const info = JSON.parse(stdout);
|
||||
const stream = (info.streams || []).find((s: any) => s.codec_type === 'video');
|
||||
if (!stream) { reject(new Error('未找到视频流')); return; }
|
||||
resolve({
|
||||
duration: parseFloat(info.format?.duration || stream.duration || '0'),
|
||||
width: stream.width || 0,
|
||||
height: stream.height || 0
|
||||
});
|
||||
} catch { reject(new Error('ffprobe 输出解析失败')); }
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function extractVideoFrames(filePath: string, maxFrames: number, maxWidth: number): Promise<{ success: boolean; frames?: ExtractedFrame[]; videoInfo?: VideoInfo; error?: string }> {
|
||||
return new Promise(async (resolve) => {
|
||||
const tmpDir = path.join(os.tmpdir(), `metona-video-${crypto.randomBytes(6).toString('hex')}`);
|
||||
try {
|
||||
await fs.promises.mkdir(tmpDir, { recursive: true });
|
||||
|
||||
// 1. 获取视频信息
|
||||
const videoInfo = await ffprobeVideo(filePath);
|
||||
if (videoInfo.duration <= 0) {
|
||||
await fs.promises.rm(tmpDir, { recursive: true, force: true });
|
||||
resolve({ success: false, error: '无法获取视频时长' });
|
||||
return;
|
||||
}
|
||||
|
||||
// 2. 计算帧提取策略:固定 1fps,按视频实际时长
|
||||
const frameCount = Math.min(Math.ceil(videoInfo.duration), maxFrames);
|
||||
const scaleFilter = videoInfo.width > maxWidth
|
||||
? `scale=${maxWidth}:-1`
|
||||
: 'scale=iw:ih'; // 不放大
|
||||
|
||||
// 3. ffmpeg 提取帧:固定 1fps
|
||||
const outPattern = path.join(tmpDir, 'frame_%04d.jpg');
|
||||
await new Promise<void>((ffResolve, ffReject) => {
|
||||
execFile('ffmpeg', [
|
||||
'-y',
|
||||
'-i', filePath,
|
||||
'-vf', `${scaleFilter},fps=1`,
|
||||
'-q:v', '3', // JPEG 质量 (2-31, 越小越好, 3≈80%)
|
||||
'-frames:v', String(frameCount),
|
||||
'-threads', '2',
|
||||
outPattern
|
||||
], { timeout: 120_000 }, (err) => {
|
||||
if (err) { ffReject(new Error(`ffmpeg 执行失败: ${err.message}`)); return; }
|
||||
ffResolve();
|
||||
});
|
||||
});
|
||||
|
||||
// 4. 读取提取的帧为 base64,附时间戳
|
||||
const files = await fs.promises.readdir(tmpDir);
|
||||
const jpegs = files.filter(f => f.endsWith('.jpg')).sort();
|
||||
if (jpegs.length === 0) {
|
||||
await fs.promises.rm(tmpDir, { recursive: true, force: true });
|
||||
resolve({ success: false, error: '没有提取到视频帧' });
|
||||
return;
|
||||
}
|
||||
|
||||
const frames: ExtractedFrame[] = [];
|
||||
for (let i = 0; i < jpegs.length; i++) {
|
||||
const f = jpegs[i];
|
||||
const buf = await fs.promises.readFile(path.join(tmpDir, f));
|
||||
// 跳过太小的帧(可能是纯黑/白,无信息量)
|
||||
if (buf.length < 200) continue;
|
||||
const timestampSec = i; // fps=1,所以帧索引 = 秒数
|
||||
frames.push({
|
||||
name: `frame_${String(i + 1).padStart(2, '0')}_${timestampSec.toFixed(1)}s.jpg`,
|
||||
base64: buf.toString('base64'),
|
||||
timestampSeconds: timestampSec
|
||||
});
|
||||
}
|
||||
|
||||
// 5. 清理临时文件
|
||||
await fs.promises.rm(tmpDir, { recursive: true, force: true });
|
||||
|
||||
sendLog('info', `🎬 视频帧提取完成: 1fps`, `${filePath} → ${frames.length} 帧 / ${videoInfo.duration.toFixed(1)}s (${videoInfo.width}x${videoInfo.height})`);
|
||||
resolve({ success: true, frames, videoInfo });
|
||||
|
||||
} catch (err) {
|
||||
try { await fs.promises.rm(tmpDir, { recursive: true, force: true }); } catch { /* ignore */ }
|
||||
sendLog('error', '视频帧提取失败', (err as Error).message);
|
||||
resolve({ success: false, error: (err as Error).message });
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
+1
-1
@@ -101,7 +101,7 @@ export function createMenu(): void {
|
||||
dialog.showMessageBox(mainWindow!, {
|
||||
type: 'info',
|
||||
title: '关于 Metona Ollama',
|
||||
message: 'Metona Ollama Desktop v0.11.6',
|
||||
message: 'Metona Ollama Desktop v0.11.7',
|
||||
detail: 'TypeScript + Electron Ollama AI 聊天客户端\n\nhttps://gitee.com/thzxx/metona-ollama',
|
||||
icon: getIconPath()
|
||||
});
|
||||
|
||||
@@ -124,5 +124,9 @@ contextBridge.exposeInMainWorld('metonaDesktop', {
|
||||
getTools: () => ipcRenderer.invoke('mcp:getTools'),
|
||||
getStatuses: () => ipcRenderer.invoke('mcp:getStatuses'),
|
||||
refreshTools: (name: string) => ipcRenderer.invoke('mcp:refreshTools', name)
|
||||
},
|
||||
video: {
|
||||
extractFrames: (filePath: string, options?: { maxFrames?: number; maxWidth?: number; maxSize?: number }) =>
|
||||
ipcRenderer.invoke('video:extractFrames', filePath, options)
|
||||
}
|
||||
});
|
||||
|
||||
@@ -26,8 +26,9 @@ let chatInputEl: HTMLTextAreaElement;
|
||||
let btnSendEl: HTMLButtonElement;
|
||||
let imagePreviewEl: HTMLElement;
|
||||
let filePreviewEl: HTMLElement;
|
||||
let pendingImages: Array<{ name: string; base64: string }> = [];
|
||||
let pendingImages: Array<{ name: string; base64: string; timestampSeconds?: number }> = [];
|
||||
let pendingFiles: Array<{ name: string; content: string; language: string; size: number }> = [];
|
||||
let pendingVideoInfo: { duration: number; width: number; height: number } | null = null;
|
||||
|
||||
const TEXT_EXTENSIONS = new Set([
|
||||
'txt','md','markdown','rst','log','csv','tsv',
|
||||
@@ -42,6 +43,8 @@ const TEXT_EXTENSIONS = new Set([
|
||||
|
||||
const MAX_FILE_SIZE = 500 * 1024;
|
||||
|
||||
const VIDEO_EXTENSIONS = new Set(['mp4', 'avi', 'mov', 'mkv', 'webm', 'flv', 'wmv', 'm4v']);
|
||||
|
||||
function getFileIcon(filename: string): string {
|
||||
const name = filename.toLowerCase();
|
||||
if (name === 'dockerfile') return '🐳';
|
||||
@@ -111,6 +114,21 @@ export function initInputArea(): void {
|
||||
await handleTextFilePaths(paths);
|
||||
});
|
||||
|
||||
// ── 视频上传:提取帧作为图片 ──
|
||||
document.querySelector('#btnAttachVideo')!.addEventListener('click', async () => {
|
||||
if (!isVisionAvailable()) {
|
||||
showToast('当前模型不支持图片/视频分析', 'warning');
|
||||
return;
|
||||
}
|
||||
const bridge = getBridge();
|
||||
if (!bridge) return;
|
||||
const paths = await bridge.dialog.openFile({
|
||||
filters: [{ name: '视频', extensions: Array.from(VIDEO_EXTENSIONS) }]
|
||||
});
|
||||
if (!paths || paths.length === 0) return;
|
||||
await handleVideoPath(paths[0]);
|
||||
});
|
||||
|
||||
imagePreviewEl.addEventListener('click', (e) => {
|
||||
const target = e.target as HTMLElement;
|
||||
if (target.classList.contains('remove-img')) {
|
||||
@@ -222,6 +240,45 @@ async function handleImagePaths(paths: string[]): Promise<void> {
|
||||
renderImagePreviews();
|
||||
}
|
||||
|
||||
async function handleVideoPath(filePath: string): Promise<void> {
|
||||
const bridge = getBridge();
|
||||
if (!bridge?.video) {
|
||||
showToast('视频功能需要桌面版', 'warning');
|
||||
return;
|
||||
}
|
||||
|
||||
const name = filePath.split(/[/\\]/).pop() || filePath;
|
||||
appendSystemMessage(`🎬 正在提取视频帧 (1fps): ${name}...`);
|
||||
|
||||
try {
|
||||
const result = await bridge.video.extractFrames(filePath, { maxFrames: 60, maxWidth: 512 });
|
||||
if (!result.success) {
|
||||
appendSystemMessage(`❌ 视频帧提取失败: ${result.error}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const frames = result.frames!;
|
||||
const info = result.videoInfo!;
|
||||
|
||||
// 保存视频元信息(用于消息构建时的时序标注)
|
||||
pendingVideoInfo = { duration: info.duration, width: info.width, height: info.height };
|
||||
|
||||
// 帧加入 pendingImages,保留时间戳
|
||||
for (const frame of frames) {
|
||||
pendingImages.push({ name: frame.name, base64: frame.base64, timestampSeconds: frame.timestampSeconds });
|
||||
}
|
||||
|
||||
logInfo(`视频帧提取完成: ${name}`,
|
||||
`${frames.length} 帧 (1fps, ${info.duration.toFixed(0)}s, ${info.width}x${info.height})`);
|
||||
appendSystemMessage(`✅ ${name}: 已提取 ${frames.length} 帧 (1fps 采样)`);
|
||||
|
||||
renderImagePreviews();
|
||||
} catch (err) {
|
||||
logError('视频帧提取异常', (err as Error).message);
|
||||
appendSystemMessage(`❌ 视频处理失败: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
function renderImagePreviews(): void {
|
||||
if (pendingImages.length === 0) {
|
||||
imagePreviewEl.style.display = 'none';
|
||||
@@ -843,7 +900,20 @@ async function sendMessageWithAgentLoop(text: string, currentSession: ChatSessio
|
||||
const images = pendingImages.map(img => img.base64);
|
||||
|
||||
if (pendingImages.length > 0) {
|
||||
const imgDesc = pendingImages.length === 1 ? `[上传了图片: ${pendingImages[0].name}]` : `[上传了 ${pendingImages.length} 张图片]`;
|
||||
const hasVideoFrames = pendingVideoInfo !== null && pendingImages.some(img => img.timestampSeconds !== undefined);
|
||||
let imgDesc: string;
|
||||
if (hasVideoFrames) {
|
||||
const vi = pendingVideoInfo!;
|
||||
const frameList = pendingImages
|
||||
.filter(img => img.timestampSeconds !== undefined)
|
||||
.map(img => ` ${String(img.timestampSeconds!).padStart(4, ' ')}s — ${img.name}`)
|
||||
.join('\n');
|
||||
imgDesc = `[视频帧序列 · ${pendingImages.length}帧 · 1fps · ${vi.duration.toFixed(0)}s]\n${frameList}\n\n上述帧按时间顺序排列,帧间存在时序关系。请分析视频内容。`;
|
||||
} else if (pendingImages.length === 1) {
|
||||
imgDesc = `[上传了图片: ${pendingImages[0].name}]`;
|
||||
} else {
|
||||
imgDesc = `[上传了 ${pendingImages.length} 张图片]`;
|
||||
}
|
||||
const msg: ChatMessage = {
|
||||
role: 'user',
|
||||
content: text ? `${text}\n\n${imgDesc}` : imgDesc,
|
||||
@@ -872,7 +942,7 @@ async function sendMessageWithAgentLoop(text: string, currentSession: ChatSessio
|
||||
state.update(KEYS.CURRENT_SESSION, (session: any) => ({
|
||||
...session,
|
||||
...(isFirstMsg && {
|
||||
title: truncate(text || (pendingFiles.length > 0 ? `[文件: ${pendingFiles.map(f => f.name).join(', ')}]` : '[图片消息]'), 30),
|
||||
title: truncate(text || (pendingFiles.length > 0 ? `[文件: ${pendingFiles.map(f => f.name).join(', ')}]` : pendingVideoInfo ? '[视频消息]' : '[图片消息]'), 30),
|
||||
model
|
||||
}),
|
||||
messages: [...session.messages, ...msgsToAdd],
|
||||
@@ -888,6 +958,7 @@ async function sendMessageWithAgentLoop(text: string, currentSession: ChatSessio
|
||||
chatInputEl.value = '';
|
||||
pendingImages = [];
|
||||
pendingFiles = [];
|
||||
pendingVideoInfo = null;
|
||||
imagePreviewEl.style.display = 'none';
|
||||
imagePreviewEl.innerHTML = '';
|
||||
filePreviewEl.style.display = 'none';
|
||||
|
||||
@@ -18,6 +18,7 @@ let badgeToolsEl: HTMLElement;
|
||||
let thinkCheckbox: HTMLInputElement;
|
||||
let thinkWrap: HTMLElement;
|
||||
let btnAttachImg: HTMLButtonElement;
|
||||
let btnAttachVideo: HTMLButtonElement;
|
||||
|
||||
const modelCapabilityCache = new Map<string, ModelCaps>();
|
||||
|
||||
@@ -29,6 +30,7 @@ export function initModelBar(): void {
|
||||
thinkCheckbox = document.querySelector('#toggleThink')!;
|
||||
thinkWrap = document.querySelector('#thinkToggleWrap')!;
|
||||
btnAttachImg = document.querySelector('#btnAttachImg')!;
|
||||
btnAttachVideo = document.querySelector('#btnAttachVideo')!;
|
||||
|
||||
// Think 开关:同步 state 中的 thinkEnabled
|
||||
thinkCheckbox.addEventListener('change', () => {
|
||||
@@ -252,10 +254,20 @@ function setVisionAvailable(available: boolean): void {
|
||||
btnAttachImg.disabled = false;
|
||||
btnAttachImg.title = '上传图片';
|
||||
btnAttachImg.classList.remove('disabled');
|
||||
if (btnAttachVideo) {
|
||||
btnAttachVideo.disabled = false;
|
||||
btnAttachVideo.title = '上传视频(提取帧分析)';
|
||||
btnAttachVideo.classList.remove('disabled');
|
||||
}
|
||||
} else {
|
||||
btnAttachImg.disabled = true;
|
||||
btnAttachImg.title = '当前模型不支持图片分析';
|
||||
btnAttachImg.classList.add('disabled');
|
||||
if (btnAttachVideo) {
|
||||
btnAttachVideo.disabled = true;
|
||||
btnAttachVideo.title = '当前模型不支持图片/视频分析';
|
||||
btnAttachVideo.classList.add('disabled');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@
|
||||
<div class="header-left">
|
||||
<span class="logo">🦙</span>
|
||||
<span class="app-title">Metona Ollama</span>
|
||||
<span class="app-version">v0.11.6</span>
|
||||
<span class="app-version">v0.11.7</span>
|
||||
<button class="icon-btn help-btn" id="btnHelp" title="使用帮助">
|
||||
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
|
||||
<circle cx="12" cy="12" r="10"/><path d="M9.09 9a3 3 0 0 1 5.83 1c0 2-3 3-3 3"/>
|
||||
@@ -148,6 +148,12 @@
|
||||
<polyline points="21 15 16 10 5 21"/>
|
||||
</svg>
|
||||
</button>
|
||||
<button class="icon-btn attach-btn" id="btnAttachVideo" title="上传视频(提取帧分析)">
|
||||
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
|
||||
<polygon points="23 7 16 12 23 17 23 7"/>
|
||||
<rect x="1" y="5" width="15" height="14" rx="2" ry="2"/>
|
||||
</svg>
|
||||
</button>
|
||||
<button class="icon-btn attach-btn" id="btnAttachFile" title="上传文件(文本/代码)">
|
||||
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2">
|
||||
<path d="M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8z"/>
|
||||
@@ -417,10 +423,10 @@
|
||||
</div>
|
||||
<div class="modal-body">
|
||||
<div class="help-section"><h4>🚀 快速开始</h4><ol><li>确保 Ollama 已启动(默认 <code>http://127.0.0.1:11434</code>,可在设置中修改)</li><li>顶部模型栏选择一个模型,右侧会显示模型能力徽章(🧠 Think / 👁️ Vision / 🔧 Tools)</li><li>输入消息,按 <kbd>Enter</kbd> 发送,<kbd>Shift+Enter</kbd> 换行</li></ol></div>
|
||||
<div class="help-section"><h4>💬 聊天功能</h4><ul><li><strong>流式回复</strong> — 实时打字效果,随时点 ■ 停止</li><li><strong>Think 推理</strong> — 下方 Think 按钮切换,让模型展示深度思考过程(需模型支持,如 Qwen3)</li><li><strong>上下文长度自动检测</strong> — 切换模型时自动从模型元数据获取实际支持的上下文长度(如 Qwen3.6 27B = 262,144 tokens),无需手动配置</li><li><strong>多模态</strong> — 上传图片,模型需支持 Vision 能力。图片自动压缩至合适分辨率,节省上下文</li><li><strong>文件分析</strong> — 支持 50+ 种文本/代码格式,单文件 ≤500KB,自动剥离注释并按上下文预算智能截断</li><li><strong>AI 回复顶部</strong> — 每条 AI 回复上方展示 📋 系统提示词折叠卡片,可点击查看实际发送给模型的完整上下文</li></ul></div>
|
||||
<div class="help-section"><h4>💬 聊天功能</h4><ul><li><strong>流式回复</strong> — 实时打字效果,随时点 ■ 停止</li><li><strong>Think 推理</strong> — 下方 Think 按钮切换,让模型展示深度思考过程(需模型支持,如 Qwen3)</li><li><strong>上下文长度自动检测</strong> — 切换模型时自动从模型元数据获取实际支持的上下文长度(如 Qwen3.6 27B = 262,144 tokens),无需手动配置</li><li><strong>多模态</strong> — 上传图片或视频(≤10MB),模型需支持 Vision。图片自动压缩,视频 1fps 提取帧序列带时序标注</li><li><strong>文件分析</strong> — 支持 50+ 种文本/代码格式,单文件 ≤500KB,自动剥离注释并按上下文预算智能截断</li><li><strong>AI 回复顶部</strong> — 每条 AI 回复上方展示 📋 系统提示词折叠卡片,可点击查看实际发送给模型的完整上下文</li></ul></div>
|
||||
<div class="help-section"><h4>🔧 Tool Calling(始终开启)</h4><ul><li>所有消息均通过 <strong>Agent Loop 自主调用本地工具</strong>,像一个本地 Agent,无普通聊天模式</li><li><strong>44 个工具</strong>,分为 10 类:<ul><li><strong>文件系统</strong>(16 个):read_file / write_file / list_directory / search_files / create_directory / delete_file / move_file / copy_file / edit_file / get_file_info / tree / download_file / diff_files / replace_in_files / read_multiple_files / compress</li><li><strong>命令执行</strong>(1 个):run_command(实时流式输出,支持自动/需确认/禁用三种模式,可配置超时)</li><li><strong>联网搜索</strong>(2 个):web_search(支持 SearXNG 元搜索引擎 JSON API / 四引擎 HTML 解析,双模式可切换)/ web_fetch(反爬headers+UA自动切换+自动回退浏览器渲染)</li><li><strong>Git</strong>(1 个):git(17 个子操作,push/pull/clone 内置60-120s超时保护)</li><li><strong>浏览器控制</strong>(9 个):browser_open / browser_screenshot / browser_evaluate / browser_extract / browser_click / browser_type / browser_scroll / browser_wait / browser_close</li><li><strong>记忆 & 会话 & Skill</strong>(8 个):memory_search / memory_add / memory_replace / memory_remove / session_list / session_read / skill_list / skill_view</li><li><strong>子代理</strong>(1 个):spawn_task</li><li><strong>系统工具</strong>(6 个):datetime / calculator / random / uuid / json_format / hash</li></ul></li><li>read_file 支持 binary模式读取/base64解码/字节分页/2000行默认+截断hint自动续读</li><li>write_file 支持 base64写二进制文件/mode(overwrite+append)合并append_file功能/10MB</li><li>edit_file 支持 use_regex 正则替换</li><li>search_files 支持正则表达式搜索(use_regex=true)</li><li>browser_screenshot 支持全页面截图+元素截图</li><li>browser_extract 支持CSS选择器提取特定区域</li><li>new browser_wait 等待元素出现或定时等待</li><li>仅 <code>run_command</code> 支持三模式切换:自动/需确认/禁用</li><li>危险命令(<code>rm -rf</code>、<code>mkfs</code>、反弹 shell 等)和系统路径(<code>/etc</code>、<code>~/.ssh</code> 等)被自动拦截</li><li>工具调用以<strong>可视化卡片</strong>展示状态(pending → running → success/error),在工作空间🔧工具页签中显示</li><li>默认最大 <strong>85 轮</strong>工具调用循环,上下文使用率>80%时自动缩减到3轮</li><li>独立工具自动<strong>并行执行</strong>(16个只读工具加入并行白名单),有依赖关系的工具串行执行</li></ul></div>
|
||||
<div class="help-section"><h4>🧠 Agent 记忆系统</h4><ul><li>设置中开启后,AI 从对话中<strong>自动提取关键信息</strong>(事实/偏好/规则),跨会话持续积累</li><li><strong>增量提取</strong>:长对话中每 20 轮自动触发轻量级记忆提取,不等到对话结束</li><li>对话结束时自动触发完整记忆提取</li><li>新对话时自动检索相关记忆注入上下文,让 AI "记住"你</li><li>点击顶部 🧠 按钮打开记忆面板:搜索、筛选、编辑、删除</li><li><strong>记忆容量上限 500 条</strong>,超限时自动清理低价值条目(规则类型受保护)</li><li><strong>向量语义搜索</strong>:在设置中选择嵌入模型后,支持语义级别记忆检索(更精准)</li><li>未选择嵌入模型时,使用关键词匹配检索记忆</li><li>AI 可通过 memory 工具主动管理记忆</li><li>记忆写入前自动安全扫描(prompt injection / 敏感信息 / 不可见字符检测)</li><li>记忆存储在本地 SQLite,不会上传到任何服务器</li></ul></div>
|
||||
<div class="help-section"><h4>🤖 Agent Loop v0.11.6 增强</h4><ul><li><strong>智能上下文压缩</strong>:滑动窗口 + LLM结构化JSON摘要,超过50%上下文窗口自动触发</li><li><strong>流式调用超时保护</strong>:可配置超时(默认300s),Ollama 卡死不再永久阻塞</li><li><strong>HTTP/MCP 超时可配</strong>:设置面板可分别调整 HTTP(默认30s)和 MCP(默认60s)超时</li><li><strong>Final Answer 智能检测</strong>:多模式匹配(Final Answer/最终答案/最终回答/总结),避免漏检或误触</li><li><strong>Token 感知迭代预算</strong>:上下文使用率>80%时自动缩减剩余轮次到3轮</li><li><strong>工具缓存 TTL</strong>:搜索5分钟/网页10分钟/文件30分钟,过期自动刷新</li><li><strong>自动子任务拆解</strong>:检测"同时/分别/以及"等并行关键词,自动 spawn 子代理并行处理</li><li><strong>旧工具结果自动截断</strong>:超过10轮的工具结果自动截断到500字符,控制上下文膨胀</li></ul></div>
|
||||
<div class="help-section"><h4>🤖 Agent Loop v0.11.7 增强</h4><ul><li><strong>🎬 视频上传</strong>:支持上传 .mp4/.avi/.mov/.mkv/.webm 等视频(≤10MB),自动 1fps 提取帧序列(带时间戳),多模态模型原生理解视频时序关系</li><li><strong>智能上下文压缩</strong>:滑动窗口 + LLM结构化JSON摘要,超过50%上下文窗口自动触发</li><li><strong>流式调用超时保护</strong>:可配置超时(默认300s),Ollama 卡死不再永久阻塞</li><li><strong>HTTP/MCP 超时可配</strong>:设置面板可分别调整 HTTP(默认30s)和 MCP(默认60s)超时</li><li><strong>Final Answer 智能检测</strong>:多模式匹配(Final Answer/最终答案/最终回答/总结),避免漏检或误触</li><li><strong>Token 感知迭代预算</strong>:上下文使用率>80%时自动缩减剩余轮次到3轮</li><li><strong>工具缓存 TTL</strong>:搜索5分钟/网页10分钟/文件30分钟,过期自动刷新</li><li><strong>自动子任务拆解</strong>:检测"同时/分别/以及"等并行关键词,自动 spawn 子代理并行处理</li><li><strong>旧工具结果自动截断</strong>:超过10轮的工具结果自动截断到500字符,控制上下文膨胀</li></ul></div>
|
||||
<div class="help-section"><h4>🔌 MCP(Model Context Protocol)</h4><ul><li>支持连接外部 MCP Server,动态扩展工具能力</li><li>设置面板可添加/启用/禁用/删除 MCP 服务器</li><li>MCP 工具以 <code>mcp_{server}_{tool}</code> 前缀注册,与内置工具统一调度</li><li>启动时自动连接已启用的 MCP 服务器</li></ul></div>
|
||||
<div class="help-section"><h4>🔍 SearXNG 元搜索引擎</h4><ul><li>点击顶部 🔍 按钮打开配置面板,可接入自部署的 SearXNG 实例</li><li><strong>JSON 模式</strong>:调用 SearXNG JSON API,聚合 70+ 引擎结果,结构化解析</li><li><strong>HTML 模式</strong>:获取原始搜索结果页面,交由 AI 自行分析提取信息</li><li>支持认证 Key(HTTP Header Authorization),保护私有实例</li><li>启用后替代内置四引擎方案;关闭即回退,无缝切换</li><li>所有参数(引擎、语言、安全搜索、时间范围等)均可独立配置</li></ul></div>
|
||||
<div class="help-section"><h4>📋 自定义文件(SOUL.md / AGENT.md / USER.md)</h4><ul><li>在工作空间目录创建以下文件即可自定义 AI 行为,修改后下一轮对话立即生效</li><li><strong>SOUL.md</strong> — AI 身份、性格、行为准则(<strong>永远不可被压缩</strong>,注入为最高优先级系统提示词)</li><li><strong>AGENT.md</strong> — 工具调用规则、链式调用模式、核心约束(内置精简默认版,可通过工作空间覆盖)</li><li><strong>USER.md</strong> — 用户画像:技术栈、偏好、习惯等个人信息,AI 在对话中自动参考</li><li>可在 AI 回复顶部的 📋 系统提示词卡片中查看实际注入的完整上下文</li><li>删除工作空间中的文件即可恢复为内置默认版本</li></ul></div>
|
||||
|
||||
@@ -351,6 +351,15 @@ html, body {
|
||||
background: var(--critical-bg);
|
||||
}
|
||||
|
||||
#btnAttachVideo.disabled {
|
||||
pointer-events: auto;
|
||||
}
|
||||
|
||||
#btnAttachVideo.disabled:hover {
|
||||
opacity: 0.3;
|
||||
background: var(--critical-bg);
|
||||
}
|
||||
|
||||
.icon-btn.sm {
|
||||
width: 28px;
|
||||
height: 28px;
|
||||
|
||||
Vendored
+8
@@ -271,6 +271,14 @@ export interface MetonaDesktopAPI {
|
||||
getStatuses: () => Promise<Array<{ name: string; running: boolean; toolCount: number }>>;
|
||||
refreshTools: (name: string) => Promise<Array<{ name: string; description?: string; inputSchema: Record<string, unknown> }>>;
|
||||
};
|
||||
video: {
|
||||
extractFrames: (filePath: string, options?: { maxFrames?: number; maxWidth?: number; maxSize?: number }) => Promise<{
|
||||
success: boolean;
|
||||
frames?: Array<{ name: string; base64: string; timestampSeconds: number }>;
|
||||
videoInfo?: { duration: number; width: number; height: number };
|
||||
error?: string;
|
||||
}>;
|
||||
};
|
||||
}
|
||||
|
||||
export interface AppInfo {
|
||||
|
||||
Reference in New Issue
Block a user