Files
metona-ai-desktop/electron/services/session-summary.service.ts
thzxx 4ee5100661
CI / 类型检查 + Lint + 单元测试 (push) Failing after 5m47s
CI / 全量测试 (Electron ABI) (push) Failing after 5m25s
CI / 产物编译验证 (push) Successful in 10m4s
feat: v0.5.4 多模态增强 — 多轮图片记忆 + 多模态总开关 + DeepSeek vision 模型支持
多轮图片记忆:
- 此前历史轮次的图片不回传 LLM(attachments 仅存压缩 preview,历史组装
  时被丢弃)— 跨轮对话中模型对图片内容"失忆"
- 修复:SessionSummaryService.buildHistoryMessages 从持久化的 attachments
  恢复 images(type=image 的 preview base64),历史图片随上下文回传
- Token 控制:最多注入最近 10 张(MAX_HISTORY_IMAGES,从最新消息向前
  收集)— 每张 1024px 压缩图约数百至千余 token,无上限会吃满上下文
- 摘要区间(summarizedUntilRowid 之前)的图片不恢复,符合滚动摘要语义

多模态总开关 llm.multimodalEnabled(默认关闭):
- 新配置项:CONFIG_DEFAULTS 种子 + 设置弹框 LLM 配置 Switch +
  首次引导向导 LLM 步骤 Switch(含说明文案)
- 上传入口双重判断:总开关 × 模型能力 — 未开启时即使模型支持多模态
  也不能上传图片(ChatInput 的选择/拖拽/粘贴统一拦截,Toast 区分
  "开关未开启"与"当前模型不支持"两种原因)
- 保存成功后同步 Agent Store 立即生效;App 启动时随 setProvider 加载

DeepSeek vision 模型支持:
- 新增 deepseek-v4-flash-vision-exp(OpenAI image_url content parts 格式,
  128K 上下文 / 8K 输出)
- adapter 按 isVisionModel() 判断:vision 模型将带 images 的消息转换为
  [{type:'text'},{type:'image_url'}] parts;非 vision 模型保持 images
  静默丢弃(防 API 400)

测试(236 → 243 用例):
- 多轮图片记忆 ×3(session-summary.test.ts):历史 attachments 恢复
  images / 上限 10 张从最新向前 / 摘要区间图片不恢复
- DeepSeek vision 请求格式 ×4(deepseek-vision.test.ts,契约级 mock
  fetch 断言请求体):image_url parts 转换 / 非 vision 模型丢弃 /
  max_tokens 钳制 8192 / 无图不转换
- 测试顺序修正:多轮图片用例置于 describe 末尾(插入新行消耗全局自增
  rowid,插在中间会破坏既有用例对 rowid 数值的断言)

文档: README 同步(DeepSeek 模型表 + vision 多模态列、llm.multimodalEnabled
配置项、多轮图片记忆特性行、243 用例数)

验证: lint 0 / typecheck 双工程 0 / test:electron 243 全过 / build 成功
2026-08-21 23:00:20 +08:00

246 lines
10 KiB
TypeScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* Session Summary Service — 会话摘要分层上下文(P2-11)
*
* 解决"历史消息全量加载"问题:sendMessage 原实现从 DB 加载全部历史消息进 LLM
* 上下文,超长会话(数百条消息)token 成本线性膨胀,只能依赖运行时压缩兜底。
*
* 分层策略(DB 层持久化,跨 run 生效):
* 1. 会话消息数超过阈值(50 条)后,将较早消息(保留尾部 20 条原文)交由 LLM
* 生成滚动摘要,持久化到 session_summaries 表(含 summarized_until_rowid 游标)
* 2. 下次 sendMessage 只加载:[摘要消息] + [rowid > 游标的近期原文消息]
* 3. 摘要随新消息增量滚动更新(旧摘要作为上下文参与新一轮总结)
*
* 与 engine 运行时压缩(compressMessages)的关系:
* 运行时压缩处理"单次 run 内"的上下文膨胀;本服务处理"跨 run"的 DB 历史分层,
* 两者互补,运行时压缩触发频率将显著下降。
*
* @see electron/services/database.service.ts — session_summaries 表结构
*/
import { nanoid } from 'nanoid';
import type Database from 'better-sqlite3';
import log from 'electron-log';
import type { IMetonaProviderAdapter, MetonaMessage, MetonaRequest } from '../harness/types';
import type { SessionService } from './session.service';
/** 触发摘要的最小消息总数(低于此值保持全量加载) */
const MIN_MESSAGES_TO_SUMMARIZE = 50;
/** 摘要后保留的尾部原文条数 */
const TAIL_KEEP = 20;
/** 每次摘要需新增的最小未总结消息数(避免每条消息都触发 LLM 摘要) */
const MIN_NEW_TO_SUMMARIZE = 15;
/** LLM 摘要调用超时 */
const SUMMARY_TIMEOUT_MS = 30_000;
/** 传给 LLM 的单条消息内容截断 */
const PER_MESSAGE_TRUNCATE = 600;
/** 传给 LLM 的总字符上限 */
const MAX_DIGEST_CHARS = 24_000;
/** v0.5.4: 多轮图片记忆 — 历史上下文注入的最大图片数(从最新向前收集,防 token 爆炸) */
const MAX_HISTORY_IMAGES = 10;
export class SessionSummaryService {
constructor(
private getDB: () => Database.Database,
private sessionService: SessionService,
private adapterGetter: () => IMetonaProviderAdapter,
) {}
/**
* 构建分层历史消息(sendMessage 的加载入口)
*
* @returns MetonaMessage 数组:存在摘要时为 [摘要消息, 近期原文...],否则全量原文
*/
buildHistoryMessages(sessionId: string): MetonaMessage[] {
const existing = this.getSummary(sessionId);
const tail = this.sessionService.getMessages(sessionId, {
afterRowid: existing?.summarizedUntilRowid ?? 0,
});
const messages: MetonaMessage[] = tail
.filter((m) => m.role !== 'system')
.map((m) => ({
role: m.role as MetonaMessage['role'],
content: m.content,
reasoningContent: m.reasoningContent,
toolCalls: m.toolCalls as MetonaMessage['toolCalls'],
toolResult: m.toolResult as MetonaMessage['toolResult'],
// v0.5.4: 保留 attachments 供 restoreHistoryImages 恢复图片(多轮图片记忆)
attachments: (m as { attachments?: unknown[] }).attachments,
timestamp: m.timestamp,
iteration: m.iteration,
}));
// v0.5.4: 多轮图片记忆 — 从持久化的 attachments(压缩 base64 preview)恢复 images
// 历史轮次的图片重新注入 LLM 上下文(此前仅发送当轮可见,跨轮即"失忆")。
// 数量上限防 token 爆炸:只取最近 MAX_HISTORY_IMAGES 张(从最新消息向前收集)。
// 摘要区间(summarizedUntilRowid 之前)的图片无法恢复 — 符合滚动摘要的语义。
this.restoreHistoryImages(messages);
if (existing && messages.length > 0) {
// 摘要以 assistant 角色注入(与 engine 运行时压缩的注入策略一致)
const summaryMessage: MetonaMessage = {
role: 'assistant',
content: `[Context Summary] The following is a rolling summary of earlier conversation history:\n\n${existing.summary}`,
timestamp: 0,
};
return [summaryMessage, ...messages];
}
return messages;
}
/**
* v0.5.4: 恢复历史消息的 images(多轮图片记忆)
*
* attachments 中 type=image 的 preview1024px JPEG 压缩 base64)在消息
* 持久化时已保存(与编辑重发的恢复逻辑同源)。此处将其映射回
* MetonaMessage.images,让历史轮次图片随上下文回传 LLM。
*
* 上限策略:从最新消息向前收集,最多 MAX_HISTORY_IMAGES 张 —
* 每张 1024px 图约数百至千余 token,无上限的长会话会迅速吃满上下文。
*/
private restoreHistoryImages(messages: MetonaMessage[]): void {
let remaining = MAX_HISTORY_IMAGES;
for (let i = messages.length - 1; i >= 0 && remaining > 0; i--) {
const raw = messages[i] as MetonaMessage & {
attachments?: Array<{ type?: string; preview?: string }>;
};
const imageAttachments = (raw.attachments ?? []).filter(
(a) => a.type === 'image' && typeof a.preview === 'string' && a.preview.length > 0,
);
if (imageAttachments.length === 0) continue;
const take = Math.min(imageAttachments.length, remaining);
// 优先保留靠后的图片(时间更近)
const picked = imageAttachments.slice(-take);
messages[i].images = picked.map((a) => ({
url: a.preview as string,
detail: 'auto' as const,
}));
remaining -= take;
}
}
/**
* 会话结束后评估并生成滚动摘要(fire-and-forget 调用,失败仅记录日志)
*/
async maybeSummarize(sessionId: string): Promise<void> {
const rows = this.sessionService.getMessages(sessionId);
if (rows.length < MIN_MESSAGES_TO_SUMMARIZE) return;
const existing = this.getSummary(sessionId);
const lastSummarized = existing?.summarizedUntilRowid ?? 0;
const unsummarized = rows.filter((r) => (r.rowId ?? 0) > lastSummarized);
// 未总结增量不足(保留尾部 TAIL_KEEP 后仍需 ≥ MIN_NEW_TO_SUMMARIZE 条)
if (unsummarized.length <= TAIL_KEEP + MIN_NEW_TO_SUMMARIZE) return;
const toSummarize = unsummarized.slice(0, unsummarized.length - TAIL_KEEP);
if (toSummarize.length < MIN_NEW_TO_SUMMARIZE) return;
const summary = await this.summarizeViaLLM(existing?.summary ?? '', toSummarize);
if (!summary) return;
const untilRowid = toSummarize[toSummarize.length - 1].rowId ?? 0;
this.saveSummary(sessionId, summary, untilRowid);
log.info(
`[SessionSummary] session ${sessionId}: summarized ${toSummarize.length} messages (up to rowid ${untilRowid}), kept ${TAIL_KEEP} recent`,
);
}
// ===== 摘要表 CRUD =====
getSummary(sessionId: string): { summary: string; summarizedUntilRowid: number } | null {
const db = this.getDB();
const row = db
.prepare('SELECT summary, summarized_until_rowid FROM session_summaries WHERE session_id = ?')
.get(sessionId) as { summary: string; summarized_until_rowid: number } | undefined;
return row ? { summary: row.summary, summarizedUntilRowid: row.summarized_until_rowid } : null;
}
saveSummary(sessionId: string, summary: string, untilRowid: number): void {
const db = this.getDB();
db.prepare(
`
INSERT INTO session_summaries (session_id, summary, summarized_until_rowid, updated_at)
VALUES (?, ?, ?, ?)
ON CONFLICT(session_id) DO UPDATE SET
summary = excluded.summary,
summarized_until_rowid = excluded.summarized_until_rowid,
updated_at = excluded.updated_at
`,
).run(sessionId, summary, untilRowid, Date.now());
}
// ===== LLM 摘要 =====
/**
* 调用 LLM 生成滚动摘要(复用主 Provider adapter
*
* @param priorSummary 既有摘要(滚动总结上下文,可为空)
* @param messages 本轮待总结的消息
*/
private async summarizeViaLLM(
priorSummary: string,
messages: Array<{ role: string; content: string | null }>,
): Promise<string | null> {
let total = 0;
const transcript = messages
.map((m) => {
const content = (m.content ?? '').slice(0, PER_MESSAGE_TRUNCATE);
if (total < MAX_DIGEST_CHARS) {
total += content.length;
return `[${m.role.toUpperCase()}] ${content}`;
}
return null;
})
.filter((s): s is string => s !== null)
.join('\n\n');
const request: MetonaRequest = {
meta: {
sessionId: 'session-summary',
iteration: 0,
requestId: `ss_${nanoid(12)}`,
timestamp: Date.now(),
agentVersion: '1.0.0',
},
systemPrompt: {
roleDefinition: 'You are a conversation summarizer for an AI agent application.',
outputConstraints:
'Produce a rolling summary of the conversation history below. ' +
'If a prior summary exists, merge it with the new content into one updated summary. ' +
'Preserve key facts, decisions, tool outcomes, file paths, and open questions needed for future reasoning. ' +
'Output in the same language as the conversation. Maximum 400 words. Output ONLY the summary text.',
safetyGuidelines:
'Do not include sensitive data like passwords or API keys in the summary.',
},
messages: [
{
role: 'user',
content: `${priorSummary ? `## Prior summary (merge and update):\n\n${priorSummary}\n\n` : ''}## New messages to summarize:\n\n${transcript}`,
timestamp: Date.now(),
},
],
params: {
maxTokens: 2048,
temperature: 0.0,
stream: false,
thinkingEnabled: false,
thinkingEffort: 'low',
},
};
try {
const timeoutPromise = new Promise<never>((_, reject) => {
setTimeout(() => reject(new Error('summary timeout')), SUMMARY_TIMEOUT_MS);
});
const response = await Promise.race([this.adapterGetter().send(request), timeoutPromise]);
const summary = response.content.trim();
return summary || null;
} catch (err) {
log.warn('[SessionSummary] LLM summarization failed:', (err as Error).message);
return null;
}
}
}