feat: v0.8.2 安全纵深补全 · 协议保真 · 断链修复 — 图片SSRF/根MEMORY.md保护根治 · Anthropic thinking回传+pause_turn续传 · 2523 用例全量回归 + E2E 扩充
This commit is contained in:
@@ -0,0 +1,179 @@
|
||||
/**
|
||||
* v0.8.2 P0-1: 适配器图片下载 SSRF 安全通道测试
|
||||
*
|
||||
* 锁定单元:
|
||||
* - sniffImageMediaType 魔数嗅探
|
||||
* - 协议白名单(仅 http/https)
|
||||
* - IP 直连私网/云元数据拒绝(resolvePublicAddresses 单一事实来源)
|
||||
* - DNS 解析到私网拒绝(rebinding 形态)
|
||||
* - 成功路径:下载 → base64 + 类型钳制(png/jpeg/gif/webp 白名单)
|
||||
* - 非图片 content-type 拒绝
|
||||
* - 字节上限(maxBytes + content-length 双闸)
|
||||
* - 逐跳重定向复检(每一跳重新进入完整校验)
|
||||
*/
|
||||
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
||||
|
||||
vi.mock('electron-log', () => ({
|
||||
default: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() },
|
||||
}));
|
||||
|
||||
const proxyMock = vi.hoisted(() => ({ isProxyActive: vi.fn(() => false) }));
|
||||
vi.mock('../../../../utils/network-proxy', () => ({
|
||||
isProxyActive: proxyMock.isProxyActive,
|
||||
}));
|
||||
|
||||
// undici mock:ssrfPinnedFetch 的 pinned 路径需要 Agent + fetch
|
||||
const undiciMock = vi.hoisted(() => {
|
||||
class FakeAgent {
|
||||
async close(): Promise<void> {}
|
||||
}
|
||||
const fetch = vi.fn();
|
||||
return { FakeAgent, fetch };
|
||||
});
|
||||
vi.mock('undici', () => ({
|
||||
Agent: undiciMock.FakeAgent,
|
||||
fetch: undiciMock.fetch,
|
||||
}));
|
||||
|
||||
import { fetchImageAsBase64, sniffImageMediaType, __imageFetcher } from '../ssrf-image-fetch';
|
||||
import type { ImageFetcher } from '../ssrf-image-fetch';
|
||||
|
||||
const PNG_MAGIC = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00]);
|
||||
|
||||
function imageResponse(bytes: Uint8Array, contentType = 'image/png'): Response {
|
||||
return new Response(bytes, { status: 200, headers: { 'content-type': contentType } });
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
undiciMock.fetch.mockReset();
|
||||
proxyMock.isProxyActive.mockReturnValue(false);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe('sniffImageMediaType', () => {
|
||||
it('识别 PNG / JPEG / GIF / WEBP 魔数', () => {
|
||||
const png = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
|
||||
expect(sniffImageMediaType(png)).toBe('image/png');
|
||||
const jpeg = Buffer.from([0xff, 0xd8, 0xff, 0xe0]);
|
||||
expect(sniffImageMediaType(jpeg)).toBe('image/jpeg');
|
||||
const gif = Buffer.from('GIF89a');
|
||||
expect(sniffImageMediaType(gif)).toBe('image/gif');
|
||||
const webp = Buffer.concat([
|
||||
Buffer.from('RIFF'),
|
||||
Buffer.from([0x00, 0x00, 0x00, 0x00]),
|
||||
Buffer.from('WEBP'),
|
||||
]);
|
||||
expect(sniffImageMediaType(webp)).toBe('image/webp');
|
||||
});
|
||||
|
||||
it('非图片内容返回 null', () => {
|
||||
expect(sniffImageMediaType(Buffer.from('hello world, plain text'))).toBeNull();
|
||||
expect(sniffImageMediaType(Buffer.alloc(0))).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('fetchImageAsBase64 — 校验拒绝矩阵', () => {
|
||||
it('拒绝非 http/https 协议', async () => {
|
||||
await expect(fetchImageAsBase64('file:///etc/passwd')).rejects.toThrow(/protocol not allowed/);
|
||||
await expect(fetchImageAsBase64('ftp://example.com/a.png')).rejects.toThrow(
|
||||
/protocol not allowed/,
|
||||
);
|
||||
});
|
||||
|
||||
it('拒绝 IP 直连私网/回环/云元数据', async () => {
|
||||
await expect(fetchImageAsBase64('http://127.0.0.1/img.png')).rejects.toThrow(/Blocked SSRF/);
|
||||
await expect(fetchImageAsBase64('http://10.1.2.3/img.png')).rejects.toThrow(/Blocked SSRF/);
|
||||
await expect(fetchImageAsBase64('http://192.168.1.1/img.png')).rejects.toThrow(/Blocked SSRF/);
|
||||
await expect(fetchImageAsBase64('http://172.16.0.9/img.png')).rejects.toThrow(/Blocked SSRF/);
|
||||
// 可回读通道的核心威胁:云元数据
|
||||
await expect(fetchImageAsBase64('http://169.254.169.254/latest/meta-data')).rejects.toThrow(
|
||||
/Blocked SSRF/,
|
||||
);
|
||||
});
|
||||
|
||||
it('拒绝域名解析到私网(DNS rebinding 形态)', async () => {
|
||||
await expect(fetchImageAsBase64('http://nx.invalid.example/img.png')).rejects.toThrow(
|
||||
/Blocked SSRF/,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('fetchImageAsBase64 — 成功与钳制路径', () => {
|
||||
it('公网图片下载 → base64 + mediaType(IP 直连,无需 DNS)', async () => {
|
||||
const original = __imageFetcher.current;
|
||||
__imageFetcher.current = (async () => imageResponse(PNG_MAGIC)) as unknown as ImageFetcher;
|
||||
try {
|
||||
const r = await fetchImageAsBase64('http://93.184.216.34/a.png');
|
||||
expect(r.mediaType).toBe('image/png');
|
||||
expect(Buffer.from(r.base64, 'base64').equals(PNG_MAGIC)).toBe(true);
|
||||
} finally {
|
||||
__imageFetcher.current = original;
|
||||
}
|
||||
});
|
||||
|
||||
it('非图片 content-type(text/html)拒绝', async () => {
|
||||
const original = __imageFetcher.current;
|
||||
__imageFetcher.current = (async () =>
|
||||
imageResponse(Buffer.from('<html></html>'), 'text/html')) as unknown as ImageFetcher;
|
||||
try {
|
||||
await expect(fetchImageAsBase64('http://93.184.216.34/a.png')).rejects.toThrow(
|
||||
/non-image content-type/,
|
||||
);
|
||||
} finally {
|
||||
__imageFetcher.current = original;
|
||||
}
|
||||
});
|
||||
|
||||
it('白名单外图片类型(image/bmp)拒绝', async () => {
|
||||
const original = __imageFetcher.current;
|
||||
__imageFetcher.current = (async () =>
|
||||
imageResponse(Buffer.from('BMxx'), 'image/bmp')) as unknown as ImageFetcher;
|
||||
try {
|
||||
await expect(fetchImageAsBase64('http://93.184.216.34/a.bmp')).rejects.toThrow(
|
||||
/unsupported media type/,
|
||||
);
|
||||
} finally {
|
||||
__imageFetcher.current = original;
|
||||
}
|
||||
});
|
||||
|
||||
it('字节上限:超过 maxBytes 拒绝(防大图内存峰值)', async () => {
|
||||
const original = __imageFetcher.current;
|
||||
__imageFetcher.current = (async () =>
|
||||
imageResponse(Buffer.alloc(64, 1))) as unknown as ImageFetcher;
|
||||
try {
|
||||
await expect(
|
||||
fetchImageAsBase64('http://93.184.216.34/a.png', { maxBytes: 8 }),
|
||||
).rejects.toThrow(/size limit/);
|
||||
} finally {
|
||||
__imageFetcher.current = original;
|
||||
}
|
||||
});
|
||||
|
||||
it('逐跳重定向复检:302 跳转后以下一跳 URL 再次下载,终图可用', async () => {
|
||||
const original = __imageFetcher.current;
|
||||
const seen: string[] = [];
|
||||
const fetcher: ImageFetcher = async (url: string) => {
|
||||
seen.push(url);
|
||||
if (seen.length === 1) {
|
||||
return new Response(null, {
|
||||
status: 302,
|
||||
headers: { location: 'http://93.184.216.34/final.png' },
|
||||
});
|
||||
}
|
||||
return imageResponse(PNG_MAGIC);
|
||||
};
|
||||
__imageFetcher.current = fetcher;
|
||||
try {
|
||||
const r = await fetchImageAsBase64('http://93.184.216.34/redirect.png');
|
||||
expect(r.mediaType).toBe('image/png');
|
||||
expect(seen).toEqual(['http://93.184.216.34/redirect.png', 'http://93.184.216.34/final.png']);
|
||||
} finally {
|
||||
__imageFetcher.current = original;
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -102,6 +102,8 @@ export abstract class OpenAICompatibleAdapter extends BaseAdapter {
|
||||
request.meta.requestId,
|
||||
request.meta.sessionId,
|
||||
request.meta.iteration,
|
||||
// v0.8.2 P3-3: 流式消费阶段的中断贯通
|
||||
this.getExternalAbortSignal(),
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -53,13 +53,22 @@ export class SseUpstreamError extends Error {
|
||||
* 引擎 chatStreamWithRetry 的 catch,自动走既有重试/故障转移通道。
|
||||
* SSE / Ollama NDJSON / Anthropic 事件机三处读循环共用,杜绝三份重复实现漂移。
|
||||
*
|
||||
* v0.8.2 P3-3 根治: 新增外部中断贯通 —— 此前 abort 监听只在 fetch 头阶段有效
|
||||
* (BaseAdapter.fetchWithTimeout 在响应头返回后解除监听),流式消费阶段的
|
||||
* reader.read() 对用户中断完全无感:配合保活/心跳型上游,"中断"按钮无法真正
|
||||
* 终止挂起的 run(E2E 中断链路实测暴露)。现把外部 signal 传入本辅助:
|
||||
* abort 触发时 cancel reader 并抛 AbortError —— 引擎 chatStreamWithRetry 由
|
||||
* this.aborted 拦截原样抛出,executeRunStream 以 USER_INTERRUPT 收尾。
|
||||
*
|
||||
* @param reader 流的 reader
|
||||
* @param idleTimeoutMs 空闲超时(默认 60s — 慢速思考模型正常 chunk 间隔可达数十秒)
|
||||
* @param externalSignal 外部中断信号(引擎 abortController;可选)
|
||||
* @returns { done, value },done=true 表示流正常结束
|
||||
*/
|
||||
export async function readStreamChunkWithIdleTimeout(
|
||||
reader: ReadableStreamDefaultReader<Uint8Array>,
|
||||
idleTimeoutMs = 60_000,
|
||||
externalSignal?: AbortSignal,
|
||||
): Promise<{ done: boolean; value: Uint8Array | undefined }> {
|
||||
let idleExpired = false;
|
||||
let idleTimer: NodeJS.Timeout | undefined;
|
||||
@@ -70,8 +79,37 @@ export async function readStreamChunkWithIdleTimeout(
|
||||
}, idleTimeoutMs);
|
||||
});
|
||||
|
||||
// 外部中断竞速(流式消费阶段的中断贯通)
|
||||
let onExternalAbort: (() => void) | null = null;
|
||||
const abortController = externalSignal
|
||||
? new Promise<never>((_, reject) => {
|
||||
if (externalSignal.aborted) {
|
||||
reject(new Error('Aborted'));
|
||||
return;
|
||||
}
|
||||
onExternalAbort = () => reject(new Error('Aborted'));
|
||||
externalSignal.addEventListener('abort', onExternalAbort, { once: true });
|
||||
})
|
||||
: null;
|
||||
|
||||
const abortError = (): Error => {
|
||||
const err = new Error('Aborted');
|
||||
err.name = 'AbortError';
|
||||
void reader.cancel().catch(() => {
|
||||
/* 连接销毁时 cancel 可能失败,忽略 */
|
||||
});
|
||||
return err;
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await Promise.race([reader.read(), idleController]);
|
||||
const result = await Promise.race([
|
||||
reader.read(),
|
||||
idleController,
|
||||
...(abortController ? [abortController] : []),
|
||||
]);
|
||||
if (externalSignal?.aborted) {
|
||||
throw abortError();
|
||||
}
|
||||
if (idleExpired) {
|
||||
throw new SseUpstreamError(
|
||||
`Stream idle timeout after ${idleTimeoutMs}ms (no data received)`,
|
||||
@@ -79,8 +117,17 @@ export async function readStreamChunkWithIdleTimeout(
|
||||
);
|
||||
}
|
||||
return result;
|
||||
} catch (err) {
|
||||
// 中断竞速赢时把底层读错误替换为标准 AbortError(reader.read 会因 cancel 拒绝)
|
||||
if (externalSignal?.aborted) {
|
||||
throw abortError();
|
||||
}
|
||||
throw err;
|
||||
} finally {
|
||||
if (idleTimer) clearTimeout(idleTimer);
|
||||
if (externalSignal && onExternalAbort) {
|
||||
externalSignal.removeEventListener('abort', onExternalAbort);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -90,13 +137,18 @@ interface SseStreamFrame {
|
||||
delta?: {
|
||||
content?: string;
|
||||
reasoning_content?: string;
|
||||
annotations?: unknown;
|
||||
tool_calls?: Array<{
|
||||
index?: number;
|
||||
function?: { name?: string; arguments?: string };
|
||||
}>;
|
||||
};
|
||||
/** 部分网关在最后一个 chunk 附带完整 message(含 annotations) */
|
||||
message?: { annotations?: unknown };
|
||||
finish_reason?: string;
|
||||
}>;
|
||||
/** v0.8.2 P2-6: MiMo 联网搜索引用注释(服务端 web_search 工具) */
|
||||
annotations?: unknown;
|
||||
usage?: {
|
||||
prompt_tokens?: number;
|
||||
completion_tokens?: number;
|
||||
@@ -108,6 +160,54 @@ interface SseStreamFrame {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* v0.8.2 P2-6 根治: MiMo 联网搜索引用(annotations)全链路丢失的收口。
|
||||
*
|
||||
*MiMo enableWebSearch 服务端工具在响应中返回 annotations[].{title,url,site_name,...}
|
||||
*(非流式在 choices[].message.annotations;流式按文档"其余字段与非流式相同",
|
||||
* 可能出现在最后一个 chunk 的顶层 / message / delta)。此前 sse-stream 完全不
|
||||
* 读取该字段 —— mimo.adapter 注释宣称"由上层归并为文本内容展示"实为断头链路,
|
||||
* 引用信息全链路丢失。
|
||||
*
|
||||
* 采集策略:按 url 去重累积;流结束时([DONE] / 断流兜底)把引用格式化为
|
||||
* Markdown 列表以 TEXT_DELTA 追加到正文 —— 引擎/渲染层按既有文本管线自然
|
||||
* 消费,无需新事件类型。
|
||||
*/
|
||||
function collectAnnotations(
|
||||
annotations: unknown,
|
||||
sink: Map<string, { title: string; url: string; siteName?: string }>,
|
||||
): void {
|
||||
if (!Array.isArray(annotations)) return;
|
||||
for (const item of annotations) {
|
||||
if (!item || typeof item !== 'object') continue;
|
||||
const rec = item as Record<string, unknown>;
|
||||
const url = typeof rec.url === 'string' ? rec.url : '';
|
||||
if (!url || sink.has(url)) continue;
|
||||
sink.set(url, {
|
||||
title: typeof rec.title === 'string' && rec.title ? rec.title : url,
|
||||
url,
|
||||
siteName: typeof rec.site_name === 'string' ? rec.site_name : undefined,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function formatAnnotationsBlock(
|
||||
sink: Map<string, { title: string; url: string; siteName?: string }>,
|
||||
): string | null {
|
||||
if (sink.size === 0) return null;
|
||||
const MAX_CITATIONS = 20;
|
||||
const lines: string[] = ['', '', '**References**', ''];
|
||||
let count = 0;
|
||||
for (const { title, url, siteName } of sink.values()) {
|
||||
if (count >= MAX_CITATIONS) break;
|
||||
const safeUrl = /^https?:\/\//i.test(url) ? url : '';
|
||||
if (!safeUrl) continue;
|
||||
lines.push(`- [${title}](${safeUrl})${siteName ? ` — ${siteName}` : ''}`);
|
||||
count++;
|
||||
}
|
||||
return count > 0 ? lines.join('\n') : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 从一条已 JSON.parse 的 SSE 数据帧中提取上游错误信息。
|
||||
* 兼容三种形态:
|
||||
@@ -312,6 +412,7 @@ function* flushToolCallBuffer(
|
||||
* @param requestId - 对应的请求 ID
|
||||
* @param sessionId - 会话 ID
|
||||
* @param iteration - 当前迭代轮次
|
||||
* @param externalSignal - 外部中断信号(v0.8.2 P3-3: 流式消费阶段的中断贯通,可选)
|
||||
* @yields MetonaStreamEvent
|
||||
*/
|
||||
export async function* parseSSEStream(
|
||||
@@ -319,6 +420,7 @@ export async function* parseSSEStream(
|
||||
requestId: string,
|
||||
sessionId: string,
|
||||
iteration: number,
|
||||
externalSignal?: AbortSignal,
|
||||
): AsyncGenerator<MetonaStreamEvent> {
|
||||
const reader = responseBody.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
@@ -341,11 +443,17 @@ export async function* parseSSEStream(
|
||||
|
||||
// 工具调用缓冲区:index → { name, argsBuffer }
|
||||
const toolCallsBuffer = new Map<number, { name: string; argsBuffer: string }>();
|
||||
// v0.8.2 P2-6: MiMo 联网搜索引用采集(按 url 去重,流结束时回填正文)
|
||||
const annotationsSink = new Map<string, { title: string; url: string; siteName?: string }>();
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
// 数据到达即重置空闲窗口(辅助函数内部实现)
|
||||
const { done, value } = await readStreamChunkWithIdleTimeout(reader, IDLE_TIMEOUT_MS);
|
||||
const { done, value } = await readStreamChunkWithIdleTimeout(
|
||||
reader,
|
||||
IDLE_TIMEOUT_MS,
|
||||
externalSignal,
|
||||
);
|
||||
if (done) break;
|
||||
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
@@ -366,6 +474,20 @@ export async function* parseSSEStream(
|
||||
// L-4 修复: 使用 flushToolCallBuffer 替代重复的遍历代码
|
||||
yield* flushToolCallBuffer(toolCallsBuffer, requestId, sessionId, iteration, seqRef);
|
||||
|
||||
// v0.8.2 P2-6: 引用注释回填正文(在 DONE 之前以 TEXT_DELTA 追加)
|
||||
const citationBlock = formatAnnotationsBlock(annotationsSink);
|
||||
if (citationBlock) {
|
||||
yield {
|
||||
type: MetonaStreamEventType.TEXT_DELTA,
|
||||
requestId,
|
||||
sessionId,
|
||||
iteration,
|
||||
seq: seqRef.seq++,
|
||||
timestamp: Date.now(),
|
||||
delta: citationBlock,
|
||||
};
|
||||
}
|
||||
|
||||
yield {
|
||||
type: MetonaStreamEventType.DONE,
|
||||
requestId,
|
||||
@@ -405,6 +527,11 @@ export async function* parseSSEStream(
|
||||
|
||||
const delta = chunk.choices?.[0]?.delta;
|
||||
|
||||
// v0.8.2 P2-6: 采集引用注释(顶层 / message / delta 三处兼容)
|
||||
collectAnnotations(chunk.annotations, annotationsSink);
|
||||
collectAnnotations(chunk.choices?.[0]?.message?.annotations, annotationsSink);
|
||||
collectAnnotations(delta?.annotations, annotationsSink);
|
||||
|
||||
// 文本内容增量
|
||||
if (delta?.content) {
|
||||
yield {
|
||||
@@ -532,6 +659,19 @@ export async function* parseSSEStream(
|
||||
'[SSE] Stream ended without [DONE] marker — flushing buffers (connection likely dropped)',
|
||||
);
|
||||
yield* flushToolCallBuffer(toolCallsBuffer, requestId, sessionId, iteration, seqRef);
|
||||
// v0.8.2 P2-6: 断流兜底路径同样回填引用注释
|
||||
const citationBlock = formatAnnotationsBlock(annotationsSink);
|
||||
if (citationBlock) {
|
||||
yield {
|
||||
type: MetonaStreamEventType.TEXT_DELTA,
|
||||
requestId,
|
||||
sessionId,
|
||||
iteration,
|
||||
seq: seqRef.seq++,
|
||||
timestamp: Date.now(),
|
||||
delta: citationBlock,
|
||||
};
|
||||
}
|
||||
yield {
|
||||
type: MetonaStreamEventType.DONE,
|
||||
requestId,
|
||||
@@ -568,8 +708,16 @@ export function parseOpenAICompatibleResponse(data: Record<string, unknown>): {
|
||||
const usage = data.usage as Record<string, unknown> | undefined;
|
||||
const rawToolCalls = message?.tool_calls as Array<Record<string, unknown>> | undefined;
|
||||
|
||||
// v0.8.2 P2-6: 非流式路径的引用注释回填(MiMo 联网搜索)
|
||||
const annotationsSink = new Map<string, { title: string; url: string; siteName?: string }>();
|
||||
collectAnnotations(message?.annotations, annotationsSink);
|
||||
collectAnnotations(data.annotations, annotationsSink);
|
||||
let content = (message?.content as string) ?? '';
|
||||
const citationBlock = formatAnnotationsBlock(annotationsSink);
|
||||
if (citationBlock) content += citationBlock;
|
||||
|
||||
return {
|
||||
content: (message?.content as string) ?? '',
|
||||
content,
|
||||
reasoningContent: message?.reasoning_content as string | undefined,
|
||||
toolCalls: rawToolCalls?.map((tc) => {
|
||||
const fn = tc.function as Record<string, unknown>;
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
/**
|
||||
* v0.8.2 P0-1: 适配器图片下载的 SSRF 安全通道(Anthropic / Ollama 共用)
|
||||
*
|
||||
* 背景:此前 anthropic.adapter.toImageBlock 与 ollama.adapter.resolveImageToBase64
|
||||
* 对消息里的 http(s) 图片 URL 直接 fetchWithTimeout 下载转 base64 —— 未接 SSRF
|
||||
* 校验、无字节上限、无 content-type 约束。与其他工具"SSRF 拦截即断"不同,
|
||||
* 该通道的下载结果会**以图片块进入模型上下文(数据可回读)**:模型可诱导用户
|
||||
* 发送 http://169.254.169.254/... 图片链接回读内网数据,属于可回读外泄通道;
|
||||
* 同时无上限的 arrayBuffer 会造成内存峰值。
|
||||
*
|
||||
* 收口方案(根治):
|
||||
* - 校验与连接同源:resolvePublicAddresses(ssrf-guard 单一事实来源)+ DNS
|
||||
* pinning(ssrfPinnedFetch),代理激活时按既有语义退化为"仅入口校验";
|
||||
* - 逐跳重定向复检:redirect:'manual' + resolveRedirectTarget,每一跳都重新
|
||||
* 走完整校验(对齐 web_fetch 的逐跳语义),最多 3 跳;
|
||||
* - 字节上限:content-length 预检 + 流式增量累计双闸(10MB,防大图内存峰值);
|
||||
* - 类型白名单:content-type 必须 image/*(或缺失/二进制时按魔数嗅探),
|
||||
* 并钳制到 Provider 实际支持集合(png/jpeg/gif/webp)。
|
||||
*
|
||||
* 失败语义由调用方决定:适配器保持"跳过该图、不阻断请求"(log.warn + 占位)。
|
||||
*/
|
||||
|
||||
import { ssrfPinnedFetch, resolveRedirectTarget } from '../../tools/built-in/ssrf-dispatcher';
|
||||
|
||||
/** 实际下载器签名(与 ssrfPinnedFetch 对齐) */
|
||||
export type ImageFetcher = (
|
||||
url: string,
|
||||
init: RequestInit,
|
||||
timeoutMs: number,
|
||||
signal?: AbortSignal,
|
||||
) => Promise<Response>;
|
||||
|
||||
/**
|
||||
* v0.8.2 P0-1: 下载器注入点 —— 生产恒为 ssrfPinnedFetch(校验与连接同源);
|
||||
* 单元测试通过替换 current 注入受控下载器(undici 客户端无法经 global fetch 打桩)。
|
||||
*/
|
||||
export const __imageFetcher: { current: ImageFetcher } = { current: ssrfPinnedFetch };
|
||||
|
||||
/** 单张图片下载字节上限(10MB,对齐 view_image 工具 5MB×2 的量级) */
|
||||
export const MAX_IMAGE_BYTES = 10 * 1024 * 1024;
|
||||
|
||||
/** 允许的图片 media type(Anthropic Messages API 支持集合;Ollama 同样接受) */
|
||||
const ALLOWED_MEDIA_TYPES = new Set(['image/png', 'image/jpeg', 'image/gif', 'image/webp']);
|
||||
|
||||
const MAX_REDIRECTS = 3;
|
||||
|
||||
export interface ImageFetchOptions {
|
||||
timeoutMs?: number;
|
||||
maxBytes?: number;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export interface ImageFetchResult {
|
||||
base64: string;
|
||||
mediaType: string;
|
||||
}
|
||||
|
||||
/** 魔数嗅探:content-type 缺失或为通用二进制类型时判定真实图片类型 */
|
||||
export function sniffImageMediaType(buf: Buffer): string | null {
|
||||
if (buf.length >= 8 && buf.subarray(0, 4).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47]))) {
|
||||
return 'image/png';
|
||||
}
|
||||
if (buf.length >= 3 && buf[0] === 0xff && buf[1] === 0xd8 && buf[2] === 0xff) {
|
||||
return 'image/jpeg';
|
||||
}
|
||||
if (buf.length >= 6 && buf.subarray(0, 3).toString('ascii') === 'GIF') {
|
||||
return 'image/gif';
|
||||
}
|
||||
if (
|
||||
buf.length >= 12 &&
|
||||
buf.subarray(0, 4).toString('ascii') === 'RIFF' &&
|
||||
buf.subarray(8, 12).toString('ascii') === 'WEBP'
|
||||
) {
|
||||
return 'image/webp';
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** 流式增量读取并强制字节上限(content-length 可伪造,以实际累计为准) */
|
||||
async function readBodyWithCap(res: Response, maxBytes: number): Promise<Buffer> {
|
||||
const declared = res.headers.get('content-length');
|
||||
const declaredBytes = declared ? Number(declared) : NaN;
|
||||
if (Number.isFinite(declaredBytes) && declaredBytes > maxBytes) {
|
||||
throw new Error(`Image exceeds size limit: ${declaredBytes} > ${maxBytes} bytes`);
|
||||
}
|
||||
const body = res.body;
|
||||
if (!body) {
|
||||
throw new Error('Image response has no body');
|
||||
}
|
||||
const reader = body.getReader();
|
||||
const chunks: Buffer[] = [];
|
||||
let total = 0;
|
||||
try {
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
if (value) {
|
||||
total += value.byteLength;
|
||||
if (total > maxBytes) {
|
||||
throw new Error(`Image exceeds size limit: > ${maxBytes} bytes`);
|
||||
}
|
||||
chunks.push(Buffer.from(value));
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
return Buffer.concat(chunks);
|
||||
}
|
||||
|
||||
/**
|
||||
* SSRF 安全的图片下载:校验 → pinned 连接 → 逐跳重定向复检 → 类型/尺寸钳制。
|
||||
*
|
||||
* @throws 任何校验失败/网络失败/超限/类型不符均抛 Error(消息含 Blocked SSRF /
|
||||
* Image exceeds / non-image 等),由调用方按"跳过该图"处理。
|
||||
*/
|
||||
export async function fetchImageAsBase64(
|
||||
url: string,
|
||||
options: ImageFetchOptions = {},
|
||||
): Promise<ImageFetchResult> {
|
||||
const { timeoutMs = 30_000, maxBytes = MAX_IMAGE_BYTES, signal } = options;
|
||||
|
||||
if (!/^https?:/i.test(url)) {
|
||||
throw new Error(`Blocked image fetch: protocol not allowed (url=${url.slice(0, 120)})`);
|
||||
}
|
||||
|
||||
let currentUrl = url;
|
||||
let res: Response | null = null;
|
||||
for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {
|
||||
// __imageFetcher(生产 = ssrfPinnedFetch)内部先 resolvePublicAddresses 全量
|
||||
// 校验再 pinned 连接;每一跳都进入本调用 —— 重定向目标同样受完整 SSRF 校验。
|
||||
res = await __imageFetcher.current(currentUrl, { redirect: 'manual' }, timeoutMs, signal);
|
||||
const next = resolveRedirectTarget(
|
||||
{ status: res.status, headers: { get: (n: string) => res!.headers.get(n) } },
|
||||
currentUrl,
|
||||
);
|
||||
if (!next) break;
|
||||
if (hop === MAX_REDIRECTS) {
|
||||
throw new Error(`Image fetch exceeded ${MAX_REDIRECTS} redirects`);
|
||||
}
|
||||
currentUrl = next;
|
||||
}
|
||||
if (!res) throw new Error('Image fetch failed: no response');
|
||||
|
||||
if (!res.ok) {
|
||||
throw new Error(`Image fetch failed: HTTP ${res.status} (url=${currentUrl.slice(0, 120)})`);
|
||||
}
|
||||
|
||||
const buf = await readBodyWithCap(res, maxBytes);
|
||||
if (buf.length === 0) {
|
||||
throw new Error('Image fetch failed: empty body');
|
||||
}
|
||||
|
||||
// 类型钳制:content-type 声明优先(必须 image/*),缺失/通用二进制按魔数嗅探;
|
||||
// 明确的非图片类型(text/html、application/json 等)直接拒绝。
|
||||
const declaredType = (res.headers.get('content-type') ?? '').split(';')[0].trim().toLowerCase();
|
||||
let mediaType: string | null = null;
|
||||
if (
|
||||
declaredType === '' ||
|
||||
declaredType === 'application/octet-stream' ||
|
||||
declaredType === 'binary/octet-stream'
|
||||
) {
|
||||
mediaType = sniffImageMediaType(buf);
|
||||
} else if (declaredType.startsWith('image/')) {
|
||||
mediaType = declaredType;
|
||||
} else {
|
||||
throw new Error(`Blocked image fetch: non-image content-type "${declaredType}"`);
|
||||
}
|
||||
if (!mediaType || !ALLOWED_MEDIA_TYPES.has(mediaType)) {
|
||||
throw new Error(
|
||||
`Blocked image fetch: unsupported media type "${mediaType ?? declaredType}" (allowed: png/jpeg/gif/webp)`,
|
||||
);
|
||||
}
|
||||
|
||||
return { base64: buf.toString('base64'), mediaType };
|
||||
}
|
||||
Reference in New Issue
Block a user