feat: v0.7.3 成本收口 · 状态一致 · 死账清理 — Prompt Cache 根治 + SSRF DNS Pinning + 87 用例扩充全量回归
P1 修复面收口: Prompt Cache 根治(日期/记忆/附件三类易变内容出 system 入用户消息 前置块 user-context.ts, system 跨 run 字节级稳定; Anthropic system 块数组化 + cache_control ephemeral 断言, DeepSeek 自动缓存前缀命中 — 多轮对话输入 token 成本降数量级); 编辑重发/重新生成幽灵 Trace 双侧根治(DB truncateMessagesAfter 同步过滤 metadata.traceSteps + 前端 trimTraceStepsByAnchor 镜像, 严格小于锚点 时间戳, 同毫秒等值判废); sessions:deleteMessage 死通道全链路删除(渲染层零调用 + message_count 漂移面); Ollama vision 能力门控全链路(MetonaModelInfo .supportsVision 贯穿 adapter/IPC/store/UI, model-capabilities.ts 三道判定纯函数, 未知保守放行); 记忆固化节流(consolidation-policy 纯函数: 总开关 + 内容门控 [回答>=200字符或存在成功工具调用] + 会话级 10 分钟频率窗口, 三 memory.* 配置键) P2 安全纵深: SSRF DNS Pinning 关闭 rebinding 窗口(ssrf-guard 重构 resolvePublicAddresses 单源; ssrf-dispatcher 以 undici Agent.connect.lookup 钉死校验 IP, TLS SNI 保持原域名, 一次性 dispatcher 用后即毁; 代理激活显式 退化为仅入口校验); web_fetch 重写手动逐跳重定向循环(每跳先校验后连接, 替代 redirect:follow 内核跟跳的中间跳裸奔, 上限 5 跳); http_request 换用 pinned fetch; web_search 可达性预检加固(私有 URL 零请求 + 不跟跳, 3xx 视为 可达); Agent 浏览器 CORS 通配收紧为 Origin 回显 + Vary: Origin; ConfirmationHook.forgetSession 会话终态清理(会话删除/abort 联动/SubAgent 终结三处接线, 根治 rememberedDecisions 泄漏) P3 架构还债: agent.enableReflection 死配置全链路接线(main→shared→引擎→ Orchestrator→设置开关, REFLECTING 状态真实可达); AgentLoopConfig.timeoutMs 死字段删除; MemoryManager.cleanupExpired 挂入健康检查周期(expires_at 回收 管道真实化); buildSafeEnv 收敛 utils/safe-env.ts 单源(run_command 与 MCP stdio 共用, 终结双实现漂移); Trace 生命周期治理(metadata 只保留最近 20 个 run — keepRecentRuns 纯函数; JSONL 录制启动自动清理保留 200 个 + 设置页 手动清理); SLO/健康快照可视化(app:healthSnapshot IPC + 设置页只读卡片 + 审计链一键校验) P4 能力演进: 会话标题 LLM 自动生成(TitleGenerator — 每会话幂等/并发重入复用 同一 Promise/自定义标题不覆盖/失败静默回退, Sidebar 经 config:changed 实时 刷新); MCP 自动重连(5s/15s/60s 退避最多 3 次, reconnecting 状态机, teardownConnection 内部拆除保留簿记 — 用户断开/开关关闭即时取消, 设置页 显示第 N/3 次); 死循环检测 ABAB 乒乓模式(最近4轮 A→B→A→B 交替判定, 补齐 docs 第五章"两状态反复切换"检测契约); i18n 第三阶段(ChatInput/LLMSettings/ OnboardingWizard/MemoryViewer 主链路文案出层, zh-CN + en-US 双字典补齐) 测试: 737 → 824 用例(+87, 新增 8 个测试文件 + 扩展 3 个)。新覆盖: user-context 分组/空值收缩/拼接契约、context-builder 字节级稳定性、Anthropic cache_control 四态、consolidation-policy 九路判定矩阵、ssrf-dispatcher(pinned lookup/重定向 解析/IP 校验)、forget-session 会话隔离、trace-lifecycle run 淘汰、 trace-trim 严格小于边界、safe-env 净化矩阵、mcp-reconnect 退避状态机 (fake timers)、title-generator 并发重入、SQLite 侧 truncate×TRACE 联动 (Electron ABI)。测试驱动修复: GIT_*/ 注释终止块注释、重连计数被自身重试 前置断开重置(拆 teardownConnection 保留簿记)、TitleGenerator 幂等占位与 并发去重的检查顺序竞态(去重先于幂等) 版本: 0.7.3; README 同步(配置表新增 agent.enableReflection/memory.*/mcp.autoReconnect) 回归: typecheck 双端 0 错误; ESLint 0/0; 系统 Node 771 通过 53 跳过 (better-sqlite3 ABI); Electron ABI 全量 824/824 零跳过
This commit is contained in:
@@ -0,0 +1,103 @@
|
||||
/**
|
||||
* SSRF DNS Pinning 测试(v0.7.3 P2-1)
|
||||
*
|
||||
* 锁定三个单元:
|
||||
* D1 createPinnedLookup —— 只返回校验阶段锁定的 IP 集合(过滤非法 family),
|
||||
* 空集合返回 ENOTFOUND(防御)。
|
||||
* D2 resolveRedirectTarget —— 重定向状态识别 + 相对 Location 解析 +
|
||||
* 非法/缺失 Location 返回 null。
|
||||
* D3 resolvePinnedIps —— IP 直连与私网拒绝(走 ssrf-guard 单一事实来源;
|
||||
* 域名解析路径由 ssrf-guard 表测覆盖,此处不重复触网)。
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { createPinnedLookup, resolveRedirectTarget, resolvePinnedIps } from '../ssrf-dispatcher';
|
||||
import type { LookupCallback } from '../ssrf-dispatcher';
|
||||
|
||||
describe('createPinnedLookup', () => {
|
||||
it('D1: 仅返回钉死的 IP 集合(忽略 hostname),family 正确标注', async () => {
|
||||
const lookup = createPinnedLookup(['93.184.216.34', '2606:2800:220:1:248:1893:25c8:1946']);
|
||||
const result = await new Promise<{ address: string; family: number }[]>((resolve, reject) => {
|
||||
const cb: LookupCallback = (err, addresses) => (err ? reject(err) : resolve(addresses!));
|
||||
lookup('attacker.example', {}, cb);
|
||||
});
|
||||
expect(result).toHaveLength(2);
|
||||
expect(result[0]).toEqual({ address: '93.184.216.34', family: 4 });
|
||||
expect(result[1].family).toBe(6);
|
||||
});
|
||||
|
||||
it('D1: 非法 family(非 IPv4/IPv6 字符串)被过滤', async () => {
|
||||
const lookup = createPinnedLookup(['not-an-ip']);
|
||||
await expect(
|
||||
new Promise((resolve, reject) => {
|
||||
const cb: LookupCallback = (err, addresses) => (err ? reject(err) : resolve(addresses));
|
||||
lookup('h', {}, cb as never);
|
||||
}),
|
||||
).rejects.toMatchObject({ code: 'ENOTFOUND' });
|
||||
});
|
||||
|
||||
it('D1: 空集合 → ENOTFOUND(防御:调用方不应构造空 pin dispatcher)', async () => {
|
||||
const lookup = createPinnedLookup([]);
|
||||
await expect(
|
||||
new Promise((resolve, reject) => {
|
||||
const cb: LookupCallback = (err, addresses) => (err ? reject(err) : resolve(addresses));
|
||||
lookup('h', {}, cb as never);
|
||||
}),
|
||||
).rejects.toMatchObject({ code: 'ENOTFOUND' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveRedirectTarget', () => {
|
||||
const makeResponse = (status: number, location?: string) => ({
|
||||
status,
|
||||
headers: {
|
||||
get: (name: string) => (name.toLowerCase() === 'location' ? (location ?? null) : null),
|
||||
},
|
||||
});
|
||||
|
||||
it('D2: 301/302/303/307/308 识别并解析绝对 Location', () => {
|
||||
for (const status of [301, 302, 303, 307, 308]) {
|
||||
expect(
|
||||
resolveRedirectTarget(makeResponse(status, 'https://cdn.example.com/x'), 'https://a.test/'),
|
||||
).toBe('https://cdn.example.com/x');
|
||||
}
|
||||
});
|
||||
|
||||
it('D2: 相对 Location 以当前 URL 为基解析(RFC 7231)', () => {
|
||||
expect(resolveRedirectTarget(makeResponse(302, '/next?a=1'), 'https://a.test/dir/page')).toBe(
|
||||
'https://a.test/next?a=1',
|
||||
);
|
||||
});
|
||||
|
||||
it('D2: 非 3xx 状态 → null(终态)', () => {
|
||||
expect(resolveRedirectTarget(makeResponse(200), 'https://a.test/')).toBeNull();
|
||||
expect(resolveRedirectTarget(makeResponse(404), 'https://a.test/')).toBeNull();
|
||||
});
|
||||
|
||||
it('D2: 缺失/非法 Location → null', () => {
|
||||
expect(resolveRedirectTarget(makeResponse(302), 'https://a.test/')).toBeNull();
|
||||
expect(resolveRedirectTarget(makeResponse(302, ''), 'https://a.test/')).toBeNull();
|
||||
expect(resolveRedirectTarget(makeResponse(302, 'http://[::bad'), 'https://a.test/')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolvePinnedIps', () => {
|
||||
it('D3: IP 直连 URL —— 公网 IP 直接返回', async () => {
|
||||
const ips = await resolvePinnedIps('https://93.184.216.34/x');
|
||||
expect(ips).toEqual(['93.184.216.34']);
|
||||
});
|
||||
|
||||
it('D3: 私有/回环 IP 直连被拒(单一事实来源 ssrf-guard)', async () => {
|
||||
for (const host of ['127.0.0.1', '10.0.0.5', '169.254.169.254', '192.168.1.1', '[::1]']) {
|
||||
await expect(resolvePinnedIps(`http://${host}/latest`)).rejects.toThrow(/Blocked SSRF/);
|
||||
}
|
||||
});
|
||||
|
||||
it('D3: 非 http/https 协议被拒', async () => {
|
||||
await expect(resolvePinnedIps('ftp://example.com')).rejects.toThrow(/not allowed/);
|
||||
});
|
||||
|
||||
it('D3: 非法 URL 被拒', async () => {
|
||||
await expect(resolvePinnedIps('not a url')).rejects.toThrow(/Invalid URL/);
|
||||
});
|
||||
});
|
||||
@@ -9,6 +9,8 @@
|
||||
|
||||
import { BrowserWindow, session } from 'electron';
|
||||
import log from 'electron-log';
|
||||
// v0.7.3 P2-2: CORS Origin 回显(纯函数在 network-utils,可表测)
|
||||
import { corsAllowOrigin, extractOriginHeader } from './network-utils';
|
||||
|
||||
/** Agent 浏览器专用 session partition — 与主应用 default session 完全隔离 */
|
||||
const AGENT_PARTITION = 'persist:metona-agent-browser';
|
||||
@@ -128,12 +130,22 @@ export class BrowserWindowManager {
|
||||
|
||||
// v0.3.0 修复: 使用 CORS 放行替代 webSecurity: false
|
||||
// 仅对 agent session 放行 CORS,不影响主应用
|
||||
// v0.7.3 P2-2 收紧: ACAO 从通配 '*' 改为回显请求 Origin —— 通配值让任意
|
||||
// 第三方页面都能借该分区跨域读取;回显等价保留截图/页面自身跨域能力,
|
||||
// 并附加 Vary: Origin 防止共享缓存把定向值串到其他 Origin。
|
||||
const agentSession = session.fromPartition(AGENT_PARTITION);
|
||||
agentSession.webRequest.onHeadersReceived((details, callback) => {
|
||||
// Electron 类型在此版本的 OnHeadersReceivedListenerDetails 上不暴露
|
||||
// requestHeaders —— 显式声明读取面(Origin 大小写不敏感提取)
|
||||
const requestHeaders = (
|
||||
details as unknown as { requestHeaders?: Record<string, string | string[] | undefined> }
|
||||
).requestHeaders;
|
||||
const originHeader = extractOriginHeader(requestHeaders);
|
||||
callback({
|
||||
responseHeaders: {
|
||||
...details.responseHeaders,
|
||||
'Access-Control-Allow-Origin': ['*'],
|
||||
'Access-Control-Allow-Origin': corsAllowOrigin(originHeader),
|
||||
Vary: [...(details.responseHeaders?.Vary ?? []), 'Origin'],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
@@ -25,6 +25,9 @@ import type { MetonaToolDef } from '../../../harness/types';
|
||||
import { MetonaToolCategory, MetonaRiskLevel } from '../../../harness/types';
|
||||
import { commandTouchesProtectedFile, isPathWithinWorkspace } from './file-guard';
|
||||
import type { SandboxManager } from '../../sandbox/sandbox';
|
||||
// v0.7.3 P3-2: 子进程环境净化收敛到 utils/safe-env.ts 单源
|
||||
// (与 MCP stdio 启动共用同一黑名单,历史双实现已漂移过一次)
|
||||
import { buildSafeChildEnv } from '../../../utils/safe-env';
|
||||
|
||||
const execAsync = promisify(exec);
|
||||
const execFileAsync = promisify(execFile);
|
||||
@@ -52,47 +55,22 @@ function decodeBuffer(buf: Buffer): string {
|
||||
/**
|
||||
* #9 修复 + 审查修复: 构建安全的子进程环境变量
|
||||
*
|
||||
* 审查修复: 原白名单方案遗漏了 GIT_* / PYTHONPATH / HTTP_PROXY 等常用变量,导致子进程功能破坏。
|
||||
* 改为黑名单方案:剔除包含敏感后缀的变量,保留其余。
|
||||
* v0.7.3 P3-2: 实现收敛到 utils/safe-env.ts(buildSafeChildEnv)——
|
||||
* 与 MCP stdio 启动共用同一黑名单,本文件仅保留 run_command 的运行时差异注入
|
||||
* (Windows 中文编码变量)。黑名单方案的设计原因见 safe-env.ts 模块注释:
|
||||
* 白名单方案会遗漏 GIT_* / PYTHONPATH / HTTP_PROXY 等常用变量导致子进程功能破坏。
|
||||
*/
|
||||
function buildSafeCommandEnv(isWindows: boolean): Record<string, string> {
|
||||
// 敏感变量后缀黑名单
|
||||
const SENSITIVE_SUFFIXES = [
|
||||
'_API_KEY',
|
||||
'_TOKEN',
|
||||
'_SECRET',
|
||||
'_PASSWORD',
|
||||
'_PASSWD',
|
||||
'_CREDENTIAL',
|
||||
'_CREDENTIALS',
|
||||
'_PRIVATE_KEY',
|
||||
];
|
||||
// 敏感变量名黑名单(精确匹配)
|
||||
const SENSITIVE_KEYS = new Set([
|
||||
'DEEPSEEK_API_KEY',
|
||||
'AGNES_API_KEY',
|
||||
'MIMO_API_KEY',
|
||||
'GITEA_PASSWORD',
|
||||
'DATABASE_PASSWORD',
|
||||
]);
|
||||
|
||||
const env: Record<string, string> = {};
|
||||
for (const [key, val] of Object.entries(process.env)) {
|
||||
if (!val) continue;
|
||||
if (SENSITIVE_KEYS.has(key)) continue;
|
||||
if (SENSITIVE_SUFFIXES.some((suffix) => key.toUpperCase().endsWith(suffix))) continue;
|
||||
env[key] = val;
|
||||
}
|
||||
|
||||
// 添加必要的运行时变量
|
||||
env.NODE_ENV = 'production';
|
||||
if (isWindows) {
|
||||
env.PYTHONIOENCODING = 'utf-8';
|
||||
env.LANG = 'zh_CN.UTF-8';
|
||||
env.LC_ALL = 'zh_CN.UTF-8';
|
||||
}
|
||||
|
||||
return env;
|
||||
return buildSafeChildEnv({
|
||||
runtime: isWindows
|
||||
? {
|
||||
NODE_ENV: 'production',
|
||||
PYTHONIOENCODING: 'utf-8',
|
||||
LANG: 'zh_CN.UTF-8',
|
||||
LC_ALL: 'zh_CN.UTF-8',
|
||||
}
|
||||
: { NODE_ENV: 'production' },
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -14,7 +14,11 @@ import type { MetonaToolDef } from '../../../harness/types';
|
||||
import { MetonaToolCategory, MetonaRiskLevel } from '../../../harness/types';
|
||||
// v0.6.4 P2-2: SSRF 校验收敛到共享模块 ssrf-guard.ts —— 原实现是本文件私有逻辑,
|
||||
// web_fetch 无校验造成工具层最大的安全不对称。单源后所有网络工具行为一致。
|
||||
// v0.7.3 P2-1 根治: 请求层升级为 ssrfPinnedFetch —— 校验通过的 IP 集合 pin 到
|
||||
// 连接层(undici connect.lookup),校验与连接共用同一批 IP,DNS rebinding
|
||||
// 窗口(M7 已知限制)就此关闭;代理激活时自动退化为仅入口校验(见模块注释)。
|
||||
import { validateSSRF } from './ssrf-guard';
|
||||
import { ssrfPinnedFetch } from './ssrf-dispatcher';
|
||||
|
||||
const ALLOWED_METHODS = ['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD'] as const;
|
||||
const MAX_BODY_BYTES = 50 * 1024; // 50KB
|
||||
@@ -52,7 +56,8 @@ const MAX_BODY_BYTES = 50 * 1024; // 50KB
|
||||
export class HttpRequestTool implements IMetonaTool {
|
||||
readonly definition: MetonaToolDef = {
|
||||
name: 'http_request',
|
||||
description: 'Send an HTTP/REST API request. Supports GET/POST/PUT/PATCH/DELETE/HEAD methods with custom headers and body. Response body is truncated to 50KB.',
|
||||
description:
|
||||
'Send an HTTP/REST API request. Supports GET/POST/PUT/PATCH/DELETE/HEAD methods with custom headers and body. Response body is truncated to 50KB.',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
@@ -64,7 +69,10 @@ export class HttpRequestTool implements IMetonaTool {
|
||||
},
|
||||
headers: { type: 'object', description: 'Request headers as key-value pairs' },
|
||||
body: { type: 'string', description: 'Request body (string)' },
|
||||
timeout: { type: 'number', description: 'Timeout in milliseconds (default 30000, max 60000)' },
|
||||
timeout: {
|
||||
type: 'number',
|
||||
description: 'Timeout in milliseconds (default 30000, max 60000)',
|
||||
},
|
||||
},
|
||||
required: ['url'],
|
||||
},
|
||||
@@ -102,15 +110,12 @@ export class HttpRequestTool implements IMetonaTool {
|
||||
};
|
||||
}
|
||||
|
||||
// 超时控制
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), timeout);
|
||||
|
||||
try {
|
||||
// 超时控制由 ssrfPinnedFetch 内部管理(超时 → ETIMEDOUT;
|
||||
// 工具执行层的 abort signal 经 context 传入 registry 兜底)
|
||||
{
|
||||
const fetchOptions: RequestInit = {
|
||||
method,
|
||||
headers,
|
||||
signal: controller.signal,
|
||||
// #10 修复: 禁用自动重定向跟随 — 防止重定向到内网地址绕过 SSRF 校验
|
||||
// 重定向后的 URL 由用户自行处理(响应中会包含 Location 头)
|
||||
redirect: 'manual',
|
||||
@@ -120,7 +125,9 @@ export class HttpRequestTool implements IMetonaTool {
|
||||
fetchOptions.body = body;
|
||||
}
|
||||
|
||||
const response = await fetch(url, fetchOptions);
|
||||
// v0.7.3 P2-1: pinned fetch —— 校验通过的 IP pin 到连接层,
|
||||
// 关闭校验-连接之间的 DNS rebinding 窗口
|
||||
const response = await ssrfPinnedFetch(url, fetchOptions, timeout);
|
||||
const text = await response.text();
|
||||
|
||||
// 截断到 50KB
|
||||
@@ -144,14 +151,14 @@ export class HttpRequestTool implements IMetonaTool {
|
||||
body: safeBody,
|
||||
truncated,
|
||||
ok: response.ok,
|
||||
success: true, // v0.3.1 修复 WARN-4: 成功路径添加 success 字段
|
||||
success: true, // v0.3.1 修复 WARN-4: 成功路径添加 success 字段
|
||||
};
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
} catch (error) {
|
||||
// 区分超时(AbortError)与其他网络错误
|
||||
if (error instanceof Error && error.name === 'AbortError') {
|
||||
// 区分超时与其他网络错误:AbortError(外部中断)与
|
||||
// ETIMEDOUT(ssrfPinnedFetch 超时转译,v0.7.3 P2-1)均归为超时语义
|
||||
const err = error as Error & { code?: string };
|
||||
if (err?.name === 'AbortError' || err?.code === 'ETIMEDOUT') {
|
||||
return { error: 'Request timeout', success: false };
|
||||
}
|
||||
const errMsg = error instanceof Error ? error.message : String(error);
|
||||
|
||||
@@ -34,7 +34,8 @@ export const UA_POOL = [
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0',
|
||||
];
|
||||
|
||||
export const MOBILE_UA = 'Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1';
|
||||
export const MOBILE_UA =
|
||||
'Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1';
|
||||
|
||||
export const ACCEPT_LANGUAGE_POOL = [
|
||||
'zh-CN,zh;q=0.9,en;q=0.8',
|
||||
@@ -54,21 +55,25 @@ export function buildAntiCrawlHeaders(
|
||||
const userAgent = mobileUA ? MOBILE_UA : UA_POOL[uaIdx];
|
||||
|
||||
let origin = '';
|
||||
try { origin = new URL(url).origin; } catch { /* ignore */ }
|
||||
try {
|
||||
origin = new URL(url).origin;
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
|
||||
return {
|
||||
'User-Agent': userAgent,
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
|
||||
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
|
||||
'Accept-Language': ACCEPT_LANGUAGE_POOL[langIdx],
|
||||
'Accept-Encoding': 'gzip, deflate, br',
|
||||
'Cache-Control': 'no-cache',
|
||||
'DNT': '1',
|
||||
'Referer': origin || '',
|
||||
DNT: '1',
|
||||
Referer: origin || '',
|
||||
'Sec-Fetch-Dest': 'document',
|
||||
'Sec-Fetch-Mode': 'navigate',
|
||||
'Sec-Fetch-Site': 'none',
|
||||
'Sec-Fetch-User': '?1',
|
||||
'Pragma': 'no-cache',
|
||||
Pragma: 'no-cache',
|
||||
};
|
||||
}
|
||||
|
||||
@@ -107,7 +112,15 @@ export function normalizeUrl(url: string): string {
|
||||
const u = new URL(url);
|
||||
|
||||
// 去除追踪参数
|
||||
const trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'gclid', 'fbclid'];
|
||||
const trackingParams = [
|
||||
'utm_source',
|
||||
'utm_medium',
|
||||
'utm_campaign',
|
||||
'utm_term',
|
||||
'utm_content',
|
||||
'gclid',
|
||||
'fbclid',
|
||||
];
|
||||
for (const p of trackingParams) u.searchParams.delete(p);
|
||||
|
||||
// H-6 增强: 排序查询参数(确保参数顺序一致,便于去重)
|
||||
@@ -125,7 +138,7 @@ export function normalizeUrl(url: string): string {
|
||||
(u.protocol === 'https:' && u.port === '443') ||
|
||||
(u.protocol === 'ws:' && u.port === '80') ||
|
||||
(u.protocol === 'wss:' && u.port === '443');
|
||||
const portSuffix = isDefaultPort ? '' : (u.port ? `:${u.port}` : '');
|
||||
const portSuffix = isDefaultPort ? '' : u.port ? `:${u.port}` : '';
|
||||
|
||||
// 强制小写 host
|
||||
return `${u.protocol}//${u.hostname.toLowerCase()}${portSuffix}${path}${u.search}${u.hash}`;
|
||||
@@ -158,47 +171,68 @@ export function isInterceptedPage(html: string): boolean {
|
||||
// ===== HTML → 纯文本转换 =====
|
||||
|
||||
const HTML_ENTITY_MAP: Record<string, string> = {
|
||||
' ': ' ', '<': '<', '>': '>', '&': '&', '"': '"',
|
||||
''': "'", '…': '…', '—': '—', '–': '–',
|
||||
'«': '«', '»': '»', '×': '×', '÷': '÷',
|
||||
'©': '©', '®': '®', '™': '™', '€': '€',
|
||||
'£': '£', '¥': '¥', '¢': '¢', '°': '°',
|
||||
' ': ' ',
|
||||
'<': '<',
|
||||
'>': '>',
|
||||
'&': '&',
|
||||
'"': '"',
|
||||
''': "'",
|
||||
'…': '…',
|
||||
'—': '—',
|
||||
'–': '–',
|
||||
'«': '«',
|
||||
'»': '»',
|
||||
'×': '×',
|
||||
'÷': '÷',
|
||||
'©': '©',
|
||||
'®': '®',
|
||||
'™': '™',
|
||||
'€': '€',
|
||||
'£': '£',
|
||||
'¥': '¥',
|
||||
'¢': '¢',
|
||||
'°': '°',
|
||||
};
|
||||
|
||||
export function htmlToText(html: string): string {
|
||||
return html
|
||||
// 移除噪声标签及内容
|
||||
.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
|
||||
.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
|
||||
.replace(/<noscript[^>]*>[\s\S]*?<\/noscript>/gi, '')
|
||||
.replace(/<nav[^>]*>[\s\S]*?<\/nav>/gi, '')
|
||||
.replace(/<header[^>]*>[\s\S]*?<\/header>/gi, '')
|
||||
.replace(/<footer[^>]*>[\s\S]*?<\/footer>/gi, '')
|
||||
.replace(/<aside[^>]*>[\s\S]*?<\/aside>/gi, '')
|
||||
.replace(/<iframe[^>]*>[\s\S]*?<\/iframe>/gi, '')
|
||||
.replace(/<svg[^>]*>[\s\S]*?<\/svg>/gi, '')
|
||||
// 移除 HTML 注释
|
||||
.replace(/<!--[\s\S]*?-->/g, '')
|
||||
// 块级标签转换行
|
||||
.replace(/<\/?(p|div|h[1-6]|li|tr|blockquote|section|article|pre|br|hr)[^>]*>/gi, '\n')
|
||||
// 表格单元格转制表符
|
||||
.replace(/<\/?(td|th)[^>]*>/gi, '\t')
|
||||
// 移除剩余标签
|
||||
.replace(/<[^>]+>/g, '')
|
||||
// 解码 HTML 实体
|
||||
.replace(/&#(\d+);/g, (_, n) => String.fromCharCode(Number(n)))
|
||||
.replace(/&#x([0-9a-f]+);/gi, (_, h) => String.fromCharCode(parseInt(h, 16)))
|
||||
.replace(/&[a-z]+;/gi, (m) => HTML_ENTITY_MAP[m.toLowerCase()] ?? m)
|
||||
// 清理空白
|
||||
.replace(/\n{3,}/g, '\n\n')
|
||||
.replace(/[ \t]+/g, ' ')
|
||||
.replace(/^[ \t]+/gm, '')
|
||||
.trim();
|
||||
return (
|
||||
html
|
||||
// 移除噪声标签及内容
|
||||
.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
|
||||
.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
|
||||
.replace(/<noscript[^>]*>[\s\S]*?<\/noscript>/gi, '')
|
||||
.replace(/<nav[^>]*>[\s\S]*?<\/nav>/gi, '')
|
||||
.replace(/<header[^>]*>[\s\S]*?<\/header>/gi, '')
|
||||
.replace(/<footer[^>]*>[\s\S]*?<\/footer>/gi, '')
|
||||
.replace(/<aside[^>]*>[\s\S]*?<\/aside>/gi, '')
|
||||
.replace(/<iframe[^>]*>[\s\S]*?<\/iframe>/gi, '')
|
||||
.replace(/<svg[^>]*>[\s\S]*?<\/svg>/gi, '')
|
||||
// 移除 HTML 注释
|
||||
.replace(/<!--[\s\S]*?-->/g, '')
|
||||
// 块级标签转换行
|
||||
.replace(/<\/?(p|div|h[1-6]|li|tr|blockquote|section|article|pre|br|hr)[^>]*>/gi, '\n')
|
||||
// 表格单元格转制表符
|
||||
.replace(/<\/?(td|th)[^>]*>/gi, '\t')
|
||||
// 移除剩余标签
|
||||
.replace(/<[^>]+>/g, '')
|
||||
// 解码 HTML 实体
|
||||
.replace(/&#(\d+);/g, (_, n) => String.fromCharCode(Number(n)))
|
||||
.replace(/&#x([0-9a-f]+);/gi, (_, h) => String.fromCharCode(parseInt(h, 16)))
|
||||
.replace(/&[a-z]+;/gi, (m) => HTML_ENTITY_MAP[m.toLowerCase()] ?? m)
|
||||
// 清理空白
|
||||
.replace(/\n{3,}/g, '\n\n')
|
||||
.replace(/[ \t]+/g, ' ')
|
||||
.replace(/^[ \t]+/gm, '')
|
||||
.trim()
|
||||
);
|
||||
}
|
||||
|
||||
// ===== 流式读取(大文件保护,10MB 上限) =====
|
||||
|
||||
export async function readBodyWithLimit(response: Response, maxBytes = 10 * 1024 * 1024): Promise<string> {
|
||||
export async function readBodyWithLimit(
|
||||
response: Response,
|
||||
maxBytes = 10 * 1024 * 1024,
|
||||
): Promise<string> {
|
||||
const contentLength = response.headers.get('content-length');
|
||||
if (contentLength && parseInt(contentLength) > maxBytes) {
|
||||
throw new Error(`Response too large: ${contentLength} bytes (limit: ${maxBytes})`);
|
||||
@@ -249,6 +283,40 @@ export function logTool(toolName: string, message: string): void {
|
||||
log.info(`[Tool:${toolName}] ${message}`);
|
||||
}
|
||||
|
||||
// ===== v0.7.3 P2-2: Agent 浏览器 CORS Origin 回显 =====
|
||||
|
||||
/**
|
||||
* 计算响应应携带的 Access-Control-Allow-Origin 值(纯函数,表测锁定)。
|
||||
*
|
||||
* 背景:Agent 浏览器专用 session 此前对所有响应注入 ACAO:* —— 任意被 Agent
|
||||
* 打开的第三方页面都能借该分区无差别跨域读取。改为回显请求 Origin(等价能力:
|
||||
* 页面对自己的 Origin 仍可跨域读,如截图所需),无 Origin(同源导航/非浏览器
|
||||
* 客户端)回退 '*' 保持既有能力不回退。
|
||||
*
|
||||
* @param requestOrigin 请求头 Origin(可能为 undefined / 任意字符串)
|
||||
* @returns 应写入响应的 ACAO 值(单元素数组,供 Electron responseHeaders 使用)
|
||||
*/
|
||||
export function corsAllowOrigin(requestOrigin: string | undefined | null): string[] {
|
||||
const origin = requestOrigin?.trim();
|
||||
if (origin && /^https?:\/\//i.test(origin)) {
|
||||
return [origin];
|
||||
}
|
||||
return ['*'];
|
||||
}
|
||||
|
||||
/** 从请求头集合中大小写不敏感地提取 Origin 值 */
|
||||
export function extractOriginHeader(
|
||||
requestHeaders: Record<string, string | string[] | undefined> | undefined,
|
||||
): string | undefined {
|
||||
if (!requestHeaders) return undefined;
|
||||
for (const [key, value] of Object.entries(requestHeaders)) {
|
||||
if (key.toLowerCase() === 'origin') {
|
||||
return Array.isArray(value) ? value[0] : value;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// ===== v0.6.4 P4-4: HTML → Markdown 转换(web_fetch extract_mode='markdown') =====
|
||||
//
|
||||
// v0.6.4 收尾:私有 npm 凭据解锁后,按开发规范第一铁律把第一轮的临时自写实现
|
||||
@@ -285,11 +353,9 @@ const LIST_LINE = /^\s*(?:- |\d+\. )/;
|
||||
* 2. 仅当"空行两侧都是同一列表的条目行"时移除该空行(绝不吞条目、不影响段落间距)。
|
||||
*/
|
||||
function collapseListGaps(markdown: string): string {
|
||||
const lines = markdown.split('\n').map((line) =>
|
||||
line
|
||||
.replace(/^(\s*)- {2,}/, '$1- ')
|
||||
.replace(/^(\s*\d+\.)\s{2,}/, '$1 '),
|
||||
);
|
||||
const lines = markdown
|
||||
.split('\n')
|
||||
.map((line) => line.replace(/^(\s*)- {2,}/, '$1- ').replace(/^(\s*\d+\.)\s{2,}/, '$1 '));
|
||||
|
||||
const isListItem = (l: string | undefined): boolean => (l ?? '').length > 0 && LIST_LINE.test(l!);
|
||||
|
||||
@@ -321,5 +387,7 @@ export function htmlToMarkdown(html: string): string {
|
||||
return '';
|
||||
}
|
||||
|
||||
return collapseListGaps(md).replace(/\n{3,}/g, '\n\n').trim();
|
||||
return collapseListGaps(md)
|
||||
.replace(/\n{3,}/g, '\n\n')
|
||||
.trim();
|
||||
}
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
/**
|
||||
* SSRF DNS Pinning Dispatcher(v0.7.3 P2-1)
|
||||
*
|
||||
* 关闭 M7 审查确认的 DNS rebinding 窗口:此前 validateSSRF 在校验阶段解析一次
|
||||
* DNS,fetch 实际连接时 undici 再次解析 —— 两次解析之间攻击者可切换 DNS 记录
|
||||
* (TTL=0)把连接导向内网。原注释断言"Node fetch 下无法彻底关闭",该结论只对
|
||||
* 全局 fetch 成立;主进程已依赖 undici(network-proxy 的 setGlobalDispatcher),
|
||||
* undici 的 Agent 支持 connect.lookup 自定义 —— 校验通过的 IP 集合可精确 pin 到
|
||||
* 连接层,TLS SNI/证书校验仍基于原始域名(undici 将 servername 保持为 hostname)。
|
||||
*
|
||||
* 契约:
|
||||
* - resolvePublicAddresses(ssrf-guard)是校验与地址解析的唯一事实来源,
|
||||
* 本模块 pin 的就是它返回的那批 IP —— 校验与连接同源,无双解析窗口;
|
||||
* - 代理激活(network-proxy.isProxyActive())时 pinning 不可实现(DNS 在代理端
|
||||
* 解析)且 per-request dispatcher 会旁路用户代理 —— 退化为"仅入口校验",
|
||||
* 走全局 dispatcher(保持既有语义与代理兼容);
|
||||
* - fetchWithTimeoutPinned 合并外部 abort signal 与超时控制,语义对齐
|
||||
* BaseAdapter.fetchWithTimeout(超时 → ETIMEDOUT 可重试;外部中断原样抛出);
|
||||
* - 每次 pinned 请求构造一次性 Agent 并在 finally 中 close(连接池即用即毁,
|
||||
* 防止把"上一请求的 pin 集合"泄漏给后续请求)。
|
||||
*/
|
||||
|
||||
import { Agent, fetch as undiciFetch } from 'undici';
|
||||
import { isIP } from 'node:net';
|
||||
import { resolvePublicAddresses } from './ssrf-guard';
|
||||
import { fetchWithTimeout } from './network-utils';
|
||||
import { isProxyActive } from '../../../utils/network-proxy';
|
||||
import log from 'electron-log';
|
||||
|
||||
/** 标准 dns.lookup 回调签名(undici connect.lookup 消费) */
|
||||
export type LookupCallback = (
|
||||
err: NodeJS.ErrnoException | null,
|
||||
addresses?: Array<{ address: string; family: number }>,
|
||||
) => void;
|
||||
|
||||
/** undici connect.lookup 的函数签名形态 */
|
||||
export type PinnedLookup = (hostname: string, options: unknown, callback: LookupCallback) => void;
|
||||
|
||||
/**
|
||||
* 构造"钉死 IP 集合"的 lookup 函数:无论传入什么 hostname,都只返回校验阶段
|
||||
* 锁定的公网地址(过滤非法 family)。集合为空时返回 ENOTFOUND(防御性——
|
||||
* 调用方在集合为空时不应构造 dispatcher)。
|
||||
*/
|
||||
export function createPinnedLookup(allowedIps: string[]): PinnedLookup {
|
||||
return (_hostname, _options, callback) => {
|
||||
process.nextTick(() => {
|
||||
const addresses = allowedIps
|
||||
.map((ip) => ({ address: ip, family: isIP(ip) }))
|
||||
.filter((a): a is { address: string; family: number } => a.family === 4 || a.family === 6);
|
||||
if (addresses.length === 0) {
|
||||
const err: NodeJS.ErrnoException = new Error('pinned lookup: no allowed addresses');
|
||||
err.code = 'ENOTFOUND';
|
||||
callback(err, undefined);
|
||||
return;
|
||||
}
|
||||
callback(null, addresses);
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* 校验 URL 并返回 pinning 用的公网 IP 集合。
|
||||
* 校验失败原样抛出(调用方按 SSRF 阻断处理)。
|
||||
*/
|
||||
export async function resolvePinnedIps(url: string): Promise<string[]> {
|
||||
return resolvePublicAddresses(url);
|
||||
}
|
||||
|
||||
/**
|
||||
* 带 SSRF pinning 的 fetch(http_request / web_fetch Phase1 / 可达性预检共用)。
|
||||
*
|
||||
* 行为:
|
||||
* 1. 代理激活 → 退化为普通 fetchWithTimeout(仅入口校验语义,见模块注释);
|
||||
* 2. 否则 → 解析并校验公网 IP → 一次性 undici Agent(pinned lookup)发起请求;
|
||||
* 3. 超时/外部中断语义与 fetchWithTimeout 对齐;
|
||||
* 4. 返回 Response 与全局 fetch 兼容(status/ok/headers/text/url/body)。
|
||||
*
|
||||
* @param url 目标 URL(调用方已保证 http/https;本函数再做一次全量 SSRF 校验)
|
||||
* @param init RequestInit(redirect 等由调用方决定)
|
||||
* @param timeoutMs 请求超时
|
||||
* @param externalSignal 外部 abort 信号(引擎中断透传,可选)
|
||||
*/
|
||||
export async function ssrfPinnedFetch(
|
||||
url: string,
|
||||
init: RequestInit,
|
||||
timeoutMs: number,
|
||||
externalSignal?: AbortSignal,
|
||||
): Promise<Response> {
|
||||
const ips = await resolvePublicAddresses(url);
|
||||
|
||||
// 代理激活:DNS 在代理端解析,pinning 不可实现;走全局 dispatcher 保持代理语义
|
||||
if (isProxyActive()) {
|
||||
return fetchWithTimeout(url, init, timeoutMs);
|
||||
}
|
||||
|
||||
// 外部信号已中止 → 直接抛 AbortError(对齐 fetchWithTimeout 行为)
|
||||
if (externalSignal?.aborted) {
|
||||
const err = new Error('Aborted');
|
||||
err.name = 'AbortError';
|
||||
throw err;
|
||||
}
|
||||
|
||||
const controller = new AbortController();
|
||||
let timedOut = false;
|
||||
const timer = setTimeout(() => {
|
||||
timedOut = true;
|
||||
controller.abort();
|
||||
}, timeoutMs);
|
||||
|
||||
const onExternalAbort = () => controller.abort();
|
||||
if (externalSignal) {
|
||||
externalSignal.addEventListener('abort', onExternalAbort, { once: true });
|
||||
}
|
||||
|
||||
// 一次性 pinned Agent(connect 超时对齐 network-proxy 的 15s 连接上限)
|
||||
const dispatcher = new Agent({
|
||||
connect: { timeout: 15_000, lookup: createPinnedLookup(ips) as never },
|
||||
});
|
||||
|
||||
try {
|
||||
const response = await undiciFetch(url, {
|
||||
...(init as Record<string, unknown>),
|
||||
signal: controller.signal,
|
||||
dispatcher,
|
||||
} as never);
|
||||
return response as unknown as Response;
|
||||
} catch (err) {
|
||||
const externalAborted = externalSignal?.aborted === true;
|
||||
if (timedOut && !externalAborted) {
|
||||
const timeoutError = new Error(
|
||||
`Request timed out after ${timeoutMs}ms (url=${String(url).slice(0, 120)})`,
|
||||
);
|
||||
(timeoutError as Error & { code: string }).code = 'ETIMEDOUT';
|
||||
throw timeoutError;
|
||||
}
|
||||
throw err;
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
if (externalSignal) {
|
||||
externalSignal.removeEventListener('abort', onExternalAbort);
|
||||
}
|
||||
// 一次性 dispatcher 用后即毁(连接池不跨请求复用,防止 pin 集合泄漏)
|
||||
void dispatcher.close().catch((closeErr) => {
|
||||
log.debug(`[SSRFDispatcher] dispatcher close failed: ${(closeErr as Error).message}`);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const REDIRECT_STATUS = new Set([301, 302, 303, 307, 308]);
|
||||
|
||||
/**
|
||||
* 解析重定向目标(纯函数,表测锁定)。
|
||||
*
|
||||
* @returns 下一跳绝对 URL;非重定向状态/缺失/非法 Location 返回 null
|
||||
* (表示当前响应即终态或无法跟随,由调用方按既有语义处理)。
|
||||
* 相对 Location 以 currentUrl 为基解析(RFC 7231)。
|
||||
*/
|
||||
export function resolveRedirectTarget(
|
||||
response: { status: number; headers: { get(name: string): string | null } },
|
||||
currentUrl: string,
|
||||
): string | null {
|
||||
if (!REDIRECT_STATUS.has(response.status)) return null;
|
||||
const location = response.headers.get('location');
|
||||
if (!location) return null;
|
||||
try {
|
||||
return new URL(location, currentUrl).toString();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -30,21 +30,21 @@ export function isPrivateIP(ip: string): boolean {
|
||||
// IPv4 直接检测
|
||||
if (isIP(ip) === 4) {
|
||||
const parts = ip.split('.').map(Number);
|
||||
if (parts[0] === 127) return true; // 回环
|
||||
if (parts[0] === 10) return true; // 内网
|
||||
if (parts[0] === 127) return true; // 回环
|
||||
if (parts[0] === 10) return true; // 内网
|
||||
if (parts[0] === 192 && parts[1] === 168) return true; // 内网
|
||||
if (parts[0] === 172 && parts[1] >= 16 && parts[1] <= 31) return true; // 内网
|
||||
if (parts[0] === 169 && parts[1] === 254) return true; // 链路本地(含云元数据)
|
||||
if (parts[0] === 0) return true; // 0.0.0.0/8
|
||||
if (parts[0] >= 224) return true; // 组播 + 保留
|
||||
if (parts[0] === 169 && parts[1] === 254) return true; // 链路本地(含云元数据)
|
||||
if (parts[0] === 0) return true; // 0.0.0.0/8
|
||||
if (parts[0] >= 224) return true; // 组播 + 保留
|
||||
return false;
|
||||
}
|
||||
|
||||
// IPv6 检测
|
||||
if (isIP(ip) === 6) {
|
||||
const lower = ip.toLowerCase();
|
||||
if (lower === '::1') return true; // 回环
|
||||
if (lower.startsWith('fe80:')) return true; // 链路本地
|
||||
if (lower === '::1') return true; // 回环
|
||||
if (lower.startsWith('fe80:')) return true; // 链路本地
|
||||
if (lower.startsWith('fc') || lower.startsWith('fd')) return true; // 唯一本地
|
||||
// ::ffff: 映射的 IPv4 — 提取 IPv4 部分递归检测
|
||||
const v4MappedMatch = lower.match(/::ffff:(\d+\.\d+\.\d+\.\d+)$/);
|
||||
@@ -57,16 +57,22 @@ export function isPrivateIP(ip: string): boolean {
|
||||
}
|
||||
|
||||
/**
|
||||
* SSRF 校验 — 解析 URL 域名并校验 IP
|
||||
* SSRF 校验 + 公网地址解析(单一事实来源)
|
||||
*
|
||||
* v0.7.3 P2-1 重构:resolvePublicAddresses 承载全部校验逻辑并返回解析出的
|
||||
* 公网 IP 集合;validateSSRF 成为它的"只要不抛错"薄包装。这样 pinning 层
|
||||
* (ssrf-dispatcher)能拿到与校验完全同一批 IP,避免"校验一次解析、连接
|
||||
* 再解析一次"的双解析不一致。
|
||||
*
|
||||
* 1. 协议白名单:仅允许 http/https
|
||||
* 2. hostname 为 IP 时直接检测
|
||||
* 3. 域名 — DNS 解析后检测所有 IP;任意一个 IP 为私有即拒绝
|
||||
* (防止 DNS rebinding 中只校验第一个 IP 的绕过)
|
||||
*
|
||||
* @returns 校验通过的全部公网 IP(供 DNS pinning 使用)
|
||||
* @throws 如果 URL 指向私有/内网/回环地址或协议不被允许
|
||||
*/
|
||||
export async function validateSSRF(url: string): Promise<void> {
|
||||
export async function resolvePublicAddresses(url: string): Promise<string[]> {
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
@@ -86,7 +92,7 @@ export async function validateSSRF(url: string): Promise<void> {
|
||||
if (isPrivateIP(hostname)) {
|
||||
throw new Error(`Blocked SSRF: ${hostname} is a private/loopback address`);
|
||||
}
|
||||
return;
|
||||
return [hostname];
|
||||
}
|
||||
|
||||
// 域名 — DNS 解析后检测所有 IP
|
||||
@@ -94,22 +100,39 @@ export async function validateSSRF(url: string): Promise<void> {
|
||||
try {
|
||||
addresses = await lookup(hostname, { all: true });
|
||||
} catch (err) {
|
||||
throw new Error(`Blocked SSRF: DNS resolution failed for ${hostname}: ${(err as Error).message}`);
|
||||
throw new Error(
|
||||
`Blocked SSRF: DNS resolution failed for ${hostname}: ${(err as Error).message}`,
|
||||
);
|
||||
}
|
||||
|
||||
if (addresses.length === 0) {
|
||||
throw new Error(`Blocked SSRF: no DNS records for ${hostname}`);
|
||||
}
|
||||
|
||||
const publicIps: string[] = [];
|
||||
for (const { address } of addresses) {
|
||||
if (isPrivateIP(address)) {
|
||||
throw new Error(`Blocked SSRF: ${hostname} resolves to private IP ${address}`);
|
||||
}
|
||||
publicIps.push(address);
|
||||
}
|
||||
return publicIps;
|
||||
}
|
||||
|
||||
/**
|
||||
* SSRF 校验 — 解析 URL 域名并校验 IP(v0.7.3 起为 resolvePublicAddresses 的
|
||||
* "仅校验不取值"包装,校验逻辑单一来源在后者)
|
||||
*
|
||||
* @throws 如果 URL 指向私有/内网/回环地址或协议不被允许
|
||||
*/
|
||||
export async function validateSSRF(url: string): Promise<void> {
|
||||
await resolvePublicAddresses(url);
|
||||
}
|
||||
|
||||
/** validateSSRF 的不抛错包装:返回结构化结果供工具 execute 直接 return */
|
||||
export async function safeValidateSSRF(url: string): Promise<{ ok: true } | { ok: false; error: string }> {
|
||||
export async function safeValidateSSRF(
|
||||
url: string,
|
||||
): Promise<{ ok: true } | { ok: false; error: string }> {
|
||||
try {
|
||||
await validateSSRF(url);
|
||||
return { ok: true };
|
||||
|
||||
@@ -18,7 +18,6 @@ import { MetonaToolCategory, MetonaRiskLevel } from '../../../harness/types';
|
||||
import {
|
||||
fetchCache,
|
||||
buildAntiCrawlHeaders,
|
||||
fetchWithTimeout,
|
||||
htmlToText,
|
||||
isInterceptedPage,
|
||||
readBodyWithLimit,
|
||||
@@ -30,14 +29,21 @@ import { getBrowserManager } from './browser';
|
||||
// v0.6.4 P2-2 根治安全不对称:web_fetch 此前完全没有 SSRF 校验(仅协议检查)且
|
||||
// requiresPermission:false —— LLM 可直接抓取 http://127.0.0.1:<port>、
|
||||
// http://169.254.169.254/latest/meta-data 等内网/云元数据地址,浏览器回退通道
|
||||
// 同样可达内网。现复用共享 ssrf-guard 模块(与 http_request 同源同行为):
|
||||
// 入口校验 + HTTP 重定向终态 URL 复检(堵 redirect:'follow' 绕道内网的口子)。
|
||||
// 同样可达内网。现复用共享 ssrf-guard 模块(与 http_request 同源同行为)。
|
||||
// v0.7.3 P2-1 根治: 抓取层升级为 ssrfPinnedFetch —— 校验通过的 IP 集合 pin 到
|
||||
// 连接层(undici connect.lookup),关闭校验-连接之间的 DNS rebinding 窗口;
|
||||
// 重定向改为逐跳手动跟随,每一跳都先校验后连接(原 redirect:'follow' 下
|
||||
// 中间跳转在"终态复检"之前已真实发出,可触达内网)。
|
||||
import { validateSSRF } from './ssrf-guard';
|
||||
import { resolveRedirectTarget, ssrfPinnedFetch } from './ssrf-dispatcher';
|
||||
|
||||
// ===== 跳过重试的状态码 =====
|
||||
|
||||
const SKIP_RETRY_STATUS = new Set([403, 429, 502, 503]);
|
||||
|
||||
/** v0.7.3 P2-1: 单次抓取允许的最大重定向跳数(每跳均经校验 + pinning) */
|
||||
const MAX_REDIRECT_HOPS = 5;
|
||||
|
||||
// ===== WebFetchTool =====
|
||||
|
||||
export class WebFetchTool implements IMetonaTool {
|
||||
@@ -188,25 +194,55 @@ export class WebFetchTool implements IMetonaTool {
|
||||
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
||||
try {
|
||||
const headers = buildAntiCrawlHeaders(url, attempt, mobileUA);
|
||||
const response = await fetchWithTimeout(url, { headers, redirect: 'follow' }, 20_000);
|
||||
|
||||
// v0.6.4 P2-2: 重定向终态复检 —— redirect:'follow' 下 fetch 可能跟随跳转
|
||||
// 到与入口校验不同的目标;SSRF 校验 initial URL 后再对 response.url(终态)
|
||||
// 复检,堵住"外网跳内网"绕道。终态指向私有地址时按拦截处理转入浏览器通道
|
||||
// 也会被浏览器侧域名校验拒绝。
|
||||
if (response.url && response.url !== url) {
|
||||
try {
|
||||
await validateSSRF(response.url);
|
||||
} catch (ssrfErr) {
|
||||
// ===== v0.7.3 P2-1 根治: 手动逐跳重定向 + 每跳 SSRF 校验 + DNS pinning =====
|
||||
// 原 redirect:'follow' 下 undici 在内核自动跟跳:跳转目标仅在"终态复检"
|
||||
// 时被校验,中间跳转的请求已经真实发出(可触达内网/元数据地址)。
|
||||
// 现改为逐跳手动跟随:每跳由 ssrfPinnedFetch 发出(校验通过 IP pin 到
|
||||
// 连接层),Location 目标显式校验通过后才允许下一跳;私有地址/非法
|
||||
// 协议目标按 blocked 语义立即终止(禁止重试与浏览器回退)。
|
||||
let currentUrl = url;
|
||||
let response: Response | null = null;
|
||||
let redirectBlocked: string | null = null;
|
||||
|
||||
for (let hop = 0; hop <= MAX_REDIRECT_HOPS; hop++) {
|
||||
response = await ssrfPinnedFetch(currentUrl, { headers, redirect: 'manual' }, 20_000);
|
||||
const next = resolveRedirectTarget(response, currentUrl);
|
||||
if (next === null) break; // 非重定向(或无/非法 Location)—— 当前响应即终态
|
||||
if (hop === MAX_REDIRECT_HOPS) {
|
||||
return {
|
||||
success: false,
|
||||
html: '',
|
||||
text: '',
|
||||
intercepted: false,
|
||||
blocked: true,
|
||||
reason: `Redirect target blocked by SSRF guard: ${(ssrfErr as Error).message}`,
|
||||
reason: `Too many redirects (>${MAX_REDIRECT_HOPS})`,
|
||||
};
|
||||
}
|
||||
// 下一跳目标显式校验(ssrfPinnedFetch 内部还会再次校验+pinning;
|
||||
// 这里提前拦截以保证 blocked 语义:不重试、不进浏览器回退)
|
||||
try {
|
||||
await validateSSRF(next);
|
||||
} catch (ssrfErr) {
|
||||
redirectBlocked = `Redirect target blocked by SSRF guard: ${(ssrfErr as Error).message}`;
|
||||
break;
|
||||
}
|
||||
currentUrl = next;
|
||||
response = null; // 丢弃中间跳转响应,下一跳重新抓取
|
||||
}
|
||||
|
||||
if (redirectBlocked) {
|
||||
return {
|
||||
success: false,
|
||||
html: '',
|
||||
text: '',
|
||||
intercepted: false,
|
||||
blocked: true,
|
||||
reason: redirectBlocked,
|
||||
};
|
||||
}
|
||||
if (!response) {
|
||||
// 防御性:循环正常结束必然携带终态响应
|
||||
return { success: false, html: '', text: '', intercepted: false, reason: 'No response' };
|
||||
}
|
||||
|
||||
// 跳过重试的状态码 → 直接进入浏览器回退
|
||||
|
||||
@@ -26,6 +26,9 @@ import {
|
||||
buildSearXNGAuthHeaders,
|
||||
logTool,
|
||||
} from './network-utils';
|
||||
// v0.7.3 P2-1: 可达性预检经 SSRF 校验 + DNS pinning(结果 URL 是不可信外部输入)
|
||||
import { safeValidateSSRF } from './ssrf-guard';
|
||||
import { ssrfPinnedFetch } from './ssrf-dispatcher';
|
||||
import type { WebFetchTool } from './web-fetch';
|
||||
|
||||
// ===== 类型定义 =====
|
||||
@@ -330,6 +333,10 @@ function parse360Regex(html: string): SearchResult[] {
|
||||
}
|
||||
|
||||
// ===== 可达性预检 =====
|
||||
// v0.7.3 P2-1: 可达性预检的 URL 来自不可信的搜索结果 —— HEAD 探测同样不得
|
||||
// 触达内网/元数据地址。校验失败的 URL 直接标记不可达(不发起任何请求);
|
||||
// 探测经 ssrfPinnedFetch(DNS pinning),重定向不自动跟随(3xx 即视为可达 ——
|
||||
// 链接活性已证明,且跟跳目标不再绕过校验)。
|
||||
|
||||
async function checkReachability(urls: string[], concurrency = 5): Promise<Map<string, boolean>> {
|
||||
const result = new Map<string, boolean>();
|
||||
@@ -337,8 +344,13 @@ async function checkReachability(urls: string[], concurrency = 5): Promise<Map<s
|
||||
const batch = urls.slice(i, i + concurrency);
|
||||
const checks = batch.map(async (url) => {
|
||||
try {
|
||||
const resp = await fetchWithTimeout(url, { method: 'HEAD', redirect: 'follow' }, 3_000);
|
||||
result.set(url, resp.ok);
|
||||
const ssrf = await safeValidateSSRF(url);
|
||||
if (!ssrf.ok) {
|
||||
result.set(url, false);
|
||||
return;
|
||||
}
|
||||
const resp = await ssrfPinnedFetch(url, { method: 'HEAD', redirect: 'manual' }, 3_000);
|
||||
result.set(url, resp.ok || (resp.status >= 300 && resp.status < 400));
|
||||
} catch {
|
||||
result.set(url, false);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user