optimize: enhance web_search, web_fetch, browser tools

- browser.ts: Fix XSS injection in browserClick/browserType via JSON.stringify
- browser.ts: Replace fixed 1000ms wait with did-finish-load event + 30s timeout
- web_fetch: Add 15s HTTP timeout via AbortController
- web_fetch: Improve HTML→text conversion (block-level tags→newlines, nav/header/footer removal, comprehensive HTML entity decoding)
- web_fetch: Add content-length pre-check (10MB limit)
- web_search: Add 15s HTTP timeout per engine request
- web_search: Add URL deduplication across results
- web_search: Clean residual HTML tags/entities from snippets
- download_file: Add 60s download timeout
- tool-registry: Fix missing logWarn import
- Extract shared decodeHTMLEntities with 40+ entity mappings
This commit is contained in:
OpenClaw Agent
2026-04-28 11:30:55 +08:00
parent 45b8b6c330
commit 4781abadf0
3 changed files with 193 additions and 60 deletions
+39 -8
View File
@@ -37,13 +37,35 @@ function getAgentBrowser(): BrowserWindow {
return agentBrowser; return agentBrowser;
} }
/** 页面加载超时(毫秒) */
const PAGE_LOAD_TIMEOUT = 30_000;
/** 打开 URL */ /** 打开 URL */
export async function browserOpen(url: string): Promise<{ success: boolean; title?: string; url?: string; error?: string }> { export async function browserOpen(url: string): Promise<{ success: boolean; title?: string; url?: string; error?: string }> {
try { try {
const win = getAgentBrowser(); const win = getAgentBrowser();
sendLog('info', `🌐 browser 打开`, url.slice(0, 100)); sendLog('info', `🌐 browser 打开`, url.slice(0, 100));
await win.loadURL(url);
await new Promise(r => setTimeout(r, 1000)); // 等待页面实际加载完成,而非固定延时
const loadPromise = new Promise<void>((resolve, reject) => {
const timeout = setTimeout(() => {
win.webContents.removeListener('did-finish-load', onFinish);
reject(new Error(`页面加载超时 (${PAGE_LOAD_TIMEOUT / 1000}s)`));
}, PAGE_LOAD_TIMEOUT);
const onFinish = () => {
clearTimeout(timeout);
resolve();
};
win.webContents.once('did-finish-load', onFinish);
win.loadURL(url).catch((err) => {
clearTimeout(timeout);
win.webContents.removeListener('did-finish-load', onFinish);
reject(err);
});
});
await loadPromise;
const title = win.webContents.getTitle(); const title = win.webContents.getTitle();
sendLog('success', `🌐 browser 已加载`, title); sendLog('success', `🌐 browser 已加载`, title);
return { success: true, title, url: win.webContents.getURL() }; return { success: true, title, url: win.webContents.getURL() };
@@ -144,7 +166,15 @@ export async function browserClick(selector: string): Promise<{ success: boolean
} }
const win = agentBrowser!; const win = agentBrowser!;
sendLog('info', `🌐 browser 点击`, selector); sendLog('info', `🌐 browser 点击`, selector);
await win.webContents.executeJavaScript(`document.querySelector('${selector.replace(/'/g, "\\'")}').click()`, true); // 安全注入:通过 JSON.stringify 防止选择器中的 JS 注入
await win.webContents.executeJavaScript(`
(() => {
const sel = ${JSON.stringify(selector)};
const el = document.querySelector(sel);
if (!el) throw new Error('元素未找到: ' + sel);
el.click();
})()
`, true);
await new Promise(r => setTimeout(r, 500)); await new Promise(r => setTimeout(r, 500));
sendLog('success', `🌐 browser 点击完成`, selector); sendLog('success', `🌐 browser 点击完成`, selector);
return { success: true }; return { success: true };
@@ -161,15 +191,16 @@ export async function browserType(selector: string, text: string): Promise<{ suc
return { success: false, error: '浏览器未打开,请先使用 browser_open 加载页面' }; return { success: false, error: '浏览器未打开,请先使用 browser_open 加载页面' };
} }
const win = agentBrowser!; const win = agentBrowser!;
const escapedSelector = selector.replace(/'/g, "\\'");
const escapedText = text.replace(/'/g, "\\'").replace(/\n/g, "\\n");
sendLog('info', `🌐 browser 输入`, `${selector}: ${text.slice(0, 50)}`); sendLog('info', `🌐 browser 输入`, `${selector}: ${text.slice(0, 50)}`);
// 安全注入:通过 JSON.stringify 防止选择器和文本中的 JS 注入
await win.webContents.executeJavaScript(` await win.webContents.executeJavaScript(`
(() => { (() => {
const el = document.querySelector('${escapedSelector}'); const sel = ${JSON.stringify(selector)};
if (!el) throw new Error('元素未找到: ${escapedSelector}'); const val = ${JSON.stringify(text)};
const el = document.querySelector(sel);
if (!el) throw new Error('元素未找到: ' + sel);
el.focus(); el.focus();
el.value = '${escapedText}'; el.value = val;
el.dispatchEvent(new Event('input', { bubbles: true })); el.dispatchEvent(new Event('input', { bubbles: true }));
el.dispatchEvent(new Event('change', { bubbles: true })); el.dispatchEvent(new Event('change', { bubbles: true }));
})() })()
+153 -51
View File
@@ -118,6 +118,69 @@ export function killToolProcess(): boolean {
/** HTTP 请求超时(毫秒) */
const HTTP_TIMEOUT = 15_000;
/** 完整 HTML 实体映射(常见实体) */
const HTML_ENTITIES: Record<string, string> = {
'&nbsp;': ' ', '&lt;': '<', '&gt;': '>', '&amp;': '&', '&quot;': '"',
'&#39;': "'", '&apos;': "'", '&ensp;': ' ', '&emsp;': ' ',
'&copy;': '\u00A9', '&reg;': '\u00AE', '&trade;': '\u2122', '&euro;': '\u20AC',
'&pound;': '\u00A3', '&yen;': '\u00A5', '&deg;': '\u00B0', '&middot;': '\u00B7',
'&hellip;': '\u2026', '&mdash;': '\u2014', '&ndash;': '\u2013',
'&lsquo;': '\u2018', '&rsquo;': '\u2019', '&ldquo;': '\u201C', '&rdquo;': '\u201D',
'&bull;': '\u2022',
'&times;': '\u00D7', '&divide;': '\u00F7', '&plusmn;': '\u00B1', '&micro;': '\u00B5',
'&para;': '\u00B6', '&sect;': '\u00A7', '&laquo;': '\u00AB', '&raquo;': '\u00BB',
'&iexcl;': '\u00A1', '&iquest;': '\u00BF', '&not;': '\u00AC', '&shy;': '\u00AD',
'&macr;': '\u00AF', '&acute;': '\u00B4', '&cedil;': '\u00B8',
'&OElig;': '\u0152', '&oelig;': '\u0153', '&Scaron;': '\u0160', '&scaron;': '\u0161',
'&Yuml;': '\u0178', '&circ;': '\u02C6', '&tilde;': '\u02DC',
};
/** 解码 HTML 实体 */
function decodeHTMLEntities(text: string): string {
let result = text;
for (const [entity, char] of Object.entries(HTML_ENTITIES)) {
result = result.replaceAll(entity, char);
}
// 数字实体: &#123; 和 &#x1F;
result = result.replace(/&#x([0-9a-fA-F]+);/g, (_, hex) => String.fromCharCode(parseInt(hex, 16)));
result = result.replace(/&#(\d+);/g, (_, dec) => String.fromCharCode(parseInt(dec, 10)));
return result;
}
/** 将 HTML 转换为可读文本(保留结构) */
function htmlToText(html: string): string {
let text = html;
// 移除 script/style/nav/header/footer 等噪音标签及其内容
text = text.replace(/<script[\s\S]*?<\/script>/gi, '');
text = text.replace(/<style[\s\S]*?<\/style>/gi, '');
text = text.replace(/<noscript[\s\S]*?<\/noscript>/gi, '');
text = text.replace(/<nav[\s\S]*?<\/nav>/gi, '');
text = text.replace(/<header[\s\S]*?<\/header>/gi, '');
text = text.replace(/<footer[\s\S]*?<\/footer>/gi, '');
text = text.replace(/<aside[\s\S]*?<\/aside>/gi, '');
text = text.replace(/<iframe[\s\S]*?<\/iframe>/gi, '');
text = text.replace(/<svg[\s\S]*?<\/svg>/gi, '');
// 移除 HTML 注释
text = text.replace(/<!--[\s\S]*?-->/g, '');
// 块级标签转为换行
text = text.replace(/<\/(p|div|h[1-6]|li|tr|blockquote|section|article|pre|br|hr)[^>]*>/gi, '\n');
text = text.replace(/<(br|hr)[^>]*\/?>/gi, '\n');
// 表格单元用制表符分隔
text = text.replace(/<\/(td|th)[^>]*>/gi, '\t');
// 移除剩余标签
text = text.replace(/<[^>]+>/g, '');
// 解码 HTML 实体
text = decodeHTMLEntities(text);
// 清理多余空白(保留换行结构)
text = text.replace(/[ \t]+/g, ' ');
text = text.replace(/\n\s*\n\s*\n+/g, '\n\n');
text = text.split('\n').map(l => l.trim()).join('\n');
return text.trim();
}
export async function handleWebFetch(params: { url: string; max_chars?: number; extract_mode?: string }): Promise<ToolResult> { export async function handleWebFetch(params: { url: string; max_chars?: number; extract_mode?: string }): Promise<ToolResult> {
try { try {
const url = params.url; const url = params.url;
@@ -128,13 +191,24 @@ export async function handleWebFetch(params: { url: string; max_chars?: number;
const maxChars = params.max_chars || 0; // 0 = 不截断,返回全部内容 const maxChars = params.max_chars || 0; // 0 = 不截断,返回全部内容
sendLog('info', `🌐 web_fetch`, `${url} (${maxChars > 0 ? maxChars + ' chars' : '完整内容'})`); sendLog('info', `🌐 web_fetch`, `${url} (${maxChars > 0 ? maxChars + ' chars' : '完整内容'})`);
const resp = await fetch(url, {
headers: { // 带超时的 fetch
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36', const controller = new AbortController();
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', const timeoutId = setTimeout(() => controller.abort(), HTTP_TIMEOUT);
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8' let resp: Response;
} try {
}); resp = await fetch(url, {
headers: {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8'
},
signal: controller.signal,
redirect: 'follow',
});
} finally {
clearTimeout(timeoutId);
}
if (!resp.ok) { if (!resp.ok) {
return { success: false, error: `HTTP ${resp.status}: ${resp.statusText}` }; return { success: false, error: `HTTP ${resp.status}: ${resp.statusText}` };
@@ -145,20 +219,17 @@ export async function handleWebFetch(params: { url: string; max_chars?: number;
return { success: false, error: `不支持的内容类型: ${contentType}` }; return { success: false, error: `不支持的内容类型: ${contentType}` };
} }
// 预检查内容长度,避免下载过大的页面
const contentLength = resp.headers.get('content-length');
if (contentLength && parseInt(contentLength, 10) > 10 * 1024 * 1024) {
return { success: false, error: `页面过大 (${(parseInt(contentLength, 10) / 1024 / 1024).toFixed(1)}MB),超过 10MB 限制` };
}
let text = await resp.text(); let text = await resp.text();
// 简单 HTML → 文本提取 // HTML → 可读文本提取
if (contentType.includes('html')) { if (contentType.includes('html')) {
text = text text = htmlToText(text);
.replace(/<script[\s\S]*?<\/script>/gi, '')
.replace(/<style[\s\S]*?<\/style>/gi, '')
.replace(/<[^>]+>/g, ' ')
.replace(/&nbsp;/g, ' ')
.replace(/&lt;/g, '<')
.replace(/&gt;/g, '>')
.replace(/&amp;/g, '&')
.replace(/\s+/g, ' ')
.trim();
} }
const truncated = maxChars > 0 && text.length > maxChars; const truncated = maxChars > 0 && text.length > maxChars;
@@ -175,7 +246,11 @@ export async function handleWebFetch(params: { url: string; max_chars?: number;
length: text.length length: text.length
}; };
} catch (err) { } catch (err) {
return { success: false, error: (err as Error).message }; const errMsg = (err as Error).message;
if (errMsg.includes('abort')) {
return { success: false, error: `请求超时 (${HTTP_TIMEOUT / 1000}s)` };
}
return { success: false, error: errMsg };
} }
} }
@@ -200,16 +275,27 @@ export async function handleWebSearch(params: { query: string; max_results?: num
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8' 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8'
}; };
/** 带超时的搜索请求 */
async function fetchWithTimeout(url: string): Promise<Response | null> {
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), HTTP_TIMEOUT);
try {
const resp = await fetch(url, { headers: searchHeaders, signal: controller.signal });
clearTimeout(timeoutId);
return resp;
} catch {
clearTimeout(timeoutId);
return null;
}
}
// 1. Bing 搜索(首选) // 1. Bing 搜索(首选)
let results: Array<{ title: string; url: string; snippet: string }> = []; let results: Array<{ title: string; url: string; snippet: string }> = [];
try { try {
const bingUrl = `https://www.bing.com/search?q=${encodeURIComponent(query)}&count=${maxResults}`; const bingUrl = `https://www.bing.com/search?q=${encodeURIComponent(query)}&count=${maxResults}`;
const resp = await fetch(bingUrl, { const resp = await fetchWithTimeout(bingUrl);
headers: searchHeaders, if (resp?.ok) {
});
if (resp.ok) {
const html = await resp.text(); const html = await resp.text();
results = parseBingResults(html, maxResults); results = parseBingResults(html, maxResults);
} }
@@ -221,11 +307,8 @@ export async function handleWebSearch(params: { query: string; max_results?: num
if (results.length === 0) { if (results.length === 0) {
try { try {
const baiduUrl = `https://www.baidu.com/s?wd=${encodeURIComponent(query)}&rn=${maxResults}`; const baiduUrl = `https://www.baidu.com/s?wd=${encodeURIComponent(query)}&rn=${maxResults}`;
const resp = await fetch(baiduUrl, { const resp = await fetchWithTimeout(baiduUrl);
headers: searchHeaders, if (resp?.ok) {
});
if (resp.ok) {
const html = await resp.text(); const html = await resp.text();
results = parseBaiduResults(html, maxResults); results = parseBaiduResults(html, maxResults);
} }
@@ -238,11 +321,8 @@ export async function handleWebSearch(params: { query: string; max_results?: num
if (results.length === 0) { if (results.length === 0) {
try { try {
const googleUrl = `https://www.google.com/search?q=${encodeURIComponent(query)}&num=${maxResults}`; const googleUrl = `https://www.google.com/search?q=${encodeURIComponent(query)}&num=${maxResults}`;
const resp = await fetch(googleUrl, { const resp = await fetchWithTimeout(googleUrl);
headers: searchHeaders, if (resp?.ok) {
});
if (resp.ok) {
const html = await resp.text(); const html = await resp.text();
results = parseGoogleResults(html, maxResults); results = parseGoogleResults(html, maxResults);
} }
@@ -255,20 +335,37 @@ export async function handleWebSearch(params: { query: string; max_results?: num
return { success: false, error: '未找到搜索结果,请尝试其他关键词' }; return { success: false, error: '未找到搜索结果,请尝试其他关键词' };
} }
// URL 去重:同一 URL 只保留第一个结果
const seenUrls = new Set<string>();
const deduped: typeof results = [];
for (const r of results) {
const normalizedUrl = r.url.replace(/\/+$/, '').toLowerCase();
if (!seenUrls.has(normalizedUrl)) {
seenUrls.add(normalizedUrl);
deduped.push(r);
}
}
// 清理摘要中的残留 HTML 标签和多余空白
for (const r of deduped) {
r.title = r.title.replace(/<[^>]+>/g, '').replace(/\s+/g, ' ').trim();
r.snippet = r.snippet.replace(/<[^>]+>/g, '').replace(/\s+/g, ' ').trim();
}
// 构建格式化的搜索结果(附带当前日期) // 构建格式化的搜索结果(附带当前日期)
const now = new Date(); const now = new Date();
const dateStr = `${now.getFullYear()}${now.getMonth() + 1}${now.getDate()}`; const dateStr = `${now.getFullYear()}${now.getMonth() + 1}${now.getDate()}`;
const formatted = `[当前日期: ${dateStr}(搜索时的实时日期)]\n\n` + results.map((r, i) => const formatted = `[当前日期: ${dateStr}(搜索时的实时日期)]\n\n` + deduped.map((r, i) =>
`[${i + 1}] ${r.title}\n URL: ${r.url}\n ${r.snippet}` `[${i + 1}] ${r.title}\n URL: ${r.url}\n ${r.snippet}`
).join('\n\n'); ).join('\n\n');
sendLog('success', `🔍 web_search 完成`, `"${query}" → ${results.length} 条结果`); sendLog('success', `🔍 web_search 完成`, `"${query}" → ${deduped.length} 条结果`);
return { return {
success: true, success: true,
query, query,
results, results: deduped,
total: results.length, total: deduped.length,
formatted formatted
}; };
} catch (err) { } catch (err) {
@@ -375,17 +472,9 @@ function parseGoogleResults(html: string, maxResults: number): Array<{ title: st
return results; return results;
} }
/** 简单 HTML 实体解码 */ /** 简单 HTML 实体解码(用于搜索结果解析) */
function decodeHTML(text: string): string { function decodeHTML(text: string): string {
return text return decodeHTMLEntities(text);
.replace(/&amp;/g, '&')
.replace(/&lt;/g, '<')
.replace(/&gt;/g, '>')
.replace(/&quot;/g, '"')
.replace(/&#39;/g, "'")
.replace(/&nbsp;/g, ' ')
.replace(/&#x([0-9a-f]+);/gi, (_, hex) => String.fromCharCode(parseInt(hex, 16)))
.replace(/&#(\d+);/g, (_, dec) => String.fromCharCode(parseInt(dec, 10)));
} }
export async function handleDownloadFile(params: { url: string; destination: string }): Promise<ToolResult> { export async function handleDownloadFile(params: { url: string; destination: string }): Promise<ToolResult> {
@@ -399,7 +488,17 @@ export async function handleDownloadFile(params: { url: string; destination: str
} }
sendLog('info', `⬇️ download_file`, `${params.url}${destPath}`); sendLog('info', `⬇️ download_file`, `${params.url}${destPath}`);
const resp = await fetch(params.url);
// 带超时的下载(大文件给更长超时)
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), 60_000);
let resp: Response;
try {
resp = await fetch(params.url, { signal: controller.signal });
} finally {
clearTimeout(timeoutId);
}
if (!resp.ok) return { success: false, error: `HTTP ${resp.status}: ${resp.statusText}` }; if (!resp.ok) return { success: false, error: `HTTP ${resp.status}: ${resp.statusText}` };
const buf = Buffer.from(await resp.arrayBuffer()); const buf = Buffer.from(await resp.arrayBuffer());
@@ -420,8 +519,11 @@ export async function handleDownloadFile(params: { url: string; destination: str
content_type: resp.headers.get('content-type') || 'unknown' content_type: resp.headers.get('content-type') || 'unknown'
}; };
} catch (err) { } catch (err) {
sendLog('error', `⬇️ download_file 失败`, (err as Error).message); const errMsg = (err as Error).message;
return { success: false, error: (err as Error).message }; if (errMsg.includes('abort')) {
return { success: false, error: '下载超时 (60s)' };
}
return { success: false, error: errMsg };
} }
} }
+1 -1
View File
@@ -4,7 +4,7 @@
*/ */
import type { ToolDefinition, ToolResult } from '../types.js'; import type { ToolDefinition, ToolResult } from '../types.js';
import { logToolStart, logToolResult, logError, logInfo } from './log-service.js'; import { logToolStart, logToolResult, logError, logInfo, logWarn } from './log-service.js';
import { getMCPToolDefinitions } from './mcp-client.js'; import { getMCPToolDefinitions } from './mcp-client.js';
export const TOOL_DEFINITIONS: ToolDefinition[] = [ export const TOOL_DEFINITIONS: ToolDefinition[] = [