feat: 浏览器控制工具全面增强 — browser_open(wait_selector)/screenshot(full_page+selector)/extract(selector+max_chars)/click(wait滚动到视图)/type(clear+submit)/scroll(selector)/新增browser_wait

This commit is contained in:
thzxx
2026-06-10 14:23:56 +08:00
parent d6cf7d846f
commit b3de861faf
3 changed files with 240 additions and 46 deletions
+184 -24
View File
@@ -1,5 +1,5 @@
/**
* Browser - Agent 浏览器控制 (v5.0)
* Browser - Agent 浏览器控制 (v5.1 增强版)
* 通过隐藏 BrowserWindow 实现网页加载、截图、JS 执行
*/
@@ -52,10 +52,25 @@ function getAgentBrowser(): BrowserWindow {
/** 页面加载超时(毫秒) */
const PAGE_LOAD_TIMEOUT = 30_000;
/** 等待 CSS 选择器出现在页面中(最多 waitMs 毫秒) */
async function waitForSelector(win: BrowserWindow, selector: string, waitMs = 10_000): Promise<boolean> {
const start = Date.now();
while (Date.now() - start < waitMs) {
try {
const found = await win.webContents.executeJavaScript(`
(() => { const el = document.querySelector(${JSON.stringify(selector)}); return el !== null; })()
`, true);
if (found) return true;
} catch { /* retry */ }
await new Promise(r => setTimeout(r, 300));
}
return false;
}
/** 打开 URL */
export async function browserOpen(url: string): Promise<{ success: boolean; title?: string; url?: string; error?: string }> {
export async function browserOpen(url: string, waitSelector?: string): Promise<{ success: boolean; title?: string; url?: string; waited?: boolean; error?: string }> {
try {
// 如果已有 BrowserWindow 且加载了不同 URL,先关闭重建,避免状态残留
// 如果已有 BrowserWindow 且加载了不同 URL,先关闭重建
if (agentBrowser && !agentBrowser.isDestroyed()) {
const currentUrl = agentBrowser.webContents.getURL();
if (currentUrl !== 'about:blank' && currentUrl !== url) {
@@ -68,7 +83,6 @@ export async function browserOpen(url: string): Promise<{ success: boolean; titl
const win = getAgentBrowser();
sendLog('info', `🌐 browser 打开`, url.slice(0, 100));
// 等待页面实际加载完成,而非固定延时
const loadPromise = new Promise<void>((resolve, reject) => {
const timeout = setTimeout(() => {
win.webContents.removeListener('did-finish-load', onFinish);
@@ -88,9 +102,18 @@ export async function browserOpen(url: string): Promise<{ success: boolean; titl
});
await loadPromise;
let waited = false;
// 等待特定元素出现(用于 SPA 页面)
if (waitSelector) {
sendLog('info', `🌐 browser 等待元素`, waitSelector);
waited = await waitForSelector(win, waitSelector);
if (!waited) sendLog('warn', `🌐 browser 等待超时`, `未找到: ${waitSelector}`);
}
const title = win.webContents.getTitle();
sendLog('success', `🌐 browser 已加载`, title);
return { success: true, title, url: win.webContents.getURL() };
sendLog('success', `🌐 browser 已加载`, title + (waited ? ' [等待元素完成]' : ''));
return { success: true, title, url: win.webContents.getURL(), waited };
} catch (err) {
sendLog('error', `🌐 browser 打开失败`, (err as Error).message);
return { success: false, error: (err as Error).message };
@@ -98,15 +121,73 @@ export async function browserOpen(url: string): Promise<{ success: boolean; titl
}
/** 截图 */
export async function browserScreenshot(): Promise<{ success: boolean; png?: string; error?: string }> {
export async function browserScreenshot(params: { full_page?: boolean; selector?: string } = {}): Promise<{ success: boolean; png?: string; width?: number; height?: number; error?: string }> {
try {
const ready = ensureBrowserReady();
if (!ready.ok) return { success: false, error: ready.error };
const win = ready.win!;
if (params.selector) {
// 截取特定元素的截图
const rect = await win.webContents.executeJavaScript(`
(() => {
const el = document.querySelector(${JSON.stringify(params.selector)});
if (!el) return null;
const r = el.getBoundingClientRect();
return { x: r.x, y: r.y, width: r.width, height: r.height };
})()
`, true) as { x: number; y: number; width: number; height: number } | null;
if (!rect || rect.width === 0 || rect.height === 0) {
return { success: false, error: `未找到元素或元素不可见: ${params.selector}` };
}
const image = await win.webContents.capturePage({ x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height) });
const buffer = image.toPNG();
sendLog('info', `🌐 browser 截图(元素)`, `${params.selector}${buffer.length} bytes, ${rect.width}x${rect.height}`);
return { success: true, png: buffer.toString('base64'), width: Math.round(rect.width), height: Math.round(rect.height) };
}
if (params.full_page) {
// 全页面截图:先滚动获取完整高度,截完恢复
const pageSize = await win.webContents.executeJavaScript(`
(() => {
const w = Math.max(document.documentElement.scrollWidth, document.body.scrollWidth, window.innerWidth);
const h = Math.max(document.documentElement.scrollHeight, document.body.scrollHeight, window.innerHeight);
const origScroll = { x: window.scrollX, y: window.scrollY };
window.scrollTo(0, 0);
return { width: w, height: h, origScroll };
})()
`, true) as { width: number; height: number; origScroll: { x: number; y: number } };
// 分块截图拼接(超过窗口高度的页面)
const viewHeight = 800;
const chunks: Buffer[] = [];
for (let y = 0; y < pageSize.height; y += viewHeight) {
await win.webContents.executeJavaScript(`window.scrollTo(0, ${y})`, true);
await new Promise(r => setTimeout(r, 200));
const image = await win.webContents.capturePage({
x: 0, y: 0,
width: pageSize.width,
height: Math.min(viewHeight, pageSize.height - y)
});
chunks.push(image.toPNG());
}
// 恢复滚动位置
await win.webContents.executeJavaScript(`window.scrollTo(${pageSize.origScroll.x}, ${pageSize.origScroll.y})`, true);
// 拼接所有截图(简单合并 base64)
const fullPng = Buffer.concat(chunks).toString('base64');
sendLog('success', `🌐 browser 截图(全页)`, `${pageSize.width}x${pageSize.height}, ${fullPng.length} chars`);
return { success: true, png: fullPng, width: pageSize.width, height: pageSize.height };
}
// 视口截图(默认)
const image = await win.webContents.capturePage();
const buffer = image.toPNG();
sendLog('info', `🌐 browser 截图`, `${buffer.length} bytes`);
return { success: true, png: buffer.toString('base64') };
const size = image.getSize();
sendLog('info', `🌐 browser 截图(视口)`, `${buffer.length} bytes, ${size.width}x${size.height}`);
return { success: true, png: buffer.toString('base64'), width: size.width, height: size.height };
} catch (err) {
sendLog('error', `🌐 browser 截图失败`, (err as Error).message);
return { success: false, error: (err as Error).message };
@@ -130,12 +211,13 @@ export async function browserEvaluate(js: string): Promise<{ success: boolean; r
}
}
/** 提取页面文本和链接 */
export async function browserExtract(): Promise<{ success: boolean; title?: string; text?: string; links?: Array<{text: string; href: string}>; error?: string }> {
/** 提取页面内容 */
export async function browserExtract(params: { selector?: string; max_chars?: number } = {}): Promise<{ success: boolean; title?: string; text?: string; links?: Array<{text: string; href: string}>; error?: string }> {
try {
const ready = ensureBrowserReady();
if (!ready.ok) return { success: false, error: ready.error };
const win = ready.win!;
const maxChars = params.max_chars || 15000;
const result = await win.webContents.executeJavaScript(`
(() => {
@@ -143,24 +225,35 @@ export async function browserExtract(): Promise<{ success: boolean; title?: stri
const getText = (el) => {
if (!el) return '';
const clone = el.cloneNode(true);
clone.querySelectorAll('script,style,noscript,iframe').forEach(e => e.remove());
clone.querySelectorAll('script,style,noscript,iframe,svg').forEach(e => e.remove());
return clone.innerText?.trim() || '';
};
const body = getText(document.body);
const text = body.length > 15000 ? body.slice(0, 15000) + '\\\\n... (已截断)' : body;
let root = document.body;
${params.selector ? `
const selEl = document.querySelector(${JSON.stringify(params.selector)});
if (selEl) root = selEl;
` : ''}
const body = getText(root);
const maxC = ${maxChars};
const text = body.length > maxC ? body.slice(0, maxC) + '\\\\n... (已截断)' : body;
const links = [];
document.querySelectorAll('a[href]').forEach(a => {
root.querySelectorAll('a[href]').forEach(a => {
const href = a.href;
const text = a.textContent?.trim();
if (href && text && href.startsWith('http')) {
links.push({ text: text.slice(0, 100), href });
const t = a.textContent?.trim();
if (href && t && href.startsWith('http')) {
links.push({ text: t.slice(0, 100), href });
}
});
return { title, text, links: links.slice(0, 50) };
})()
`, true);
sendLog('success', `🌐 browser 提取完成`, `标题: ${result.title}, 文本: ${result.text?.length || 0}字, 链接: ${result.links?.length || 0}`);
const extractType = params.selector ? `selector:${params.selector}` : '全文';
sendLog('success', `🌐 browser 提取完成(${extractType})`, `标题: ${result.title}, 文本: ${result.text?.length || 0}字, 链接: ${result.links?.length || 0}`);
return { success: true, ...result };
} catch (err) {
sendLog('error', `🌐 browser 提取失败`, (err as Error).message);
@@ -169,17 +262,25 @@ export async function browserExtract(): Promise<{ success: boolean; title?: stri
}
/** 点击元素 */
export async function browserClick(selector: string): Promise<{ success: boolean; error?: string }> {
export async function browserClick(selector: string, wait = false): Promise<{ success: boolean; error?: string }> {
try {
const ready = ensureBrowserReady();
if (!ready.ok) return { success: false, error: ready.error };
const win = ready.win!;
if (wait) {
const found = await waitForSelector(win, selector);
if (!found) return { success: false, error: `等待超时: 未找到元素 ${selector}` };
sendLog('info', `🌐 browser 等待完成`, selector);
}
sendLog('info', `🌐 browser 点击`, selector);
await win.webContents.executeJavaScript(`
(() => {
const sel = ${JSON.stringify(selector)};
const el = document.querySelector(sel);
if (!el) throw new Error('元素未找到: ' + sel);
el.scrollIntoView({ behavior: 'smooth', block: 'center' });
el.click();
})()
`, true);
@@ -193,7 +294,7 @@ export async function browserClick(selector: string): Promise<{ success: boolean
}
/** 输入文本 */
export async function browserType(selector: string, text: string): Promise<{ success: boolean; error?: string }> {
export async function browserType(selector: string, text: string, clear = true, submit = false): Promise<{ success: boolean; error?: string }> {
try {
const ready = ensureBrowserReady();
if (!ready.ok) return { success: false, error: ready.error };
@@ -206,12 +307,21 @@ export async function browserType(selector: string, text: string): Promise<{ suc
const el = document.querySelector(sel);
if (!el) throw new Error('元素未找到: ' + sel);
el.focus();
el.value = val;
${clear ? `el.value = val;` : `el.value += val;`}
el.dispatchEvent(new Event('input', { bubbles: true }));
el.dispatchEvent(new Event('change', { bubbles: true }));
${submit ? `
const form = el.closest('form');
if (form) {
form.dispatchEvent(new Event('submit', { bubbles: true, cancelable: true }));
} else {
el.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', code: 'Enter', keyCode: 13, bubbles: true }));
}
` : ''}
})()
`, true);
sendLog('success', `🌐 browser 输入完成`, selector);
await new Promise(r => setTimeout(r, submit ? 1000 : 300));
sendLog('success', `🌐 browser 输入完成`, selector + (submit ? ' [已提交]' : ''));
return { success: true };
} catch (err) {
sendLog('error', `🌐 browser 输入失败`, `${selector}: ${(err as Error).message}`);
@@ -220,11 +330,29 @@ export async function browserType(selector: string, text: string): Promise<{ suc
}
/** 滚动页面 */
export async function browserScroll(direction: 'down' | 'up' | 'top' | 'bottom'): Promise<{ success: boolean; error?: string }> {
export async function browserScroll(params: { direction?: string; selector?: string } = {}): Promise<{ success: boolean; error?: string }> {
try {
const ready = ensureBrowserReady();
if (!ready.ok) return { success: false, error: ready.error };
const win = ready.win!;
const direction = params.direction || 'down';
if (params.selector) {
// 滚动到特定元素
const found = await win.webContents.executeJavaScript(`
(() => {
const el = document.querySelector(${JSON.stringify(params.selector)});
if (!el) return false;
el.scrollIntoView({ behavior: 'smooth', block: 'center' });
return true;
})()
`, true);
if (!found) return { success: false, error: `未找到元素: ${params.selector}` };
await new Promise(r => setTimeout(r, 500));
sendLog('info', `🌐 browser 滚动到元素`, params.selector);
return { success: true };
}
const js = direction === 'down' ? 'window.scrollBy(0, 500)'
: direction === 'up' ? 'window.scrollBy(0, -500)'
: direction === 'top' ? 'window.scrollTo(0, 0)'
@@ -239,6 +367,38 @@ export async function browserScroll(direction: 'down' | 'up' | 'top' | 'bottom')
}
}
/** 等待条件 */
export async function browserWait(params: { selector?: string; time_ms?: number }): Promise<{ success: boolean; found?: boolean; error?: string }> {
try {
const ready = ensureBrowserReady();
if (!ready.ok) return { success: false, error: ready.error };
const win = ready.win!;
if (params.selector) {
sendLog('info', `🌐 browser 等待元素`, params.selector);
const found = await waitForSelector(win, params.selector, params.time_ms || 10_000);
if (found) {
sendLog('success', `🌐 browser 元素已出现`, params.selector);
} else {
sendLog('warn', `🌐 browser 等待超时`, params.selector);
}
return { success: true, found };
}
if (params.time_ms) {
sendLog('info', `🌐 browser 等待`, `${params.time_ms}ms`);
await new Promise(r => setTimeout(r, params.time_ms));
return { success: true };
}
// 默认等待 1 秒
await new Promise(r => setTimeout(r, 1000));
return { success: true };
} catch (err) {
return { success: false, error: (err as Error).message };
}
}
/** 关闭浏览器 */
export function browserClose(): void {
if (agentBrowser && !agentBrowser.isDestroyed()) {
+9 -8
View File
@@ -48,7 +48,7 @@ import {
handleGit,
handleCompress
} from './tool-handlers.js';
import { browserOpen, browserScreenshot, browserEvaluate, browserExtract, browserClick, browserType, browserScroll, browserClose } from './browser.js';
import { browserOpen, browserScreenshot, browserEvaluate, browserExtract, browserClick, browserType, browserScroll, browserClose, browserWait } from './browser.js';
import { startServer, stopServer, stopAllServers, callTool, getAllTools, getServerStatuses, refreshTools, setMCPTimeout } from './mcp-manager.js';
import { getAllowedDirs, getBlockedDirs, setAllowedDirs } from './tool-security.js';
import { startProcess, killProcess, getWorkspaceDir, setWorkspaceDir, listWorkspaceDir } from './workspace.js';
@@ -190,14 +190,15 @@ export async function setupIPC(): Promise<void> {
case 'read_multiple_files':result = await handleReadMultipleFiles(args as { paths: string[]; max_chars_per_file?: number }); break;
case 'git': result = await handleGit(args as { action: string; path?: string; files?: string[]; message?: string; branch?: string; remote?: string; remote_url?: string; count?: number; all?: boolean; staged?: boolean; new_branch?: boolean; delete_branch?: boolean; force?: boolean; url?: string }); break;
case 'compress': result = await handleCompress(args as { action: string; path: string; destination?: string; format?: string }); break;
// v5.0 Browser 控制
case 'browser_open': result = await browserOpen(args.url as string); break;
case 'browser_screenshot': result = await browserScreenshot(); break;
// v5.1 Browser 控制(增强版)
case 'browser_open': result = await browserOpen(args.url as string, args.wait_selector as string | undefined); break;
case 'browser_screenshot': result = await browserScreenshot({ full_page: args.full_page as boolean, selector: args.selector as string }); break;
case 'browser_evaluate': result = await browserEvaluate(args.js as string); break;
case 'browser_extract': result = await browserExtract(); break;
case 'browser_click': result = await browserClick(args.selector as string); break;
case 'browser_type': result = await browserType(args.selector as string, args.text as string); break;
case 'browser_scroll': result = await browserScroll((args.direction as 'down' | 'up' | 'top' | 'bottom') || 'down'); break;
case 'browser_extract': result = await browserExtract({ selector: args.selector as string, max_chars: args.max_chars as number }); break;
case 'browser_click': result = await browserClick(args.selector as string, (args.wait as boolean) || false); break;
case 'browser_type': result = await browserType(args.selector as string, args.text as string, args.clear !== false, args.submit as boolean || false); break;
case 'browser_scroll': result = await browserScroll({ direction: args.direction as string, selector: args.selector as string }); break;
case 'browser_wait': result = await browserWait({ selector: args.selector as string, time_ms: args.time_ms as number }); break;
case 'browser_close': browserClose(); result = { success: true }; break;
default: return { success: false, error: `未知工具: ${toolName}` };
}
+47 -14
View File
@@ -508,16 +508,19 @@ export const TOOL_DEFINITIONS: ToolDefinition[] = [
// ══════════════════════════════════════════════
// v5.0 新增工具:浏览器控制
// ══════════════════════════════════════════════
// v5.1 Browser 控制工具(增强版)
// ══════════════════════════════════════════════
{
type: 'function',
function: {
name: 'browser_open',
description: 'Open a URL in the agent browser. Loads the page and waits for rendering. Returns the page title and final URL.',
description: 'Open a URL in the agent browser. Waits for page load. Use wait_selector to wait for a specific element (useful for SPA pages). Returns page title and URL.',
parameters: {
type: 'object',
required: ['url'],
properties: {
url: { type: 'string', description: 'The URL to open (http/https).' }
url: { type: 'string', description: 'The URL to open (http/https).' },
wait_selector: { type: 'string', description: 'CSS selector to wait for after page load. Useful for SPA/dynamic pages. Default: none.' }
}
}
}
@@ -526,15 +529,21 @@ export const TOOL_DEFINITIONS: ToolDefinition[] = [
type: 'function',
function: {
name: 'browser_screenshot',
description: 'Take a screenshot of the current browser page. Returns the image as base64 PNG. Use this to see what the page looks like.',
parameters: { type: 'object', properties: {} }
description: 'Take a screenshot of the current page. Supports viewport (default), full_page (entire scrollable page), or selector (specific element only). Returns base64 PNG.',
parameters: {
type: 'object',
properties: {
full_page: { type: 'boolean', description: 'Capture the entire scrollable page. Default: false.' },
selector: { type: 'string', description: 'CSS selector of element to capture. Overrides full_page.' }
}
}
}
},
{
type: 'function',
function: {
name: 'browser_evaluate',
description: 'Execute JavaScript code in the browser page context. Use to interact with the page, read DOM elements, fill forms, click buttons, etc. Returns the result of the last expression.',
description: 'Execute JavaScript in the browser page. Returns the result as JSON string. Use to read DOM, extract data, or interact programmatically.',
parameters: {
type: 'object',
required: ['js'],
@@ -548,20 +557,27 @@ export const TOOL_DEFINITIONS: ToolDefinition[] = [
type: 'function',
function: {
name: 'browser_extract',
description: 'Extract all text content and links from the current browser page. Returns title, full text, and up to 50 links.',
parameters: { type: 'object', properties: {} }
description: 'Extract text and links from the current page. Use selector to extract only from a specific element (e.g. "main", "#content", "article"). Without selector, extracts full page. Returns title, text, and up to 50 links.',
parameters: {
type: 'object',
properties: {
selector: { type: 'string', description: 'CSS selector to extract from. Default: entire body.' },
max_chars: { type: 'integer', description: 'Max characters to return. Default: 15000.' }
}
}
}
},
{
type: 'function',
function: {
name: 'browser_click',
description: 'Click an element on the page using a CSS selector.',
description: 'Click an element by CSS selector. Use wait=true to wait up to 10s for the element to appear first (useful after page navigation). Scrolls element into view before clicking.',
parameters: {
type: 'object',
required: ['selector'],
properties: {
selector: { type: 'string', description: 'CSS selector for the element to click.' }
selector: { type: 'string', description: 'CSS selector for the element to click.' },
wait: { type: 'boolean', description: 'Wait for element to appear before clicking (max 10s). Default: false.' }
}
}
}
@@ -570,13 +586,15 @@ export const TOOL_DEFINITIONS: ToolDefinition[] = [
type: 'function',
function: {
name: 'browser_type',
description: 'Type text into an input field on the page using a CSS selector.',
description: 'Type text into an input field. Use clear=false to append instead of replacing. Use submit=true to press Enter or submit the parent form after typing.',
parameters: {
type: 'object',
required: ['selector', 'text'],
properties: {
selector: { type: 'string', description: 'CSS selector for the input element.' },
text: { type: 'string', description: 'Text to type into the input.' }
text: { type: 'string', description: 'Text to type into the input.' },
clear: { type: 'boolean', description: 'Clear existing text first. Default: true.' },
submit: { type: 'boolean', description: 'Submit the form or press Enter after typing. Default: false.' }
}
}
}
@@ -585,11 +603,26 @@ export const TOOL_DEFINITIONS: ToolDefinition[] = [
type: 'function',
function: {
name: 'browser_scroll',
description: 'Scroll the browser page.',
description: 'Scroll the page. Use direction for up/down/top/bottom, or selector to scroll a specific element into view.',
parameters: {
type: 'object',
properties: {
direction: { type: 'string', enum: ['down', 'up', 'top', 'bottom'], description: 'Scroll direction. Default: down.' }
direction: { type: 'string', enum: ['down', 'up', 'top', 'bottom'], description: 'Scroll direction. Default: down.' },
selector: { type: 'string', description: 'CSS selector to scroll into view (overrides direction).' }
}
}
}
},
{
type: 'function',
function: {
name: 'browser_wait',
description: 'Wait for a condition. Use selector to wait for an element to appear, or time_ms for a fixed delay. Default: 1 second.',
parameters: {
type: 'object',
properties: {
selector: { type: 'string', description: 'CSS selector to wait for. Returns found: true/false.' },
time_ms: { type: 'integer', description: 'Fixed wait time in milliseconds. Default: 1000.' }
}
}
}
@@ -598,7 +631,7 @@ export const TOOL_DEFINITIONS: ToolDefinition[] = [
type: 'function',
function: {
name: 'browser_close',
description: 'Close the agent browser and release resources.',
description: 'Close the agent browser and free resources.',
parameters: { type: 'object', properties: {} }
}
}