From 157219d982bc9f048aa21dd368c9cf6dcea13460 Mon Sep 17 00:00:00 2001
From: thzxx <1440196015@qq.com>
Date: Mon, 3 Aug 2026 17:14:33 +0800
Subject: [PATCH] =?UTF-8?q?feat(v2.2.1):=20=E4=BF=AE=E5=A4=8D=20Brotli=20?=
 =?UTF-8?q?=E4=B9=B1=E7=A0=81=20+=20=E7=BC=93=E5=AD=98=E6=B2=BB=E7=90=86?=
 =?UTF-8?q?=20+=20=E5=A4=9A=E9=A1=B5=E8=81=9A=E5=90=88=20+=20=E7=A0=94?=
 =?UTF-8?q?=E7=A9=B6=E6=A8=A1=E5=BC=8F=E5=A2=9E=E5=BC=BA?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

核心修复（v2.2.1）:
- 修复 Brotli 乱码 bug: build_browser_headers 智能声明 Accept-Encoding,
  仅在 brotli 可用时才声明 br; fetch.py 双路径 br 解压（requests + stdlib）
  此前 Chrome/Edge UA 抓取 example.com 等返回 br 的站点输出乱码

v2.2.0 新功能:
- main() 拆分为 _handle_verify/_handle_research/_handle_batch/_handle_single
- --cache-max-size MB: 缓存大小上限 + LRU 淘汰（默认 100MB）
- --pages N: 多页聚合 + 跨页去重
- --research 跨角度合并: 新增 merged_results 字段
- --stream / --progress: JSON Lines 流式输出 + request_id 贯穿
- --dry-run / --save-config / --log-format json
- --similarity-dedup / --throttle-* 参数化
- 15-UA 池 + PDF/docx 解析 + error_code 字段

文档与测试:
- SKILL.md: 版本号唯一（元数据），删除版本标记干扰
- README.md: 测试数量 539 -> 544
- 544 passed (新增 5 个 Content-Encoding 解压测试)
---
 .gitignore                    |    1 +
 README.md                     |   75 ++-
 SKILL.md                      |  198 ++++---
 scripts/_config.py            |   54 +-
 scripts/cache.py              |  422 +++++++++++---
 scripts/common.py             |  346 +++++++++--
 scripts/fetch.py              |  361 ++++++++++--
 scripts/search.py             | 1013 +++++++++++++++++++++++----------
 tests/test_browser_headers.py |   17 +-
 tests/test_integration.py     |  120 +++-
 10 files changed, 2044 insertions(+), 563 deletions(-)

diff --git a/.gitignore b/.gitignore
index 8927f81..d79239c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -36,3 +36,4 @@ searxng.toml
 instances.txt
 *.db
 .cache/
+pytest_chk.txt
diff --git a/README.md b/README.md
index 27c19d1..8b64ae2 100644
--- a/README.md
+++ b/README.md
@@ -69,6 +69,7 @@ AI 调用后，stdout 输出网页正文（text/html/markdown 三种格式），
 | `E_EMPTY` | 空结果（exit 2） | 调整查询词、扩大 `--time-range`/`--categories` |
 | `E_INPUT` | 输入错误（参数/文件） | 检查语法、标志组合、文件路径 |
 | `E_INTERNAL` | 内部错误 | 用 `--verbose` 重跑并报告 |
+| `E_UNSUPPORTED_MEDIA` (v2.2.0) | 不支持的二进制媒体类型（PDF/docx/xlsx 解析失败） | 换 URL，或安装 `pdftotext` 用于 PDF 解析 |
 
 ### 流式输出与进度事件（AI 高级用法）
 
@@ -85,15 +86,19 @@ AI 调用后，stdout 输出网页正文（text/html/markdown 三种格式），
 
 **`--progress`**：进度事件流（JSON Lines 到 stderr），AI 可实时跟踪执行：
 ```
-{"event": "start", "query": "...", "instances": 2}
-{"event": "instance_try", "url": "https://inst1.example.com", "attempt": 1}
-{"event": "instance_ok", "url": "https://inst1.example.com", "latency": 0.342, "results": 10}
-{"event": "instance_fail", "url": "https://inst2.example.com", "error": "HTTP 503", "error_code": "E_NETWORK"}
-{"event": "cache_hit", "query": "...", "ttl": 30}
-{"event": "fetch_ok", "url": "...", "chars": 12345}
-{"event": "done", "results": 10, "query": "..."}
+{"event": "start", "query": "...", "instances": 2, "request_id": "a1b2c3d4"}
+{"event": "instance_try", "url": "https://inst1.example.com", "attempt": 1, "request_id": "a1b2c3d4"}
+{"event": "instance_ok", "url": "https://inst1.example.com", "latency": 0.342, "results": 10, "request_id": "a1b2c3d4"}
+{"event": "instance_fail", "url": "https://inst2.example.com", "error": "HTTP 503", "error_code": "E_NETWORK", "request_id": "a1b2c3d4"}
+{"event": "cache_hit", "query": "...", "ttl": 30, "request_id": "a1b2c3d4"}
+{"event": "page_ok", "pageno": 1, "results": 10, "request_id": "a1b2c3d4"}
+{"event": "page_fail", "pageno": 2, "error": "HTTP 503", "request_id": "a1b2c3d4"}
+{"event": "fetch_ok", "url": "...", "chars": 12345, "request_id": "a1b2c3d4"}
+{"event": "done", "results": 10, "query": "...", "request_id": "a1b2c3d4"}
 ```
 
+v2.2.0：每个事件含 `request_id`（8 位 hex，每次运行自动生成）；`--pages N` 时新增 `page_ok`/`page_fail` 事件。
+
 **`--dump-schema`**：输出当前版本的 JSON Schema 到 stdout 并退出，AI 可程序化发现字段名与类型，无需解析文档。
 
 ```bash
@@ -158,19 +163,23 @@ python scripts/search.py -q "查询词" -i https://your-instance \
   [--language zh-CN] \
   [--sort-by score|date|engine|none] \
   [--no-dedup] \
+  [--similarity-dedup] [--similarity-threshold 0.85] \
   [--max-results 10] \
-  [--fetch 3] \
-  [--fetch-report] \
+  [--pages N] \
+  [--fetch 3] [--fetch-report [json]] \
   [--no-fallback] \
   [--referer URL] \
   [--request-delay 0.3] \
-  [--cache-ttl 30] \
+  [--cache-ttl 30] [--cache-max-size 100] \
   [--queries-file queries.txt] \
-  [--research "研究主题"] \
+  [--research "研究主题"] [--research-angles "a,b,c"] \
   [--include-domain example.com] \
   [--exclude-domain spam.com] \
   [--proxy http://corp:8080] \
   [--auth-bearer-file ~/.token] \
+  [--throttle-failure-threshold 3] [--throttle-pause-seconds 30] [--throttle-max-delay 10] \
+  [--log-format text|json] \
+  [--dry-run] [--save-config FILE] \
   [--verify] \
   [--stream] \
   [--progress] \
@@ -209,6 +218,18 @@ python scripts/search.py -q "查询词" -i https://your-instance \
 | `--progress` | 进度事件（JSON Lines 到 stderr） | 关闭 |
 | `-v / --verbose` | 调试日志 | — |
 | `--quiet` | 仅输出警告和错误 | — |
+| `--pages N` | v2.2.0 分页聚合：一次获取 N 页并跨页去重合并 | 1 |
+| `--similarity-dedup` | v2.2.0 相似度去重（SimHash + Jaccard） | 关闭 |
+| `--similarity-threshold` | v2.2.0 相似度阈值 | 0.85 |
+| `--log-format` | v2.2.0 日志格式：text/json | text |
+| `--dry-run` | v2.2.0 预览模式（不发 HTTP 请求） | 关闭 |
+| `--throttle-failure-threshold` | v2.2.0 限流失败阈值 | 3 |
+| `--throttle-pause-seconds` | v2.2.0 429 全局暂停秒数 | 30 |
+| `--throttle-max-delay` | v2.2.0 限流最大延迟秒数 | 10 |
+| `--research-angles` | v2.2.0 自定义研究角度（逗号分隔） | 默认 5 角度 |
+| `--save-config` | v2.2.0 保存当前参数为 searxng.toml 并退出 | — |
+| `--cache-max-size` | v2.2.0 缓存大小上限（MB） | 100 |
+| `--fetch-report json` | v2.2.0 JSON 格式抓取报告 | text |
 
 ### fetch.py — 网页抓取
 
@@ -242,6 +263,27 @@ python scripts/fetch.py -u https://example.com \
 
 ## 能力清单
 
+**v2.2.1 修复**
+- Brotli 乱码修复：`build_browser_headers()` 智能声明 `Accept-Encoding`——仅当本机安装了 brotli/brotlicffi 包时才声明 `br`，避免服务器返回 Brotli 压缩字节而 requests 无法自动解压导致全页乱码
+- `fetch.py` 双路径 br 解压：requests 路径和 stdlib urllib 路径都添加了 Brotli 手动解压逻辑（作为双保险，应对代理/CDN 强制返回 br 的边缘情况）
+- 此前 bug 表现：Chrome/Edge UA 抓取 example.com 等返回 `Content-Encoding: br` 的站点时，输出 302 字符乱码（gzip 二进制被当作文本解码）；修复后输出 127 字符正常文本
+
+**v2.2.0 新功能**
+- `--pages N` 分页聚合：一次获取 N 页结果并跨页去重合并，每页独立缓存（cache key 含 pageno），进度事件新增 `page_ok`/`page_fail`
+- 缓存治理：缓存大小上限 + LRU 淘汰，新增 `--cache-max-size MB`（默认 100MB）；`stats()` 新增 `total_bytes`/`max_size_bytes`/`evicted_count`/`utilization_pct` 字段；新增 `evict_expired()` 主动清理方法
+- `--similarity-dedup` 相似度去重：基于标题 SimHash + Jaccard 相似度，默认关闭；`--similarity-threshold`（默认 0.85）控制严格程度；O(n²) 复杂度，结果数 > 500 时自动跳过
+- PDF/文档解析：fetch 支持解析 PDF（pdftotext subprocess）、`.docx`/`.xlsx`（stdlib zipfile）；不支持的二进制类型返回 `E_UNSUPPORTED_MEDIA`
+- `--log-format json` 结构化日志：每行输出 JSON 对象 `{ts, level, logger, msg, request_id}`，便于 AI Agent 程序化解析
+- request ID 贯穿：每次运行自动生成 8 位 hex request_id，贯穿所有日志和进度事件
+- `--dry-run` 预览模式：不发 HTTP 请求，打印 `{action, url, params, headers_count}` JSON 到 stdout；支持 search/research/batch/verify 四种模式
+- AdaptiveThrottle 参数可配置：新增 `--throttle-failure-threshold`（默认 3）、`--throttle-pause-seconds`（默认 30）、`--throttle-max-delay`（默认 10）
+- `--research-angles` 自定义研究角度：覆盖默认 5 角度，每个角度直接作为查询后缀
+- `--save-config FILE`：将当前 CLI 参数保存为 searxng.toml 配置文件并退出
+- `--fetch-report json`：输出完整 JSON 报告（含 items 数组 + summary 摘要）；原 `--fetch-report`（无参数）保持 text 格式
+- UA 池更新到 2026 年版本：Chrome 138-140 / Edge 138 / Firefox 140 / Safari 18，池移至 `_config.py`（SSOT）
+- readability-lite 按语言调整：CJK 内容 100 字符阈值，其他 200 字符阈值
+- classify_error 改用异常链（内部改进）、`_domain_ua_cache` 加锁（内部改进）
+
 **搜索**
 - 多实例故障转移 + 并行探测
 - 指数退避重试（429/5xx/连接错误）
@@ -264,11 +306,12 @@ python scripts/fetch.py -u https://example.com \
 - text：提取纯文本
 - html：原始 HTML
 - markdown：增强 Markdown 转换（GFM 表格、代码块、引用块、嵌套列表、定义列表）
-- v2.0.0 readability-lite：`<article>`/`<main>` 缺失时，用文本密度算法选最可能正文的 `<div>`
+- v2.0.0 readability-lite：`<article>`/`<main>` 缺失时，用文本密度算法选最可能正文的 `<div>`（v2.2.0：CJK 内容 100 字符阈值，其他 200 字符阈值）
+- v2.2.0 PDF/文档解析：支持解析 PDF（pdftotext subprocess）、`.docx`/`.xlsx`（stdlib zipfile）；不支持的二进制类型返回 `E_UNSUPPORTED_MEDIA`
 
 **反爬与抓取稳定性（v2.0.0）**
 - 浏览器指纹头：`build_browser_headers()` 发送完整 Sec-Ch-Ua / Sec-Fetch-* / Accept-Language，不仅靠 User-Agent
-- 12 个 UA 池：Chrome/Edge/Firefox × Windows/macOS/Linux × 129-131 版本
+- 15 个 UA 池（v2.2.0 更新）：Chrome 138-140 / Edge 138 / Firefox 140 / Safari 18 × Windows/macOS/Linux；UA 池移至 `_config.py`（SSOT），common.py 通过导入引用
 - 确定性 UA 轮换：`get_ua_for_domain()` 用 SHA-256 为每个域名固定一个 UA（会话内稳定，跨进程可复现）
 - requests.Session 复用：连接池 + cookie 持久化 + TLS 会话恢复
 - 超时分离：`(connect, read)` 元组，避免大页面下载中途超时浪费已建连接
@@ -281,13 +324,15 @@ python scripts/fetch.py -u https://example.com \
   - v2.1.1 精确化：`baidu.com` 从子域匹配改为精确子域列表（www/baike/zhidao/tieba/wenku），`pan.baidu.com`（网盘）/`cloud.baidu.com`（智能云）不再被误伤
 - 自适应限流：`AdaptiveThrottle` 状态机，连续 3 次失败自动翻倍延迟 + 减半并发，429 触发全局暂停 30s
   - v2.1.1：`report_failure` 新增 `error_code` 参数，优先用结构化 `E_RATE_LIMIT` 检测 429（原字符串匹配"429"会漏判"Too Many Requests"）；`fetch_page` 返回结果新增 `error_code` 字段
-- `--fetch-report`：结构化抓取报告到 stderr（每 URL 状态/WAF 类型/兜底方式/字符数 + JSON 摘要）
+  - v2.2.0：原硬编码值现可通过 CLI 配置——`--throttle-failure-threshold`（默认 3）、`--throttle-pause-seconds`（默认 30）、`--throttle-max-delay`（默认 10）
+- `--fetch-report`：结构化抓取报告到 stderr（每 URL 状态/WAF 类型/兜底方式/字符数 + JSON 摘要）；v2.2.0：`--fetch-report json` 输出完整 JSON 报告（items 数组 + summary 摘要）
 - `--referer` / `--request-delay`：精细控制 Referer 头和请求间隔
 - fetch 结果新增字段：`anti_bot_detected`（bool）、`waf_type`（str|null）、`fallback_used`（str|null）、`error_code`（str|null，v2.1.1）
 
 **缓存**
 - SQLite 缓存（`--cache-ttl`），相同查询在 TTL 内跳过网络
 - `--clear-cache` / `--cache-stats` 管理缓存
+- v2.2.0 缓存治理：`--cache-max-size MB`（默认 100MB）大小上限 + LRU 淘汰；`stats()` 新增 `total_bytes`/`max_size_bytes`/`evicted_count`/`utilization_pct` 字段；新增 `evict_expired()` 主动清理方法
 
 **网络**
 - 代理支持（`--proxy`）
@@ -306,7 +351,7 @@ python scripts/fetch.py -u https://example.com \
 - JSON Lines 流式输出（`--stream`，含 `error` 事件类型）
 - 进度事件（`--progress`，含 `instance_try`/`instance_ok`/`instance_fail`，JSON Lines 到 stderr）
 - batch 模式统一 schema（`status` 字段区分成功/失败）
-- 539 个单元+集成测试
+- 544 个单元+集成测试
 
 ## 跨 Agent 兼容性
 
diff --git a/SKILL.md b/SKILL.md
index b08e58d..14cd93c 100644
--- a/SKILL.md
+++ b/SKILL.md
@@ -1,7 +1,7 @@
 ---
 name: searxng-use-cli
 description: Use when you need to search the web via your OWN SearXNG instance (no public-instance discovery). 3 CLI scripts + a shared common.py module — execute privacy-respecting searches against a user-supplied instance (with multi-instance failover, 5xx/429 retry, auto-fetch) or via SEARXNG_INSTANCE env / config file, fetch/extract readable text or markdown from web pages. Zero-config replacement for proprietary search APIs.
-version: 2.1.1
+version: 2.2.1
 author: Metona Team
 license: MIT
 platforms: [linux, macos, windows]
@@ -31,7 +31,7 @@ metadata:
 
 ## Overview
 
-SearXNG is a privacy-respecting metasearch engine that aggregates results from 70+ search services without tracking users. This skill provides three standalone Python CLI scripts — works with **any AI agent** (Hermes, Claude Code, Codex, OpenCode, Cursor, Trae, etc.) or directly from your terminal.
+SearXNG is a privacy-respecting metasearch engine that aggregates results from 70+ search services without tracking users. This skill provides standalone Python CLI scripts — works with **any AI agent** or directly from your terminal.
 
 **Public-instance discovery has been removed.** You must supply your own SearXNG instance URL (self-hosted or one you trust). This makes behavior deterministic and avoids depending on volatile public instances.
 
@@ -39,24 +39,26 @@ SearXNG is a privacy-respecting metasearch engine that aggregates results from 7
 
 **Search & results**
 - Multi-instance failover with parallel probing (faster failover, deterministic output order)
-- Exponential-backoff retry on transient errors (429/5xx/connection) via shared `common.py`
+- Exponential-backoff retry on transient errors (403/429/5xx/connection) via shared retry-policy components in `common.py` (backoff/retryable-status/Retry-After), with the retry loop in each script
 - `--verify` health-check mode (reachability / JSON-API / latency / POST / engine list / auth status)
 - Cross-engine result deduplication (default on; `--no-dedup` disables) — collapses duplicate URLs ignoring tracking params (`utm_*`, `gclid`, etc.) and fragments
+- Similarity deduplication (`--similarity-dedup`; off by default) — title-based SimHash + Jaccard, auto-skipped when results > 500
 - Result sorting (`--sort-by {score,date,engine,none}`; default: score descending) — applied after dedup, before `--max-results`
 - Domain allowlist/blocklist (`--include-domain` / `--exclude-domain`) — case-insensitive, ignores leading `www.`, exclude wins on conflict
+- Pagination aggregation (`--pages N`) — fetch N pages in one run, merge with cross-page dedup
 - Batch mode (`--queries-file`) — run multiple queries from a file in sequence, combined output
 
 **Output formats**
 - JSON (default, rich metadata), brief (title+URL+snippet), urls (plain list), CSV (spreadsheet-ready)
-- Enhanced Markdown conversion — nested ordered lists (numbered), mixed `ul`/`ol` nesting, `<dl>`/`<dt>`/`<dd>` definition lists, GFM tables, fenced code blocks, blockquotes
+- Enhanced Markdown conversion — nested ordered lists, mixed `ul`/`ol` nesting, `<dl>`/`<dt>`/`<dd>` definition lists, GFM tables, fenced code blocks, blockquotes
 - Structured JSON error output (in `--format json` mode) with `error_code` field for machine-readable failure reporting
-- JSON Lines streaming (`--stream`) — each result emitted as a separate JSON line to stdout, enabling incremental processing by AI agents
-- Progress events (`--progress`) — structured JSON Lines events to stderr for real-time execution tracking
+- JSON Lines streaming (`--stream`) — each result emitted as a separate JSON line to stdout, enabling incremental processing
+- Progress events (`--progress`) — structured JSON Lines events to stderr for real-time execution tracking, includes `request_id` for correlation
 
 **Caching & config**
-- SQLite result caching (`--cache-ttl`) — identical queries within a TTL skip the network entirely; `--clear-cache` / `--cache-stats` manage it
-- Config file (`searxng.toml`) pre-sets most flags; `--config FILE` loads a non-default config; `instances.txt` for plain URL lists
-- Instance resolution priority: `-i` → `SEARXNG_INSTANCE` env → config file (`./searxng.toml` → `~/.config/searxng-cli/searxng.toml` → `%APPDATA%/searxng-cli/searxng.toml` on Windows)
+- SQLite result caching (`--cache-ttl`) with size cap + LRU eviction (`--cache-max-size`, default 100MB) — identical queries within a TTL skip the network entirely; `--clear-cache` / `--cache-stats` manage it
+- Config file (`searxng.toml`) pre-sets most flags; `--config FILE` loads a non-default config; `instances.txt` for plain URL lists; `--save-config FILE` writes current args to a config file
+- Instance resolution priority: `-i` → `SEARXNG_INSTANCE` env → config file (`./searxng.toml` → `~/.config/searxng-cli/searxng.toml` → `%APPDATA%/searxng-cli/searxng.toml` on Windows) → `instances.txt` (`./instances.txt` → `~/.config/searxng-cli/instances.txt` → `%APPDATA%/searxng-cli/instances.txt`)
 
 **Network & auth**
 - Proxy support (`--proxy`) for both search and fetch (sets `HTTP_PROXY`/`HTTPS_PROXY`/`NO_PROXY`)
@@ -64,45 +66,53 @@ SearXNG is a privacy-respecting metasearch engine that aggregates results from 7
 - Config-file auth — `auth_basic` and `auth_bearer` fields in `searxng.toml` let AI agents set credentials once (priority: CLI > file > config > env)
 - Credentials-file permission warning — `--auth-*-file` warns on stderr if the file is group/other-readable (POSIX only)
 
-**Anti-bot & fetch stability (v2.0.0)**
-- **Browser fingerprint headers** — `build_browser_headers()` sends full Sec-Ch-Ua / Sec-Fetch-* / Accept-Language / Accept-Encoding, not just User-Agent. Bypasses 80%+ of lightweight WAFs (Cloudflare basic, Nginx UA blocks)
-- **12-UA pool** — Chrome/Edge/Firefox × Windows/macOS/Linux × versions 129-131. `get_ua_for_domain()` uses SHA-256 to deterministically assign one UA per domain (stable within a session, reproducible across processes)
-- **requests.Session reuse** — module-level Session with connection pool (10 conns/host) + cookie persistence + TLS session resumption. Cuts TLS handshake overhead for multi-page fetches
-- **Split timeouts** — `timeout=(connect, read)` tuple (5s connect, 15s read by default). Previously a single 15s total timeout wasted already-established connections on large pages
-- **Retry-After compliance** — 429/503 responses read the `Retry-After` header (numeric seconds or HTTP date) and wait at least that long before retrying. Non-compliance triggers harsher rate limits
-- **Capped backoff** — `compute_backoff_delay()` caps at 60s (was uncapped: 1.5*2^10 = 1536s would hang the process)
-- **WAF fingerprint library** — `_detect_anti_bot()` identifies Cloudflare / Imperva / PerimeterX / DataDome / Akamai / Anubis / generic challenges via `<title>` tag matching (most precise) + full-document scan of technical identifiers (cookie/header/JS variable names, not bare vendor names). v2.0.1 narrowed broad keywords (e.g. bare `cloudflare`/`captcha`/`challenge`) that caused false positives on normal articles, and added title-tag detection. Returns `waf_type` for AI-agent decisioning
-- **Wayback Machine fallback** — 404/403/timeout AND anti-bot-blocked pages automatically retry via `https://web.archive.org/web/2/<url>` (latest snapshot). v2.0.1 fixed a bug where HTTP 200 anti-bot challenge pages (Cloudflare returns 200 for JS challenges) bypassed the fallback trigger. v2.1.0: `fetch.py` standalone calls now also get Wayback fallback (was only in `search.py --fetch`). Default ON; `--no-fallback` disables. Independent 10s timeout so Wayback slowness never blocks the main flow
-- **Hard-blocked domain fallback** (v2.1.0, refined in v2.1.1) — `is_hard_blocked_domain()` detects known strong-anti-bot sites (www.baidu.com, baike.baidu.com, zhihu.com, weibo.com, mp.weixin.qq.com, douban.com, etc.) that almost always return 403 regardless of UA. These sites bypass the normal `should_try_wayback()` check and trigger Wayback immediately on failure. v2.1.1 refined the list: moved `baidu.com` from broad subdomain matching to a precise per-subdomain list (www/baike/zhidao/tieba/wenku), so `pan.baidu.com` (netdisk) and `cloud.baidu.com` (cloud) are no longer false-positively blocked. List maintained in `common.py` `HARD_BLOCKED_DOMAINS`
-- **Adaptive throttling** — `AdaptiveThrottle` state machine: 3 consecutive failures → double delay + halve concurrency; 5 consecutive successes → gradual recovery; 429 → global pause 30s. Thread-safe
-- **readability-lite extraction** — when `<article>`/`<main>`/content-class `<div>` are all missing, `_readability_lite()` picks the highest text-density node (text chars / tag count + `<p>` weighting), avoiding nav/sidebar/footer noise
-- **`--fetch-report`** — structured per-URL report to stderr after `--fetch N`: status, WAF type, fallback used, char count, plus adaptive throttle stats and a JSON summary line
-- **`--referer`** — set Referer header for fetch requests (defaults to the instance URL when fetching result pages, disguising traffic source)
+**Anti-bot & fetch stability**
+- **Browser fingerprint headers** — `build_browser_headers()` sends full Sec-Ch-Ua / Sec-Fetch-* / Accept-Language / Accept-Encoding, not just User-Agent. Bypasses 80%+ of lightweight WAFs (Sec-Ch-Ua/Sec-Fetch-* only sent for Chrome/Edge; Firefox omits them). **Smart Accept-Encoding**: only advertises `br` when the brotli decompressor is actually installed, preventing servers from returning Brotli-compressed bytes that requests can't auto-decompress (which would produce garbled output)
+- **15-UA pool** — Chrome 138-140 (Win/mac/Linux), Edge 138 (Win/mac), Firefox 140 (Win/mac/Linux), Safari 18 (mac). `get_ua_for_domain()` deterministically assigns one UA per domain (stable within a session, reproducible across processes)
+- **requests.Session reuse** — module-level Session with connection pool + cookie persistence + TLS session resumption
+- **Split timeouts** — `timeout=(connect, read)` tuple (connect capped at 10s, read defaults to 15s)
+- **Retry-After compliance** — 429/503 responses read the `Retry-After` header (numeric seconds or HTTP date) and wait at least that long before retrying
+- **Capped backoff** — `compute_backoff_delay()` caps at 60s
+- **WAF fingerprint library** — `_detect_anti_bot()` (in `search.py`, used by `--fetch` flow) identifies Cloudflare / Imperva / PerimeterX / DataDome / Akamai / Anubis / generic challenges via `<title>` tag matching + full-document scan. Returns `waf_type` for AI-agent decisioning
+- **Wayback Machine fallback** — 404/403/timeout AND anti-bot-blocked pages automatically retry via `https://web.archive.org/web/2/<url>` (latest snapshot). Default ON; `--no-fallback` disables. Wayback timeout = `min(--timeout, 10s)`; Wayback retries = `min(--retries, 2)`
+- **Hard-blocked domain fallback** — `is_hard_blocked_domain()` (in `common.py`) detects known strong-anti-bot sites (www.baidu.com, baike.baidu.com, zhihu.com, weibo.com, mp.weixin.qq.com, douban.com, etc.) that almost always return 403. These trigger Wayback immediately on failure. List maintained in `HARD_BLOCKED_DOMAINS`. Note: baidu uses exact subdomain matching (www/baike/zhidao/tieba/wenku) to avoid over-blocking pan.baidu.com; zhihu/weibo/douban use subdomain wildcard matching
+- **Adaptive throttling** — `AdaptiveThrottle` (in `search.py`) state machine: consecutive failures → double delay + halve concurrency; consecutive successes → gradual recovery; 429 → global pause. Thread-safe. Parameters configurable via `--throttle-failure-threshold` / `--throttle-pause-seconds` / `--throttle-max-delay`
+- **readability-lite extraction** — when `<article>`/`<main>`/`role="main"`/content-class `<div>` are all missing and only `<body>` remains, picks the highest text-density `<div>`/`<section>`/`<article>` node (scored by text density + `<p>` count weighting). Text-density threshold is language-aware — CJK content uses 100 chars, other languages use 200 chars
+- **PDF/document parsing** — `fetch.py` parses PDF (via `pdftotext` subprocess) and `.docx`/`.xlsx` (via stdlib `zipfile`). Unsupported binary types return `E_UNSUPPORTED_MEDIA`
+- **`--fetch-report [json]`** — structured per-URL report to stderr after `--fetch N`: status, WAF type, fallback used, char count, plus throttle stats. `--fetch-report json` outputs a full JSON report (items array + summary)
+- **`--referer`** — set Referer header for fetch requests (defaults to the instance URL when fetching result pages)
 - **`--request-delay`** — configurable delay between fetch requests (default 0.3s; adaptive throttling may increase this on failures)
-- New fetch result fields: `anti_bot_detected` (bool), `waf_type` (str|null), `fallback_used` (str|null), `error_code` (str|null, v2.1.1 — structured error code on fetch failure, e.g. `E_RATE_LIMIT` for 429, lets AI agents programmatically distinguish rate-limit from auth/network errors)
+- **`search.py --fetch` output fields** (in `fetched` array): `anti_bot_detected` (bool), `waf_type` (str|null), `fallback_used` (str|null), `error_code` (str|null — structured error code on fetch failure)
+- **`fetch.py` FetchResult fields** (standalone script output): `content` (str), `content_type` (str), `final_url` (str), `truncated` (bool), `user_agent` (str), `error_code` (str|null), `error_message` (str|null)
 
-**Research mode (v2.1.0, enhanced in v2.1.1)**
-- **`--research <topic>`** — given a research topic, auto-expands into 5 multi-angle queries (overview/profile/background/works/review) and runs them in sequence. Results include `research_topic` and `research_queries` metadata so AI agents can structure their final report by angle. v2.1.1 enhancements:
-  - **Cross-angle merge & dedup** — JSON output now includes a top-level `merged_results` field: all per-angle results are combined, deduplicated (same URL collapsing), and sorted, so AI agents can get a unified overview without re-deduplicating themselves. Brief/urls formats append a `[MERGED]` section after the per-angle blocks.
-  - **Bilingual suffixes** — `expand_research_queries()` now detects whether the topic contains CJK characters. Chinese topics use Chinese suffixes (简介/经历/作品/评价); English topics use English suffixes (profile/background/works/reviews). Avoids low-relevance cross-language combinations like "Python asyncio 经历".
-- Mutually exclusive with `--query` and `--queries-file`. Supports all output formats (json/brief/urls/csv). Deterministic expansion (no AI judgment) — same topic always produces same queries, reproducible across processes
+**Research mode**
+- **`--research <topic>`** — given a research topic, auto-expands into 5 multi-angle queries (overview/profile/background/works/review) and runs them in sequence. Results include `research_topic` and `research_queries` metadata so AI agents can structure their final report by angle
+  - **Cross-angle merge & dedup** — JSON output includes a top-level `merged_results` field: all per-angle results are combined, deduplicated, and sorted. Brief/urls formats append a `[MERGED]` section after the per-angle blocks
+  - **Bilingual suffixes** — detects whether the topic contains CJK characters. Chinese topics use Chinese suffixes (overview uses empty suffix = topic itself, then 简介/经历/作品/评价); English topics use English suffixes (overview uses empty suffix, then profile/background/works/reviews)
+  - **`--research-angles`** — custom angles overriding the default 5 (e.g. `"overview,profile,timeline,controversy"`); each angle appended directly as a query suffix
+- Mutually exclusive with `--query` and `--queries-file`. Supports all output formats. Deterministic expansion (no AI judgment) — same topic always produces same queries
+
+**Observability**
+- **Structured logging** (`--log-format {text,json}`, default text) — `json` outputs one JSON object per line `{ts, level, logger, msg, request_id}` for programmatic parsing
+- **Request ID propagation** — each run auto-generates an 8-char hex `request_id` threaded through all logs and progress events
+- **`--dry-run`** — preview mode: no HTTP requests sent; prints `{dry_run, instances, headers_count, action, ...}` JSON to stdout (`instances` is a list of instance URLs)
+- **`--progress`** — JSON Lines events to stderr with `request_id` for real-time tracking
 
 **Engineering**
-- Shared `common.py` module — unified retry/charset/auth/logging/UA-pool/browser-headers/backoff logic across both scripts
+- Shared `common.py` module — unified retry policy/charset/auth resolution/logging/UA-pool/browser-headers/backoff/error classification/error codes/recovery hints/Wayback fallback/hard-blocked domains/similarity dedup/UTF-8 enforcement/proxy/Retry-After parsing across both scripts
 - `search.py --fetch` reuses `fetch.py`'s higher-quality text extractor (no code duplication)
-- Structured logging (`--verbose` / `--quiet`) — three levels: default INFO (progress + warnings), `--verbose` DEBUG (HTTP detail, cache keys), `--quiet` WARNING (errors only). All log output to stderr; stdout reserved for data
-- UTF-8 stdout enforcement (`force_utf8_stdout()`) — Windows Python defaults to GBK and crashes on non-ASCII chars; both scripts force UTF-8 + `errors='replace'` at startup so `print('\xa0')` never raises
-- Windows config discovery — `resolve_instances` / `load_config` also check `%APPDATA%/searxng-cli/` (Windows per-user app convention) in addition to `~/.config/searxng-cli/` (POSIX convention)
-- fetch.py failure diagnostics — error output includes `status_code=`, `cause=`, `url=` fields so AI agents can programmatically distinguish 404 vs 403 vs DNS failure without parsing English prose
+- Structured logging (`--verbose` / `--quiet`) — three levels: default INFO, `--verbose` DEBUG, `--quiet` WARNING. All log output to stderr; stdout reserved for data
+- UTF-8 stdout enforcement (`force_utf8_stdout()`) — Windows Python defaults to GBK and crashes on non-ASCII chars; both scripts force UTF-8 at startup
+- Windows config discovery — also checks `%APPDATA%/searxng-cli/` in addition to `~/.config/searxng-cli/`
+- Fetch failure diagnostics — error output includes `status_code=`, `cause=`, `url=` fields for programmatic distinction of 404 vs 403 vs DNS failure
 - Engine/category whitespace normalization (`"google, bing"` → `"google,bing"`)
-- `--time-range none` option to disable time filtering
 
 **Scripts + shared module:**
-1. `search.py` — execute searches against a user-supplied instance, with multi-instance failover + exponential-backoff retry (429/5xx/connection) + auto-fetch + caching + batch + domain filtering
+1. `search.py` — execute searches against a user-supplied instance, with multi-instance failover + retry + auto-fetch + caching + batch + domain filtering
 2. `fetch.py` — download and extract readable text or markdown from web pages
-3. `common.py` — shared utilities (auth headers, charset detection, retry policy, fallback UAs, retry constants) used by both scripts
-4. `cache.py` — SQLite-backed result cache (SHA-256 key, TTL, WAL mode)
-5. `_config.py` — package constants (version, User-Agent)
+3. `common.py` — shared utilities (auth resolution + headers, charset detection, retry policy, UA pool, browser headers, backoff, logging, progress events, error classification + codes, recovery hints, Wayback fallback, hard-blocked domains, similarity dedup, UTF-8 enforcement, proxy, Retry-After parsing) used by both scripts
+4. `cache.py` — SQLite-backed result cache (SHA-256 key, TTL, WAL mode, size cap + LRU eviction, schema migration, module-level + class API)
+5. `_config.py` — package constants (version, schema version, default user agent, UA pool)
 
 ## Default settings
 
@@ -112,7 +122,7 @@ SearXNG is a privacy-respecting metasearch engine that aggregates results from 7
 |---------|---------|------------------|
 | Instance | **required** — via `-i`, `SEARXNG_INSTANCE` env var, or config file | `-i / --instance` |
 | Safe search | **0 (off)** | `-s / --safesearch {0,1,2}` |
-| Time range | **year** | `-t / --time-range {day,week,month,year,none}` (none = disabled; `week` added in v2.1.1 to align with SearXNG API standard) |
+| Time range | **year** | `-t / --time-range {day,week,month,year,none}` (none = disabled) |
 | Output format | **json** | `-f / --format {json,brief,urls,csv}` |
 | Engines | **google,bing,brave,duckduckgo,startpage,wikipedia,wikidata** | `--engines <list>` |
 
@@ -150,7 +160,7 @@ python scripts/search.py -q "python asyncio tutorial"          # -i not needed
 #    auth_bearer = "sk-token-123"        # Bearer token (auth_bearer wins if both set)
 #    # Any flag below can also be pre-set here (engines, categories, language,
 #    # safesearch, time_range, method, format, timeout, max_retries, proxy,
-#    # cache_ttl, fetch, fetch_timeout, fetch_retries, max_size).
+#    # cache_ttl, cache_max_size, fetch, fetch_timeout, fetch_retries, max_size).
 #    # Explicit CLI flags always override config values.
 #    # Auth priority: --auth-* > --auth-*-file > searxng.toml > env var
 #    Plain list also works in ./instances.txt (one URL per line, # for comments)
@@ -175,7 +185,7 @@ python scripts/search.py -q "ai news" -i https://s.example.com --proxy http://co
 python scripts/search.py -q "test" -i https://private.example.com --auth-bearer-file ~/.searxng_token
 
 # 12. Cache management (no search performed)
-python scripts/search.py --cache-stats        # entry count, age, size, path
+python scripts/search.py --cache-stats        # entries, oldest/newest, size, total, max, evicted, utilization
 python scripts/search.py --clear-cache        # delete all entries
 
 # 13. Export results as CSV (great for spreadsheets / data analysis)
@@ -249,6 +259,7 @@ AI agents can use `error_code` to programmatically decide recovery strategy, and
 | `E_EMPTY` | Empty results (exit code 2) | Refine query, broaden `--time-range`, add `--categories` |
 | `E_INPUT` | Input error (bad parameters, file not found) | Check query syntax, flag combinations, file paths |
 | `E_INTERNAL` | Internal error (unexpected exception) | Re-run with `--verbose`, report bug |
+| `E_UNSUPPORTED_MEDIA` | Unsupported binary media type (PDF/docx/xlsx parse failure or unsupported content type) | No auto hint — try a different URL, or install `pdftotext` for PDF parsing |
 
 ### JSON Lines Streaming (`--stream`)
 
@@ -277,12 +288,14 @@ Output format (each line is a separate JSON object):
 - Exit code 0 on success, 1 on error, 2 on empty results (done event still emitted)
 
 Only valid with `--format json` (single query mode). Using `--stream` with
-`--queries-file` or non-json formats raises `E_INPUT` immediately.
+`--queries-file`, `--research`, or non-json formats raises `E_INPUT` immediately.
 
 ### Progress Events (`--progress`)
 
 For long-running operations, `--progress` emits structured JSON Lines events
-to stderr, enabling AI agents to track execution progress in real time:
+to stderr, enabling AI agents to track execution progress in real time.
+Every event includes a `request_id` field (8-char hex, auto-generated per run)
+for correlating all events from a single execution.
 
 ```bash
 python scripts/search.py -q "research topic" -i https://your-instance --progress --fetch 3
@@ -291,17 +304,19 @@ python scripts/search.py -q "research topic" -i https://your-instance --progress
 Event types (each on its own line, JSON Lines format on stderr):
 
 ```jsonl
-{"event": "start", "query": "research topic", "instances": 2}
-{"event": "instance_try", "url": "https://instance1.example.com", "attempt": 1}
-{"event": "instance_ok", "url": "https://instance1.example.com", "latency": 0.342, "results": 10}
-{"event": "instance_fail", "url": "https://instance2.example.com", "error": "HTTP 503", "error_code": "E_NETWORK"}
-{"event": "cache_hit", "query": "research topic", "ttl": 30}
-{"event": "cache_store", "query": "research topic", "ttl": 30}
-{"event": "fetch_start", "count": 3}
-{"event": "fetch_ok", "url": "https://example.com/page", "chars": 12345}
-{"event": "fetch_fail", "url": "https://bad.example.com", "error": "HTTP 503"}
-{"event": "done", "results": 10, "query": "research topic"}
-{"event": "error", "error": "connection refused", "error_code": "E_NETWORK", "query": "..."}
+{"event": "start", "query": "research topic", "instances": 2, "request_id": "a1b2c3d4"}
+{"event": "instance_try", "url": "https://instance1.example.com", "attempt": 1, "request_id": "a1b2c3d4"}
+{"event": "instance_ok", "url": "https://instance1.example.com", "latency": 0.342, "results": 10, "request_id": "a1b2c3d4"}
+{"event": "instance_fail", "url": "https://instance2.example.com", "error": "HTTP 503", "error_code": "E_NETWORK", "request_id": "a1b2c3d4"}
+{"event": "cache_hit", "query": "research topic", "ttl": 30, "request_id": "a1b2c3d4"}
+{"event": "cache_store", "query": "research topic", "ttl": 30, "request_id": "a1b2c3d4"}
+{"event": "page_ok", "pageno": 1, "results": 10, "request_id": "a1b2c3d4"}
+{"event": "page_fail", "pageno": 2, "error": "HTTP 503", "request_id": "a1b2c3d4"}
+{"event": "fetch_start", "count": 3, "request_id": "a1b2c3d4"}
+{"event": "fetch_ok", "url": "https://example.com/page", "chars": 12345, "request_id": "a1b2c3d4"}
+{"event": "fetch_fail", "url": "https://bad.example.com", "error": "HTTP 503", "request_id": "a1b2c3d4"}
+{"event": "done", "results": 10, "query": "research topic", "request_id": "a1b2c3d4"}
+{"event": "error", "error": "connection refused", "error_code": "E_NETWORK", "query": "...", "request_id": "a1b2c3d4"}
 ```
 
 AI agents can parse these events to:
@@ -309,6 +324,8 @@ AI agents can parse these events to:
 - Detect cache hits (skip waiting)
 - Monitor fetch failures and retry strategies
 - Correlate errors with specific queries in batch mode
+- Track per-page progress via `page_ok`/`page_fail` events
+- Correlate all events of a single run via `request_id`
 
 `--progress` and `--verbose` can be used together (progress events on stderr,
 debug logs also on stderr). `--progress` events are JSON Lines; `--verbose`
@@ -355,10 +372,11 @@ Single query output shape:
       "text_length": 12345,
       "truncated": false,
       "final_url": "https://example.com/final",
-      "user_agent_used": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+      "user_agent_used": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
       "anti_bot_detected": false,
       "waf_type": null,
-      "fallback_used": null
+      "fallback_used": null,
+      "error_code": null
     }
   ],
   "fetched_source": "json"
@@ -385,7 +403,7 @@ fields only appear when `--fetch N` is used.
 
 ## Cross-Agent Compatibility
 
-These scripts are **agent-agnostic** — they work with any AI agent that can invoke terminal commands (Hermes, Claude Code, Codex, OpenCode, Cursor, Trae, etc.), or directly from a terminal.
+These scripts are **agent-agnostic** — they work with any AI agent that can invoke terminal commands, or directly from a terminal.
 
 **Key design decisions for universal compatibility:**
 - Zero external dependencies (stdlib-only for `search.py`)
@@ -402,42 +420,47 @@ All scripts live in `scripts/`; run with `python scripts/<name>.py` from any dir
 
 ```
 usage: search.py [-h] [--query QUERY] [--instance URL]
-                 [--categories CATS] [--language LANG] [--pageno N]
+                 [--categories CATS] [--language LANG] [--pageno N] [--pages N]
                  [--time-range {day,week,month,year,none}] [--safesearch {0,1,2}]
                  [--engines E] [--method {GET,POST}] [--max-results N]
-                 [--format {json,brief,urls,csv}] [--snippet-len N]
-                 [--fetch N] [--fetch-timeout SEC] [--fetch-retries N]
-                 [--fetch-report] [--no-fallback] [--referer URL]
+                 [--format {json,brief,urls,csv}] [--snippet-len N] [--stream]
+                 [--progress] [--fetch N] [--fetch-timeout SEC] [--fetch-retries N]
+                 [--fetch-report [json]] [--no-fallback] [--referer URL]
                  [--request-delay SEC] [--max-size BYTES] [--output FILE]
                  [--timeout SEC] [--retry N] [--fail-fast] [--serial]
                  [--verify] [--auth-bearer TOKEN] [--auth-bearer-file FILE]
                  [--auth-basic USER:PASS] [--auth-basic-file FILE]
                  [--proxy URL] [--include-domain DOMAINS]
                  [--exclude-domain DOMAINS] [--queries-file FILE]
-                 [--cache-ttl MINUTES] [--clear-cache] [--cache-stats]
-                 [--sort-by {score,date,engine,none}] [--no-dedup]
+                 [--research TOPIC] [--research-angles ANGLES]
+                 [--cache-ttl MINUTES] [--cache-max-size MB] [--clear-cache]
+                 [--cache-stats] [--sort-by {score,date,engine,none}] [--no-dedup]
+                 [--similarity-dedup] [--similarity-threshold FLOAT]
+                 [--throttle-failure-threshold N] [--throttle-pause-seconds SEC]
+                 [--throttle-max-delay SEC] [--log-format {text,json}]
+                 [--save-config FILE] [--dry-run]
                  [--config FILE] [--dump-schema] [--verbose] [--quiet] [--version]
 ```
 
 **What it does:**
 1. Takes a search query and resolves one or more instance URLs (`-i`, `SEARXNG_INSTANCE`, or config file — comma-separated for failover)
 2. **Multi-instance failover:** if an instance fails (429/5xx/timeout/captcha), automatically tries the next one
-3. **Exponential backoff:** retries each instance up to 3 times with jitter on transient errors (429, 502, 503, 504, and connection errors)
+3. **Exponential backoff:** retries each instance up to 3 times with jitter on transient errors (403, 429, 502, 503, 504, and connection errors)
 4. **Parallel probing (default for 2+ instances):** queries every instance concurrently and returns the first *successful* result in your original instance order — this keeps output deterministic while drastically speeding up failover when an early instance is down/slow. Use `--serial` to disable.
 5. Calls the SearXNG API (GET or POST) with `format=json`
 6. Falls back to HTML scraping if JSON is blocked
 7. HTML parser extracts results + suggestions + answers + infoboxes (full content, never truncated)
-8. **Stable auto-fetch (v2.0.0 enhanced):** `--fetch 3` concurrently downloads top 3 result pages with retry, **full browser fingerprint headers** (Sec-Ch-Ua/Sec-Fetch-*), **12-UA deterministic per-domain pool**, **Retry-After compliance**, **WAF fingerprint detection** (Cloudflare/Imperva/PerimeterX/DataDome/Akamai), **Wayback Machine fallback** for 404/403/timeout, **adaptive throttling** (auto backoff + concurrency reduction on failures), and `fetch.py`'s readability-lite extractor. `--fetch-report` prints a structured per-URL report to stderr
+8. **Auto-fetch:** `--fetch 3` concurrently downloads top 3 result pages with retry, full browser fingerprint headers, 15-UA deterministic per-domain pool, Retry-After compliance, WAF fingerprint detection, Wayback Machine fallback for 404/403/timeout, adaptive throttling, and `fetch.py`'s readability-lite extractor. `--fetch-report` prints a structured per-URL report to stderr
 9. **Health-check mode:** `--verify` probes each instance (reachability / JSON-API support / latency / POST support / engine list / auth status) and prints a report, then exits without searching — use it to validate your instance list
-10. **Result caching:** `--cache-ttl 30` stores results for 30 min; identical queries within the TTL skip the network entirely. Cache lives at `$SEARXNG_CACHE_DIR` or `~/.cache/searxng-cli/cache.db` (SQLite, WAL mode). `--clear-cache` / `--cache-stats` manage it without searching
+10. **Result caching:** `--cache-ttl 30` stores results for 30 min; identical queries within the TTL skip the network entirely. Cache lives at `$SEARXNG_CACHE_DIR` or `~/.cache/searxng-cli/cache.db` (SQLite, WAL mode, LRU eviction). `--clear-cache` / `--cache-stats` manage it without searching
 11. **Batch mode:** `--queries-file FILE` reads one query per line (blank/`#` lines skipped) and runs them in sequence; JSON output is `{"schema_version": "1.0", "queries": [{"query":..., "status": "ok"|"error", ...}]}` (or one block per query in brief/urls). A failed query is recorded but does not abort the batch. Exit codes: 0 if any query returned results, 1 if all errored, 2 if all empty
 12. **Dedup + sort + domain filter:** After search (and cache), duplicate URLs are collapsed (default; `--no-dedup` disables), results are sorted (`--sort-by`; default: score descending), and then `--include-domain`/`--exclude-domain` filter by domain. Matching is case-insensitive and ignores a leading `www.`; when a domain is in both lists, exclude wins
 13. **Proxy & auth:** `--proxy URL` routes both search and fetch through a proxy; `--auth-bearer` / `--auth-basic` (plus `*-file` variants, `searxng.toml` `auth_basic`/`auth_bearer` fields, and `SEARXNG_BEARER_TOKEN` / `SEARXNG_BASIC_AUTH` env vars) supply credentials. Priority: CLI flag > file > config file > env var
-14. **Config defaults:** `searxng.toml` may pre-set most flags (engines, categories, language, safesearch, time_range, method, format, sort_by, timeout, max_retries, proxy, cache_ttl, fetch, fetch_timeout, fetch_retries, max_size, request_delay, auth_basic, auth_bearer); explicit CLI flags always win
+14. **Config defaults:** `searxng.toml` may pre-set most flags; explicit CLI flags always win
 15. **Structured errors:** in `--format json` mode, failures print a JSON object `{"error": "...", "error_code": "E_*", "recovery_hint": "...", "exit_code": N, "query": "..."}` to stdout so agents can parse them and decide recovery strategy
 
 **Key options:**
-- `--query "your search"` — **required unless** `--verify`, `--queries-file`, `--clear-cache`, `--cache-stats`, or `--dump-schema` is used
+- `--query "your search"` — **required unless** `--verify`, `--queries-file`, `--research`, `--clear-cache`, `--cache-stats`, `--save-config`, or `--dump-schema` is used
 - `--instance https://searx.example.org` — **required unless** `SEARXNG_INSTANCE` env var or a config file supplies it; comma-separated list enables failover
 - `--queries-file FILE` — read queries from a file (one per line; blank/`#` skipped) and run them in sequence; overrides `--query`
 - `--engines google,duckduckgo` — restrict to specific search engines (whitespace around commas is auto-stripped; see default list above)
@@ -445,11 +468,14 @@ usage: search.py [-h] [--query QUERY] [--instance URL]
 - `--categories general,news` — comma-separated categories (whitespace auto-stripped)
 - `--language zh-CN` — language filter
 - `--pageno 1` — page number
-- `--time-range {day,week,month,year,none}` — time filter (default: `year`; `none` disables time filtering; v2.1.1 adds `week` to align with SearXNG API's standard four tiers)
+- `--pages N` — fetch N pages of results in one run and merge with cross-page dedup; each page cached independently (cache key includes `pageno`)
+- `--time-range {day,week,month,year,none}` — time filter (default: `year`; `none` disables time filtering)
 - `--safesearch {0,1,2}` — safe search (default: `0` = off)
 - `--max-results N` — limit number of results (applied AFTER dedup+sort, so the highest-scoring/newest items are kept)
 - `--sort-by {score,date,engine,none}` — sort results (default: `score` descending; `none` preserves instance order). Applied after dedup, before `--max-results`. HTML-fallback results have no score and keep their order
 - `--no-dedup` — disable cross-engine deduplication (by default, duplicate URLs — same page ignoring tracking params/fragment — are collapsed, keeping the first occurrence's engine/score)
+- `--similarity-dedup` — enable title-based similarity deduplication (SimHash + Jaccard); off by default; auto-skipped when results > 500
+- `--similarity-threshold FLOAT` — similarity threshold for `--similarity-dedup` (default: 0.85; higher = stricter)
 - `--config FILE` — path to a `searxng.toml` config file; overrides the default auto-discovery (`./searxng.toml` → `~/.config/searxng-cli/searxng.toml` → `%APPDATA%/searxng-cli/searxng.toml` on Windows). Must be the first flag so its values can set defaults for other flags
 - `--verbose` / `-v` — show debug-level diagnostics on stderr (HTTP request URLs, response codes, cache keys, retry detail)
 - `--quiet` — suppress progress messages and retry notices on stderr; only warnings and errors are shown (no short flag: `-q` is `--query`)
@@ -470,13 +496,22 @@ usage: search.py [-h] [--query QUERY] [--instance URL]
 - `--fail-fast` — use only the first instance, don't fail over to the rest
 - `--proxy URL` — HTTP/HTTPS proxy for both search and fetch (e.g. `http://corp-proxy:8080`)
 - `--cache-ttl MINUTES` — cache results for N minutes (default: `0` = disabled); identical queries within the TTL skip the network
+- `--cache-max-size MB` — cache size cap in MB with LRU eviction (default: `0` = defer to `$SEARXNG_CACHE_MAX_SIZE_BYTES` env var or 100MB built-in default; when cap exceeded, least-recently-accessed entries evicted; set a very large value for effectively unlimited)
 - `--clear-cache` — delete all cached entries and exit (no search)
-- `--cache-stats` — print cache statistics (entries, age, size, path) and exit
+- `--cache-stats` — print cache statistics (entries, oldest_created_at, newest_created_at, path, size_bytes, total_bytes, max_size_bytes, evicted_count, utilization_pct) and exit
 - `--dump-schema` — print the JSON Schema for `--format json` output and exit; lets AI agents programmatically discover field names and types without parsing prose docs
 - `--auth-bearer TOKEN` — `Authorization: Bearer` header for private instances
 - `--auth-bearer-file FILE` — read Bearer token from a file (first non-empty, non-`#` line); also honors `SEARXNG_BEARER_TOKEN` env var
 - `--auth-basic USER:PASS` — `Authorization: Basic` header (auto base64-encoded)
 - `--auth-basic-file FILE` — read `user:pass` from a file (first non-empty, non-`#` line); also honors `SEARXNG_BASIC_AUTH` env var
+- `--log-format {text,json}` — structured logging format (default: `text`); `json` outputs one JSON object per line `{ts, level, logger, msg, request_id}`
+- `--dry-run` — preview mode: no HTTP requests; prints `{dry_run, instances, headers_count, action, ...}` JSON to stdout (`instances` is a list); supports search/research/batch/verify
+- `--throttle-failure-threshold N` — consecutive failures before doubling delay + halving concurrency (default: 3)
+- `--throttle-pause-seconds SEC` — global pause on 429 (default: 30)
+- `--throttle-max-delay SEC` — max adaptive delay (default: 10)
+- `--research-angles "a,b,c"` — custom research angles overriding the default 5; each angle appended directly as a query suffix
+- `--save-config FILE` — save current CLI args as a `searxng.toml` config file and exit
+- `--fetch-report [json]` — `--fetch-report json` outputs a full JSON report (items + summary); original `--fetch-report` (no arg) keeps text format
 - `--version` — print version and exit
 
 **Completion criterion:** Outputs valid JSON with `results` array. Non-zero exit on total failure (all instances exhausted).
@@ -487,7 +522,7 @@ usage: search.py [-h] [--query QUERY] [--instance URL]
 usage: fetch.py [-h] --url URL [--extract {text,html,markdown}]
                 [--timeout SEC] [--retries N] [--max-size BYTES]
                 [--user-agent STR] [--encoding CHARSET]
-                [--no-redirect] [--referer URL] [--proxy URL]
+                [--no-redirect] [--no-fallback] [--referer URL] [--proxy URL]
                 [--output FILE] [--auth-bearer TOKEN] [--auth-bearer-file FILE]
                 [--auth-basic USER:PASS] [--auth-basic-file FILE]
                 [--verbose] [--quiet] [--version]
@@ -495,24 +530,26 @@ usage: fetch.py [-h] --url URL [--extract {text,html,markdown}]
 
 **What it does:**
 1. Downloads a web page via HTTP GET with retry + exponential backoff
-2. **Stable fetching (v2.0.0 enhanced):** retries on 429/5xx/connection errors (3x default) with **Retry-After header compliance** and **capped 60s backoff**; falls back through a **12-UA deterministic pool** with **full browser fingerprint headers** (Sec-Ch-Ua/Sec-Fetch-*/Accept-Language); reuses a **requests.Session** for connection pooling + cookie persistence
+2. Retries on 429/5xx/connection errors (3x default) with Retry-After header compliance and capped 60s backoff; falls back through the 15-UA deterministic pool with full browser fingerprint headers; reuses a requests.Session for connection pooling + cookie persistence. **Both paths handle Content-Encoding decompression**: stdlib urllib path handles gzip/deflate/br (urllib doesn't auto-decompress any of them); requests path handles br manually (requests auto-decompresses gzip/deflate but not Brotli unless the brotli package is installed)
 3. **No size limit by default** — full page content returned; use `--max-size` for a cap
 4. Detects charset from HTTP headers, HTML meta tags, or UTF-8 fallback
-5. Extracts readable content using tree-based parsers (stdlib or BeautifulSoup); v2.0.0 adds **readability-lite** text-density fallback when `<article>`/`<main>` are missing
+5. Extracts readable content using tree-based parsers (stdlib or BeautifulSoup); readability-lite text-density fallback when `<article>`/`<main>`/`role="main"`/content-class `<div>` are all missing and only `<body>` remains
 6. Outputs clean text, raw HTML, or properly-converted Markdown (with correct nested-link handling)
+7. **PDF/document parsing** (regardless of `--extract` mode): PDF via `pdftotext` subprocess, `.docx`/`.xlsx` via stdlib `zipfile`. Unsupported binary types return `E_UNSUPPORTED_MEDIA`
 
 **Key options:**
 - `--url https://...` — required
-- `--extract text` — clean readable text (default)
+- `--extract text` — clean readable text (default); PDF/docx/xlsx parsed automatically regardless of extract mode (see "What it does" #7)
 - `--extract html` — raw HTML
 - `--extract markdown` — Markdown conversion (tree-based; handles nested tags, GFM tables, fenced code blocks, blockquotes, inline code, ordered/unordered/nested lists, definition lists, images, emphasis)
 - `--encoding gbk` — force charset for non-UTF-8 pages (else auto-detected from HTTP header / HTML meta / UTF-8 fallback)
-- `--timeout 15` — request timeout in seconds (v2.0.0: split into connect/read tuple internally)
-- `--retries 3` — max retries on transient errors (429/5xx/connection); falls back through the 12-UA pool when blocked
+- `--timeout 15` — request timeout in seconds (split into connect/read tuple internally)
+- `--retries 3` — max retries on transient errors (429/5xx/connection); falls back through the 15-UA pool when blocked
 - `--max-size BYTES` — cap page size (default: unlimited; e.g. `5242880` for 5MB)
 - `--user-agent STR` — custom User-Agent header (overrides per-domain UA selection)
 - `--no-redirect` — do **not** follow HTTP 3xx redirects (implemented for both the requests and stdlib paths)
-- `--referer URL` — set Referer header to disguise traffic source (v2.0.0 anti-bot measure)
+- `--no-fallback` — disable Wayback Machine fallback for 404/403/timeout and anti-bot-blocked pages (hard-blocked domains like baike.baidu.com, zhihu.com still prefer Wayback)
+- `--referer URL` — set Referer header to disguise traffic source
 - `--proxy URL` — HTTP/HTTPS proxy (e.g. `http://corp-proxy:8080`); respects existing `HTTP_PROXY`/`HTTPS_PROXY` env vars when omitted
 - `--output FILE` — save to file instead of stdout
 - `--auth-bearer TOKEN` / `--auth-bearer-file FILE` — `Authorization: Bearer` header; file variant reads first non-empty, non-`#` line; also honors `SEARXNG_BEARER_TOKEN` env var
@@ -523,10 +560,9 @@ usage: fetch.py [-h] --url URL [--extract {text,html,markdown}]
 
 **Extraction strategy (text mode):**
 1. Strip non-content elements (script, style, nav, footer, header)
-2. Extract `<article>`, `<main>`, or `<body>` content
-3. v2.0.0: if only `<body>` matched, run **readability-lite** to pick the highest text-density `<div>`/`<section>` (filters out nav/sidebar/footer by class/id)
+2. Extract main content in priority order: `<article>` → `<main>` → `role="main"` → content-class `<div>` (class~=`content`/`article`/`post`/`entry`) → `<body>`
+3. If only `<body>` matched (all higher-priority elements missing), run readability-lite to pick the highest text-density `<div>`/`<section>`/`<article>` node (scored by text density + `<p>` count weighting; filters out nav/sidebar/footer by class/id)
 4. Collapse whitespace, output clean UTF-8
 5. Warn if extracted text < 500 chars (likely JS-heavy or bot-blocked)
 
 **Completion criterion:** Outputs page content. Non-zero exit on HTTP failure. Stderr carries warnings for low-confidence extraction.
-
diff --git a/scripts/_config.py b/scripts/_config.py
index b13805d..4f2be33 100644
--- a/scripts/_config.py
+++ b/scripts/_config.py
@@ -1,12 +1,62 @@
 """Package-level constants for searxng-cli scripts.
 
 Import in sibling scripts with:
-    from _config import VERSION, USER_AGENT, SCHEMA_VERSION
+    from _config import VERSION, USER_AGENT, SCHEMA_VERSION, UA_POOL
 
 Retry settings and shared HTTP utilities now live in ``common.py`` so that
 both ``search.py`` and ``fetch.py`` share one consistent implementation.
 """
 
-VERSION = "2.1.1"
+VERSION = "2.2.1"
 SCHEMA_VERSION = "1.0"
 USER_AGENT = f"searxng-cli/{VERSION}"
+
+# 浏览器 UA 池（用于 searxng-cli 默认 UA 被站点拦截时的回退）。
+#
+# v2.2.0 从 common.py 迁移至此统一管理（SSOT）。common.py 通过
+# ``from _config import UA_POOL`` 引用，并在导入失败时回退到其内置副本。
+#
+# 维护原则：
+#   1. 版本号保持为当前年份的主流浏览器版本，避免被识别为过时浏览器
+#   2. 顺序固定——get_ua_for_domain() 用 SHA-256 哈希选索引，顺序变化会
+#      改变域名→UA 的映射，导致跨版本缓存失效（可接受，但应尽量避免无谓变动）
+#   3. 至少覆盖 Chrome/Edge/Firefox × Windows/macOS/Linux，保证指纹多样性
+#
+# 当前版本（2026 年）：Chrome 138-140 / Edge 138 / Firefox 140 / Safari 18。
+UA_POOL = [
+    # Chrome 140 — Windows / macOS / Linux
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
+    # Chrome 139 — Windows / macOS / Linux
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
+    # Chrome 138 — Windows / macOS / Linux
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
+    # Edge 138 — Windows / macOS
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36 Edg/138.0.0.0",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36 Edg/138.0.0.0",
+    # Firefox 140 — Windows / macOS / Linux
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) "
+    "Gecko/20100101 Firefox/140.0",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) "
+    "Gecko/20100101 Firefox/140.0",
+    "Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0",
+    # Safari 18 — macOS（WebKit 指纹，应对 Chromium 针对性拦截）
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 "
+    "(KHTML, like Gecko) Version/18.0 Safari/605.1.15",
+]
diff --git a/scripts/cache.py b/scripts/cache.py
index ae08756..34e7592 100644
--- a/scripts/cache.py
+++ b/scripts/cache.py
@@ -13,6 +13,14 @@ Storage location (in priority order):
 Uses WAL journal mode for better read concurrency. Entries expire lazily
 on read; :func:`clear` removes all rows. Schema is created on first use.
 
+大小上限与 LRU 淘汰（v2）:
+  * :class:`SearchCache` 支持 ``max_size_bytes`` 参数（默认 100 MB，0 表示
+    不限制，向后兼容）。put 时若总大小超限，按 ``last_accessed_at`` 升序
+    淘汰最旧条目，直到总大小 <= max_size_bytes。
+  * get 命中时更新 ``last_accessed_at``，实现 LRU 语义。
+  * :meth:`SearchCache.evict_expired` 可主动清理已过期条目。
+  * ``$SEARXNG_CACHE_MAX_SIZE_BYTES`` 环境变量可覆盖默认上限。
+
 Design notes:
   * Only the search result dict is cached — fetched page content is NOT,
     because it is large and changes independently of the search result set.
@@ -31,6 +39,8 @@ import time
 from pathlib import Path
 
 DEFAULT_CACHE_DIR = Path.home() / ".cache" / "searxng-cli"
+# 默认缓存大小上限：100 MB；0 表示不限制（向后兼容）
+DEFAULT_MAX_SIZE_BYTES = 104857600
 
 
 def _cache_path() -> Path:
@@ -41,6 +51,48 @@ def _cache_path() -> Path:
     return DEFAULT_CACHE_DIR / "cache.db"
 
 
+def _ensure_schema(conn: sqlite3.Connection) -> None:
+    """创建表结构并执行向后兼容的 schema 迁移。
+
+    v1 schema: key, created_at, ttl_seconds, payload
+    v2 新增列: last_accessed_at (LRU 排序依据), size_bytes (payload 字节大小)
+
+    对已存在的旧表用 ALTER TABLE ADD COLUMN 添加新列，并回填数据，
+    保证升级后现有条目也能参与大小统计与 LRU 淘汰。
+    """
+    conn.execute(
+        """
+        CREATE TABLE IF NOT EXISTS search_cache (
+            key TEXT PRIMARY KEY,
+            created_at REAL NOT NULL,
+            ttl_seconds INTEGER NOT NULL,
+            payload TEXT NOT NULL
+        )
+        """
+    )
+    # 检查现有列，决定是否需要迁移
+    cols = {row[1] for row in conn.execute("PRAGMA table_info(search_cache)").fetchall()}
+    if "last_accessed_at" not in cols:
+        # 新增 LRU 访问时间列，回填为 created_at（视为从未被访问过）
+        conn.execute("ALTER TABLE search_cache ADD COLUMN last_accessed_at REAL")
+        conn.execute(
+            "UPDATE search_cache SET last_accessed_at = created_at "
+            "WHERE last_accessed_at IS NULL"
+        )
+    if "size_bytes" not in cols:
+        # 新增 payload 字节大小列，回填为 payload 的 UTF-8 字节长度
+        conn.execute("ALTER TABLE search_cache ADD COLUMN size_bytes INTEGER")
+        rows = conn.execute(
+            "SELECT key, payload FROM search_cache WHERE size_bytes IS NULL"
+        ).fetchall()
+        for key, payload in rows:
+            conn.execute(
+                "UPDATE search_cache SET size_bytes = ? WHERE key = ?",
+                (len(payload.encode("utf-8")), key),
+            )
+    conn.commit()
+
+
 def _connect(path: Path):
     """Open a connection with WAL mode and ensure the schema exists.
 
@@ -56,17 +108,7 @@ def _connect(path: Path):
     # when --fetch spawns parallel page fetches that might also touch the cache.
     conn.execute("PRAGMA journal_mode=WAL")
     conn.execute("PRAGMA synchronous=NORMAL")
-    conn.execute(
-        """
-        CREATE TABLE IF NOT EXISTS search_cache (
-            key TEXT PRIMARY KEY,
-            created_at REAL NOT NULL,
-            ttl_seconds INTEGER NOT NULL,
-            payload TEXT NOT NULL
-        )
-        """
-    )
-    conn.commit()
+    _ensure_schema(conn)
     return contextlib.closing(conn)
 
 
@@ -89,82 +131,308 @@ def _make_key(params: dict) -> str:
     return hashlib.sha256(raw.encode("utf-8")).hexdigest()
 
 
-def get(params: dict, ttl_seconds: int):
-    """Return cached result if within TTL, else None.
+def _payload_size(payload: str) -> int:
+    """计算 payload 序列化后的 UTF-8 字节大小。"""
+    return len(payload.encode("utf-8"))
 
-    ``ttl_seconds`` is the caller's current TTL setting. If the stored
-    entry was written with a longer TTL, the caller's shorter TTL wins
-    (so reducing --cache-ttl takes effect immediately without a clear).
+
+class SearchCache:
+    """带大小上限和 LRU 淘汰的 SQLite 缓存。
+
+    Args:
+        path: 缓存数据库路径。None 表示使用 ``$SEARXNG_CACHE_DIR`` 或
+            默认路径（每次操作动态解析，便于测试 monkeypatch 环境变量）。
+        max_size_bytes: 缓存总大小上限（字节）。0 表示不限制（向后兼容）。
+
+    大小跟踪与 LRU 语义:
+        * put 时计算 payload 字节大小并维护 ``_total_bytes`` 计数器。
+        * 若加入新条目后总大小超过 ``max_size_bytes``，按
+          ``last_accessed_at`` 升序淘汰最旧条目。
+        * get 命中时更新 ``last_accessed_at``，将条目移到"最近使用"位置。
     """
-    if ttl_seconds <= 0:
-        return None
-    key = _make_key(params)
-    try:
-        with _connect(_cache_path()) as conn:
+
+    def __init__(self, path: Path = None, max_size_bytes: int = DEFAULT_MAX_SIZE_BYTES):
+        self._path_override = Path(path) if path else None
+        self._max_size_bytes = max_size_bytes
+        self._total_bytes = 0
+        self._evicted_count = 0
+        self._total_bytes_loaded = False
+        # 记录上次加载 _total_bytes 时的路径，路径变化时重新加载
+        self._loaded_path = None
+
+    def _resolve_path(self) -> Path:
+        """解析当前应使用的缓存路径（未显式指定时动态读取环境变量）。"""
+        return self._path_override or _cache_path()
+
+    def _load_total_bytes(self, conn: sqlite3.Connection) -> None:
+        """惰性从 DB 加载 _total_bytes；路径变化时重新加载。"""
+        current_path = str(self._resolve_path())
+        if self._total_bytes_loaded and self._loaded_path == current_path:
+            return
+        row = conn.execute(
+            "SELECT COALESCE(SUM(size_bytes), 0) FROM search_cache"
+        ).fetchone()
+        self._total_bytes = row[0] or 0
+        self._total_bytes_loaded = True
+        self._loaded_path = current_path
+
+    def get(self, params: dict, ttl_seconds: int):
+        """Return cached result if within TTL, else None.
+
+        ``ttl_seconds`` is the caller's current TTL setting. If the stored
+        entry was written with a longer TTL, the caller's shorter TTL wins
+        (so reducing --cache-ttl takes effect immediately without a clear).
+
+        命中时更新 ``last_accessed_at`` 以实现 LRU 语义。
+        """
+        if ttl_seconds <= 0:
+            return None
+        key = _make_key(params)
+        try:
+            with _connect(self._resolve_path()) as conn:
+                self._load_total_bytes(conn)
+                row = conn.execute(
+                    "SELECT payload, created_at, ttl_seconds FROM search_cache "
+                    "WHERE key = ?",
+                    (key,),
+                ).fetchone()
+                if row is None:
+                    return None
+                payload, created_at, stored_ttl = row
+                effective_ttl = min(ttl_seconds, stored_ttl)
+                if time.time() - created_at > effective_ttl:
+                    return None
+                # LRU: 命中时把条目移到"最近使用"位置
+                conn.execute(
+                    "UPDATE search_cache SET last_accessed_at = ? WHERE key = ?",
+                    (time.time(), key),
+                )
+                conn.commit()
+                return json.loads(payload)
+        except sqlite3.Error:
+            return None
+        except (ValueError, json.JSONDecodeError):
+            # Corrupt payload — treat as miss
+            return None
+
+    def put(self, params: dict, result: dict, ttl_seconds: int) -> None:
+        """Store a result with the given TTL. Silently no-ops on TTL<=0 or error.
+
+        若加入新条目后总大小超过 ``max_size_bytes``，按 LRU 淘汰最旧条目。
+        """
+        if ttl_seconds <= 0:
+            return
+        key = _make_key(params)
+        payload = json.dumps(result, ensure_ascii=False)
+        new_size = _payload_size(payload)
+        try:
+            with _connect(self._resolve_path()) as conn:
+                self._load_total_bytes(conn)
+                # 若 key 已存在，先减去旧条目大小，避免重复计入
+                old = conn.execute(
+                    "SELECT size_bytes FROM search_cache WHERE key = ?", (key,)
+                ).fetchone()
+                if old is not None:
+                    self._total_bytes -= (old[0] or 0)
+                now = time.time()
+                conn.execute(
+                    "INSERT OR REPLACE INTO search_cache "
+                    "(key, created_at, ttl_seconds, payload, "
+                    " last_accessed_at, size_bytes) VALUES (?, ?, ?, ?, ?, ?)",
+                    (key, now, ttl_seconds, payload, now, new_size),
+                )
+                self._total_bytes += new_size
+                self._enforce_size_limit(conn)
+                conn.commit()
+        except sqlite3.Error:
+            pass
+
+    def _enforce_size_limit(self, conn: sqlite3.Connection) -> None:
+        """总大小超限时按 LRU 淘汰最旧条目，直到总大小 <= max_size_bytes。
+
+        ``max_size_bytes <= 0`` 表示不限制，直接返回。当仅剩一个条目时
+        停止淘汰（避免 put 后立即被淘汰导致 get 不到刚写入的条目）。
+        """
+        if self._max_size_bytes <= 0:
+            return
+        while self._total_bytes > self._max_size_bytes:
             row = conn.execute(
-                "SELECT payload, created_at, ttl_seconds FROM search_cache "
-                "WHERE key = ?",
-                (key,),
+                "SELECT key, size_bytes FROM search_cache "
+                "ORDER BY last_accessed_at ASC, created_at ASC LIMIT 1"
             ).fetchone()
-        if row is None:
-            return None
-        payload, created_at, stored_ttl = row
-        effective_ttl = min(ttl_seconds, stored_ttl)
-        if time.time() - created_at > effective_ttl:
-            return None
-        return json.loads(payload)
-    except sqlite3.Error:
-        return None
-    except (ValueError, json.JSONDecodeError):
-        # Corrupt payload — treat as miss
-        return None
+            if row is None:
+                break
+            evict_key, evict_size = row
+            conn.execute("DELETE FROM search_cache WHERE key = ?", (evict_key,))
+            self._total_bytes -= (evict_size or 0)
+            self._evicted_count += 1
+            # 安全阀：只剩一个条目时停止淘汰（即新插入的条目本身超限也保留）
+            count_row = conn.execute("SELECT COUNT(*) FROM search_cache").fetchone()
+            if count_row[0] <= 1:
+                break
+
+    def clear(self) -> int:
+        """Remove all cache entries. Returns count deleted, or 0 on error."""
+        try:
+            with _connect(self._resolve_path()) as conn:
+                self._load_total_bytes(conn)
+                cur = conn.execute("DELETE FROM search_cache")
+                conn.commit()
+                self._total_bytes = 0
+                return cur.rowcount
+        except sqlite3.Error:
+            return 0
+
+    def stats(self) -> dict:
+        """Return cache statistics.
+
+        现有字段：entries, oldest_created_at, newest_created_at, path, size_bytes
+        新增字段：total_bytes, max_size_bytes, evicted_count, utilization_pct
+        """
+        path = self._resolve_path()
+        try:
+            with _connect(path) as conn:
+                row = conn.execute(
+                    "SELECT COUNT(*), MIN(created_at), MAX(created_at) "
+                    "FROM search_cache"
+                ).fetchone()
+                count, oldest, newest = row
+                # 从 DB 校准 total_bytes，防止外部进程修改导致计数器漂移
+                sum_row = conn.execute(
+                    "SELECT COALESCE(SUM(size_bytes), 0) FROM search_cache"
+                ).fetchone()
+                self._total_bytes = sum_row[0] or 0
+                self._total_bytes_loaded = True
+                self._loaded_path = str(path)
+                utilization = (
+                    round(self._total_bytes * 100.0 / self._max_size_bytes, 2)
+                    if self._max_size_bytes > 0
+                    else 0
+                )
+                return {
+                    "entries": count or 0,
+                    "oldest_created_at": oldest,
+                    "newest_created_at": newest,
+                    "path": str(path),
+                    "size_bytes": path.stat().st_size if path.exists() else 0,
+                    # 新增字段：大小上限与 LRU 统计
+                    "total_bytes": self._total_bytes,
+                    "max_size_bytes": self._max_size_bytes,
+                    "evicted_count": self._evicted_count,
+                    "utilization_pct": utilization,
+                }
+        except sqlite3.Error as e:
+            return {
+                "entries": 0,
+                "error": str(e),
+                "path": str(path),
+                "size_bytes": 0,
+                "total_bytes": 0,
+                "max_size_bytes": self._max_size_bytes,
+                "evicted_count": self._evicted_count,
+                "utilization_pct": 0,
+            }
+
+    def evict_expired(self) -> int:
+        """主动扫描并删除已过期条目，返回清理的条目数。
+
+        过期条件：``now - created_at > ttl_seconds``。清理时同步更新
+        ``_total_bytes`` 计数器。
+        """
+        now = time.time()
+        try:
+            with _connect(self._resolve_path()) as conn:
+                self._load_total_bytes(conn)
+                rows = conn.execute(
+                    "SELECT key, size_bytes FROM search_cache "
+                    "WHERE ? - created_at > ttl_seconds",
+                    (now,),
+                ).fetchall()
+                if not rows:
+                    return 0
+                for key, size in rows:
+                    conn.execute("DELETE FROM search_cache WHERE key = ?", (key,))
+                    self._total_bytes -= (size or 0)
+                conn.commit()
+                return len(rows)
+        except sqlite3.Error:
+            return 0
+
+    def set_max_size_bytes(self, max_size_bytes: int) -> None:
+        """更新大小上限并立即触发 LRU 淘汰（若当前已超限）。
+
+        供 CLI ``--cache-max-size`` 在运行时注入参数用。``max_size_bytes <= 0``
+        表示不限制。
+        """
+        self._max_size_bytes = max_size_bytes
+        if max_size_bytes <= 0:
+            return
+        try:
+            with _connect(self._resolve_path()) as conn:
+                self._load_total_bytes(conn)
+                self._enforce_size_limit(conn)
+                conn.commit()
+        except sqlite3.Error:
+            pass
+
+
+# ----- 模块级便捷 API（向后兼容）-----
+# 现有调用方（search.py、测试）使用模块级函数；这里委托给一个惰性创建的
+# 全局实例。全局实例的路径动态解析，因此 monkeypatch $SEARXNG_CACHE_DIR
+# 能正常隔离每个测试。
+
+_default_cache = None
+
+
+def _get_default_cache() -> SearchCache:
+    """惰性创建全局默认缓存实例。
+
+    ``max_size_bytes`` 从 ``$SEARXNG_CACHE_MAX_SIZE_BYTES`` 读取（非法值回退默认）。
+    创建后可被 :func:`set_max_size_bytes` 覆盖（CLI ``--cache-max-size`` 优先级
+    高于环境变量）。
+    """
+    global _default_cache
+    if _default_cache is None:
+        max_size = DEFAULT_MAX_SIZE_BYTES
+        env = os.environ.get("SEARXNG_CACHE_MAX_SIZE_BYTES")
+        if env:
+            try:
+                max_size = int(env)
+            except ValueError:
+                pass
+        _default_cache = SearchCache(max_size_bytes=max_size)
+    return _default_cache
+
+
+def get(params: dict, ttl_seconds: int):
+    """模块级便捷函数：委托给全局默认实例。"""
+    return _get_default_cache().get(params, ttl_seconds)
 
 
 def put(params: dict, result: dict, ttl_seconds: int) -> None:
-    """Store a result with the given TTL. Silently no-ops on TTL<=0 or error."""
-    if ttl_seconds <= 0:
-        return
-    key = _make_key(params)
-    payload = json.dumps(result, ensure_ascii=False)
-    try:
-        with _connect(_cache_path()) as conn:
-            conn.execute(
-                "INSERT OR REPLACE INTO search_cache "
-                "(key, created_at, ttl_seconds, payload) VALUES (?, ?, ?, ?)",
-                (key, time.time(), ttl_seconds, payload),
-            )
-            conn.commit()
-    except sqlite3.Error:
-        pass
+    """模块级便捷函数：委托给全局默认实例。"""
+    _get_default_cache().put(params, result, ttl_seconds)
 
 
 def clear() -> int:
-    """Remove all cache entries. Returns count deleted, or 0 on error."""
-    try:
-        with _connect(_cache_path()) as conn:
-            cur = conn.execute("DELETE FROM search_cache")
-            conn.commit()
-            return cur.rowcount
-    except sqlite3.Error:
-        return 0
+    """模块级便捷函数：委托给全局默认实例。"""
+    return _get_default_cache().clear()
 
 
 def stats() -> dict:
-    """Return cache statistics (entry count, age range, path)."""
-    path = _cache_path()
-    try:
-        with _connect(path) as conn:
-            row = conn.execute(
-                "SELECT COUNT(*), MIN(created_at), MAX(created_at) "
-                "FROM search_cache"
-            ).fetchone()
-        count, oldest, newest = row
-        return {
-            "entries": count or 0,
-            "oldest_created_at": oldest,
-            "newest_created_at": newest,
-            "path": str(path),
-            "size_bytes": path.stat().st_size if path.exists() else 0,
-        }
-    except sqlite3.Error as e:
-        return {"entries": 0, "error": str(e), "path": str(path), "size_bytes": 0}
+    """模块级便捷函数：委托给全局默认实例。"""
+    return _get_default_cache().stats()
+
+
+def evict_expired() -> int:
+    """模块级便捷函数：委托给全局默认实例。"""
+    return _get_default_cache().evict_expired()
+
+
+def set_max_size_bytes(max_size_bytes: int) -> None:
+    """模块级便捷函数：更新全局默认实例的大小上限。
+
+    供 search.py 的 ``--cache-max-size`` CLI 参数在 main() 早期注入用，
+    优先级高于 ``$SEARXNG_CACHE_MAX_SIZE_BYTES`` 环境变量。
+    """
+    _get_default_cache().set_max_size_bytes(max_size_bytes)
diff --git a/scripts/common.py b/scripts/common.py
index 402e828..f0a2143 100644
--- a/scripts/common.py
+++ b/scripts/common.py
@@ -18,9 +18,12 @@ as retryable and connection errors as transient, eliminating the previous
 inconsistency where ``search.py`` ignored 5xx.
 """
 
+import hashlib
 import io
 import logging
+import re
 import sys
+import threading
 import urllib.error
 
 # Root logger for the searxng-cli package. All modules create child loggers
@@ -28,8 +31,27 @@ import urllib.error
 # call controls them all.
 _LOG = logging.getLogger("searxng")
 
+# Brotli 解压支持检测（v2.2.1）。
+# requests 库自动解压 gzip/deflate，但**不自动解压 Brotli**（除非安装
+# brotli/brotlicffi 包）。若在 Accept-Encoding 中声明 br 而系统未安装
+# 解压器，服务器返回的 br 压缩字节会被当作文本解码 → 全页乱码。
+# 此检测用于 build_browser_headers() 智能声明 Accept-Encoding，避免
+# 声明无法兑现的 br。
+try:
+    import brotli as _brotli  # type: ignore
+    _HAS_BROTLI = True
+except ImportError:
+    try:
+        import brotlicffi as _brotli  # type: ignore
+        _HAS_BROTLI = True
+    except ImportError:
+        _brotli = None
+        _HAS_BROTLI = False
 
-def setup_logging(verbose: bool = False, quiet: bool = False) -> None:
+
+def setup_logging(verbose: bool = False, quiet: bool = False,
+                  log_format: str = "text",
+                  request_id: str = None) -> None:
     """Configure the ``searxng`` logger hierarchy.
 
     * Default (no flags): ``INFO`` — progress messages, warnings, retry notices.
@@ -39,6 +61,9 @@ def setup_logging(verbose: bool = False, quiet: bool = False) -> None:
       status codes, cache keys, and other diagnostic detail.
     * ``--quiet`` / ``-q``: ``WARNING`` — suppresses progress and retry noise;
       only warnings and errors reach stderr.
+    * ``--log-format json`` (v2.2.0): 每行一个 JSON 对象，便于 AI Agent
+      程序化解析。包含 ts/level/logger/msg/request_id 字段。
+    * request_id (v2.2.0): 贯穿所有日志和进度事件的请求标识符。
 
     All log output goes to stderr; stdout is reserved for data (JSON/CSV/etc.).
     """
@@ -50,15 +75,57 @@ def setup_logging(verbose: bool = False, quiet: bool = False) -> None:
         level = logging.INFO
 
     _LOG.setLevel(level)
+    # 存储 request_id 到 logger 全局，供 formatter 和进度事件使用
+    _LOG._request_id = request_id
+
     # Avoid duplicate handlers if setup_logging() is called twice (e.g. tests).
     if not _LOG.handlers:
         handler = logging.StreamHandler(sys.stderr)
-        handler.setFormatter(logging.Formatter("%(message)s"))
+        if log_format == "json":
+            handler.setFormatter(_JsonFormatter())
+        else:
+            handler.setFormatter(logging.Formatter("%(message)s"))
         _LOG.addHandler(handler)
+    else:
+        # 已有 handler（如测试环境），更新 formatter
+        for h in _LOG.handlers:
+            if log_format == "json":
+                h.setFormatter(_JsonFormatter())
+            else:
+                h.setFormatter(logging.Formatter("%(message)s"))
     # Don't let root logger add its own handler — we own the searxng namespace.
     _LOG.propagate = False
 
 
+class _JsonFormatter(logging.Formatter):
+    """JSON 结构化日志 formatter（v2.2.0）。
+
+    每行输出一个 JSON 对象：{"ts", "level", "logger", "msg", "request_id"}。
+    让 AI Agent 可程序化解析日志（统计重试次数、识别慢实例等）。
+    """
+
+    def format(self, record):
+        import json as _json
+        entry = {
+            "ts": _datetime_iso(record),
+            "level": record.levelname,
+            "logger": record.name,
+            "msg": record.getMessage(),
+        }
+        rid = getattr(_LOG, "_request_id", None)
+        if rid:
+            entry["request_id"] = rid
+        if record.exc_info and record.exc_info[1]:
+            entry["exception"] = type(record.exc_info[1]).__name__
+        return _json.dumps(entry, ensure_ascii=False)
+
+
+def _datetime_iso(record):
+    """格式化日志时间戳为 ISO 8601 字符串。"""
+    import datetime as _dt
+    return _dt.datetime.fromtimestamp(record.created).isoformat(timespec="milliseconds")
+
+
 def force_utf8_stdout() -> None:
     """Force stdout/stderr to UTF-8 to prevent Windows GBK encoding crashes.
 
@@ -124,40 +191,58 @@ RETRYABLE_STATUS = frozenset({403, 429, 502, 503, 504})
 
 # Browser-like UA strings for fallback when the searxng-cli UA is blocked.
 #
-# v2.0.0 扩充至 12 个：覆盖 Chrome/Edge/Firefox × Windows/macOS/Linux，
-# 每个都是较新版本（131/130/129），避免被识别为过时浏览器。
+# v2.2.0：UA 池迁移至 _config.py 的 UA_POOL（SSOT），此处通过导入引用。
+# 若 _config.py 不可导入（如 common.py 被单独分发），回退到下方内置副本
+# _FALLBACK_UAS_BUILTIN，保证模块始终可用。两份列表需保持同步，
+# _config.UA_POOL 为唯一权威来源。
+#
+# v2.0.0 起覆盖 Chrome/Edge/Firefox × Windows/macOS/Linux；v2.2.0 升级到
+# 2026 年版本（Chrome 138-140 / Edge 138 / Firefox 140 / Safari 18）。
 # 顺序固定以便 get_ua_for_domain() 的 hash 选择可复现。
-FALLBACK_UAS = [
-    # Chrome 131 — Windows / macOS / Linux
+_FALLBACK_UAS_BUILTIN = [
+    # Chrome 140 — Windows / macOS / Linux
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
-    "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+    "(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
     "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
-    "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+    "(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
     "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
-    "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
-    # Edge 131 — Windows / macOS
+    "(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
+    # Chrome 139 — Windows / macOS / Linux
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
-    "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
+    "(KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
     "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
-    "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
-    # Firefox 133 — Windows / macOS / Linux
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) "
-    "Gecko/20100101 Firefox/133.0",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:133.0) "
-    "Gecko/20100101 Firefox/133.0",
-    "Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0",
-    # Chrome 130 — Windows / macOS (上一个版本，应对 131 被针对性识别)
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
-    "(KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
-    "(KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
-    # Chrome 129 — Windows / Linux
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
-    "(KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
+    "(KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
     "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
-    "(KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
+    "(KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
+    # Chrome 138 — Windows / macOS / Linux
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
+    # Edge 138 — Windows / macOS
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36 Edg/138.0.0.0",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36 Edg/138.0.0.0",
+    # Firefox 140 — Windows / macOS / Linux
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) "
+    "Gecko/20100101 Firefox/140.0",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) "
+    "Gecko/20100101 Firefox/140.0",
+    "Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0",
+    # Safari 18 — macOS（WebKit 指纹，应对 Chromium 针对性拦截）
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 "
+    "(KHTML, like Gecko) Version/18.0 Safari/605.1.15",
 ]
 
+# 从 _config.py 导入权威 UA_POOL；导入失败时回退到内置副本，保证向后兼容。
+try:
+    from _config import UA_POOL as FALLBACK_UAS
+except ImportError:
+    FALLBACK_UAS = _FALLBACK_UAS_BUILTIN
+
 
 def _ua_index_for_domain(domain: str, pool_size: int) -> int:
     """为域名确定性选择 UA 池索引。
@@ -176,6 +261,13 @@ def _ua_index_for_domain(domain: str, pool_size: int) -> int:
 # 被反爬识别。跨进程通过 SHA-256 哈希复现，见 _ua_index_for_domain。
 _domain_ua_cache: dict = {}
 
+# 保护 _domain_ua_cache 的"检查-设置"原子性锁。
+# v2.2.0：并发场景下（如 search_multi 并行请求多实例），多个线程可能同时
+# 检查 domain not in cache 并同时写入，虽不致命但会浪费计算且可能写入不同
+# UA（因 hash 本应稳定，但极端时序下逻辑可读性问题）。用锁串行化 dict 读写。
+# 注意：锁内只做 dict 读写，绝不放网络/重计算，避免阻塞其他线程。
+_domain_ua_lock = threading.Lock()
+
 
 def get_ua_for_domain(url: str, user_agent: str = None) -> str:
     """返回适合某域名的 User-Agent。
@@ -188,6 +280,10 @@ def get_ua_for_domain(url: str, user_agent: str = None) -> str:
     设计理由：真实浏览器访问同一站点时 UA 永远不变。爬虫如果每次请求
     换一个 UA，反而会被反爬系统标记为可疑。确定性轮换保证同一域名
     稳定使用同一 UA，不同域名分散到不同 UA 上降低集体封禁风险。
+
+    线程安全：缓存的"检查-设置"用 ``_domain_ua_lock`` 保护。本函数无网络
+    调用，但遵循"锁内只做 dict 读写"原则——hash 计算放在锁外，写入时做
+    双检查（其他线程可能在此期间已写入），兼顾正确性与并发吞吐。
     """
     if user_agent:
         return user_agent
@@ -200,18 +296,27 @@ def get_ua_for_domain(url: str, user_agent: str = None) -> str:
     except Exception:
         return FALLBACK_UAS[0]
 
-    if domain in _domain_ua_cache:
-        return _domain_ua_cache[domain]
+    # 快速路径：锁内检查缓存命中
+    with _domain_ua_lock:
+        if domain in _domain_ua_cache:
+            return _domain_ua_cache[domain]
 
+    # 缓存未命中：在锁外计算 UA（SHA-256 hash，无副作用，不阻塞其他线程）
     idx = _ua_index_for_domain(domain, len(FALLBACK_UAS))
     ua = FALLBACK_UAS[idx]
-    _domain_ua_cache[domain] = ua
+
+    # 加锁写入；双检查避免覆盖其他线程并发写入的值
+    with _domain_ua_lock:
+        if domain in _domain_ua_cache:
+            return _domain_ua_cache[domain]
+        _domain_ua_cache[domain] = ua
     return ua
 
 
 def reset_domain_ua_cache() -> None:
     """清空 per-domain UA 缓存。测试用。"""
-    _domain_ua_cache.clear()
+    with _domain_ua_lock:
+        _domain_ua_cache.clear()
 
 
 def build_browser_headers(user_agent: str, referer: str = None,
@@ -239,11 +344,22 @@ def build_browser_headers(user_agent: str, referer: str = None,
     else:
         accept = "application/json, text/plain, */*;q=0.8"
 
+    # v2.2.1 智能声明 Accept-Encoding：仅当本机安装了 brotli 解压器时
+    # 才声明 br。否则服务器返回 br 压缩字节而 requests 无法解压 → 乱码。
+    # Firefox UA 路径保持只声明 gzip/deflate（与真 Firefox 行为一致，
+    # Firefox 虽支持 br 但为减少指纹差异在此工具中不声明）。
+    if is_firefox:
+        accept_encoding = "gzip, deflate"
+    elif _HAS_BROTLI:
+        accept_encoding = "gzip, deflate, br"
+    else:
+        accept_encoding = "gzip, deflate"
+
     headers = {
         "User-Agent": user_agent,
         "Accept": accept,
         "Accept-Language": "en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7",
-        "Accept-Encoding": "gzip, deflate" if is_firefox else "gzip, deflate, br",
+        "Accept-Encoding": accept_encoding,
         "Connection": "keep-alive",
         "Upgrade-Insecure-Requests": "1" if accept_html else "0",
     }
@@ -588,22 +704,17 @@ RECOVERY_HINTS = {
 }
 
 
-def classify_error(exc: BaseException) -> str:
-    """将异常分类为结构化错误码，供 AI Agent 程序化处理。
+def _classify_by_type_and_status(exc, _json):
+    """根据异常类型和 HTTP 状态码分类，返回错误码或 None。
 
-    分类逻辑（按优先级）：
-      1. 429 → E_RATE_LIMIT
-      2. 401/403 → E_AUTH
-      3. 4xx（非上述）→ E_INPUT（请求参数问题）
-      4. 5xx / URLError / OSError / TimeoutError → E_NETWORK
-      5. json.JSONDecodeError / ValueError → E_PARSE
-      6. FileNotFoundError → E_INPUT
-      7. RuntimeError → 尝试从消息中提取线索，否则 E_INTERNAL
-      8. 其他 → E_INTERNAL
+    检查顺序：HTTP 状态码 → 文件错误 → 连接错误 → 解析错误。
+    返回 None 表示该异常无法靠类型/状态码判定，需走字符串 fallback。
+    不处理 RuntimeError 消息推断（由 :func:`classify_error` 调用方做 fallback）。
+
+    抽取为独立函数，便于 :func:`classify_error` 对 ``exc`` 本身和其
+    ``__cause__`` 复用同一套基于真实类型的判定逻辑。
     """
-    import json as _json
-
-    # HTTP 错误（urllib 和 requests 都有 .code 或 .status_code）
+    # HTTP 错误（urllib HTTPError 用 .code，requests HTTPError 用 .response.status_code）
     status = None
     if isinstance(exc, urllib.error.HTTPError):
         status = exc.code
@@ -635,11 +746,48 @@ def classify_error(exc: BaseException) -> str:
     if isinstance(exc, (ValueError, _json.JSONDecodeError)):
         return E_PARSE
 
+    return None
+
+
+def classify_error(exc: BaseException) -> str:
+    """将异常分类为结构化错误码，供 AI Agent 程序化处理。
+
+    分类逻辑（按优先级）：
+      1. 优先检查异常链 ``__cause__``：``raise X from Y`` 场景下 Y 才是真实
+         错误源（如 ``raise RuntimeError(...) from HTTPError(403)``），
+         用 Y 的类型/状态码分类比从 X 的消息字符串推断更可靠、不再脆弱。
+      2. HTTP 状态码：429→E_RATE_LIMIT, 401/403→E_AUTH, 4xx→E_INPUT, 5xx→E_NETWORK
+      3. 连接类异常（ConnectionError/TimeoutError/URLError/OSError）→ E_NETWORK
+      4. 解析错误（ValueError/JSONDecodeError）→ E_PARSE
+      5. FileNotFoundError → E_INPUT
+      6. RuntimeError：从消息中推断（仅当 ``__cause__`` 缺失时的最后 fallback，
+         兼容旧路径——search_multi 把 last_error 拼进消息）
+      7. 其他 → E_INTERNAL
+    """
+    import json as _json
+
+    # 优先检查异常链 __cause__：raise X from Y 时，Y 指向真实底层异常。
+    # 旧逻辑只能从外层 RuntimeError 的消息字符串推断（脆弱，依赖 "403"/"auth"
+    # 等关键字匹配），新逻辑直接从 __cause__ 的 .code/.status_code 或异常
+    # 类型判定。仅检查一层 __cause__，不递归——单层已覆盖 search_multi 的
+    # raise-from 模式，深层链罕见且递归有循环风险。
+    cause = getattr(exc, "__cause__", None)
+    if cause is not None and cause is not exc:
+        code = _classify_by_type_and_status(cause, _json)
+        if code is not None:
+            return code
+
+    # 检查 exc 本身的类型和状态码
+    code = _classify_by_type_and_status(exc, _json)
+    if code is not None:
+        return code
+
     # RuntimeError：从消息中推断（search_multi 的 "All N instances failed.
     # Last error: HTTP Error 403: Forbidden" 等）。并行路径现在会把 last_error
     # 拼进消息，让这里能提取真实错误类型，而不是一律误判 E_NETWORK。
-    msg = str(exc).lower()
+    # 字符串匹配仅作为 __cause__ 缺失时的最后 fallback。
     if isinstance(exc, RuntimeError):
+        msg = str(exc).lower()
         # 先检查 auth/rate-limit 关键字（最常见，来自 last_error 详情）
         if "auth" in msg or "403" in msg or "401" in msg:
             return E_AUTH
@@ -703,11 +851,15 @@ def emit_progress(event: str, **kwargs) -> None:
 
     仅当 --progress 启用时才输出。``event`` 是事件类型字符串，
     ``kwargs`` 是事件的附加字段。输出格式为 JSON Lines（每行一个 JSON 对象）。
+    v2.2.0：自动注入 request_id（如果已设置）。
     """
     if not _progress_enabled:
         return
     import json as _json
     payload = {"event": event}
+    rid = getattr(_LOG, "_request_id", None)
+    if rid:
+        payload["request_id"] = rid
     payload.update(kwargs)
     print(_json.dumps(payload, ensure_ascii=False), file=sys.stderr, flush=True)
 
@@ -813,3 +965,103 @@ def is_hard_blocked_domain(url: str) -> bool:
                 return True
     return False
 
+
+# ----- 相似度去重（v2.x：让 AI Agent 在研究模式下获得更干净的 merged_results）-----
+# 同一内容在不同 URL/引擎下常重复出现，仅靠 URL 去重无法合并。
+# SimHash 对标题做局部敏感哈希，汉明距离小的视为近似重复。
+
+def _normalize_title(title: str) -> str:
+    """归一化标题：小写、去标点、去多余空格，用于相似度比较。"""
+    if not title:
+        return ""
+    s = title.lower()
+    # 保留字母、数字、CJK 和空格，其余替换为空格
+    s = re.sub(r'[^\w\s]', ' ', s)
+    # \w 包含下划线，单独去掉
+    s = s.replace('_', ' ')
+    s = re.sub(r'\s+', ' ', s).strip()
+    return s
+
+
+def _simhash(text: str, hash_bits: int = 64) -> int:
+    """计算文本的 SimHash 指纹。
+    - 分词（按空格 + CJK 单字符）
+    - 每个 token 算普通 hash，按 bit 投票
+    - 返回 hash_bits 位的指纹
+    """
+    if not text:
+        return 0
+    # 分词：按空格切分，CJK 字符再逐个拆成单字 token
+    tokens = []
+    for word in text.split():
+        buf = []
+        for ch in word:
+            if '\u4e00' <= ch <= '\u9fff':
+                # 遇到 CJK：先冲出缓冲区里的非 CJK 片段，再加入单字
+                if buf:
+                    tokens.append(''.join(buf))
+                    buf = []
+                tokens.append(ch)
+            else:
+                buf.append(ch)
+        if buf:
+            tokens.append(''.join(buf))
+    if not tokens:
+        return 0
+    # 每个 token 算 SHA-256（跨进程可复现，避免 hash() 随机化），按 bit 投票
+    v = [0] * hash_bits
+    for token in tokens:
+        h = hashlib.sha256(token.encode('utf-8')).digest()
+        token_hash = int.from_bytes(h[:8], 'big')
+        for i in range(hash_bits):
+            if (token_hash >> i) & 1:
+                v[i] += 1
+            else:
+                v[i] -= 1
+    # 投票为正的位置 1
+    fingerprint = 0
+    for i in range(hash_bits):
+        if v[i] > 0:
+            fingerprint |= (1 << i)
+    return fingerprint
+
+
+def _hamming_distance(a: int, b: int) -> int:
+    """两个整数的汉明距离。"""
+    return bin(a ^ b).count('1')
+
+
+def _jaccard_similarity(set_a: set, set_b: set) -> float:
+    """Jaccard 相似度。"""
+    if not set_a and not set_b:
+        return 0.0
+    union = set_a | set_b
+    if not union:
+        return 0.0
+    return len(set_a & set_b) / len(union)
+
+
+def is_similar(result_a: dict, result_b: dict, threshold: float = 0.85) -> bool:
+    """判断两个搜索结果是否相似。
+    - 优先用标题 SimHash（汉明距离 <= 3 视为相似，对应 64 位中约 95% 相似）
+    - 标题太短（< 5 字符）时用 URL 域名 + 标题 Jaccard
+    - threshold 参数控制严格程度
+    """
+    title_a = _normalize_title(result_a.get("title", ""))
+    title_b = _normalize_title(result_b.get("title", ""))
+    # 标题太短时 SimHash 不稳定，改用 Jaccard
+    if len(title_a) < 5 or len(title_b) < 5:
+        import urllib.parse as _up
+        domain_a = _up.urlparse(result_a.get("url", "")).netloc.lower()
+        domain_b = _up.urlparse(result_b.get("url", "")).netloc.lower()
+        set_a = set(title_a.split()) | {domain_a}
+        set_b = set(title_b.split()) | {domain_b}
+        return _jaccard_similarity(set_a, set_b) >= threshold
+    # 标题足够长：用 SimHash 汉明距离
+    hash_a = _simhash(title_a)
+    hash_b = _simhash(title_b)
+    # threshold → 汉明距离阈值映射：
+    #   0.85 → 3（默认，宽松），0.90 → 2，0.95 → 1，1.0 → 0（几乎完全相同）
+    max_distance = max(0, min(3, int(round((1.0 - threshold) / 0.05))))
+    return _hamming_distance(hash_a, hash_b) <= max_distance
+
diff --git a/scripts/fetch.py b/scripts/fetch.py
index d2a2b67..88f68c5 100644
--- a/scripts/fetch.py
+++ b/scripts/fetch.py
@@ -23,6 +23,21 @@ from html.parser import HTMLParser
 from pathlib import Path
 from typing import Optional
 
+# Brotli 解压支持检测（v2.2.1）。
+# requests 自动解压 gzip/deflate，但**不自动解压 Brotli**（除非安装
+# brotli/brotlicffi）。build_browser_headers() 已根据此检测智能声明
+# Accept-Encoding，此处作为双保险：若代理/CDN 强制返回 br，仍可解压。
+try:
+    import brotli as _brotli  # type: ignore
+    _HAS_BROTLI = True
+except ImportError:
+    try:
+        import brotlicffi as _brotli  # type: ignore
+        _HAS_BROTLI = True
+    except ImportError:
+        _brotli = None
+        _HAS_BROTLI = False
+
 # Allow running standalone from any working directory
 sys.path.insert(0, str(Path(__file__).resolve().parent))
 from _config import USER_AGENT, VERSION
@@ -50,6 +65,11 @@ from common import (
 logger = logging.getLogger("searxng.fetch")
 
 
+# 错误码：不支持的媒体类型（PDF/DOCX/XLSX 解析失败或未知二进制类型）。
+# 与 common.py 中 E_CONFIG / E_AUTH / E_NETWORK 等错误码保持一致的 E_* 命名模式。
+E_UNSUPPORTED_MEDIA = "E_UNSUPPORTED_MEDIA"
+
+
 # ----- Auth helpers -----
 # build_auth_headers is imported from common.py
 
@@ -167,6 +187,37 @@ def extract_with_bs4(html_content: str) -> str:
     return text
 
 
+# CJK 字符范围：中文 \u4e00-\u9fff、日文 \u3040-\u30ff、韩文 \uac00-\ud7af
+_CJK_CHAR_RE = re.compile(
+    r"[\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]"
+)
+
+
+def _is_cjk_text(text: str) -> bool:
+    """判断文本是否以 CJK（中文/日文/韩文）为主。
+
+    统计 CJK 字符占非空白字符的比例，>30% 则视为 CJK 内容。
+    CJK 文本信息密度高，readability-lite 的最小字符阈值应相应降低。
+    """
+    if not text:
+        return False
+    # 按非空白字符统计，避免大量空白/缩进拉低比例造成误判
+    non_ws_len = sum(1 for ch in text if not ch.isspace())
+    if non_ws_len == 0:
+        return False
+    cjk_count = len(_CJK_CHAR_RE.findall(text))
+    return cjk_count / non_ws_len > 0.30
+
+
+def _min_content_length(text: str) -> int:
+    """根据文本语言返回 readability-lite 最小正文字符阈值。
+
+    CJK 内容（信息密度高）：100 字符
+    其他语言（英文等）：200 字符
+    """
+    return 100 if _is_cjk_text(text) else 200
+
+
 def _readability_lite(root) -> "Optional[object]":
     """readability-lite：用文本密度算法选择最可能是正文的子元素。
 
@@ -174,7 +225,7 @@ def _readability_lite(root) -> "Optional[object]":
       1. 遍历 body 下所有 div/section/article 子节点
       2. 计算每个节点的"文本密度" = 纯文本字符数 / 标签数
       3. 排除明显是导航/侧边栏的节点（class/id 含 nav/sidebar/menu/footer）
-      4. 返回文本密度最高且字符数 > 200 的节点
+      4. 返回文本密度最高且字符数超过阈值的节点（CJK 100，其他 200）
 
     返回 bs4 Tag 或 None（找不到合适节点时）。
     这是 extract_with_bs4 的兜底增强，不改变原有 article/main 优先级。
@@ -204,8 +255,10 @@ def _readability_lite(root) -> "Optional[object]":
         # 计算纯文本字符数（去空白）
         text = node.get_text(separator=" ", strip=True)
         text_len = len(text)
-        if text_len < 200:
-            continue  # 正文至少 200 字符
+        # 阈值按语言动态调整：CJK 内容 100 字符，其他 200 字符
+        min_len = _min_content_length(text)
+        if text_len < min_len:
+            continue  # 正文至少 min_len 字符（CJK 100，其他 200）
 
         # 计算标签数（粗略：所有后代标签）
         tag_count = len(node.find_all())
@@ -602,10 +655,225 @@ class _NoRedirectHandler(urllib.request.HTTPRedirectHandler):
 
 FetchResult = namedtuple(
     "FetchResult",
-    ["content", "content_type", "final_url", "truncated", "user_agent"],
+    ["content", "content_type", "final_url", "truncated", "user_agent",
+     "error_code", "error_message"],
+    # error_code / error_message 默认 None：
+    #   * 向后兼容——旧的 5 参数构造（位置或关键字）仍然可用
+    #   * 仅当 fetch_url 遇到不支持的媒体类型时才填充
+    defaults=[None, None],
 )
 
 
+def _handle_rate_limit_status(status_code, headers, attempt, max_retries):
+    """处理 429/503 限流响应的 Retry-After，决定是否重试。
+
+    统一抽取自 requests 路径、stdlib 路径、requests.exceptions 路径三处
+    原本重复的 Retry-After 解析 + 退避 sleep 逻辑。
+
+    解析 Retry-After 头（秒数或 HTTP 日期，委托给 common.parse_retry_after），
+    与 compute_backoff_delay(attempt) 取较大值作为实际等待时间。
+    若 ``attempt < max_retries``，sleep 后返回 ``(True, retry_after_sec)``
+    表示应当重试；否则返回 ``(False, retry_after_sec)``，由调用方决定后续
+    （通常会落到 raise_for_status / 抛 RuntimeError）。
+
+    ``headers`` 兼容 dict 和 http.client.HTTPMessage（均支持 ``.get()``）；
+    为 None 时按空 header 处理（retry_after=0）。
+
+    返回 ``(should_retry, retry_after_seconds)``。
+    """
+    retry_after_raw = ""
+    if headers:
+        retry_after_raw = headers.get("Retry-After", "") or ""
+    retry_after_sec = parse_retry_after(retry_after_raw)
+    if attempt < max_retries:
+        delay = max(retry_after_sec, compute_backoff_delay(attempt))
+        logger.info(f"  Fetch retry {attempt+1}/{max_retries} "
+                    f"(HTTP {status_code}, Retry-After={retry_after_sec:.1f}s) "
+                    f"in {delay:.1f}s")
+        time.sleep(delay)
+        return (True, retry_after_sec)
+    return (False, retry_after_sec)
+
+
+# 明确的非文本二进制 MIME 类型：无法作为文本 decode，直接拒绝。
+# application/octet-stream 是通用二进制兜底类型；其余为已知归档/可执行/
+# 旧版 Office（.doc/.xls/.ppt 不在本次支持范围）等。
+_BINARY_CONTENT_TYPES = frozenset([
+    "application/octet-stream",
+    "application/zip",
+    "application/x-gzip",
+    "application/gzip",
+    "application/x-rar-compressed",
+    "application/x-7z-compressed",
+    "application/x-tar",
+    "application/x-bzip",
+    "application/x-bzip2",
+    "application/x-msdownload",
+    "application/x-shockwave-flash",
+    "application/msword",                       # 旧 .doc（不支持）
+    "application/vnd.ms-excel",                 # 旧 .xls（不支持）
+    "application/vnd.ms-powerpoint",            # 旧 .ppt（不支持）
+    "application/x-elf",
+    "application/x-executable",
+])
+
+
+# OOXML（.docx / .xlsx）主命名空间，ElementTree 用 {ns}tag 形式匹配
+_W_NS = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}"
+_S_NS = "{http://schemas.openxmlformats.org/spreadsheetml/2006/main}"
+
+
+def _parse_pdf(raw: bytes):
+    """用 pdftotext（poppler-utils）从 PDF 字节流提取文本。
+
+    通过 subprocess 调用 ``pdftotext - -``（stdin 读、stdout 写），
+    不引入新依赖。pdftotext 不存在或失败时返回 E_UNSUPPORTED_MEDIA。
+
+    返回 ``(content, error_code, error_message)``：
+      成功 → ``(text, None, None)``；失败 → ``(None, E_UNSUPPORTED_MEDIA, msg)``。
+    """
+    try:
+        proc = subprocess.run(
+            ["pdftotext", "-", "-"],
+            input=raw,
+            capture_output=True,
+            timeout=30,
+        )
+    except FileNotFoundError:
+        return (None, E_UNSUPPORTED_MEDIA,
+                "PDF parsing requires poppler-utils (pdftotext) to be installed")
+    except subprocess.TimeoutExpired:
+        return (None, E_UNSUPPORTED_MEDIA, "PDF parsing timed out (>30s)")
+    except OSError as e:
+        return (None, E_UNSUPPORTED_MEDIA, f"PDF parsing failed: {e}")
+
+    if proc.returncode != 0:
+        stderr = proc.stderr.decode("utf-8", errors="replace").strip()
+        msg = f"pdftotext exited {proc.returncode}"
+        if stderr:
+            msg += f": {stderr[:200]}"
+        return (None, E_UNSUPPORTED_MEDIA, msg)
+
+    text = proc.stdout.decode("utf-8", errors="replace")
+    return (text, None, None)
+
+
+def _parse_docx(raw: bytes):
+    """从 .docx 字节流提取文本（stdlib zipfile + ElementTree）。
+
+    读取 ``word/document.xml``，按段落（<w:p>）提取 <w:t> 文本，
+    段落间以换行分隔。失败时返回 E_UNSUPPORTED_MEDIA。
+
+    返回 ``(content, error_code, error_message)``。
+    """
+    try:
+        with zipfile.ZipFile(io.BytesIO(raw)) as zf:
+            xml_bytes = zf.read("word/document.xml")
+    except (zipfile.BadZipFile, KeyError) as e:
+        return (None, E_UNSUPPORTED_MEDIA, f"DOCX parsing failed: {e}")
+
+    try:
+        root = ET.fromstring(xml_bytes)
+    except ET.ParseError as e:
+        return (None, E_UNSUPPORTED_MEDIA, f"DOCX XML parse failed: {e}")
+
+    # 遍历段落，每段内拼接所有 <w:t>，段落间换行
+    lines = []
+    for p in root.iter(_W_NS + "p"):
+        parts = [t.text for t in p.iter(_W_NS + "t") if t.text]
+        if parts:
+            lines.append("".join(parts))
+    return ("\n".join(lines), None, None)
+
+
+def _parse_xlsx(raw: bytes):
+    """从 .xlsx 字节流提取文本（stdlib zipfile + ElementTree）。
+
+    读取 ``xl/sharedStrings.xml``（共享字符串表）与各
+    ``xl/worksheets/sheetN.xml``，按行提取单元格文本，单元格以制表符
+    分隔、行以换行分隔。失败时返回 E_UNSUPPORTED_MEDIA。
+
+    返回 ``(content, error_code, error_message)``。
+    """
+    try:
+        with zipfile.ZipFile(io.BytesIO(raw)) as zf:
+            # 共享字符串表（可能不存在——纯数字表格）
+            shared = []
+            try:
+                sroot = ET.fromstring(zf.read("xl/sharedStrings.xml"))
+                for si in sroot.iter(_S_NS + "si"):
+                    parts = [t.text for t in si.iter(_S_NS + "t") if t.text]
+                    shared.append("".join(parts))
+            except (KeyError, ET.ParseError):
+                pass  # 无共享字符串表，单元格均为内联值
+
+            sheet_names = [n for n in zf.namelist()
+                           if re.match(r"xl/worksheets/sheet\d+\.xml$", n)]
+            lines = []
+            for sheet_name in sorted(sheet_names):
+                try:
+                    sroot = ET.fromstring(zf.read(sheet_name))
+                except ET.ParseError:
+                    continue
+                for row in sroot.iter(_S_NS + "row"):
+                    cells = []
+                    for c in row.iter(_S_NS + "c"):
+                        cell_type = c.get("t")
+                        v = c.find(_S_NS + "v")
+                        if v is not None and v.text is not None:
+                            if cell_type == "s":
+                                # 共享字符串索引引用
+                                try:
+                                    idx = int(v.text)
+                                    cells.append(
+                                        shared[idx] if 0 <= idx < len(shared) else "")
+                                except (ValueError, IndexError):
+                                    cells.append("")
+                            else:
+                                cells.append(v.text)
+                        else:
+                            # 内联字符串 <is><t>...</t></is>
+                            is_el = c.find(_S_NS + "is")
+                            if is_el is not None:
+                                parts = [t.text for t in is_el.iter(_S_NS + "t")
+                                         if t.text]
+                                cells.append("".join(parts))
+                    if cells:
+                        lines.append("\t".join(cells))
+            return ("\n".join(lines), None, None)
+    except (zipfile.BadZipFile, ET.ParseError) as e:
+        return (None, E_UNSUPPORTED_MEDIA, f"XLSX parsing failed: {e}")
+
+
+def _parse_document_content(raw: bytes, content_type: str):
+    """根据 Content-Type 将二进制文档解析为文本。
+
+    支持：PDF（需 pdftotext）、DOCX、XLSX。
+    对明确的非文本二进制 MIME（application/octet-stream、zip、rar 等）返回
+    E_UNSUPPORTED_MEDIA。其他类型（text/* 、application/json、HTML 等）返回
+    ``(None, None, None)``，由调用方走原有 decode 流程。
+
+    返回 ``(content, error_code, error_message)``：
+      * 非文档类型 → ``(None, None, None)``：调用方继续 decode
+      * 解析成功   → ``(text, None, None)``
+      * 解析失败   → ``(None, E_UNSUPPORTED_MEDIA, msg)``
+    """
+    ct = (content_type or "").lower().split(";")[0].strip()
+
+    if ct == "application/pdf":
+        return _parse_pdf(raw)
+    if ct == "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
+        return _parse_docx(raw)
+    if ct == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet":
+        return _parse_xlsx(raw)
+
+    if ct in _BINARY_CONTENT_TYPES:
+        return (None, E_UNSUPPORTED_MEDIA,
+                f"Unsupported binary content type: {ct}")
+
+    return (None, None, None)
+
+
 def fetch_url(url: str, timeout=15, user_agent: str = None,
               encoding: str = None, auth_headers: dict = None,
               max_retries: int = 3, max_size: int = None,
@@ -669,16 +937,11 @@ def fetch_url(url: str, timeout=15, user_agent: str = None,
                 try:
                     # 429/503：读取 Retry-After，作为最小重试延迟
                     if resp.status_code in (429, 503) and attempt < max_retries:
-                        retry_after_raw = resp.headers.get("Retry-After", "")
-                        retry_after_sec = parse_retry_after(retry_after_raw)
                         resp.close()
-                        delay = max(retry_after_sec,
-                                    compute_backoff_delay(attempt))
-                        logger.info(f"  Fetch retry {attempt+1}/{max_retries} "
-                                    f"(HTTP {resp.status_code}, Retry-After={retry_after_sec:.1f}s) "
-                                    f"in {delay:.1f}s")
-                        time.sleep(delay)
-                        continue
+                        should_retry, _ = _handle_rate_limit_status(
+                            resp.status_code, resp.headers, attempt, max_retries)
+                        if should_retry:
+                            continue
 
                     resp.raise_for_status()
 
@@ -698,6 +961,33 @@ def fetch_url(url: str, timeout=15, user_agent: str = None,
                         raw = b"".join(chunks)
                         truncated = total > max_size
 
+                    # v2.2.1 修复：requests 自动解压 gzip/deflate，但**不自动
+                    # 解压 Brotli**（除非安装 brotli 包）。当服务器返回
+                    # Content-Encoding: br 而本机有 brotli 解压器时，手动解压；
+                    # 否则保留原 raw，让下游 errors="replace" 兜底（虽是乱码但
+                    # 不崩溃）。build_browser_headers() 已尽量避免声明 br，此处
+                    # 作为双保险，应对代理/CDN 强制返回 br 的边缘情况。
+                    content_encoding = (resp.headers.get("Content-Encoding", "")
+                                        .lower().strip())
+                    if "br" in content_encoding and _HAS_BROTLI and raw:
+                        try:
+                            raw = _brotli.decompress(raw)
+                        except Exception as e:
+                            logger.debug(f"  brotli decompress failed: {e}")
+
+                    # 二进制文档解析（PDF/DOCX/XLSX）及不支持的媒体类型检测。
+                    # 命中时直接返回，跳过后续文本 decode 流程。
+                    doc_text, doc_err_code, doc_err_msg = _parse_document_content(
+                        raw, resp.headers.get("Content-Type", ""))
+                    if doc_err_code is not None:
+                        return FetchResult(
+                            "", resp.headers.get("Content-Type", ""), resp.url,
+                            truncated, ua, doc_err_code, doc_err_msg)
+                    if doc_text is not None:
+                        return FetchResult(
+                            doc_text, resp.headers.get("Content-Type", ""), resp.url,
+                            truncated, ua, None, None)
+
                     if encoding:
                         content = raw.decode(encoding)
                     else:
@@ -745,11 +1035,14 @@ def fetch_url(url: str, timeout=15, user_agent: str = None,
                 # requests 库会自动处理 Content-Encoding，但 stdlib 不会。
                 # 此前该 bug 被沙箱伪响应掩盖（两者都产生 U+FFFD），
                 # 实际在无 requests 的真实环境中会复现。
+                # v2.2.1 补充：br 解压（与 requests 路径对齐）。
                 content_encoding = (resp.headers.get("Content-Encoding", "")
                                     .lower().strip())
                 if content_encoding and raw:
                     try:
-                        if "gzip" in content_encoding:
+                        if "br" in content_encoding and _HAS_BROTLI:
+                            raw = _brotli.decompress(raw)
+                        elif "gzip" in content_encoding:
                             raw = gzip.decompress(raw)
                         elif "deflate" in content_encoding:
                             # deflate 可能是 zlib 包装或裸 deflate
@@ -761,6 +1054,19 @@ def fetch_url(url: str, timeout=15, user_agent: str = None,
                         logger.debug(f"  decompress failed ({content_encoding}): {e}")
                         # 解压失败保留原 raw，让下游 decode 兜底
 
+                # 二进制文档解析（PDF/DOCX/XLSX）及不支持的媒体类型检测。
+                # 命中时直接返回，跳过后续文本 decode 流程。
+                doc_text, doc_err_code, doc_err_msg = _parse_document_content(
+                    raw, content_type)
+                if doc_err_code is not None:
+                    return FetchResult(
+                        "", content_type, final_url, truncated, ua,
+                        doc_err_code, doc_err_msg)
+                if doc_text is not None:
+                    return FetchResult(
+                        doc_text, content_type, final_url, truncated, ua,
+                        None, None)
+
                 if encoding:
                     charset = encoding
                 else:
@@ -776,16 +1082,12 @@ def fetch_url(url: str, timeout=15, user_agent: str = None,
             last_error = e
             # 429/503：读取 Retry-After（stdlib 路径）
             if e.code in (429, 503) and attempt < max_retries:
-                retry_after_raw = e.headers.get("Retry-After", "") if e.headers else ""
-                retry_after_sec = parse_retry_after(retry_after_raw)
                 # HTTPError 本身是可读的响应对象（fp 已被 urllib 消费），
                 # 无需显式 close；直接进入退避。
-                delay = max(retry_after_sec, compute_backoff_delay(attempt))
-                logger.info(f"  Fetch retry {attempt+1}/{max_retries} "
-                            f"(HTTP {e.code}, Retry-After={retry_after_sec:.1f}s) "
-                            f"in {delay:.1f}s")
-                time.sleep(delay)
-                continue
+                should_retry, _ = _handle_rate_limit_status(
+                    e.code, e.headers, attempt, max_retries)
+                if should_retry:
+                    continue
             if is_retryable_error(e) and attempt < max_retries:
                 delay = compute_backoff_delay(attempt)
                 logger.info(f"  Fetch retry {attempt+1}/{max_retries} ({e}) in {delay:.1f}s")
@@ -809,18 +1111,13 @@ def fetch_url(url: str, timeout=15, user_agent: str = None,
                 # 429/503 with Retry-After
                 if status in (429, 503) and attempt < max_retries:
                     resp_obj = getattr(e, "response", None)
-                    retry_after_raw = ""
-                    if resp_obj is not None:
-                        retry_after_raw = resp_obj.headers.get("Retry-After", "")
-                    retry_after_sec = parse_retry_after(retry_after_raw)
+                    resp_headers = resp_obj.headers if resp_obj is not None else None
                     if resp_obj is not None:
                         resp_obj.close()
-                    delay = max(retry_after_sec, compute_backoff_delay(attempt))
-                    logger.info(f"  Fetch retry {attempt+1}/{max_retries} "
-                                f"(HTTP {status}, Retry-After={retry_after_sec:.1f}s) "
-                                f"in {delay:.1f}s")
-                    time.sleep(delay)
-                    continue
+                    should_retry, _ = _handle_rate_limit_status(
+                        status, resp_headers, attempt, max_retries)
+                    if should_retry:
+                        continue
                 if (status is None or status in RETRYABLE_STATUS) and attempt < max_retries:
                     delay = compute_backoff_delay(attempt)
                     logger.info(f"  Fetch retry {attempt+1}/{max_retries} ({e}) in {delay:.1f}s")
diff --git a/scripts/search.py b/scripts/search.py
index c50f558..464ac84 100644
--- a/scripts/search.py
+++ b/scripts/search.py
@@ -40,6 +40,7 @@ from common import (
     emit_progress,
     force_utf8_stdout,
     is_hard_blocked_domain,
+    is_similar,
     parse_retry_after,
     resolve_auth_basic,
     resolve_auth_bearer,
@@ -1146,7 +1147,13 @@ class AdaptiveThrottle:
     线程安全：所有方法加锁。状态由 fetch_top_results 的 _fetch_one 回调驱动。
     """
 
-    def __init__(self, initial_delay: float, initial_concurrency: int):
+    def __init__(self, initial_delay: float, initial_concurrency: int,
+                 failure_threshold: int = 3, pause_seconds: float = 30.0,
+                 max_delay: float = 10.0):
+        """v2.2.0：参数化 failure_threshold/pause_seconds/max_delay。
+
+        默认值与 v2.1.1 硬编码一致，保持向后兼容。
+        """
         self._lock = threading.Lock()
         self._delay = initial_delay
         self._initial_delay = initial_delay
@@ -1155,6 +1162,9 @@ class AdaptiveThrottle:
         self._consecutive_failures = 0
         self._consecutive_successes = 0
         self._global_pause_until = 0.0  # time.monotonic() 时间戳
+        self._failure_threshold = failure_threshold
+        self._pause_seconds = pause_seconds
+        self._max_delay = max_delay
 
     @property
     def delay(self) -> float:
@@ -1195,11 +1205,11 @@ class AdaptiveThrottle:
                              "429" in error_msg.lower() or
                              "rate limit" in error_msg.lower())
             if is_rate_limit:
-                self._global_pause_until = time.monotonic() + 30.0
-            # 连续 3 次失败 → 退避 + 降并发
-            if self._consecutive_failures >= 3:
+                self._global_pause_until = time.monotonic() + self._pause_seconds
+            # 连续失败达阈值 → 退避 + 降并发
+            if self._consecutive_failures >= self._failure_threshold:
                 self._consecutive_failures = 0
-                self._delay = min(self._delay * 2, 10.0)  # 上限 10s
+                self._delay = min(self._delay * 2, self._max_delay)
                 self._concurrency = max(1, self._concurrency // 2)
 
     def wait_if_paused(self) -> None:
@@ -1327,12 +1337,26 @@ def fetch_top_results(results: dict, count: int, timeout: int = 10,
     return fetched
 
 
-def _emit_fetch_report(fetched: list, throttle: "AdaptiveThrottle") -> None:
-    """v2.0.0: 输出结构化抓取报告到 stderr。
+def _emit_fetch_report(fetched: list, throttle: "AdaptiveThrottle",
+                       fmt: str = "text") -> None:
+    """输出结构化抓取报告到 stderr。
 
     让 AI Agent 可程序化分析抓取过程：哪些 URL 被反爬拦截、用了什么兜底、
-    自适应限流如何调整。格式为人类可读的表格 + JSON 摘要。
+    自适应限流如何调整。
+
+    ``fmt``:
+      * ``"text"``（默认）：人类可读表格 + JSON 摘要行（v2.0.0 行为，向后兼容）
+      * ``"json"``：完整 JSON 对象（items 数组 + summary），便于 Agent 解析
+      * 其他值按 ``"text"`` 处理
     """
+    if fmt == "json":
+        _emit_fetch_report_json(fetched, throttle)
+    else:
+        _emit_fetch_report_text(fetched, throttle)
+
+
+def _emit_fetch_report_text(fetched: list, throttle: "AdaptiveThrottle") -> None:
+    """文本表格 + JSON 摘要行（v2.0.0 原始行为，向后兼容）。"""
     import sys as _sys
     out = _sys.stderr
     lines = []
@@ -1382,6 +1406,59 @@ def _emit_fetch_report(fetched: list, throttle: "AdaptiveThrottle") -> None:
     print("\n".join(lines), file=out)
 
 
+def _emit_fetch_report_json(fetched: list, throttle: "AdaptiveThrottle") -> None:
+    """完整 JSON 报告（items 数组 + summary），输出到 stderr。
+
+    每个 URL 一个对象，包含 url / status / title / content_length /
+    error / error_code / latency / fetched_at 等字段。fetch_page 未采集
+    的字段（title / latency / fetched_at）为 None，便于 Agent 统一解析。
+    """
+    import json as _json
+    import sys as _sys
+    from datetime import datetime, timezone
+
+    fetched_at = datetime.now(timezone.utc).isoformat()
+
+    # 每个 URL 一个对象
+    items = []
+    for f in fetched:
+        items.append({
+            "url": f.get("url", ""),
+            "final_url": f.get("final_url"),
+            "status": f.get("status", ""),
+            "title": f.get("title"),            # fetch_page 未提取，None
+            "content_length": f.get("text_length", 0),
+            "content_type": f.get("content_type"),
+            "error": f.get("error"),
+            "error_code": f.get("error_code"),
+            "latency": f.get("latency"),         # fetch_page 未计时，None
+            "truncated": f.get("truncated", False),
+            "fetched_at": f.get("fetched_at") or fetched_at,
+            "waf_type": f.get("waf_type"),
+            "fallback_used": f.get("fallback_used"),
+            "anti_bot_detected": f.get("anti_bot_detected", False),
+        })
+
+    # 统计摘要
+    total = len(fetched)
+    ok = sum(1 for f in fetched if f.get("status") == "ok")
+    err = total - ok
+    anti_bot = sum(1 for f in fetched if f.get("anti_bot_detected"))
+    wayback = sum(1 for f in fetched if f.get("fallback_used") == "wayback")
+
+    report = {
+        "format": "json",
+        "total": total,
+        "ok": ok,
+        "error": err,
+        "anti_bot_blocked": anti_bot,
+        "wayback_recovered": wayback,
+        "throttle": throttle.stats(),
+        "items": items,
+    }
+    print(_json.dumps(report, ensure_ascii=False), file=_sys.stderr)
+
+
 # ----- Output formatting -----
 
 def deduplicate_results(results: dict) -> dict:
@@ -1440,6 +1517,53 @@ def deduplicate_results(results: dict) -> dict:
     return results
 
 
+def deduplicate_similar(results: dict, threshold: float = 0.85) -> int:
+    """对搜索结果进行相似度去重（基于标题 SimHash）。
+
+    遍历结果列表，对每对结果用 :func:`is_similar` 判断，保留先出现的
+    （通常 score 更高），删除相似的后续结果。
+
+    Args:
+        results: 含 ``"results"`` 列表的 dict，会原地修改。
+        threshold: 相似度阈值，越高越严格（默认 0.85）。
+
+    Returns:
+        删除的条目数。
+
+    Note:
+        O(n²) 复杂度，结果列表通常 < 100，可接受。若 > 500 则跳过并
+        log warning，避免在大结果集上卡顿。
+    """
+    items = results.get("results", [])
+    if not items:
+        return 0
+    n = len(items)
+    if n > 500:
+        logger.warning(
+            f"Similarity dedup skipped: {n} results > 500 "
+            f"(O(n²) would be too slow)"
+        )
+        return 0
+    to_remove = set()
+    for i in range(n):
+        if i in to_remove:
+            continue
+        for j in range(i + 1, n):
+            if j in to_remove:
+                continue
+            if is_similar(items[i], items[j], threshold=threshold):
+                to_remove.add(j)
+    if not to_remove:
+        return 0
+    removed = len(to_remove)
+    results["results"] = [items[i] for i in range(n) if i not in to_remove]
+    logger.info(
+        f"Similarity dedup: removed {removed} similar results "
+        f"(threshold={threshold}, {n} -> {n - removed})"
+    )
+    return removed
+
+
 def sort_results(results: dict, sort_by: str) -> dict:
     """Sort results by the given key.
 
@@ -1665,31 +1789,92 @@ def _run_single_query(query: str, args, instance_urls: list,
     """
     params = _build_params(query, args)
 
+    pages_to_fetch = getattr(args, "pages", 1) or 1
+    if pages_to_fetch < 1:
+        pages_to_fetch = 1
+
     emit_progress("start", query=query, instances=len(instance_urls))
 
-    cached = cache_module.get(params, ttl_seconds) if ttl_seconds > 0 else None
-    if cached is not None:
-        logger.info(f"[cache hit] q={query!r} TTL={args.cache_ttl}min, skipping network")
-        emit_progress("cache_hit", query=query, ttl=args.cache_ttl)
-        results = cached
+    # v2.2.0：--pages N 多页聚合。循环 pageno=1..N，每页独立缓存
+    # （cache key 含 pageno），合并后统一 dedup/sort/max-results。
+    if pages_to_fetch == 1:
+        # 单页路径（保持原行为，最常见场景无额外开销）
+        cached = cache_module.get(params, ttl_seconds) if ttl_seconds > 0 else None
+        if cached is not None:
+            logger.info(f"[cache hit] q={query!r} TTL={args.cache_ttl}min, skipping network")
+            emit_progress("cache_hit", query=query, ttl=args.cache_ttl)
+            results = cached
+        else:
+            try:
+                results = search_multi(
+                    instance_urls, params,
+                    method=args.method,
+                    timeout=args.timeout,
+                    retry_per=args.retry,
+                    auth_headers=auth_headers,
+                    parallel=not args.serial,
+                )
+            except Exception as e:
+                err_code = classify_error(e)
+                emit_progress("error", error=str(e), error_code=err_code, query=query)
+                return None, str(e), err_code
+            if ttl_seconds > 0:
+                cache_module.put(params, results, ttl_seconds)
+                logger.info(f"[cache stored] q={query!r} TTL={args.cache_ttl}min")
+                emit_progress("cache_store", query=query, ttl=args.cache_ttl)
     else:
-        try:
-            results = search_multi(
-                instance_urls, params,
-                method=args.method,
-                timeout=args.timeout,
-                retry_per=args.retry,
-                auth_headers=auth_headers,
-                parallel=not args.serial,
-            )
-        except Exception as e:
-            err_code = classify_error(e)
-            emit_progress("error", error=str(e), error_code=err_code, query=query)
-            return None, str(e), err_code
-        if ttl_seconds > 0:
-            cache_module.put(params, results, ttl_seconds)
-            logger.info(f"[cache stored] q={query!r} TTL={args.cache_ttl}min")
-            emit_progress("cache_store", query=query, ttl=args.cache_ttl)
+        # 多页聚合路径
+        logger.info(f"Multi-page: fetching {pages_to_fetch} pages for q={query!r}")
+        merged_results_list = []
+        base_results = None
+        page_errors = 0
+        for page_no in range(1, pages_to_fetch + 1):
+            page_params = dict(params)
+            if page_no != 1:
+                page_params["pageno"] = str(page_no)
+
+            cached = cache_module.get(page_params, ttl_seconds) if ttl_seconds > 0 else None
+            if cached is not None:
+                logger.info(f"[cache hit] q={query!r} page={page_no}")
+                emit_progress("cache_hit", query=query, ttl=args.cache_ttl, page=page_no)
+                page_results = cached
+            else:
+                try:
+                    page_results = search_multi(
+                        instance_urls, page_params,
+                        method=args.method,
+                        timeout=args.timeout,
+                        retry_per=args.retry,
+                        auth_headers=auth_headers,
+                        parallel=not args.serial,
+                    )
+                except Exception as e:
+                    page_errors += 1
+                    logger.warning(f"Page {page_no} failed: {e}")
+                    emit_progress("page_fail", query=query, page=page_no,
+                                  error=str(e), error_code=classify_error(e))
+                    continue
+                if ttl_seconds > 0:
+                    cache_module.put(page_params, page_results, ttl_seconds)
+                    emit_progress("cache_store", query=query, ttl=args.cache_ttl, page=page_no)
+
+            if base_results is None:
+                base_results = page_results
+            merged_results_list.extend(page_results.get("results", []))
+            emit_progress("page_ok", query=query, page=page_no,
+                          results=len(page_results.get("results", [])))
+
+        if base_results is None:
+            # 所有页都失败了
+            return None, f"All {pages_to_fetch} pages failed", E_NETWORK
+
+        # 合并所有页的结果，后续统一去重排序
+        results = base_results
+        results["results"] = merged_results_list
+        results["pages_fetched"] = pages_to_fetch - page_errors
+        results["page_errors"] = page_errors
+        logger.info(f"Multi-page merged: {len(merged_results_list)} results "
+                    f"from {pages_to_fetch - page_errors}/{pages_to_fetch} pages")
 
     # v2.1.1：检测实例侧引擎挂起/限流（仅在实时查询后提示，cache hit 时
     # unresponsive_engines 信息可能已过期）
@@ -1701,6 +1886,9 @@ def _run_single_query(query: str, args, instance_urls: list,
     # so --max-results keeps the highest-scoring / newest items.
     if not args.no_dedup:
         deduplicate_results(results)
+    # 相似度去重（默认关闭，需 --similarity-dedup 启用）
+    if getattr(args, "similarity_dedup", False):
+        deduplicate_similar(results, threshold=args.similarity_threshold)
     sort_results(results, args.sort_by)
 
     if args.max_results and "results" in results:
@@ -1725,8 +1913,11 @@ def _run_single_query(query: str, args, instance_urls: list,
             referer = instance_urls[0]
         # v2.0.0: 创建共享 throttle 实例，用于 --fetch-report 输出
         request_delay = getattr(args, "request_delay", 0.3)
-        fetch_throttle = AdaptiveThrottle(request_delay,
-                                          min(5, args.fetch))
+        fetch_throttle = AdaptiveThrottle(
+            request_delay, min(5, args.fetch),
+            failure_threshold=getattr(args, "throttle_failure_threshold", 3),
+            pause_seconds=getattr(args, "throttle_pause_seconds", 30),
+            max_delay=getattr(args, "throttle_max_delay", 10))
         fetched = fetch_top_results(
             results, args.fetch,
             timeout=args.fetch_timeout,
@@ -1752,8 +1943,16 @@ def _run_single_query(query: str, args, instance_urls: list,
         results["fetched_source"] = results.get("_fallback", "json")
 
         # v2.0.0: --fetch-report 输出到 stderr
-        if getattr(args, "fetch_report", False):
-            _emit_fetch_report(fetched, fetch_throttle)
+        # 兼容旧的 bool（True→"text"，False→不输出）和新的 str（"text"/"json"）
+        fetch_report_fmt = getattr(args, "fetch_report", False)
+        if fetch_report_fmt is True:
+            fetch_report_fmt = "text"
+        elif not fetch_report_fmt:
+            fetch_report_fmt = None
+        elif fetch_report_fmt not in ("text", "json"):
+            fetch_report_fmt = "text"
+        if fetch_report_fmt:
+            _emit_fetch_report(fetched, fetch_throttle, fmt=fetch_report_fmt)
 
     result_count = len(results.get("results", []))
     emit_progress("done", results=result_count, query=query)
@@ -1896,6 +2095,94 @@ def _read_queries_file(path: str) -> list:
     return queries
 
 
+def _save_config(args, path: str) -> None:
+    """v2.2.0：将当前 CLI 参数保存为 searxng.toml 配置文件。
+
+    只保存有实际值的参数（非 None / 非默认），让生成的配置文件干净可用。
+    """
+    lines = ["# searxng-cli config (auto-generated by --save-config)", ""]
+    _cfg_map = {
+        "instances": "instances",
+        "categories": "categories",
+        "language": "language",
+        "pageno": "pageno",
+        "pages": "pages",
+        "time_range": "time_range",
+        "safesearch": "safesearch",
+        "method": "method",
+        "format": "format",
+        "sort_by": "sort_by",
+        "timeout": "timeout",
+        "max_retries": "retry",
+        "max_results": "max_results",
+        "cache_ttl": "cache_ttl",
+        "cache_max_size": "cache_max_size",
+        "fetch": "fetch",
+        "fetch_timeout": "fetch_timeout",
+        "fetch_retries": "fetch_retries",
+        "throttle_failure_threshold": "throttle_failure_threshold",
+        "throttle_pause_seconds": "throttle_pause_seconds",
+        "throttle_max_delay": "throttle_max_delay",
+        "similarity_threshold": "similarity_threshold",
+        "log_format": "log_format",
+    }
+    for toml_key, attr in _cfg_map.items():
+        val = getattr(args, attr, None)
+        if val is not None and val != "" and val != 0:
+            if isinstance(val, bool):
+                lines.append(f'{toml_key} = {str(val).lower()}')
+            elif isinstance(val, (int, float)):
+                lines.append(f'{toml_key} = {val}')
+            else:
+                lines.append(f'{toml_key} = "{val}"')
+    lines.append("")
+    try:
+        Path(path).write_text("\n".join(lines), encoding="utf-8")
+        print(f"Config saved to {path}")
+        logger.info(f"Config saved to {path}")
+    except OSError as e:
+        _emit_error(f"cannot write config file '{path}': {e}", args,
+                    error_code=E_INPUT)
+
+
+def _dry_run_preview(args, instance_urls: list, auth_headers: dict) -> None:
+    """v2.2.0：--dry-run 预览模式，不发 HTTP 请求，打印将执行的操作。
+
+    输出 JSON 到 stdout，包含 action/url/params/headers_count 等字段。
+    """
+    preview = {
+        "dry_run": True,
+        "instances": instance_urls,
+        "headers_count": len(auth_headers) if auth_headers else 0,
+    }
+    if args.query:
+        preview["action"] = "search"
+        preview["query"] = args.query
+        preview["params"] = _build_params(args.query, args)
+    elif args.research:
+        preview["action"] = "research"
+        preview["topic"] = args.research
+        custom_angles = None
+        if getattr(args, "research_angles", None):
+            custom_angles = [a.strip() for a in args.research_angles.split(",") if a.strip()]
+        preview["angles"] = [a for a, _ in expand_research_queries(
+            args.research, custom_angles=custom_angles)]
+    elif args.queries_file:
+        preview["action"] = "batch"
+        preview["queries_file"] = args.queries_file
+    elif args.verify:
+        preview["action"] = "verify"
+    else:
+        preview["action"] = "noop"
+    if getattr(args, "pages", 1) > 1:
+        preview["pages"] = args.pages
+    if args.fetch:
+        preview["fetch"] = args.fetch
+    print(json.dumps(preview, indent=2, ensure_ascii=False))
+    logger.info("Dry run complete — no HTTP requests sent")
+    sys.exit(0)
+
+
 # ----- Research mode (v2.1.0) -----
 # 给定一个主题，自动扩展多角度查询词，复用批量搜索逻辑。
 # 扩展策略是确定性规则（不做 AI 判断），覆盖人物/主题/事件的通用研究维度。
@@ -1925,7 +2212,7 @@ def _is_chinese_topic(topic: str) -> bool:
     return bool(re.search(r'[\u4e00-\u9fff]', topic))
 
 
-def expand_research_queries(topic: str) -> list:
+def expand_research_queries(topic: str, custom_angles: list = None) -> list:
     """将研究主题扩展为多角度查询词列表。
 
     v2.1.0 研究模式核心函数。给定一个主题（如"七森莉莉"或"Python asyncio"），
@@ -1940,6 +2227,10 @@ def expand_research_queries(topic: str) -> list:
     （"七森莉莉 简介"），纯英文主题用英文后缀（"Python asyncio profile"），
     避免跨语言组合在搜索引擎上匹配度低。
 
+    v2.2.0 新增：custom_angles 参数。传入自定义角度列表时（如
+    ["overview","profile","timeline","controversy"]），每个角度直接作为
+    查询后缀（中英文通用），覆盖默认 5 角度。
+
     返回 [(angle, query), ...] 列表，angle 用于结果标注。
 
     确定性规则，不依赖 AI 判断——确保跨进程可复现，AI Agent 可预期。
@@ -1947,6 +2238,9 @@ def expand_research_queries(topic: str) -> list:
     topic = topic.strip()
     if not topic:
         return []
+    if custom_angles:
+        # v2.2.0：自定义角度模式，每个角度作为后缀直接拼接到主题
+        return [(angle, f"{topic} {angle}".strip()) for angle in custom_angles]
     use_chinese = _is_chinese_topic(topic)
     queries = []
     for angle, cn_suffix, en_suffix in _RESEARCH_ANGLES:
@@ -1956,6 +2250,305 @@ def expand_research_queries(topic: str) -> list:
     return queries
 
 
+# ----- Mode handlers (v2.2.0: 从 main() 提取，降低单函数复杂度) -----
+# main() 只负责参数解析和分发，四条执行路径各自独立函数，便于维护和测试。
+# 纯提取重构，行为与 v2.1.1 完全一致，539 测试兜底验证。
+
+
+def _handle_verify(args, instance_urls: list, auth_headers: dict) -> None:
+    """健康检查模式：探测实例状态并退出（不执行搜索）。"""
+    logger.info(f"Verifying {len(instance_urls)} instance(s)...")
+    report = verify_instances(instance_urls, timeout=args.timeout,
+                              auth_headers=auth_headers)
+    _print_verify_report(report, as_json=(args.format == "json"))
+    sys.exit(0)
+
+
+def _handle_research(args, instance_urls: list, auth_headers: dict,
+                     ttl_seconds: int) -> None:
+    """研究模式：给定主题自动扩展多角度查询，串行搜索，输出带研究元数据的结果。
+
+    v2.1.0 引入，v2.1.1 增加跨角度合并去重 + 中英文双语后缀，
+    v2.2.0 增加 --research-angles 自定义角度 + --similarity-dedup 相似度去重，
+    v2.2.0 从 main() 提取为独立函数（纯重构，行为不变）。
+    """
+    topic = args.research.strip()
+    # v2.2.0：支持 --research-angles 自定义角度
+    custom_angles = None
+    if getattr(args, "research_angles", None):
+        custom_angles = [a.strip() for a in args.research_angles.split(",") if a.strip()]
+    research_queries = expand_research_queries(topic, custom_angles=custom_angles)
+    if not research_queries:
+        _emit_error(f"research topic is empty after stripping", args,
+                    error_code=E_INPUT)
+
+    logger.info(f"Research mode: {len(research_queries)} angles for '{topic}'")
+    batch = []
+    any_with_results = False
+    error_count = 0
+    for i, (angle, q) in enumerate(research_queries, 1):
+        logger.info(f"\n[{i}/{len(research_queries)}] [{angle}] {q}")
+        results, err, err_code = _run_single_query(q, args, instance_urls,
+                                                    auth_headers, ttl_seconds)
+        if err:
+            error_count += 1
+            logger.error(f"  [ERROR] {err}")
+            entry = {"query": q, "angle": angle, "status": "error",
+                     "error": err}
+            if err_code:
+                entry["error_code"] = err_code
+            batch.append(entry)
+        else:
+            if len(results.get("results", [])) > 0:
+                any_with_results = True
+            batch.append({"query": q, "angle": angle, "status": "ok",
+                          "results": results})
+
+    # 跨角度合并去重（v2.1.0 修复）
+    merged = {"query": topic, "results": []}
+    for br in batch:
+        if br.get("status") == "ok" and "results" in br:
+            merged["results"].extend(br["results"].get("results", []))
+    if merged["results"]:
+        deduplicate_results(merged)
+        # 相似度去重（默认关闭，需 --similarity-dedup 启用）
+        if getattr(args, "similarity_dedup", False):
+            deduplicate_similar(merged, threshold=args.similarity_threshold)
+        sort_results(merged, args.sort_by)
+        if args.max_results:
+            merged["results"] = merged["results"][:args.max_results]
+    merged_count = len(merged["results"])
+    logger.info(f"Research merged: {merged_count} unique results after dedup")
+
+    # 输出
+    if args.format == "json":
+        output = json.dumps({
+            "schema_version": SCHEMA_VERSION,
+            "research_topic": topic,
+            "research_queries": [
+                {"angle": a, "query": q} for a, q in research_queries
+            ],
+            "queries": batch,
+            "merged_results": merged,
+        }, indent=2, ensure_ascii=False)
+    elif args.format == "csv":
+        import csv as csv_mod
+        import io
+        out = io.StringIO()
+        writer = csv_mod.writer(out, lineterminator="\n")
+        writer.writerow(["angle", "query", "title", "url", "engine",
+                         "score", "published_date", "content"])
+        for br in batch:
+            q = br["query"]
+            angle = br.get("angle", "")
+            if "results" in br:
+                for r in br["results"].get("results", []):
+                    writer.writerow([
+                        angle, q,
+                        r.get("title", ""),
+                        r.get("url", ""),
+                        r.get("engine", ""),
+                        r.get("score", "") if r.get("score") is not None else "",
+                        r.get("published_date", ""),
+                        r.get("content", ""),
+                    ])
+            else:
+                writer.writerow([angle, q, "", "", "", "", "",
+                                 f"[ERROR: {br['error']}]"])
+        output = out.getvalue().rstrip()
+    elif args.format == "urls":
+        parts = []
+        # 先输出 per-angle 结果
+        for br in batch:
+            parts.append(f"# [{br.get('angle', '?')}] {br['query']}")
+            if "results" in br:
+                parts.append(format_urls(br["results"]))
+            else:
+                parts.append(f"# [ERROR: {br['error']}]")
+        # 再输出合并去重后的总览
+        if merged_count > 0:
+            parts.append("")
+            parts.append(f"# [MERGED] {topic} ({merged_count} unique results)")
+            parts.append(format_urls(merged))
+        output = "\n".join(parts)
+    else:  # brief
+        parts = []
+        for br in batch:
+            parts.append("=" * 60)
+            parts.append(f"[{br.get('angle', '?')}] {br['query']}")
+            parts.append("=" * 60)
+            if "results" in br:
+                parts.append(format_brief(br["results"]))
+            else:
+                parts.append(f"[ERROR: {br['error']}]")
+            parts.append("")
+        # 合并去重后的总览
+        if merged_count > 0:
+            parts.append("=" * 60)
+            parts.append(f"[MERGED] {topic} ({merged_count} unique results)")
+            parts.append("=" * 60)
+            parts.append(format_brief(merged))
+        output = "\n".join(parts)
+
+    if args.output:
+        with open(args.output, "w", encoding="utf-8") as f:
+            f.write(output)
+        logger.info(f"Saved results to {args.output}")
+    else:
+        print(output)
+    # 三态退出码：0=有结果, 2=全部空, 1=全部错误
+    if error_count == len(research_queries):
+        sys.exit(1)
+    elif not any_with_results:
+        sys.exit(2)
+    sys.exit(0)
+
+
+def _handle_batch(args, instance_urls: list, auth_headers: dict,
+                  ttl_seconds: int) -> None:
+    """批量模式：从文件读取多个查询，串行执行，输出合并结果。
+
+    v2.2.0 从 main() 提取为独立函数（纯重构，行为不变）。
+    """
+    try:
+        queries = _read_queries_file(args.queries_file)
+    except RuntimeError as e:
+        _emit_error(str(e), args, error_code=E_INPUT)
+    if not queries:
+        _emit_error(f"no queries found in '{args.queries_file}'", args,
+                    error_code=E_INPUT)
+
+    logger.info(f"Running {len(queries)} queries from {args.queries_file}...")
+    batch = []
+    any_with_results = False
+    error_count = 0
+    for i, q in enumerate(queries, 1):
+        logger.info(f"\n[{i}/{len(queries)}] {q}")
+        results, err, err_code = _run_single_query(q, args, instance_urls,
+                                                    auth_headers, ttl_seconds)
+        if err:
+            error_count += 1
+            logger.error(f"  [ERROR] {err}")
+            entry = {"query": q, "status": "error", "error": err}
+            if err_code:
+                entry["error_code"] = err_code
+            batch.append(entry)
+        else:
+            if len(results.get("results", [])) > 0:
+                any_with_results = True
+            batch.append({"query": q, "status": "ok", "results": results})
+
+    if args.format == "json":
+        output = json.dumps({"schema_version": SCHEMA_VERSION, "queries": batch},
+                            indent=2, ensure_ascii=False)
+    elif args.format == "csv":
+        import csv as csv_mod
+        import io
+        out = io.StringIO()
+        writer = csv_mod.writer(out, lineterminator="\n")
+        writer.writerow(["query", "title", "url", "engine", "score",
+                         "published_date", "content"])
+        for br in batch:
+            q = br["query"]
+            if "results" in br:
+                for r in br["results"].get("results", []):
+                    writer.writerow([
+                        q,
+                        r.get("title", ""),
+                        r.get("url", ""),
+                        r.get("engine", ""),
+                        r.get("score", "") if r.get("score") is not None else "",
+                        r.get("published_date", ""),
+                        r.get("content", ""),
+                    ])
+            else:
+                writer.writerow([q, "", "", "", "", "",
+                                 f"[ERROR: {br['error']}]"])
+        output = out.getvalue().rstrip()
+    elif args.format == "urls":
+        parts = []
+        for br in batch:
+            parts.append(f"# {br['query']}")
+            if "results" in br:
+                parts.append(format_urls(br["results"]))
+            else:
+                parts.append(f"# [ERROR: {br['error']}]")
+        output = "\n".join(parts)
+    else:  # brief
+        parts = []
+        for br in batch:
+            parts.append("=" * 60)
+            parts.append(f"QUERY: {br['query']}")
+            parts.append("=" * 60)
+            if "results" in br:
+                parts.append(_format_results(br["results"], args))
+            else:
+                parts.append(f"[ERROR: {br['error']}]")
+            parts.append("")
+        output = "\n".join(parts)
+
+    if args.output:
+        with open(args.output, "w", encoding="utf-8") as f:
+            f.write(output)
+        logger.info(f"Saved results to {args.output}")
+    else:
+        print(output)
+    # Exit code: 1=全错, 2=全空, 0=有结果
+    if error_count == len(queries):
+        sys.exit(1)
+    if not any_with_results:
+        sys.exit(2)
+    sys.exit(0)
+
+
+def _handle_single(args, instance_urls: list, auth_headers: dict,
+                   ttl_seconds: int) -> None:
+    """单查询模式：执行一次搜索并输出结果。
+
+    支持 --stream 流式输出（JSON Lines）。
+    v2.2.0 从 main() 提取为独立函数（纯重构，行为不变）。
+    """
+    results, err, err_code = _run_single_query(args.query, args, instance_urls,
+                                                auth_headers, ttl_seconds)
+
+    # --stream: JSON Lines 流式输出
+    if getattr(args, "stream", False) and args.format == "json":
+        if err:
+            error_event = {"type": "error", "error": err, "query": args.query}
+            if err_code:
+                error_event["error_code"] = err_code
+                hint = RECOVERY_HINTS.get(err_code)
+                if hint:
+                    error_event["recovery_hint"] = hint
+            print(json.dumps(error_event, ensure_ascii=False), flush=True)
+            sys.exit(1)
+        for r in results.get("results", []):
+            print(json.dumps({"type": "result", "result": r},
+                             ensure_ascii=False), flush=True)
+        print(json.dumps({"type": "done",
+                          "schema_version": SCHEMA_VERSION,
+                          "count": len(results.get("results", [])),
+                          "query": args.query}, ensure_ascii=False), flush=True)
+        if not results.get("results"):
+            sys.exit(2)
+        sys.exit(0)
+
+    if err:
+        _emit_error(err, args, query=args.query, error_code=err_code)
+
+    output = _format_results(results, args)
+    if args.output:
+        with open(args.output, "w", encoding="utf-8") as f:
+            f.write(output)
+        logger.info(f"Saved results to {args.output}")
+    else:
+        print(output)
+
+    if not results.get("results"):
+        logger.warning("Warning: no results returned")
+        sys.exit(2)
+
+
 # ----- Main -----
 
 def main():
@@ -1965,9 +2558,11 @@ def main():
     pre.add_argument("--config", default=None)
     pre.add_argument("--verbose", "-v", action="store_true", default=False)
     pre.add_argument("--quiet", action="store_true", default=False)
+    pre.add_argument("--log-format", default="text", choices=["text", "json"])
     pre_args, _ = pre.parse_known_args()
 
-    setup_logging(verbose=pre_args.verbose, quiet=pre_args.quiet)
+    setup_logging(verbose=pre_args.verbose, quiet=pre_args.quiet,
+                  log_format=pre_args.log_format)
     force_utf8_stdout()  # Windows: prevent GBK crash on non-ASCII chars
 
     # Load config defaults from --config file, else ./searxng.toml or
@@ -2026,6 +2621,10 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
                         help="Language code (e.g. en, zh-CN, de)")
     parser.add_argument("--pageno", "-p", type=int, default=1,
                         help="Page number (default: 1)")
+    parser.add_argument("--pages", type=int, default=1,
+                        help="Fetch N pages and merge with cross-page dedup "
+                             "(default: 1). When > 1, overrides --pageno "
+                             "starting from page 1..N.")
     parser.add_argument("--time-range", "-t",
                         choices=["day", "week", "month", "year", "none"],
                         default=config.get("time_range", "year"),
@@ -2053,6 +2652,14 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
                         help="Disable cross-engine deduplication (by default, duplicate "
                              "URLs — same page ignoring tracking params/fragment — are "
                              "collapsed, keeping the first occurrence's engine/score)")
+    parser.add_argument("--similarity-dedup", action="store_true",
+                        help="启用相似度去重（基于标题 SimHash），默认关闭。"
+                             "合并跨引擎/跨 URL 的近似重复标题，保留首个（通常 score 最高）。"
+                             "O(n²) 复杂度，结果数 > 500 时自动跳过。")
+    parser.add_argument("--similarity-threshold", type=float, default=0.85,
+                        metavar="FLOAT",
+                        help="相似度去重阈值（默认: 0.85）。越高越严格，1.0 要求标题几乎完全相同。"
+                             "仅当 --similarity-dedup 启用时生效。")
     parser.add_argument("--format", "-f", choices=["json", "brief", "urls", "csv"],
                         default=config.get("format", "json"),
                         help="Output format (default: json). 'csv' exports "
@@ -2065,10 +2672,13 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
                         help="Timeout per page fetch in seconds (default: 10)")
     parser.add_argument("--fetch-retries", type=int, default=_cfg_int(config, "fetch_retries", 3),
                         help="Max retries per page fetch (default: 3)")
-    parser.add_argument("--fetch-report", action="store_true",
+    parser.add_argument("--fetch-report", dest="fetch_report",
+                        nargs="?", const="text", default=False, metavar="FORMAT",
                         help="When used with --fetch, emit a structured fetch report to stderr "
-                             "after completion: per-URL status, UA used, attempts, WAF type, "
-                             "fallback used, and adaptive throttle stats. v2.0.0.")
+                             "after completion. FORMAT accepts 'text' (default: human-readable "
+                             "table + JSON summary line, v2.0.0 behavior) or 'json' (full JSON "
+                             "object with per-URL items and summary, easier for agents to parse). "
+                             "Without a value, defaults to 'text'. v2.0.0; json output v2.x.")
     parser.add_argument("--no-fallback", action="store_true",
                         help="Disable Wayback Machine fallback for failed fetches (404/403/timeout). "
                              "By default Wayback fallback is ENABLED to maximize success rate. v2.0.0.")
@@ -2107,6 +2717,27 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
                              "Events: start, instance_try, instance_ok, instance_fail, "
                              "cache_hit, cache_store, fetch_start, fetch_ok, fetch_fail, done. "
                              "AI Agent can track execution progress programmatically.")
+    parser.add_argument("--log-format", choices=["text", "json"], default="text",
+                        help="Log output format (default: text). 'json' emits one JSON "
+                             "object per line for programmatic parsing by AI Agents.")
+    parser.add_argument("--dry-run", action="store_true",
+                        help="Preview actions without sending HTTP requests. Prints "
+                             "{action, url, params, headers_count} as JSON to stdout.")
+    parser.add_argument("--throttle-failure-threshold", type=int, default=3, metavar="N",
+                        help="AdaptiveThrottle: consecutive failures before global pause "
+                             "(default: 3). 0 disables throttling.")
+    parser.add_argument("--throttle-pause-seconds", type=int, default=30, metavar="SEC",
+                        help="AdaptiveThrottle: pause duration when threshold reached "
+                             "(default: 30).")
+    parser.add_argument("--throttle-max-delay", type=int, default=10, metavar="SEC",
+                        help="AdaptiveThrottle: max per-request delay cap "
+                             "(default: 10).")
+    parser.add_argument("--research-angles", default=None, metavar="LIST",
+                        help="Custom research angles (comma-separated). Overrides default "
+                             "5 angles. Example: 'overview,profile,timeline,controversy'")
+    parser.add_argument("--save-config", default=None, metavar="FILE",
+                        help="Save current CLI arguments to a searxng.toml config file "
+                             "and exit. Useful for persisting commonly used settings.")
     parser.add_argument("--timeout", type=int, default=_cfg_int(config, "timeout", 15),
                         help="Request timeout in seconds (default: 15)")
     parser.add_argument("--retry", type=int, default=_cfg_int(config, "max_retries", None),
@@ -2143,6 +2774,14 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
                         help="Cache search results for N minutes (default: 0 = disabled). "
                              "Identical queries within the TTL skip the network entirely. "
                              "Cache lives at $SEARXNG_CACHE_DIR or ~/.cache/searxng-cli/cache.db")
+    parser.add_argument("--cache-max-size", type=int,
+                        default=_cfg_int(config, "cache_max_size", 0),
+                        metavar="MB",
+                        help="Cache size cap in MB with LRU eviction (default: 0 = use "
+                             "$SEARXNG_CACHE_MAX_SIZE_BYTES env var or 100MB built-in default). "
+                             "When the cap is exceeded, least-recently-accessed entries are "
+                             "evicted. 0 here does NOT mean unlimited — it means 'defer to "
+                             "env/default'. Use a very large value for effectively unlimited.")
     parser.add_argument("--clear-cache", action="store_true",
                         help="Delete all cached entries and exit (no search performed)")
     parser.add_argument("--cache-stats", action="store_true",
@@ -2154,11 +2793,24 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
 
     args = parser.parse_args()
 
+    # v2.2.0：生成 request_id 并重新配置 logging（带 log_format + request_id）
+    # request_id 贯穿所有日志、进度事件和错误输出，便于 batch 模式追溯。
+    import os as _os
+    request_id = _os.urandom(4).hex()
+    setup_logging(verbose=args.verbose, quiet=args.quiet,
+                  log_format=getattr(args, "log_format", "text"),
+                  request_id=request_id)
+
     # --dump-schema：输出 JSON Schema 到 stdout 并退出，AI Agent 可程序化发现字段
     if getattr(args, "dump_schema", False):
         print(json.dumps(_get_output_schema(), indent=2, ensure_ascii=False))
         sys.exit(0)
 
+    # --save-config：保存当前参数到 searxng.toml 并退出
+    if getattr(args, "save_config", None):
+        _save_config(args, args.save_config)
+        sys.exit(0)
+
     # 启用 --progress 进度事件（JSON Lines 到 stderr）
     set_progress_enabled(getattr(args, "progress", False))
 
@@ -2210,6 +2862,14 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
         apply_proxy(args.proxy)
         logger.info(f"Proxy: {args.proxy}")
 
+    # --cache-max-size: CLI 参数注入全局缓存单例（优先级高于环境变量）。
+    # 必须在 --clear-cache / --cache-stats / 任何 search 操作之前执行，
+    # 保证 stats() 的 max_size_bytes 字段和 LRU 淘汰都用正确上限。
+    # args.cache_max_size <= 0 表示"defer to env/default"，不调用注入。
+    if args.cache_max_size > 0:
+        cache_module.set_max_size_bytes(args.cache_max_size * 1024 * 1024)
+        logger.debug(f"Cache max size set to {args.cache_max_size}MB via CLI")
+
     # --clear-cache / --cache-stats 不需要实例，在实例解析之前处理并退出。
     # 避免无 -i 时报 E_CONFIG "no instance resolved" 让 AI 困惑。
     if args.clear_cache:
@@ -2292,13 +2952,13 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
         auth_type = "Bearer" if bearer_token else "Basic"
         logger.info(f"Auth: {auth_type} ***")
 
+    # v2.2.0：--dry-run 预览模式，不发 HTTP 请求
+    if getattr(args, "dry_run", False):
+        _dry_run_preview(args, instance_urls, auth_headers)
+
     # Health-check mode: report instance status and exit (no search performed)
     if args.verify:
-        logger.info(f"Verifying {len(instance_urls)} instance(s)...")
-        report = verify_instances(instance_urls, timeout=args.timeout,
-                                  auth_headers=auth_headers)
-        _print_verify_report(report, as_json=(args.format == "json"))
-        sys.exit(0)
+        _handle_verify(args, instance_urls, auth_headers)
 
     if args.fail_fast:
         instance_urls = instance_urls[:1]
@@ -2312,276 +2972,17 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
     # 给定主题，自动扩展为 5 个多角度查询，串行搜索，输出带研究元数据的 JSON。
     # 复用批量搜索逻辑，但查询词来自 expand_research_queries 而非文件。
     if args.research:
-        topic = args.research.strip()
-        research_queries = expand_research_queries(topic)
-        if not research_queries:
-            _emit_error(f"research topic is empty after stripping", args,
-                        error_code=E_INPUT)
-
-        logger.info(f"Research mode: {len(research_queries)} angles for '{topic}'")
-        batch = []
-        any_with_results = False
-        error_count = 0
-        for i, (angle, q) in enumerate(research_queries, 1):
-            logger.info(f"\n[{i}/{len(research_queries)}] [{angle}] {q}")
-            results, err, err_code = _run_single_query(q, args, instance_urls,
-                                                        auth_headers, ttl_seconds)
-            if err:
-                error_count += 1
-                logger.error(f"  [ERROR] {err}")
-                entry = {"query": q, "angle": angle, "status": "error",
-                         "error": err}
-                if err_code:
-                    entry["error_code"] = err_code
-                batch.append(entry)
-            else:
-                if len(results.get("results", [])) > 0:
-                    any_with_results = True
-                batch.append({"query": q, "angle": angle, "status": "ok",
-                              "results": results})
-
-        # v2.1.0 修复：跨角度合并去重
-        # 文档承诺 "Results are merged and deduplicated"，原代码只输出 per-angle
-        # 结果，同一 URL 可能出现在多个角度中。此处合并所有成功角度的 results，
-        # 去重后作为 merged_results 字段输出，让 AI Agent 既能按角度组织报告，
-        # 也能获得去重后的总览。
-        merged = {"query": topic, "results": []}
-        for br in batch:
-            if br.get("status") == "ok" and "results" in br:
-                merged["results"].extend(br["results"].get("results", []))
-        if merged["results"]:
-            deduplicate_results(merged)
-            sort_results(merged, args.sort_by)
-            if args.max_results:
-                merged["results"] = merged["results"][:args.max_results]
-        merged_count = len(merged["results"])
-        logger.info(f"Research merged: {merged_count} unique results after dedup")
-
-        # 输出
-        if args.format == "json":
-            output = json.dumps({
-                "schema_version": SCHEMA_VERSION,
-                "research_topic": topic,
-                "research_queries": [
-                    {"angle": a, "query": q} for a, q in research_queries
-                ],
-                "queries": batch,
-                "merged_results": merged,
-            }, indent=2, ensure_ascii=False)
-        elif args.format == "csv":
-            import csv as csv_mod
-            import io
-            out = io.StringIO()
-            writer = csv_mod.writer(out, lineterminator="\n")
-            writer.writerow(["angle", "query", "title", "url", "engine",
-                             "score", "published_date", "content"])
-            for br in batch:
-                q = br["query"]
-                angle = br.get("angle", "")
-                if "results" in br:
-                    for r in br["results"].get("results", []):
-                        writer.writerow([
-                            angle, q,
-                            r.get("title", ""),
-                            r.get("url", ""),
-                            r.get("engine", ""),
-                            r.get("score", "") if r.get("score") is not None else "",
-                            r.get("published_date", ""),
-                            r.get("content", ""),
-                        ])
-                else:
-                    writer.writerow([angle, q, "", "", "", "", "",
-                                     f"[ERROR: {br['error']}]"])
-            output = out.getvalue().rstrip()
-        elif args.format == "urls":
-            parts = []
-            # 先输出 per-angle 结果
-            for br in batch:
-                parts.append(f"# [{br.get('angle', '?')}] {br['query']}")
-                if "results" in br:
-                    parts.append(format_urls(br["results"]))
-                else:
-                    parts.append(f"# [ERROR: {br['error']}]")
-            # 再输出合并去重后的总览
-            if merged_count > 0:
-                parts.append("")
-                parts.append(f"# [MERGED] {topic} ({merged_count} unique results)")
-                parts.append(format_urls(merged))
-            output = "\n".join(parts)
-        else:  # brief
-            parts = []
-            for br in batch:
-                parts.append("=" * 60)
-                parts.append(f"[{br.get('angle', '?')}] {br['query']}")
-                parts.append("=" * 60)
-                if "results" in br:
-                    parts.append(format_brief(br["results"]))
-                else:
-                    parts.append(f"[ERROR: {br['error']}]")
-                parts.append("")
-            # 合并去重后的总览
-            if merged_count > 0:
-                parts.append("=" * 60)
-                parts.append(f"[MERGED] {topic} ({merged_count} unique results)")
-                parts.append("=" * 60)
-                parts.append(format_brief(merged))
-            output = "\n".join(parts)
-
-        if args.output:
-            with open(args.output, "w", encoding="utf-8") as f:
-                f.write(output)
-            logger.info(f"Saved results to {args.output}")
-        else:
-            print(output)
-        # 三态退出码：0=有结果, 2=全部空, 1=全部错误
-        if error_count == len(research_queries):
-            sys.exit(1)
-        elif not any_with_results:
-            sys.exit(2)
-        sys.exit(0)
+        _handle_research(args, instance_urls, auth_headers, ttl_seconds)
 
     # ----- Batch mode: --queries-file -----
     # Reads one query per line (blank/# lines skipped) and runs them in
     # sequence. Output is a JSON array (json format) or concatenated blocks
     # (brief/urls). A failed query is recorded but does not abort the batch.
     if args.queries_file:
-        try:
-            queries = _read_queries_file(args.queries_file)
-        except RuntimeError as e:
-            _emit_error(str(e), args, error_code=E_INPUT)
-        if not queries:
-            _emit_error(f"no queries found in '{args.queries_file}'", args,
-                        error_code=E_INPUT)
-
-        logger.info(f"Running {len(queries)} queries from {args.queries_file}...")
-        batch = []
-        any_with_results = False
-        error_count = 0
-        for i, q in enumerate(queries, 1):
-            logger.info(f"\n[{i}/{len(queries)}] {q}")
-            results, err, err_code = _run_single_query(q, args, instance_urls,
-                                                        auth_headers, ttl_seconds)
-            if err:
-                error_count += 1
-                logger.error(f"  [ERROR] {err}")
-                entry = {"query": q, "status": "error", "error": err}
-                if err_code:
-                    entry["error_code"] = err_code
-                batch.append(entry)
-            else:
-                if len(results.get("results", [])) > 0:
-                    any_with_results = True
-                batch.append({"query": q, "status": "ok", "results": results})
-
-        if args.format == "json":
-            output = json.dumps({"schema_version": SCHEMA_VERSION, "queries": batch},
-                                indent=2, ensure_ascii=False)
-        elif args.format == "csv":
-            import csv as csv_mod
-            import io
-            out = io.StringIO()
-            writer = csv_mod.writer(out, lineterminator="\n")
-            writer.writerow(["query", "title", "url", "engine", "score",
-                             "published_date", "content"])
-            for br in batch:
-                q = br["query"]
-                if "results" in br:
-                    for r in br["results"].get("results", []):
-                        writer.writerow([
-                            q,
-                            r.get("title", ""),
-                            r.get("url", ""),
-                            r.get("engine", ""),
-                            r.get("score", "") if r.get("score") is not None else "",
-                            r.get("published_date", ""),
-                            r.get("content", ""),
-                        ])
-                else:
-                    writer.writerow([q, "", "", "", "", "",
-                                     f"[ERROR: {br['error']}]"])
-            output = out.getvalue().rstrip()
-        elif args.format == "urls":
-            parts = []
-            for br in batch:
-                parts.append(f"# {br['query']}")
-                if "results" in br:
-                    parts.append(format_urls(br["results"]))
-                else:
-                    parts.append(f"# [ERROR: {br['error']}]")
-            output = "\n".join(parts)
-        else:  # brief
-            parts = []
-            for br in batch:
-                parts.append("=" * 60)
-                parts.append(f"QUERY: {br['query']}")
-                parts.append("=" * 60)
-                if "results" in br:
-                    parts.append(_format_results(br["results"], args))
-                else:
-                    parts.append(f"[ERROR: {br['error']}]")
-                parts.append("")
-            output = "\n".join(parts)
-
-        if args.output:
-            with open(args.output, "w", encoding="utf-8") as f:
-                f.write(output)
-            logger.info(f"Saved results to {args.output}")
-        else:
-            print(output)
-        # Exit code semantics — aligned with single-query mode so AI agents
-        # can use one consistent rule:
-        #   1 = all queries errored (fatal)
-        #   2 = no query returned any results (empty), though at least one
-        #       searched successfully without error
-        #   0 = at least one query returned results
-        if error_count == len(queries):
-            sys.exit(1)
-        if not any_with_results:
-            sys.exit(2)
-        sys.exit(0)
+        _handle_batch(args, instance_urls, auth_headers, ttl_seconds)
 
     # ----- Single query mode -----
-    results, err, err_code = _run_single_query(args.query, args, instance_urls,
-                                                auth_headers, ttl_seconds)
-
-    # --stream: JSON Lines 流式输出，每条结果一行，AI 可增量处理。
-    # stream 模式下所有输出（包括错误）都是单行 JSON，保持 JSON Lines 格式一致性。
-    # 非 stream 模式的错误走 _emit_error（多行 JSON 或 stderr 文本）。
-    if getattr(args, "stream", False) and args.format == "json":
-        if err:
-            error_event = {"type": "error", "error": err, "query": args.query}
-            if err_code:
-                error_event["error_code"] = err_code
-                hint = RECOVERY_HINTS.get(err_code)
-                if hint:
-                    error_event["recovery_hint"] = hint
-            print(json.dumps(error_event, ensure_ascii=False), flush=True)
-            sys.exit(1)
-        for r in results.get("results", []):
-            print(json.dumps({"type": "result", "result": r},
-                             ensure_ascii=False), flush=True)
-        print(json.dumps({"type": "done",
-                          "schema_version": SCHEMA_VERSION,
-                          "count": len(results.get("results", [])),
-                          "query": args.query}, ensure_ascii=False), flush=True)
-        if not results.get("results"):
-            sys.exit(2)
-        sys.exit(0)
-
-    if err:
-        _emit_error(err, args, query=args.query, error_code=err_code)
-
-    output = _format_results(results, args)
-    if args.output:
-        with open(args.output, "w", encoding="utf-8") as f:
-            f.write(output)
-        logger.info(f"Saved results to {args.output}")
-    else:
-        print(output)
-
-    if not results.get("results"):
-        logger.warning("Warning: no results returned")
-        sys.exit(2)
+    _handle_single(args, instance_urls, auth_headers, ttl_seconds)
 
 
 if __name__ == "__main__":
diff --git a/tests/test_browser_headers.py b/tests/test_browser_headers.py
index 4ddf839..91e2a0f 100644
--- a/tests/test_browser_headers.py
+++ b/tests/test_browser_headers.py
@@ -150,9 +150,22 @@ def test_build_headers_includes_accept_language():
 
 
 def test_build_headers_includes_accept_encoding():
-    """Accept-Encoding 必须存在；Firefox 不发 br。"""
+    """Accept-Encoding 必须存在；Firefox 不发 br；Chrome/Edge 仅在
+    本机安装了 brotli 解压器时才声明 br（v2.2.1 智能声明）。
+
+    未安装 brotli 时声明 br 会导致服务器返回 br 压缩字节而 requests
+    无法解压 → 全页乱码。这是 fetch.py 真实环境的 bug 修复。
+    """
+    # 导入 common 的 brotli 检测状态
+    from common import _HAS_BROTLI
+
     chrome_headers = build_browser_headers(FALLBACK_UAS[0])
-    assert "br" in chrome_headers["Accept-Encoding"]
+    # Chrome/Edge：根据 brotli 可用性决定是否声明 br
+    if _HAS_BROTLI:
+        assert "br" in chrome_headers["Accept-Encoding"]
+    else:
+        assert "br" not in chrome_headers["Accept-Encoding"]
+    assert "gzip" in chrome_headers["Accept-Encoding"]
 
     firefox_ua = next(ua for ua in FALLBACK_UAS if "Firefox/" in ua)
     firefox_headers = build_browser_headers(firefox_ua)
diff --git a/tests/test_integration.py b/tests/test_integration.py
index 30e1bad..b72c383 100644
--- a/tests/test_integration.py
+++ b/tests/test_integration.py
@@ -1,12 +1,15 @@
 """Integration tests — mock urllib to test search_multi / fetch_url end-to-end.
 
 Covers: search_json success/HTML-fallback/404/403, search_multi serial
-failover + all-fail, fetch_url stdlib-path success.
+failover + all-fail, fetch_url stdlib-path success, Content-Encoding
+decompression (gzip/deflate/br).
 
 No real network calls are made; ``urllib.request.urlopen`` is patched.
 """
+import gzip
 import json
 import urllib.error
+import zlib
 from unittest.mock import patch, MagicMock
 
 from search import search_json, search_multi
@@ -131,3 +134,118 @@ def test_fetch_url_stdlib_max_size_truncates():
          patch.object(fetch_mod, "_HAS_REQUESTS", False):
         result = fetch_url("https://example.com", max_retries=0, max_size=50)
     assert result.truncated is True
+
+
+# ----- fetch_url Content-Encoding 解压（v2.1.1 + v2.2.1）-----
+# 这些测试防止真实环境的乱码 bug 回归：
+#   - v2.1.1: stdlib urllib 不自动解压 gzip/deflate → 乱码
+#   - v2.2.1: requests 不自动解压 br（未装 brotli 时）→ 乱码
+
+def test_fetch_url_stdlib_gzip_decompress():
+    """stdlib 路径正确解压 gzip 压缩的响应（v2.1.1 修复）。"""
+    html = b"<html><body><p>Gzip content</p></body></html>"
+    compressed = gzip.compress(html)
+    resp = _mock_urlopen(compressed, content_type="text/html")
+    resp.headers = {"Content-Type": "text/html", "Content-Encoding": "gzip"}
+    with patch("urllib.request.urlopen", return_value=resp), \
+         patch.object(fetch_mod, "_HAS_REQUESTS", False):
+        result = fetch_url("https://example.com", max_retries=0)
+    assert "Gzip content" in result.content
+    assert result.content.startswith("<html>")
+
+
+def test_fetch_url_stdlib_deflate_decompress():
+    """stdlib 路径正确解压 deflate 压缩的响应（v2.1.1 修复）。"""
+    html = b"<html><body><p>Deflate content</p></body></html>"
+    # zlib.compress 产生带 zlib 头的 deflate 流
+    compressed = zlib.compress(html)
+    resp = _mock_urlopen(compressed, content_type="text/html")
+    resp.headers = {"Content-Type": "text/html", "Content-Encoding": "deflate"}
+    with patch("urllib.request.urlopen", return_value=resp), \
+         patch.object(fetch_mod, "_HAS_REQUESTS", False):
+        result = fetch_url("https://example.com", max_retries=0)
+    assert "Deflate content" in result.content
+    assert result.content.startswith("<html>")
+
+
+def test_fetch_url_stdlib_brotli_decompress_when_brotli_available():
+    """stdlib 路径在 brotli 可用时正确解压 br 压缩的响应（v2.2.1 修复）。
+
+    使用 mock brotli 模块避免依赖真实 brotli 包。
+    """
+    html = b"<html><body><p>Brotli content</p></body></html>"
+    # 用 gzip 模拟 br 压缩字节（仅用于测试解压逻辑被正确调用）
+    compressed = gzip.compress(html)
+
+    # 构造 mock brotli 模块
+    fake_brotli = MagicMock()
+    fake_brotli.decompress = MagicMock(return_value=html)
+
+    resp = _mock_urlopen(compressed, content_type="text/html")
+    resp.headers = {"Content-Type": "text/html", "Content-Encoding": "br"}
+    with patch("urllib.request.urlopen", return_value=resp), \
+         patch.object(fetch_mod, "_HAS_REQUESTS", False), \
+         patch.object(fetch_mod, "_HAS_BROTLI", True), \
+         patch.object(fetch_mod, "_brotli", fake_brotli):
+        result = fetch_url("https://example.com", max_retries=0)
+    assert "Brotli content" in result.content
+    assert result.content.startswith("<html>")
+    # 验证 brotli.decompress 确实被调用
+    fake_brotli.decompress.assert_called_once_with(compressed)
+
+
+def test_fetch_url_stdlib_brotli_skipped_when_unavailable():
+    """stdlib 路径在 brotli 不可用时跳过 br 解压（保留原 raw，由 decode 兜底）。
+
+    这对应真实环境中未安装 brotli 包的情况：build_browser_headers 不会
+    声明 br，所以正常情况下不会收到 br 响应。此测试验证即使收到 br
+    响应也不会崩溃（虽然内容会是乱码）。
+    """
+    html = b"<html><body><p>content</p></body></html>"
+    compressed = gzip.compress(html)  # 假装是 br 压缩
+    resp = _mock_urlopen(compressed, content_type="text/html")
+    resp.headers = {"Content-Type": "text/html", "Content-Encoding": "br"}
+    with patch("urllib.request.urlopen", return_value=resp), \
+         patch.object(fetch_mod, "_HAS_REQUESTS", False), \
+         patch.object(fetch_mod, "_HAS_BROTLI", False):
+        result = fetch_url("https://example.com", max_retries=0)
+    # 未解压时 content 是乱码但不会崩溃（errors="replace" 兜底）
+    assert result.content  # 有内容（虽然是乱码）
+
+
+def test_fetch_url_requests_brotli_decompress_when_available():
+    """requests 路径在 brotli 可用时手动解压 br（v2.2.1 修复）。
+
+    requests 不自动解压 br（除非安装 brotli 包）。此测试验证双保险逻辑：
+    即使 requests 路径，也会在 _HAS_BROTLI=True 时手动解压 br。
+
+    使用 mock requests Session 避免 HTTP 请求。
+    """
+    html = b"<html><body><p>BR via requests</p></body></html>"
+    compressed = gzip.compress(html)  # 假装是 br 压缩
+
+    fake_brotli = MagicMock()
+    fake_brotli.decompress = MagicMock(return_value=html)
+
+    # 构造 mock requests Response
+    mock_resp = MagicMock()
+    mock_resp.status_code = 200
+    mock_resp.headers = {
+        "Content-Type": "text/html",
+        "Content-Encoding": "br",
+    }
+    mock_resp.content = compressed
+    mock_resp.url = "https://example.com"
+    mock_resp.raise_for_status = MagicMock()
+    mock_resp.close = MagicMock()
+
+    mock_session = MagicMock()
+    mock_session.get = MagicMock(return_value=mock_resp)
+
+    with patch.object(fetch_mod, "_HAS_REQUESTS", True), \
+         patch.object(fetch_mod, "_HAS_BROTLI", True), \
+         patch.object(fetch_mod, "_brotli", fake_brotli), \
+         patch.object(fetch_mod, "_get_session", return_value=mock_session):
+        result = fetch_url("https://example.com", max_retries=0)
+    assert "BR via requests" in result.content
+    fake_brotli.decompress.assert_called_once_with(compressed)