feat(v2.2.1): 修复 Brotli 乱码 + 缓存治理 + 多页聚合 + 研究模式增强

核心修复（v2.2.1）:
- 修复 Brotli 乱码 bug: build_browser_headers 智能声明 Accept-Encoding,
  仅在 brotli 可用时才声明 br; fetch.py 双路径 br 解压（requests + stdlib）
  此前 Chrome/Edge UA 抓取 example.com 等返回 br 的站点输出乱码

v2.2.0 新功能:
- main() 拆分为 _handle_verify/_handle_research/_handle_batch/_handle_single
- --cache-max-size MB: 缓存大小上限 + LRU 淘汰（默认 100MB）
- --pages N: 多页聚合 + 跨页去重
- --research 跨角度合并: 新增 merged_results 字段
- --stream / --progress: JSON Lines 流式输出 + request_id 贯穿
- --dry-run / --save-config / --log-format json
- --similarity-dedup / --throttle-* 参数化
- 15-UA 池 + PDF/docx 解析 + error_code 字段

文档与测试:
- SKILL.md: 版本号唯一（元数据），删除版本标记干扰
- README.md: 测试数量 539 -> 544
- 544 passed (新增 5 个 Content-Encoding 解压测试)

scripts/_config.py

@@ -1,12 +1,62 @@
 """Package-level constants for searxng-cli scripts.
 
 Import in sibling scripts with:
-    from _config import VERSION, USER_AGENT, SCHEMA_VERSION
+    from _config import VERSION, USER_AGENT, SCHEMA_VERSION, UA_POOL
 
 Retry settings and shared HTTP utilities now live in ``common.py`` so that
 both ``search.py`` and ``fetch.py`` share one consistent implementation.
 """
 
-VERSION = "2.1.1"
+VERSION = "2.2.1"
 SCHEMA_VERSION = "1.0"
 USER_AGENT = f"searxng-cli/{VERSION}"
+
+# 浏览器 UA 池（用于 searxng-cli 默认 UA 被站点拦截时的回退）。
+#
+# v2.2.0 从 common.py 迁移至此统一管理（SSOT）。common.py 通过
+# ``from _config import UA_POOL`` 引用，并在导入失败时回退到其内置副本。
+#
+# 维护原则：
+#   1. 版本号保持为当前年份的主流浏览器版本，避免被识别为过时浏览器
+#   2. 顺序固定——get_ua_for_domain() 用 SHA-256 哈希选索引，顺序变化会
+#      改变域名→UA 的映射，导致跨版本缓存失效（可接受，但应尽量避免无谓变动）
+#   3. 至少覆盖 Chrome/Edge/Firefox × Windows/macOS/Linux，保证指纹多样性
+#
+# 当前版本（2026 年）：Chrome 138-140 / Edge 138 / Firefox 140 / Safari 18。
+UA_POOL = [
+    # Chrome 140 — Windows / macOS / Linux
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
+    # Chrome 139 — Windows / macOS / Linux
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
+    # Chrome 138 — Windows / macOS / Linux
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
+    # Edge 138 — Windows / macOS
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36 Edg/138.0.0.0",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36 Edg/138.0.0.0",
+    # Firefox 140 — Windows / macOS / Linux
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) "
+    "Gecko/20100101 Firefox/140.0",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) "
+    "Gecko/20100101 Firefox/140.0",
+    "Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0",
+    # Safari 18 — macOS（WebKit 指纹，应对 Chromium 针对性拦截）
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 "
+    "(KHTML, like Gecko) Version/18.0 Safari/605.1.15",
+]

scripts/cache.py

@@ -13,6 +13,14 @@ Storage location (in priority order):
 Uses WAL journal mode for better read concurrency. Entries expire lazily
 on read; :func:`clear` removes all rows. Schema is created on first use.
 
+大小上限与 LRU 淘汰（v2）:
+  * :class:`SearchCache` 支持 ``max_size_bytes`` 参数（默认 100 MB，0 表示
+    不限制，向后兼容）。put 时若总大小超限，按 ``last_accessed_at`` 升序
+    淘汰最旧条目，直到总大小 <= max_size_bytes。
+  * get 命中时更新 ``last_accessed_at``，实现 LRU 语义。
+  * :meth:`SearchCache.evict_expired` 可主动清理已过期条目。
+  * ``$SEARXNG_CACHE_MAX_SIZE_BYTES`` 环境变量可覆盖默认上限。
+
 Design notes:
   * Only the search result dict is cached — fetched page content is NOT,
     because it is large and changes independently of the search result set.
@@ -31,6 +39,8 @@ import time
 from pathlib import Path
 
 DEFAULT_CACHE_DIR = Path.home() / ".cache" / "searxng-cli"
+# 默认缓存大小上限：100 MB；0 表示不限制（向后兼容）
+DEFAULT_MAX_SIZE_BYTES = 104857600
 
 
 def _cache_path() -> Path:
@@ -41,6 +51,48 @@ def _cache_path() -> Path:
     return DEFAULT_CACHE_DIR / "cache.db"
 
 
+def _ensure_schema(conn: sqlite3.Connection) -> None:
+    """创建表结构并执行向后兼容的 schema 迁移。
+
+    v1 schema: key, created_at, ttl_seconds, payload
+    v2 新增列: last_accessed_at (LRU 排序依据), size_bytes (payload 字节大小)
+
+    对已存在的旧表用 ALTER TABLE ADD COLUMN 添加新列，并回填数据，
+    保证升级后现有条目也能参与大小统计与 LRU 淘汰。
+    """
+    conn.execute(
+        """
+        CREATE TABLE IF NOT EXISTS search_cache (
+            key TEXT PRIMARY KEY,
+            created_at REAL NOT NULL,
+            ttl_seconds INTEGER NOT NULL,
+            payload TEXT NOT NULL
+        )
+        """
+    )
+    # 检查现有列，决定是否需要迁移
+    cols = {row[1] for row in conn.execute("PRAGMA table_info(search_cache)").fetchall()}
+    if "last_accessed_at" not in cols:
+        # 新增 LRU 访问时间列，回填为 created_at（视为从未被访问过）
+        conn.execute("ALTER TABLE search_cache ADD COLUMN last_accessed_at REAL")
+        conn.execute(
+            "UPDATE search_cache SET last_accessed_at = created_at "
+            "WHERE last_accessed_at IS NULL"
+        )
+    if "size_bytes" not in cols:
+        # 新增 payload 字节大小列，回填为 payload 的 UTF-8 字节长度
+        conn.execute("ALTER TABLE search_cache ADD COLUMN size_bytes INTEGER")
+        rows = conn.execute(
+            "SELECT key, payload FROM search_cache WHERE size_bytes IS NULL"
+        ).fetchall()
+        for key, payload in rows:
+            conn.execute(
+                "UPDATE search_cache SET size_bytes = ? WHERE key = ?",
+                (len(payload.encode("utf-8")), key),
+            )
+    conn.commit()
+
+
 def _connect(path: Path):
     """Open a connection with WAL mode and ensure the schema exists.
 
@@ -56,17 +108,7 @@ def _connect(path: Path):
     # when --fetch spawns parallel page fetches that might also touch the cache.
     conn.execute("PRAGMA journal_mode=WAL")
     conn.execute("PRAGMA synchronous=NORMAL")
-    conn.execute(
-        """
-        CREATE TABLE IF NOT EXISTS search_cache (
-            key TEXT PRIMARY KEY,
-            created_at REAL NOT NULL,
-            ttl_seconds INTEGER NOT NULL,
-            payload TEXT NOT NULL
-        )
-        """
-    )
-    conn.commit()
+    _ensure_schema(conn)
     return contextlib.closing(conn)
 
 
@@ -89,82 +131,308 @@ def _make_key(params: dict) -> str:
     return hashlib.sha256(raw.encode("utf-8")).hexdigest()
 
 
-def get(params: dict, ttl_seconds: int):
-    """Return cached result if within TTL, else None.
+def _payload_size(payload: str) -> int:
+    """计算 payload 序列化后的 UTF-8 字节大小。"""
+    return len(payload.encode("utf-8"))
 
-    ``ttl_seconds`` is the caller's current TTL setting. If the stored
-    entry was written with a longer TTL, the caller's shorter TTL wins
-    (so reducing --cache-ttl takes effect immediately without a clear).
+
+class SearchCache:
+    """带大小上限和 LRU 淘汰的 SQLite 缓存。
+
+    Args:
+        path: 缓存数据库路径。None 表示使用 ``$SEARXNG_CACHE_DIR`` 或
+            默认路径（每次操作动态解析，便于测试 monkeypatch 环境变量）。
+        max_size_bytes: 缓存总大小上限（字节）。0 表示不限制（向后兼容）。
+
+    大小跟踪与 LRU 语义:
+        * put 时计算 payload 字节大小并维护 ``_total_bytes`` 计数器。
+        * 若加入新条目后总大小超过 ``max_size_bytes``，按
+          ``last_accessed_at`` 升序淘汰最旧条目。
+        * get 命中时更新 ``last_accessed_at``，将条目移到"最近使用"位置。
     """
-    if ttl_seconds <= 0:
-        return None
-    key = _make_key(params)
-    try:
-        with _connect(_cache_path()) as conn:
+
+    def __init__(self, path: Path = None, max_size_bytes: int = DEFAULT_MAX_SIZE_BYTES):
+        self._path_override = Path(path) if path else None
+        self._max_size_bytes = max_size_bytes
+        self._total_bytes = 0
+        self._evicted_count = 0
+        self._total_bytes_loaded = False
+        # 记录上次加载 _total_bytes 时的路径，路径变化时重新加载
+        self._loaded_path = None
+
+    def _resolve_path(self) -> Path:
+        """解析当前应使用的缓存路径（未显式指定时动态读取环境变量）。"""
+        return self._path_override or _cache_path()
+
+    def _load_total_bytes(self, conn: sqlite3.Connection) -> None:
+        """惰性从 DB 加载 _total_bytes；路径变化时重新加载。"""
+        current_path = str(self._resolve_path())
+        if self._total_bytes_loaded and self._loaded_path == current_path:
+            return
+        row = conn.execute(
+            "SELECT COALESCE(SUM(size_bytes), 0) FROM search_cache"
+        ).fetchone()
+        self._total_bytes = row[0] or 0
+        self._total_bytes_loaded = True
+        self._loaded_path = current_path
+
+    def get(self, params: dict, ttl_seconds: int):
+        """Return cached result if within TTL, else None.
+
+        ``ttl_seconds`` is the caller's current TTL setting. If the stored
+        entry was written with a longer TTL, the caller's shorter TTL wins
+        (so reducing --cache-ttl takes effect immediately without a clear).
+
+        命中时更新 ``last_accessed_at`` 以实现 LRU 语义。
+        """
+        if ttl_seconds <= 0:
+            return None
+        key = _make_key(params)
+        try:
+            with _connect(self._resolve_path()) as conn:
+                self._load_total_bytes(conn)
+                row = conn.execute(
+                    "SELECT payload, created_at, ttl_seconds FROM search_cache "
+                    "WHERE key = ?",
+                    (key,),
+                ).fetchone()
+                if row is None:
+                    return None
+                payload, created_at, stored_ttl = row
+                effective_ttl = min(ttl_seconds, stored_ttl)
+                if time.time() - created_at > effective_ttl:
+                    return None
+                # LRU: 命中时把条目移到"最近使用"位置
+                conn.execute(
+                    "UPDATE search_cache SET last_accessed_at = ? WHERE key = ?",
+                    (time.time(), key),
+                )
+                conn.commit()
+                return json.loads(payload)
+        except sqlite3.Error:
+            return None
+        except (ValueError, json.JSONDecodeError):
+            # Corrupt payload — treat as miss
+            return None
+
+    def put(self, params: dict, result: dict, ttl_seconds: int) -> None:
+        """Store a result with the given TTL. Silently no-ops on TTL<=0 or error.
+
+        若加入新条目后总大小超过 ``max_size_bytes``，按 LRU 淘汰最旧条目。
+        """
+        if ttl_seconds <= 0:
+            return
+        key = _make_key(params)
+        payload = json.dumps(result, ensure_ascii=False)
+        new_size = _payload_size(payload)
+        try:
+            with _connect(self._resolve_path()) as conn:
+                self._load_total_bytes(conn)
+                # 若 key 已存在，先减去旧条目大小，避免重复计入
+                old = conn.execute(
+                    "SELECT size_bytes FROM search_cache WHERE key = ?", (key,)
+                ).fetchone()
+                if old is not None:
+                    self._total_bytes -= (old[0] or 0)
+                now = time.time()
+                conn.execute(
+                    "INSERT OR REPLACE INTO search_cache "
+                    "(key, created_at, ttl_seconds, payload, "
+                    " last_accessed_at, size_bytes) VALUES (?, ?, ?, ?, ?, ?)",
+                    (key, now, ttl_seconds, payload, now, new_size),
+                )
+                self._total_bytes += new_size
+                self._enforce_size_limit(conn)
+                conn.commit()
+        except sqlite3.Error:
+            pass
+
+    def _enforce_size_limit(self, conn: sqlite3.Connection) -> None:
+        """总大小超限时按 LRU 淘汰最旧条目，直到总大小 <= max_size_bytes。
+
+        ``max_size_bytes <= 0`` 表示不限制，直接返回。当仅剩一个条目时
+        停止淘汰（避免 put 后立即被淘汰导致 get 不到刚写入的条目）。
+        """
+        if self._max_size_bytes <= 0:
+            return
+        while self._total_bytes > self._max_size_bytes:
             row = conn.execute(
-                "SELECT payload, created_at, ttl_seconds FROM search_cache "
-                "WHERE key = ?",
-                (key,),
+                "SELECT key, size_bytes FROM search_cache "
+                "ORDER BY last_accessed_at ASC, created_at ASC LIMIT 1"
             ).fetchone()
-        if row is None:
-            return None
-        payload, created_at, stored_ttl = row
-        effective_ttl = min(ttl_seconds, stored_ttl)
-        if time.time() - created_at > effective_ttl:
-            return None
-        return json.loads(payload)
-    except sqlite3.Error:
-        return None
-    except (ValueError, json.JSONDecodeError):
-        # Corrupt payload — treat as miss
-        return None
+            if row is None:
+                break
+            evict_key, evict_size = row
+            conn.execute("DELETE FROM search_cache WHERE key = ?", (evict_key,))
+            self._total_bytes -= (evict_size or 0)
+            self._evicted_count += 1
+            # 安全阀：只剩一个条目时停止淘汰（即新插入的条目本身超限也保留）
+            count_row = conn.execute("SELECT COUNT(*) FROM search_cache").fetchone()
+            if count_row[0] <= 1:
+                break
+
+    def clear(self) -> int:
+        """Remove all cache entries. Returns count deleted, or 0 on error."""
+        try:
+            with _connect(self._resolve_path()) as conn:
+                self._load_total_bytes(conn)
+                cur = conn.execute("DELETE FROM search_cache")
+                conn.commit()
+                self._total_bytes = 0
+                return cur.rowcount
+        except sqlite3.Error:
+            return 0
+
+    def stats(self) -> dict:
+        """Return cache statistics.
+
+        现有字段：entries, oldest_created_at, newest_created_at, path, size_bytes
+        新增字段：total_bytes, max_size_bytes, evicted_count, utilization_pct
+        """
+        path = self._resolve_path()
+        try:
+            with _connect(path) as conn:
+                row = conn.execute(
+                    "SELECT COUNT(*), MIN(created_at), MAX(created_at) "
+                    "FROM search_cache"
+                ).fetchone()
+                count, oldest, newest = row
+                # 从 DB 校准 total_bytes，防止外部进程修改导致计数器漂移
+                sum_row = conn.execute(
+                    "SELECT COALESCE(SUM(size_bytes), 0) FROM search_cache"
+                ).fetchone()
+                self._total_bytes = sum_row[0] or 0
+                self._total_bytes_loaded = True
+                self._loaded_path = str(path)
+                utilization = (
+                    round(self._total_bytes * 100.0 / self._max_size_bytes, 2)
+                    if self._max_size_bytes > 0
+                    else 0
+                )
+                return {
+                    "entries": count or 0,
+                    "oldest_created_at": oldest,
+                    "newest_created_at": newest,
+                    "path": str(path),
+                    "size_bytes": path.stat().st_size if path.exists() else 0,
+                    # 新增字段：大小上限与 LRU 统计
+                    "total_bytes": self._total_bytes,
+                    "max_size_bytes": self._max_size_bytes,
+                    "evicted_count": self._evicted_count,
+                    "utilization_pct": utilization,
+                }
+        except sqlite3.Error as e:
+            return {
+                "entries": 0,
+                "error": str(e),
+                "path": str(path),
+                "size_bytes": 0,
+                "total_bytes": 0,
+                "max_size_bytes": self._max_size_bytes,
+                "evicted_count": self._evicted_count,
+                "utilization_pct": 0,
+            }
+
+    def evict_expired(self) -> int:
+        """主动扫描并删除已过期条目，返回清理的条目数。
+
+        过期条件：``now - created_at > ttl_seconds``。清理时同步更新
+        ``_total_bytes`` 计数器。
+        """
+        now = time.time()
+        try:
+            with _connect(self._resolve_path()) as conn:
+                self._load_total_bytes(conn)
+                rows = conn.execute(
+                    "SELECT key, size_bytes FROM search_cache "
+                    "WHERE ? - created_at > ttl_seconds",
+                    (now,),
+                ).fetchall()
+                if not rows:
+                    return 0
+                for key, size in rows:
+                    conn.execute("DELETE FROM search_cache WHERE key = ?", (key,))
+                    self._total_bytes -= (size or 0)
+                conn.commit()
+                return len(rows)
+        except sqlite3.Error:
+            return 0
+
+    def set_max_size_bytes(self, max_size_bytes: int) -> None:
+        """更新大小上限并立即触发 LRU 淘汰（若当前已超限）。
+
+        供 CLI ``--cache-max-size`` 在运行时注入参数用。``max_size_bytes <= 0``
+        表示不限制。
+        """
+        self._max_size_bytes = max_size_bytes
+        if max_size_bytes <= 0:
+            return
+        try:
+            with _connect(self._resolve_path()) as conn:
+                self._load_total_bytes(conn)
+                self._enforce_size_limit(conn)
+                conn.commit()
+        except sqlite3.Error:
+            pass
+
+
+# ----- 模块级便捷 API（向后兼容）-----
+# 现有调用方（search.py、测试）使用模块级函数；这里委托给一个惰性创建的
+# 全局实例。全局实例的路径动态解析，因此 monkeypatch $SEARXNG_CACHE_DIR
+# 能正常隔离每个测试。
+
+_default_cache = None
+
+
+def _get_default_cache() -> SearchCache:
+    """惰性创建全局默认缓存实例。
+
+    ``max_size_bytes`` 从 ``$SEARXNG_CACHE_MAX_SIZE_BYTES`` 读取（非法值回退默认）。
+    创建后可被 :func:`set_max_size_bytes` 覆盖（CLI ``--cache-max-size`` 优先级
+    高于环境变量）。
+    """
+    global _default_cache
+    if _default_cache is None:
+        max_size = DEFAULT_MAX_SIZE_BYTES
+        env = os.environ.get("SEARXNG_CACHE_MAX_SIZE_BYTES")
+        if env:
+            try:
+                max_size = int(env)
+            except ValueError:
+                pass
+        _default_cache = SearchCache(max_size_bytes=max_size)
+    return _default_cache
+
+
+def get(params: dict, ttl_seconds: int):
+    """模块级便捷函数：委托给全局默认实例。"""
+    return _get_default_cache().get(params, ttl_seconds)
 
 
 def put(params: dict, result: dict, ttl_seconds: int) -> None:
-    """Store a result with the given TTL. Silently no-ops on TTL<=0 or error."""
-    if ttl_seconds <= 0:
-        return
-    key = _make_key(params)
-    payload = json.dumps(result, ensure_ascii=False)
-    try:
-        with _connect(_cache_path()) as conn:
-            conn.execute(
-                "INSERT OR REPLACE INTO search_cache "
-                "(key, created_at, ttl_seconds, payload) VALUES (?, ?, ?, ?)",
-                (key, time.time(), ttl_seconds, payload),
-            )
-            conn.commit()
-    except sqlite3.Error:
-        pass
+    """模块级便捷函数：委托给全局默认实例。"""
+    _get_default_cache().put(params, result, ttl_seconds)
 
 
 def clear() -> int:
-    """Remove all cache entries. Returns count deleted, or 0 on error."""
-    try:
-        with _connect(_cache_path()) as conn:
-            cur = conn.execute("DELETE FROM search_cache")
-            conn.commit()
-            return cur.rowcount
-    except sqlite3.Error:
-        return 0
+    """模块级便捷函数：委托给全局默认实例。"""
+    return _get_default_cache().clear()
 
 
 def stats() -> dict:
-    """Return cache statistics (entry count, age range, path)."""
-    path = _cache_path()
-    try:
-        with _connect(path) as conn:
-            row = conn.execute(
-                "SELECT COUNT(*), MIN(created_at), MAX(created_at) "
-                "FROM search_cache"
-            ).fetchone()
-        count, oldest, newest = row
-        return {
-            "entries": count or 0,
-            "oldest_created_at": oldest,
-            "newest_created_at": newest,
-            "path": str(path),
-            "size_bytes": path.stat().st_size if path.exists() else 0,
-        }
-    except sqlite3.Error as e:
-        return {"entries": 0, "error": str(e), "path": str(path), "size_bytes": 0}
+    """模块级便捷函数：委托给全局默认实例。"""
+    return _get_default_cache().stats()
+
+
+def evict_expired() -> int:
+    """模块级便捷函数：委托给全局默认实例。"""
+    return _get_default_cache().evict_expired()
+
+
+def set_max_size_bytes(max_size_bytes: int) -> None:
+    """模块级便捷函数：更新全局默认实例的大小上限。
+
+    供 search.py 的 ``--cache-max-size`` CLI 参数在 main() 早期注入用，
+    优先级高于 ``$SEARXNG_CACHE_MAX_SIZE_BYTES`` 环境变量。
+    """
+    _get_default_cache().set_max_size_bytes(max_size_bytes)

scripts/common.py

@@ -18,9 +18,12 @@ as retryable and connection errors as transient, eliminating the previous
 inconsistency where ``search.py`` ignored 5xx.
 """
 
+import hashlib
 import io
 import logging
+import re
 import sys
+import threading
 import urllib.error
 
 # Root logger for the searxng-cli package. All modules create child loggers
@@ -28,8 +31,27 @@ import urllib.error
 # call controls them all.
 _LOG = logging.getLogger("searxng")
 
+# Brotli 解压支持检测（v2.2.1）。
+# requests 库自动解压 gzip/deflate，但**不自动解压 Brotli**（除非安装
+# brotli/brotlicffi 包）。若在 Accept-Encoding 中声明 br 而系统未安装
+# 解压器，服务器返回的 br 压缩字节会被当作文本解码 → 全页乱码。
+# 此检测用于 build_browser_headers() 智能声明 Accept-Encoding，避免
+# 声明无法兑现的 br。
+try:
+    import brotli as _brotli  # type: ignore
+    _HAS_BROTLI = True
+except ImportError:
+    try:
+        import brotlicffi as _brotli  # type: ignore
+        _HAS_BROTLI = True
+    except ImportError:
+        _brotli = None
+        _HAS_BROTLI = False
 
-def setup_logging(verbose: bool = False, quiet: bool = False) -> None:
+
+def setup_logging(verbose: bool = False, quiet: bool = False,
+                  log_format: str = "text",
+                  request_id: str = None) -> None:
     """Configure the ``searxng`` logger hierarchy.
 
     * Default (no flags): ``INFO`` — progress messages, warnings, retry notices.
@@ -39,6 +61,9 @@ def setup_logging(verbose: bool = False, quiet: bool = False) -> None:
       status codes, cache keys, and other diagnostic detail.
     * ``--quiet`` / ``-q``: ``WARNING`` — suppresses progress and retry noise;
       only warnings and errors reach stderr.
+    * ``--log-format json`` (v2.2.0): 每行一个 JSON 对象，便于 AI Agent
+      程序化解析。包含 ts/level/logger/msg/request_id 字段。
+    * request_id (v2.2.0): 贯穿所有日志和进度事件的请求标识符。
 
     All log output goes to stderr; stdout is reserved for data (JSON/CSV/etc.).
     """
@@ -50,15 +75,57 @@ def setup_logging(verbose: bool = False, quiet: bool = False) -> None:
         level = logging.INFO
 
     _LOG.setLevel(level)
+    # 存储 request_id 到 logger 全局，供 formatter 和进度事件使用
+    _LOG._request_id = request_id
+
     # Avoid duplicate handlers if setup_logging() is called twice (e.g. tests).
     if not _LOG.handlers:
         handler = logging.StreamHandler(sys.stderr)
-        handler.setFormatter(logging.Formatter("%(message)s"))
+        if log_format == "json":
+            handler.setFormatter(_JsonFormatter())
+        else:
+            handler.setFormatter(logging.Formatter("%(message)s"))
         _LOG.addHandler(handler)
+    else:
+        # 已有 handler（如测试环境），更新 formatter
+        for h in _LOG.handlers:
+            if log_format == "json":
+                h.setFormatter(_JsonFormatter())
+            else:
+                h.setFormatter(logging.Formatter("%(message)s"))
     # Don't let root logger add its own handler — we own the searxng namespace.
     _LOG.propagate = False
 
 
+class _JsonFormatter(logging.Formatter):
+    """JSON 结构化日志 formatter（v2.2.0）。
+
+    每行输出一个 JSON 对象：{"ts", "level", "logger", "msg", "request_id"}。
+    让 AI Agent 可程序化解析日志（统计重试次数、识别慢实例等）。
+    """
+
+    def format(self, record):
+        import json as _json
+        entry = {
+            "ts": _datetime_iso(record),
+            "level": record.levelname,
+            "logger": record.name,
+            "msg": record.getMessage(),
+        }
+        rid = getattr(_LOG, "_request_id", None)
+        if rid:
+            entry["request_id"] = rid
+        if record.exc_info and record.exc_info[1]:
+            entry["exception"] = type(record.exc_info[1]).__name__
+        return _json.dumps(entry, ensure_ascii=False)
+
+
+def _datetime_iso(record):
+    """格式化日志时间戳为 ISO 8601 字符串。"""
+    import datetime as _dt
+    return _dt.datetime.fromtimestamp(record.created).isoformat(timespec="milliseconds")
+
+
 def force_utf8_stdout() -> None:
     """Force stdout/stderr to UTF-8 to prevent Windows GBK encoding crashes.
 
@@ -124,40 +191,58 @@ RETRYABLE_STATUS = frozenset({403, 429, 502, 503, 504})
 
 # Browser-like UA strings for fallback when the searxng-cli UA is blocked.
 #
-# v2.0.0 扩充至 12 个：覆盖 Chrome/Edge/Firefox × Windows/macOS/Linux，
-# 每个都是较新版本（131/130/129），避免被识别为过时浏览器。
+# v2.2.0：UA 池迁移至 _config.py 的 UA_POOL（SSOT），此处通过导入引用。
+# 若 _config.py 不可导入（如 common.py 被单独分发），回退到下方内置副本
+# _FALLBACK_UAS_BUILTIN，保证模块始终可用。两份列表需保持同步，
+# _config.UA_POOL 为唯一权威来源。
+#
+# v2.0.0 起覆盖 Chrome/Edge/Firefox × Windows/macOS/Linux；v2.2.0 升级到
+# 2026 年版本（Chrome 138-140 / Edge 138 / Firefox 140 / Safari 18）。
 # 顺序固定以便 get_ua_for_domain() 的 hash 选择可复现。
-FALLBACK_UAS = [
-    # Chrome 131 — Windows / macOS / Linux
+_FALLBACK_UAS_BUILTIN = [
+    # Chrome 140 — Windows / macOS / Linux
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
-    "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+    "(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
     "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
-    "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+    "(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
     "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
-    "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
-    # Edge 131 — Windows / macOS
+    "(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
+    # Chrome 139 — Windows / macOS / Linux
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
-    "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
+    "(KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
     "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
-    "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
-    # Firefox 133 — Windows / macOS / Linux
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) "
-    "Gecko/20100101 Firefox/133.0",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:133.0) "
-    "Gecko/20100101 Firefox/133.0",
-    "Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0",
-    # Chrome 130 — Windows / macOS (上一个版本，应对 131 被针对性识别)
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
-    "(KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
-    "(KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
-    # Chrome 129 — Windows / Linux
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
-    "(KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
+    "(KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
     "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
-    "(KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
+    "(KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
+    # Chrome 138 — Windows / macOS / Linux
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
+    # Edge 138 — Windows / macOS
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36 Edg/138.0.0.0",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36 Edg/138.0.0.0",
+    # Firefox 140 — Windows / macOS / Linux
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) "
+    "Gecko/20100101 Firefox/140.0",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) "
+    "Gecko/20100101 Firefox/140.0",
+    "Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0",
+    # Safari 18 — macOS（WebKit 指纹，应对 Chromium 针对性拦截）
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 "
+    "(KHTML, like Gecko) Version/18.0 Safari/605.1.15",
 ]
 
+# 从 _config.py 导入权威 UA_POOL；导入失败时回退到内置副本，保证向后兼容。
+try:
+    from _config import UA_POOL as FALLBACK_UAS
+except ImportError:
+    FALLBACK_UAS = _FALLBACK_UAS_BUILTIN
+
 
 def _ua_index_for_domain(domain: str, pool_size: int) -> int:
     """为域名确定性选择 UA 池索引。
@@ -176,6 +261,13 @@ def _ua_index_for_domain(domain: str, pool_size: int) -> int:
 # 被反爬识别。跨进程通过 SHA-256 哈希复现，见 _ua_index_for_domain。
 _domain_ua_cache: dict = {}
 
+# 保护 _domain_ua_cache 的"检查-设置"原子性锁。
+# v2.2.0：并发场景下（如 search_multi 并行请求多实例），多个线程可能同时
+# 检查 domain not in cache 并同时写入，虽不致命但会浪费计算且可能写入不同
+# UA（因 hash 本应稳定，但极端时序下逻辑可读性问题）。用锁串行化 dict 读写。
+# 注意：锁内只做 dict 读写，绝不放网络/重计算，避免阻塞其他线程。
+_domain_ua_lock = threading.Lock()
+
 
 def get_ua_for_domain(url: str, user_agent: str = None) -> str:
     """返回适合某域名的 User-Agent。
@@ -188,6 +280,10 @@ def get_ua_for_domain(url: str, user_agent: str = None) -> str:
     设计理由：真实浏览器访问同一站点时 UA 永远不变。爬虫如果每次请求
     换一个 UA，反而会被反爬系统标记为可疑。确定性轮换保证同一域名
     稳定使用同一 UA，不同域名分散到不同 UA 上降低集体封禁风险。
+
+    线程安全：缓存的"检查-设置"用 ``_domain_ua_lock`` 保护。本函数无网络
+    调用，但遵循"锁内只做 dict 读写"原则——hash 计算放在锁外，写入时做
+    双检查（其他线程可能在此期间已写入），兼顾正确性与并发吞吐。
     """
     if user_agent:
         return user_agent
@@ -200,18 +296,27 @@ def get_ua_for_domain(url: str, user_agent: str = None) -> str:
     except Exception:
         return FALLBACK_UAS[0]
 
-    if domain in _domain_ua_cache:
-        return _domain_ua_cache[domain]
+    # 快速路径：锁内检查缓存命中
+    with _domain_ua_lock:
+        if domain in _domain_ua_cache:
+            return _domain_ua_cache[domain]
 
+    # 缓存未命中：在锁外计算 UA（SHA-256 hash，无副作用，不阻塞其他线程）
     idx = _ua_index_for_domain(domain, len(FALLBACK_UAS))
     ua = FALLBACK_UAS[idx]
-    _domain_ua_cache[domain] = ua
+
+    # 加锁写入；双检查避免覆盖其他线程并发写入的值
+    with _domain_ua_lock:
+        if domain in _domain_ua_cache:
+            return _domain_ua_cache[domain]
+        _domain_ua_cache[domain] = ua
     return ua
 
 
 def reset_domain_ua_cache() -> None:
     """清空 per-domain UA 缓存。测试用。"""
-    _domain_ua_cache.clear()
+    with _domain_ua_lock:
+        _domain_ua_cache.clear()
 
 
 def build_browser_headers(user_agent: str, referer: str = None,
@@ -239,11 +344,22 @@ def build_browser_headers(user_agent: str, referer: str = None,
     else:
         accept = "application/json, text/plain, */*;q=0.8"
 
+    # v2.2.1 智能声明 Accept-Encoding：仅当本机安装了 brotli 解压器时
+    # 才声明 br。否则服务器返回 br 压缩字节而 requests 无法解压 → 乱码。
+    # Firefox UA 路径保持只声明 gzip/deflate（与真 Firefox 行为一致，
+    # Firefox 虽支持 br 但为减少指纹差异在此工具中不声明）。
+    if is_firefox:
+        accept_encoding = "gzip, deflate"
+    elif _HAS_BROTLI:
+        accept_encoding = "gzip, deflate, br"
+    else:
+        accept_encoding = "gzip, deflate"
+
     headers = {
         "User-Agent": user_agent,
         "Accept": accept,
         "Accept-Language": "en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7",
-        "Accept-Encoding": "gzip, deflate" if is_firefox else "gzip, deflate, br",
+        "Accept-Encoding": accept_encoding,
         "Connection": "keep-alive",
         "Upgrade-Insecure-Requests": "1" if accept_html else "0",
     }
@@ -588,22 +704,17 @@ RECOVERY_HINTS = {
 }
 
 
-def classify_error(exc: BaseException) -> str:
-    """将异常分类为结构化错误码，供 AI Agent 程序化处理。
+def _classify_by_type_and_status(exc, _json):
+    """根据异常类型和 HTTP 状态码分类，返回错误码或 None。
 
-    分类逻辑（按优先级）：
-      1. 429 → E_RATE_LIMIT
-      2. 401/403 → E_AUTH
-      3. 4xx（非上述）→ E_INPUT（请求参数问题）
-      4. 5xx / URLError / OSError / TimeoutError → E_NETWORK
-      5. json.JSONDecodeError / ValueError → E_PARSE
-      6. FileNotFoundError → E_INPUT
-      7. RuntimeError → 尝试从消息中提取线索，否则 E_INTERNAL
-      8. 其他 → E_INTERNAL
+    检查顺序：HTTP 状态码 → 文件错误 → 连接错误 → 解析错误。
+    返回 None 表示该异常无法靠类型/状态码判定，需走字符串 fallback。
+    不处理 RuntimeError 消息推断（由 :func:`classify_error` 调用方做 fallback）。
+
+    抽取为独立函数，便于 :func:`classify_error` 对 ``exc`` 本身和其
+    ``__cause__`` 复用同一套基于真实类型的判定逻辑。
     """
-    import json as _json
-
-    # HTTP 错误（urllib 和 requests 都有 .code 或 .status_code）
+    # HTTP 错误（urllib HTTPError 用 .code，requests HTTPError 用 .response.status_code）
     status = None
     if isinstance(exc, urllib.error.HTTPError):
         status = exc.code
@@ -635,11 +746,48 @@ def classify_error(exc: BaseException) -> str:
     if isinstance(exc, (ValueError, _json.JSONDecodeError)):
         return E_PARSE
 
+    return None
+
+
+def classify_error(exc: BaseException) -> str:
+    """将异常分类为结构化错误码，供 AI Agent 程序化处理。
+
+    分类逻辑（按优先级）：
+      1. 优先检查异常链 ``__cause__``：``raise X from Y`` 场景下 Y 才是真实
+         错误源（如 ``raise RuntimeError(...) from HTTPError(403)``），
+         用 Y 的类型/状态码分类比从 X 的消息字符串推断更可靠、不再脆弱。
+      2. HTTP 状态码：429→E_RATE_LIMIT, 401/403→E_AUTH, 4xx→E_INPUT, 5xx→E_NETWORK
+      3. 连接类异常（ConnectionError/TimeoutError/URLError/OSError）→ E_NETWORK
+      4. 解析错误（ValueError/JSONDecodeError）→ E_PARSE
+      5. FileNotFoundError → E_INPUT
+      6. RuntimeError：从消息中推断（仅当 ``__cause__`` 缺失时的最后 fallback，
+         兼容旧路径——search_multi 把 last_error 拼进消息）
+      7. 其他 → E_INTERNAL
+    """
+    import json as _json
+
+    # 优先检查异常链 __cause__：raise X from Y 时，Y 指向真实底层异常。
+    # 旧逻辑只能从外层 RuntimeError 的消息字符串推断（脆弱，依赖 "403"/"auth"
+    # 等关键字匹配），新逻辑直接从 __cause__ 的 .code/.status_code 或异常
+    # 类型判定。仅检查一层 __cause__，不递归——单层已覆盖 search_multi 的
+    # raise-from 模式，深层链罕见且递归有循环风险。
+    cause = getattr(exc, "__cause__", None)
+    if cause is not None and cause is not exc:
+        code = _classify_by_type_and_status(cause, _json)
+        if code is not None:
+            return code
+
+    # 检查 exc 本身的类型和状态码
+    code = _classify_by_type_and_status(exc, _json)
+    if code is not None:
+        return code
+
     # RuntimeError：从消息中推断（search_multi 的 "All N instances failed.
     # Last error: HTTP Error 403: Forbidden" 等）。并行路径现在会把 last_error
     # 拼进消息，让这里能提取真实错误类型，而不是一律误判 E_NETWORK。
-    msg = str(exc).lower()
+    # 字符串匹配仅作为 __cause__ 缺失时的最后 fallback。
     if isinstance(exc, RuntimeError):
+        msg = str(exc).lower()
         # 先检查 auth/rate-limit 关键字（最常见，来自 last_error 详情）
         if "auth" in msg or "403" in msg or "401" in msg:
             return E_AUTH
@@ -703,11 +851,15 @@ def emit_progress(event: str, **kwargs) -> None:
 
     仅当 --progress 启用时才输出。``event`` 是事件类型字符串，
     ``kwargs`` 是事件的附加字段。输出格式为 JSON Lines（每行一个 JSON 对象）。
+    v2.2.0：自动注入 request_id（如果已设置）。
     """
     if not _progress_enabled:
         return
     import json as _json
     payload = {"event": event}
+    rid = getattr(_LOG, "_request_id", None)
+    if rid:
+        payload["request_id"] = rid
     payload.update(kwargs)
     print(_json.dumps(payload, ensure_ascii=False), file=sys.stderr, flush=True)
 
@@ -813,3 +965,103 @@ def is_hard_blocked_domain(url: str) -> bool:
                 return True
     return False
 
+
+# ----- 相似度去重（v2.x：让 AI Agent 在研究模式下获得更干净的 merged_results）-----
+# 同一内容在不同 URL/引擎下常重复出现，仅靠 URL 去重无法合并。
+# SimHash 对标题做局部敏感哈希，汉明距离小的视为近似重复。
+
+def _normalize_title(title: str) -> str:
+    """归一化标题：小写、去标点、去多余空格，用于相似度比较。"""
+    if not title:
+        return ""
+    s = title.lower()
+    # 保留字母、数字、CJK 和空格，其余替换为空格
+    s = re.sub(r'[^\w\s]', ' ', s)
+    # \w 包含下划线，单独去掉
+    s = s.replace('_', ' ')
+    s = re.sub(r'\s+', ' ', s).strip()
+    return s
+
+
+def _simhash(text: str, hash_bits: int = 64) -> int:
+    """计算文本的 SimHash 指纹。
+    - 分词（按空格 + CJK 单字符）
+    - 每个 token 算普通 hash，按 bit 投票
+    - 返回 hash_bits 位的指纹
+    """
+    if not text:
+        return 0
+    # 分词：按空格切分，CJK 字符再逐个拆成单字 token
+    tokens = []
+    for word in text.split():
+        buf = []
+        for ch in word:
+            if '\u4e00' <= ch <= '\u9fff':
+                # 遇到 CJK：先冲出缓冲区里的非 CJK 片段，再加入单字
+                if buf:
+                    tokens.append(''.join(buf))
+                    buf = []
+                tokens.append(ch)
+            else:
+                buf.append(ch)
+        if buf:
+            tokens.append(''.join(buf))
+    if not tokens:
+        return 0
+    # 每个 token 算 SHA-256（跨进程可复现，避免 hash() 随机化），按 bit 投票
+    v = [0] * hash_bits
+    for token in tokens:
+        h = hashlib.sha256(token.encode('utf-8')).digest()
+        token_hash = int.from_bytes(h[:8], 'big')
+        for i in range(hash_bits):
+            if (token_hash >> i) & 1:
+                v[i] += 1
+            else:
+                v[i] -= 1
+    # 投票为正的位置 1
+    fingerprint = 0
+    for i in range(hash_bits):
+        if v[i] > 0:
+            fingerprint |= (1 << i)
+    return fingerprint
+
+
+def _hamming_distance(a: int, b: int) -> int:
+    """两个整数的汉明距离。"""
+    return bin(a ^ b).count('1')
+
+
+def _jaccard_similarity(set_a: set, set_b: set) -> float:
+    """Jaccard 相似度。"""
+    if not set_a and not set_b:
+        return 0.0
+    union = set_a | set_b
+    if not union:
+        return 0.0
+    return len(set_a & set_b) / len(union)
+
+
+def is_similar(result_a: dict, result_b: dict, threshold: float = 0.85) -> bool:
+    """判断两个搜索结果是否相似。
+    - 优先用标题 SimHash（汉明距离 <= 3 视为相似，对应 64 位中约 95% 相似）
+    - 标题太短（< 5 字符）时用 URL 域名 + 标题 Jaccard
+    - threshold 参数控制严格程度
+    """
+    title_a = _normalize_title(result_a.get("title", ""))
+    title_b = _normalize_title(result_b.get("title", ""))
+    # 标题太短时 SimHash 不稳定，改用 Jaccard
+    if len(title_a) < 5 or len(title_b) < 5:
+        import urllib.parse as _up
+        domain_a = _up.urlparse(result_a.get("url", "")).netloc.lower()
+        domain_b = _up.urlparse(result_b.get("url", "")).netloc.lower()
+        set_a = set(title_a.split()) | {domain_a}
+        set_b = set(title_b.split()) | {domain_b}
+        return _jaccard_similarity(set_a, set_b) >= threshold
+    # 标题足够长：用 SimHash 汉明距离
+    hash_a = _simhash(title_a)
+    hash_b = _simhash(title_b)
+    # threshold → 汉明距离阈值映射：
+    #   0.85 → 3（默认，宽松），0.90 → 2，0.95 → 1，1.0 → 0（几乎完全相同）
+    max_distance = max(0, min(3, int(round((1.0 - threshold) / 0.05))))
+    return _hamming_distance(hash_a, hash_b) <= max_distance
+

scripts/fetch.py

@@ -23,6 +23,21 @@ from html.parser import HTMLParser
 from pathlib import Path
 from typing import Optional
 
+# Brotli 解压支持检测（v2.2.1）。
+# requests 自动解压 gzip/deflate，但**不自动解压 Brotli**（除非安装
+# brotli/brotlicffi）。build_browser_headers() 已根据此检测智能声明
+# Accept-Encoding，此处作为双保险：若代理/CDN 强制返回 br，仍可解压。
+try:
+    import brotli as _brotli  # type: ignore
+    _HAS_BROTLI = True
+except ImportError:
+    try:
+        import brotlicffi as _brotli  # type: ignore
+        _HAS_BROTLI = True
+    except ImportError:
+        _brotli = None
+        _HAS_BROTLI = False
+
 # Allow running standalone from any working directory
 sys.path.insert(0, str(Path(__file__).resolve().parent))
 from _config import USER_AGENT, VERSION
@@ -50,6 +65,11 @@ from common import (
 logger = logging.getLogger("searxng.fetch")
 
 
+# 错误码：不支持的媒体类型（PDF/DOCX/XLSX 解析失败或未知二进制类型）。
+# 与 common.py 中 E_CONFIG / E_AUTH / E_NETWORK 等错误码保持一致的 E_* 命名模式。
+E_UNSUPPORTED_MEDIA = "E_UNSUPPORTED_MEDIA"
+
+
 # ----- Auth helpers -----
 # build_auth_headers is imported from common.py
 
@@ -167,6 +187,37 @@ def extract_with_bs4(html_content: str) -> str:
     return text
 
 
+# CJK 字符范围：中文 \u4e00-\u9fff、日文 \u3040-\u30ff、韩文 \uac00-\ud7af
+_CJK_CHAR_RE = re.compile(
+    r"[\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]"
+)
+
+
+def _is_cjk_text(text: str) -> bool:
+    """判断文本是否以 CJK（中文/日文/韩文）为主。
+
+    统计 CJK 字符占非空白字符的比例，>30% 则视为 CJK 内容。
+    CJK 文本信息密度高，readability-lite 的最小字符阈值应相应降低。
+    """
+    if not text:
+        return False
+    # 按非空白字符统计，避免大量空白/缩进拉低比例造成误判
+    non_ws_len = sum(1 for ch in text if not ch.isspace())
+    if non_ws_len == 0:
+        return False
+    cjk_count = len(_CJK_CHAR_RE.findall(text))
+    return cjk_count / non_ws_len > 0.30
+
+
+def _min_content_length(text: str) -> int:
+    """根据文本语言返回 readability-lite 最小正文字符阈值。
+
+    CJK 内容（信息密度高）：100 字符
+    其他语言（英文等）：200 字符
+    """
+    return 100 if _is_cjk_text(text) else 200
+
+
 def _readability_lite(root) -> "Optional[object]":
     """readability-lite：用文本密度算法选择最可能是正文的子元素。
 
@@ -174,7 +225,7 @@ def _readability_lite(root) -> "Optional[object]":
       1. 遍历 body 下所有 div/section/article 子节点
       2. 计算每个节点的"文本密度" = 纯文本字符数 / 标签数
       3. 排除明显是导航/侧边栏的节点（class/id 含 nav/sidebar/menu/footer）
-      4. 返回文本密度最高且字符数 > 200 的节点
+      4. 返回文本密度最高且字符数超过阈值的节点（CJK 100，其他 200）
 
     返回 bs4 Tag 或 None（找不到合适节点时）。
     这是 extract_with_bs4 的兜底增强，不改变原有 article/main 优先级。
@@ -204,8 +255,10 @@ def _readability_lite(root) -> "Optional[object]":
         # 计算纯文本字符数（去空白）
         text = node.get_text(separator=" ", strip=True)
         text_len = len(text)
-        if text_len < 200:
-            continue  # 正文至少 200 字符
+        # 阈值按语言动态调整：CJK 内容 100 字符，其他 200 字符
+        min_len = _min_content_length(text)
+        if text_len < min_len:
+            continue  # 正文至少 min_len 字符（CJK 100，其他 200）
 
         # 计算标签数（粗略：所有后代标签）
         tag_count = len(node.find_all())
@@ -602,10 +655,225 @@ class _NoRedirectHandler(urllib.request.HTTPRedirectHandler):
 
 FetchResult = namedtuple(
     "FetchResult",
-    ["content", "content_type", "final_url", "truncated", "user_agent"],
+    ["content", "content_type", "final_url", "truncated", "user_agent",
+     "error_code", "error_message"],
+    # error_code / error_message 默认 None：
+    #   * 向后兼容——旧的 5 参数构造（位置或关键字）仍然可用
+    #   * 仅当 fetch_url 遇到不支持的媒体类型时才填充
+    defaults=[None, None],
 )
 
 
+def _handle_rate_limit_status(status_code, headers, attempt, max_retries):
+    """处理 429/503 限流响应的 Retry-After，决定是否重试。
+
+    统一抽取自 requests 路径、stdlib 路径、requests.exceptions 路径三处
+    原本重复的 Retry-After 解析 + 退避 sleep 逻辑。
+
+    解析 Retry-After 头（秒数或 HTTP 日期，委托给 common.parse_retry_after），
+    与 compute_backoff_delay(attempt) 取较大值作为实际等待时间。
+    若 ``attempt < max_retries``，sleep 后返回 ``(True, retry_after_sec)``
+    表示应当重试；否则返回 ``(False, retry_after_sec)``，由调用方决定后续
+    （通常会落到 raise_for_status / 抛 RuntimeError）。
+
+    ``headers`` 兼容 dict 和 http.client.HTTPMessage（均支持 ``.get()``）；
+    为 None 时按空 header 处理（retry_after=0）。
+
+    返回 ``(should_retry, retry_after_seconds)``。
+    """
+    retry_after_raw = ""
+    if headers:
+        retry_after_raw = headers.get("Retry-After", "") or ""
+    retry_after_sec = parse_retry_after(retry_after_raw)
+    if attempt < max_retries:
+        delay = max(retry_after_sec, compute_backoff_delay(attempt))
+        logger.info(f"  Fetch retry {attempt+1}/{max_retries} "
+                    f"(HTTP {status_code}, Retry-After={retry_after_sec:.1f}s) "
+                    f"in {delay:.1f}s")
+        time.sleep(delay)
+        return (True, retry_after_sec)
+    return (False, retry_after_sec)
+
+
+# 明确的非文本二进制 MIME 类型：无法作为文本 decode，直接拒绝。
+# application/octet-stream 是通用二进制兜底类型；其余为已知归档/可执行/
+# 旧版 Office（.doc/.xls/.ppt 不在本次支持范围）等。
+_BINARY_CONTENT_TYPES = frozenset([
+    "application/octet-stream",
+    "application/zip",
+    "application/x-gzip",
+    "application/gzip",
+    "application/x-rar-compressed",
+    "application/x-7z-compressed",
+    "application/x-tar",
+    "application/x-bzip",
+    "application/x-bzip2",
+    "application/x-msdownload",
+    "application/x-shockwave-flash",
+    "application/msword",                       # 旧 .doc（不支持）
+    "application/vnd.ms-excel",                 # 旧 .xls（不支持）
+    "application/vnd.ms-powerpoint",            # 旧 .ppt（不支持）
+    "application/x-elf",
+    "application/x-executable",
+])
+
+
+# OOXML（.docx / .xlsx）主命名空间，ElementTree 用 {ns}tag 形式匹配
+_W_NS = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}"
+_S_NS = "{http://schemas.openxmlformats.org/spreadsheetml/2006/main}"
+
+
+def _parse_pdf(raw: bytes):
+    """用 pdftotext（poppler-utils）从 PDF 字节流提取文本。
+
+    通过 subprocess 调用 ``pdftotext - -``（stdin 读、stdout 写），
+    不引入新依赖。pdftotext 不存在或失败时返回 E_UNSUPPORTED_MEDIA。
+
+    返回 ``(content, error_code, error_message)``：
+      成功 → ``(text, None, None)``；失败 → ``(None, E_UNSUPPORTED_MEDIA, msg)``。
+    """
+    try:
+        proc = subprocess.run(
+            ["pdftotext", "-", "-"],
+            input=raw,
+            capture_output=True,
+            timeout=30,
+        )
+    except FileNotFoundError:
+        return (None, E_UNSUPPORTED_MEDIA,
+                "PDF parsing requires poppler-utils (pdftotext) to be installed")
+    except subprocess.TimeoutExpired:
+        return (None, E_UNSUPPORTED_MEDIA, "PDF parsing timed out (>30s)")
+    except OSError as e:
+        return (None, E_UNSUPPORTED_MEDIA, f"PDF parsing failed: {e}")
+
+    if proc.returncode != 0:
+        stderr = proc.stderr.decode("utf-8", errors="replace").strip()
+        msg = f"pdftotext exited {proc.returncode}"
+        if stderr:
+            msg += f": {stderr[:200]}"
+        return (None, E_UNSUPPORTED_MEDIA, msg)
+
+    text = proc.stdout.decode("utf-8", errors="replace")
+    return (text, None, None)
+
+
+def _parse_docx(raw: bytes):
+    """从 .docx 字节流提取文本（stdlib zipfile + ElementTree）。
+
+    读取 ``word/document.xml``，按段落（<w:p>）提取 <w:t> 文本，
+    段落间以换行分隔。失败时返回 E_UNSUPPORTED_MEDIA。
+
+    返回 ``(content, error_code, error_message)``。
+    """
+    try:
+        with zipfile.ZipFile(io.BytesIO(raw)) as zf:
+            xml_bytes = zf.read("word/document.xml")
+    except (zipfile.BadZipFile, KeyError) as e:
+        return (None, E_UNSUPPORTED_MEDIA, f"DOCX parsing failed: {e}")
+
+    try:
+        root = ET.fromstring(xml_bytes)
+    except ET.ParseError as e:
+        return (None, E_UNSUPPORTED_MEDIA, f"DOCX XML parse failed: {e}")
+
+    # 遍历段落，每段内拼接所有 <w:t>，段落间换行
+    lines = []
+    for p in root.iter(_W_NS + "p"):
+        parts = [t.text for t in p.iter(_W_NS + "t") if t.text]
+        if parts:
+            lines.append("".join(parts))
+    return ("\n".join(lines), None, None)
+
+
+def _parse_xlsx(raw: bytes):
+    """从 .xlsx 字节流提取文本（stdlib zipfile + ElementTree）。
+
+    读取 ``xl/sharedStrings.xml``（共享字符串表）与各
+    ``xl/worksheets/sheetN.xml``，按行提取单元格文本，单元格以制表符
+    分隔、行以换行分隔。失败时返回 E_UNSUPPORTED_MEDIA。
+
+    返回 ``(content, error_code, error_message)``。
+    """
+    try:
+        with zipfile.ZipFile(io.BytesIO(raw)) as zf:
+            # 共享字符串表（可能不存在——纯数字表格）
+            shared = []
+            try:
+                sroot = ET.fromstring(zf.read("xl/sharedStrings.xml"))
+                for si in sroot.iter(_S_NS + "si"):
+                    parts = [t.text for t in si.iter(_S_NS + "t") if t.text]
+                    shared.append("".join(parts))
+            except (KeyError, ET.ParseError):
+                pass  # 无共享字符串表，单元格均为内联值
+
+            sheet_names = [n for n in zf.namelist()
+                           if re.match(r"xl/worksheets/sheet\d+\.xml$", n)]
+            lines = []
+            for sheet_name in sorted(sheet_names):
+                try:
+                    sroot = ET.fromstring(zf.read(sheet_name))
+                except ET.ParseError:
+                    continue
+                for row in sroot.iter(_S_NS + "row"):
+                    cells = []
+                    for c in row.iter(_S_NS + "c"):
+                        cell_type = c.get("t")
+                        v = c.find(_S_NS + "v")
+                        if v is not None and v.text is not None:
+                            if cell_type == "s":
+                                # 共享字符串索引引用
+                                try:
+                                    idx = int(v.text)
+                                    cells.append(
+                                        shared[idx] if 0 <= idx < len(shared) else "")
+                                except (ValueError, IndexError):
+                                    cells.append("")
+                            else:
+                                cells.append(v.text)
+                        else:
+                            # 内联字符串 <is><t>...</t></is>
+                            is_el = c.find(_S_NS + "is")
+                            if is_el is not None:
+                                parts = [t.text for t in is_el.iter(_S_NS + "t")
+                                         if t.text]
+                                cells.append("".join(parts))
+                    if cells:
+                        lines.append("\t".join(cells))
+            return ("\n".join(lines), None, None)
+    except (zipfile.BadZipFile, ET.ParseError) as e:
+        return (None, E_UNSUPPORTED_MEDIA, f"XLSX parsing failed: {e}")
+
+
+def _parse_document_content(raw: bytes, content_type: str):
+    """根据 Content-Type 将二进制文档解析为文本。
+
+    支持：PDF（需 pdftotext）、DOCX、XLSX。
+    对明确的非文本二进制 MIME（application/octet-stream、zip、rar 等）返回
+    E_UNSUPPORTED_MEDIA。其他类型（text/* 、application/json、HTML 等）返回
+    ``(None, None, None)``，由调用方走原有 decode 流程。
+
+    返回 ``(content, error_code, error_message)``：
+      * 非文档类型 → ``(None, None, None)``：调用方继续 decode
+      * 解析成功   → ``(text, None, None)``
+      * 解析失败   → ``(None, E_UNSUPPORTED_MEDIA, msg)``
+    """
+    ct = (content_type or "").lower().split(";")[0].strip()
+
+    if ct == "application/pdf":
+        return _parse_pdf(raw)
+    if ct == "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
+        return _parse_docx(raw)
+    if ct == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet":
+        return _parse_xlsx(raw)
+
+    if ct in _BINARY_CONTENT_TYPES:
+        return (None, E_UNSUPPORTED_MEDIA,
+                f"Unsupported binary content type: {ct}")
+
+    return (None, None, None)
+
+
 def fetch_url(url: str, timeout=15, user_agent: str = None,
               encoding: str = None, auth_headers: dict = None,
               max_retries: int = 3, max_size: int = None,
@@ -669,16 +937,11 @@ def fetch_url(url: str, timeout=15, user_agent: str = None,
                 try:
                     # 429/503：读取 Retry-After，作为最小重试延迟
                     if resp.status_code in (429, 503) and attempt < max_retries:
-                        retry_after_raw = resp.headers.get("Retry-After", "")
-                        retry_after_sec = parse_retry_after(retry_after_raw)
                         resp.close()
-                        delay = max(retry_after_sec,
-                                    compute_backoff_delay(attempt))
-                        logger.info(f"  Fetch retry {attempt+1}/{max_retries} "
-                                    f"(HTTP {resp.status_code}, Retry-After={retry_after_sec:.1f}s) "
-                                    f"in {delay:.1f}s")
-                        time.sleep(delay)
-                        continue
+                        should_retry, _ = _handle_rate_limit_status(
+                            resp.status_code, resp.headers, attempt, max_retries)
+                        if should_retry:
+                            continue
 
                     resp.raise_for_status()
 
@@ -698,6 +961,33 @@ def fetch_url(url: str, timeout=15, user_agent: str = None,
                         raw = b"".join(chunks)
                         truncated = total > max_size
 
+                    # v2.2.1 修复：requests 自动解压 gzip/deflate，但**不自动
+                    # 解压 Brotli**（除非安装 brotli 包）。当服务器返回
+                    # Content-Encoding: br 而本机有 brotli 解压器时，手动解压；
+                    # 否则保留原 raw，让下游 errors="replace" 兜底（虽是乱码但
+                    # 不崩溃）。build_browser_headers() 已尽量避免声明 br，此处
+                    # 作为双保险，应对代理/CDN 强制返回 br 的边缘情况。
+                    content_encoding = (resp.headers.get("Content-Encoding", "")
+                                        .lower().strip())
+                    if "br" in content_encoding and _HAS_BROTLI and raw:
+                        try:
+                            raw = _brotli.decompress(raw)
+                        except Exception as e:
+                            logger.debug(f"  brotli decompress failed: {e}")
+
+                    # 二进制文档解析（PDF/DOCX/XLSX）及不支持的媒体类型检测。
+                    # 命中时直接返回，跳过后续文本 decode 流程。
+                    doc_text, doc_err_code, doc_err_msg = _parse_document_content(
+                        raw, resp.headers.get("Content-Type", ""))
+                    if doc_err_code is not None:
+                        return FetchResult(
+                            "", resp.headers.get("Content-Type", ""), resp.url,
+                            truncated, ua, doc_err_code, doc_err_msg)
+                    if doc_text is not None:
+                        return FetchResult(
+                            doc_text, resp.headers.get("Content-Type", ""), resp.url,
+                            truncated, ua, None, None)
+
                     if encoding:
                         content = raw.decode(encoding)
                     else:
@@ -745,11 +1035,14 @@ def fetch_url(url: str, timeout=15, user_agent: str = None,
                 # requests 库会自动处理 Content-Encoding，但 stdlib 不会。
                 # 此前该 bug 被沙箱伪响应掩盖（两者都产生 U+FFFD），
                 # 实际在无 requests 的真实环境中会复现。
+                # v2.2.1 补充：br 解压（与 requests 路径对齐）。
                 content_encoding = (resp.headers.get("Content-Encoding", "")
                                     .lower().strip())
                 if content_encoding and raw:
                     try:
-                        if "gzip" in content_encoding:
+                        if "br" in content_encoding and _HAS_BROTLI:
+                            raw = _brotli.decompress(raw)
+                        elif "gzip" in content_encoding:
                             raw = gzip.decompress(raw)
                         elif "deflate" in content_encoding:
                             # deflate 可能是 zlib 包装或裸 deflate
@@ -761,6 +1054,19 @@ def fetch_url(url: str, timeout=15, user_agent: str = None,
                         logger.debug(f"  decompress failed ({content_encoding}): {e}")
                         # 解压失败保留原 raw，让下游 decode 兜底
 
+                # 二进制文档解析（PDF/DOCX/XLSX）及不支持的媒体类型检测。
+                # 命中时直接返回，跳过后续文本 decode 流程。
+                doc_text, doc_err_code, doc_err_msg = _parse_document_content(
+                    raw, content_type)
+                if doc_err_code is not None:
+                    return FetchResult(
+                        "", content_type, final_url, truncated, ua,
+                        doc_err_code, doc_err_msg)
+                if doc_text is not None:
+                    return FetchResult(
+                        doc_text, content_type, final_url, truncated, ua,
+                        None, None)
+
                 if encoding:
                     charset = encoding
                 else:
@@ -776,16 +1082,12 @@ def fetch_url(url: str, timeout=15, user_agent: str = None,
             last_error = e
             # 429/503：读取 Retry-After（stdlib 路径）
             if e.code in (429, 503) and attempt < max_retries:
-                retry_after_raw = e.headers.get("Retry-After", "") if e.headers else ""
-                retry_after_sec = parse_retry_after(retry_after_raw)
                 # HTTPError 本身是可读的响应对象（fp 已被 urllib 消费），
                 # 无需显式 close；直接进入退避。
-                delay = max(retry_after_sec, compute_backoff_delay(attempt))
-                logger.info(f"  Fetch retry {attempt+1}/{max_retries} "
-                            f"(HTTP {e.code}, Retry-After={retry_after_sec:.1f}s) "
-                            f"in {delay:.1f}s")
-                time.sleep(delay)
-                continue
+                should_retry, _ = _handle_rate_limit_status(
+                    e.code, e.headers, attempt, max_retries)
+                if should_retry:
+                    continue
             if is_retryable_error(e) and attempt < max_retries:
                 delay = compute_backoff_delay(attempt)
                 logger.info(f"  Fetch retry {attempt+1}/{max_retries} ({e}) in {delay:.1f}s")
@@ -809,18 +1111,13 @@ def fetch_url(url: str, timeout=15, user_agent: str = None,
                 # 429/503 with Retry-After
                 if status in (429, 503) and attempt < max_retries:
                     resp_obj = getattr(e, "response", None)
-                    retry_after_raw = ""
-                    if resp_obj is not None:
-                        retry_after_raw = resp_obj.headers.get("Retry-After", "")
-                    retry_after_sec = parse_retry_after(retry_after_raw)
+                    resp_headers = resp_obj.headers if resp_obj is not None else None
                     if resp_obj is not None:
                         resp_obj.close()
-                    delay = max(retry_after_sec, compute_backoff_delay(attempt))
-                    logger.info(f"  Fetch retry {attempt+1}/{max_retries} "
-                                f"(HTTP {status}, Retry-After={retry_after_sec:.1f}s) "
-                                f"in {delay:.1f}s")
-                    time.sleep(delay)
-                    continue
+                    should_retry, _ = _handle_rate_limit_status(
+                        status, resp_headers, attempt, max_retries)
+                    if should_retry:
+                        continue
                 if (status is None or status in RETRYABLE_STATUS) and attempt < max_retries:
                     delay = compute_backoff_delay(attempt)
                     logger.info(f"  Fetch retry {attempt+1}/{max_retries} ({e}) in {delay:.1f}s")