feat(v2.3.0): fetch 结构化 JSON 契约 + 并发批量 + 真实并发门控

迭代 1 — 正确性修复: - 修复 --pages N 多页聚合的 unresponsive-engine 警告误判: 原用循环末次 cached 变量判断, 缓存命中时警告被错误跳过/误触发; 改用独立 performed_live_query 标记 - UA 池单一来源: 删除 common.py 手工副本 _FALLBACK_UAS_BUILTIN, FALLBACK_UAS 直接引用 _config.UA_POOL, 消除双份漂移 - search_html 解码修复: 硬编码 utf-8 改为 detect_charset(header/meta 自动检测), 新增 --encoding 强制覆盖, 贯穿 search_multi 全链 - AdaptiveThrottle 真实并发门控: acquire_slot()/release_slot() 槽位机制, 退避降并发后新请求被快速拒绝(E_RATE_LIMIT), 实现持久降并发而非名义降并发迭代 2 — fetch JSON 契约 + 批量并发: - fetch.py --format json: 成功 {status,url,final_url,content_type,extract, truncated,text_length,user_agent}; 失败 {status,error,error_code, status_code,url}, 对齐 search.py 错误码体系 - fetch_page 采集 title + latency, 填充 --fetch-report json 空字段 - --queries-file --parallel-queries N (1-8): 并发批量, 输出保序, 受 AdaptiveThrottle 门控; 并发模式禁用 --fetch(嵌套并行不安全) - queries 文件编码自动检测 (UTF-8 → GBK 回退) 迭代 3 — 工程化: - 新增 pyproject.toml (searxng-search/searxng-fetch 入口点) - 收敛 20+ 处函数内冗余导入 - --dump-schema 扩展: fetched.items 补全 15 字段, 新增 defs.batch/research - 新增 17 个测试 (tests/test_v230_features.py), 全量 561 测试通过 - 文档同步 (SKILL.md/README.md, 版本号 2.3.0)
2026-08-05 20:14:07 +08:00
parent 6cedba9042
commit 4df521dc9d
8 changed files with 1006 additions and 226 deletions
@@ -18,13 +18,20 @@ as retryable and connection errors as transient, eliminating the previous
 inconsistency where ``search.py`` ignored 5xx.
 """

+import base64
+import datetime
 import hashlib
 import io
+import json
 import logging
+import os
+import random
 import re
 import sys
 import threading
 import urllib.error
+import urllib.parse
+from pathlib import Path

 # Root logger for the searxng-cli package. All modules create child loggers
 # via ``logging.getLogger("searxng.<module>")`` so a single setup_logging()
@@ -105,7 +112,6 @@ class _JsonFormatter(logging.Formatter):
    """

    def format(self, record):
-        import json as _json
        entry = {
            "ts": _datetime_iso(record),
            "level": record.levelname,
@@ -117,13 +123,12 @@ class _JsonFormatter(logging.Formatter):
            entry["request_id"] = rid
        if record.exc_info and record.exc_info[1]:
            entry["exception"] = type(record.exc_info[1]).__name__
-        return _json.dumps(entry, ensure_ascii=False)
+        return json.dumps(entry, ensure_ascii=False)


 def _datetime_iso(record):
    """格式化日志时间戳为 ISO 8601 字符串。"""
-    import datetime as _dt
-    return _dt.datetime.fromtimestamp(record.created).isoformat(timespec="milliseconds")
+    return datetime.datetime.fromtimestamp(record.created).isoformat(timespec="milliseconds")


 def force_utf8_stdout() -> None:
@@ -191,57 +196,17 @@ RETRYABLE_STATUS = frozenset({403, 429, 502, 503, 504})

 # Browser-like UA strings for fallback when the searxng-cli UA is blocked.
 #
-# v2.2.0：UA 池迁移至 _config.py 的 UA_POOL（SSOT），此处通过导入引用。
-# 若 _config.py 不可导入（如 common.py 被单独分发），回退到下方内置副本
-# _FALLBACK_UAS_BUILTIN，保证模块始终可用。两份列表需保持同步，
-# _config.UA_POOL 为唯一权威来源。
+# v2.2.2：单一来源（SSOT）。UA 池唯一权威定义在 _config.UA_POOL，
+# 此处直接导入——删除原有的内置副本 _FALLBACK_UAS_BUILTIN。
+# v2.2.0 曾保留一份手工同步副本，两份列表漂移会导致跨脚本 UA 行为不一致
+# （fetch.py 用 FALLBACK_UAS 轮换、search.py 的 --dry-run 报告引用同一池），
+# 且注释要求"保持同步"无任何机制保证。直接引用同一对象后，改一处即全局生效。
 #
-# v2.0.0 起覆盖 Chrome/Edge/Firefox × Windows/macOS/Linux；v2.2.0 升级到
-# 2026 年版本（Chrome 138-140 / Edge 138 / Firefox 140 / Safari 18）。
-# 顺序固定以便 get_ua_for_domain() 的 hash 选择可复现。
-_FALLBACK_UAS_BUILTIN = [
-    # Chrome 140 — Windows / macOS / Linux
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
-    "(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
-    "(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
-    "(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
-    # Chrome 139 — Windows / macOS / Linux
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
-    "(KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
-    "(KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
-    "(KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
-    # Chrome 138 — Windows / macOS / Linux
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
-    "(KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
-    "(KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
-    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
-    "(KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
-    # Edge 138 — Windows / macOS
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
-    "(KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36 Edg/138.0.0.0",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
-    "(KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36 Edg/138.0.0.0",
-    # Firefox 140 — Windows / macOS / Linux
-    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) "
-    "Gecko/20100101 Firefox/140.0",
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) "
-    "Gecko/20100101 Firefox/140.0",
-    "Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0",
-    # Safari 18 — macOS（WebKit 指纹，应对 Chromium 针对性拦截）
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 "
-    "(KHTML, like Gecko) Version/18.0 Safari/605.1.15",
-]
-
-# 从 _config.py 导入权威 UA_POOL；导入失败时回退到内置副本，保证向后兼容。
-try:
-    from _config import UA_POOL as FALLBACK_UAS
-except ImportError:
-    FALLBACK_UAS = _FALLBACK_UAS_BUILTIN
+# 维护原则（见 _config.UA_POOL 注释）：
+#   1. 版本号保持为当前年份的主流浏览器版本
+#   2. 顺序固定——get_ua_for_domain() 用 SHA-256 哈希选索引
+#   3. 至少覆盖 Chrome/Edge/Firefox × Windows/macOS/Linux
+from _config import UA_POOL as FALLBACK_UAS


 def _ua_index_for_domain(domain: str, pool_size: int) -> int:
@@ -251,7 +216,6 @@ def _ua_index_for_domain(domain: str, pool_size: int) -> int:
    随机化（PYTHONHASHSEED），跨进程不可复现。SHA-256 保证同一域名
    永远映射到同一索引，跨进程一致——这对调试和日志分析至关重要。
    """
-    import hashlib
    h = hashlib.sha256(domain.encode("utf-8")).digest()
    # 取前 8 字节作为无符号整数，避免负数和短字符串的分布不均
    return int.from_bytes(h[:8], "big") % pool_size
@@ -288,9 +252,8 @@ def get_ua_for_domain(url: str, user_agent: str = None) -> str:
    if user_agent:
        return user_agent

-    import urllib.parse as _up
    try:
-        domain = _up.urlparse(url).netloc.lower()
+        domain = urllib.parse.urlparse(url).netloc.lower()
        if not domain:
            return FALLBACK_UAS[0]
    except Exception:
@@ -367,7 +330,6 @@ def build_browser_headers(user_agent: str, referer: str = None,
    # Sec-Ch-Ua 系列仅 Chrome/Edge 发送，Firefox 不发
    if not is_firefox:
        # 从 UA 提取主版本号，构造 Sec-Ch-Ua
-        import re
        m = re.search(r"Chrome/(\d+)", user_agent)
        ver = m.group(1) if m else "131"
        not_a_brand = '"Not_A Brand";v="99"' if ver != "99" else '"Not/A)Brand";v="99"'
@@ -445,7 +407,6 @@ def compute_backoff_delay(attempt: int, base: float = RETRY_BACKOFF_BASE,
    ``base * 2^attempt + jitter``，但不超过 ``cap``。
    v2.0.0 新增封顶：原公式无上限，N=10 时达 1536s 会卡死进程。
    """
-    import random
    delay = base * (2 ** attempt) + random.uniform(0, 1)
    return min(delay, cap)

@@ -459,8 +420,6 @@ def build_auth_headers(bearer_token: str = None, basic_auth: str = None) -> dict
    If both are provided, Bearer takes precedence (more common for APIs).
    Returns a dict to merge into request headers, or an empty dict.
    """
-    import base64
-
    headers = {}
    if bearer_token:
        headers["Authorization"] = f"Bearer {bearer_token}"
@@ -476,7 +435,6 @@ def _warn_file_perms(path: str) -> None:
    On Windows the Unix permission bits in ``st_mode`` do not reflect the
    actual ACL, so the check is skipped to avoid false alarms.
    """
-    import os
    if os.name != "posix":
        return
    log = logging.getLogger("searxng.common")
@@ -513,7 +471,6 @@ def resolve_auth_basic(cli_value: str = None, file_path: str = None,

    if file_path:
        try:
-            from pathlib import Path
            text = Path(file_path).read_text(encoding="utf-8")
            _warn_file_perms(file_path)
            for line in text.splitlines():
@@ -527,7 +484,6 @@ def resolve_auth_basic(cli_value: str = None, file_path: str = None,
    if config_value:
        return config_value

-    import os
    return os.environ.get(env_var)


@@ -550,7 +506,6 @@ def resolve_auth_bearer(cli_value: str = None, file_path: str = None,

    if file_path:
        try:
-            from pathlib import Path
            text = Path(file_path).read_text(encoding="utf-8")
            _warn_file_perms(file_path)
            for line in text.splitlines():
@@ -564,7 +519,6 @@ def resolve_auth_bearer(cli_value: str = None, file_path: str = None,
    if config_value:
        return config_value

-    import os
    return os.environ.get(env_var)


@@ -583,7 +537,6 @@ def apply_proxy(proxy_url: str) -> None:
    Pass an empty string to clear the proxy env vars (rarely needed; the
    default unset state already means "no proxy").
    """
-    import os
    if not proxy_url:
        return
    os.environ["HTTP_PROXY"] = proxy_url
@@ -597,8 +550,6 @@ def detect_charset(raw: bytes, content_type: str) -> str:

    Falls back to UTF-8 (with replacement) if nothing reliable is found.
    """
-    import re
-
    # 1. HTTP header
    if "charset=" in content_type:
        charset = content_type.split("charset=")[-1].split(";")[0].strip()
@@ -704,7 +655,7 @@ RECOVERY_HINTS = {
 }


-def _classify_by_type_and_status(exc, _json):
+def _classify_by_type_and_status(exc) -> str:
    """根据异常类型和 HTTP 状态码分类，返回错误码或 None。

    检查顺序：HTTP 状态码 → 文件错误 → 连接错误 → 解析错误。
@@ -743,7 +694,7 @@ def _classify_by_type_and_status(exc, _json):
        return E_NETWORK

    # 解析错误
-    if isinstance(exc, (ValueError, _json.JSONDecodeError)):
+    if isinstance(exc, (ValueError, json.JSONDecodeError)):
        return E_PARSE

    return None
@@ -764,8 +715,6 @@ def classify_error(exc: BaseException) -> str:
         兼容旧路径——search_multi 把 last_error 拼进消息）
      7. 其他 → E_INTERNAL
    """
-    import json as _json
-
    # 优先检查异常链 __cause__：raise X from Y 时，Y 指向真实底层异常。
    # 旧逻辑只能从外层 RuntimeError 的消息字符串推断（脆弱，依赖 "403"/"auth"
    # 等关键字匹配），新逻辑直接从 __cause__ 的 .code/.status_code 或异常
@@ -773,12 +722,12 @@ def classify_error(exc: BaseException) -> str:
    # raise-from 模式，深层链罕见且递归有循环风险。
    cause = getattr(exc, "__cause__", None)
    if cause is not None and cause is not exc:
-        code = _classify_by_type_and_status(cause, _json)
+        code = _classify_by_type_and_status(cause)
        if code is not None:
            return code

    # 检查 exc 本身的类型和状态码
-    code = _classify_by_type_and_status(exc, _json)
+    code = _classify_by_type_and_status(exc)
    if code is not None:
        return code

@@ -795,8 +744,7 @@ def classify_error(exc: BaseException) -> str:
            return E_RATE_LIMIT
        # 从 "http error NNN" / "http NNN" 模式中提取状态码，
        # 正确分类 "All instances failed. Last error: HTTP Error 403" 等
-        import re as _re
-        status_match = _re.search(r'http(?: error)? (\d{3})', msg)
+        status_match = re.search(r'http(?: error)? (\d{3})', msg)
        if status_match:
            status = int(status_match.group(1))
            if status == 429:
@@ -855,13 +803,12 @@ def emit_progress(event: str, **kwargs) -> None:
    """
    if not _progress_enabled:
        return
-    import json as _json
    payload = {"event": event}
    rid = getattr(_LOG, "_request_id", None)
    if rid:
        payload["request_id"] = rid
    payload.update(kwargs)
-    print(_json.dumps(payload, ensure_ascii=False), file=sys.stderr, flush=True)
+    print(json.dumps(payload, ensure_ascii=False), file=sys.stderr, flush=True)


 # ----- Wayback Machine 兜底（v2.1.0 共享逻辑）-----
@@ -945,8 +892,7 @@ def is_hard_blocked_domain(url: str) -> bool:
        return False
    # 提取域名
    try:
-        from urllib.parse import urlparse
-        host = urlparse(url).hostname or ""
+        host = urllib.parse.urlparse(url).hostname or ""
    except Exception:
        host = ""
    if not host:
@@ -1051,9 +997,8 @@ def is_similar(result_a: dict, result_b: dict, threshold: float = 0.85) -> bool:
    title_b = _normalize_title(result_b.get("title", ""))
    # 标题太短时 SimHash 不稳定，改用 Jaccard
    if len(title_a) < 5 or len(title_b) < 5:
-        import urllib.parse as _up
-        domain_a = _up.urlparse(result_a.get("url", "")).netloc.lower()
-        domain_b = _up.urlparse(result_b.get("url", "")).netloc.lower()
+        domain_a = urllib.parse.urlparse(result_a.get("url", "")).netloc.lower()
+        domain_b = urllib.parse.urlparse(result_b.get("url", "")).netloc.lower()
        set_a = set(title_a.split()) | {domain_a}
        set_b = set(title_b.split()) | {domain_b}
        return _jaccard_similarity(set_a, set_b) >= threshold