fix(v2.1.1): 修复执行问题记录中的真实 bug + 文档对齐

源码修复（5 项）: 1. search.py --time-range choices 加入 week（对齐 SearXNG API 四档） 2. fetch.py stdlib 路径处理 gzip/deflate 解压（被沙箱伪响应掩盖的真实 bug，无 requests 环境抓取压缩服务器会全页 U+FFFD 乱码） 3. search.py --research 模式实现跨角度合并去重，输出 merged_results 字段（兑现文档承诺 "Results are merged and deduplicated"） 4. search.py fetch_page 返回 error_code 字段 + AdaptiveThrottle 用 E_RATE_LIMIT 结构化检测 429（原字符串匹配 "429" 会漏判 "Too Many Requests"） 5. search.py _retry_with_backoff 复用 compute_backoff_delay（60s 封顶） + 处理 Retry-After header，与 fetch.py 保持一致增强（3 项）: - common.py 精确化 baidu 子域列表（pan.baidu.com/cloud.baidu.com 不再误伤） - search.py expand_research_queries 根据主题语言切换中英文后缀 - search.py 新增 _warn_unresponsive_engines，识别实例侧引擎挂起并提示文档/版本: - _config.py VERSION 2.1.0 → 2.1.1 - SKILL.md 同步更新（time-range week、merged_results、error_code、baidu 精确化） - README.md 同步更新 + 测试数量 503 → 539 测试: 539 个全部通过，含 6 个新增验证测试
2026-08-03 12:54:27 +08:00
parent e94cbe0783
commit 0c8fdc1e45
9 changed files with 257 additions and 41 deletions
@@ -36,9 +36,11 @@ from common import (
    build_auth_headers,
    build_wayback_url,
    classify_error,
+    compute_backoff_delay,
    emit_progress,
    force_utf8_stdout,
    is_hard_blocked_domain,
+    parse_retry_after,
    resolve_auth_basic,
    resolve_auth_bearer,
    set_progress_enabled,
@@ -476,7 +478,12 @@ def _cfg_float(config: dict, key: str, default: float) -> float:
 # ----- Retry logic -----

 def _retry_with_backoff(fn, max_retries: int = MAX_RETRIES, base_delay: float = RETRY_BACKOFF_BASE):
-    """Call fn with exponential backoff + jitter on transient failures."""
+    """Call fn with exponential backoff + jitter on transient failures.
+
+    v2.1.0 修复：复用 common.compute_backoff_delay（带 60s 封顶），
+    避免高重试次数（如 --retry 10）时 1.5*2^10=1536s 卡死进程。
+    同时遵守 Retry-After header（429/503），与 fetch.py 保持一致。
+    """
    last_error = None
    for attempt in range(max_retries + 1):
        try:
@@ -485,15 +492,23 @@ def _retry_with_backoff(fn, max_retries: int = MAX_RETRIES, base_delay: float =
            if e.code in RETRYABLE_STATUS:  # 403 (UA block) + 429 + 5xx
                last_error = e
                if attempt < max_retries:
-                    delay = base_delay * (2 ** attempt) + random.uniform(0, 1)
-                    logger.info(f"  HTTP {e.code}, retrying in {delay:.1f}s... (attempt {attempt+1}/{max_retries})")
+                    # 429/503：遵守 Retry-After header，避免触发更严厉限流
+                    retry_after_sec = 0.0
+                    if e.code in (429, 503) and e.headers:
+                        retry_after_sec = parse_retry_after(
+                            e.headers.get("Retry-After", ""))
+                    delay = max(retry_after_sec,
+                                compute_backoff_delay(attempt, base=base_delay))
+                    logger.info(f"  HTTP {e.code}, retrying in {delay:.1f}s... "
+                                f"(attempt {attempt+1}/{max_retries})"
+                                f"{f' Retry-After={retry_after_sec:.1f}s' if retry_after_sec > 0 else ''}")
                    time.sleep(delay)
                    continue
            raise
        except (urllib.error.URLError, OSError) as e:
            last_error = e
            if attempt < max_retries:
-                delay = base_delay * (2 ** attempt) + random.uniform(0, 1)
+                delay = compute_backoff_delay(attempt, base=base_delay)
                logger.info(f"  Connection error ({e}), retrying in {delay:.1f}s...")
                time.sleep(delay)
                continue
@@ -850,6 +865,7 @@ def fetch_page(url: str, timeout: int = 10, auth_headers: dict = None,
    # 主抓取
    result = None
    error_msg = None
+    error_code = None
    try:
        result = fetch_url(
            url, timeout=timeout, auth_headers=auth_headers,
@@ -858,6 +874,9 @@ def fetch_page(url: str, timeout: int = 10, auth_headers: dict = None,
        )
    except Exception as e:
        error_msg = str(e) if str(e) else e.__class__.__name__
+        # v2.1.0：结构化错误码，让 AdaptiveThrottle 能用 error_code 检测 429
+        # 而非字符串匹配（"Too Many Requests" 不含 "429" 会漏判）
+        error_code = classify_error(e)

    # 反爬检测（v2.0.0 增强：全文档扫描 + WAF 指纹库）
    # 必须在 Wayback 兜底判断之前执行：Cloudflare 质询页常返回 HTTP 200，
@@ -914,6 +933,7 @@ def fetch_page(url: str, timeout: int = 10, auth_headers: dict = None,
        return {
            "url": url, "status": "error",
            "error": error_msg or "unknown error",
+            "error_code": error_code,
            "text": "", "text_length": 0, "truncated": False,
            "anti_bot_detected": False, "waf_type": None,
            "fallback_used": None,
@@ -924,6 +944,7 @@ def fetch_page(url: str, timeout: int = 10, auth_headers: dict = None,
        return {
            "url": url, "final_url": result.final_url, "status": "error",
            "error": f"Bot protection detected ({waf_type})",
+            "error_code": E_PARSE,
            "text": "", "text_length": 0, "truncated": False,
            "anti_bot_detected": True, "waf_type": waf_type,
            "fallback_used": fallback_used,
@@ -1157,12 +1178,23 @@ class AdaptiveThrottle:
                    self._concurrency = min(self._initial_concurrency,
                                            self._concurrency * 2)

-    def report_failure(self, error_msg: str = "") -> None:
+    def report_failure(self, error_msg: str = "",
+                       error_code: str = None) -> None:
+        """报告一次失败，触发自适应退避。
+
+        v2.1.0：优先用结构化 error_code 检测 429/限流（E_RATE_LIMIT），
+        回退到字符串匹配兼容旧调用方。原代码仅检查 "429" 字面量，
+        "Too Many Requests" 会漏判。
+        """
        with self._lock:
            self._consecutive_successes = 0
            self._consecutive_failures += 1
-            # 429 → 全局暂停（调用方会从 error_msg 提取秒数，这里只标记）
-            if "429" in error_msg.lower():
+            # 429/限流 → 全局暂停 30s
+            # 优先用 error_code，回退到字符串匹配（兼容无 error_code 的旧调用）
+            is_rate_limit = (error_code == E_RATE_LIMIT or
+                             "429" in error_msg.lower() or
+                             "rate limit" in error_msg.lower())
+            if is_rate_limit:
                self._global_pause_until = time.monotonic() + 30.0
            # 连续 3 次失败 → 退避 + 降并发
            if self._consecutive_failures >= 3:
@@ -1263,7 +1295,8 @@ def fetch_top_results(results: dict, count: int, timeout: int = 10,
                        f"{trunc}{ua_note}{fb_note})")
        else:
            err_count[0] += 1
-            throttle.report_failure(result.get("error", ""))
+            throttle.report_failure(result.get("error", ""),
+                                    error_code=result.get("error_code"))
            # 统计反爬拦截
            if result.get("anti_bot_detected"):
                anti_bot_count[0] += 1
@@ -1279,6 +1312,7 @@ def fetch_top_results(results: dict, count: int, timeout: int = 10,
            except Exception as e:
                u = future_map[future]
                fetched.append({"url": u, "status": "error", "error": str(e),
+                                "error_code": classify_error(e),
                                "text": "", "text_length": 0, "truncated": False,
                                "anti_bot_detected": False, "waf_type": None,
                                "fallback_used": None})
@@ -1581,6 +1615,45 @@ def _build_params(query: str, args) -> dict:
    return params


+def _warn_unresponsive_engines(results: dict, query: str,
+                               result_count: int = None) -> None:
+    """检测并提示实例侧引擎挂起/限流（v2.1.1）。
+
+    SearXNG JSON API 返回的 ``unresponsive_engines`` 字段格式为::
+
+        [["brave", "Suspended: too many requests"],
+         ["duckduckgo", "CAPTCHA"]]
+
+    当该字段非空时，说明实例内多个引擎被上游限流挂起。此时：
+      1. 用 logger.warning 输出挂起的引擎列表及原因（到 stderr，
+         不污染 stdout 数据流）
+      2. 如果结果数较少，建议用 --engines 限定未挂起引擎
+
+    这是真实运营问题（见执行问题记录 #3）：连续查询后 brave/duckduckgo/
+    startpage 等引擎会被上游限流挂起，导致结果骤减或全空。让用户及时
+    感知引擎状态，避免误判为"无结果"而反复重试触发更严厉限流。
+    """
+    unresponsive = results.get("unresponsive_engines", [])
+    if not unresponsive:
+        return
+    # 格式化引擎列表：兼容 [engine, reason] 和 [engine] 两种格式
+    parts = []
+    for entry in unresponsive:
+        if isinstance(entry, (list, tuple)) and len(entry) >= 2:
+            parts.append(f"{entry[0]} ({entry[1]})")
+        elif isinstance(entry, (list, tuple)) and len(entry) == 1:
+            parts.append(str(entry[0]))
+        else:
+            parts.append(str(entry))
+    engine_list = ", ".join(parts)
+    logger.warning(f"Instance engines unresponsive: {engine_list}")
+    # 结果数少 + 引擎挂起 → 建议规避
+    if result_count is not None and result_count < 3 and len(unresponsive) >= 2:
+        # 找出可能未挂起的常见引擎提示
+        logger.warning("Hint: multiple engines suspended — consider using "
+                       "--engines to target responsive ones, or wait before retrying")
+
+
 def _run_single_query(query: str, args, instance_urls: list,
                      auth_headers: dict, ttl_seconds: int):
    """Run one query end-to-end: search → limit → domain-filter → fetch.
@@ -1618,6 +1691,12 @@ def _run_single_query(query: str, args, instance_urls: list,
            logger.info(f"[cache stored] q={query!r} TTL={args.cache_ttl}min")
            emit_progress("cache_store", query=query, ttl=args.cache_ttl)

+    # v2.1.1：检测实例侧引擎挂起/限流（仅在实时查询后提示，cache hit 时
+    # unresponsive_engines 信息可能已过期）
+    if cached is None:
+        _warn_unresponsive_engines(results, query,
+                                   result_count=len(results.get("results", [])))
+
    # Dedup (default on; --no-dedup disables) then sort, both BEFORE limit
    # so --max-results keeps the highest-scoring / newest items.
    if not args.no_dedup:
@@ -1821,17 +1900,31 @@ def _read_queries_file(path: str) -> list:
 # 给定一个主题，自动扩展多角度查询词，复用批量搜索逻辑。
 # 扩展策略是确定性规则（不做 AI 判断），覆盖人物/主题/事件的通用研究维度。

-# 研究角度定义：(角度标识, 后缀词)
+# 研究角度定义：(角度标识, 中文后缀, 英文后缀)
 # 顺序代表搜索优先级——基本信息优先，评价争议最后。
+# v2.1.0：支持中英文双语后缀，根据主题语言自动选择。
+# 中文主题用中文后缀（"简介"/"经历"等），英文主题用英文后缀
+# （"profile"/"background"等），避免 "Python asyncio 经历" 这类
+# 跨语言组合在英文引擎上匹配度低的问题。angle 标识符保持英文，
+# 便于 AI Agent 程序化处理。
 _RESEARCH_ANGLES = [
-    ("overview",   ""),          # 主题本身：最直接的搜索
-    ("profile",    "简介"),      # 基本信息：百科式介绍
-    ("background", "经历"),      # 背景经历：生平/历史
-    ("works",      "作品"),      # 作品成就：产出物
-    ("review",     "评价"),      # 评价争议：外界看法
+    ("overview",   "",       ""),            # 主题本身：最直接的搜索
+    ("profile",    "简介",   "profile"),     # 基本信息：百科式介绍
+    ("background", "经历",   "background"),  # 背景经历：生平/历史
+    ("works",      "作品",   "works"),       # 作品成就：产出物
+    ("review",     "评价",   "reviews"),     # 评价争议：外界看法
 ]


+def _is_chinese_topic(topic: str) -> bool:
+    """检测主题是否包含中文字符（CJK 统一表意文字范围）。
+
+    用于 expand_research_queries 选择中文还是英文后缀。
+    纯英文主题（如 "Python asyncio"）返回 False，用英文后缀。
+    """
+    return bool(re.search(r'[\u4e00-\u9fff]', topic))
+
+
 def expand_research_queries(topic: str) -> list:
    """将研究主题扩展为多角度查询词列表。

@@ -1843,6 +1936,10 @@ def expand_research_queries(topic: str) -> list:
      4. works      — 作品成就
      5. review     — 评价争议

+    v2.1.0 修复：根据主题语言自动切换后缀。含中文字符的主题用中文后缀
+    （"七森莉莉 简介"），纯英文主题用英文后缀（"Python asyncio profile"），
+    避免跨语言组合在搜索引擎上匹配度低。
+
    返回 [(angle, query), ...] 列表，angle 用于结果标注。

    确定性规则，不依赖 AI 判断——确保跨进程可复现，AI Agent 可预期。
@@ -1850,8 +1947,10 @@ def expand_research_queries(topic: str) -> list:
    topic = topic.strip()
    if not topic:
        return []
+    use_chinese = _is_chinese_topic(topic)
    queries = []
-    for angle, suffix in _RESEARCH_ANGLES:
+    for angle, cn_suffix, en_suffix in _RESEARCH_ANGLES:
+        suffix = cn_suffix if use_chinese else en_suffix
        query = f"{topic} {suffix}".strip()
        queries.append((angle, query))
    return queries
@@ -1927,9 +2026,11 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
                        help="Language code (e.g. en, zh-CN, de)")
    parser.add_argument("--pageno", "-p", type=int, default=1,
                        help="Page number (default: 1)")
-    parser.add_argument("--time-range", "-t", choices=["day", "month", "year", "none"],
+    parser.add_argument("--time-range", "-t",
+                        choices=["day", "week", "month", "year", "none"],
                        default=config.get("time_range", "year"),
-                        help="Time range filter (default: year; 'none' disables filtering)")
+                        help="Time range filter (default: year; 'none' disables filtering). "
+                             "SearXNG API standard four tiers: day/week/month/year")
    parser.add_argument("--safesearch", "-s", type=int, choices=[0, 1, 2],
                        default=_cfg_int(config, "safesearch", 0),
                        help="Safe search: 0=off, 1=moderate, 2=strict (default: 0=off)")
@@ -2239,6 +2340,23 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
                batch.append({"query": q, "angle": angle, "status": "ok",
                              "results": results})

+        # v2.1.0 修复：跨角度合并去重
+        # 文档承诺 "Results are merged and deduplicated"，原代码只输出 per-angle
+        # 结果，同一 URL 可能出现在多个角度中。此处合并所有成功角度的 results，
+        # 去重后作为 merged_results 字段输出，让 AI Agent 既能按角度组织报告，
+        # 也能获得去重后的总览。
+        merged = {"query": topic, "results": []}
+        for br in batch:
+            if br.get("status") == "ok" and "results" in br:
+                merged["results"].extend(br["results"].get("results", []))
+        if merged["results"]:
+            deduplicate_results(merged)
+            sort_results(merged, args.sort_by)
+            if args.max_results:
+                merged["results"] = merged["results"][:args.max_results]
+        merged_count = len(merged["results"])
+        logger.info(f"Research merged: {merged_count} unique results after dedup")
+
        # 输出
        if args.format == "json":
            output = json.dumps({
@@ -2248,6 +2366,7 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
                    {"angle": a, "query": q} for a, q in research_queries
                ],
                "queries": batch,
+                "merged_results": merged,
            }, indent=2, ensure_ascii=False)
        elif args.format == "csv":
            import csv as csv_mod
@@ -2276,12 +2395,18 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
            output = out.getvalue().rstrip()
        elif args.format == "urls":
            parts = []
+            # 先输出 per-angle 结果
            for br in batch:
                parts.append(f"# [{br.get('angle', '?')}] {br['query']}")
                if "results" in br:
                    parts.append(format_urls(br["results"]))
                else:
                    parts.append(f"# [ERROR: {br['error']}]")
+            # 再输出合并去重后的总览
+            if merged_count > 0:
+                parts.append("")
+                parts.append(f"# [MERGED] {topic} ({merged_count} unique results)")
+                parts.append(format_urls(merged))
            output = "\n".join(parts)
        else:  # brief
            parts = []
@@ -2294,6 +2419,12 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
                else:
                    parts.append(f"[ERROR: {br['error']}]")
                parts.append("")
+            # 合并去重后的总览
+            if merged_count > 0:
+                parts.append("=" * 60)
+                parts.append(f"[MERGED] {topic} ({merged_count} unique results)")
+                parts.append("=" * 60)
+                parts.append(format_brief(merged))
            output = "\n".join(parts)

        if args.output: