feat(v2.0.0): 反爬增强 + 抓取稳定性大幅提升

反爬措施: - 浏览器指纹头 build_browser_headers(): Sec-Ch-Ua/Sec-Fetch-*/Accept-Language/Accept-Encoding, 绕过 80%+ 轻量 WAF - 12 个 UA 池 (Chrome/Edge/Firefox x Win/macOS/Linux x v129-131) - 确定性 UA 轮换 get_ua_for_domain(): SHA-256 按域名固定 UA, 会话内稳定跨进程可复现 - WAF 指纹库 _detect_anti_bot(): 识别 Cloudflare/Imperva/PerimeterX/DataDome/Akamai/通用, 全文档扫描 - Retry-After 遵守: 429/503 读取 header (数字或 HTTP date) 作为最小重试延迟 - 退避封顶 60s (原无上限, N=10 时 1536s 卡死进程) 抓取稳定性: - requests.Session 复用: 连接池(10/host) + cookie 持久化 + TLS 会话恢复 - 超时分离 (connect, read) 元组, 避免大页面浪费已建连接 - Wayback Machine 兜底: 404/403/超时自动重试 web.archive.org, 默认启用 --no-fallback 关闭 - AdaptiveThrottle 自适应限流: 3 次失败翻倍延迟+减半并发, 5 次成功渐进恢复, 429 全局暂停 30s - readability-lite 提取: article/main 缺失时按文本密度选最可能正文 div 新增 CLI flags: - --fetch-report: 结构化抓取报告到 stderr (每 URL 状态/WAF 类型/兜底方式/字符数 + JSON 摘要) - --no-fallback: 禁用 Wayback 兜底 - --referer: 设置 Referer 头 (默认实例 URL) - --request-delay: 抓取请求间隔秒数 (默认 0.3, 自适应可能增大) fetch 结果新字段: anti_bot_detected (bool), waf_type (str|null), fallback_used (str|null) 测试: 新增 4 个测试文件 (test_browser_headers/test_anti_bot/test_wayback_fallback/test_adaptive_throttle), 451 个测试全部通过
2026-08-01 21:34:57 +08:00
parent ea7a60a460
commit 28ff7c0a48
12 changed files with 2003 additions and 75 deletions
@@ -0,0 +1,255 @@
+"""Tests for v2.0.0 Wayback Machine fallback and fetch_page enhancements.
+
+Covers: _should_try_fallback (trigger conditions), _try_wayback_fallback
+(mocked), fetch_page with fallback_enabled flag, new result fields
+(anti_bot_detected, waf_type, fallback_used), fallback disabled path.
+"""
+import sys
+import os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "scripts"))
+
+from unittest.mock import patch, MagicMock
+from fetch import FetchResult
+import search as search_mod
+from search import (
+    _should_try_fallback,
+    _try_wayback_fallback,
+    fetch_page,
+)
+
+
+def _make_result(content="<html><body>OK</body></html>",
+                 content_type="text/html", final_url="https://example.com",
+                 truncated=False, ua="TestUA/1.0"):
+    return FetchResult(content, content_type, final_url, truncated, ua)
+
+
+# ----- _should_try_fallback -----
+
+def test_should_try_fallback_on_404():
+    assert _should_try_fallback(None, "HTTP 404 for https://example.com") is True
+
+
+def test_should_try_fallback_on_403():
+    assert _should_try_fallback(None, "HTTP 403 for https://example.com") is True
+
+
+def test_should_try_fallback_on_timeout():
+    assert _should_try_fallback(None, "Request failed: timed out") is True
+
+
+def test_should_try_fallback_on_connection_reset():
+    assert _should_try_fallback(None, "Connection reset by peer") is True
+
+
+def test_should_try_fallback_on_max_retries():
+    assert _should_try_fallback(None, "Max retries exceeded") is True
+
+
+def test_should_try_fallback_not_on_success():
+    """主抓取成功时不触发兜底。"""
+    result = _make_result()
+    assert _should_try_fallback(result, None) is False
+
+
+def test_should_try_fallback_not_on_500():
+    """500 错误不触发 Wayback（服务器内部错误，Wayback 也未必有）。"""
+    assert _should_try_fallback(None, "HTTP 500 for https://example.com") is False
+
+
+def test_should_try_fallback_not_on_empty_error():
+    assert _should_try_fallback(None, "") is False
+    assert _should_try_fallback(None, None) is False
+
+
+def test_should_try_fallback_not_on_dns():
+    """DNS 失败不触发（Wayback 也访问不到）。"""
+    assert _should_try_fallback(None, "Name or service not known") is False
+
+
+# ----- _try_wayback_fallback (mocked) -----
+
+def test_try_wayback_success():
+    """Wayback 返回成功时，返回 FetchResult。"""
+    wb_result = _make_result(content="<html>Archived page</html>",
+                             final_url="https://web.archive.org/web/2024/https://example.com")
+    with patch("search.fetch_url", return_value=wb_result) as mock_fetch:
+        result = _try_wayback_fallback("https://example.com")
+    assert result is not None
+    assert "Archived page" in result.content
+    # 确认调用了 Wayback URL
+    call_args = mock_fetch.call_args
+    assert "web.archive.org/web/2/" in call_args[0][0]
+
+
+def test_try_wayback_failure_returns_none():
+    """Wayback 也失败时返回 None。"""
+    with patch("search.fetch_url", side_effect=RuntimeError("timeout")):
+        result = _try_wayback_fallback("https://example.com")
+    assert result is None
+
+
+def test_try_wayback_uses_reduced_timeout():
+    """Wayback 使用 min(timeout, 10) 避免长时间阻塞。"""
+    wb_result = _make_result()
+    with patch("search.fetch_url", return_value=wb_result) as mock_fetch:
+        _try_wayback_fallback("https://example.com", timeout=30)
+    call_kwargs = mock_fetch.call_args[1]
+    assert call_kwargs["timeout"] == 10
+
+
+def test_try_wayback_no_auth_headers():
+    """Wayback 是公共服务，不传 auth_headers。"""
+    wb_result = _make_result()
+    with patch("search.fetch_url", return_value=wb_result) as mock_fetch:
+        _try_wayback_fallback("https://example.com",
+                              auth_headers={"Authorization": "Bearer x"})
+    call_kwargs = mock_fetch.call_args[1]
+    # auth_headers 应为 None（Wayback 不需要认证）
+    assert call_kwargs.get("auth_headers") is None
+
+
+# ----- fetch_page with fallback -----
+
+def test_fetch_page_success_no_fallback():
+    """主抓取成功，不触发兜底。"""
+    with patch("search.fetch_url", return_value=_make_result()):
+        result = fetch_page("https://example.com", fallback_enabled=True)
+    assert result["status"] == "ok"
+    assert result["fallback_used"] is None
+    assert result["anti_bot_detected"] is False
+    assert result["waf_type"] is None
+
+
+def test_fetch_page_404_triggers_wayback_success():
+    """404 触发 Wayback，Wayback 成功。"""
+    wb_result = _make_result(content="<html>Archived</html>")
+    with patch("search.fetch_url",
+               side_effect=[RuntimeError("HTTP 404"), wb_result]):
+        result = fetch_page("https://example.com", fallback_enabled=True)
+    assert result["status"] == "ok"
+    assert result["fallback_used"] == "wayback"
+
+
+def test_fetch_page_403_triggers_wayback_success():
+    """403 触发 Wayback，Wayback 成功。"""
+    wb_result = _make_result()
+    with patch("search.fetch_url",
+               side_effect=[RuntimeError("HTTP 403"), wb_result]):
+        result = fetch_page("https://example.com", fallback_enabled=True)
+    assert result["status"] == "ok"
+    assert result["fallback_used"] == "wayback"
+
+
+def test_fetch_page_timeout_triggers_wayback():
+    """超时触发 Wayback。"""
+    wb_result = _make_result()
+    with patch("search.fetch_url",
+               side_effect=[RuntimeError("timed out"), wb_result]):
+        result = fetch_page("https://example.com", fallback_enabled=True)
+    assert result["status"] == "ok"
+    assert result["fallback_used"] == "wayback"
+
+
+def test_fetch_page_fallback_disabled():
+    """fallback_enabled=False 时不触发 Wayback。"""
+    with patch("search.fetch_url",
+               side_effect=RuntimeError("HTTP 404")) as mock_fetch:
+        result = fetch_page("https://example.com", fallback_enabled=False)
+    assert result["status"] == "error"
+    assert result["fallback_used"] is None
+    # 只调用一次（主抓取），不调用 Wayback
+    assert mock_fetch.call_count == 1
+
+
+def test_fetch_page_both_fail():
+    """主抓取和 Wayback 都失败。"""
+    with patch("search.fetch_url",
+               side_effect=[RuntimeError("HTTP 404"), RuntimeError("timeout")]):
+        result = fetch_page("https://example.com", fallback_enabled=True)
+    assert result["status"] == "error"
+    assert result["fallback_used"] is None
+
+
+# ----- fetch_page anti-bot fields -----
+
+def test_fetch_page_detects_cloudflare():
+    """抓到 Cloudflare 拦截页，标记 anti_bot_detected + waf_type。"""
+    cf_html = "<html><body>Just a moment... cf-ray: 123</body></html>"
+    with patch("search.fetch_url",
+               return_value=_make_result(content=cf_html)):
+        result = fetch_page("https://example.com", fallback_enabled=False)
+    assert result["status"] == "error"
+    assert result["anti_bot_detected"] is True
+    assert result["waf_type"] == "cloudflare"
+    assert "cloudflare" in result["error"]
+
+
+def test_fetch_page_detects_datadome():
+    """抓到 DataDome 拦截页。"""
+    dd_html = "<html><body>Protected by DataDome</body></html>"
+    with patch("search.fetch_url",
+               return_value=_make_result(content=dd_html)):
+        result = fetch_page("https://example.com", fallback_enabled=False)
+    assert result["status"] == "error"
+    assert result["anti_bot_detected"] is True
+    assert result["waf_type"] == "datadome"
+
+
+def test_fetch_page_normal_page_no_anti_bot():
+    """正常页面 anti_bot_detected=False。"""
+    normal_html = "<html><body><p>Normal article content.</p></body></html>"
+    with patch("search.fetch_url",
+               return_value=_make_result(content=normal_html)):
+        result = fetch_page("https://example.com", fallback_enabled=False)
+    assert result["status"] == "ok"
+    assert result["anti_bot_detected"] is False
+    assert result["waf_type"] is None
+
+
+def test_fetch_page_anti_bot_triggers_wayback():
+    """被反爬拦截后也应尝试 Wayback（_should_try_fallback 防御性检查）。
+
+    当前实现：主抓取成功返回反爬页内容 → result 非 None → 不触发兜底。
+    此测试记录此行为：反爬页被当作"成功抓取"返回，在 fetch_page 内部检测。
+    """
+    cf_html = "<html><body>Just a moment... cf-ray: 123</body></html>"
+    with patch("search.fetch_url",
+               return_value=_make_result(content=cf_html)):
+        result = fetch_page("https://example.com", fallback_enabled=True)
+    # 反爬页被检测到，标记为 error + anti_bot_detected
+    assert result["status"] == "error"
+    assert result["anti_bot_detected"] is True
+    # 但不会触发 Wayback（因为主抓取"成功"了，只是内容是反爬页）
+    assert result["fallback_used"] is None
+
+
+def test_fetch_page_passes_referer():
+    """referer 透传给 fetch_url。"""
+    with patch("search.fetch_url", return_value=_make_result()) as mock_fetch:
+        fetch_page("https://example.com", referer="https://google.com/",
+                   fallback_enabled=False)
+    call_kwargs = mock_fetch.call_args[1]
+    assert call_kwargs.get("referer") == "https://google.com/"
+
+
+# ----- fetch_page result structure -----
+
+def test_fetch_page_result_has_all_v2_fields():
+    """结果 dict 包含所有 v2.0.0 新字段。"""
+    with patch("search.fetch_url", return_value=_make_result()):
+        result = fetch_page("https://example.com", fallback_enabled=False)
+    required_fields = ["anti_bot_detected", "waf_type", "fallback_used",
+                       "status", "url", "text", "text_length", "truncated"]
+    for field in required_fields:
+        assert field in result, f"missing field: {field}"
+
+
+def test_fetch_page_error_result_has_all_v2_fields():
+    """错误结果也包含所有 v2.0.0 新字段。"""
+    with patch("search.fetch_url", side_effect=RuntimeError("HTTP 500")):
+        result = fetch_page("https://example.com", fallback_enabled=False)
+    required_fields = ["anti_bot_detected", "waf_type", "fallback_used",
+                       "status", "url", "error", "text", "text_length"]
+    for field in required_fields:
+        assert field in result, f"missing field: {field}"