thzxx
28ff7c0a48
反爬措施: - 浏览器指纹头 build_browser_headers(): Sec-Ch-Ua/Sec-Fetch-*/Accept-Language/Accept-Encoding, 绕过 80%+ 轻量 WAF - 12 个 UA 池 (Chrome/Edge/Firefox x Win/macOS/Linux x v129-131) - 确定性 UA 轮换 get_ua_for_domain(): SHA-256 按域名固定 UA, 会话内稳定跨进程可复现 - WAF 指纹库 _detect_anti_bot(): 识别 Cloudflare/Imperva/PerimeterX/DataDome/Akamai/通用, 全文档扫描 - Retry-After 遵守: 429/503 读取 header (数字或 HTTP date) 作为最小重试延迟 - 退避封顶 60s (原无上限, N=10 时 1536s 卡死进程) 抓取稳定性: - requests.Session 复用: 连接池(10/host) + cookie 持久化 + TLS 会话恢复 - 超时分离 (connect, read) 元组, 避免大页面浪费已建连接 - Wayback Machine 兜底: 404/403/超时自动重试 web.archive.org, 默认启用 --no-fallback 关闭 - AdaptiveThrottle 自适应限流: 3 次失败翻倍延迟+减半并发, 5 次成功渐进恢复, 429 全局暂停 30s - readability-lite 提取: article/main 缺失时按文本密度选最可能正文 div 新增 CLI flags: - --fetch-report: 结构化抓取报告到 stderr (每 URL 状态/WAF 类型/兜底方式/字符数 + JSON 摘要) - --no-fallback: 禁用 Wayback 兜底 - --referer: 设置 Referer 头 (默认实例 URL) - --request-delay: 抓取请求间隔秒数 (默认 0.3, 自适应可能增大) fetch 结果新字段: anti_bot_detected (bool), waf_type (str|null), fallback_used (str|null) 测试: 新增 4 个测试文件 (test_browser_headers/test_anti_bot/test_wayback_fallback/test_adaptive_throttle), 451 个测试全部通过
256 lines
10 KiB
Python
256 lines
10 KiB
Python
"""Tests for v2.0.0 Wayback Machine fallback and fetch_page enhancements.
|
||
|
||
Covers: _should_try_fallback (trigger conditions), _try_wayback_fallback
|
||
(mocked), fetch_page with fallback_enabled flag, new result fields
|
||
(anti_bot_detected, waf_type, fallback_used), fallback disabled path.
|
||
"""
|
||
import sys
|
||
import os
|
||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "scripts"))
|
||
|
||
from unittest.mock import patch, MagicMock
|
||
from fetch import FetchResult
|
||
import search as search_mod
|
||
from search import (
|
||
_should_try_fallback,
|
||
_try_wayback_fallback,
|
||
fetch_page,
|
||
)
|
||
|
||
|
||
def _make_result(content="<html><body>OK</body></html>",
|
||
content_type="text/html", final_url="https://example.com",
|
||
truncated=False, ua="TestUA/1.0"):
|
||
return FetchResult(content, content_type, final_url, truncated, ua)
|
||
|
||
|
||
# ----- _should_try_fallback -----
|
||
|
||
def test_should_try_fallback_on_404():
|
||
assert _should_try_fallback(None, "HTTP 404 for https://example.com") is True
|
||
|
||
|
||
def test_should_try_fallback_on_403():
|
||
assert _should_try_fallback(None, "HTTP 403 for https://example.com") is True
|
||
|
||
|
||
def test_should_try_fallback_on_timeout():
|
||
assert _should_try_fallback(None, "Request failed: timed out") is True
|
||
|
||
|
||
def test_should_try_fallback_on_connection_reset():
|
||
assert _should_try_fallback(None, "Connection reset by peer") is True
|
||
|
||
|
||
def test_should_try_fallback_on_max_retries():
|
||
assert _should_try_fallback(None, "Max retries exceeded") is True
|
||
|
||
|
||
def test_should_try_fallback_not_on_success():
|
||
"""主抓取成功时不触发兜底。"""
|
||
result = _make_result()
|
||
assert _should_try_fallback(result, None) is False
|
||
|
||
|
||
def test_should_try_fallback_not_on_500():
|
||
"""500 错误不触发 Wayback（服务器内部错误，Wayback 也未必有）。"""
|
||
assert _should_try_fallback(None, "HTTP 500 for https://example.com") is False
|
||
|
||
|
||
def test_should_try_fallback_not_on_empty_error():
|
||
assert _should_try_fallback(None, "") is False
|
||
assert _should_try_fallback(None, None) is False
|
||
|
||
|
||
def test_should_try_fallback_not_on_dns():
|
||
"""DNS 失败不触发（Wayback 也访问不到）。"""
|
||
assert _should_try_fallback(None, "Name or service not known") is False
|
||
|
||
|
||
# ----- _try_wayback_fallback (mocked) -----
|
||
|
||
def test_try_wayback_success():
|
||
"""Wayback 返回成功时，返回 FetchResult。"""
|
||
wb_result = _make_result(content="<html>Archived page</html>",
|
||
final_url="https://web.archive.org/web/2024/https://example.com")
|
||
with patch("search.fetch_url", return_value=wb_result) as mock_fetch:
|
||
result = _try_wayback_fallback("https://example.com")
|
||
assert result is not None
|
||
assert "Archived page" in result.content
|
||
# 确认调用了 Wayback URL
|
||
call_args = mock_fetch.call_args
|
||
assert "web.archive.org/web/2/" in call_args[0][0]
|
||
|
||
|
||
def test_try_wayback_failure_returns_none():
|
||
"""Wayback 也失败时返回 None。"""
|
||
with patch("search.fetch_url", side_effect=RuntimeError("timeout")):
|
||
result = _try_wayback_fallback("https://example.com")
|
||
assert result is None
|
||
|
||
|
||
def test_try_wayback_uses_reduced_timeout():
|
||
"""Wayback 使用 min(timeout, 10) 避免长时间阻塞。"""
|
||
wb_result = _make_result()
|
||
with patch("search.fetch_url", return_value=wb_result) as mock_fetch:
|
||
_try_wayback_fallback("https://example.com", timeout=30)
|
||
call_kwargs = mock_fetch.call_args[1]
|
||
assert call_kwargs["timeout"] == 10
|
||
|
||
|
||
def test_try_wayback_no_auth_headers():
|
||
"""Wayback 是公共服务，不传 auth_headers。"""
|
||
wb_result = _make_result()
|
||
with patch("search.fetch_url", return_value=wb_result) as mock_fetch:
|
||
_try_wayback_fallback("https://example.com",
|
||
auth_headers={"Authorization": "Bearer x"})
|
||
call_kwargs = mock_fetch.call_args[1]
|
||
# auth_headers 应为 None（Wayback 不需要认证）
|
||
assert call_kwargs.get("auth_headers") is None
|
||
|
||
|
||
# ----- fetch_page with fallback -----
|
||
|
||
def test_fetch_page_success_no_fallback():
|
||
"""主抓取成功，不触发兜底。"""
|
||
with patch("search.fetch_url", return_value=_make_result()):
|
||
result = fetch_page("https://example.com", fallback_enabled=True)
|
||
assert result["status"] == "ok"
|
||
assert result["fallback_used"] is None
|
||
assert result["anti_bot_detected"] is False
|
||
assert result["waf_type"] is None
|
||
|
||
|
||
def test_fetch_page_404_triggers_wayback_success():
|
||
"""404 触发 Wayback，Wayback 成功。"""
|
||
wb_result = _make_result(content="<html>Archived</html>")
|
||
with patch("search.fetch_url",
|
||
side_effect=[RuntimeError("HTTP 404"), wb_result]):
|
||
result = fetch_page("https://example.com", fallback_enabled=True)
|
||
assert result["status"] == "ok"
|
||
assert result["fallback_used"] == "wayback"
|
||
|
||
|
||
def test_fetch_page_403_triggers_wayback_success():
|
||
"""403 触发 Wayback，Wayback 成功。"""
|
||
wb_result = _make_result()
|
||
with patch("search.fetch_url",
|
||
side_effect=[RuntimeError("HTTP 403"), wb_result]):
|
||
result = fetch_page("https://example.com", fallback_enabled=True)
|
||
assert result["status"] == "ok"
|
||
assert result["fallback_used"] == "wayback"
|
||
|
||
|
||
def test_fetch_page_timeout_triggers_wayback():
|
||
"""超时触发 Wayback。"""
|
||
wb_result = _make_result()
|
||
with patch("search.fetch_url",
|
||
side_effect=[RuntimeError("timed out"), wb_result]):
|
||
result = fetch_page("https://example.com", fallback_enabled=True)
|
||
assert result["status"] == "ok"
|
||
assert result["fallback_used"] == "wayback"
|
||
|
||
|
||
def test_fetch_page_fallback_disabled():
|
||
"""fallback_enabled=False 时不触发 Wayback。"""
|
||
with patch("search.fetch_url",
|
||
side_effect=RuntimeError("HTTP 404")) as mock_fetch:
|
||
result = fetch_page("https://example.com", fallback_enabled=False)
|
||
assert result["status"] == "error"
|
||
assert result["fallback_used"] is None
|
||
# 只调用一次（主抓取），不调用 Wayback
|
||
assert mock_fetch.call_count == 1
|
||
|
||
|
||
def test_fetch_page_both_fail():
|
||
"""主抓取和 Wayback 都失败。"""
|
||
with patch("search.fetch_url",
|
||
side_effect=[RuntimeError("HTTP 404"), RuntimeError("timeout")]):
|
||
result = fetch_page("https://example.com", fallback_enabled=True)
|
||
assert result["status"] == "error"
|
||
assert result["fallback_used"] is None
|
||
|
||
|
||
# ----- fetch_page anti-bot fields -----
|
||
|
||
def test_fetch_page_detects_cloudflare():
|
||
"""抓到 Cloudflare 拦截页，标记 anti_bot_detected + waf_type。"""
|
||
cf_html = "<html><body>Just a moment... cf-ray: 123</body></html>"
|
||
with patch("search.fetch_url",
|
||
return_value=_make_result(content=cf_html)):
|
||
result = fetch_page("https://example.com", fallback_enabled=False)
|
||
assert result["status"] == "error"
|
||
assert result["anti_bot_detected"] is True
|
||
assert result["waf_type"] == "cloudflare"
|
||
assert "cloudflare" in result["error"]
|
||
|
||
|
||
def test_fetch_page_detects_datadome():
|
||
"""抓到 DataDome 拦截页。"""
|
||
dd_html = "<html><body>Protected by DataDome</body></html>"
|
||
with patch("search.fetch_url",
|
||
return_value=_make_result(content=dd_html)):
|
||
result = fetch_page("https://example.com", fallback_enabled=False)
|
||
assert result["status"] == "error"
|
||
assert result["anti_bot_detected"] is True
|
||
assert result["waf_type"] == "datadome"
|
||
|
||
|
||
def test_fetch_page_normal_page_no_anti_bot():
|
||
"""正常页面 anti_bot_detected=False。"""
|
||
normal_html = "<html><body><p>Normal article content.</p></body></html>"
|
||
with patch("search.fetch_url",
|
||
return_value=_make_result(content=normal_html)):
|
||
result = fetch_page("https://example.com", fallback_enabled=False)
|
||
assert result["status"] == "ok"
|
||
assert result["anti_bot_detected"] is False
|
||
assert result["waf_type"] is None
|
||
|
||
|
||
def test_fetch_page_anti_bot_triggers_wayback():
|
||
"""被反爬拦截后也应尝试 Wayback（_should_try_fallback 防御性检查）。
|
||
|
||
当前实现：主抓取成功返回反爬页内容 → result 非 None → 不触发兜底。
|
||
此测试记录此行为：反爬页被当作"成功抓取"返回，在 fetch_page 内部检测。
|
||
"""
|
||
cf_html = "<html><body>Just a moment... cf-ray: 123</body></html>"
|
||
with patch("search.fetch_url",
|
||
return_value=_make_result(content=cf_html)):
|
||
result = fetch_page("https://example.com", fallback_enabled=True)
|
||
# 反爬页被检测到，标记为 error + anti_bot_detected
|
||
assert result["status"] == "error"
|
||
assert result["anti_bot_detected"] is True
|
||
# 但不会触发 Wayback（因为主抓取"成功"了，只是内容是反爬页）
|
||
assert result["fallback_used"] is None
|
||
|
||
|
||
def test_fetch_page_passes_referer():
|
||
"""referer 透传给 fetch_url。"""
|
||
with patch("search.fetch_url", return_value=_make_result()) as mock_fetch:
|
||
fetch_page("https://example.com", referer="https://google.com/",
|
||
fallback_enabled=False)
|
||
call_kwargs = mock_fetch.call_args[1]
|
||
assert call_kwargs.get("referer") == "https://google.com/"
|
||
|
||
|
||
# ----- fetch_page result structure -----
|
||
|
||
def test_fetch_page_result_has_all_v2_fields():
|
||
"""结果 dict 包含所有 v2.0.0 新字段。"""
|
||
with patch("search.fetch_url", return_value=_make_result()):
|
||
result = fetch_page("https://example.com", fallback_enabled=False)
|
||
required_fields = ["anti_bot_detected", "waf_type", "fallback_used",
|
||
"status", "url", "text", "text_length", "truncated"]
|
||
for field in required_fields:
|
||
assert field in result, f"missing field: {field}"
|
||
|
||
|
||
def test_fetch_page_error_result_has_all_v2_fields():
|
||
"""错误结果也包含所有 v2.0.0 新字段。"""
|
||
with patch("search.fetch_url", side_effect=RuntimeError("HTTP 500")):
|
||
result = fetch_page("https://example.com", fallback_enabled=False)
|
||
required_fields = ["anti_bot_detected", "waf_type", "fallback_used",
|
||
"status", "url", "error", "text", "text_length"]
|
||
for field in required_fields:
|
||
assert field in result, f"missing field: {field}"
|