thzxx
28ff7c0a48
反爬措施: - 浏览器指纹头 build_browser_headers(): Sec-Ch-Ua/Sec-Fetch-*/Accept-Language/Accept-Encoding, 绕过 80%+ 轻量 WAF - 12 个 UA 池 (Chrome/Edge/Firefox x Win/macOS/Linux x v129-131) - 确定性 UA 轮换 get_ua_for_domain(): SHA-256 按域名固定 UA, 会话内稳定跨进程可复现 - WAF 指纹库 _detect_anti_bot(): 识别 Cloudflare/Imperva/PerimeterX/DataDome/Akamai/通用, 全文档扫描 - Retry-After 遵守: 429/503 读取 header (数字或 HTTP date) 作为最小重试延迟 - 退避封顶 60s (原无上限, N=10 时 1536s 卡死进程) 抓取稳定性: - requests.Session 复用: 连接池(10/host) + cookie 持久化 + TLS 会话恢复 - 超时分离 (connect, read) 元组, 避免大页面浪费已建连接 - Wayback Machine 兜底: 404/403/超时自动重试 web.archive.org, 默认启用 --no-fallback 关闭 - AdaptiveThrottle 自适应限流: 3 次失败翻倍延迟+减半并发, 5 次成功渐进恢复, 429 全局暂停 30s - readability-lite 提取: article/main 缺失时按文本密度选最可能正文 div 新增 CLI flags: - --fetch-report: 结构化抓取报告到 stderr (每 URL 状态/WAF 类型/兜底方式/字符数 + JSON 摘要) - --no-fallback: 禁用 Wayback 兜底 - --referer: 设置 Referer 头 (默认实例 URL) - --request-delay: 抓取请求间隔秒数 (默认 0.3, 自适应可能增大) fetch 结果新字段: anti_bot_detected (bool), waf_type (str|null), fallback_used (str|null) 测试: 新增 4 个测试文件 (test_browser_headers/test_anti_bot/test_wayback_fallback/test_adaptive_throttle), 451 个测试全部通过
207 lines
6.6 KiB
Python
207 lines
6.6 KiB
Python
"""Tests for v2.0.0 anti-bot / WAF detection enhancements.
|
|
|
|
Covers: _detect_anti_bot (WAF fingerprint library: Cloudflare/Imperva/
|
|
PerimeterX/DataDome/Akamai/generic), full-document scanning (not just
|
|
first 2000 chars), _is_blocked_page backward compatibility.
|
|
"""
|
|
import sys
|
|
import os
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "scripts"))
|
|
|
|
from search import _detect_anti_bot, _is_blocked_page, WAF_FINGERPRINTS
|
|
|
|
|
|
# ----- Cloudflare detection -----
|
|
|
|
def test_detect_cloudflare_cf_ray():
|
|
html = "<html><head><title>Just a moment...</title></head>" \
|
|
"<body>cf-ray: 8abc123</body></html>"
|
|
assert _detect_anti_bot(html) == "cloudflare"
|
|
|
|
|
|
def test_detect_cloudflare_just_a_moment():
|
|
html = "<html><body>Just a moment...</body></html>"
|
|
assert _detect_anti_bot(html) == "cloudflare"
|
|
|
|
|
|
def test_detect_cloudflare_checking_browser():
|
|
html = "<html><body>Checking your browser before accessing</body></html>"
|
|
assert _detect_anti_bot(html) == "cloudflare"
|
|
|
|
|
|
def test_detect_cloudflare_attention_required():
|
|
html = "<html><body>Attention Required! | Cloudflare</body></html>"
|
|
assert _detect_anti_bot(html) == "cloudflare"
|
|
|
|
|
|
# ----- Imperva detection -----
|
|
|
|
def test_detect_imperva_incap_ses():
|
|
html = "<html><body>incap_ses_123_cookie</body></html>"
|
|
assert _detect_anti_bot(html) == "imperva"
|
|
|
|
|
|
def test_detect_imperva_incapsula():
|
|
html = "<html><body>Request unsuccessful. Incapsula incident ID: 123</body></html>"
|
|
assert _detect_anti_bot(html) == "imperva"
|
|
|
|
|
|
# ----- PerimeterX detection -----
|
|
|
|
def test_detect_perimeterx_px_captcha():
|
|
html = "<html><body>px-captcha challenge</body></html>"
|
|
assert _detect_anti_bot(html) == "perimeterx"
|
|
|
|
|
|
def test_detect_perimeterx_press_hold():
|
|
html = "<html><body>Press & hold to confirm you are a human</body></html>"
|
|
assert _detect_anti_bot(html) == "perimeterx"
|
|
|
|
|
|
# ----- DataDome detection -----
|
|
|
|
def test_detect_datadome_protected():
|
|
html = "<html><body>Protected by DataDome</body></html>"
|
|
assert _detect_anti_bot(html) == "datadome"
|
|
|
|
|
|
def test_detect_datadome_cookie():
|
|
html = "<html><body>datadome cookie set</body></html>"
|
|
assert _detect_anti_bot(html) == "datadome"
|
|
|
|
|
|
# ----- Akamai detection -----
|
|
|
|
def test_detect_akamai_bm_sz():
|
|
html = "<html><body>bm_sz cookie</body></html>"
|
|
assert _detect_anti_bot(html) == "akamai"
|
|
|
|
|
|
def test_detect_akamai_reference():
|
|
html = "<html><body>Reference #123.akamaighost</body></html>"
|
|
assert _detect_anti_bot(html) == "akamai"
|
|
|
|
|
|
# ----- Generic detection -----
|
|
|
|
def test_detect_generic_captcha():
|
|
html = "<html><body>Please complete the CAPTCHA</body></html>"
|
|
assert _detect_anti_bot(html) == "generic"
|
|
|
|
|
|
def test_detect_generic_verify_human():
|
|
html = "<html><body>Verify you are human</body></html>"
|
|
assert _detect_anti_bot(html) == "generic"
|
|
|
|
|
|
def test_detect_generic_access_denied():
|
|
html = "<html><body>Access Denied</body></html>"
|
|
assert _detect_anti_bot(html) == "generic"
|
|
|
|
|
|
def test_detect_generic_blocked():
|
|
html = "<html><body>You have been blocked</body></html>"
|
|
assert _detect_anti_bot(html) == "generic"
|
|
|
|
|
|
def test_detect_generic_unusual_traffic():
|
|
html = "<html><body>unusual traffic from your computer</body></html>"
|
|
assert _detect_anti_bot(html) == "generic"
|
|
|
|
|
|
def test_detect_generic_robot():
|
|
html = "<html><body>Are you a robot?</body></html>"
|
|
assert _detect_anti_bot(html) == "generic"
|
|
|
|
|
|
# ----- Full-document scanning (v2.0.0 key improvement) -----
|
|
|
|
def test_detect_anti_bot_beyond_2000_chars():
|
|
"""反爬指示词在前 2000 字符之外也能检测到。
|
|
|
|
v2.0.0 核心改进：旧版只扫前 2000 字符，大页面反爬页可能漏检。
|
|
"""
|
|
# 构造 3000 字符的无意义填充 + 反爬关键词
|
|
padding = "x" * 2500
|
|
html = f"<html><body>{padding}<div>Just a moment...</div></body></html>"
|
|
assert _detect_anti_bot(html) == "cloudflare"
|
|
|
|
|
|
def test_detect_anti_bot_large_page_end():
|
|
"""反爬关键词在文档末尾也能检测到。"""
|
|
padding = "y" * 5000
|
|
html = f"<html><body>{padding}captcha</body></html>"
|
|
assert _detect_anti_bot(html) == "generic"
|
|
|
|
|
|
# ----- Negative cases -----
|
|
|
|
def test_detect_anti_bot_normal_page():
|
|
"""正常页面不触发检测。"""
|
|
html = "<html><body><h1>Welcome</h1><p>This is a normal article about Python programming.</p></body></html>"
|
|
assert _detect_anti_bot(html) is None
|
|
|
|
|
|
def test_detect_anti_bot_empty_content():
|
|
"""空内容返回 None。"""
|
|
assert _detect_anti_bot("") is None
|
|
assert _detect_anti_bot(None) is None
|
|
|
|
|
|
def test_detect_anti_bot_article_mentions_captcha_in_context():
|
|
"""文章讨论 captcha 但不是反爬页（上下文判断的局限——接受误报）。
|
|
|
|
注意：当前实现是关键词匹配，无法区分"讨论 captcha 的文章"和
|
|
"captcha 拦截页"。这是已知局限，测试记录此行为。
|
|
"""
|
|
html = "<html><body><p>This article explains how CAPTCHA works.</p></body></html>"
|
|
# 关键词匹配会误报为 generic
|
|
assert _detect_anti_bot(html) == "generic"
|
|
|
|
|
|
# ----- Priority: specialized WAF before generic -----
|
|
|
|
def test_detect_priority_cloudflare_over_generic():
|
|
"""同时匹配 cloudflare 和 generic 时，返回 cloudflare（优先级）。"""
|
|
# "just a moment" 是 cloudflare 专用，"captcha" 是 generic
|
|
# cloudflare 在 WAF_FINGERPRINTS 中排在 generic 之前
|
|
html = "<html><body>Just a moment... captcha</body></html>"
|
|
assert _detect_anti_bot(html) == "cloudflare"
|
|
|
|
|
|
# ----- WAF_FINGERPRINTS structure -----
|
|
|
|
def test_waf_fingerprints_has_six_types():
|
|
"""指纹库覆盖 6 种 WAF 类型。"""
|
|
types = [waf_type for waf_type, _ in WAF_FINGERPRINTS]
|
|
assert "cloudflare" in types
|
|
assert "imperva" in types
|
|
assert "perimeterx" in types
|
|
assert "datadome" in types
|
|
assert "akamai" in types
|
|
assert "generic" in types
|
|
|
|
|
|
def test_waf_fingerprints_generic_is_last():
|
|
"""generic 排在最后（优先级最低）。"""
|
|
assert WAF_FINGERPRINTS[-1][0] == "generic"
|
|
|
|
|
|
def test_waf_fingerprints_all_have_indicators():
|
|
"""每个 WAF 类型都有至少 2 个指示词。"""
|
|
for waf_type, indicators in WAF_FINGERPRINTS:
|
|
assert len(indicators) >= 2, f"{waf_type} has too few indicators"
|
|
|
|
|
|
# ----- _is_blocked_page backward compat -----
|
|
|
|
def test_is_blocked_page_delegates_to_detect():
|
|
"""_is_blocked_page 应委托给 _detect_anti_bot。"""
|
|
assert _is_blocked_page("<html>captcha</html>") is True
|
|
assert _is_blocked_page("<html>normal content</html>") is False
|
|
|
|
|
|
def test_is_blocked_page_cloudflare():
|
|
"""Cloudflare 页面被检测为 blocked。"""
|
|
assert _is_blocked_page("<html>cf-ray: 123</html>") is True
|