thzxx
d9a08716bc
Bug 1: Wayback 兜底未触发 (search.py fetch_page) - 根因: Cloudflare JS 质询页常返回 HTTP 200 (非 403), 主抓取 result 非 None - _should_try_fallback 在 result 非 None 时直接返回 False, 跳过兜底 - 反爬检测在兜底判断之后执行, 错过兜底入口 - 修复: 反爬检测提前到兜底判断之前, anti_bot_detected=True 也触发 Wayback - Wayback 结果重新做反爬检测 (防御性) Bug 2: WAF 指纹库误判正常内容 (search.py WAF_FINGERPRINTS) - 根因: 裸公司名 (cloudflare/akamai) 和宽泛词 (captcha/challenge/dd-) 做全文匹配 - DataCamp 文章引用 cloudflare.com 文档链接 -> 误判为 cloudflare WAF - 'coding challenges' 正常内容 -> 误判为 generic 反爬 - Wayback 归档正文被误判, 兜底返回的有效内容被丢弃 - 修复: 移除裸公司名和宽泛词, 改用技术标识符 (cf-ray/incap_ses/bm_sz 等) - 通用文案用完整短语 (please complete the captcha) 替代单词 - 增加 <title> 标签精准检测 (反爬页 title 是特征文案, 误判率极低) - 新增 Anubis 反爬系统检测 (anubis_challenge/miserere) 真实测试验证 (search.metona.cn 实例): - v2.0.0: --fetch 3 全部失败 (3 ERR: cloudflare/generic, Wayback 未触发) - v2.0.1: --fetch 3 全部成功 (3 OK: 38100/93442/1945 chars, UA 轮换绕过 Cloudflare) 测试: 458 个全部通过 (新增 7 个测试覆盖修复行为)
238 lines
8.2 KiB
Python
238 lines
8.2 KiB
Python
"""Tests for v2.0.0 anti-bot / WAF detection enhancements.
|
||
|
||
Covers: _detect_anti_bot (WAF fingerprint library: Cloudflare/Imperva/
|
||
PerimeterX/DataDome/Akamai/generic), full-document scanning (not just
|
||
first 2000 chars), _is_blocked_page backward compatibility.
|
||
"""
|
||
import sys
|
||
import os
|
||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "scripts"))
|
||
|
||
from search import _detect_anti_bot, _is_blocked_page, WAF_FINGERPRINTS
|
||
|
||
|
||
# ----- Cloudflare detection -----
|
||
|
||
def test_detect_cloudflare_cf_ray():
|
||
html = "<html><head><title>Just a moment...</title></head>" \
|
||
"<body>cf-ray: 8abc123</body></html>"
|
||
assert _detect_anti_bot(html) == "cloudflare"
|
||
|
||
|
||
def test_detect_cloudflare_just_a_moment():
|
||
html = "<html><body>Just a moment...</body></html>"
|
||
assert _detect_anti_bot(html) == "cloudflare"
|
||
|
||
|
||
def test_detect_cloudflare_checking_browser():
|
||
html = "<html><body>Checking your browser before accessing</body></html>"
|
||
assert _detect_anti_bot(html) == "cloudflare"
|
||
|
||
|
||
def test_detect_cloudflare_attention_required():
|
||
html = "<html><body>Attention Required! | Cloudflare</body></html>"
|
||
assert _detect_anti_bot(html) == "cloudflare"
|
||
|
||
|
||
# ----- Imperva detection -----
|
||
|
||
def test_detect_imperva_incap_ses():
|
||
html = "<html><body>incap_ses_123_cookie</body></html>"
|
||
assert _detect_anti_bot(html) == "imperva"
|
||
|
||
|
||
def test_detect_imperva_incapsula():
|
||
html = "<html><body>Request unsuccessful. Incapsula incident ID: 123</body></html>"
|
||
assert _detect_anti_bot(html) == "imperva"
|
||
|
||
|
||
# ----- PerimeterX detection -----
|
||
|
||
def test_detect_perimeterx_px_captcha():
|
||
html = "<html><body>px-captcha challenge</body></html>"
|
||
assert _detect_anti_bot(html) == "perimeterx"
|
||
|
||
|
||
def test_detect_perimeterx_press_hold():
|
||
html = "<html><body>Press & hold to confirm you are a human</body></html>"
|
||
assert _detect_anti_bot(html) == "perimeterx"
|
||
|
||
|
||
# ----- DataDome detection -----
|
||
|
||
def test_detect_datadome_protected():
|
||
html = "<html><body>Protected by DataDome</body></html>"
|
||
assert _detect_anti_bot(html) == "datadome"
|
||
|
||
|
||
def test_detect_datadome_cookie():
|
||
html = "<html><body>datadome cookie set</body></html>"
|
||
assert _detect_anti_bot(html) == "datadome"
|
||
|
||
|
||
# ----- Akamai detection -----
|
||
|
||
def test_detect_akamai_bm_sz():
|
||
html = "<html><body>bm_sz cookie</body></html>"
|
||
assert _detect_anti_bot(html) == "akamai"
|
||
|
||
|
||
def test_detect_akamai_reference():
|
||
html = "<html><body>Reference #123.akamaighost</body></html>"
|
||
assert _detect_anti_bot(html) == "akamai"
|
||
|
||
|
||
# ----- Generic detection -----
|
||
|
||
def test_detect_generic_captcha():
|
||
"""v2.0.1：收窄为完整短语 'please complete the captcha'，避免正文误判。"""
|
||
html = "<html><body>Please complete the CAPTCHA</body></html>"
|
||
assert _detect_anti_bot(html) == "generic"
|
||
|
||
|
||
def test_detect_generic_verify_human():
|
||
html = "<html><body>Verify you are human</body></html>"
|
||
assert _detect_anti_bot(html) == "generic"
|
||
|
||
|
||
def test_detect_generic_access_denied():
|
||
"""v2.0.1：裸 'access denied' 从全文档扫描移除（避免权限文章误判），
|
||
改由 <title> 标签检测覆盖。反爬页 title 常为 'Access Denied'。"""
|
||
html = "<html><head><title>Access Denied</title></head><body>Access Denied</body></html>"
|
||
assert _detect_anti_bot(html) == "generic"
|
||
|
||
|
||
def test_detect_generic_access_denied_sucuri():
|
||
"""v2.0.1：Sucuri WAF 的完整文案在全文档扫描中保留。"""
|
||
html = "<html><body>Access Denied - Sucuri Website Firewall</body></html>"
|
||
assert _detect_anti_bot(html) == "generic"
|
||
|
||
|
||
def test_detect_generic_blocked():
|
||
html = "<html><body>You have been blocked</body></html>"
|
||
assert _detect_anti_bot(html) == "generic"
|
||
|
||
|
||
def test_detect_generic_unusual_traffic():
|
||
html = "<html><body>unusual traffic from your computer</body></html>"
|
||
assert _detect_anti_bot(html) == "generic"
|
||
|
||
|
||
def test_detect_generic_robot():
|
||
html = "<html><body>Are you a robot?</body></html>"
|
||
assert _detect_anti_bot(html) == "generic"
|
||
|
||
|
||
# ----- Full-document scanning (v2.0.0 key improvement) -----
|
||
|
||
def test_detect_anti_bot_beyond_2000_chars():
|
||
"""反爬指示词在前 2000 字符之外也能检测到。
|
||
|
||
v2.0.0 核心改进：旧版只扫前 2000 字符，大页面反爬页可能漏检。
|
||
"""
|
||
# 构造 3000 字符的无意义填充 + 反爬关键词
|
||
padding = "x" * 2500
|
||
html = f"<html><body>{padding}<div>Just a moment...</div></body></html>"
|
||
assert _detect_anti_bot(html) == "cloudflare"
|
||
|
||
|
||
def test_detect_anti_bot_large_page_end():
|
||
"""反爬关键词在文档末尾也能检测到。"""
|
||
padding = "y" * 5000
|
||
# v2.0.1：用完整短语而非裸 captcha，避免误判
|
||
html = f"<html><body>{padding}please complete the captcha</body></html>"
|
||
assert _detect_anti_bot(html) == "generic"
|
||
|
||
|
||
# ----- Negative cases -----
|
||
|
||
def test_detect_anti_bot_normal_page():
|
||
"""正常页面不触发检测。"""
|
||
html = "<html><body><h1>Welcome</h1><p>This is a normal article about Python programming.</p></body></html>"
|
||
assert _detect_anti_bot(html) is None
|
||
|
||
|
||
def test_detect_anti_bot_empty_content():
|
||
"""空内容返回 None。"""
|
||
assert _detect_anti_bot("") is None
|
||
assert _detect_anti_bot(None) is None
|
||
|
||
|
||
def test_detect_anti_bot_article_mentions_captcha_in_context():
|
||
"""v2.0.1 修复：文章讨论 captcha 但不是反爬页，不再误判。
|
||
|
||
v2.0.0 用裸 "captcha" 做全文匹配，导致"讨论 captcha 工作原理的文章"
|
||
被误判为反爬页。v2.0.1 收窄为 "please complete the captcha" 等完整短语，
|
||
正常讨论 captcha 的文章不再触发。
|
||
"""
|
||
html = "<html><body><p>This article explains how CAPTCHA works.</p></body></html>"
|
||
# v2.0.1：收窄后不再误判
|
||
assert _detect_anti_bot(html) is None
|
||
|
||
|
||
def test_detect_anti_bot_article_mentions_cloudflare_in_context():
|
||
"""v2.0.1 修复：文章引用 Cloudflare 文档链接，不再误判为 cloudflare WAF。
|
||
|
||
v2.0.0 用裸 "cloudflare" 做全文匹配，Wayback 归档的 DataCamp 文章
|
||
正文里有 cloudflare.com 链接，被误判为反爬页导致兜底失败。
|
||
"""
|
||
html = ("<html><body><article>"
|
||
"<p>Learn more at <a href='https://www.cloudflare.com/learn/'>Cloudflare</a></p>"
|
||
"<p>Join our daily coding challenges!</p>"
|
||
"</article></body></html>")
|
||
assert _detect_anti_bot(html) is None
|
||
|
||
|
||
def test_detect_anti_bot_article_mentions_challenge_in_context():
|
||
"""v2.0.1 修复：文章含 'coding challenge' 等正常内容，不再误判。"""
|
||
html = "<html><body><p>Daily 5-minute coding challenges.</p></body></html>"
|
||
assert _detect_anti_bot(html) is None
|
||
|
||
|
||
# ----- Priority: specialized WAF before generic -----
|
||
|
||
def test_detect_priority_cloudflare_over_generic():
|
||
"""同时匹配 cloudflare 和 generic 时，返回 cloudflare（优先级）。"""
|
||
# "just a moment" 是 cloudflare 专用，"captcha" 是 generic
|
||
# cloudflare 在 WAF_FINGERPRINTS 中排在 generic 之前
|
||
html = "<html><body>Just a moment... captcha</body></html>"
|
||
assert _detect_anti_bot(html) == "cloudflare"
|
||
|
||
|
||
# ----- WAF_FINGERPRINTS structure -----
|
||
|
||
def test_waf_fingerprints_has_six_types():
|
||
"""指纹库覆盖 6 种 WAF 类型。"""
|
||
types = [waf_type for waf_type, _ in WAF_FINGERPRINTS]
|
||
assert "cloudflare" in types
|
||
assert "imperva" in types
|
||
assert "perimeterx" in types
|
||
assert "datadome" in types
|
||
assert "akamai" in types
|
||
assert "generic" in types
|
||
|
||
|
||
def test_waf_fingerprints_generic_is_last():
|
||
"""generic 排在最后（优先级最低）。"""
|
||
assert WAF_FINGERPRINTS[-1][0] == "generic"
|
||
|
||
|
||
def test_waf_fingerprints_all_have_indicators():
|
||
"""每个 WAF 类型都有至少 2 个指示词。"""
|
||
for waf_type, indicators in WAF_FINGERPRINTS:
|
||
assert len(indicators) >= 2, f"{waf_type} has too few indicators"
|
||
|
||
|
||
# ----- _is_blocked_page backward compat -----
|
||
|
||
def test_is_blocked_page_delegates_to_detect():
|
||
"""_is_blocked_page 应委托给 _detect_anti_bot。"""
|
||
# v2.0.1：用完整反爬短语而非裸 captcha
|
||
assert _is_blocked_page("<html><body>please complete the captcha</body></html>") is True
|
||
assert _is_blocked_page("<html>normal content</html>") is False
|
||
|
||
|
||
def test_is_blocked_page_cloudflare():
|
||
"""Cloudflare 页面被检测为 blocked。"""
|
||
assert _is_blocked_page("<html>cf-ray: 123</html>") is True
|