"""Tests for v2.0.0 anti-bot / WAF detection enhancements.
Covers: _detect_anti_bot (WAF fingerprint library: Cloudflare/Imperva/
PerimeterX/DataDome/Akamai/generic), full-document scanning (not just
first 2000 chars), _is_blocked_page backward compatibility.
"""
import sys
import os
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "scripts"))
from search import _detect_anti_bot, _is_blocked_page, WAF_FINGERPRINTS
# ----- Cloudflare detection -----
def test_detect_cloudflare_cf_ray():
html = "Just a moment..." \
"cf-ray: 8abc123"
assert _detect_anti_bot(html) == "cloudflare"
def test_detect_cloudflare_just_a_moment():
html = "Just a moment..."
assert _detect_anti_bot(html) == "cloudflare"
def test_detect_cloudflare_checking_browser():
html = "Checking your browser before accessing"
assert _detect_anti_bot(html) == "cloudflare"
def test_detect_cloudflare_attention_required():
html = "Attention Required! | Cloudflare"
assert _detect_anti_bot(html) == "cloudflare"
# ----- Imperva detection -----
def test_detect_imperva_incap_ses():
html = "incap_ses_123_cookie"
assert _detect_anti_bot(html) == "imperva"
def test_detect_imperva_incapsula():
html = "Request unsuccessful. Incapsula incident ID: 123"
assert _detect_anti_bot(html) == "imperva"
# ----- PerimeterX detection -----
def test_detect_perimeterx_px_captcha():
html = "px-captcha challenge"
assert _detect_anti_bot(html) == "perimeterx"
def test_detect_perimeterx_press_hold():
html = "Press & hold to confirm you are a human"
assert _detect_anti_bot(html) == "perimeterx"
# ----- DataDome detection -----
def test_detect_datadome_protected():
html = "Protected by DataDome"
assert _detect_anti_bot(html) == "datadome"
def test_detect_datadome_cookie():
html = "datadome cookie set"
assert _detect_anti_bot(html) == "datadome"
# ----- Akamai detection -----
def test_detect_akamai_bm_sz():
html = "bm_sz cookie"
assert _detect_anti_bot(html) == "akamai"
def test_detect_akamai_reference():
html = "Reference #123.akamaighost"
assert _detect_anti_bot(html) == "akamai"
# ----- Generic detection -----
def test_detect_generic_captcha():
html = "Please complete the CAPTCHA"
assert _detect_anti_bot(html) == "generic"
def test_detect_generic_verify_human():
html = "Verify you are human"
assert _detect_anti_bot(html) == "generic"
def test_detect_generic_access_denied():
html = "Access Denied"
assert _detect_anti_bot(html) == "generic"
def test_detect_generic_blocked():
html = "You have been blocked"
assert _detect_anti_bot(html) == "generic"
def test_detect_generic_unusual_traffic():
html = "unusual traffic from your computer"
assert _detect_anti_bot(html) == "generic"
def test_detect_generic_robot():
html = "Are you a robot?"
assert _detect_anti_bot(html) == "generic"
# ----- Full-document scanning (v2.0.0 key improvement) -----
def test_detect_anti_bot_beyond_2000_chars():
"""反爬指示词在前 2000 字符之外也能检测到。
v2.0.0 核心改进：旧版只扫前 2000 字符，大页面反爬页可能漏检。
"""
# 构造 3000 字符的无意义填充 + 反爬关键词
padding = "x" * 2500
html = f"{padding}Just a moment...
"
assert _detect_anti_bot(html) == "cloudflare"
def test_detect_anti_bot_large_page_end():
"""反爬关键词在文档末尾也能检测到。"""
padding = "y" * 5000
html = f"{padding}captcha"
assert _detect_anti_bot(html) == "generic"
# ----- Negative cases -----
def test_detect_anti_bot_normal_page():
"""正常页面不触发检测。"""
html = "Welcome
This is a normal article about Python programming.
"
assert _detect_anti_bot(html) is None
def test_detect_anti_bot_empty_content():
"""空内容返回 None。"""
assert _detect_anti_bot("") is None
assert _detect_anti_bot(None) is None
def test_detect_anti_bot_article_mentions_captcha_in_context():
"""文章讨论 captcha 但不是反爬页（上下文判断的局限——接受误报）。
注意：当前实现是关键词匹配，无法区分"讨论 captcha 的文章"和
"captcha 拦截页"。这是已知局限，测试记录此行为。
"""
html = "This article explains how CAPTCHA works.
"
# 关键词匹配会误报为 generic
assert _detect_anti_bot(html) == "generic"
# ----- Priority: specialized WAF before generic -----
def test_detect_priority_cloudflare_over_generic():
"""同时匹配 cloudflare 和 generic 时，返回 cloudflare（优先级）。"""
# "just a moment" 是 cloudflare 专用，"captcha" 是 generic
# cloudflare 在 WAF_FINGERPRINTS 中排在 generic 之前
html = "Just a moment... captcha"
assert _detect_anti_bot(html) == "cloudflare"
# ----- WAF_FINGERPRINTS structure -----
def test_waf_fingerprints_has_six_types():
"""指纹库覆盖 6 种 WAF 类型。"""
types = [waf_type for waf_type, _ in WAF_FINGERPRINTS]
assert "cloudflare" in types
assert "imperva" in types
assert "perimeterx" in types
assert "datadome" in types
assert "akamai" in types
assert "generic" in types
def test_waf_fingerprints_generic_is_last():
"""generic 排在最后（优先级最低）。"""
assert WAF_FINGERPRINTS[-1][0] == "generic"
def test_waf_fingerprints_all_have_indicators():
"""每个 WAF 类型都有至少 2 个指示词。"""
for waf_type, indicators in WAF_FINGERPRINTS:
assert len(indicators) >= 2, f"{waf_type} has too few indicators"
# ----- _is_blocked_page backward compat -----
def test_is_blocked_page_delegates_to_detect():
"""_is_blocked_page 应委托给 _detect_anti_bot。"""
assert _is_blocked_page("captcha") is True
assert _is_blocked_page("normal content") is False
def test_is_blocked_page_cloudflare():
"""Cloudflare 页面被检测为 blocked。"""
assert _is_blocked_page("cf-ray: 123") is True