thzxx
157219d982
核心修复（v2.2.1）: - 修复 Brotli 乱码 bug: build_browser_headers 智能声明 Accept-Encoding, 仅在 brotli 可用时才声明 br; fetch.py 双路径 br 解压（requests + stdlib） 此前 Chrome/Edge UA 抓取 example.com 等返回 br 的站点输出乱码 v2.2.0 新功能: - main() 拆分为 _handle_verify/_handle_research/_handle_batch/_handle_single - --cache-max-size MB: 缓存大小上限 + LRU 淘汰（默认 100MB） - --pages N: 多页聚合 + 跨页去重 - --research 跨角度合并: 新增 merged_results 字段 - --stream / --progress: JSON Lines 流式输出 + request_id 贯穿 - --dry-run / --save-config / --log-format json - --similarity-dedup / --throttle-* 参数化 - 15-UA 池 + PDF/docx 解析 + error_code 字段 文档与测试: - SKILL.md: 版本号唯一（元数据），删除版本标记干扰 - README.md: 测试数量 539 -> 544 - 544 passed (新增 5 个 Content-Encoding 解压测试)
338 lines
11 KiB
Python
338 lines
11 KiB
Python
"""Tests for v2.0.0 browser fingerprint headers and UA pool enhancements.
|
||
|
||
Covers: build_browser_headers (Chrome/Edge/Firefox variants, Referer,
|
||
Accept modes), get_ua_for_domain (deterministic per-domain, caching,
|
||
explicit override), parse_retry_after (numeric/HTTP date/edge cases),
|
||
compute_backoff_delay (cap enforcement), UA pool size and diversity.
|
||
"""
|
||
import sys
|
||
import os
|
||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "scripts"))
|
||
|
||
from common import (
|
||
FALLBACK_UAS,
|
||
RETRY_BACKOFF_CAP,
|
||
build_browser_headers,
|
||
compute_backoff_delay,
|
||
get_ua_for_domain,
|
||
parse_retry_after,
|
||
reset_domain_ua_cache,
|
||
_ua_index_for_domain,
|
||
)
|
||
|
||
|
||
# ----- UA pool -----
|
||
|
||
def test_ua_pool_has_at_least_12_entries():
|
||
"""v2.0.0: expanded from 3 to 12 for diversity."""
|
||
assert len(FALLBACK_UAS) >= 12
|
||
|
||
|
||
def test_ua_pool_covers_multiple_browsers():
|
||
browsers = []
|
||
for ua in FALLBACK_UAS:
|
||
if "Edg/" in ua:
|
||
browsers.append("edge")
|
||
elif "Firefox/" in ua:
|
||
browsers.append("firefox")
|
||
elif "Chrome/" in ua:
|
||
browsers.append("chrome")
|
||
# 至少三种浏览器
|
||
assert "chrome" in browsers
|
||
assert "firefox" in browsers
|
||
assert "edge" in browsers
|
||
|
||
|
||
def test_ua_pool_covers_multiple_platforms():
|
||
platforms = []
|
||
for ua in FALLBACK_UAS:
|
||
if "Windows" in ua:
|
||
platforms.append("windows")
|
||
elif "Macintosh" in ua:
|
||
platforms.append("macos")
|
||
elif "Linux" in ua:
|
||
platforms.append("linux")
|
||
assert "windows" in platforms
|
||
assert "macos" in platforms
|
||
assert "linux" in platforms
|
||
|
||
|
||
# ----- get_ua_for_domain -----
|
||
|
||
def test_get_ua_for_domain_is_deterministic():
|
||
"""同一域名永远返回同一 UA（跨调用一致）。"""
|
||
reset_domain_ua_cache()
|
||
url = "https://example.com/page1"
|
||
ua1 = get_ua_for_domain(url)
|
||
ua2 = get_ua_for_domain(url)
|
||
assert ua1 == ua2
|
||
assert ua1 in FALLBACK_UAS
|
||
|
||
|
||
def test_get_ua_for_domain_same_domain_different_paths():
|
||
"""同域名不同路径返回同 UA。"""
|
||
reset_domain_ua_cache()
|
||
ua1 = get_ua_for_domain("https://example.com/a")
|
||
ua2 = get_ua_for_domain("https://example.com/b/c/d")
|
||
assert ua1 == ua2
|
||
|
||
|
||
def test_get_ua_for_domain_explicit_override():
|
||
"""显式 user_agent 优先于域名缓存。"""
|
||
reset_domain_ua_cache()
|
||
custom = "MyCustomBot/1.0"
|
||
ua = get_ua_for_domain("https://example.com", user_agent=custom)
|
||
assert ua == custom
|
||
|
||
|
||
def test_get_ua_for_domain_different_domains_may_differ():
|
||
"""不同域名可能映射到不同 UA（不一定，但缓存独立）。"""
|
||
reset_domain_ua_cache()
|
||
ua1 = get_ua_for_domain("https://aaa.example.com")
|
||
ua2 = get_ua_for_domain("https://bbb.example.com")
|
||
# 都是合法 UA
|
||
assert ua1 in FALLBACK_UAS
|
||
assert ua2 in FALLBACK_UAS
|
||
|
||
|
||
def test_get_ua_for_domain_invalid_url_returns_default():
|
||
"""无效 URL 返回第一个 UA（兜底）。"""
|
||
reset_domain_ua_cache()
|
||
ua = get_ua_for_domain("not-a-url")
|
||
assert ua == FALLBACK_UAS[0]
|
||
|
||
|
||
def test_get_ua_for_domain_caches_across_calls():
|
||
"""缓存生效：第二次调用不重新计算。"""
|
||
reset_domain_ua_cache()
|
||
url = "https://cached.example.com"
|
||
ua1 = get_ua_for_domain(url)
|
||
# 直接从缓存取
|
||
from common import _domain_ua_cache
|
||
domain_key = "cached.example.com"
|
||
assert domain_key in _domain_ua_cache
|
||
assert _domain_ua_cache[domain_key] == ua1
|
||
|
||
|
||
def test_ua_index_for_domain_is_stable_across_processes():
|
||
"""SHA-256 hash 保证跨进程一致（不像内置 hash 受 PYTHONHASHSEED 影响）。"""
|
||
idx1 = _ua_index_for_domain("example.com", len(FALLBACK_UAS))
|
||
idx2 = _ua_index_for_domain("example.com", len(FALLBACK_UAS))
|
||
assert idx1 == idx2
|
||
assert 0 <= idx1 < len(FALLBACK_UAS)
|
||
|
||
|
||
# ----- build_browser_headers -----
|
||
|
||
def test_build_headers_chrome_includes_sec_ch_ua():
|
||
"""Chrome UA 应生成 Sec-Ch-Ua 系列头。"""
|
||
chrome_ua = FALLBACK_UAS[0] # Chrome 131 Windows
|
||
headers = build_browser_headers(chrome_ua)
|
||
assert "Sec-Ch-Ua" in headers
|
||
assert "Sec-Ch-Ua-Mobile" in headers
|
||
assert "Sec-Ch-Ua-Platform" in headers
|
||
assert "Windows" in headers["Sec-Ch-Ua-Platform"]
|
||
|
||
|
||
def test_build_headers_firefox_excludes_sec_ch_ua():
|
||
"""Firefox UA 不应生成 Sec-Ch-Ua（Firefox 不发送此头）。"""
|
||
firefox_ua = next(ua for ua in FALLBACK_UAS if "Firefox/" in ua)
|
||
headers = build_browser_headers(firefox_ua)
|
||
assert "Sec-Ch-Ua" not in headers
|
||
assert "Sec-Ch-Ua-Mobile" not in headers
|
||
|
||
|
||
def test_build_headers_includes_accept_language():
|
||
"""所有浏览器都应有 Accept-Language。"""
|
||
headers = build_browser_headers(FALLBACK_UAS[0])
|
||
assert "Accept-Language" in headers
|
||
assert "en-US" in headers["Accept-Language"]
|
||
|
||
|
||
def test_build_headers_includes_accept_encoding():
|
||
"""Accept-Encoding 必须存在；Firefox 不发 br；Chrome/Edge 仅在
|
||
本机安装了 brotli 解压器时才声明 br（v2.2.1 智能声明）。
|
||
|
||
未安装 brotli 时声明 br 会导致服务器返回 br 压缩字节而 requests
|
||
无法解压 → 全页乱码。这是 fetch.py 真实环境的 bug 修复。
|
||
"""
|
||
# 导入 common 的 brotli 检测状态
|
||
from common import _HAS_BROTLI
|
||
|
||
chrome_headers = build_browser_headers(FALLBACK_UAS[0])
|
||
# Chrome/Edge：根据 brotli 可用性决定是否声明 br
|
||
if _HAS_BROTLI:
|
||
assert "br" in chrome_headers["Accept-Encoding"]
|
||
else:
|
||
assert "br" not in chrome_headers["Accept-Encoding"]
|
||
assert "gzip" in chrome_headers["Accept-Encoding"]
|
||
|
||
firefox_ua = next(ua for ua in FALLBACK_UAS if "Firefox/" in ua)
|
||
firefox_headers = build_browser_headers(firefox_ua)
|
||
assert "br" not in firefox_headers["Accept-Encoding"]
|
||
assert "gzip" in firefox_headers["Accept-Encoding"]
|
||
|
||
|
||
def test_build_headers_html_mode_includes_upgrade_insecure():
|
||
"""HTML 模式下 Upgrade-Insecure-Requests=1。"""
|
||
headers = build_browser_headers(FALLBACK_UAS[0], accept_html=True)
|
||
assert headers["Upgrade-Insecure-Requests"] == "1"
|
||
|
||
|
||
def test_build_headers_json_mode_excludes_upgrade_insecure_navigation():
|
||
"""JSON 模式下不发送导航相关头。"""
|
||
headers = build_browser_headers(FALLBACK_UAS[0], accept_html=False)
|
||
assert headers["Upgrade-Insecure-Requests"] == "0"
|
||
assert headers["Sec-Fetch-Mode"] == "cors"
|
||
assert headers["Sec-Fetch-Dest"] == "empty"
|
||
|
||
|
||
def test_build_headers_sec_fetch_dest_document_for_html():
|
||
"""HTML 模式 Sec-Fetch-Dest=document。"""
|
||
headers = build_browser_headers(FALLBACK_UAS[0], accept_html=True)
|
||
assert headers["Sec-Fetch-Dest"] == "document"
|
||
assert headers["Sec-Fetch-Mode"] == "navigate"
|
||
|
||
|
||
def test_build_headers_referer_set_when_provided():
|
||
"""传入 referer 时设置 Referer 头。"""
|
||
headers = build_browser_headers(FALLBACK_UAS[0],
|
||
referer="https://google.com/")
|
||
assert headers["Referer"] == "https://google.com/"
|
||
|
||
|
||
def test_build_headers_no_referer_when_absent():
|
||
"""不传 referer 时不设置 Referer 头。"""
|
||
headers = build_browser_headers(FALLBACK_UAS[0])
|
||
assert "Referer" not in headers
|
||
|
||
|
||
def test_build_headers_sec_fetch_site_none_without_referer():
|
||
"""无 Referer 时 Sec-Fetch-Site=none（像地址栏直接访问）。"""
|
||
headers = build_browser_headers(FALLBACK_UAS[0], accept_html=True)
|
||
assert headers["Sec-Fetch-Site"] == "none"
|
||
|
||
|
||
def test_build_headers_sec_fetch_site_cross_site_with_referer():
|
||
"""有 Referer 时 Sec-Fetch-Site=cross-site。"""
|
||
headers = build_browser_headers(FALLBACK_UAS[0],
|
||
accept_html=True,
|
||
referer="https://google.com/")
|
||
assert headers["Sec-Fetch-Site"] == "cross-site"
|
||
|
||
|
||
def test_build_headers_edge_includes_edge_brand():
|
||
"""Edge UA 的 Sec-Ch-Ua 应包含 Microsoft Edge 品牌。"""
|
||
edge_ua = next(ua for ua in FALLBACK_UAS if "Edg/" in ua)
|
||
headers = build_browser_headers(edge_ua)
|
||
assert "Microsoft Edge" in headers["Sec-Ch-Ua"]
|
||
|
||
|
||
def test_build_headers_macos_platform():
|
||
"""macOS UA 的 Sec-Ch-Ua-Platform=macOS。"""
|
||
mac_ua = next(ua for ua in FALLBACK_UAS if "Macintosh" in ua and "Edg/" not in ua)
|
||
headers = build_browser_headers(mac_ua)
|
||
assert "macOS" in headers["Sec-Ch-Ua-Platform"]
|
||
|
||
|
||
def test_build_headers_linux_platform():
|
||
"""Linux UA 的 Sec-Ch-Ua-Platform=Linux。"""
|
||
linux_ua = next(ua for ua in FALLBACK_UAS if "Linux" in ua and "Firefox/" not in ua)
|
||
headers = build_browser_headers(linux_ua)
|
||
assert "Linux" in headers["Sec-Ch-Ua-Platform"]
|
||
|
||
|
||
def test_build_headers_user_agent_set():
|
||
"""UA 必须设置到 User-Agent 头。"""
|
||
headers = build_browser_headers(FALLBACK_UAS[0])
|
||
assert headers["User-Agent"] == FALLBACK_UAS[0]
|
||
|
||
|
||
def test_build_headers_connection_keep_alive():
|
||
"""Connection: keep-alive 支持 HTTP 持久连接。"""
|
||
headers = build_browser_headers(FALLBACK_UAS[0])
|
||
assert headers["Connection"] == "keep-alive"
|
||
|
||
|
||
# ----- parse_retry_after -----
|
||
|
||
def test_parse_retry_after_numeric_seconds():
|
||
"""纯数字格式：秒数。"""
|
||
assert parse_retry_after("30") == 30.0
|
||
assert parse_retry_after("0") == 0.0
|
||
assert parse_retry_after("120") == 120.0
|
||
|
||
|
||
def test_parse_retry_after_decimal():
|
||
"""小数秒数。"""
|
||
assert parse_retry_after("1.5") == 1.5
|
||
|
||
|
||
def test_parse_retry_after_empty():
|
||
"""空字符串返回 0。"""
|
||
assert parse_retry_after("") == 0.0
|
||
assert parse_retry_after(None) == 0.0
|
||
|
||
|
||
def test_parse_retry_after_http_date_future():
|
||
"""HTTP date 格式（未来时间）返回正秒数。"""
|
||
from datetime import datetime, timezone, timedelta
|
||
future = datetime.now(timezone.utc) + timedelta(seconds=60)
|
||
from email.utils import format_datetime
|
||
date_str = format_datetime(future)
|
||
seconds = parse_retry_after(date_str)
|
||
assert 50 < seconds < 70 # 允许一点时间漂移
|
||
|
||
|
||
def test_parse_retry_after_http_date_past():
|
||
"""HTTP date 格式（过去时间）返回 0（已过期）。"""
|
||
from datetime import datetime, timezone, timedelta
|
||
past = datetime.now(timezone.utc) - timedelta(seconds=60)
|
||
from email.utils import format_datetime
|
||
date_str = format_datetime(past)
|
||
assert parse_retry_after(date_str) == 0.0
|
||
|
||
|
||
def test_parse_retry_after_garbage():
|
||
"""无法解析的值返回 0。"""
|
||
assert parse_retry_after("not-a-date-or-number") == 0.0
|
||
|
||
|
||
def test_parse_retry_after_negative_numeric():
|
||
"""负数秒返回 0（不允许负等待）。"""
|
||
assert parse_retry_after("-5") == 0.0
|
||
|
||
|
||
# ----- compute_backoff_delay -----
|
||
|
||
def test_backoff_caps_at_60_seconds():
|
||
"""退避延迟不超过 60s 上限。"""
|
||
# attempt=20 会产生 1.5*2^20 ≈ 1.5M，远超上限
|
||
delay = compute_backoff_delay(20)
|
||
assert delay <= RETRY_BACKOFF_CAP
|
||
|
||
|
||
def test_backoff_increases_with_attempt():
|
||
"""退避延迟随 attempt 增加（允许抖动误差）。"""
|
||
# 多次取均值避免抖动干扰
|
||
import random
|
||
random.seed(42)
|
||
delays = [compute_backoff_delay(0) for _ in range(100)]
|
||
avg0 = sum(delays) / len(delays)
|
||
random.seed(42)
|
||
delays = [compute_backoff_delay(3) for _ in range(100)]
|
||
avg3 = sum(delays) / len(delays)
|
||
assert avg3 > avg0
|
||
|
||
|
||
def test_backoff_custom_cap():
|
||
"""自定义上限生效。"""
|
||
delay = compute_backoff_delay(20, cap=5.0)
|
||
assert delay <= 5.0
|
||
|
||
|
||
def test_backoff_attempt_zero_positive():
|
||
"""attempt=0 时延迟为正。"""
|
||
delay = compute_backoff_delay(0)
|
||
assert delay > 0
|