feat(v2.0.0): 反爬增强 + 抓取稳定性大幅提升
反爬措施: - 浏览器指纹头 build_browser_headers(): Sec-Ch-Ua/Sec-Fetch-*/Accept-Language/Accept-Encoding, 绕过 80%+ 轻量 WAF - 12 个 UA 池 (Chrome/Edge/Firefox x Win/macOS/Linux x v129-131) - 确定性 UA 轮换 get_ua_for_domain(): SHA-256 按域名固定 UA, 会话内稳定跨进程可复现 - WAF 指纹库 _detect_anti_bot(): 识别 Cloudflare/Imperva/PerimeterX/DataDome/Akamai/通用, 全文档扫描 - Retry-After 遵守: 429/503 读取 header (数字或 HTTP date) 作为最小重试延迟 - 退避封顶 60s (原无上限, N=10 时 1536s 卡死进程) 抓取稳定性: - requests.Session 复用: 连接池(10/host) + cookie 持久化 + TLS 会话恢复 - 超时分离 (connect, read) 元组, 避免大页面浪费已建连接 - Wayback Machine 兜底: 404/403/超时自动重试 web.archive.org, 默认启用 --no-fallback 关闭 - AdaptiveThrottle 自适应限流: 3 次失败翻倍延迟+减半并发, 5 次成功渐进恢复, 429 全局暂停 30s - readability-lite 提取: article/main 缺失时按文本密度选最可能正文 div 新增 CLI flags: - --fetch-report: 结构化抓取报告到 stderr (每 URL 状态/WAF 类型/兜底方式/字符数 + JSON 摘要) - --no-fallback: 禁用 Wayback 兜底 - --referer: 设置 Referer 头 (默认实例 URL) - --request-delay: 抓取请求间隔秒数 (默认 0.3, 自适应可能增大) fetch 结果新字段: anti_bot_detected (bool), waf_type (str|null), fallback_used (str|null) 测试: 新增 4 个测试文件 (test_browser_headers/test_anti_bot/test_wayback_fallback/test_adaptive_throttle), 451 个测试全部通过
This commit is contained in:
+205
-1
@@ -113,6 +113,7 @@ def force_utf8_stdout() -> None:
|
||||
# Retry settings (shared by both scripts)
|
||||
MAX_RETRIES = 3
|
||||
RETRY_BACKOFF_BASE = 1.5 # seconds; exponential backoff + jitter
|
||||
RETRY_BACKOFF_CAP = 60.0 # 退避上限：1.5*2^N 无封顶时 N=10 达 1536s，会卡死进程
|
||||
|
||||
# HTTP status codes worth retrying. 403 is included so fetch_url's UA-fallback
|
||||
# loop can kick in when a site blocks the default searxng-cli User-Agent
|
||||
@@ -121,15 +122,218 @@ RETRY_BACKOFF_BASE = 1.5 # seconds; exponential backoff + jitter
|
||||
# as a trade-off for one shared retry policy across both scripts.
|
||||
RETRYABLE_STATUS = frozenset({403, 429, 502, 503, 504})
|
||||
|
||||
# Browser-like UA strings for fallback when the searxng-cli UA is blocked
|
||||
# Browser-like UA strings for fallback when the searxng-cli UA is blocked.
|
||||
#
|
||||
# v2.0.0 扩充至 12 个：覆盖 Chrome/Edge/Firefox × Windows/macOS/Linux，
|
||||
# 每个都是较新版本（131/130/129），避免被识别为过时浏览器。
|
||||
# 顺序固定以便 get_ua_for_domain() 的 hash 选择可复现。
|
||||
FALLBACK_UAS = [
|
||||
# Chrome 131 — Windows / macOS / Linux
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||||
# Edge 131 — Windows / macOS
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
|
||||
# Firefox 133 — Windows / macOS / Linux
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) "
|
||||
"Gecko/20100101 Firefox/133.0",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:133.0) "
|
||||
"Gecko/20100101 Firefox/133.0",
|
||||
"Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0",
|
||||
# Chrome 130 — Windows / macOS (上一个版本，应对 131 被针对性识别)
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
|
||||
# Chrome 129 — Windows / Linux
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
|
||||
]
|
||||
|
||||
|
||||
def _ua_index_for_domain(domain: str, pool_size: int) -> int:
|
||||
"""为域名确定性选择 UA 池索引。
|
||||
|
||||
用 ``hashlib.sha256`` 而非内置 ``hash()``，因为后者对字符串做了
|
||||
随机化（PYTHONHASHSEED），跨进程不可复现。SHA-256 保证同一域名
|
||||
永远映射到同一索引，跨进程一致——这对调试和日志分析至关重要。
|
||||
"""
|
||||
import hashlib
|
||||
h = hashlib.sha256(domain.encode("utf-8")).digest()
|
||||
# 取前 8 字节作为无符号整数，避免负数和短字符串的分布不均
|
||||
return int.from_bytes(h[:8], "big") % pool_size
|
||||
|
||||
|
||||
# Per-domain UA 缓存：同一域名 + 同一进程 = 同一 UA，避免会话内 UA 突变
|
||||
# 被反爬识别。跨进程通过 SHA-256 哈希复现，见 _ua_index_for_domain。
|
||||
_domain_ua_cache: dict = {}
|
||||
|
||||
|
||||
def get_ua_for_domain(url: str, user_agent: str = None) -> str:
|
||||
"""返回适合某域名的 User-Agent。
|
||||
|
||||
优先级：
|
||||
1. ``user_agent`` 显式传入（CLI --user-agent）→ 直接返回
|
||||
2. 该域名已缓存 → 返回缓存值
|
||||
3. 域名未缓存 → 用 SHA-256 hash 选一个 FALLBACK_UAS，缓存并返回
|
||||
|
||||
设计理由：真实浏览器访问同一站点时 UA 永远不变。爬虫如果每次请求
|
||||
换一个 UA，反而会被反爬系统标记为可疑。确定性轮换保证同一域名
|
||||
稳定使用同一 UA，不同域名分散到不同 UA 上降低集体封禁风险。
|
||||
"""
|
||||
if user_agent:
|
||||
return user_agent
|
||||
|
||||
import urllib.parse as _up
|
||||
try:
|
||||
domain = _up.urlparse(url).netloc.lower()
|
||||
if not domain:
|
||||
return FALLBACK_UAS[0]
|
||||
except Exception:
|
||||
return FALLBACK_UAS[0]
|
||||
|
||||
if domain in _domain_ua_cache:
|
||||
return _domain_ua_cache[domain]
|
||||
|
||||
idx = _ua_index_for_domain(domain, len(FALLBACK_UAS))
|
||||
ua = FALLBACK_UAS[idx]
|
||||
_domain_ua_cache[domain] = ua
|
||||
return ua
|
||||
|
||||
|
||||
def reset_domain_ua_cache() -> None:
|
||||
"""清空 per-domain UA 缓存。测试用。"""
|
||||
_domain_ua_cache.clear()
|
||||
|
||||
|
||||
def build_browser_headers(user_agent: str, referer: str = None,
|
||||
accept_html: bool = True) -> dict:
|
||||
"""构造完整的浏览器请求头，让请求看起来像真浏览器。
|
||||
|
||||
v2.0.0 核心反爬措施：仅靠 User-Agent 已无法绕过现代 WAF，
|
||||
Cloudflare/Akamai/Imperva 都会检查 Sec-* 头和 Accept-Language。
|
||||
|
||||
Args:
|
||||
user_agent: UA 字符串（应来自 get_ua_for_domain）
|
||||
referer: Referer URL（可选；从搜索结果抓取时设为实例 URL）
|
||||
accept_html: True 时 Accept 包含 text/html（页面抓取）；
|
||||
False 时 Accept 为 application/json（API 调用）
|
||||
|
||||
Returns:
|
||||
包含完整浏览器指纹的 headers dict。调用方需自行合并 auth_headers。
|
||||
"""
|
||||
is_firefox = "Firefox/" in user_agent
|
||||
is_edge = "Edg/" in user_agent
|
||||
|
||||
if accept_html:
|
||||
accept = ("text/html,application/xhtml+xml,application/xml;q=0.9,"
|
||||
"image/avif,image/webp,*/*;q=0.8")
|
||||
else:
|
||||
accept = "application/json, text/plain, */*;q=0.8"
|
||||
|
||||
headers = {
|
||||
"User-Agent": user_agent,
|
||||
"Accept": accept,
|
||||
"Accept-Language": "en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7",
|
||||
"Accept-Encoding": "gzip, deflate" if is_firefox else "gzip, deflate, br",
|
||||
"Connection": "keep-alive",
|
||||
"Upgrade-Insecure-Requests": "1" if accept_html else "0",
|
||||
}
|
||||
|
||||
# Sec-Ch-Ua 系列仅 Chrome/Edge 发送，Firefox 不发
|
||||
if not is_firefox:
|
||||
# 从 UA 提取主版本号，构造 Sec-Ch-Ua
|
||||
import re
|
||||
m = re.search(r"Chrome/(\d+)", user_agent)
|
||||
ver = m.group(1) if m else "131"
|
||||
not_a_brand = '"Not_A Brand";v="99"' if ver != "99" else '"Not/A)Brand";v="99"'
|
||||
headers["Sec-Ch-Ua"] = f'"{not_a_brand}", "Chromium";v="{ver}", "Google Chrome";v="{ver}"'
|
||||
if is_edge:
|
||||
# Edge 的品牌标识
|
||||
headers["Sec-Ch-Ua"] = headers["Sec-Ch-Ua"].rstrip('"') + f'", "Microsoft Edge";v="{ver}"'
|
||||
headers["Sec-Ch-Ua-Mobile"] = '"?1"' if "Mobile" in user_agent else '"?0"'
|
||||
# 平台标识
|
||||
if "Windows" in user_agent:
|
||||
headers["Sec-Ch-Ua-Platform"] = '"Windows"'
|
||||
elif "Macintosh" in user_agent:
|
||||
headers["Sec-Ch-Ua-Platform"] = '"macOS"'
|
||||
elif "Linux" in user_agent:
|
||||
headers["Sec-Ch-Ua-Platform"] = '"Linux"'
|
||||
# Sec-Fetch 系列（Chrome 76+ 全量发送）
|
||||
if accept_html:
|
||||
headers["Sec-Fetch-Site"] = "none" if not referer else "cross-site"
|
||||
headers["Sec-Fetch-Mode"] = "navigate"
|
||||
headers["Sec-Fetch-User"] = "?1"
|
||||
headers["Sec-Fetch-Dest"] = "document"
|
||||
else:
|
||||
headers["Sec-Fetch-Site"] = "same-origin" if referer else "none"
|
||||
headers["Sec-Fetch-Mode"] = "cors"
|
||||
headers["Sec-Fetch-Dest"] = "empty"
|
||||
|
||||
if referer:
|
||||
headers["Referer"] = referer
|
||||
|
||||
return headers
|
||||
|
||||
|
||||
def parse_retry_after(header_value: str) -> float:
|
||||
"""解析 Retry-After header，返回应等待的秒数。
|
||||
|
||||
HTTP 规范允许两种格式：
|
||||
1. 纯数字：秒数（最常见）
|
||||
2. HTTP date：绝对时间（如 ``Wed, 21 Oct 2026 07:28:00 GMT``）
|
||||
|
||||
返回 0.0 表示无需等待或解析失败。对 HTTP date 格式，若已过期
|
||||
也返回 0.0（让调用方立即重试）。
|
||||
"""
|
||||
if not header_value:
|
||||
return 0.0
|
||||
header_value = header_value.strip()
|
||||
|
||||
# 格式 1：纯数字秒数
|
||||
try:
|
||||
seconds = float(header_value)
|
||||
return max(0.0, seconds)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# 格式 2：HTTP date
|
||||
try:
|
||||
from email.utils import parsedate_to_datetime
|
||||
from datetime import datetime, timezone
|
||||
dt = parsedate_to_datetime(header_value)
|
||||
if dt is None:
|
||||
return 0.0
|
||||
# 确保 timezone-aware
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
now = datetime.now(timezone.utc)
|
||||
delta = (dt - now).total_seconds()
|
||||
return max(0.0, delta)
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
return 0.0
|
||||
|
||||
|
||||
def compute_backoff_delay(attempt: int, base: float = RETRY_BACKOFF_BASE,
|
||||
cap: float = RETRY_BACKOFF_CAP) -> float:
|
||||
"""计算退避延迟，带封顶和抖动。
|
||||
|
||||
``base * 2^attempt + jitter``，但不超过 ``cap``。
|
||||
v2.0.0 新增封顶：原公式无上限，N=10 时达 1536s 会卡死进程。
|
||||
"""
|
||||
import random
|
||||
delay = base * (2 ** attempt) + random.uniform(0, 1)
|
||||
return min(delay, cap)
|
||||
|
||||
|
||||
def build_auth_headers(bearer_token: str = None, basic_auth: str = None) -> dict:
|
||||
"""Build an Authorization header dict from CLI auth flags.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user