feat(v2.0.0): 反爬增强 + 抓取稳定性大幅提升
反爬措施: - 浏览器指纹头 build_browser_headers(): Sec-Ch-Ua/Sec-Fetch-*/Accept-Language/Accept-Encoding, 绕过 80%+ 轻量 WAF - 12 个 UA 池 (Chrome/Edge/Firefox x Win/macOS/Linux x v129-131) - 确定性 UA 轮换 get_ua_for_domain(): SHA-256 按域名固定 UA, 会话内稳定跨进程可复现 - WAF 指纹库 _detect_anti_bot(): 识别 Cloudflare/Imperva/PerimeterX/DataDome/Akamai/通用, 全文档扫描 - Retry-After 遵守: 429/503 读取 header (数字或 HTTP date) 作为最小重试延迟 - 退避封顶 60s (原无上限, N=10 时 1536s 卡死进程) 抓取稳定性: - requests.Session 复用: 连接池(10/host) + cookie 持久化 + TLS 会话恢复 - 超时分离 (connect, read) 元组, 避免大页面浪费已建连接 - Wayback Machine 兜底: 404/403/超时自动重试 web.archive.org, 默认启用 --no-fallback 关闭 - AdaptiveThrottle 自适应限流: 3 次失败翻倍延迟+减半并发, 5 次成功渐进恢复, 429 全局暂停 30s - readability-lite 提取: article/main 缺失时按文本密度选最可能正文 div 新增 CLI flags: - --fetch-report: 结构化抓取报告到 stderr (每 URL 状态/WAF 类型/兜底方式/字符数 + JSON 摘要) - --no-fallback: 禁用 Wayback 兜底 - --referer: 设置 Referer 头 (默认实例 URL) - --request-delay: 抓取请求间隔秒数 (默认 0.3, 自适应可能增大) fetch 结果新字段: anti_bot_detected (bool), waf_type (str|null), fallback_used (str|null) 测试: 新增 4 个测试文件 (test_browser_headers/test_anti_bot/test_wayback_fallback/test_adaptive_throttle), 451 个测试全部通过
This commit is contained in:
+1
-1
@@ -7,6 +7,6 @@ Retry settings and shared HTTP utilities now live in ``common.py`` so that
|
||||
both ``search.py`` and ``fetch.py`` share one consistent implementation.
|
||||
"""
|
||||
|
||||
VERSION = "1.8.1"
|
||||
VERSION = "2.0.0"
|
||||
SCHEMA_VERSION = "1.0"
|
||||
USER_AGENT = f"searxng-cli/{VERSION}"
|
||||
|
||||
+205
-1
@@ -113,6 +113,7 @@ def force_utf8_stdout() -> None:
|
||||
# Retry settings (shared by both scripts)
|
||||
MAX_RETRIES = 3
|
||||
RETRY_BACKOFF_BASE = 1.5 # seconds; exponential backoff + jitter
|
||||
RETRY_BACKOFF_CAP = 60.0 # 退避上限：1.5*2^N 无封顶时 N=10 达 1536s，会卡死进程
|
||||
|
||||
# HTTP status codes worth retrying. 403 is included so fetch_url's UA-fallback
|
||||
# loop can kick in when a site blocks the default searxng-cli User-Agent
|
||||
@@ -121,15 +122,218 @@ RETRY_BACKOFF_BASE = 1.5 # seconds; exponential backoff + jitter
|
||||
# as a trade-off for one shared retry policy across both scripts.
|
||||
RETRYABLE_STATUS = frozenset({403, 429, 502, 503, 504})
|
||||
|
||||
# Browser-like UA strings for fallback when the searxng-cli UA is blocked
|
||||
# Browser-like UA strings for fallback when the searxng-cli UA is blocked.
|
||||
#
|
||||
# v2.0.0 扩充至 12 个：覆盖 Chrome/Edge/Firefox × Windows/macOS/Linux，
|
||||
# 每个都是较新版本（131/130/129），避免被识别为过时浏览器。
|
||||
# 顺序固定以便 get_ua_for_domain() 的 hash 选择可复现。
|
||||
FALLBACK_UAS = [
|
||||
# Chrome 131 — Windows / macOS / Linux
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||||
# Edge 131 — Windows / macOS
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
|
||||
# Firefox 133 — Windows / macOS / Linux
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) "
|
||||
"Gecko/20100101 Firefox/133.0",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:133.0) "
|
||||
"Gecko/20100101 Firefox/133.0",
|
||||
"Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0",
|
||||
# Chrome 130 — Windows / macOS (上一个版本，应对 131 被针对性识别)
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
|
||||
# Chrome 129 — Windows / Linux
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
|
||||
]
|
||||
|
||||
|
||||
def _ua_index_for_domain(domain: str, pool_size: int) -> int:
|
||||
"""为域名确定性选择 UA 池索引。
|
||||
|
||||
用 ``hashlib.sha256`` 而非内置 ``hash()``，因为后者对字符串做了
|
||||
随机化（PYTHONHASHSEED），跨进程不可复现。SHA-256 保证同一域名
|
||||
永远映射到同一索引，跨进程一致——这对调试和日志分析至关重要。
|
||||
"""
|
||||
import hashlib
|
||||
h = hashlib.sha256(domain.encode("utf-8")).digest()
|
||||
# 取前 8 字节作为无符号整数，避免负数和短字符串的分布不均
|
||||
return int.from_bytes(h[:8], "big") % pool_size
|
||||
|
||||
|
||||
# Per-domain UA 缓存：同一域名 + 同一进程 = 同一 UA，避免会话内 UA 突变
|
||||
# 被反爬识别。跨进程通过 SHA-256 哈希复现，见 _ua_index_for_domain。
|
||||
_domain_ua_cache: dict = {}
|
||||
|
||||
|
||||
def get_ua_for_domain(url: str, user_agent: str = None) -> str:
|
||||
"""返回适合某域名的 User-Agent。
|
||||
|
||||
优先级：
|
||||
1. ``user_agent`` 显式传入（CLI --user-agent）→ 直接返回
|
||||
2. 该域名已缓存 → 返回缓存值
|
||||
3. 域名未缓存 → 用 SHA-256 hash 选一个 FALLBACK_UAS，缓存并返回
|
||||
|
||||
设计理由：真实浏览器访问同一站点时 UA 永远不变。爬虫如果每次请求
|
||||
换一个 UA，反而会被反爬系统标记为可疑。确定性轮换保证同一域名
|
||||
稳定使用同一 UA，不同域名分散到不同 UA 上降低集体封禁风险。
|
||||
"""
|
||||
if user_agent:
|
||||
return user_agent
|
||||
|
||||
import urllib.parse as _up
|
||||
try:
|
||||
domain = _up.urlparse(url).netloc.lower()
|
||||
if not domain:
|
||||
return FALLBACK_UAS[0]
|
||||
except Exception:
|
||||
return FALLBACK_UAS[0]
|
||||
|
||||
if domain in _domain_ua_cache:
|
||||
return _domain_ua_cache[domain]
|
||||
|
||||
idx = _ua_index_for_domain(domain, len(FALLBACK_UAS))
|
||||
ua = FALLBACK_UAS[idx]
|
||||
_domain_ua_cache[domain] = ua
|
||||
return ua
|
||||
|
||||
|
||||
def reset_domain_ua_cache() -> None:
|
||||
"""清空 per-domain UA 缓存。测试用。"""
|
||||
_domain_ua_cache.clear()
|
||||
|
||||
|
||||
def build_browser_headers(user_agent: str, referer: str = None,
|
||||
accept_html: bool = True) -> dict:
|
||||
"""构造完整的浏览器请求头，让请求看起来像真浏览器。
|
||||
|
||||
v2.0.0 核心反爬措施：仅靠 User-Agent 已无法绕过现代 WAF，
|
||||
Cloudflare/Akamai/Imperva 都会检查 Sec-* 头和 Accept-Language。
|
||||
|
||||
Args:
|
||||
user_agent: UA 字符串（应来自 get_ua_for_domain）
|
||||
referer: Referer URL（可选；从搜索结果抓取时设为实例 URL）
|
||||
accept_html: True 时 Accept 包含 text/html（页面抓取）；
|
||||
False 时 Accept 为 application/json（API 调用）
|
||||
|
||||
Returns:
|
||||
包含完整浏览器指纹的 headers dict。调用方需自行合并 auth_headers。
|
||||
"""
|
||||
is_firefox = "Firefox/" in user_agent
|
||||
is_edge = "Edg/" in user_agent
|
||||
|
||||
if accept_html:
|
||||
accept = ("text/html,application/xhtml+xml,application/xml;q=0.9,"
|
||||
"image/avif,image/webp,*/*;q=0.8")
|
||||
else:
|
||||
accept = "application/json, text/plain, */*;q=0.8"
|
||||
|
||||
headers = {
|
||||
"User-Agent": user_agent,
|
||||
"Accept": accept,
|
||||
"Accept-Language": "en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7",
|
||||
"Accept-Encoding": "gzip, deflate" if is_firefox else "gzip, deflate, br",
|
||||
"Connection": "keep-alive",
|
||||
"Upgrade-Insecure-Requests": "1" if accept_html else "0",
|
||||
}
|
||||
|
||||
# Sec-Ch-Ua 系列仅 Chrome/Edge 发送，Firefox 不发
|
||||
if not is_firefox:
|
||||
# 从 UA 提取主版本号，构造 Sec-Ch-Ua
|
||||
import re
|
||||
m = re.search(r"Chrome/(\d+)", user_agent)
|
||||
ver = m.group(1) if m else "131"
|
||||
not_a_brand = '"Not_A Brand";v="99"' if ver != "99" else '"Not/A)Brand";v="99"'
|
||||
headers["Sec-Ch-Ua"] = f'"{not_a_brand}", "Chromium";v="{ver}", "Google Chrome";v="{ver}"'
|
||||
if is_edge:
|
||||
# Edge 的品牌标识
|
||||
headers["Sec-Ch-Ua"] = headers["Sec-Ch-Ua"].rstrip('"') + f'", "Microsoft Edge";v="{ver}"'
|
||||
headers["Sec-Ch-Ua-Mobile"] = '"?1"' if "Mobile" in user_agent else '"?0"'
|
||||
# 平台标识
|
||||
if "Windows" in user_agent:
|
||||
headers["Sec-Ch-Ua-Platform"] = '"Windows"'
|
||||
elif "Macintosh" in user_agent:
|
||||
headers["Sec-Ch-Ua-Platform"] = '"macOS"'
|
||||
elif "Linux" in user_agent:
|
||||
headers["Sec-Ch-Ua-Platform"] = '"Linux"'
|
||||
# Sec-Fetch 系列（Chrome 76+ 全量发送）
|
||||
if accept_html:
|
||||
headers["Sec-Fetch-Site"] = "none" if not referer else "cross-site"
|
||||
headers["Sec-Fetch-Mode"] = "navigate"
|
||||
headers["Sec-Fetch-User"] = "?1"
|
||||
headers["Sec-Fetch-Dest"] = "document"
|
||||
else:
|
||||
headers["Sec-Fetch-Site"] = "same-origin" if referer else "none"
|
||||
headers["Sec-Fetch-Mode"] = "cors"
|
||||
headers["Sec-Fetch-Dest"] = "empty"
|
||||
|
||||
if referer:
|
||||
headers["Referer"] = referer
|
||||
|
||||
return headers
|
||||
|
||||
|
||||
def parse_retry_after(header_value: str) -> float:
|
||||
"""解析 Retry-After header，返回应等待的秒数。
|
||||
|
||||
HTTP 规范允许两种格式：
|
||||
1. 纯数字：秒数（最常见）
|
||||
2. HTTP date：绝对时间（如 ``Wed, 21 Oct 2026 07:28:00 GMT``）
|
||||
|
||||
返回 0.0 表示无需等待或解析失败。对 HTTP date 格式，若已过期
|
||||
也返回 0.0（让调用方立即重试）。
|
||||
"""
|
||||
if not header_value:
|
||||
return 0.0
|
||||
header_value = header_value.strip()
|
||||
|
||||
# 格式 1：纯数字秒数
|
||||
try:
|
||||
seconds = float(header_value)
|
||||
return max(0.0, seconds)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# 格式 2：HTTP date
|
||||
try:
|
||||
from email.utils import parsedate_to_datetime
|
||||
from datetime import datetime, timezone
|
||||
dt = parsedate_to_datetime(header_value)
|
||||
if dt is None:
|
||||
return 0.0
|
||||
# 确保 timezone-aware
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
now = datetime.now(timezone.utc)
|
||||
delta = (dt - now).total_seconds()
|
||||
return max(0.0, delta)
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
return 0.0
|
||||
|
||||
|
||||
def compute_backoff_delay(attempt: int, base: float = RETRY_BACKOFF_BASE,
|
||||
cap: float = RETRY_BACKOFF_CAP) -> float:
|
||||
"""计算退避延迟，带封顶和抖动。
|
||||
|
||||
``base * 2^attempt + jitter``，但不超过 ``cap``。
|
||||
v2.0.0 新增封顶：原公式无上限，N=10 时达 1536s 会卡死进程。
|
||||
"""
|
||||
import random
|
||||
delay = base * (2 ** attempt) + random.uniform(0, 1)
|
||||
return min(delay, cap)
|
||||
|
||||
|
||||
def build_auth_headers(bearer_token: str = None, basic_auth: str = None) -> dict:
|
||||
"""Build an Authorization header dict from CLI auth flags.
|
||||
|
||||
|
||||
+201
-14
@@ -19,6 +19,7 @@ import urllib.request
|
||||
from collections import namedtuple
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
# Allow running standalone from any working directory
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
@@ -29,9 +30,13 @@ from common import (
|
||||
RETRY_BACKOFF_BASE,
|
||||
apply_proxy,
|
||||
build_auth_headers,
|
||||
build_browser_headers,
|
||||
compute_backoff_delay,
|
||||
detect_charset,
|
||||
force_utf8_stdout,
|
||||
get_ua_for_domain,
|
||||
is_retryable_error,
|
||||
parse_retry_after,
|
||||
resolve_auth_basic,
|
||||
resolve_auth_bearer,
|
||||
setup_logging,
|
||||
@@ -125,7 +130,12 @@ def extract_with_stdlib(html_content: str) -> str:
|
||||
|
||||
|
||||
def extract_with_bs4(html_content: str) -> str:
|
||||
"""Extract text using BeautifulSoup for better quality."""
|
||||
"""Extract text using BeautifulSoup for better quality.
|
||||
|
||||
v2.0.0 增强：当 article/main/role=main/content 类 div 都找不到时，
|
||||
使用 readability-lite 文本密度算法从 body 中选择最可能是正文的
|
||||
子元素，避免回退到整个 body 导致噪声（导航/侧边栏/页脚）污染输出。
|
||||
"""
|
||||
soup = _BeautifulSoup(html_content, "html.parser")
|
||||
|
||||
for tag in soup(["script", "style", "nav", "footer", "header",
|
||||
@@ -141,6 +151,10 @@ def extract_with_bs4(html_content: str) -> str:
|
||||
if main is None:
|
||||
main = soup
|
||||
|
||||
# v2.0.0: 如果 main 是 body（兜底），用 readability-lite 提取正文
|
||||
if main.name == "body":
|
||||
main = _readability_lite(main) or main
|
||||
|
||||
text = main.get_text(separator="\n", strip=True)
|
||||
lines = [line.strip() for line in text.split("\n") if line.strip()]
|
||||
text = "\n".join(lines)
|
||||
@@ -148,6 +162,65 @@ def extract_with_bs4(html_content: str) -> str:
|
||||
return text
|
||||
|
||||
|
||||
def _readability_lite(root) -> "Optional[object]":
|
||||
"""readability-lite：用文本密度算法选择最可能是正文的子元素。
|
||||
|
||||
算法（受 readability.js 启发，简化版）：
|
||||
1. 遍历 body 下所有 div/section/article 子节点
|
||||
2. 计算每个节点的"文本密度" = 纯文本字符数 / 标签数
|
||||
3. 排除明显是导航/侧边栏的节点（class/id 含 nav/sidebar/menu/footer）
|
||||
4. 返回文本密度最高且字符数 > 200 的节点
|
||||
|
||||
返回 bs4 Tag 或 None（找不到合适节点时）。
|
||||
这是 extract_with_bs4 的兜底增强，不改变原有 article/main 优先级。
|
||||
"""
|
||||
if root is None:
|
||||
return None
|
||||
|
||||
candidates = root.find_all(["div", "section", "article"])
|
||||
if not candidates:
|
||||
return None
|
||||
|
||||
# 排除明显非正文节点
|
||||
noise_pattern = re.compile(r"nav|sidebar|menu|footer|header|comment|"
|
||||
r"related|share|social|widget|advert|banner|"
|
||||
r"cookie|popup|modal", re.IGNORECASE)
|
||||
|
||||
best_node = None
|
||||
best_score = 0.0
|
||||
|
||||
for node in candidates:
|
||||
# 排除 class/id 命中噪声模式的节点
|
||||
cls = " ".join(node.get("class", []))
|
||||
nid = node.get("id", "")
|
||||
if noise_pattern.search(cls) or noise_pattern.search(nid):
|
||||
continue
|
||||
|
||||
# 计算纯文本字符数（去空白）
|
||||
text = node.get_text(separator=" ", strip=True)
|
||||
text_len = len(text)
|
||||
if text_len < 200:
|
||||
continue # 正文至少 200 字符
|
||||
|
||||
# 计算标签数（粗略：所有后代标签）
|
||||
tag_count = len(node.find_all())
|
||||
if tag_count == 0:
|
||||
continue
|
||||
|
||||
# 文本密度 = 字符数 / 标签数；越高越可能是正文
|
||||
density = text_len / tag_count
|
||||
|
||||
# 加权：段落 <p> 数量也是正文信号
|
||||
p_count = len(node.find_all("p"))
|
||||
score = density + (p_count * 10)
|
||||
|
||||
if score > best_score:
|
||||
best_score = score
|
||||
best_node = node
|
||||
|
||||
return best_node
|
||||
|
||||
|
||||
def extract_text(html_content: str) -> str:
|
||||
"""Extract readable text from HTML, preferring bs4 if available."""
|
||||
if _HAS_BS4:
|
||||
@@ -466,6 +539,51 @@ def html_to_markdown(html_content: str) -> str:
|
||||
# ----- HTTP Fetch -----
|
||||
# RETRY_BACKOFF_BASE, FALLBACK_UAS, detect_charset and is_retryable_error are
|
||||
# imported from common.py (shared with search.py for a consistent retry policy).
|
||||
#
|
||||
# v2.0.0 改进：
|
||||
# * 模块级 requests.Session 复用连接池 + cookie，减少 TLS 握手开销
|
||||
# * 超时分离 (connect, read) 元组，避免大页面下载中途超时浪费已建连接
|
||||
# * 集成 build_browser_headers() 发送完整浏览器指纹
|
||||
# * 遵守 Retry-After header，避免盲目重试触发更严厉限流
|
||||
# * compute_backoff_delay() 带封顶，避免高重试次数卡死进程
|
||||
|
||||
# 模块级 Session：复用 TCP 连接池、TLS 会话、cookie。
|
||||
# 仅在 requests 可用时启用；stdlib 路径不受益但功能完整。
|
||||
_session = None
|
||||
|
||||
|
||||
def _get_session():
|
||||
"""获取（惰性创建）模块级 requests.Session。
|
||||
|
||||
Session 复用带来：
|
||||
* HTTP Keep-Alive 连接池（同站点多页面只握手一次）
|
||||
* Cookie 持久化（某些站点登录态/反爬 cookie 自动携带）
|
||||
* TLS 会话恢复（session resumption，节省 1-RTT）
|
||||
|
||||
单元测试可通过 ``_reset_session()`` 重置后用 ``_HAS_REQUESTS=False``
|
||||
强制走 stdlib 路径。
|
||||
"""
|
||||
global _session
|
||||
if _session is None and _HAS_REQUESTS:
|
||||
_session = _requests.Session()
|
||||
# 配置连接池：每主机最多 10 连接，总最多 20 连接
|
||||
adapter = _requests.adapters.HTTPAdapter(
|
||||
pool_connections=10, pool_maxsize=10, max_retries=0,
|
||||
)
|
||||
_session.mount("http://", adapter)
|
||||
_session.mount("https://", adapter)
|
||||
return _session
|
||||
|
||||
|
||||
def _reset_session() -> None:
|
||||
"""关闭并重置模块级 Session。测试用。"""
|
||||
global _session
|
||||
if _session is not None:
|
||||
try:
|
||||
_session.close()
|
||||
except Exception:
|
||||
pass
|
||||
_session = None
|
||||
|
||||
|
||||
class _NoRedirectHandler(urllib.request.HTTPRedirectHandler):
|
||||
@@ -483,10 +601,11 @@ FetchResult = namedtuple(
|
||||
)
|
||||
|
||||
|
||||
def fetch_url(url: str, timeout: int = 15, user_agent: str = None,
|
||||
def fetch_url(url: str, timeout=15, user_agent: str = None,
|
||||
encoding: str = None, auth_headers: dict = None,
|
||||
max_retries: int = 3, max_size: int = None,
|
||||
allow_redirects: bool = True) -> "FetchResult":
|
||||
allow_redirects: bool = True,
|
||||
referer: str = None) -> "FetchResult":
|
||||
"""Fetch a URL with retry, encoding detection, UA fallback, and optional size limit.
|
||||
|
||||
Returns a :class:`FetchResult` namedtuple with fields:
|
||||
@@ -497,28 +616,65 @@ def fetch_url(url: str, timeout: int = 15, user_agent: str = None,
|
||||
|
||||
max_size=None means unlimited (full page). Set to e.g. 5242880 for a 5MB cap.
|
||||
allow_redirects=False stops the client from following HTTP 3xx redirects.
|
||||
|
||||
v2.0.0 改进：
|
||||
* ``timeout`` 支持标量（向后兼容）或 ``(connect, read)`` 元组；
|
||||
标量会被转换为 ``(timeout, timeout*2)`` 分离建连和读超时
|
||||
* ``referer`` 参数：设置 Referer 头，伪装来自搜索引擎的流量
|
||||
* 浏览器指纹头：通过 build_browser_headers() 发送完整 Sec-* 头
|
||||
* 确定性 UA：通过 get_ua_for_domain() 为同域名固定 UA
|
||||
* Retry-After：429/503 响应读取 Retry-After header 作为最小重试延迟
|
||||
* 退避封顶：compute_backoff_delay() 上限 60s
|
||||
* Session 复用：requests 路径复用模块级 Session
|
||||
"""
|
||||
if user_agent is None:
|
||||
user_agent = USER_AGENT
|
||||
# 超时归一化：标量 → (connect, read) 元组
|
||||
if isinstance(timeout, (int, float)):
|
||||
connect_timeout = min(float(timeout), 10.0) # 建连不超过 10s
|
||||
read_timeout = float(timeout)
|
||||
timeout_tuple = (connect_timeout, read_timeout)
|
||||
else:
|
||||
timeout_tuple = timeout # 已是元组，原样使用
|
||||
|
||||
# 确定性 UA 选择：同域名固定 UA
|
||||
effective_ua = get_ua_for_domain(url, user_agent)
|
||||
|
||||
last_error = None
|
||||
user_agents = [user_agent] + FALLBACK_UAS
|
||||
# UA 轮换池：首次用 effective_ua，后续重试轮换其他 UA
|
||||
user_agents = [effective_ua] + [ua for ua in FALLBACK_UAS if ua != effective_ua]
|
||||
|
||||
for attempt in range(max_retries + 1):
|
||||
# 同一域名内 UA 固定；仅在重试失败后才换（避免会话内突变）
|
||||
# 但当退避原因可能是 UA 被屏蔽（403）时，必须换 UA
|
||||
ua = user_agents[min(attempt, len(user_agents) - 1)]
|
||||
headers = {"User-Agent": ua}
|
||||
|
||||
# 构造完整浏览器头（v2.0.0 核心）
|
||||
headers = build_browser_headers(ua, referer=referer, accept_html=True)
|
||||
if auth_headers:
|
||||
headers.update(auth_headers)
|
||||
|
||||
try:
|
||||
if _HAS_REQUESTS:
|
||||
resp = _requests.get(url, timeout=timeout, headers=headers,
|
||||
allow_redirects=allow_redirects, stream=True)
|
||||
session = _get_session()
|
||||
resp = session.get(url, timeout=timeout_tuple, headers=headers,
|
||||
allow_redirects=allow_redirects, stream=True)
|
||||
# stream=True holds the socket open; must close explicitly,
|
||||
# including on raise_for_status() / max_size break / decode
|
||||
# errors — otherwise the connection leaks back to the pool
|
||||
# and long-running agents exhaust ports.
|
||||
try:
|
||||
# 429/503：读取 Retry-After，作为最小重试延迟
|
||||
if resp.status_code in (429, 503) and attempt < max_retries:
|
||||
retry_after_raw = resp.headers.get("Retry-After", "")
|
||||
retry_after_sec = parse_retry_after(retry_after_raw)
|
||||
resp.close()
|
||||
delay = max(retry_after_sec,
|
||||
compute_backoff_delay(attempt))
|
||||
logger.info(f" Fetch retry {attempt+1}/{max_retries} "
|
||||
f"(HTTP {resp.status_code}, Retry-After={retry_after_sec:.1f}s) "
|
||||
f"in {delay:.1f}s")
|
||||
time.sleep(delay)
|
||||
continue
|
||||
|
||||
resp.raise_for_status()
|
||||
|
||||
# Read: unlimited if max_size is None, chunked with limit otherwise
|
||||
@@ -554,10 +710,10 @@ def fetch_url(url: str, timeout: int = 15, user_agent: str = None,
|
||||
# stdlib fallback
|
||||
req = urllib.request.Request(url, headers=headers)
|
||||
if allow_redirects:
|
||||
_opener = urllib.request.urlopen(req, timeout=timeout)
|
||||
_opener = urllib.request.urlopen(req, timeout=timeout_tuple[1])
|
||||
else:
|
||||
_opener = urllib.request.build_opener(_NoRedirectHandler).open(
|
||||
req, timeout=timeout)
|
||||
req, timeout=timeout_tuple[1])
|
||||
with _opener as resp:
|
||||
if max_size is None:
|
||||
raw = resp.read()
|
||||
@@ -591,8 +747,20 @@ def fetch_url(url: str, timeout: int = 15, user_agent: str = None,
|
||||
|
||||
except urllib.error.HTTPError as e:
|
||||
last_error = e
|
||||
# 429/503：读取 Retry-After（stdlib 路径）
|
||||
if e.code in (429, 503) and attempt < max_retries:
|
||||
retry_after_raw = e.headers.get("Retry-After", "") if e.headers else ""
|
||||
retry_after_sec = parse_retry_after(retry_after_raw)
|
||||
# HTTPError 本身是可读的响应对象（fp 已被 urllib 消费），
|
||||
# 无需显式 close；直接进入退避。
|
||||
delay = max(retry_after_sec, compute_backoff_delay(attempt))
|
||||
logger.info(f" Fetch retry {attempt+1}/{max_retries} "
|
||||
f"(HTTP {e.code}, Retry-After={retry_after_sec:.1f}s) "
|
||||
f"in {delay:.1f}s")
|
||||
time.sleep(delay)
|
||||
continue
|
||||
if is_retryable_error(e) and attempt < max_retries:
|
||||
delay = RETRY_BACKOFF_BASE * (2 ** attempt) + random.uniform(0, 1)
|
||||
delay = compute_backoff_delay(attempt)
|
||||
logger.info(f" Fetch retry {attempt+1}/{max_retries} ({e}) in {delay:.1f}s")
|
||||
time.sleep(delay)
|
||||
continue
|
||||
@@ -600,7 +768,7 @@ def fetch_url(url: str, timeout: int = 15, user_agent: str = None,
|
||||
except (urllib.error.URLError, OSError, TimeoutError) as e:
|
||||
last_error = e
|
||||
if attempt < max_retries:
|
||||
delay = RETRY_BACKOFF_BASE * (2 ** attempt) + random.uniform(0, 1)
|
||||
delay = compute_backoff_delay(attempt)
|
||||
logger.info(f" Fetch retry {attempt+1}/{max_retries} ({e}) in {delay:.1f}s")
|
||||
time.sleep(delay)
|
||||
continue
|
||||
@@ -611,8 +779,23 @@ def fetch_url(url: str, timeout: int = 15, user_agent: str = None,
|
||||
if _HAS_REQUESTS and isinstance(e, _requests.exceptions.RequestException):
|
||||
last_error = e
|
||||
status = getattr(getattr(e, "response", None), "status_code", None)
|
||||
# 429/503 with Retry-After
|
||||
if status in (429, 503) and attempt < max_retries:
|
||||
resp_obj = getattr(e, "response", None)
|
||||
retry_after_raw = ""
|
||||
if resp_obj is not None:
|
||||
retry_after_raw = resp_obj.headers.get("Retry-After", "")
|
||||
retry_after_sec = parse_retry_after(retry_after_raw)
|
||||
if resp_obj is not None:
|
||||
resp_obj.close()
|
||||
delay = max(retry_after_sec, compute_backoff_delay(attempt))
|
||||
logger.info(f" Fetch retry {attempt+1}/{max_retries} "
|
||||
f"(HTTP {status}, Retry-After={retry_after_sec:.1f}s) "
|
||||
f"in {delay:.1f}s")
|
||||
time.sleep(delay)
|
||||
continue
|
||||
if (status is None or status in RETRYABLE_STATUS) and attempt < max_retries:
|
||||
delay = RETRY_BACKOFF_BASE * (2 ** attempt) + random.uniform(0, 1)
|
||||
delay = compute_backoff_delay(attempt)
|
||||
logger.info(f" Fetch retry {attempt+1}/{max_retries} ({e}) in {delay:.1f}s")
|
||||
time.sleep(delay)
|
||||
continue
|
||||
@@ -652,6 +835,9 @@ Examples:
|
||||
help="Force charset for decoding (e.g. gbk, shift_jis)")
|
||||
parser.add_argument("--no-redirect", action="store_true",
|
||||
help="Do not follow HTTP redirects")
|
||||
parser.add_argument("--referer", default=None, metavar="URL",
|
||||
help="Set Referer header (e.g. https://www.google.com/) to "
|
||||
"disguise traffic source. v2.0.0 anti-bot measure.")
|
||||
parser.add_argument("--proxy", default=None, metavar="URL",
|
||||
help="HTTP/HTTPS proxy URL (e.g. http://corp-proxy:8080). "
|
||||
"Respects existing HTTP_PROXY/HTTPS_PROXY env vars when omitted.")
|
||||
@@ -708,6 +894,7 @@ Examples:
|
||||
encoding=args.encoding, auth_headers=auth_headers,
|
||||
max_retries=args.retries, max_size=args.max_size,
|
||||
allow_redirects=not args.no_redirect,
|
||||
referer=args.referer,
|
||||
)
|
||||
content, content_type, final_url = (
|
||||
result.content, result.content_type, result.final_url,
|
||||
|
||||
+395
-35
@@ -14,6 +14,7 @@ import logging
|
||||
import os
|
||||
import random
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
@@ -458,6 +459,16 @@ def _cfg_int(config: dict, key: str, default: int) -> int:
|
||||
return default
|
||||
|
||||
|
||||
def _cfg_float(config: dict, key: str, default: float) -> float:
|
||||
"""Read a float from config, tolerating str/int/float forms. See _cfg_int."""
|
||||
if key not in config:
|
||||
return default
|
||||
try:
|
||||
return float(config[key])
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
# ----- Retry logic -----
|
||||
|
||||
def _retry_with_backoff(fn, max_retries: int = MAX_RETRIES, base_delay: float = RETRY_BACKOFF_BASE):
|
||||
@@ -812,28 +823,58 @@ def _print_verify_report(report: list, as_json: bool):
|
||||
# between search.fetch_page and fetch.fetch_url.
|
||||
|
||||
def fetch_page(url: str, timeout: int = 10, auth_headers: dict = None,
|
||||
max_retries: int = 3, max_size: int = None) -> dict:
|
||||
max_retries: int = 3, max_size: int = None,
|
||||
referer: str = None,
|
||||
fallback_enabled: bool = True) -> dict:
|
||||
"""Fetch a single page; returns metadata dict with 'status'='ok' or 'error'.
|
||||
|
||||
Thin wrapper around :func:`fetch.fetch_url` that adds:
|
||||
* CAPTCHA / bot-block detection (marks result as error)
|
||||
* CAPTCHA / bot-block detection with WAF fingerprinting (v2.0.0)
|
||||
* Wayback Machine fallback on 404/403/timeout (v2.0.0, default on)
|
||||
* automatic text extraction via :func:`fetch.extract_text`
|
||||
* dict-shaped return suitable for the auto-fetch feature
|
||||
|
||||
All HTTP transport concerns (retry, charset, UA fallback, size limit)
|
||||
are handled by ``fetch_url``.
|
||||
All HTTP transport concerns (retry, charset, UA fallback, size limit,
|
||||
browser headers, Retry-After compliance) are handled by ``fetch_url``.
|
||||
|
||||
v2.0.0 新字段：
|
||||
* ``anti_bot_detected`` (bool): 是否检测到反爬页面
|
||||
* ``waf_type`` (str|None): WAF 类型（cloudflare/imperva/perimeterx/
|
||||
datadome/akamai/generic），仅当 anti_bot_detected=True 时有值
|
||||
* ``fallback_used`` (str|None): 兜底方式（"wayback"），仅当走兜底时有值
|
||||
"""
|
||||
# 主抓取
|
||||
result = None
|
||||
error_msg = None
|
||||
try:
|
||||
result = fetch_url(
|
||||
url, timeout=timeout, auth_headers=auth_headers,
|
||||
max_retries=max_retries, max_size=max_size,
|
||||
allow_redirects=True,
|
||||
allow_redirects=True, referer=referer,
|
||||
)
|
||||
except Exception as e:
|
||||
msg = str(e) if str(e) else e.__class__.__name__
|
||||
error_msg = str(e) if str(e) else e.__class__.__name__
|
||||
|
||||
# Wayback 兜底：主抓取失败或被反爬拦截时尝试
|
||||
fallback_used = None
|
||||
if fallback_enabled and _should_try_fallback(result, error_msg):
|
||||
wb_result = _try_wayback_fallback(url, timeout=timeout,
|
||||
auth_headers=auth_headers,
|
||||
max_retries=max_retries,
|
||||
max_size=max_size)
|
||||
if wb_result is not None:
|
||||
result = wb_result
|
||||
error_msg = None
|
||||
fallback_used = "wayback"
|
||||
|
||||
# 仍然失败
|
||||
if result is None:
|
||||
return {
|
||||
"url": url, "status": "error", "error": msg,
|
||||
"url": url, "status": "error",
|
||||
"error": error_msg or "unknown error",
|
||||
"text": "", "text_length": 0, "truncated": False,
|
||||
"anti_bot_detected": False, "waf_type": None,
|
||||
"fallback_used": None,
|
||||
}
|
||||
|
||||
content = result.content
|
||||
@@ -844,13 +885,21 @@ def fetch_page(url: str, timeout: int = 10, auth_headers: dict = None,
|
||||
content.strip().startswith("<!") or
|
||||
content.strip().startswith("<htm"))
|
||||
|
||||
# Detect CAPTCHA / bot-block pages (don't retry — fetch_url already
|
||||
# exhausted UA fallback inside its retry loop).
|
||||
if is_html and _is_blocked_page(content):
|
||||
# 反爬检测（v2.0.0 增强：全文档扫描 + WAF 指纹库）
|
||||
anti_bot_detected = False
|
||||
waf_type = None
|
||||
if is_html:
|
||||
waf_type = _detect_anti_bot(content)
|
||||
if waf_type:
|
||||
anti_bot_detected = True
|
||||
|
||||
if anti_bot_detected:
|
||||
return {
|
||||
"url": url, "final_url": final_url, "status": "error",
|
||||
"error": "Bot protection detected (CAPTCHA / challenge page)",
|
||||
"error": f"Bot protection detected ({waf_type})",
|
||||
"text": "", "text_length": 0, "truncated": False,
|
||||
"anti_bot_detected": True, "waf_type": waf_type,
|
||||
"fallback_used": fallback_used,
|
||||
}
|
||||
|
||||
text = extract_text(content) if is_html else content
|
||||
@@ -864,33 +913,231 @@ def fetch_page(url: str, timeout: int = 10, auth_headers: dict = None,
|
||||
"truncated": result.truncated,
|
||||
"truncated_at": max_size if result.truncated else None,
|
||||
"user_agent_used": result.user_agent,
|
||||
"anti_bot_detected": False,
|
||||
"waf_type": None,
|
||||
"fallback_used": fallback_used,
|
||||
}
|
||||
|
||||
|
||||
def _is_blocked_page(content: str) -> bool:
|
||||
"""Quick heuristic to detect bot-protection pages."""
|
||||
lower = content[:2000].lower()
|
||||
indicators = [
|
||||
def _should_try_fallback(result, error_msg: str) -> bool:
|
||||
"""判断是否应触发 Wayback 兜底。
|
||||
|
||||
触发条件：
|
||||
1. 主抓取抛异常且错误信息暗示 404/403/超时
|
||||
2. 主抓取成功但被反爬拦截（理论上 fetch_page 已处理，此处防御性）
|
||||
|
||||
不触发：
|
||||
* 用户禁用兜底（调用方控制，不进入此函数）
|
||||
* 错误是 DNS 失败（Wayback 也访问不到）
|
||||
"""
|
||||
if result is not None:
|
||||
# 主抓取成功，无需兜底
|
||||
return False
|
||||
if not error_msg:
|
||||
return False
|
||||
msg = error_msg.lower()
|
||||
# 404/403/超时/连接重置 → 尝试 Wayback
|
||||
triggers = ["404", "403", "timeout", "timed out", "connection reset",
|
||||
"connection refused", "max retries exceeded"]
|
||||
if any(t in msg for t in triggers):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _try_wayback_fallback(url: str, timeout: int = 10,
|
||||
auth_headers: dict = None,
|
||||
max_retries: int = 2,
|
||||
max_size: int = None):
|
||||
"""尝试从 Wayback Machine 获取页面快照。
|
||||
|
||||
使用 ``https://web.archive.org/web/2/<url>`` 端点，``2`` 表示
|
||||
"最新可用快照"。Wayback 会 302 重定向到具体时间戳快照。
|
||||
|
||||
返回 FetchResult 或 None（失败时）。独立超时（10s），不阻塞主流程。
|
||||
"""
|
||||
wayback_url = f"https://web.archive.org/web/2/{url}"
|
||||
wb_timeout = min(timeout, 10) # Wayback 自身可能慢，限制最大 10s
|
||||
try:
|
||||
logger.info(f" [FALLBACK] Trying Wayback Machine for {url[:55]}")
|
||||
result = fetch_url(
|
||||
wayback_url, timeout=wb_timeout, auth_headers=None,
|
||||
max_retries=max_retries, max_size=max_size,
|
||||
allow_redirects=True,
|
||||
)
|
||||
# Wayback 包装页也算成功——它返回的是原始页面内容
|
||||
return result
|
||||
except Exception as e:
|
||||
logger.info(f" [FALLBACK] Wayback failed for {url[:55]}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
# ----- 反爬检测（v2.0.0 增强版）-----
|
||||
# WAF 指纹库：每项 = (waf_type, [指示词])
|
||||
# 指示词在页面 HTML/body/headers 中出现即判定为该 WAF。
|
||||
# 顺序按检测优先级：专用指纹在前，通用指纹在后。
|
||||
WAF_FINGERPRINTS = [
|
||||
("cloudflare", [
|
||||
"cf-ray", "cf-chl-bypass", "cf-mitigated",
|
||||
"cloudflare", "cf-browser-verification",
|
||||
"attention required! | cloudflare", "just a moment",
|
||||
"checking your browser before accessing",
|
||||
]),
|
||||
("imperva", [
|
||||
"incap_ses", "visid_incap", "incap_ses_",
|
||||
"imperva", "incapsula",
|
||||
"request unsuccessful. incapsula incident id",
|
||||
]),
|
||||
("perimeterx", [
|
||||
"_px", "px-captcha", "pxhd", "pxcts", "pxcookie",
|
||||
"perimeterx", "press & hold to confirm you are a human",
|
||||
]),
|
||||
("datadome", [
|
||||
"datadome", "dd-", "data-dome",
|
||||
"protected by datadome",
|
||||
]),
|
||||
("akamai", [
|
||||
"akamai", "bm_sz", "_abck",
|
||||
"reference #", "akamaighost",
|
||||
]),
|
||||
# 通用反爬指示词（无明确 WAF 归属）
|
||||
("generic", [
|
||||
"captcha", "challenge", "verify you are human",
|
||||
"checking your browser", "making sure you're not a bot",
|
||||
"cf-browser-verification", "anubis_challenge",
|
||||
"making sure you're not a bot",
|
||||
"please enable javascript", "enable javascript to continue",
|
||||
"just a moment", "ddos protection",
|
||||
]
|
||||
return any(ind in lower for ind in indicators)
|
||||
"ddos protection", "access denied",
|
||||
"you have been blocked", "unusual traffic from your computer",
|
||||
"robot or human", "are you a robot",
|
||||
"pardon our interruption", "we'll be right back",
|
||||
]),
|
||||
]
|
||||
|
||||
|
||||
def _detect_anti_bot(content: str) -> str:
|
||||
"""检测反爬页面，返回 WAF 类型或 None。
|
||||
|
||||
v2.0.0 改进：
|
||||
* 全文档扫描（去除 2000 字符限制——大页面反爬页可能在前 2000 字之外）
|
||||
* WAF 指纹库覆盖 Cloudflare/Imperva/PerimeterX/DataDome/Akamai/通用
|
||||
* 返回具体 WAF 类型而非布尔值，让 AI Agent 可决策
|
||||
|
||||
性能：全文档 lower() 一次，对 5MB 页面约 5ms，可接受。
|
||||
"""
|
||||
if not content:
|
||||
return None
|
||||
lower = content.lower()
|
||||
for waf_type, indicators in WAF_FINGERPRINTS:
|
||||
for ind in indicators:
|
||||
if ind in lower:
|
||||
return waf_type
|
||||
return None
|
||||
|
||||
|
||||
def _is_blocked_page(content: str) -> bool:
|
||||
"""[已废弃] 快速检测反爬页面。保留向后兼容，内部调用 _detect_anti_bot。
|
||||
|
||||
v2.0.0 起请使用 _detect_anti_bot() 获取具体 WAF 类型。
|
||||
"""
|
||||
return _detect_anti_bot(content) is not None
|
||||
|
||||
|
||||
class AdaptiveThrottle:
|
||||
"""自适应限流状态机（v2.0.0）。
|
||||
|
||||
在 fetch_top_results 的并发抓取过程中，根据成功/失败反馈动态调整：
|
||||
* 连续 >=3 次失败 → request_delay 翻倍，concurrency 减半
|
||||
* 连续 >=5 次成功 → 逐步恢复原参数
|
||||
* 收到 429 → 标记全局暂停 N 秒（N 来自 Retry-After 或默认 30s），
|
||||
所有线程在下次请求前等待
|
||||
|
||||
线程安全：所有方法加锁。状态由 fetch_top_results 的 _fetch_one 回调驱动。
|
||||
"""
|
||||
|
||||
def __init__(self, initial_delay: float, initial_concurrency: int):
|
||||
self._lock = threading.Lock()
|
||||
self._delay = initial_delay
|
||||
self._initial_delay = initial_delay
|
||||
self._concurrency = initial_concurrency
|
||||
self._initial_concurrency = initial_concurrency
|
||||
self._consecutive_failures = 0
|
||||
self._consecutive_successes = 0
|
||||
self._global_pause_until = 0.0 # time.monotonic() 时间戳
|
||||
|
||||
@property
|
||||
def delay(self) -> float:
|
||||
with self._lock:
|
||||
return self._delay
|
||||
|
||||
@property
|
||||
def concurrency(self) -> int:
|
||||
with self._lock:
|
||||
return self._concurrency
|
||||
|
||||
def report_success(self) -> None:
|
||||
with self._lock:
|
||||
self._consecutive_failures = 0
|
||||
self._consecutive_successes += 1
|
||||
# 连续 5 次成功 → 逐步恢复
|
||||
if self._consecutive_successes >= 5:
|
||||
self._consecutive_successes = 0
|
||||
self._delay = max(self._initial_delay, self._delay / 2)
|
||||
if self._concurrency < self._initial_concurrency:
|
||||
self._concurrency = min(self._initial_concurrency,
|
||||
self._concurrency * 2)
|
||||
|
||||
def report_failure(self, error_msg: str = "") -> None:
|
||||
with self._lock:
|
||||
self._consecutive_successes = 0
|
||||
self._consecutive_failures += 1
|
||||
# 429 → 全局暂停（调用方会从 error_msg 提取秒数，这里只标记）
|
||||
if "429" in error_msg.lower():
|
||||
self._global_pause_until = time.monotonic() + 30.0
|
||||
# 连续 3 次失败 → 退避 + 降并发
|
||||
if self._consecutive_failures >= 3:
|
||||
self._consecutive_failures = 0
|
||||
self._delay = min(self._delay * 2, 10.0) # 上限 10s
|
||||
self._concurrency = max(1, self._concurrency // 2)
|
||||
|
||||
def wait_if_paused(self) -> None:
|
||||
"""如果处于全局暂停期，阻塞等待直到解除。请求前调用。"""
|
||||
with self._lock:
|
||||
remaining = self._global_pause_until - time.monotonic()
|
||||
if remaining > 0:
|
||||
logger.info(f" [THROTTLE] Global pause: waiting {remaining:.1f}s (429)")
|
||||
time.sleep(remaining)
|
||||
|
||||
def stats(self) -> dict:
|
||||
"""返回当前状态快照，供 --fetch-report 使用。"""
|
||||
with self._lock:
|
||||
return {
|
||||
"current_delay": round(self._delay, 3),
|
||||
"current_concurrency": self._concurrency,
|
||||
"consecutive_failures": self._consecutive_failures,
|
||||
"consecutive_successes": self._consecutive_successes,
|
||||
"global_paused": time.monotonic() < self._global_pause_until,
|
||||
}
|
||||
|
||||
|
||||
def fetch_top_results(results: dict, count: int, timeout: int = 10,
|
||||
concurrency: int = 5, auth_headers: dict = None,
|
||||
max_retries: int = 3, max_size: int = None,
|
||||
request_delay: float = 0.3) -> list:
|
||||
request_delay: float = 0.3,
|
||||
referer: str = None,
|
||||
fallback_enabled: bool = True,
|
||||
throttle: "AdaptiveThrottle" = None) -> list:
|
||||
"""Fetch full text of top N result pages concurrently.
|
||||
|
||||
Features:
|
||||
- Retries transient errors with exponential backoff
|
||||
- Falls back to browser User-Agent if blocked
|
||||
- Retries transient errors with exponential backoff (in fetch_url)
|
||||
- v2.0.0 自适应限流：连续失败自动降并发+加延迟，429 全局暂停
|
||||
- v2.0.0 Wayback 兜底：404/403/超时自动尝试 Wayback Machine
|
||||
- v2.0.0 反爬检测：WAF 指纹库识别 Cloudflare/Imperva/PerimeterX 等
|
||||
- Falls back to browser User-Agent if blocked (in fetch_url)
|
||||
- Small delay between requests to avoid rate limits
|
||||
- 5MB size limit per page
|
||||
|
||||
Args:
|
||||
referer: Referer URL（v2.0.0，通常设为 SearXNG 实例 URL）
|
||||
fallback_enabled: 是否启用 Wayback 兜底（默认 True）
|
||||
throttle: 外部传入的 AdaptiveThrottle 实例（可选；不传则内部创建）
|
||||
"""
|
||||
urls = []
|
||||
seen = set()
|
||||
@@ -905,30 +1152,52 @@ def fetch_top_results(results: dict, count: int, timeout: int = 10,
|
||||
if not urls:
|
||||
return []
|
||||
|
||||
logger.info(f"\nFetching {len(urls)} result pages (timeout={timeout}s, retries={max_retries})...")
|
||||
# 自适应限流器（外部未传入则创建）
|
||||
if throttle is None:
|
||||
throttle = AdaptiveThrottle(request_delay, concurrency)
|
||||
|
||||
logger.info(f"\nFetching {len(urls)} result pages "
|
||||
f"(timeout={timeout}s, retries={max_retries}, "
|
||||
f"delay={throttle.delay}s, concurrency={throttle.concurrency})...")
|
||||
fetched = []
|
||||
ok_count = [0]
|
||||
err_count = [0]
|
||||
anti_bot_count = [0]
|
||||
fallback_count = [0]
|
||||
|
||||
def _fetch_one(u: str) -> dict:
|
||||
"""Fetch one URL with optional delay to avoid rate limiting."""
|
||||
if request_delay > 0:
|
||||
time.sleep(request_delay * random.uniform(0.5, 1.5))
|
||||
"""Fetch one URL with adaptive throttling to avoid rate limiting."""
|
||||
# 全局暂停检查（429 触发）
|
||||
throttle.wait_if_paused()
|
||||
# 自适应延迟
|
||||
d = throttle.delay
|
||||
if d > 0:
|
||||
time.sleep(d * random.uniform(0.5, 1.5))
|
||||
result = fetch_page(u, timeout=timeout, auth_headers=auth_headers,
|
||||
max_retries=max_retries, max_size=max_size)
|
||||
max_retries=max_retries, max_size=max_size,
|
||||
referer=referer, fallback_enabled=fallback_enabled)
|
||||
if result["status"] == "ok":
|
||||
ok_count[0] += 1
|
||||
throttle.report_success()
|
||||
trunc = ", TRUNCATED" if result.get("truncated") else ""
|
||||
ua_note = ""
|
||||
if result.get("user_agent_used") != USER_AGENT:
|
||||
ua_note = " [fallback UA]"
|
||||
logger.info(f" [OK] {u[:55]} ({result['text_length']:,} chars{trunc}{ua_note})")
|
||||
fb_note = " [wayback]" if result.get("fallback_used") else ""
|
||||
if fb_note:
|
||||
fallback_count[0] += 1
|
||||
logger.info(f" [OK] {u[:55]} ({result['text_length']:,} chars"
|
||||
f"{trunc}{ua_note}{fb_note})")
|
||||
else:
|
||||
err_count[0] += 1
|
||||
throttle.report_failure(result.get("error", ""))
|
||||
# 统计反爬拦截
|
||||
if result.get("anti_bot_detected"):
|
||||
anti_bot_count[0] += 1
|
||||
logger.error(f" [ERR] {u[:55]} ({result.get('error', 'unknown')})")
|
||||
return result
|
||||
|
||||
with ThreadPoolExecutor(max_workers=min(concurrency, len(urls))) as ex:
|
||||
with ThreadPoolExecutor(max_workers=min(throttle.concurrency, len(urls))) as ex:
|
||||
future_map = {ex.submit(_fetch_one, u): u for u in urls}
|
||||
for future in as_completed(future_map):
|
||||
try:
|
||||
@@ -937,17 +1206,75 @@ def fetch_top_results(results: dict, count: int, timeout: int = 10,
|
||||
except Exception as e:
|
||||
u = future_map[future]
|
||||
fetched.append({"url": u, "status": "error", "error": str(e),
|
||||
"text": "", "text_length": 0, "truncated": False})
|
||||
"text": "", "text_length": 0, "truncated": False,
|
||||
"anti_bot_detected": False, "waf_type": None,
|
||||
"fallback_used": None})
|
||||
logger.error(f" [ERR] {u[:55]} (thread error: {e})")
|
||||
|
||||
# Reorder to match original result order
|
||||
url_order = {u: i for i, u in enumerate(urls)}
|
||||
fetched.sort(key=lambda f: url_order.get(f["url"], 999))
|
||||
|
||||
logger.info(f"Fetched: {ok_count[0]} ok, {err_count[0]} errors")
|
||||
logger.info(f"Fetched: {ok_count[0]} ok, {err_count[0]} errors"
|
||||
f" (anti-bot: {anti_bot_count[0]}, wayback: {fallback_count[0]})")
|
||||
return fetched
|
||||
|
||||
|
||||
def _emit_fetch_report(fetched: list, throttle: "AdaptiveThrottle") -> None:
|
||||
"""v2.0.0: 输出结构化抓取报告到 stderr。
|
||||
|
||||
让 AI Agent 可程序化分析抓取过程：哪些 URL 被反爬拦截、用了什么兜底、
|
||||
自适应限流如何调整。格式为人类可读的表格 + JSON 摘要。
|
||||
"""
|
||||
import sys as _sys
|
||||
out = _sys.stderr
|
||||
lines = []
|
||||
lines.append("\n" + "=" * 72)
|
||||
lines.append("FETCH REPORT (v2.0.0)")
|
||||
lines.append("=" * 72)
|
||||
|
||||
# Per-URL 表
|
||||
header = f"{'URL':<45} {'Status':<8} {'WAF':<12} {'Fallback':<10} {'Chars':>10}"
|
||||
lines.append(header)
|
||||
lines.append("-" * len(header))
|
||||
for f in fetched:
|
||||
url = f.get("url", "")[:44]
|
||||
status = "OK" if f.get("status") == "ok" else "ERR"
|
||||
waf = f.get("waf_type") or "-"
|
||||
fb = f.get("fallback_used") or "-"
|
||||
chars = f.get("text_length", 0)
|
||||
lines.append(f"{url:<45} {status:<8} {waf:<12} {fb:<10} {chars:>10,}")
|
||||
|
||||
# 统计摘要
|
||||
total = len(fetched)
|
||||
ok = sum(1 for f in fetched if f.get("status") == "ok")
|
||||
err = total - ok
|
||||
anti_bot = sum(1 for f in fetched if f.get("anti_bot_detected"))
|
||||
wayback = sum(1 for f in fetched if f.get("fallback_used") == "wayback")
|
||||
lines.append("-" * len(header))
|
||||
lines.append(f"Total: {total} | OK: {ok} | Error: {err} | "
|
||||
f"Anti-bot blocked: {anti_bot} | Wayback recovered: {wayback}")
|
||||
|
||||
# 自适应限流状态
|
||||
s = throttle.stats()
|
||||
lines.append(f"Throttle: delay={s['current_delay']}s "
|
||||
f"concurrency={s['current_concurrency']} "
|
||||
f"paused={s['global_paused']} "
|
||||
f"consec_fail={s['consecutive_failures']} "
|
||||
f"consec_ok={s['consecutive_successes']}")
|
||||
|
||||
# JSON 摘要（一行，便于 Agent 解析）
|
||||
import json as _json
|
||||
summary = {
|
||||
"total": total, "ok": ok, "error": err,
|
||||
"anti_bot_blocked": anti_bot, "wayback_recovered": wayback,
|
||||
"throttle": s,
|
||||
}
|
||||
lines.append("JSON: " + _json.dumps(summary, ensure_ascii=False))
|
||||
lines.append("=" * 72 + "\n")
|
||||
print("\n".join(lines), file=out)
|
||||
|
||||
|
||||
# ----- Output formatting -----
|
||||
|
||||
def deduplicate_results(results: dict) -> dict:
|
||||
@@ -1240,24 +1567,42 @@ def _run_single_query(query: str, args, instance_urls: list,
|
||||
|
||||
if args.fetch > 0 and results.get("results"):
|
||||
emit_progress("fetch_start", count=args.fetch)
|
||||
# v2.0.0: Referer 默认设为首个实例 URL，伪装流量来自搜索引擎
|
||||
referer = getattr(args, "referer", None)
|
||||
if referer is None and instance_urls:
|
||||
referer = instance_urls[0]
|
||||
# v2.0.0: 创建共享 throttle 实例，用于 --fetch-report 输出
|
||||
request_delay = getattr(args, "request_delay", 0.3)
|
||||
fetch_throttle = AdaptiveThrottle(request_delay,
|
||||
min(5, args.fetch))
|
||||
fetched = fetch_top_results(
|
||||
results, args.fetch,
|
||||
timeout=args.fetch_timeout,
|
||||
auth_headers=auth_headers,
|
||||
max_retries=args.fetch_retries,
|
||||
max_size=args.max_size,
|
||||
request_delay=request_delay,
|
||||
referer=referer,
|
||||
fallback_enabled=not getattr(args, "no_fallback", False),
|
||||
throttle=fetch_throttle,
|
||||
)
|
||||
# Emit fetch_ok / fetch_fail events
|
||||
for f in fetched:
|
||||
if f.get("status") == "ok":
|
||||
emit_progress("fetch_ok", url=f.get("url", ""),
|
||||
chars=f.get("text_length", 0))
|
||||
chars=f.get("text_length", 0),
|
||||
fallback=f.get("fallback_used"))
|
||||
else:
|
||||
emit_progress("fetch_fail", url=f.get("url", ""),
|
||||
error=f.get("error", "unknown"))
|
||||
error=f.get("error", "unknown"),
|
||||
waf_type=f.get("waf_type"))
|
||||
results["fetched"] = fetched
|
||||
results["fetched_source"] = results.get("_fallback", "json")
|
||||
|
||||
# v2.0.0: --fetch-report 输出到 stderr
|
||||
if getattr(args, "fetch_report", False):
|
||||
_emit_fetch_report(fetched, fetch_throttle)
|
||||
|
||||
result_count = len(results.get("results", []))
|
||||
emit_progress("done", results=result_count, query=query)
|
||||
# 清理内部 _fallback 字段，避免泄漏到 JSON 输出。
|
||||
@@ -1506,6 +1851,21 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
|
||||
help="Timeout per page fetch in seconds (default: 10)")
|
||||
parser.add_argument("--fetch-retries", type=int, default=_cfg_int(config, "fetch_retries", 3),
|
||||
help="Max retries per page fetch (default: 3)")
|
||||
parser.add_argument("--fetch-report", action="store_true",
|
||||
help="When used with --fetch, emit a structured fetch report to stderr "
|
||||
"after completion: per-URL status, UA used, attempts, WAF type, "
|
||||
"fallback used, and adaptive throttle stats. v2.0.0.")
|
||||
parser.add_argument("--no-fallback", action="store_true",
|
||||
help="Disable Wayback Machine fallback for failed fetches (404/403/timeout). "
|
||||
"By default Wayback fallback is ENABLED to maximize success rate. v2.0.0.")
|
||||
parser.add_argument("--referer", default=None, metavar="URL",
|
||||
help="Set Referer header for fetch requests (e.g. the SearXNG instance URL). "
|
||||
"Defaults to the instance URL when fetching result pages. v2.0.0.")
|
||||
parser.add_argument("--request-delay", type=float,
|
||||
default=_cfg_float(config, "request_delay", 0.3),
|
||||
metavar="SECONDS",
|
||||
help="Delay between fetch requests to avoid rate limiting (default: 0.3s). "
|
||||
"v2.0.0: adaptive throttling may increase this on consecutive failures.")
|
||||
parser.add_argument("--max-size", type=int, default=_cfg_int(config, "max_size", None),
|
||||
metavar="BYTES",
|
||||
help="Max page size in bytes (default: unlimited). Set to 5242880 for 5MB cap.")
|
||||
|
||||
Reference in New Issue
Block a user