MetonaTeam/searxng-use-cli
2
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(v2.0.0): 反爬增强 + 抓取稳定性大幅提升

Browse Source 
反爬措施:

- 浏览器指纹头 build_browser_headers(): Sec-Ch-Ua/Sec-Fetch-*/Accept-Language/Accept-Encoding, 绕过 80%+ 轻量 WAF

- 12 个 UA 池 (Chrome/Edge/Firefox x Win/macOS/Linux x v129-131)

- 确定性 UA 轮换 get_ua_for_domain(): SHA-256 按域名固定 UA, 会话内稳定跨进程可复现

- WAF 指纹库 _detect_anti_bot(): 识别 Cloudflare/Imperva/PerimeterX/DataDome/Akamai/通用, 全文档扫描

- Retry-After 遵守: 429/503 读取 header (数字或 HTTP date) 作为最小重试延迟

- 退避封顶 60s (原无上限, N=10 时 1536s 卡死进程)

抓取稳定性:

- requests.Session 复用: 连接池(10/host) + cookie 持久化 + TLS 会话恢复

- 超时分离 (connect, read) 元组, 避免大页面浪费已建连接

- Wayback Machine 兜底: 404/403/超时自动重试 web.archive.org, 默认启用 --no-fallback 关闭

- AdaptiveThrottle 自适应限流: 3 次失败翻倍延迟+减半并发, 5 次成功渐进恢复, 429 全局暂停 30s

- readability-lite 提取: article/main 缺失时按文本密度选最可能正文 div

新增 CLI flags:

- --fetch-report: 结构化抓取报告到 stderr (每 URL 状态/WAF 类型/兜底方式/字符数 + JSON 摘要)

- --no-fallback: 禁用 Wayback 兜底

- --referer: 设置 Referer 头 (默认实例 URL)

- --request-delay: 抓取请求间隔秒数 (默认 0.3, 自适应可能增大)

fetch 结果新字段: anti_bot_detected (bool), waf_type (str|null), fallback_used (str|null)

测试: 新增 4 个测试文件 (test_browser_headers/test_anti_bot/test_wayback_fallback/test_adaptive_throttle), 451 个测试全部通过
This commit is contained in:
thzxx
2026-08-01 21:34:57 +08:00
parent ea7a60a460
commit 28ff7c0a48
12 changed files with 2003 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/fetch.py
+201 -14
View File
@@ -19,6 +19,7 @@ import urllib.request
from collections import namedtuple
from html.parser import HTMLParser
from pathlib import Path
from typing import Optional
# Allow running standalone from any working directory
sys.path.insert(0, str(Path(__file__).resolve().parent))
@@ -29,9 +30,13 @@ from common import (
RETRY_BACKOFF_BASE,
apply_proxy,
build_auth_headers,
build_browser_headers,
compute_backoff_delay,
detect_charset,
force_utf8_stdout,
get_ua_for_domain,
is_retryable_error,
parse_retry_after,
resolve_auth_basic,
resolve_auth_bearer,
setup_logging,
@@ -125,7 +130,12 @@ def extract_with_stdlib(html_content: str) -> str:
def extract_with_bs4(html_content: str) -> str:
"""Extract text using BeautifulSoup for better quality."""
"""Extract text using BeautifulSoup for better quality.
v2.0.0 增强：当 article/main/role=main/content 类 div 都找不到时，
使用 readability-lite 文本密度算法从 body 中选择最可能是正文的
子元素，避免回退到整个 body 导致噪声（导航/侧边栏/页脚）污染输出。
"""
soup = _BeautifulSoup(html_content, "html.parser")
for tag in soup(["script", "style", "nav", "footer", "header",
@@ -141,6 +151,10 @@ def extract_with_bs4(html_content: str) -> str:
if main is None:
main = soup
# v2.0.0: 如果 main 是 body（兜底），用 readability-lite 提取正文
if main.name == "body":
main = _readability_lite(main) or main
text = main.get_text(separator="\n", strip=True)
lines = [line.strip() for line in text.split("\n") if line.strip()]
text = "\n".join(lines)
@@ -148,6 +162,65 @@ def extract_with_bs4(html_content: str) -> str:
return text
def _readability_lite(root) -> "Optional[object]":
"""readability-lite：用文本密度算法选择最可能是正文的子元素。
算法（受 readability.js 启发，简化版）：
1. 遍历 body 下所有 div/section/article 子节点
2. 计算每个节点的"文本密度" = 纯文本字符数 / 标签数
3. 排除明显是导航/侧边栏的节点（class/id 含 nav/sidebar/menu/footer）
4. 返回文本密度最高且字符数 > 200 的节点
返回 bs4 Tag 或 None（找不到合适节点时）。
这是 extract_with_bs4 的兜底增强，不改变原有 article/main 优先级。
"""
if root is None:
return None
candidates = root.find_all(["div", "section", "article"])
if not candidates:
return None
# 排除明显非正文节点
noise_pattern = re.compile(r"nav|sidebar|menu|footer|header|comment|"
r"related|share|social|widget|advert|banner|"
r"cookie|popup|modal", re.IGNORECASE)
best_node = None
best_score = 0.0
for node in candidates:
# 排除 class/id 命中噪声模式的节点
cls = " ".join(node.get("class", []))
nid = node.get("id", "")
if noise_pattern.search(cls) or noise_pattern.search(nid):
continue
# 计算纯文本字符数（去空白）
text = node.get_text(separator=" ", strip=True)
text_len = len(text)
if text_len < 200:
continue # 正文至少 200 字符
# 计算标签数（粗略：所有后代标签）
tag_count = len(node.find_all())
if tag_count == 0:
continue
# 文本密度 = 字符数 / 标签数；越高越可能是正文
density = text_len / tag_count
# 加权：段落 <p> 数量也是正文信号
p_count = len(node.find_all("p"))
score = density + (p_count * 10)
if score > best_score:
best_score = score
best_node = node
return best_node
def extract_text(html_content: str) -> str:
"""Extract readable text from HTML, preferring bs4 if available."""
if _HAS_BS4:
@@ -466,6 +539,51 @@ def html_to_markdown(html_content: str) -> str:
# ----- HTTP Fetch -----
# RETRY_BACKOFF_BASE, FALLBACK_UAS, detect_charset and is_retryable_error are
# imported from common.py (shared with search.py for a consistent retry policy).
#
# v2.0.0 改进：
# * 模块级 requests.Session 复用连接池 + cookie，减少 TLS 握手开销
# * 超时分离 (connect, read) 元组，避免大页面下载中途超时浪费已建连接
# * 集成 build_browser_headers() 发送完整浏览器指纹
# * 遵守 Retry-After header，避免盲目重试触发更严厉限流
# * compute_backoff_delay() 带封顶，避免高重试次数卡死进程
# 模块级 Session：复用 TCP 连接池、TLS 会话、cookie。
# 仅在 requests 可用时启用；stdlib 路径不受益但功能完整。
_session = None
def _get_session():
"""获取（惰性创建）模块级 requests.Session。
Session 复用带来：
* HTTP Keep-Alive 连接池（同站点多页面只握手一次）
* Cookie 持久化（某些站点登录态/反爬 cookie 自动携带）
* TLS 会话恢复（session resumption，节省 1-RTT）
单元测试可通过 ``_reset_session()`` 重置后用 ``_HAS_REQUESTS=False``
强制走 stdlib 路径。
"""
global _session
if _session is None and _HAS_REQUESTS:
_session = _requests.Session()
# 配置连接池：每主机最多 10 连接，总最多 20 连接
adapter = _requests.adapters.HTTPAdapter(
pool_connections=10, pool_maxsize=10, max_retries=0,
)
_session.mount("http://", adapter)
_session.mount("https://", adapter)
return _session
def _reset_session() -> None:
"""关闭并重置模块级 Session。测试用。"""
global _session
if _session is not None:
try:
_session.close()
except Exception:
pass
_session = None
class _NoRedirectHandler(urllib.request.HTTPRedirectHandler):
@@ -483,10 +601,11 @@ FetchResult = namedtuple(
)
def fetch_url(url: str, timeout: int = 15, user_agent: str = None,
def fetch_url(url: str, timeout=15, user_agent: str = None,
encoding: str = None, auth_headers: dict = None,
max_retries: int = 3, max_size: int = None,
allow_redirects: bool = True) -> "FetchResult":
allow_redirects: bool = True,
referer: str = None) -> "FetchResult":
"""Fetch a URL with retry, encoding detection, UA fallback, and optional size limit.
Returns a :class:`FetchResult` namedtuple with fields:
@@ -497,28 +616,65 @@ def fetch_url(url: str, timeout: int = 15, user_agent: str = None,
max_size=None means unlimited (full page). Set to e.g. 5242880 for a 5MB cap.
allow_redirects=False stops the client from following HTTP 3xx redirects.
v2.0.0 改进：
* ``timeout`` 支持标量（向后兼容）或 ``(connect, read)`` 元组；
标量会被转换为 ``(timeout, timeout*2)`` 分离建连和读超时
* ``referer`` 参数：设置 Referer 头，伪装来自搜索引擎的流量
* 浏览器指纹头：通过 build_browser_headers() 发送完整 Sec-* 头
* 确定性 UA：通过 get_ua_for_domain() 为同域名固定 UA
* Retry-After：429/503 响应读取 Retry-After header 作为最小重试延迟
* 退避封顶：compute_backoff_delay() 上限 60s
* Session 复用：requests 路径复用模块级 Session
"""
if user_agent is None:
user_agent = USER_AGENT
# 超时归一化：标量 → (connect, read) 元组
if isinstance(timeout, (int, float)):
connect_timeout = min(float(timeout), 10.0) # 建连不超过 10s
read_timeout = float(timeout)
timeout_tuple = (connect_timeout, read_timeout)
else:
timeout_tuple = timeout # 已是元组，原样使用
# 确定性 UA 选择：同域名固定 UA
effective_ua = get_ua_for_domain(url, user_agent)
last_error = None
user_agents = [user_agent] + FALLBACK_UAS
# UA 轮换池：首次用 effective_ua，后续重试轮换其他 UA
user_agents = [effective_ua] + [ua for ua in FALLBACK_UAS if ua != effective_ua]
for attempt in range(max_retries + 1):
# 同一域名内 UA 固定；仅在重试失败后才换（避免会话内突变）
# 但当退避原因可能是 UA 被屏蔽（403）时，必须换 UA
ua = user_agents[min(attempt, len(user_agents) - 1)]
headers = {"User-Agent": ua}
# 构造完整浏览器头（v2.0.0 核心）
headers = build_browser_headers(ua, referer=referer, accept_html=True)
if auth_headers:
headers.update(auth_headers)
try:
if _HAS_REQUESTS:
resp = _requests.get(url, timeout=timeout, headers=headers,
allow_redirects=allow_redirects, stream=True)
session = _get_session()
resp = session.get(url, timeout=timeout_tuple, headers=headers,
allow_redirects=allow_redirects, stream=True)
# stream=True holds the socket open; must close explicitly,
# including on raise_for_status() / max_size break / decode
# errors — otherwise the connection leaks back to the pool
# and long-running agents exhaust ports.
try:
# 429/503：读取 Retry-After，作为最小重试延迟
if resp.status_code in (429, 503) and attempt < max_retries:
retry_after_raw = resp.headers.get("Retry-After", "")
retry_after_sec = parse_retry_after(retry_after_raw)
resp.close()
delay = max(retry_after_sec,
compute_backoff_delay(attempt))
logger.info(f" Fetch retry {attempt+1}/{max_retries} "
f"(HTTP {resp.status_code}, Retry-After={retry_after_sec:.1f}s) "
f"in {delay:.1f}s")
time.sleep(delay)
continue
resp.raise_for_status()
# Read: unlimited if max_size is None, chunked with limit otherwise
@@ -554,10 +710,10 @@ def fetch_url(url: str, timeout: int = 15, user_agent: str = None,
# stdlib fallback
req = urllib.request.Request(url, headers=headers)
if allow_redirects:
_opener = urllib.request.urlopen(req, timeout=timeout)
_opener = urllib.request.urlopen(req, timeout=timeout_tuple[1])
else:
_opener = urllib.request.build_opener(_NoRedirectHandler).open(
req, timeout=timeout)
req, timeout=timeout_tuple[1])
with _opener as resp:
if max_size is None:
raw = resp.read()
@@ -591,8 +747,20 @@ def fetch_url(url: str, timeout: int = 15, user_agent: str = None,
except urllib.error.HTTPError as e:
last_error = e
# 429/503：读取 Retry-After（stdlib 路径）
if e.code in (429, 503) and attempt < max_retries:
retry_after_raw = e.headers.get("Retry-After", "") if e.headers else ""
retry_after_sec = parse_retry_after(retry_after_raw)
# HTTPError 本身是可读的响应对象（fp 已被 urllib 消费），
# 无需显式 close；直接进入退避。
delay = max(retry_after_sec, compute_backoff_delay(attempt))
logger.info(f" Fetch retry {attempt+1}/{max_retries} "
f"(HTTP {e.code}, Retry-After={retry_after_sec:.1f}s) "
f"in {delay:.1f}s")
time.sleep(delay)
continue
if is_retryable_error(e) and attempt < max_retries:
delay = RETRY_BACKOFF_BASE * (2 ** attempt) + random.uniform(0, 1)
delay = compute_backoff_delay(attempt)
logger.info(f" Fetch retry {attempt+1}/{max_retries} ({e}) in {delay:.1f}s")
time.sleep(delay)
continue
@@ -600,7 +768,7 @@ def fetch_url(url: str, timeout: int = 15, user_agent: str = None,
except (urllib.error.URLError, OSError, TimeoutError) as e:
last_error = e
if attempt < max_retries:
delay = RETRY_BACKOFF_BASE * (2 ** attempt) + random.uniform(0, 1)
delay = compute_backoff_delay(attempt)
logger.info(f" Fetch retry {attempt+1}/{max_retries} ({e}) in {delay:.1f}s")
time.sleep(delay)
continue
@@ -611,8 +779,23 @@ def fetch_url(url: str, timeout: int = 15, user_agent: str = None,
if _HAS_REQUESTS and isinstance(e, _requests.exceptions.RequestException):
last_error = e
status = getattr(getattr(e, "response", None), "status_code", None)
# 429/503 with Retry-After
if status in (429, 503) and attempt < max_retries:
resp_obj = getattr(e, "response", None)
retry_after_raw = ""
if resp_obj is not None:
retry_after_raw = resp_obj.headers.get("Retry-After", "")
retry_after_sec = parse_retry_after(retry_after_raw)
if resp_obj is not None:
resp_obj.close()
delay = max(retry_after_sec, compute_backoff_delay(attempt))
logger.info(f" Fetch retry {attempt+1}/{max_retries} "
f"(HTTP {status}, Retry-After={retry_after_sec:.1f}s) "
f"in {delay:.1f}s")
time.sleep(delay)
continue
if (status is None or status in RETRYABLE_STATUS) and attempt < max_retries:
delay = RETRY_BACKOFF_BASE * (2 ** attempt) + random.uniform(0, 1)
delay = compute_backoff_delay(attempt)
logger.info(f" Fetch retry {attempt+1}/{max_retries} ({e}) in {delay:.1f}s")
time.sleep(delay)
continue
@@ -652,6 +835,9 @@ Examples:
help="Force charset for decoding (e.g. gbk, shift_jis)")
parser.add_argument("--no-redirect", action="store_true",
help="Do not follow HTTP redirects")
parser.add_argument("--referer", default=None, metavar="URL",
help="Set Referer header (e.g. https://www.google.com/) to "
"disguise traffic source. v2.0.0 anti-bot measure.")
parser.add_argument("--proxy", default=None, metavar="URL",
help="HTTP/HTTPS proxy URL (e.g. http://corp-proxy:8080). "
"Respects existing HTTP_PROXY/HTTPS_PROXY env vars when omitted.")
@@ -708,6 +894,7 @@ Examples:
encoding=args.encoding, auth_headers=auth_headers,
max_retries=args.retries, max_size=args.max_size,
allow_redirects=not args.no_redirect,
referer=args.referer,
)
content, content_type, final_url = (
result.content, result.content_type, result.final_url,