diff --git a/README.md b/README.md index 420a863..4ebc64b 100644 --- a/README.md +++ b/README.md @@ -297,7 +297,7 @@ python scripts/fetch.py -u https://example.com \ - JSON Lines 流式输出（`--stream`，含 `error` 事件类型） - 进度事件（`--progress`，含 `instance_try`/`instance_ok`/`instance_fail`，JSON Lines 到 stderr） - batch 模式统一 schema（`status` 字段区分成功/失败） -- 451 个单元+集成测试 +- 458 个单元+集成测试 ## 跨 Agent 兼容性 @@ -325,7 +325,7 @@ pip install pytest pytest -q ``` -451 个测试覆盖：缓存操作、认证解析、域名过滤、Markdown 转换、搜索逻辑、集成流程、日志配置、HTML 回退、自动抓取、健康检查、输出格式化、实例解析、并行搜索、CLI 端到端、错误码分类、流式输出、进度事件、配置文件认证、schema_version、recovery_hint、batch 统一 schema、--dump-schema、UTF-8 stdout 强制、Windows APPDATA 路径、v2.0.0 浏览器指纹头、WAF 反爬检测、Wayback Machine 兜底、自适应限流。 +458 个测试覆盖：缓存操作、认证解析、域名过滤、Markdown 转换、搜索逻辑、集成流程、日志配置、HTML 回退、自动抓取、健康检查、输出格式化、实例解析、并行搜索、CLI 端到端、错误码分类、流式输出、进度事件、配置文件认证、schema_version、recovery_hint、batch 统一 schema、--dump-schema、UTF-8 stdout 强制、Windows APPDATA 路径、v2.0.0 浏览器指纹头、WAF 反爬检测（v2.0.1 收窄误判 + title 精准检测）、Wayback Machine 兜底（v2.0.1 修复 HTTP 200 反爬页触发）、自适应限流。 ## 项目结构 diff --git a/SKILL.md b/SKILL.md index 8875ef8..344123a 100644 --- a/SKILL.md +++ b/SKILL.md @@ -1,7 +1,7 @@ --- name: searxng-use-cli description: Use when you need to search the web via your OWN SearXNG instance (no public-instance discovery). 3 CLI scripts + a shared common.py module — execute privacy-respecting searches against a user-supplied instance (with multi-instance failover, 5xx/429 retry, auto-fetch) or via SEARXNG_INSTANCE env / config file, fetch/extract readable text or markdown from web pages. Zero-config replacement for proprietary search APIs. -version: 2.0.0 +version: 2.0.1 author: Metona Team license: MIT platforms: [linux, macos, windows] @@ -69,8 +69,8 @@ SearXNG is a privacy-respecting metasearch engine that aggregates results from 7 - **Split timeouts** — `timeout=(connect, read)` tuple (5s connect, 15s read by default). Previously a single 15s total timeout wasted already-established connections on large pages - **Retry-After compliance** — 429/503 responses read the `Retry-After` header (numeric seconds or HTTP date) and wait at least that long before retrying. Non-compliance triggers harsher rate limits - **Capped backoff** — `compute_backoff_delay()` caps at 60s (was uncapped: 1.5*2^10 = 1536s would hang the process) -- **WAF fingerprint library** — `_detect_anti_bot()` identifies Cloudflare / Imperva / PerimeterX / DataDome / Akamai / generic challenges via full-document scan (was first 2000 chars only). Returns `waf_type` for AI-agent decisioning -- **Wayback Machine fallback** — 404/403/timeout automatically retries via `https://web.archive.org/web/2/` (latest snapshot). Default ON; `--no-fallback` disables. Independent 10s timeout so Wayback slowness never blocks the main flow +- **WAF fingerprint library** — `_detect_anti_bot()` identifies Cloudflare / Imperva / PerimeterX / DataDome / Akamai / Anubis / generic challenges via `` tag matching (most precise) + full-document scan of technical identifiers (cookie/header/JS variable names, not bare vendor names). v2.0.1 narrowed broad keywords (e.g. bare `cloudflare`/`captcha`/`challenge`) that caused false positives on normal articles, and added title-tag detection. Returns `waf_type` for AI-agent decisioning +- **Wayback Machine fallback** — 404/403/timeout AND anti-bot-blocked pages automatically retry via `https://web.archive.org/web/2/<url>` (latest snapshot). v2.0.1 fixed a bug where HTTP 200 anti-bot challenge pages (Cloudflare returns 200 for JS challenges) bypassed the fallback trigger. Default ON; `--no-fallback` disables. Independent 10s timeout so Wayback slowness never blocks the main flow - **Adaptive throttling** — `AdaptiveThrottle` state machine: 3 consecutive failures → double delay + halve concurrency; 5 consecutive successes → gradual recovery; 429 → global pause 30s. Thread-safe - **readability-lite extraction** — when `<article>`/`<main>`/content-class `<div>` are all missing, `_readability_lite()` picks the highest text-density node (text chars / tag count + `<p>` weighting), avoiding nav/sidebar/footer noise - **`--fetch-report`** — structured per-URL report to stderr after `--fetch N`: status, WAF type, fallback used, char count, plus adaptive throttle stats and a JSON summary line diff --git a/scripts/_config.py b/scripts/_config.py index cc84971..b7696f9 100644 --- a/scripts/_config.py +++ b/scripts/_config.py @@ -7,6 +7,6 @@ Retry settings and shared HTTP utilities now live in ``common.py`` so that both ``search.py`` and ``fetch.py`` share one consistent implementation. """ -VERSION = "2.0.0" +VERSION = "2.0.1" SCHEMA_VERSION = "1.0" USER_AGENT = f"searxng-cli/{VERSION}" diff --git a/scripts/search.py b/scripts/search.py index 5e67dba..851905e 100644 --- a/scripts/search.py +++ b/scripts/search.py @@ -13,6 +13,7 @@ import json import logging import os import random +import re import sys import threading import time @@ -855,9 +856,35 @@ def fetch_page(url: str, timeout: int = 10, auth_headers: dict = None, except Exception as e: error_msg = str(e) if str(e) else e.__class__.__name__ - # Wayback 兜底：主抓取失败或被反爬拦截时尝试 + # 反爬检测（v2.0.0 增强：全文档扫描 + WAF 指纹库） + # 必须在 Wayback 兜底判断之前执行：Cloudflare 质询页常返回 HTTP 200， + # 此时 result 非 None 但内容是反爬页，必须识别出来才能触发兜底。 + anti_bot_detected = False + waf_type = None + if result is not None: + content = result.content + content_type = result.content_type or "" + is_html = ("html" in content_type.lower() or + content.strip().startswith("<!") or + content.strip().startswith("<htm")) + if is_html: + waf_type = _detect_anti_bot(content) + if waf_type: + anti_bot_detected = True + + # Wayback 兜底触发条件（v2.0.1 修复）： + # 1. 主抓取抛异常且错误暗示 404/403/超时（_should_try_fallback） + # 2. 主抓取"成功"但被反爬拦截（anti_bot_detected=True） + # 原 v2.0.0 bug：仅条件 1 触发兜底，条件 2 漏网（Cloudflare 200 质询页）。 fallback_used = None - if fallback_enabled and _should_try_fallback(result, error_msg): + need_fallback = False + if fallback_enabled: + if anti_bot_detected: + need_fallback = True + elif result is None and _should_try_fallback(result, error_msg): + need_fallback = True + + if need_fallback: wb_result = _try_wayback_fallback(url, timeout=timeout, auth_headers=auth_headers, max_retries=max_retries, @@ -866,6 +893,18 @@ def fetch_page(url: str, timeout: int = 10, auth_headers: dict = None, result = wb_result error_msg = None fallback_used = "wayback" + # Wayback 结果重新做反爬检测（防御性：Wayback 快照极少是反爬页） + anti_bot_detected = False + waf_type = None + content = result.content + content_type = result.content_type or "" + is_html = ("html" in content_type.lower() or + content.strip().startswith("<!") or + content.strip().startswith("<htm")) + if is_html: + waf_type = _detect_anti_bot(content) + if waf_type: + anti_bot_detected = True # 仍然失败 if result is None: @@ -877,25 +916,10 @@ def fetch_page(url: str, timeout: int = 10, auth_headers: dict = None, "fallback_used": None, } - content = result.content - content_type = result.content_type - final_url = result.final_url - - is_html = ("html" in content_type.lower() or - content.strip().startswith("<!") or - content.strip().startswith("<htm")) - - # 反爬检测（v2.0.0 增强：全文档扫描 + WAF 指纹库） - anti_bot_detected = False - waf_type = None - if is_html: - waf_type = _detect_anti_bot(content) - if waf_type: - anti_bot_detected = True - + # 反爬仍被检测到（Wayback 也无能为力或兜底被禁用） if anti_bot_detected: return { - "url": url, "final_url": final_url, "status": "error", + "url": url, "final_url": result.final_url, "status": "error", "error": f"Bot protection detected ({waf_type})", "text": "", "text_length": 0, "truncated": False, "anti_bot_detected": True, "waf_type": waf_type, @@ -905,7 +929,7 @@ def fetch_page(url: str, timeout: int = 10, auth_headers: dict = None, text = extract_text(content) if is_html else content return { "url": url, - "final_url": final_url, + "final_url": result.final_url, "status": "ok", "content_type": content_type, "text": text, @@ -971,59 +995,105 @@ def _try_wayback_fallback(url: str, timeout: int = 10, return None -# ----- 反爬检测（v2.0.0 增强版）----- +# ----- 反爬检测（v2.0.1 收窄误判 + title 精准检测）----- # WAF 指纹库：每项 = (waf_type, [指示词]) -# 指示词在页面 HTML/body/headers 中出现即判定为该 WAF。 -# 顺序按检测优先级：专用指纹在前，通用指纹在后。 +# +# v2.0.1 修复：v2.0.0 用裸公司名（cloudflare/akamai）和宽泛词（captcha/ +# challenge/dd-）做全文匹配，导致正常文章（如引用 Cloudflare 文档、讨论 +# "coding challenges" 的文章、Wayback 归档正文）被误判为反爬页，Wayback +# 兜底返回的有效内容也被丢弃。收窄原则： +# 1. 专用指纹只用 WAF 厂商的技术标识符（cookie 名/HTTP header 名/JS 变量名） +# ——这些不会出现在文章正文里 +# 2. 通用文案用完整短语而非单词（如 "please complete the captcha" 而非 +# "captcha"），避免正常内容误判 +# 3. 增加 <title> 标签检测——反爬页 title 是特征文案，最精准 WAF_FINGERPRINTS = [ ("cloudflare", [ + # Cloudflare 技术标识符（cookie/header/JS 变量名，不会出现在正文） "cf-ray", "cf-chl-bypass", "cf-mitigated", - "cloudflare", "cf-browser-verification", - "attention required! | cloudflare", "just a moment", - "checking your browser before accessing", + "cf-browser-verification", "cf-error-details", "cf-error-code", + # 质询页特征文案（足够具体，正常内容不会完整出现） + "just a moment", "checking your browser before accessing", + "attention required! | cloudflare", + "enable javascript and cookies to continue", ]), ("imperva", [ + # Incapsula cookie/技术标识符 "incap_ses", "visid_incap", "incap_ses_", - "imperva", "incapsula", - "request unsuccessful. incapsula incident id", + "incapsula incident id", "request unsuccessful. incapsula", + "visit denied by incapsula", ]), ("perimeterx", [ - "_px", "px-captcha", "pxhd", "pxcts", "pxcookie", - "perimeterx", "press & hold to confirm you are a human", + # PerimeterX 专有标识符（_px 太短会匹配 CSS 类名，已移除） + "px-captcha", "pxhd", "pxcts", "pxcookie", + "_pxff", "_pxhd", + "press & hold to confirm you are a human", ]), ("datadome", [ - "datadome", "dd-", "data-dome", - "protected by datadome", + # DataDome 专有标识符（dd- 太宽泛会匹配 dd-class 等，已移除） + "datadome", "data-dome", + "protected by datadome", "datadome-bot-protect", ]), ("akamai", [ - "akamai", "bm_sz", "_abck", - "reference #", "akamaighost", + # Akamai Bot Manager cookie/标识符（akamai 裸名会匹配正文引用，已移除） + "bm_sz", "_abck", "akamaighost", "akamai-bot-manager", + "ak_bmsc", ]), - # 通用反爬指示词（无明确 WAF 归属） + # 通用反爬指示词（v2.0.1 收窄：完整短语而非单词，避免正文误判） ("generic", [ - "captcha", "challenge", "verify you are human", - "making sure you're not a bot", - "please enable javascript", "enable javascript to continue", - "ddos protection", "access denied", + "verify you are human", "verify that you are human", + "making sure you're not a bot", "are you a robot", + "robot or human", "human verification", + "please complete the captcha", "complete the security check", + "please enable javascript to continue", + "enable javascript to continue", + "ddos protection by", "access denied - sucuri", "you have been blocked", "unusual traffic from your computer", - "robot or human", "are you a robot", "pardon our interruption", "we'll be right back", + "bot protection", "anti-bot protection", + "anubis_challenge", "miserere", # Anubis 反爬系统（拦截 AI 爬虫） ]), ] +# <title> 标签检测：反爬页 title 通常是特征文案，比全文扫描更精准。 +# key = title 中的特征子串（小写），value = 对应 WAF 类型 +_TITLE_ANTI_BOT_SIGNATURES = { + "just a moment": "cloudflare", + "attention required": "cloudflare", + "access denied": "generic", + "are you a robot": "generic", + "robot check": "generic", + "human verification": "generic", + "please verify you are human": "generic", + "security check": "generic", + "verify you are human": "generic", +} + +_TITLE_RE = re.compile(r"<title[^>]*>(.*?)", re.IGNORECASE | re.DOTALL) + def _detect_anti_bot(content: str) -> str: """检测反爬页面，返回 WAF 类型或 None。 - v2.0.0 改进： - * 全文档扫描（去除 2000 字符限制——大页面反爬页可能在前 2000 字之外） - * WAF 指纹库覆盖 Cloudflare/Imperva/PerimeterX/DataDome/Akamai/通用 - * 返回具体 WAF 类型而非布尔值，让 AI Agent 可决策 + v2.0.1 改进： + * 增加 标签精准检测（反爬页 title 是特征文案，误判率极低） + * 收窄 WAF 指纹库关键词（移除裸公司名和宽泛词，改用技术标识符 + 完整短语） + * 保留全文档扫描（v2.0.0 改进，大页面反爬页可能在前 2000 字之外） - 性能：全文档 lower() 一次，对 5MB 页面约 5ms，可接受。 + 检测顺序：title 标签（最精准）→ 全文档指纹扫描（技术标识符 + 文案短语）。 + + 性能：全文档 lower() 一次 + title regex 一次，对 5MB 页面约 5ms，可接受。 """ if not content: return None + # 1. <title> 标签检测（最精准，误判率极低） + title_match = _TITLE_RE.search(content) + if title_match: + title_lower = title_match.group(1).strip().lower() + for sig, waf_type in _TITLE_ANTI_BOT_SIGNATURES.items(): + if sig in title_lower: + return waf_type + # 2. 全文档指纹扫描（收窄后的技术标识符 + 完整文案） lower = content.lower() for waf_type, indicators in WAF_FINGERPRINTS: for ind in indicators: diff --git a/tests/test_anti_bot.py b/tests/test_anti_bot.py index e8888fc..f0b6d25 100644 --- a/tests/test_anti_bot.py +++ b/tests/test_anti_bot.py @@ -85,6 +85,7 @@ def test_detect_akamai_reference(): # ----- Generic detection ----- def test_detect_generic_captcha(): + """v2.0.1：收窄为完整短语 'please complete the captcha'，避免正文误判。""" html = "<html><body>Please complete the CAPTCHA</body></html>" assert _detect_anti_bot(html) == "generic" @@ -95,7 +96,15 @@ def test_detect_generic_verify_human(): def test_detect_generic_access_denied(): - html = "<html><body>Access Denied</body></html>" + """v2.0.1：裸 'access denied' 从全文档扫描移除（避免权限文章误判）， + 改由 <title> 标签检测覆盖。反爬页 title 常为 'Access Denied'。""" + html = "<html><head><title>Access DeniedAccess Denied" + assert _detect_anti_bot(html) == "generic" + + +def test_detect_generic_access_denied_sucuri(): + """v2.0.1：Sucuri WAF 的完整文案在全文档扫描中保留。""" + html = "Access Denied - Sucuri Website Firewall" assert _detect_anti_bot(html) == "generic" @@ -130,7 +139,8 @@ def test_detect_anti_bot_beyond_2000_chars(): def test_detect_anti_bot_large_page_end(): """反爬关键词在文档末尾也能检测到。""" padding = "y" * 5000 - html = f"{padding}captcha" + # v2.0.1：用完整短语而非裸 captcha，避免误判 + html = f"{padding}please complete the captcha" assert _detect_anti_bot(html) == "generic" @@ -149,14 +159,34 @@ def test_detect_anti_bot_empty_content(): def test_detect_anti_bot_article_mentions_captcha_in_context(): - """文章讨论 captcha 但不是反爬页（上下文判断的局限——接受误报）。 + """v2.0.1 修复：文章讨论 captcha 但不是反爬页，不再误判。 - 注意：当前实现是关键词匹配，无法区分"讨论 captcha 的文章"和 - "captcha 拦截页"。这是已知局限，测试记录此行为。 + v2.0.0 用裸 "captcha" 做全文匹配，导致"讨论 captcha 工作原理的文章" + 被误判为反爬页。v2.0.1 收窄为 "please complete the captcha" 等完整短语， + 正常讨论 captcha 的文章不再触发。 """ html = "

This article explains how CAPTCHA works.

" - # 关键词匹配会误报为 generic - assert _detect_anti_bot(html) == "generic" + # v2.0.1：收窄后不再误判 + assert _detect_anti_bot(html) is None + + +def test_detect_anti_bot_article_mentions_cloudflare_in_context(): + """v2.0.1 修复：文章引用 Cloudflare 文档链接，不再误判为 cloudflare WAF。 + + v2.0.0 用裸 "cloudflare" 做全文匹配，Wayback 归档的 DataCamp 文章 + 正文里有 cloudflare.com 链接，被误判为反爬页导致兜底失败。 + """ + html = ("
" + "

Learn more at Cloudflare

" + "

Join our daily coding challenges!

" + "
") + assert _detect_anti_bot(html) is None + + +def test_detect_anti_bot_article_mentions_challenge_in_context(): + """v2.0.1 修复：文章含 'coding challenge' 等正常内容，不再误判。""" + html = "

Daily 5-minute coding challenges.

" + assert _detect_anti_bot(html) is None # ----- Priority: specialized WAF before generic ----- @@ -197,7 +227,8 @@ def test_waf_fingerprints_all_have_indicators(): def test_is_blocked_page_delegates_to_detect(): """_is_blocked_page 应委托给 _detect_anti_bot。""" - assert _is_blocked_page("captcha") is True + # v2.0.1：用完整反爬短语而非裸 captcha + assert _is_blocked_page("please complete the captcha") is True assert _is_blocked_page("normal content") is False diff --git a/tests/test_fetch_auto.py b/tests/test_fetch_auto.py index 393de8c..806fea1 100644 --- a/tests/test_fetch_auto.py +++ b/tests/test_fetch_auto.py @@ -46,9 +46,12 @@ def test_blocked_page_checks_first_2000_chars(): v2.0.0 changed this to full-document scanning because large anti-bot pages (e.g. Cloudflare challenges with big JS blobs) may place the telltale keyword beyond the 2000-char boundary. + + v2.0.1: 用完整短语 'please complete the captcha' 替代裸 'captcha'， + 避免正常内容误判。 """ padding = "x" * 2500 - html = f"{padding}captcha" + html = f"{padding}please complete the captcha" # v2.0.0: now detected (was: not detected) assert _is_blocked_page(html) diff --git a/tests/test_wayback_fallback.py b/tests/test_wayback_fallback.py index f8e92ff..23a9c15 100644 --- a/tests/test_wayback_fallback.py +++ b/tests/test_wayback_fallback.py @@ -208,22 +208,67 @@ def test_fetch_page_normal_page_no_anti_bot(): def test_fetch_page_anti_bot_triggers_wayback(): - """被反爬拦截后也应尝试 Wayback（_should_try_fallback 防御性检查）。 + """被反爬拦截后应尝试 Wayback 兜底（v2.0.1 修复）。 - 当前实现：主抓取成功返回反爬页内容 → result 非 None → 不触发兜底。 - 此测试记录此行为：反爬页被当作"成功抓取"返回，在 fetch_page 内部检测。 + v2.0.0 bug：主抓取"成功"返回反爬页（HTTP 200 + Cloudflare 质询）时， + result 非 None 导致 _should_try_fallback 返回 False，Wayback 永不触发。 + v2.0.1 修复：反爬检测提前到兜底判断之前，反爬阳性也触发 Wayback。 + + 本测试 mock fetch_url 两次调用： + 1. 主抓取 → 返回 Cloudflare 质询页 + 2. Wayback 兜底 → 返回正常归档页 """ cf_html = "Just a moment... cf-ray: 123" + wb_html = "

Archived article content.

" + cf_result = _make_result(content=cf_html, final_url="https://example.com") + wb_result = _make_result(content=wb_html, + final_url="https://web.archive.org/web/2024/https://example.com") with patch("search.fetch_url", - return_value=_make_result(content=cf_html)): + side_effect=[cf_result, wb_result]) as mock_fetch: result = fetch_page("https://example.com", fallback_enabled=True) - # 反爬页被检测到，标记为 error + anti_bot_detected + # Wayback 兜底成功，反爬标记清除，status=ok + assert result["status"] == "ok" + assert result["anti_bot_detected"] is False + assert result["waf_type"] is None + assert result["fallback_used"] == "wayback" + assert "Archived article content." in result["text"] + # 确认 fetch_url 被调用两次：主抓取 + Wayback + assert mock_fetch.call_count == 2 + # 第二次调用应该是 Wayback URL + assert "web.archive.org/web/2/" in mock_fetch.call_args_list[1][0][0] + + +def test_fetch_page_anti_bot_wayback_also_blocked(): + """主抓取反爬 + Wayback 也反爬/失败 → 最终返回 error。 + + Wayback 兜底返回 None（失败）时，保留主抓取的反爬检测结果。 + """ + cf_html = "Just a moment... cf-ray: 123" + cf_result = _make_result(content=cf_html, final_url="https://example.com") + with patch("search.fetch_url", + side_effect=[cf_result, RuntimeError("wayback timeout")]): + result = fetch_page("https://example.com", fallback_enabled=True) + # Wayback 失败，保留反爬 error assert result["status"] == "error" assert result["anti_bot_detected"] is True - # 但不会触发 Wayback（因为主抓取"成功"了，只是内容是反爬页） + assert result["waf_type"] == "cloudflare" assert result["fallback_used"] is None +def test_fetch_page_anti_bot_fallback_disabled(): + """--no-fallback 时反爬页直接返回 error，不尝试 Wayback。""" + cf_html = "Just a moment... cf-ray: 123" + with patch("search.fetch_url", + return_value=_make_result(content=cf_html)) as mock_fetch: + result = fetch_page("https://example.com", fallback_enabled=False) + assert result["status"] == "error" + assert result["anti_bot_detected"] is True + assert result["waf_type"] == "cloudflare" + assert result["fallback_used"] is None + # 只调用一次（主抓取），没有 Wayback + assert mock_fetch.call_count == 1 + + def test_fetch_page_passes_referer(): """referer 透传给 fetch_url。""" with patch("search.fetch_url", return_value=_make_result()) as mock_fetch: