feat(v2.4.0): 新增 E_BLOCKED 错误码——细分 fetch 403 反爬拦截
真实环境问题: 抓取被墙/反爬站点 (如 baike.baidu.com) 返回 403 时, classify_error 统一归为 E_AUTH, AI Agent 会误判为凭证问题而做无效的 认证重试。被封锁不是认证失败。 改动: - common.py: 新增 E_BLOCKED 错误码 + recovery_hint (提示换 URL/镜像/ 用 Wayback 兜底/--exclude-domain); 新增 classify_fetch_error() 与 _extract_status_code(): fetch 场景 403→E_BLOCKED, 401→E_AUTH, 其余委托 classify_error (搜索场景 403 仍为 E_AUTH, 不变) - fetch.py main(): 错误路径改用 classify_fetch_error (替代 classify_error) - search.py: fetch_page 与 fetch_top_results 线程错误路径同样切换 - 新增 11 个测试 (test_v240_blocked_code.py), 全量 572 测试通过 - 真实环境验证: baike 403 → E_BLOCKED, 正常站点不受影响 - 文档同步 (SKILL.md/README.md 错误码表, 版本号 2.4.0)
This commit is contained in:
+1
-1
@@ -7,7 +7,7 @@ Retry settings and shared HTTP utilities now live in ``common.py`` so that
|
||||
both ``search.py`` and ``fetch.py`` share one consistent implementation.
|
||||
"""
|
||||
|
||||
VERSION = "2.3.0"
|
||||
VERSION = "2.4.0"
|
||||
SCHEMA_VERSION = "1.0"
|
||||
USER_AGENT = f"searxng-cli/{VERSION}"
|
||||
|
||||
|
||||
@@ -624,6 +624,7 @@ def is_retryable_error(exc: BaseException) -> bool:
|
||||
# 错误码常量（供 search.py / fetch.py 引用）
|
||||
E_CONFIG = "E_CONFIG"
|
||||
E_AUTH = "E_AUTH"
|
||||
E_BLOCKED = "E_BLOCKED" # fetch 被站点反爬拦截（403），非凭证问题
|
||||
E_NETWORK = "E_NETWORK"
|
||||
E_RATE_LIMIT = "E_RATE_LIMIT"
|
||||
E_PARSE = "E_PARSE"
|
||||
@@ -639,6 +640,11 @@ RECOVERY_HINTS = {
|
||||
E_AUTH: "Verify --auth-bearer/--auth-basic credentials or "
|
||||
"SEARXNG_BEARER_TOKEN/SEARXNG_BASIC_AUTH env vars. Check token "
|
||||
"expiry and instance access permissions.",
|
||||
E_BLOCKED: "The site is blocking automated access (WAF / anti-bot / "
|
||||
"geo-block) — this is NOT a credentials problem; no auth "
|
||||
"change will help. Use the Wayback Machine fallback (on by "
|
||||
"default; --no-fallback disables), fetch a mirror or a "
|
||||
"different URL, or drop the domain with --exclude-domain.",
|
||||
E_NETWORK: "Retry with backoff, or try a different SearXNG instance. "
|
||||
"Check network connectivity, proxy settings, and instance uptime.",
|
||||
E_RATE_LIMIT: "Wait before retrying (exponential backoff). Reduce query "
|
||||
@@ -767,6 +773,44 @@ def classify_error(exc: BaseException) -> str:
|
||||
return E_INTERNAL
|
||||
|
||||
|
||||
def _extract_status_code(exc) -> int:
|
||||
"""从异常及其 ``__cause__`` 链中提取 HTTP 状态码（无则返回 None）。
|
||||
|
||||
检查顺序：urllib ``.code`` → requests ``.response.status_code`` →
|
||||
消息中的 ``HTTP NNN`` 模式（fetch_url 抛出的 RuntimeError 消息）。
|
||||
"""
|
||||
target = getattr(exc, "__cause__", None) or exc
|
||||
status = getattr(target, "code", None)
|
||||
if status is None:
|
||||
resp = getattr(target, "response", None)
|
||||
status = getattr(resp, "status_code", None)
|
||||
if status is None:
|
||||
m = re.search(r"HTTP (\d{3})", str(exc))
|
||||
if m:
|
||||
status = int(m.group(1))
|
||||
return status
|
||||
|
||||
|
||||
def classify_fetch_error(exc: BaseException, status_code: int = None) -> str:
|
||||
"""fetch 场景的错误分类：403 → E_BLOCKED，401 → E_AUTH，其余委托 classify_error。
|
||||
|
||||
网页抓取（fetch.py / search.py --fetch）遇到的 403 绝大多数是站点反爬
|
||||
拦截（UA/JS 指纹、WAF、区域封锁），**不是凭证错误**。classify_error
|
||||
把 401/403 统一归为 E_AUTH 是为 SearXNG 实例认证设计的——若 fetch 也
|
||||
用它，AI Agent 会误判为"需要检查凭证"而去做无效的认证重试。本函数在
|
||||
classify_error 基础上仅做 fetch 场景的细分。
|
||||
|
||||
``status_code`` 可选：调用方已从异常链提取时可直接传入，避免重复解析。
|
||||
"""
|
||||
if status_code is None:
|
||||
status_code = _extract_status_code(exc)
|
||||
if status_code == 403:
|
||||
return E_BLOCKED
|
||||
if status_code == 401:
|
||||
return E_AUTH
|
||||
return classify_error(exc)
|
||||
|
||||
|
||||
# ----- Progress event emitter (for --progress flag) -----
|
||||
#
|
||||
# 当 --progress 启用时，search.py 会调用 emit_progress() 发射结构化事件到
|
||||
|
||||
+2
-2
@@ -54,7 +54,7 @@ from common import (
|
||||
build_auth_headers,
|
||||
build_browser_headers,
|
||||
build_wayback_url,
|
||||
classify_error,
|
||||
classify_fetch_error,
|
||||
compute_backoff_delay,
|
||||
detect_charset,
|
||||
force_utf8_stdout,
|
||||
@@ -1346,7 +1346,7 @@ Examples:
|
||||
# v2.1.0: Wayback Machine 兜底
|
||||
# 触发条件：兜底启用 + (错误可恢复 OR 命中被墙站点)
|
||||
fatal_error = str(e) if str(e) else e.__class__.__name__
|
||||
fatal_error_code = classify_error(e)
|
||||
fatal_error_code = classify_fetch_error(e, status_code=status_code)
|
||||
fatal_status_code = status_code
|
||||
error_msg = fatal_error
|
||||
if fallback_enabled and (should_try_wayback(error_msg) or hard_blocked):
|
||||
|
||||
+6
-3
@@ -38,6 +38,7 @@ from common import (
|
||||
build_auth_headers,
|
||||
build_wayback_url,
|
||||
classify_error,
|
||||
classify_fetch_error,
|
||||
compute_backoff_delay,
|
||||
detect_charset,
|
||||
emit_progress,
|
||||
@@ -906,8 +907,10 @@ def fetch_page(url: str, timeout: int = 10, auth_headers: dict = None,
|
||||
except Exception as e:
|
||||
error_msg = str(e) if str(e) else e.__class__.__name__
|
||||
# v2.1.0：结构化错误码，让 AdaptiveThrottle 能用 error_code 检测 429
|
||||
# 而非字符串匹配（"Too Many Requests" 不含 "429" 会漏判）
|
||||
error_code = classify_error(e)
|
||||
# 而非字符串匹配（"Too Many Requests" 不含 "429" 会漏判）。
|
||||
# v2.4.0：抓取场景用 classify_fetch_error——403 反爬拦截细分为
|
||||
# E_BLOCKED，避免误判为 E_AUTH（凭证问题）。
|
||||
error_code = classify_fetch_error(e)
|
||||
|
||||
# fetch_url 对 PDF/DOCX/XLSX 解析失败不抛异常，而是返回带 error_code
|
||||
# 的 FetchResult——此处必须显式检查，否则失败会被当作成功处理
|
||||
@@ -1430,7 +1433,7 @@ def fetch_top_results(results: dict, count: int, timeout: int = 10,
|
||||
except Exception as e:
|
||||
u = future_map[future]
|
||||
fetched.append({"url": u, "status": "error", "error": str(e),
|
||||
"error_code": classify_error(e),
|
||||
"error_code": classify_fetch_error(e),
|
||||
"text": "", "text_length": 0, "truncated": False,
|
||||
"anti_bot_detected": False, "waf_type": None,
|
||||
"fallback_used": None,
|
||||
|
||||
Reference in New Issue
Block a user