fix(v2.1.1): 修复执行问题记录中的真实 bug + 文档对齐
源码修复（5 项）: 1. search.py --time-range choices 加入 week（对齐 SearXNG API 四档） 2. fetch.py stdlib 路径处理 gzip/deflate 解压（被沙箱伪响应掩盖的真实 bug， 无 requests 环境抓取压缩服务器会全页 U+FFFD 乱码） 3. search.py --research 模式实现跨角度合并去重，输出 merged_results 字段 （兑现文档承诺 "Results are merged and deduplicated"） 4. search.py fetch_page 返回 error_code 字段 + AdaptiveThrottle 用 E_RATE_LIMIT 结构化检测 429（原字符串匹配 "429" 会漏判 "Too Many Requests"） 5. search.py _retry_with_backoff 复用 compute_backoff_delay（60s 封顶） + 处理 Retry-After header，与 fetch.py 保持一致 增强（3 项）: - common.py 精确化 baidu 子域列表（pan.baidu.com/cloud.baidu.com 不再误伤） - search.py expand_research_queries 根据主题语言切换中英文后缀 - search.py 新增 _warn_unresponsive_engines，识别实例侧引擎挂起并提示 文档/版本: - _config.py VERSION 2.1.0 → 2.1.1 - SKILL.md 同步更新（time-range week、merged_results、error_code、baidu 精确化） - README.md 同步更新 + 测试数量 503 → 539 测试: 539 个全部通过，含 6 个新增验证测试
This commit is contained in:
+1
-1
@@ -7,6 +7,6 @@ Retry settings and shared HTTP utilities now live in ``common.py`` so that
|
||||
both ``search.py`` and ``fetch.py`` share one consistent implementation.
|
||||
"""
|
||||
|
||||
VERSION = "2.1.0"
|
||||
VERSION = "2.1.1"
|
||||
SCHEMA_VERSION = "1.0"
|
||||
USER_AGENT = f"searxng-cli/{VERSION}"
|
||||
|
||||
+5
-3
@@ -751,6 +751,7 @@ def build_wayback_url(url: str) -> str:
|
||||
# 3. 列表按域名匹配（子域名也算命中）
|
||||
|
||||
HARD_BLOCKED_DOMAINS = frozenset([
|
||||
"www.baidu.com", # 百度搜索：强反爬 + Cookie 检测
|
||||
"baike.baidu.com", # 百度百科：强反爬 + Cookie 检测
|
||||
"zhidao.baidu.com", # 百度知道：同上
|
||||
"tieba.baidu.com", # 百度贴吧：同上
|
||||
@@ -764,15 +765,16 @@ HARD_BLOCKED_DOMAINS = frozenset([
|
||||
"www.douban.com", # 豆瓣主站
|
||||
"book.douban.com", # 豆瓣读书
|
||||
"movie.douban.com", # 豆瓣电影
|
||||
"tieba.baidu.com", # 百度贴吧（重复，确保子域匹配）
|
||||
])
|
||||
|
||||
# 部分域名需要子域匹配（如 *.zhihu.com, *.weibo.com, *.douban.com）
|
||||
# 需要子域匹配的域名（如 *.zhihu.com, *.weibo.com, *.douban.com）
|
||||
# 注意：baidu.com 不在此列——其反爬子域（www/baike/zhidao/wenku/tieba）已在
|
||||
# 上方精确列表中，而 pan.baidu.com（网盘）/cloud.baidu.com（智能云）等
|
||||
# 子域可正常抓取，整体匹配会误伤。zhihu/weibo/douban 的子域基本都被墙。
|
||||
_SUBDOMAIN_BLOCKED = frozenset([
|
||||
"zhihu.com",
|
||||
"weibo.com",
|
||||
"douban.com",
|
||||
"baidu.com",
|
||||
])
|
||||
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ for improved extraction quality (optional, falls back to stdlib).
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import gzip
|
||||
import logging
|
||||
import random
|
||||
import re
|
||||
@@ -16,6 +17,7 @@ import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import zlib
|
||||
from collections import namedtuple
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path
|
||||
@@ -737,6 +739,28 @@ def fetch_url(url: str, timeout=15, user_agent: str = None,
|
||||
content_type = resp.headers.get("Content-Type", "")
|
||||
final_url = resp.geturl()
|
||||
|
||||
# v2.1.1 修复：stdlib urllib 不自动解压 gzip/deflate
|
||||
# 服务器返回压缩字节流时 raw.decode() 会失败 →
|
||||
# errors="replace" → 全页 U+FFFD 乱码。
|
||||
# requests 库会自动处理 Content-Encoding，但 stdlib 不会。
|
||||
# 此前该 bug 被沙箱伪响应掩盖（两者都产生 U+FFFD），
|
||||
# 实际在无 requests 的真实环境中会复现。
|
||||
content_encoding = (resp.headers.get("Content-Encoding", "")
|
||||
.lower().strip())
|
||||
if content_encoding and raw:
|
||||
try:
|
||||
if "gzip" in content_encoding:
|
||||
raw = gzip.decompress(raw)
|
||||
elif "deflate" in content_encoding:
|
||||
# deflate 可能是 zlib 包装或裸 deflate
|
||||
try:
|
||||
raw = zlib.decompress(raw)
|
||||
except zlib.error:
|
||||
raw = zlib.decompress(raw, -zlib.MAX_WBITS)
|
||||
except (OSError, zlib.error) as e:
|
||||
logger.debug(f" decompress failed ({content_encoding}): {e}")
|
||||
# 解压失败保留原 raw，让下游 decode 兜底
|
||||
|
||||
if encoding:
|
||||
charset = encoding
|
||||
else:
|
||||
|
||||
+148
-17
@@ -36,9 +36,11 @@ from common import (
|
||||
build_auth_headers,
|
||||
build_wayback_url,
|
||||
classify_error,
|
||||
compute_backoff_delay,
|
||||
emit_progress,
|
||||
force_utf8_stdout,
|
||||
is_hard_blocked_domain,
|
||||
parse_retry_after,
|
||||
resolve_auth_basic,
|
||||
resolve_auth_bearer,
|
||||
set_progress_enabled,
|
||||
@@ -476,7 +478,12 @@ def _cfg_float(config: dict, key: str, default: float) -> float:
|
||||
# ----- Retry logic -----
|
||||
|
||||
def _retry_with_backoff(fn, max_retries: int = MAX_RETRIES, base_delay: float = RETRY_BACKOFF_BASE):
|
||||
"""Call fn with exponential backoff + jitter on transient failures."""
|
||||
"""Call fn with exponential backoff + jitter on transient failures.
|
||||
|
||||
v2.1.0 修复：复用 common.compute_backoff_delay（带 60s 封顶），
|
||||
避免高重试次数（如 --retry 10）时 1.5*2^10=1536s 卡死进程。
|
||||
同时遵守 Retry-After header（429/503），与 fetch.py 保持一致。
|
||||
"""
|
||||
last_error = None
|
||||
for attempt in range(max_retries + 1):
|
||||
try:
|
||||
@@ -485,15 +492,23 @@ def _retry_with_backoff(fn, max_retries: int = MAX_RETRIES, base_delay: float =
|
||||
if e.code in RETRYABLE_STATUS: # 403 (UA block) + 429 + 5xx
|
||||
last_error = e
|
||||
if attempt < max_retries:
|
||||
delay = base_delay * (2 ** attempt) + random.uniform(0, 1)
|
||||
logger.info(f" HTTP {e.code}, retrying in {delay:.1f}s... (attempt {attempt+1}/{max_retries})")
|
||||
# 429/503：遵守 Retry-After header，避免触发更严厉限流
|
||||
retry_after_sec = 0.0
|
||||
if e.code in (429, 503) and e.headers:
|
||||
retry_after_sec = parse_retry_after(
|
||||
e.headers.get("Retry-After", ""))
|
||||
delay = max(retry_after_sec,
|
||||
compute_backoff_delay(attempt, base=base_delay))
|
||||
logger.info(f" HTTP {e.code}, retrying in {delay:.1f}s... "
|
||||
f"(attempt {attempt+1}/{max_retries})"
|
||||
f"{f' Retry-After={retry_after_sec:.1f}s' if retry_after_sec > 0 else ''}")
|
||||
time.sleep(delay)
|
||||
continue
|
||||
raise
|
||||
except (urllib.error.URLError, OSError) as e:
|
||||
last_error = e
|
||||
if attempt < max_retries:
|
||||
delay = base_delay * (2 ** attempt) + random.uniform(0, 1)
|
||||
delay = compute_backoff_delay(attempt, base=base_delay)
|
||||
logger.info(f" Connection error ({e}), retrying in {delay:.1f}s...")
|
||||
time.sleep(delay)
|
||||
continue
|
||||
@@ -850,6 +865,7 @@ def fetch_page(url: str, timeout: int = 10, auth_headers: dict = None,
|
||||
# 主抓取
|
||||
result = None
|
||||
error_msg = None
|
||||
error_code = None
|
||||
try:
|
||||
result = fetch_url(
|
||||
url, timeout=timeout, auth_headers=auth_headers,
|
||||
@@ -858,6 +874,9 @@ def fetch_page(url: str, timeout: int = 10, auth_headers: dict = None,
|
||||
)
|
||||
except Exception as e:
|
||||
error_msg = str(e) if str(e) else e.__class__.__name__
|
||||
# v2.1.0：结构化错误码，让 AdaptiveThrottle 能用 error_code 检测 429
|
||||
# 而非字符串匹配（"Too Many Requests" 不含 "429" 会漏判）
|
||||
error_code = classify_error(e)
|
||||
|
||||
# 反爬检测（v2.0.0 增强：全文档扫描 + WAF 指纹库）
|
||||
# 必须在 Wayback 兜底判断之前执行：Cloudflare 质询页常返回 HTTP 200，
|
||||
@@ -914,6 +933,7 @@ def fetch_page(url: str, timeout: int = 10, auth_headers: dict = None,
|
||||
return {
|
||||
"url": url, "status": "error",
|
||||
"error": error_msg or "unknown error",
|
||||
"error_code": error_code,
|
||||
"text": "", "text_length": 0, "truncated": False,
|
||||
"anti_bot_detected": False, "waf_type": None,
|
||||
"fallback_used": None,
|
||||
@@ -924,6 +944,7 @@ def fetch_page(url: str, timeout: int = 10, auth_headers: dict = None,
|
||||
return {
|
||||
"url": url, "final_url": result.final_url, "status": "error",
|
||||
"error": f"Bot protection detected ({waf_type})",
|
||||
"error_code": E_PARSE,
|
||||
"text": "", "text_length": 0, "truncated": False,
|
||||
"anti_bot_detected": True, "waf_type": waf_type,
|
||||
"fallback_used": fallback_used,
|
||||
@@ -1157,12 +1178,23 @@ class AdaptiveThrottle:
|
||||
self._concurrency = min(self._initial_concurrency,
|
||||
self._concurrency * 2)
|
||||
|
||||
def report_failure(self, error_msg: str = "") -> None:
|
||||
def report_failure(self, error_msg: str = "",
|
||||
error_code: str = None) -> None:
|
||||
"""报告一次失败，触发自适应退避。
|
||||
|
||||
v2.1.0：优先用结构化 error_code 检测 429/限流（E_RATE_LIMIT），
|
||||
回退到字符串匹配兼容旧调用方。原代码仅检查 "429" 字面量，
|
||||
"Too Many Requests" 会漏判。
|
||||
"""
|
||||
with self._lock:
|
||||
self._consecutive_successes = 0
|
||||
self._consecutive_failures += 1
|
||||
# 429 → 全局暂停（调用方会从 error_msg 提取秒数，这里只标记）
|
||||
if "429" in error_msg.lower():
|
||||
# 429/限流 → 全局暂停 30s
|
||||
# 优先用 error_code，回退到字符串匹配（兼容无 error_code 的旧调用）
|
||||
is_rate_limit = (error_code == E_RATE_LIMIT or
|
||||
"429" in error_msg.lower() or
|
||||
"rate limit" in error_msg.lower())
|
||||
if is_rate_limit:
|
||||
self._global_pause_until = time.monotonic() + 30.0
|
||||
# 连续 3 次失败 → 退避 + 降并发
|
||||
if self._consecutive_failures >= 3:
|
||||
@@ -1263,7 +1295,8 @@ def fetch_top_results(results: dict, count: int, timeout: int = 10,
|
||||
f"{trunc}{ua_note}{fb_note})")
|
||||
else:
|
||||
err_count[0] += 1
|
||||
throttle.report_failure(result.get("error", ""))
|
||||
throttle.report_failure(result.get("error", ""),
|
||||
error_code=result.get("error_code"))
|
||||
# 统计反爬拦截
|
||||
if result.get("anti_bot_detected"):
|
||||
anti_bot_count[0] += 1
|
||||
@@ -1279,6 +1312,7 @@ def fetch_top_results(results: dict, count: int, timeout: int = 10,
|
||||
except Exception as e:
|
||||
u = future_map[future]
|
||||
fetched.append({"url": u, "status": "error", "error": str(e),
|
||||
"error_code": classify_error(e),
|
||||
"text": "", "text_length": 0, "truncated": False,
|
||||
"anti_bot_detected": False, "waf_type": None,
|
||||
"fallback_used": None})
|
||||
@@ -1581,6 +1615,45 @@ def _build_params(query: str, args) -> dict:
|
||||
return params
|
||||
|
||||
|
||||
def _warn_unresponsive_engines(results: dict, query: str,
|
||||
result_count: int = None) -> None:
|
||||
"""检测并提示实例侧引擎挂起/限流（v2.1.1）。
|
||||
|
||||
SearXNG JSON API 返回的 ``unresponsive_engines`` 字段格式为::
|
||||
|
||||
[["brave", "Suspended: too many requests"],
|
||||
["duckduckgo", "CAPTCHA"]]
|
||||
|
||||
当该字段非空时，说明实例内多个引擎被上游限流挂起。此时：
|
||||
1. 用 logger.warning 输出挂起的引擎列表及原因（到 stderr，
|
||||
不污染 stdout 数据流）
|
||||
2. 如果结果数较少，建议用 --engines 限定未挂起引擎
|
||||
|
||||
这是真实运营问题（见执行问题记录 #3）：连续查询后 brave/duckduckgo/
|
||||
startpage 等引擎会被上游限流挂起，导致结果骤减或全空。让用户及时
|
||||
感知引擎状态，避免误判为"无结果"而反复重试触发更严厉限流。
|
||||
"""
|
||||
unresponsive = results.get("unresponsive_engines", [])
|
||||
if not unresponsive:
|
||||
return
|
||||
# 格式化引擎列表：兼容 [engine, reason] 和 [engine] 两种格式
|
||||
parts = []
|
||||
for entry in unresponsive:
|
||||
if isinstance(entry, (list, tuple)) and len(entry) >= 2:
|
||||
parts.append(f"{entry[0]} ({entry[1]})")
|
||||
elif isinstance(entry, (list, tuple)) and len(entry) == 1:
|
||||
parts.append(str(entry[0]))
|
||||
else:
|
||||
parts.append(str(entry))
|
||||
engine_list = ", ".join(parts)
|
||||
logger.warning(f"Instance engines unresponsive: {engine_list}")
|
||||
# 结果数少 + 引擎挂起 → 建议规避
|
||||
if result_count is not None and result_count < 3 and len(unresponsive) >= 2:
|
||||
# 找出可能未挂起的常见引擎提示
|
||||
logger.warning("Hint: multiple engines suspended — consider using "
|
||||
"--engines to target responsive ones, or wait before retrying")
|
||||
|
||||
|
||||
def _run_single_query(query: str, args, instance_urls: list,
|
||||
auth_headers: dict, ttl_seconds: int):
|
||||
"""Run one query end-to-end: search → limit → domain-filter → fetch.
|
||||
@@ -1618,6 +1691,12 @@ def _run_single_query(query: str, args, instance_urls: list,
|
||||
logger.info(f"[cache stored] q={query!r} TTL={args.cache_ttl}min")
|
||||
emit_progress("cache_store", query=query, ttl=args.cache_ttl)
|
||||
|
||||
# v2.1.1：检测实例侧引擎挂起/限流（仅在实时查询后提示，cache hit 时
|
||||
# unresponsive_engines 信息可能已过期）
|
||||
if cached is None:
|
||||
_warn_unresponsive_engines(results, query,
|
||||
result_count=len(results.get("results", [])))
|
||||
|
||||
# Dedup (default on; --no-dedup disables) then sort, both BEFORE limit
|
||||
# so --max-results keeps the highest-scoring / newest items.
|
||||
if not args.no_dedup:
|
||||
@@ -1821,17 +1900,31 @@ def _read_queries_file(path: str) -> list:
|
||||
# 给定一个主题，自动扩展多角度查询词，复用批量搜索逻辑。
|
||||
# 扩展策略是确定性规则（不做 AI 判断），覆盖人物/主题/事件的通用研究维度。
|
||||
|
||||
# 研究角度定义：(角度标识, 后缀词)
|
||||
# 研究角度定义：(角度标识, 中文后缀, 英文后缀)
|
||||
# 顺序代表搜索优先级——基本信息优先，评价争议最后。
|
||||
# v2.1.0：支持中英文双语后缀，根据主题语言自动选择。
|
||||
# 中文主题用中文后缀（"简介"/"经历"等），英文主题用英文后缀
|
||||
# （"profile"/"background"等），避免 "Python asyncio 经历" 这类
|
||||
# 跨语言组合在英文引擎上匹配度低的问题。angle 标识符保持英文，
|
||||
# 便于 AI Agent 程序化处理。
|
||||
_RESEARCH_ANGLES = [
|
||||
("overview", ""), # 主题本身：最直接的搜索
|
||||
("profile", "简介"), # 基本信息：百科式介绍
|
||||
("background", "经历"), # 背景经历：生平/历史
|
||||
("works", "作品"), # 作品成就：产出物
|
||||
("review", "评价"), # 评价争议：外界看法
|
||||
("overview", "", ""), # 主题本身：最直接的搜索
|
||||
("profile", "简介", "profile"), # 基本信息：百科式介绍
|
||||
("background", "经历", "background"), # 背景经历：生平/历史
|
||||
("works", "作品", "works"), # 作品成就：产出物
|
||||
("review", "评价", "reviews"), # 评价争议：外界看法
|
||||
]
|
||||
|
||||
|
||||
def _is_chinese_topic(topic: str) -> bool:
|
||||
"""检测主题是否包含中文字符（CJK 统一表意文字范围）。
|
||||
|
||||
用于 expand_research_queries 选择中文还是英文后缀。
|
||||
纯英文主题（如 "Python asyncio"）返回 False，用英文后缀。
|
||||
"""
|
||||
return bool(re.search(r'[\u4e00-\u9fff]', topic))
|
||||
|
||||
|
||||
def expand_research_queries(topic: str) -> list:
|
||||
"""将研究主题扩展为多角度查询词列表。
|
||||
|
||||
@@ -1843,6 +1936,10 @@ def expand_research_queries(topic: str) -> list:
|
||||
4. works — 作品成就
|
||||
5. review — 评价争议
|
||||
|
||||
v2.1.0 修复：根据主题语言自动切换后缀。含中文字符的主题用中文后缀
|
||||
（"七森莉莉 简介"），纯英文主题用英文后缀（"Python asyncio profile"），
|
||||
避免跨语言组合在搜索引擎上匹配度低。
|
||||
|
||||
返回 [(angle, query), ...] 列表，angle 用于结果标注。
|
||||
|
||||
确定性规则，不依赖 AI 判断——确保跨进程可复现，AI Agent 可预期。
|
||||
@@ -1850,8 +1947,10 @@ def expand_research_queries(topic: str) -> list:
|
||||
topic = topic.strip()
|
||||
if not topic:
|
||||
return []
|
||||
use_chinese = _is_chinese_topic(topic)
|
||||
queries = []
|
||||
for angle, suffix in _RESEARCH_ANGLES:
|
||||
for angle, cn_suffix, en_suffix in _RESEARCH_ANGLES:
|
||||
suffix = cn_suffix if use_chinese else en_suffix
|
||||
query = f"{topic} {suffix}".strip()
|
||||
queries.append((angle, query))
|
||||
return queries
|
||||
@@ -1927,9 +2026,11 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
|
||||
help="Language code (e.g. en, zh-CN, de)")
|
||||
parser.add_argument("--pageno", "-p", type=int, default=1,
|
||||
help="Page number (default: 1)")
|
||||
parser.add_argument("--time-range", "-t", choices=["day", "month", "year", "none"],
|
||||
parser.add_argument("--time-range", "-t",
|
||||
choices=["day", "week", "month", "year", "none"],
|
||||
default=config.get("time_range", "year"),
|
||||
help="Time range filter (default: year; 'none' disables filtering)")
|
||||
help="Time range filter (default: year; 'none' disables filtering). "
|
||||
"SearXNG API standard four tiers: day/week/month/year")
|
||||
parser.add_argument("--safesearch", "-s", type=int, choices=[0, 1, 2],
|
||||
default=_cfg_int(config, "safesearch", 0),
|
||||
help="Safe search: 0=off, 1=moderate, 2=strict (default: 0=off)")
|
||||
@@ -2239,6 +2340,23 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
|
||||
batch.append({"query": q, "angle": angle, "status": "ok",
|
||||
"results": results})
|
||||
|
||||
# v2.1.0 修复：跨角度合并去重
|
||||
# 文档承诺 "Results are merged and deduplicated"，原代码只输出 per-angle
|
||||
# 结果，同一 URL 可能出现在多个角度中。此处合并所有成功角度的 results，
|
||||
# 去重后作为 merged_results 字段输出，让 AI Agent 既能按角度组织报告，
|
||||
# 也能获得去重后的总览。
|
||||
merged = {"query": topic, "results": []}
|
||||
for br in batch:
|
||||
if br.get("status") == "ok" and "results" in br:
|
||||
merged["results"].extend(br["results"].get("results", []))
|
||||
if merged["results"]:
|
||||
deduplicate_results(merged)
|
||||
sort_results(merged, args.sort_by)
|
||||
if args.max_results:
|
||||
merged["results"] = merged["results"][:args.max_results]
|
||||
merged_count = len(merged["results"])
|
||||
logger.info(f"Research merged: {merged_count} unique results after dedup")
|
||||
|
||||
# 输出
|
||||
if args.format == "json":
|
||||
output = json.dumps({
|
||||
@@ -2248,6 +2366,7 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
|
||||
{"angle": a, "query": q} for a, q in research_queries
|
||||
],
|
||||
"queries": batch,
|
||||
"merged_results": merged,
|
||||
}, indent=2, ensure_ascii=False)
|
||||
elif args.format == "csv":
|
||||
import csv as csv_mod
|
||||
@@ -2276,12 +2395,18 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
|
||||
output = out.getvalue().rstrip()
|
||||
elif args.format == "urls":
|
||||
parts = []
|
||||
# 先输出 per-angle 结果
|
||||
for br in batch:
|
||||
parts.append(f"# [{br.get('angle', '?')}] {br['query']}")
|
||||
if "results" in br:
|
||||
parts.append(format_urls(br["results"]))
|
||||
else:
|
||||
parts.append(f"# [ERROR: {br['error']}]")
|
||||
# 再输出合并去重后的总览
|
||||
if merged_count > 0:
|
||||
parts.append("")
|
||||
parts.append(f"# [MERGED] {topic} ({merged_count} unique results)")
|
||||
parts.append(format_urls(merged))
|
||||
output = "\n".join(parts)
|
||||
else: # brief
|
||||
parts = []
|
||||
@@ -2294,6 +2419,12 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
|
||||
else:
|
||||
parts.append(f"[ERROR: {br['error']}]")
|
||||
parts.append("")
|
||||
# 合并去重后的总览
|
||||
if merged_count > 0:
|
||||
parts.append("=" * 60)
|
||||
parts.append(f"[MERGED] {topic} ({merged_count} unique results)")
|
||||
parts.append("=" * 60)
|
||||
parts.append(format_brief(merged))
|
||||
output = "\n".join(parts)
|
||||
|
||||
if args.output:
|
||||
|
||||
Reference in New Issue
Block a user