feat(v1.8.0): 稳定性修复 + AI Agent 体验增强
稳定性修复: - 修复 cache.py SQLite 连接泄漏（contextlib.closing 包装） - 修复 fetch.py requests stream=True 连接泄漏（try/finally resp.close()） - RETRYABLE_STATUS 新增 403，激活 UA fallback 切换逻辑 - --cache-stats 移至实例解析前，无需实例即可查询 - classify_error 从错误消息提取 HTTP 状态码，正确分类 E_AUTH/E_RATE_LIMIT - --stream 与 --queries-file 互斥检查，违规报 E_INPUT - batch 退出码语义统一（0=有结果 / 1=全部错误 / 2=全部空结果） AI Agent 体验增强: - 错误码体系完善：E_CONFIG/E_AUTH/E_NETWORK/E_RATE_LIMIT/E_PARSE/E_EMPTY/E_INPUT/E_INTERNAL - recovery_hint 恢复提示字段，AI Agent 可程序化决策恢复策略 - stream 模式新增 error 事件类型（含 error_code + recovery_hint） - 进度事件扩展：instance_try/instance_ok/instance_fail - batch 模式统一 schema（status 字段区分 success/failed） - JSON 输出含 schema_version 字段确保版本兼容 测试与文档: - 测试覆盖：330 -> 352 - SKILL.md / README.md 同步更新
This commit is contained in:
+3
-2
@@ -1,11 +1,12 @@
|
||||
"""Package-level constants for searxng-cli scripts.
|
||||
|
||||
Import in sibling scripts with:
|
||||
from _config import VERSION, USER_AGENT
|
||||
from _config import VERSION, USER_AGENT, SCHEMA_VERSION
|
||||
|
||||
Retry settings and shared HTTP utilities now live in ``common.py`` so that
|
||||
both ``search.py`` and ``fetch.py`` share one consistent implementation.
|
||||
"""
|
||||
|
||||
VERSION = "1.7.0"
|
||||
VERSION = "1.8.0"
|
||||
SCHEMA_VERSION = "1.0"
|
||||
USER_AGENT = f"searxng-cli/{VERSION}"
|
||||
|
||||
+11
-3
@@ -22,6 +22,7 @@ Design notes:
|
||||
cache failure must never break a search.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
@@ -40,8 +41,15 @@ def _cache_path() -> Path:
|
||||
return DEFAULT_CACHE_DIR / "cache.db"
|
||||
|
||||
|
||||
def _connect(path: Path) -> sqlite3.Connection:
|
||||
"""Open a connection with WAL mode and ensure the schema exists."""
|
||||
def _connect(path: Path):
|
||||
"""Open a connection with WAL mode and ensure the schema exists.
|
||||
|
||||
Returns a :class:`contextlib.closing` wrapper so ``with _connect(...) as conn:``
|
||||
closes the connection on exit. ``sqlite3.Connection.__exit__`` only commits /
|
||||
rolls back the transaction — it does **not** call ``close()``, which leaks
|
||||
file descriptors across many cache operations (especially under
|
||||
``--queries-file`` + ``--cache-ttl`` batch runs).
|
||||
"""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
conn = sqlite3.connect(str(path), timeout=10)
|
||||
# WAL allows concurrent readers alongside a single writer, which matters
|
||||
@@ -59,7 +67,7 @@ def _connect(path: Path) -> sqlite3.Connection:
|
||||
"""
|
||||
)
|
||||
conn.commit()
|
||||
return conn
|
||||
return contextlib.closing(conn)
|
||||
|
||||
|
||||
def _make_key(params: dict) -> str:
|
||||
|
||||
+49
-4
@@ -60,8 +60,12 @@ def setup_logging(verbose: bool = False, quiet: bool = False) -> None:
|
||||
MAX_RETRIES = 3
|
||||
RETRY_BACKOFF_BASE = 1.5 # seconds; exponential backoff + jitter
|
||||
|
||||
# HTTP status codes that are worth retrying (rate limit + gateway errors)
|
||||
RETRYABLE_STATUS = frozenset({429, 502, 503, 504})
|
||||
# HTTP status codes worth retrying. 403 is included so fetch_url's UA-fallback
|
||||
# loop can kick in when a site blocks the default searxng-cli User-Agent
|
||||
# (a fresh UA is tried on each retry attempt). search.py also retries 403 —
|
||||
# it doesn't switch UAs, so true auth failures waste ~3 attempts, accepted
|
||||
# as a trade-off for one shared retry policy across both scripts.
|
||||
RETRYABLE_STATUS = frozenset({403, 429, 502, 503, 504})
|
||||
|
||||
# Browser-like UA strings for fallback when the searxng-cli UA is blocked
|
||||
FALLBACK_UAS = [
|
||||
@@ -302,6 +306,29 @@ E_EMPTY = "E_EMPTY"
|
||||
E_INPUT = "E_INPUT"
|
||||
E_INTERNAL = "E_INTERNAL"
|
||||
|
||||
# 每个 E_* 配套的可操作恢复建议，让 AI Agent 能自决策下一步动作，
|
||||
# 而不是盲目重试或放弃。在 _emit_error 的 JSON 输出中作为 recovery_hint 字段。
|
||||
RECOVERY_HINTS = {
|
||||
E_CONFIG: "Provide -i/--instance, set SEARXNG_INSTANCE env var, or create "
|
||||
"searxng.toml/instances.txt config file.",
|
||||
E_AUTH: "Verify --auth-bearer/--auth-basic credentials or "
|
||||
"SEARXNG_BEARER_TOKEN/SEARXNG_BASIC_AUTH env vars. Check token "
|
||||
"expiry and instance access permissions.",
|
||||
E_NETWORK: "Retry with backoff, or try a different SearXNG instance. "
|
||||
"Check network connectivity, proxy settings, and instance uptime.",
|
||||
E_RATE_LIMIT: "Wait before retrying (exponential backoff). Reduce query "
|
||||
"frequency, narrow --time-range, or distribute load across "
|
||||
"multiple instances.",
|
||||
E_INPUT: "Check query syntax, --categories values, --time-range format, "
|
||||
"and flag combinations. Use --help for valid options.",
|
||||
E_PARSE: "The instance returned malformed data. Try a different instance, "
|
||||
"switch --method, or check if the instance version is compatible.",
|
||||
E_EMPTY: "Refine the query (more specific terms), broaden --time-range, "
|
||||
"add --categories, or increase --pageno to find more results.",
|
||||
E_INTERNAL: "This is likely a bug. Re-run with --verbose and report the "
|
||||
"full output for diagnosis.",
|
||||
}
|
||||
|
||||
|
||||
def classify_error(exc: BaseException) -> str:
|
||||
"""将异常分类为结构化错误码，供 AI Agent 程序化处理。
|
||||
@@ -350,18 +377,36 @@ def classify_error(exc: BaseException) -> str:
|
||||
if isinstance(exc, (ValueError, _json.JSONDecodeError)):
|
||||
return E_PARSE
|
||||
|
||||
# RuntimeError：从消息中推断（search_multi 的 "All N instances failed" 等）
|
||||
# RuntimeError：从消息中推断（search_multi 的 "All N instances failed.
|
||||
# Last error: HTTP Error 403: Forbidden" 等）。并行路径现在会把 last_error
|
||||
# 拼进消息，让这里能提取真实错误类型，而不是一律误判 E_NETWORK。
|
||||
msg = str(exc).lower()
|
||||
if isinstance(exc, RuntimeError):
|
||||
# 先检查 auth/rate-limit 关键字（最常见，来自 last_error 详情）
|
||||
if "auth" in msg or "403" in msg or "401" in msg:
|
||||
return E_AUTH
|
||||
if "rate" in msg or "429" in msg:
|
||||
return E_RATE_LIMIT
|
||||
# 从 "http error NNN" / "http NNN" 模式中提取状态码，
|
||||
# 正确分类 "All instances failed. Last error: HTTP Error 403" 等
|
||||
import re as _re
|
||||
status_match = _re.search(r'http(?: error)? (\d{3})', msg)
|
||||
if status_match:
|
||||
status = int(status_match.group(1))
|
||||
if status == 429:
|
||||
return E_RATE_LIMIT
|
||||
if status in (401, 403):
|
||||
return E_AUTH
|
||||
if 400 <= status < 500:
|
||||
return E_INPUT
|
||||
if 500 <= status < 600:
|
||||
return E_NETWORK
|
||||
# 所有实例失败的通用模式（无更具体的 HTTP 状态码时才判为网络错误）
|
||||
if "all" in msg and "instance" in msg and "fail" in msg:
|
||||
return E_NETWORK
|
||||
if "parse" in msg or "json" in msg or "html" in msg:
|
||||
return E_PARSE
|
||||
if "not found" in msg or "no " in msg and "instance" in msg:
|
||||
if "not found" in msg or ("no " in msg and "instance" in msg):
|
||||
return E_CONFIG
|
||||
return E_INTERNAL
|
||||
|
||||
|
||||
+37
-30
@@ -513,35 +513,42 @@ def fetch_url(url: str, timeout: int = 15, user_agent: str = None,
|
||||
if _HAS_REQUESTS:
|
||||
resp = _requests.get(url, timeout=timeout, headers=headers,
|
||||
allow_redirects=allow_redirects, stream=True)
|
||||
resp.raise_for_status()
|
||||
# stream=True holds the socket open; must close explicitly,
|
||||
# including on raise_for_status() / max_size break / decode
|
||||
# errors — otherwise the connection leaks back to the pool
|
||||
# and long-running agents exhaust ports.
|
||||
try:
|
||||
resp.raise_for_status()
|
||||
|
||||
# Read: unlimited if max_size is None, chunked with limit otherwise
|
||||
if max_size is None:
|
||||
raw = resp.content
|
||||
truncated = False
|
||||
else:
|
||||
chunks = []
|
||||
total = 0
|
||||
for chunk in resp.iter_content(chunk_size=65536, decode_unicode=False):
|
||||
if chunk:
|
||||
chunks.append(chunk)
|
||||
total += len(chunk)
|
||||
if total > max_size:
|
||||
break
|
||||
raw = b"".join(chunks)
|
||||
truncated = total > max_size
|
||||
# Read: unlimited if max_size is None, chunked with limit otherwise
|
||||
if max_size is None:
|
||||
raw = resp.content
|
||||
truncated = False
|
||||
else:
|
||||
chunks = []
|
||||
total = 0
|
||||
for chunk in resp.iter_content(chunk_size=65536, decode_unicode=False):
|
||||
if chunk:
|
||||
chunks.append(chunk)
|
||||
total += len(chunk)
|
||||
if total > max_size:
|
||||
break
|
||||
raw = b"".join(chunks)
|
||||
truncated = total > max_size
|
||||
|
||||
if encoding:
|
||||
content = raw.decode(encoding)
|
||||
else:
|
||||
charset = detect_charset(raw, resp.headers.get("Content-Type", ""))
|
||||
try:
|
||||
content = raw.decode(charset)
|
||||
except (UnicodeDecodeError, LookupError):
|
||||
content = raw.decode("utf-8", errors="replace")
|
||||
if encoding:
|
||||
content = raw.decode(encoding)
|
||||
else:
|
||||
charset = detect_charset(raw, resp.headers.get("Content-Type", ""))
|
||||
try:
|
||||
content = raw.decode(charset)
|
||||
except (UnicodeDecodeError, LookupError):
|
||||
content = raw.decode("utf-8", errors="replace")
|
||||
|
||||
return FetchResult(content, resp.headers.get("Content-Type", ""),
|
||||
resp.url, truncated, ua)
|
||||
return FetchResult(content, resp.headers.get("Content-Type", ""),
|
||||
resp.url, truncated, ua)
|
||||
finally:
|
||||
resp.close()
|
||||
|
||||
# stdlib fallback
|
||||
req = urllib.request.Request(url, headers=headers)
|
||||
@@ -588,7 +595,7 @@ def fetch_url(url: str, timeout: int = 15, user_agent: str = None,
|
||||
logger.info(f" Fetch retry {attempt+1}/{max_retries} ({e}) in {delay:.1f}s")
|
||||
time.sleep(delay)
|
||||
continue
|
||||
raise RuntimeError(f"HTTP {e.code} for {url}")
|
||||
raise RuntimeError(f"HTTP {e.code} for {url}") from e
|
||||
except (urllib.error.URLError, OSError, TimeoutError) as e:
|
||||
last_error = e
|
||||
if attempt < max_retries:
|
||||
@@ -596,7 +603,7 @@ def fetch_url(url: str, timeout: int = 15, user_agent: str = None,
|
||||
logger.info(f" Fetch retry {attempt+1}/{max_retries} ({e}) in {delay:.1f}s")
|
||||
time.sleep(delay)
|
||||
continue
|
||||
raise RuntimeError(f"Request failed for {url}: {e}")
|
||||
raise RuntimeError(f"Request failed for {url}: {e}") from e
|
||||
except Exception as e:
|
||||
# requests backend: retry only on connection errors (no response)
|
||||
# or transient 429/5xx; do NOT retry permanent errors like 404.
|
||||
@@ -608,9 +615,9 @@ def fetch_url(url: str, timeout: int = 15, user_agent: str = None,
|
||||
logger.info(f" Fetch retry {attempt+1}/{max_retries} ({e}) in {delay:.1f}s")
|
||||
time.sleep(delay)
|
||||
continue
|
||||
raise RuntimeError(f"Request failed for {url}: {e}")
|
||||
raise RuntimeError(f"Request failed for {url}: {e}") from e
|
||||
|
||||
raise RuntimeError(f"All {max_retries+1} attempts failed for {url}: {last_error}")
|
||||
raise RuntimeError(f"All {max_retries+1} attempts failed for {url}: {last_error}") from last_error
|
||||
|
||||
|
||||
# ----- Main -----
|
||||
|
||||
+242
-45
@@ -24,11 +24,12 @@ from pathlib import Path
|
||||
|
||||
# Allow running standalone from any working directory
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from _config import USER_AGENT, VERSION
|
||||
from _config import SCHEMA_VERSION, USER_AGENT, VERSION
|
||||
from common import (
|
||||
RETRYABLE_STATUS,
|
||||
RETRY_BACKOFF_BASE,
|
||||
MAX_RETRIES,
|
||||
RECOVERY_HINTS,
|
||||
apply_proxy,
|
||||
build_auth_headers,
|
||||
classify_error,
|
||||
@@ -143,7 +144,7 @@ class SearXNGHTMLParser(HTMLParser):
|
||||
self._in_time = True
|
||||
self._text_buf = []
|
||||
elif tag in ("script", "style"):
|
||||
self._skip_depth = 1
|
||||
self._skip_depth += 1
|
||||
|
||||
# Suggestions: <div id="suggestions"> or class containing "suggestion"
|
||||
if tag_id == "suggestions" or "suggestion" in classes:
|
||||
@@ -312,6 +313,17 @@ def _read_instance_file(path: Path) -> list:
|
||||
continue
|
||||
out.extend(parse_instances(line))
|
||||
return out
|
||||
except RuntimeError as e:
|
||||
# tomllib 缺失（Python 3.8-3.10 未装 tomli）是可恢复的——可改用
|
||||
# instances.txt——但必须明确提示用户，而不是静默返回空列表让 main
|
||||
# 报 "no instance resolved"，让用户困惑真正的失败原因。
|
||||
msg = str(e).lower()
|
||||
if "toml" in msg and ("3.11" in msg or "tomli" in msg):
|
||||
logger.error(f"Cannot parse '{path}': {e} "
|
||||
f"(consider 'pip install tomli' or use instances.txt)")
|
||||
else:
|
||||
logger.warning(f"Warning: cannot read instance file '{path}': {e}")
|
||||
return []
|
||||
except Exception as e:
|
||||
logger.warning(f"Warning: cannot read instance file '{path}': {e}")
|
||||
return []
|
||||
@@ -423,7 +435,7 @@ def _retry_with_backoff(fn, max_retries: int = MAX_RETRIES, base_delay: float =
|
||||
try:
|
||||
return fn()
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code in RETRYABLE_STATUS: # 429 rate-limit + 5xx gateway errors
|
||||
if e.code in RETRYABLE_STATUS: # 403 (UA block) + 429 + 5xx
|
||||
last_error = e
|
||||
if attempt < max_retries:
|
||||
delay = base_delay * (2 ** attempt) + random.uniform(0, 1)
|
||||
@@ -524,14 +536,22 @@ def search_multi(instance_urls: list, params: dict, method: str = "GET",
|
||||
last_error = None
|
||||
for instance in instance_urls:
|
||||
logger.info(f"Trying {instance}...")
|
||||
emit_progress("instance_try", url=instance, attempt=1)
|
||||
start = time.time()
|
||||
try:
|
||||
def _do():
|
||||
return search_single(instance, params, method=method,
|
||||
timeout=timeout, auth_headers=auth_headers)
|
||||
return _retry_with_backoff(_do, max_retries=retry_per)
|
||||
result = _retry_with_backoff(_do, max_retries=retry_per)
|
||||
emit_progress("instance_ok", url=instance,
|
||||
latency=round(time.time() - start, 3),
|
||||
results=len(result.get("results", [])) if result else 0)
|
||||
return result
|
||||
except Exception as e:
|
||||
last_error = e
|
||||
logger.info(f" Failed: {e}")
|
||||
emit_progress("instance_fail", url=instance,
|
||||
error=str(e), error_code=classify_error(e))
|
||||
continue
|
||||
raise RuntimeError(f"All {len(instance_urls)} instances failed. Last error: {last_error}")
|
||||
|
||||
@@ -540,15 +560,24 @@ def search_multi(instance_urls: list, params: dict, method: str = "GET",
|
||||
logger.info(f"Trying {len(instance_urls)} instances in parallel...")
|
||||
|
||||
def _task(instance: str):
|
||||
emit_progress("instance_try", url=instance, attempt=1)
|
||||
start = time.time()
|
||||
def _do():
|
||||
return search_single(instance, params, method=method,
|
||||
timeout=timeout, auth_headers=auth_headers)
|
||||
try:
|
||||
return instance, _retry_with_backoff(_do, max_retries=retry_per)
|
||||
result = _retry_with_backoff(_do, max_retries=retry_per)
|
||||
emit_progress("instance_ok", url=instance,
|
||||
latency=round(time.time() - start, 3),
|
||||
results=len(result.get("results", [])) if result else 0)
|
||||
return instance, result
|
||||
except Exception as e:
|
||||
emit_progress("instance_fail", url=instance,
|
||||
error=str(e), error_code=classify_error(e))
|
||||
return instance, e
|
||||
|
||||
results_by_url = {}
|
||||
last_parallel_error = None
|
||||
with ThreadPoolExecutor(max_workers=min(len(instance_urls), 8)) as ex:
|
||||
futures = {ex.submit(_task, u): u for u in instance_urls}
|
||||
for fut in as_completed(futures):
|
||||
@@ -558,6 +587,10 @@ def search_multi(instance_urls: list, params: dict, method: str = "GET",
|
||||
except Exception:
|
||||
continue
|
||||
if isinstance(res, Exception):
|
||||
# 保留最后一个失败详情，让 classify_error 能从消息中提取
|
||||
# 真实错误类型（401/403→E_AUTH，429→E_RATE_LIMIT 等），
|
||||
# 而不是一律误判为 E_NETWORK。
|
||||
last_parallel_error = res
|
||||
logger.info(f" Failed {u}: {res}")
|
||||
else:
|
||||
results_by_url[u] = res
|
||||
@@ -567,7 +600,8 @@ def search_multi(instance_urls: list, params: dict, method: str = "GET",
|
||||
if u in results_by_url:
|
||||
return results_by_url[u]
|
||||
|
||||
raise RuntimeError(f"All {len(instance_urls)} instances failed (parallel).")
|
||||
raise RuntimeError(f"All {len(instance_urls)} instances failed (parallel). "
|
||||
f"Last error: {last_parallel_error}")
|
||||
|
||||
|
||||
# ----- Output formatting -----
|
||||
@@ -675,7 +709,10 @@ def verify_instances(instance_urls: list, timeout: int = 15,
|
||||
# users know whether their token is wrong or the instance is down.
|
||||
auth_status = ("rejected" if has_auth and e.code in (401, 403)
|
||||
else ("ok" if has_auth else "n/a"))
|
||||
return {"url": u, "reachable": e.code not in (401, 403, 404),
|
||||
# 5xx 是服务器错误，实例虽然响应了但不可用，应视为不可达。
|
||||
# 400 可能只是请求格式问题，实例本身在线，仍算可达。
|
||||
is_5xx = 500 <= e.code < 600
|
||||
return {"url": u, "reachable": e.code not in (401, 403, 404) and not is_5xx,
|
||||
"json_supported": False, "post_supported": None,
|
||||
"config_endpoint": None, "engines": [],
|
||||
"latency": round(time.time() - start, 3),
|
||||
@@ -900,7 +937,9 @@ def deduplicate_results(results: dict) -> dict:
|
||||
if not results.get("results"):
|
||||
return results
|
||||
|
||||
TRACKING_PREFIXES = ("utm_", "gclid", "fbclid", "mc_", "ref", "ref_")
|
||||
# 跟踪参数前缀。裸 "ref" 过于宽泛（会误删 reference/refcode 等正常参数），
|
||||
# 收紧为 "ref_" 只匹配 ref_source/ref_campaign 等跟踪参数。
|
||||
TRACKING_PREFIXES = ("utm_", "gclid", "fbclid", "mc_", "ref_")
|
||||
|
||||
def _normalize_url(url: str) -> str:
|
||||
try:
|
||||
@@ -1052,6 +1091,7 @@ def _format_results(results: dict, args) -> str:
|
||||
the exact same formatting for each per-query block.
|
||||
"""
|
||||
if args.format == "json":
|
||||
results["schema_version"] = SCHEMA_VERSION
|
||||
return json.dumps(results, indent=2, ensure_ascii=False)
|
||||
if args.format == "urls":
|
||||
return format_urls(results)
|
||||
@@ -1188,9 +1228,89 @@ def _run_single_query(query: str, args, instance_urls: list,
|
||||
|
||||
result_count = len(results.get("results", []))
|
||||
emit_progress("done", results=result_count, query=query)
|
||||
# 清理内部 _fallback 字段，避免泄漏到 JSON 输出。
|
||||
# fetched_source（对 AI 有用的公开字段）已在 --fetch 路径中设置。
|
||||
results.pop("_fallback", None)
|
||||
return results, None, None
|
||||
|
||||
|
||||
def _get_output_schema():
|
||||
"""Return the JSON Schema describing --format json output.
|
||||
|
||||
Used by ``--dump-schema`` so AI agents can programmatically discover the
|
||||
output structure without parsing prose documentation.
|
||||
"""
|
||||
return {
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "SearXNG CLI Search Result",
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"description": "Output schema for 'python search.py --format json' (single query). "
|
||||
"Batch mode (--queries-file) wraps results in "
|
||||
'{"schema_version, queries:[]}.',
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"schema_version": {
|
||||
"type": "string",
|
||||
"const": SCHEMA_VERSION,
|
||||
"description": "Output schema version. Bump on breaking field changes.",
|
||||
},
|
||||
"query": {"type": "string", "description": "The search query string."},
|
||||
"number_of_results": {
|
||||
"type": "integer",
|
||||
"description": "Total matches reported by SearXNG (JSON path) or "
|
||||
"count of parsed results (HTML fallback path).",
|
||||
},
|
||||
"results": {
|
||||
"type": "array",
|
||||
"description": "Search result items, ordered by relevance (score desc).",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"title": {"type": "string"},
|
||||
"url": {"type": "string", "format": "uri"},
|
||||
"engine": {"type": "string", "description": "Source engine name."},
|
||||
"score": {"type": ["number", "null"]},
|
||||
"published_date": {"type": ["string", "null"]},
|
||||
"content": {"type": "string", "description": "Snippet/summary text."},
|
||||
},
|
||||
"required": ["title", "url"],
|
||||
},
|
||||
},
|
||||
"unresponsive_engines": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "Engines that failed to respond.",
|
||||
},
|
||||
"suggestions": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "Related query suggestions from the instance.",
|
||||
},
|
||||
"fetched": {
|
||||
"type": "array",
|
||||
"description": "Present only when --fetch N is used. Page content "
|
||||
"for the top N results.",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"url": {"type": "string"},
|
||||
"text": {"type": "string"},
|
||||
"text_length": {"type": "integer"},
|
||||
"error": {"type": "string"},
|
||||
},
|
||||
},
|
||||
},
|
||||
"fetched_source": {
|
||||
"type": "string",
|
||||
"enum": ["json", "html"],
|
||||
"description": "Present only when --fetch is used. Indicates whether "
|
||||
"search results came from JSON API or HTML fallback.",
|
||||
},
|
||||
},
|
||||
"required": ["query", "results"],
|
||||
}
|
||||
|
||||
|
||||
def _emit_error(message: str, args, query: str = None, exit_code: int = 1,
|
||||
error_code: str = None):
|
||||
"""Emit an error and exit.
|
||||
@@ -1201,22 +1321,30 @@ def _emit_error(message: str, args, query: str = None, exit_code: int = 1,
|
||||
|
||||
The JSON shape is::
|
||||
|
||||
{"error": "...", "exit_code": N, "error_code": "E_*", "query": "..."}
|
||||
{"error": "...", "exit_code": N, "error_code": "E_*",
|
||||
"recovery_hint": "...", "query": "..."}
|
||||
|
||||
``error_code`` 是结构化错误码（E_CONFIG/E_AUTH/E_NETWORK 等），让 AI
|
||||
Agent 程序化判断错误类型并采取恢复策略。``query`` 仅在提供时包含。
|
||||
Agent 程序化判断错误类型并采取恢复策略。``recovery_hint`` 给出可操作的
|
||||
恢复建议，让 AI 能自决策下一步动作。``query`` 仅在提供时包含。
|
||||
"""
|
||||
if getattr(args, "format", None) == "json":
|
||||
payload = {"error": message, "exit_code": exit_code}
|
||||
if error_code:
|
||||
payload["error_code"] = error_code
|
||||
hint = RECOVERY_HINTS.get(error_code)
|
||||
if hint:
|
||||
payload["recovery_hint"] = hint
|
||||
if query:
|
||||
payload["query"] = query
|
||||
print(json.dumps(payload, indent=2, ensure_ascii=False))
|
||||
else:
|
||||
prefix = f"[query: {query}] " if query else ""
|
||||
code_prefix = f"[{error_code}] " if error_code else ""
|
||||
logger.error(f"{prefix}{code_prefix}Error: {message}")
|
||||
hint_suffix = ""
|
||||
if error_code and error_code in RECOVERY_HINTS:
|
||||
hint_suffix = f"\n Hint: {RECOVERY_HINTS[error_code]}"
|
||||
logger.error(f"{prefix}{code_prefix}Error: {message}{hint_suffix}")
|
||||
sys.exit(exit_code)
|
||||
|
||||
|
||||
@@ -1406,13 +1534,35 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
|
||||
help="Delete all cached entries and exit (no search performed)")
|
||||
parser.add_argument("--cache-stats", action="store_true",
|
||||
help="Print cache statistics (entry count, age, size, path) and exit")
|
||||
parser.add_argument("--dump-schema", action="store_true",
|
||||
help="Print the JSON Schema for --format json output and exit. "
|
||||
"Lets AI agents programmatically discover field names and types.")
|
||||
parser.add_argument("--version", action="version", version=f"searxng-cli v{VERSION}")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# --dump-schema：输出 JSON Schema 到 stdout 并退出，AI Agent 可程序化发现字段
|
||||
if getattr(args, "dump_schema", False):
|
||||
print(json.dumps(_get_output_schema(), indent=2, ensure_ascii=False))
|
||||
sys.exit(0)
|
||||
|
||||
# 启用 --progress 进度事件（JSON Lines 到 stderr）
|
||||
set_progress_enabled(getattr(args, "progress", False))
|
||||
|
||||
# --stream 只在单查询 + --format json 下有效。batch 模式输出 JSON 数组，
|
||||
# 非 json 格式无 JSON Lines 语义；两种组合都显式报错 E_INPUT，避免静默失效
|
||||
# 让 AI Agent 误以为流式输出已生效。
|
||||
if getattr(args, "stream", False):
|
||||
if args.queries_file:
|
||||
_emit_error("--stream cannot be used with --queries-file: batch mode "
|
||||
"emits a JSON array, not JSON Lines. Drop --stream for "
|
||||
"batch output, or use a single --query with --stream.",
|
||||
args, error_code=E_INPUT)
|
||||
if args.format != "json":
|
||||
_emit_error(f"--stream requires --format json (current: {args.format}). "
|
||||
"JSON Lines streaming only produces valid output with json format.",
|
||||
args, error_code=E_INPUT)
|
||||
|
||||
# --query is required unless we're doing a non-search operation.
|
||||
# --queries-file is an alternative to --query for batch mode.
|
||||
if (not args.verify and not args.query and not args.queries_file
|
||||
@@ -1427,6 +1577,33 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
|
||||
apply_proxy(args.proxy)
|
||||
logger.info(f"Proxy: {args.proxy}")
|
||||
|
||||
# --clear-cache / --cache-stats 不需要实例，在实例解析之前处理并退出。
|
||||
# 避免无 -i 时报 E_CONFIG "no instance resolved" 让 AI 困惑。
|
||||
if args.clear_cache:
|
||||
removed = cache_module.clear()
|
||||
logger.info(f"Cleared {removed} cache entr{'y' if removed == 1 else 'ies'}.")
|
||||
sys.exit(0)
|
||||
|
||||
if args.cache_stats:
|
||||
s = cache_module.stats()
|
||||
if args.format == "json":
|
||||
# JSON 模式：结构化数据走 stdout，AI Agent 可管道解析
|
||||
print(json.dumps(s, indent=2, ensure_ascii=False))
|
||||
else:
|
||||
# 非 JSON 模式：人类可读的状态信息走 stderr，保持 stdout 纯净，
|
||||
# 避免 AI Agent 用 --format json 解析 stdout 时被非 JSON 污染。
|
||||
print(f"Cache path: {s.get('path', '?')}", file=sys.stderr)
|
||||
print(f"Entries: {s.get('entries', 0)}", file=sys.stderr)
|
||||
size = s.get("size_bytes", 0)
|
||||
print(f"Size: {size:,} bytes ({size / 1024:.1f} KB)", file=sys.stderr)
|
||||
if s.get("oldest_created_at"):
|
||||
print(f"Oldest: {time.ctime(s['oldest_created_at'])}", file=sys.stderr)
|
||||
if s.get("newest_created_at"):
|
||||
print(f"Newest: {time.ctime(s['newest_created_at'])}", file=sys.stderr)
|
||||
if s.get("error"):
|
||||
print(f"Error: {s['error']}", file=sys.stderr)
|
||||
sys.exit(0)
|
||||
|
||||
# Resolve instance(s): -i > SEARXNG_INSTANCE env > config file
|
||||
instance_urls = resolve_instances(args.instance)
|
||||
# Fallback: if --config was used, instance may be in the config dict
|
||||
@@ -1441,9 +1618,26 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
|
||||
instance_urls = [u if u.startswith(("http://", "https://"))
|
||||
else "https://" + u for u in raw if u and str(u).strip()]
|
||||
if not instance_urls:
|
||||
# Python 3.8-3.10 无 tomllib 时，.toml 配置文件无法读取。检查这种
|
||||
# 情况并在错误信息中附加提示，让 AI Agent 能给出可操作的恢复建议。
|
||||
toml_hint = ""
|
||||
try:
|
||||
import tomllib # Python 3.11+
|
||||
except ModuleNotFoundError:
|
||||
try:
|
||||
import tomli # type: ignore[import-not-found]
|
||||
except ModuleNotFoundError:
|
||||
toml_candidates = [
|
||||
Path.cwd() / "searxng.toml",
|
||||
Path.home() / ".config" / "searxng-cli" / "searxng.toml",
|
||||
]
|
||||
if any(p.exists() for p in toml_candidates):
|
||||
toml_hint = (" (hint: a searxng.toml file exists but cannot be "
|
||||
"read on Python < 3.11 without the 'tomli' package. "
|
||||
"Run 'pip install tomli' or use instances.txt instead.)")
|
||||
_emit_error("no SearXNG instance resolved. Provide -i/--instance, set the "
|
||||
"SEARXNG_INSTANCE environment variable, or create a searxng.toml / "
|
||||
"instances.txt config file.", args, error_code=E_CONFIG)
|
||||
"instances.txt config file." + toml_hint, args, error_code=E_CONFIG)
|
||||
|
||||
# Build auth headers if provided (needed by both verify and search).
|
||||
# Credentials may come from CLI flag, file, config file, or env var
|
||||
@@ -1473,29 +1667,6 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
|
||||
_print_verify_report(report, as_json=(args.format == "json"))
|
||||
sys.exit(0)
|
||||
|
||||
# Cache management modes (no search performed)
|
||||
if args.clear_cache:
|
||||
removed = cache_module.clear()
|
||||
logger.info(f"Cleared {removed} cache entr{'y' if removed == 1 else 'ies'}.")
|
||||
sys.exit(0)
|
||||
|
||||
if args.cache_stats:
|
||||
s = cache_module.stats()
|
||||
if args.format == "json":
|
||||
print(json.dumps(s, indent=2, ensure_ascii=False))
|
||||
else:
|
||||
print(f"Cache path: {s.get('path', '?')}")
|
||||
print(f"Entries: {s.get('entries', 0)}")
|
||||
size = s.get("size_bytes", 0)
|
||||
print(f"Size: {size:,} bytes ({size / 1024:.1f} KB)")
|
||||
if s.get("oldest_created_at"):
|
||||
print(f"Oldest: {time.ctime(s['oldest_created_at'])}")
|
||||
if s.get("newest_created_at"):
|
||||
print(f"Newest: {time.ctime(s['newest_created_at'])}")
|
||||
if s.get("error"):
|
||||
print(f"Error: {s['error']}")
|
||||
sys.exit(0)
|
||||
|
||||
if args.fail_fast:
|
||||
instance_urls = instance_urls[:1]
|
||||
|
||||
@@ -1519,23 +1690,27 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
|
||||
|
||||
logger.info(f"Running {len(queries)} queries from {args.queries_file}...")
|
||||
batch = []
|
||||
any_ok = False
|
||||
any_with_results = False
|
||||
error_count = 0
|
||||
for i, q in enumerate(queries, 1):
|
||||
logger.info(f"\n[{i}/{len(queries)}] {q}")
|
||||
results, err, err_code = _run_single_query(q, args, instance_urls,
|
||||
auth_headers, ttl_seconds)
|
||||
if err:
|
||||
error_count += 1
|
||||
logger.error(f" [ERROR] {err}")
|
||||
entry = {"query": q, "error": err}
|
||||
entry = {"query": q, "status": "error", "error": err}
|
||||
if err_code:
|
||||
entry["error_code"] = err_code
|
||||
batch.append(entry)
|
||||
else:
|
||||
any_ok = True
|
||||
batch.append({"query": q, "results": results})
|
||||
if len(results.get("results", [])) > 0:
|
||||
any_with_results = True
|
||||
batch.append({"query": q, "status": "ok", "results": results})
|
||||
|
||||
if args.format == "json":
|
||||
output = json.dumps(batch, indent=2, ensure_ascii=False)
|
||||
output = json.dumps({"schema_version": SCHEMA_VERSION, "queries": batch},
|
||||
indent=2, ensure_ascii=False)
|
||||
elif args.format == "csv":
|
||||
import csv as csv_mod
|
||||
import io
|
||||
@@ -1588,27 +1763,49 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
|
||||
logger.info(f"Saved results to {args.output}")
|
||||
else:
|
||||
print(output)
|
||||
# Exit 0 if at least one query succeeded; 1 only if all failed.
|
||||
sys.exit(0 if any_ok else 1)
|
||||
# Exit code semantics — aligned with single-query mode so AI agents
|
||||
# can use one consistent rule:
|
||||
# 1 = all queries errored (fatal)
|
||||
# 2 = no query returned any results (empty), though at least one
|
||||
# searched successfully without error
|
||||
# 0 = at least one query returned results
|
||||
if error_count == len(queries):
|
||||
sys.exit(1)
|
||||
if not any_with_results:
|
||||
sys.exit(2)
|
||||
sys.exit(0)
|
||||
|
||||
# ----- Single query mode -----
|
||||
results, err, err_code = _run_single_query(args.query, args, instance_urls,
|
||||
auth_headers, ttl_seconds)
|
||||
if err:
|
||||
_emit_error(err, args, query=args.query, error_code=err_code)
|
||||
|
||||
# --stream: JSON Lines 流式输出，每条结果一行，AI 可增量处理
|
||||
# --stream: JSON Lines 流式输出，每条结果一行，AI 可增量处理。
|
||||
# stream 模式下所有输出（包括错误）都是单行 JSON，保持 JSON Lines 格式一致性。
|
||||
# 非 stream 模式的错误走 _emit_error（多行 JSON 或 stderr 文本）。
|
||||
if getattr(args, "stream", False) and args.format == "json":
|
||||
if err:
|
||||
error_event = {"type": "error", "error": err, "query": args.query}
|
||||
if err_code:
|
||||
error_event["error_code"] = err_code
|
||||
hint = RECOVERY_HINTS.get(err_code)
|
||||
if hint:
|
||||
error_event["recovery_hint"] = hint
|
||||
print(json.dumps(error_event, ensure_ascii=False), flush=True)
|
||||
sys.exit(1)
|
||||
for r in results.get("results", []):
|
||||
print(json.dumps({"type": "result", "result": r},
|
||||
ensure_ascii=False), flush=True)
|
||||
print(json.dumps({"type": "done",
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"count": len(results.get("results", [])),
|
||||
"query": args.query}, ensure_ascii=False), flush=True)
|
||||
if not results.get("results"):
|
||||
sys.exit(2)
|
||||
sys.exit(0)
|
||||
|
||||
if err:
|
||||
_emit_error(err, args, query=args.query, error_code=err_code)
|
||||
|
||||
output = _format_results(results, args)
|
||||
if args.output:
|
||||
with open(args.output, "w", encoding="utf-8") as f:
|
||||
|
||||
Reference in New Issue
Block a user