MetonaTeam/searxng-use-cli
2
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(v1.7.0): AI 友好度增强 + 测试补全 (155→309)

Browse Source 
核心新增（面向 AI Agent 程序化使用）：
- 结构化错误码体系：E_CONFIG/E_AUTH/E_NETWORK/E_RATE_LIMIT/E_PARSE/E_EMPTY/E_INPUT/E_INTERNAL
  classify_error() 自动分类异常，JSON 错误输出含 error_code 字段
- JSON Lines 流式输出 (--stream)：每条结果独立一行，AI 可增量处理
- 进度事件 (--progress)：JSON Lines 事件流到 stderr（start/cache_hit/fetch_ok/done 等）

测试补全（+154 例，覆盖全部高风险盲区）：
- HTML 回退搜索路径 (19)
- --fetch 自动抓取 (21)
- --verify 健康检查 (15)
- 输出格式化 (15)
- 实例解析链 (20)
- 并行多实例搜索 (10)
- CLI 入口与端到端 (17)
- 错误码分类 (27)
- 流式输出与进度事件 (10)

源码改进：
- search.py: h3 内 a 标签 href 作为 url fallback，提升 SearXNG 主题兼容性
- common.py: 新增 classify_error/emit_progress/set_progress_enabled

文档同步：SKILL.md 新增 AI Agent Integration Guide 章节，README.md 更新参数与错误码表
This commit is contained in:
thzxx
2026-08-01 17:40:14 +08:00
parent 9796988da9
commit f983a9377e
14 changed files with 2516 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/search.py
+93 -20
View File
@@ -31,9 +31,20 @@ from common import (
MAX_RETRIES,
apply_proxy,
build_auth_headers,
classify_error,
emit_progress,
resolve_auth_basic,
resolve_auth_bearer,
set_progress_enabled,
setup_logging,
E_CONFIG,
E_AUTH,
E_NETWORK,
E_RATE_LIMIT,
E_PARSE,
E_EMPTY,
E_INPUT,
E_INTERNAL,
)
from fetch import extract_text, fetch_url
import cache as cache_module
@@ -112,6 +123,13 @@ class SearXNGHTMLParser(HTMLParser):
elif tag == "h3":
self._in_h3 = True
self._text_buf = []
elif tag == "a" and self._in_h3:
# h3 内的 <a href> 作为 url 的 fallback：某些 SearXNG 主题
# 不使用 url_header class，URL 仅出现在 h3 的链接中。
# url_header 优先（上面已处理），此处仅填充空 url。
href = attrs_dict.get("href", "")
if href and not self._current.get("url"):
self._current["url"] = href
elif tag == "p" and "content" in classes:
self._in_content = True
self._text_buf = []
@@ -1095,16 +1113,19 @@ def _run_single_query(query: str, args, instance_urls: list,
auth_headers: dict, ttl_seconds: int):
"""Run one query end-to-end: search → limit → domain-filter → fetch.
Returns ``(results_dict, error_str)``. On success ``error_str`` is None.
Cache hits skip the network entirely. Post-processing (limit / filter /
fetch) always runs so batch callers see the same shape as single-query
callers.
Returns ``(results_dict, error_str, error_code)``。成功时后两者为 None。
``error_code`` 是结构化错误码（E_NETWORK/E_AUTH 等），让 AI Agent
程序化判断错误类型。Cache hits 跳过网络。后处理（limit/filter/fetch）
总是执行，保证 batch 调用方看到与单查询一致的形状。
"""
params = _build_params(query, args)
emit_progress("start", query=query, instances=len(instance_urls))
cached = cache_module.get(params, ttl_seconds) if ttl_seconds > 0 else None
if cached is not None:
logger.info(f"[cache hit] q={query!r} TTL={args.cache_ttl}min, skipping network")
emit_progress("cache_hit", query=query, ttl=args.cache_ttl)
results = cached
else:
try:
@@ -1117,10 +1138,13 @@ def _run_single_query(query: str, args, instance_urls: list,
parallel=not args.serial,
)
except Exception as e:
return None, str(e)
err_code = classify_error(e)
emit_progress("error", error=str(e), error_code=err_code, query=query)
return None, str(e), err_code
if ttl_seconds > 0:
cache_module.put(params, results, ttl_seconds)
logger.info(f"[cache stored] q={query!r} TTL={args.cache_ttl}min")
emit_progress("cache_store", query=query, ttl=args.cache_ttl)
# Dedup (default on; --no-dedup disables) then sort, both BEFORE limit
# so --max-results keeps the highest-scoring / newest items.
@@ -1143,6 +1167,7 @@ def _run_single_query(query: str, args, instance_urls: list,
logger.info(f"Domain filter: {before} -> {after} results")
if args.fetch > 0 and results.get("results"):
emit_progress("fetch_start", count=args.fetch)
fetched = fetch_top_results(
results, args.fetch,
timeout=args.fetch_timeout,
@@ -1150,30 +1175,48 @@ def _run_single_query(query: str, args, instance_urls: list,
max_retries=args.fetch_retries,
max_size=args.max_size,
)
# Emit fetch_ok / fetch_fail events
for f in fetched:
if f.get("status") == "ok":
emit_progress("fetch_ok", url=f.get("url", ""),
chars=f.get("text_length", 0))
else:
emit_progress("fetch_fail", url=f.get("url", ""),
error=f.get("error", "unknown"))
results["fetched"] = fetched
results["fetched_source"] = results.get("_fallback", "json")
return results, None
result_count = len(results.get("results", []))
emit_progress("done", results=result_count, query=query)
return results, None, None
def _emit_error(message: str, args, query: str = None, exit_code: int = 1):
def _emit_error(message: str, args, query: str = None, exit_code: int = 1,
error_code: str = None):
"""Emit an error and exit.
In ``--format json`` mode the error is printed to **stdout** as a
structured JSON object so agents piping stdout can parse it. All other
formats print to stderr (keeping stdout clean for data) and exit.
The JSON shape is ``{"error": "...", "exit_code": N, "query": "..."}``
(query only included when provided).
The JSON shape is::
{"error": "...", "exit_code": N, "error_code": "E_*", "query": "..."}
``error_code`` 是结构化错误码（E_CONFIG/E_AUTH/E_NETWORK 等），让 AI
Agent 程序化判断错误类型并采取恢复策略。``query`` 仅在提供时包含。
"""
if getattr(args, "format", None) == "json":
payload = {"error": message, "exit_code": exit_code}
if error_code:
payload["error_code"] = error_code
if query:
payload["query"] = query
print(json.dumps(payload, indent=2, ensure_ascii=False))
else:
prefix = f"[query: {query}] " if query else ""
logger.error(f"{prefix}Error: {message}")
code_prefix = f"[{error_code}] " if error_code else ""
logger.error(f"{prefix}{code_prefix}Error: {message}")
sys.exit(exit_code)
@@ -1318,6 +1361,17 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
"Env var SEARXNG_BASIC_AUTH is also honored.")
parser.add_argument("--output", "-o", default=None,
help="Save to file instead of stdout")
parser.add_argument("--stream", action="store_true",
help="Stream results as JSON Lines (one JSON object per line) to stdout. "
"Each line is a {\"type\": \"result\", \"result\": {...}} event. "
"Ends with {\"type\": \"done\", \"count\": N}. "
"AI Agent can process results incrementally without waiting for full output. "
"Only valid with --format json.")
parser.add_argument("--progress", action="store_true",
help="Emit structured progress events as JSON Lines to stderr. "
"Events: start, instance_try, instance_ok, instance_fail, "
"cache_hit, cache_store, fetch_start, fetch_ok, fetch_fail, done. "
"AI Agent can track execution progress programmatically.")
parser.add_argument("--timeout", type=int, default=_cfg_int(config, "timeout", 15),
help="Request timeout in seconds (default: 15)")
parser.add_argument("--retry", type=int, default=_cfg_int(config, "max_retries", None),
@@ -1356,6 +1410,9 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
args = parser.parse_args()
# 启用 --progress 进度事件（JSON Lines 到 stderr）
set_progress_enabled(getattr(args, "progress", False))
# --query is required unless we're doing a non-search operation.
# --queries-file is an alternative to --query for batch mode.
if (not args.verify and not args.query and not args.queries_file
@@ -1375,7 +1432,7 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
if not instance_urls:
_emit_error("no SearXNG instance resolved. Provide -i/--instance, set the "
"SEARXNG_INSTANCE environment variable, or create a searxng.toml / "
"instances.txt config file.", args)
"instances.txt config file.", args, error_code=E_CONFIG)
# Build auth headers if provided (needed by both verify and search).
# Credentials may come from CLI flag, file, or env var (in priority order)
@@ -1384,7 +1441,7 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
bearer_token = resolve_auth_bearer(args.auth_bearer, args.auth_bearer_file)
basic_auth = resolve_auth_basic(args.auth_basic, args.auth_basic_file)
except RuntimeError as e:
_emit_error(str(e), args)
_emit_error(str(e), args, error_code=E_AUTH)
auth_headers = build_auth_headers(
bearer_token=bearer_token,
basic_auth=basic_auth,
@@ -1440,20 +1497,24 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
try:
queries = _read_queries_file(args.queries_file)
except RuntimeError as e:
_emit_error(str(e), args)
_emit_error(str(e), args, error_code=E_INPUT)
if not queries:
_emit_error(f"no queries found in '{args.queries_file}'", args)
_emit_error(f"no queries found in '{args.queries_file}'", args,
error_code=E_INPUT)
logger.info(f"Running {len(queries)} queries from {args.queries_file}...")
batch = []
any_ok = False
for i, q in enumerate(queries, 1):
logger.info(f"\n[{i}/{len(queries)}] {q}")
results, err = _run_single_query(q, args, instance_urls,
auth_headers, ttl_seconds)
results, err, err_code = _run_single_query(q, args, instance_urls,
auth_headers, ttl_seconds)
if err:
logger.error(f" [ERROR] {err}")
batch.append({"query": q, "error": err})
entry = {"query": q, "error": err}
if err_code:
entry["error_code"] = err_code
batch.append(entry)
else:
any_ok = True
batch.append({"query": q, "results": results})
@@ -1516,10 +1577,22 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
sys.exit(0 if any_ok else 1)
# ----- Single query mode -----
results, err = _run_single_query(args.query, args, instance_urls,
auth_headers, ttl_seconds)
results, err, err_code = _run_single_query(args.query, args, instance_urls,
auth_headers, ttl_seconds)
if err:
_emit_error(err, args, query=args.query)
_emit_error(err, args, query=args.query, error_code=err_code)
# --stream: JSON Lines 流式输出，每条结果一行，AI 可增量处理
if getattr(args, "stream", False) and args.format == "json":
for r in results.get("results", []):
print(json.dumps({"type": "result", "result": r},
ensure_ascii=False), flush=True)
print(json.dumps({"type": "done",
"count": len(results.get("results", [])),
"query": args.query}, ensure_ascii=False), flush=True)
if not results.get("results"):
sys.exit(2)
sys.exit(0)
output = _format_results(results, args)
if args.output: