feat(v1.7.0): AI 友好度增强 + 测试补全 (155→309)

核心新增（面向 AI Agent 程序化使用）： - 结构化错误码体系：E_CONFIG/E_AUTH/E_NETWORK/E_RATE_LIMIT/E_PARSE/E_EMPTY/E_INPUT/E_INTERNAL classify_error() 自动分类异常，JSON 错误输出含 error_code 字段 - JSON Lines 流式输出 (--stream)：每条结果独立一行，AI 可增量处理 - 进度事件 (--progress)：JSON Lines 事件流到 stderr（start/cache_hit/fetch_ok/done 等）测试补全（+154 例，覆盖全部高风险盲区）： - HTML 回退搜索路径 (19) - --fetch 自动抓取 (21) - --verify 健康检查 (15) - 输出格式化 (15) - 实例解析链 (20) - 并行多实例搜索 (10) - CLI 入口与端到端 (17) - 错误码分类 (27) - 流式输出与进度事件 (10) 源码改进： - search.py: h3 内 a 标签 href 作为 url fallback，提升 SearXNG 主题兼容性 - common.py: 新增 classify_error/emit_progress/set_progress_enabled 文档同步：SKILL.md 新增 AI Agent Integration Guide 章节，README.md 更新参数与错误码表
2026-08-01 17:40:14 +08:00
parent 9796988da9
commit f983a9377e
14 changed files with 2516 additions and 26 deletions
@@ -7,5 +7,5 @@ Retry settings and shared HTTP utilities now live in ``common.py`` so that
 both ``search.py`` and ``fetch.py`` share one consistent implementation.
 """

-VERSION = "1.6.0"
+VERSION = "1.7.0"
 USER_AGENT = f"searxng-cli/{VERSION}"
@@ -259,3 +259,136 @@ def is_retryable_error(exc: BaseException) -> bool:
        # requests connection/timeout error without a response -> transient
        return True
    return False
+
+
+# ----- Structured error classification -----
+#
+# 错误码体系：让 AI Agent 程序化地判断错误类型并采取恢复策略。
+# 所有错误码以 E_ 前缀，在 --format json 模式下随 error_code 字段输出。
+#
+# AI 可根据 error_code 决策：
+#   E_CONFIG     → 检查实例配置/环境变量，提示用户设置
+#   E_AUTH       → 检查 token/凭证，提示用户重新认证
+#   E_NETWORK    → 重试或切换实例/代理
+#   E_RATE_LIMIT → 等待后重试，降低请求频率
+#   E_PARSE      → 检查实例是否支持 JSON，尝试 HTML 回退
+#   E_EMPTY      → 调整查询词或时间范围
+#   E_INPUT      → 修正参数/文件路径
+#   E_INTERNAL   → 报告 bug，附带完整错误信息
+
+# 错误码常量（供 search.py / fetch.py 引用）
+E_CONFIG = "E_CONFIG"
+E_AUTH = "E_AUTH"
+E_NETWORK = "E_NETWORK"
+E_RATE_LIMIT = "E_RATE_LIMIT"
+E_PARSE = "E_PARSE"
+E_EMPTY = "E_EMPTY"
+E_INPUT = "E_INPUT"
+E_INTERNAL = "E_INTERNAL"
+
+
+def classify_error(exc: BaseException) -> str:
+    """将异常分类为结构化错误码，供 AI Agent 程序化处理。
+
+    分类逻辑（按优先级）：
+      1. 429 → E_RATE_LIMIT
+      2. 401/403 → E_AUTH
+      3. 4xx（非上述）→ E_INPUT（请求参数问题）
+      4. 5xx / URLError / OSError / TimeoutError → E_NETWORK
+      5. json.JSONDecodeError / ValueError → E_PARSE
+      6. FileNotFoundError → E_INPUT
+      7. RuntimeError → 尝试从消息中提取线索，否则 E_INTERNAL
+      8. 其他 → E_INTERNAL
+    """
+    import json as _json
+
+    # HTTP 错误（urllib 和 requests 都有 .code 或 .status_code）
+    status = None
+    if isinstance(exc, urllib.error.HTTPError):
+        status = exc.code
+    else:
+        resp = getattr(exc, "response", None)
+        status = getattr(resp, "status_code", None)
+
+    if status is not None:
+        if status == 429:
+            return E_RATE_LIMIT
+        if status in (401, 403):
+            return E_AUTH
+        if 400 <= status < 500:
+            return E_INPUT
+        if 500 <= status < 600:
+            return E_NETWORK
+
+    # 文件/输入错误（FileNotFoundError 是 OSError 子类，必须先于 OSError 检查）
+    if isinstance(exc, FileNotFoundError):
+        return E_INPUT
+
+    # 连接级错误
+    if isinstance(exc, (urllib.error.URLError, OSError, TimeoutError)):
+        return E_NETWORK
+    if isinstance(exc, ConnectionError):
+        return E_NETWORK
+
+    # 解析错误
+    if isinstance(exc, (ValueError, _json.JSONDecodeError)):
+        return E_PARSE
+
+    # RuntimeError：从消息中推断（search_multi 的 "All N instances failed" 等）
+    msg = str(exc).lower()
+    if isinstance(exc, RuntimeError):
+        if "auth" in msg or "403" in msg or "401" in msg:
+            return E_AUTH
+        if "rate" in msg or "429" in msg:
+            return E_RATE_LIMIT
+        if "all" in msg and "instance" in msg and "fail" in msg:
+            return E_NETWORK
+        if "parse" in msg or "json" in msg or "html" in msg:
+            return E_PARSE
+        if "not found" in msg or "no " in msg and "instance" in msg:
+            return E_CONFIG
+        return E_INTERNAL
+
+    return E_INTERNAL
+
+
+# ----- Progress event emitter (for --progress flag) -----
+#
+# 当 --progress 启用时，search.py 会调用 emit_progress() 发射结构化事件到
+# stderr（JSON Lines 格式）。AI Agent 可解析这些事件来跟踪执行进度。
+#
+# 事件类型：
+#   {"event": "start", "query": "...", "instances": N}
+#   {"event": "instance_try", "url": "...", "attempt": 1}
+#   {"event": "instance_ok", "url": "...", "latency": 0.5, "results": 10}
+#   {"event": "instance_fail", "url": "...", "error": "...", "error_code": "E_*"}
+#   {"event": "cache_hit", "query": "...", "ttl": 30}
+#   {"event": "cache_store", "query": "...", "ttl": 30}
+#   {"event": "fetch_start", "count": 3}
+#   {"event": "fetch_ok", "url": "...", "chars": 1234}
+#   {"event": "fetch_fail", "url": "...", "error": "..."}
+#   {"event": "done", "results": N, "query": "..."}
+#   {"event": "error", "error": "...", "error_code": "E_*", "query": "..."}
+
+_progress_enabled = False
+
+
+def set_progress_enabled(enabled: bool) -> None:
+    """全局开关：是否向 stderr 输出 JSON Lines 格式的进度事件。"""
+    global _progress_enabled
+    _progress_enabled = enabled
+
+
+def emit_progress(event: str, **kwargs) -> None:
+    """向 stderr 输出一行 JSON 格式的进度事件。
+
+    仅当 --progress 启用时才输出。``event`` 是事件类型字符串，
+    ``kwargs`` 是事件的附加字段。输出格式为 JSON Lines（每行一个 JSON 对象）。
+    """
+    if not _progress_enabled:
+        return
+    import json as _json
+    payload = {"event": event}
+    payload.update(kwargs)
+    print(_json.dumps(payload, ensure_ascii=False), file=sys.stderr, flush=True)
+
@@ -31,9 +31,20 @@ from common import (
    MAX_RETRIES,
    apply_proxy,
    build_auth_headers,
+    classify_error,
+    emit_progress,
    resolve_auth_basic,
    resolve_auth_bearer,
+    set_progress_enabled,
    setup_logging,
+    E_CONFIG,
+    E_AUTH,
+    E_NETWORK,
+    E_RATE_LIMIT,
+    E_PARSE,
+    E_EMPTY,
+    E_INPUT,
+    E_INTERNAL,
 )
 from fetch import extract_text, fetch_url
 import cache as cache_module
@@ -112,6 +123,13 @@ class SearXNGHTMLParser(HTMLParser):
            elif tag == "h3":
                self._in_h3 = True
                self._text_buf = []
+            elif tag == "a" and self._in_h3:
+                # h3 内的 <a href> 作为 url 的 fallback：某些 SearXNG 主题
+                # 不使用 url_header class，URL 仅出现在 h3 的链接中。
+                # url_header 优先（上面已处理），此处仅填充空 url。
+                href = attrs_dict.get("href", "")
+                if href and not self._current.get("url"):
+                    self._current["url"] = href
            elif tag == "p" and "content" in classes:
                self._in_content = True
                self._text_buf = []
@@ -1095,16 +1113,19 @@ def _run_single_query(query: str, args, instance_urls: list,
                      auth_headers: dict, ttl_seconds: int):
    """Run one query end-to-end: search → limit → domain-filter → fetch.

-    Returns ``(results_dict, error_str)``. On success ``error_str`` is None.
-    Cache hits skip the network entirely. Post-processing (limit / filter /
-    fetch) always runs so batch callers see the same shape as single-query
-    callers.
+    Returns ``(results_dict, error_str, error_code)``。成功时后两者为 None。
+    ``error_code`` 是结构化错误码（E_NETWORK/E_AUTH 等），让 AI Agent
+    程序化判断错误类型。Cache hits 跳过网络。后处理（limit/filter/fetch）
+    总是执行，保证 batch 调用方看到与单查询一致的形状。
    """
    params = _build_params(query, args)

+    emit_progress("start", query=query, instances=len(instance_urls))
+
    cached = cache_module.get(params, ttl_seconds) if ttl_seconds > 0 else None
    if cached is not None:
        logger.info(f"[cache hit] q={query!r} TTL={args.cache_ttl}min, skipping network")
+        emit_progress("cache_hit", query=query, ttl=args.cache_ttl)
        results = cached
    else:
        try:
@@ -1117,10 +1138,13 @@ def _run_single_query(query: str, args, instance_urls: list,
                parallel=not args.serial,
            )
        except Exception as e:
-            return None, str(e)
+            err_code = classify_error(e)
+            emit_progress("error", error=str(e), error_code=err_code, query=query)
+            return None, str(e), err_code
        if ttl_seconds > 0:
            cache_module.put(params, results, ttl_seconds)
            logger.info(f"[cache stored] q={query!r} TTL={args.cache_ttl}min")
+            emit_progress("cache_store", query=query, ttl=args.cache_ttl)

    # Dedup (default on; --no-dedup disables) then sort, both BEFORE limit
    # so --max-results keeps the highest-scoring / newest items.
@@ -1143,6 +1167,7 @@ def _run_single_query(query: str, args, instance_urls: list,
        logger.info(f"Domain filter: {before} -> {after} results")

    if args.fetch > 0 and results.get("results"):
+        emit_progress("fetch_start", count=args.fetch)
        fetched = fetch_top_results(
            results, args.fetch,
            timeout=args.fetch_timeout,
@@ -1150,30 +1175,48 @@ def _run_single_query(query: str, args, instance_urls: list,
            max_retries=args.fetch_retries,
            max_size=args.max_size,
        )
+        # Emit fetch_ok / fetch_fail events
+        for f in fetched:
+            if f.get("status") == "ok":
+                emit_progress("fetch_ok", url=f.get("url", ""),
+                              chars=f.get("text_length", 0))
+            else:
+                emit_progress("fetch_fail", url=f.get("url", ""),
+                              error=f.get("error", "unknown"))
        results["fetched"] = fetched
        results["fetched_source"] = results.get("_fallback", "json")

-    return results, None
+    result_count = len(results.get("results", []))
+    emit_progress("done", results=result_count, query=query)
+    return results, None, None


-def _emit_error(message: str, args, query: str = None, exit_code: int = 1):
+def _emit_error(message: str, args, query: str = None, exit_code: int = 1,
+                error_code: str = None):
    """Emit an error and exit.

    In ``--format json`` mode the error is printed to **stdout** as a
    structured JSON object so agents piping stdout can parse it. All other
    formats print to stderr (keeping stdout clean for data) and exit.

-    The JSON shape is ``{"error": "...", "exit_code": N, "query": "..."}``
-    (query only included when provided).
+    The JSON shape is::
+
+        {"error": "...", "exit_code": N, "error_code": "E_*", "query": "..."}
+
+    ``error_code`` 是结构化错误码（E_CONFIG/E_AUTH/E_NETWORK 等），让 AI
+    Agent 程序化判断错误类型并采取恢复策略。``query`` 仅在提供时包含。
    """
    if getattr(args, "format", None) == "json":
        payload = {"error": message, "exit_code": exit_code}
+        if error_code:
+            payload["error_code"] = error_code
        if query:
            payload["query"] = query
        print(json.dumps(payload, indent=2, ensure_ascii=False))
    else:
        prefix = f"[query: {query}] " if query else ""
-        logger.error(f"{prefix}Error: {message}")
+        code_prefix = f"[{error_code}] " if error_code else ""
+        logger.error(f"{prefix}{code_prefix}Error: {message}")
    sys.exit(exit_code)


@@ -1318,6 +1361,17 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
                             "Env var SEARXNG_BASIC_AUTH is also honored.")
    parser.add_argument("--output", "-o", default=None,
                        help="Save to file instead of stdout")
+    parser.add_argument("--stream", action="store_true",
+                        help="Stream results as JSON Lines (one JSON object per line) to stdout. "
+                             "Each line is a {\"type\": \"result\", \"result\": {...}} event. "
+                             "Ends with {\"type\": \"done\", \"count\": N}. "
+                             "AI Agent can process results incrementally without waiting for full output. "
+                             "Only valid with --format json.")
+    parser.add_argument("--progress", action="store_true",
+                        help="Emit structured progress events as JSON Lines to stderr. "
+                             "Events: start, instance_try, instance_ok, instance_fail, "
+                             "cache_hit, cache_store, fetch_start, fetch_ok, fetch_fail, done. "
+                             "AI Agent can track execution progress programmatically.")
    parser.add_argument("--timeout", type=int, default=_cfg_int(config, "timeout", 15),
                        help="Request timeout in seconds (default: 15)")
    parser.add_argument("--retry", type=int, default=_cfg_int(config, "max_retries", None),
@@ -1356,6 +1410,9 @@ Use --config FILE to load a non-default config file (overrides the auto-discover

    args = parser.parse_args()

+    # 启用 --progress 进度事件（JSON Lines 到 stderr）
+    set_progress_enabled(getattr(args, "progress", False))
+
    # --query is required unless we're doing a non-search operation.
    # --queries-file is an alternative to --query for batch mode.
    if (not args.verify and not args.query and not args.queries_file
@@ -1375,7 +1432,7 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
    if not instance_urls:
        _emit_error("no SearXNG instance resolved. Provide -i/--instance, set the "
                    "SEARXNG_INSTANCE environment variable, or create a searxng.toml / "
-                    "instances.txt config file.", args)
+                    "instances.txt config file.", args, error_code=E_CONFIG)

    # Build auth headers if provided (needed by both verify and search).
    # Credentials may come from CLI flag, file, or env var (in priority order)
@@ -1384,7 +1441,7 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
        bearer_token = resolve_auth_bearer(args.auth_bearer, args.auth_bearer_file)
        basic_auth = resolve_auth_basic(args.auth_basic, args.auth_basic_file)
    except RuntimeError as e:
-        _emit_error(str(e), args)
+        _emit_error(str(e), args, error_code=E_AUTH)
    auth_headers = build_auth_headers(
        bearer_token=bearer_token,
        basic_auth=basic_auth,
@@ -1440,20 +1497,24 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
        try:
            queries = _read_queries_file(args.queries_file)
        except RuntimeError as e:
-            _emit_error(str(e), args)
+            _emit_error(str(e), args, error_code=E_INPUT)
        if not queries:
-            _emit_error(f"no queries found in '{args.queries_file}'", args)
+            _emit_error(f"no queries found in '{args.queries_file}'", args,
+                        error_code=E_INPUT)

        logger.info(f"Running {len(queries)} queries from {args.queries_file}...")
        batch = []
        any_ok = False
        for i, q in enumerate(queries, 1):
            logger.info(f"\n[{i}/{len(queries)}] {q}")
-            results, err = _run_single_query(q, args, instance_urls,
-                                             auth_headers, ttl_seconds)
+            results, err, err_code = _run_single_query(q, args, instance_urls,
+                                                        auth_headers, ttl_seconds)
            if err:
                logger.error(f"  [ERROR] {err}")
-                batch.append({"query": q, "error": err})
+                entry = {"query": q, "error": err}
+                if err_code:
+                    entry["error_code"] = err_code
+                batch.append(entry)
            else:
                any_ok = True
                batch.append({"query": q, "results": results})
@@ -1516,10 +1577,22 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
        sys.exit(0 if any_ok else 1)

    # ----- Single query mode -----
-    results, err = _run_single_query(args.query, args, instance_urls,
-                                     auth_headers, ttl_seconds)
+    results, err, err_code = _run_single_query(args.query, args, instance_urls,
+                                                auth_headers, ttl_seconds)
    if err:
-        _emit_error(err, args, query=args.query)
+        _emit_error(err, args, query=args.query, error_code=err_code)
+
+    # --stream: JSON Lines 流式输出，每条结果一行，AI 可增量处理
+    if getattr(args, "stream", False) and args.format == "json":
+        for r in results.get("results", []):
+            print(json.dumps({"type": "result", "result": r},
+                             ensure_ascii=False), flush=True)
+        print(json.dumps({"type": "done",
+                          "count": len(results.get("results", [])),
+                          "query": args.query}, ensure_ascii=False), flush=True)
+        if not results.get("results"):
+            sys.exit(2)
+        sys.exit(0)

    output = _format_results(results, args)
    if args.output: