feat(v2.3.0): fetch 结构化 JSON 契约 + 并发批量 + 真实并发门控

迭代 1 — 正确性修复: - 修复 --pages N 多页聚合的 unresponsive-engine 警告误判: 原用循环末次 cached 变量判断, 缓存命中时警告被错误跳过/误触发; 改用独立 performed_live_query 标记 - UA 池单一来源: 删除 common.py 手工副本 _FALLBACK_UAS_BUILTIN, FALLBACK_UAS 直接引用 _config.UA_POOL, 消除双份漂移 - search_html 解码修复: 硬编码 utf-8 改为 detect_charset(header/meta 自动检测), 新增 --encoding 强制覆盖, 贯穿 search_multi 全链 - AdaptiveThrottle 真实并发门控: acquire_slot()/release_slot() 槽位机制, 退避降并发后新请求被快速拒绝(E_RATE_LIMIT), 实现持久降并发而非名义降并发迭代 2 — fetch JSON 契约 + 批量并发: - fetch.py --format json: 成功 {status,url,final_url,content_type,extract, truncated,text_length,user_agent}; 失败 {status,error,error_code, status_code,url}, 对齐 search.py 错误码体系 - fetch_page 采集 title + latency, 填充 --fetch-report json 空字段 - --queries-file --parallel-queries N (1-8): 并发批量, 输出保序, 受 AdaptiveThrottle 门控; 并发模式禁用 --fetch(嵌套并行不安全) - queries 文件编码自动检测 (UTF-8 → GBK 回退) 迭代 3 — 工程化: - 新增 pyproject.toml (searxng-search/searxng-fetch 入口点) - 收敛 20+ 处函数内冗余导入 - --dump-schema 扩展: fetched.items 补全 15 字段, 新增 defs.batch/research - 新增 17 个测试 (tests/test_v230_features.py), 全量 561 测试通过 - 文档同步 (SKILL.md/README.md, 版本号 2.3.0)
2026-08-05 20:14:07 +08:00
parent 6cedba9042
commit 4df521dc9d
8 changed files with 1006 additions and 226 deletions
@@ -9,6 +9,8 @@ Instance URLs are REQUIRED (see --instance / SEARXNG_INSTANCE / config file).
 """

 import argparse
+import csv
+import io
 import json
 import logging
 import os
@@ -37,6 +39,7 @@ from common import (
    build_wayback_url,
    classify_error,
    compute_backoff_delay,
+    detect_charset,
    emit_progress,
    force_utf8_stdout,
    is_hard_blocked_domain,
@@ -549,8 +552,13 @@ def search_json(instance: str, params: dict, method: str = "GET",


 def search_html(instance: str, params: dict, timeout: int = 15,
-                auth_headers: dict = None) -> dict:
-    """Execute search via HTML scraping fallback."""
+                auth_headers: dict = None, encoding: str = None) -> dict:
+    """Execute search via HTML scraping fallback.
+
+    v2.2.2：解码改用 common.detect_charset（此前硬编码 utf-8，GBK/Shift-JIS
+    等非 UTF-8 实例的页面会整体乱码）。``encoding`` 为显式覆盖（来自
+    ``--language`` 无关的 CLI ``--encoding``），优先级最高。
+    """
    html_params = {k: v for k, v in params.items() if k != "format"}
    query_string = urllib.parse.urlencode(html_params)
    url = f"{instance}/search?{query_string}"
@@ -559,26 +567,41 @@ def search_html(instance: str, params: dict, timeout: int = 15,
    req = urllib.request.Request(url, headers=headers)
    try:
        with urllib.request.urlopen(req, timeout=timeout) as resp:
-            html = resp.read().decode("utf-8")
+            raw = resp.read()
+            content_type = resp.headers.get("Content-Type", "")
+            if encoding:
+                try:
+                    html = raw.decode(encoding)
+                except (UnicodeDecodeError, LookupError):
+                    html = raw.decode("utf-8", errors="replace")
+            else:
+                charset = detect_charset(raw, content_type)
+                try:
+                    html = raw.decode(charset)
+                except (UnicodeDecodeError, LookupError):
+                    html = raw.decode("utf-8", errors="replace")
            return parse_html_results(html, query=params.get("q", ""))
    except Exception as e:
        raise RuntimeError(f"HTML search failed for {instance}: {e}")


 def search_single(instance: str, params: dict, method: str = "GET",
-                  timeout: int = 15, auth_headers: dict = None) -> dict:
+                  timeout: int = 15, auth_headers: dict = None,
+                  encoding: str = None) -> dict:
    """Execute one search attempt, preferring JSON with HTML fallback."""
    result = search_json(instance, params, method=method, timeout=timeout,
                         auth_headers=auth_headers)
    if result is not None:
        return result
    logger.warning(f"Warning: {instance} does not support format=json, falling back to HTML parsing")
-    return search_html(instance, params, timeout=timeout, auth_headers=auth_headers)
+    return search_html(instance, params, timeout=timeout, auth_headers=auth_headers,
+                       encoding=encoding)


 def search_multi(instance_urls: list, params: dict, method: str = "GET",
                 timeout: int = 15, retry_per: int = None,
-                 auth_headers: dict = None, parallel: bool = True) -> dict:
+                 auth_headers: dict = None, parallel: bool = True,
+                 encoding: str = None) -> dict:
    """Search across multiple instances, failing over on error.

    With parallel=True (default, multi-instance only): every instance is
@@ -589,6 +612,9 @@ def search_multi(instance_urls: list, params: dict, method: str = "GET",

    With parallel=False (or a single instance): strictly sequential, one
    request at a time, trying the next instance only after the current fails.
+
+    ``encoding`` (v2.2.2) is forwarded to the HTML-fallback path for
+    non-UTF-8 instances; JSON responses are always UTF-8.
    """
    if retry_per is None:
        retry_per = MAX_RETRIES
@@ -604,7 +630,8 @@ def search_multi(instance_urls: list, params: dict, method: str = "GET",
            try:
                def _do():
                    return search_single(instance, params, method=method,
-                                         timeout=timeout, auth_headers=auth_headers)
+                                         timeout=timeout, auth_headers=auth_headers,
+                                         encoding=encoding)
                result = _retry_with_backoff(_do, max_retries=retry_per)
                emit_progress("instance_ok", url=instance,
                              latency=round(time.time() - start, 3),
@@ -627,7 +654,8 @@ def search_multi(instance_urls: list, params: dict, method: str = "GET",
        start = time.time()
        def _do():
            return search_single(instance, params, method=method,
-                                 timeout=timeout, auth_headers=auth_headers)
+                                 timeout=timeout, auth_headers=auth_headers,
+                                 encoding=encoding)
        try:
            result = _retry_with_backoff(_do, max_retries=retry_per)
            emit_progress("instance_ok", url=instance,
@@ -864,6 +892,8 @@ def fetch_page(url: str, timeout: int = 10, auth_headers: dict = None,
      * ``fallback_used`` (str|None): 兜底方式（"wayback"），仅当走兜底时有值
    """
    # 主抓取
+    # v2.3.0: 计时整条链路（主抓取 + Wayback 兜底），填充 latency 字段。
+    _start_time = time.monotonic()
    result = None
    error_msg = None
    error_code = None
@@ -946,6 +976,9 @@ def fetch_page(url: str, timeout: int = 10, auth_headers: dict = None,
            "text": "", "text_length": 0, "truncated": False,
            "anti_bot_detected": False, "waf_type": None,
            "fallback_used": None,
+            # v2.3.0: title/latency 字段（--fetch-report json 消费）
+            "title": None,
+            "latency": round(time.monotonic() - _start_time, 3),
        }

    # 反爬仍被检测到（Wayback 也无能为力或兜底被禁用）
@@ -957,6 +990,8 @@ def fetch_page(url: str, timeout: int = 10, auth_headers: dict = None,
            "text": "", "text_length": 0, "truncated": False,
            "anti_bot_detected": True, "waf_type": waf_type,
            "fallback_used": fallback_used,
+            "title": _extract_title(content) if is_html else None,
+            "latency": round(time.monotonic() - _start_time, 3),
        }

    text = extract_text(content) if is_html else content
@@ -973,6 +1008,8 @@ def fetch_page(url: str, timeout: int = 10, auth_headers: dict = None,
        "anti_bot_detected": False,
        "waf_type": None,
        "fallback_used": fallback_used,
+        "title": _extract_title(content) if is_html else None,
+        "latency": round(time.monotonic() - _start_time, 3),
    }


@@ -1105,6 +1142,19 @@ _TITLE_ANTI_BOT_SIGNATURES = {
 _TITLE_RE = re.compile(r"<title[^>]*>(.*?)</title>", re.IGNORECASE | re.DOTALL)


+def _extract_title(content: str) -> str:
+    """从 HTML 内容提取 <title> 文本（v2.3.0，供 --fetch-report json 使用）。
+
+    截断到 200 字符，防止异常页面标题撑爆报告。非 HTML 内容返回空串。
+    """
+    if not content:
+        return ""
+    m = _TITLE_RE.search(content)
+    if m:
+        return m.group(1).strip()[:200]
+    return ""
+
+
 def _detect_anti_bot(content: str) -> str:
    """检测反爬页面，返回 WAF 类型或 None。

@@ -1173,6 +1223,11 @@ class AdaptiveThrottle:
        self._failure_threshold = failure_threshold
        self._pause_seconds = pause_seconds
        self._max_delay = max_delay
+        # v2.2.2：真实并发门控。计数信号量约束"瞬时在飞请求峰值"，
+        # _in_flight 结合当前 concurrency 判断是否应放行新请求——退避降
+        # 并发后，新请求会被快速拒绝（限流语义），而不是名义降并发。
+        self._semaphore = threading.BoundedSemaphore(initial_concurrency)
+        self._in_flight = 0

    @property
    def delay(self) -> float:
@@ -1228,6 +1283,36 @@ class AdaptiveThrottle:
            logger.info(f"  [THROTTLE] Global pause: waiting {remaining:.1f}s (429)")
            time.sleep(remaining)

+    def acquire_slot(self, timeout: float = 0.05) -> bool:
+        """获取一个并发执行槽位（v2.2.2 真实并发门控）。
+
+        线程池规模（ThreadPoolExecutor max_workers）创建时一次性固定，
+        无法随退避动态缩小。槽位机制在"请求真正发出前"做门控：
+        信号量约束瞬时峰值不超过初始并发；``_in_flight`` 结合当前
+        ``concurrency`` 判断——退避降并发后，即使信号量有空位，只要在飞
+        请求数已 >= 当前并发目标，新请求也会被快速拒绝（返回 False），
+        由调用方跳过本次抓取，实现持久降并发。
+
+        返回 True 表示拿到槽位，调用方必须在 finally 中 release_slot()。
+        """
+        if not self._semaphore.acquire(timeout=timeout):
+            return False
+        with self._lock:
+            if self._in_flight >= self._concurrency:
+                self._semaphore.release()
+                return False
+            self._in_flight += 1
+            return True
+
+    def release_slot(self) -> None:
+        """释放并发槽位。必须与 acquire_slot 成对使用。"""
+        with self._lock:
+            self._in_flight = max(0, self._in_flight - 1)
+        try:
+            self._semaphore.release()
+        except ValueError:
+            pass
+
    def stats(self) -> dict:
        """返回当前状态快照，供 --fetch-report 使用。"""
        with self._lock:
@@ -1292,34 +1377,49 @@ def fetch_top_results(results: dict, count: int, timeout: int = 10,
        """Fetch one URL with adaptive throttling to avoid rate limiting."""
        # 全局暂停检查（429 触发）
        throttle.wait_if_paused()
-        # 自适应延迟
-        d = throttle.delay
-        if d > 0:
-            time.sleep(d * random.uniform(0.5, 1.5))
-        result = fetch_page(u, timeout=timeout, auth_headers=auth_headers,
-                            max_retries=max_retries, max_size=max_size,
-                            referer=referer, fallback_enabled=fallback_enabled)
-        if result["status"] == "ok":
-            ok_count[0] += 1
-            throttle.report_success()
-            trunc = ", TRUNCATED" if result.get("truncated") else ""
-            ua_note = ""
-            if result.get("user_agent_used") != USER_AGENT:
-                ua_note = " [fallback UA]"
-            fb_note = " [wayback]" if result.get("fallback_used") else ""
-            if fb_note:
-                fallback_count[0] += 1
-            logger.info(f"  [OK] {u[:55]} ({result['text_length']:,} chars"
-                        f"{trunc}{ua_note}{fb_note})")
-        else:
-            err_count[0] += 1
-            throttle.report_failure(result.get("error", ""),
-                                    error_code=result.get("error_code"))
-            # 统计反爬拦截
-            if result.get("anti_bot_detected"):
-                anti_bot_count[0] += 1
-            logger.error(f"  [ERR] {u[:55]} ({result.get('error', 'unknown')})")
-        return result
+        # v2.2.2：真实并发门控。退避降并发后，超出当前并发目标的新请求
+        # 在这里被拒绝（不占用 fetch_page），实现持久降并发而非名义降并发。
+        if not throttle.acquire_slot():
+            logger.info(f"  [THROTTLE] concurrency cap ({throttle.concurrency}) "
+                        f"reached, skipping {u[:55]}")
+            return {"url": u, "status": "error",
+                    "error": "Throttled: concurrency limit reached",
+                    "error_code": E_RATE_LIMIT,
+                    "text": "", "text_length": 0, "truncated": False,
+                    "anti_bot_detected": False, "waf_type": None,
+                    "fallback_used": None,
+                    "title": None, "latency": None}
+        try:
+            # 自适应延迟
+            d = throttle.delay
+            if d > 0:
+                time.sleep(d * random.uniform(0.5, 1.5))
+            result = fetch_page(u, timeout=timeout, auth_headers=auth_headers,
+                                max_retries=max_retries, max_size=max_size,
+                                referer=referer, fallback_enabled=fallback_enabled)
+            if result["status"] == "ok":
+                ok_count[0] += 1
+                throttle.report_success()
+                trunc = ", TRUNCATED" if result.get("truncated") else ""
+                ua_note = ""
+                if result.get("user_agent_used") != USER_AGENT:
+                    ua_note = " [fallback UA]"
+                fb_note = " [wayback]" if result.get("fallback_used") else ""
+                if fb_note:
+                    fallback_count[0] += 1
+                logger.info(f"  [OK] {u[:55]} ({result['text_length']:,} chars"
+                            f"{trunc}{ua_note}{fb_note})")
+            else:
+                err_count[0] += 1
+                throttle.report_failure(result.get("error", ""),
+                                        error_code=result.get("error_code"))
+                # 统计反爬拦截
+                if result.get("anti_bot_detected"):
+                    anti_bot_count[0] += 1
+                logger.error(f"  [ERR] {u[:55]} ({result.get('error', 'unknown')})")
+            return result
+        finally:
+            throttle.release_slot()

    with ThreadPoolExecutor(max_workers=min(throttle.concurrency, len(urls))) as ex:
        future_map = {ex.submit(_fetch_one, u): u for u in urls}
@@ -1333,7 +1433,8 @@ def fetch_top_results(results: dict, count: int, timeout: int = 10,
                                "error_code": classify_error(e),
                                "text": "", "text_length": 0, "truncated": False,
                                "anti_bot_detected": False, "waf_type": None,
-                                "fallback_used": None})
+                                "fallback_used": None,
+                                "title": None, "latency": None})
                logger.error(f"  [ERR] {u[:55]} (thread error: {e})")

    # Reorder to match original result order
@@ -1365,8 +1466,7 @@ def _emit_fetch_report(fetched: list, throttle: "AdaptiveThrottle",

 def _emit_fetch_report_text(fetched: list, throttle: "AdaptiveThrottle") -> None:
    """文本表格 + JSON 摘要行（v2.0.0 原始行为，向后兼容）。"""
-    import sys as _sys
-    out = _sys.stderr
+    out = sys.stderr
    lines = []
    lines.append("\n" + "=" * 72)
    lines.append("FETCH REPORT (v2.0.0)")
@@ -1403,13 +1503,12 @@ def _emit_fetch_report_text(fetched: list, throttle: "AdaptiveThrottle") -> None
                 f"consec_ok={s['consecutive_successes']}")

    # JSON 摘要（一行，便于 Agent 解析）
-    import json as _json
    summary = {
        "total": total, "ok": ok, "error": err,
        "anti_bot_blocked": anti_bot, "wayback_recovered": wayback,
        "throttle": s,
    }
-    lines.append("JSON: " + _json.dumps(summary, ensure_ascii=False))
+    lines.append("JSON: " + json.dumps(summary, ensure_ascii=False))
    lines.append("=" * 72 + "\n")
    print("\n".join(lines), file=out)

@@ -1418,11 +1517,9 @@ def _emit_fetch_report_json(fetched: list, throttle: "AdaptiveThrottle") -> None
    """完整 JSON 报告（items 数组 + summary），输出到 stderr。

    每个 URL 一个对象，包含 url / status / title / content_length /
-    error / error_code / latency / fetched_at 等字段。fetch_page 未采集
-    的字段（title / latency / fetched_at）为 None，便于 Agent 统一解析。
+    error / error_code / latency / fetched_at 等字段。v2.3.0 起
+    ``title`` 与 ``latency`` 由 fetch_page 采集填充（此前恒为 None）。
    """
-    import json as _json
-    import sys as _sys
    from datetime import datetime, timezone

    fetched_at = datetime.now(timezone.utc).isoformat()
@@ -1434,12 +1531,12 @@ def _emit_fetch_report_json(fetched: list, throttle: "AdaptiveThrottle") -> None
            "url": f.get("url", ""),
            "final_url": f.get("final_url"),
            "status": f.get("status", ""),
-            "title": f.get("title"),            # fetch_page 未提取，None
+            "title": f.get("title"),            # v2.3.0: fetch_page 已采集
            "content_length": f.get("text_length", 0),
            "content_type": f.get("content_type"),
            "error": f.get("error"),
            "error_code": f.get("error_code"),
-            "latency": f.get("latency"),         # fetch_page 未计时，None
+            "latency": f.get("latency"),         # v2.3.0: fetch_page 已计时
            "truncated": f.get("truncated", False),
            "fetched_at": f.get("fetched_at") or fetched_at,
            "waf_type": f.get("waf_type"),
@@ -1464,7 +1561,7 @@ def _emit_fetch_report_json(fetched: list, throttle: "AdaptiveThrottle") -> None
        "throttle": throttle.stats(),
        "items": items,
    }
-    print(_json.dumps(report, ensure_ascii=False), file=_sys.stderr)
+    print(json.dumps(report, ensure_ascii=False), file=sys.stderr)


 # ----- Output formatting -----
@@ -1694,10 +1791,8 @@ def _format_results(results: dict, args) -> str:
    if args.format == "urls":
        return format_urls(results)
    if args.format == "csv":
-        import csv as csv_mod
-        import io
        out = io.StringIO()
-        writer = csv_mod.writer(out, lineterminator="\n")
+        writer = csv.writer(out, lineterminator="\n")
        writer.writerow(["title", "url", "engine", "score",
                         "published_date", "content"])
        for r in results.get("results", []):
@@ -1803,6 +1898,13 @@ def _run_single_query(query: str, args, instance_urls: list,

    emit_progress("start", query=query, instances=len(instance_urls))

+    # v2.2.2：区分"实时查询"与"缓存命中"。原实现用循环末次赋值的
+    # ``cached`` 变量判断是否实时查询——多页路径下该变量保存的是最后一页
+    # 的状态，导致：最后一页命中缓存但前页实时查询时，unresponsive_engines
+    # 警告被错误跳过；反之仅最后一页未命中时误触发。用独立布尔标记精确
+    # 跟踪"本次运行是否发起了至少一次实时查询"。
+    performed_live_query = False
+
    # v2.2.0：--pages N 多页聚合。循环 pageno=1..N，每页独立缓存
    # （cache key 含 pageno），合并后统一 dedup/sort/max-results。
    if pages_to_fetch == 1:
@@ -1821,11 +1923,13 @@ def _run_single_query(query: str, args, instance_urls: list,
                    retry_per=args.retry,
                    auth_headers=auth_headers,
                    parallel=not args.serial,
+                    encoding=getattr(args, "encoding", None),
                )
            except Exception as e:
                err_code = classify_error(e)
                emit_progress("error", error=str(e), error_code=err_code, query=query)
                return None, str(e), err_code
+            performed_live_query = True
            if ttl_seconds > 0:
                cache_module.put(params, results, ttl_seconds)
                logger.info(f"[cache stored] q={query!r} TTL={args.cache_ttl}min")
@@ -1855,6 +1959,7 @@ def _run_single_query(query: str, args, instance_urls: list,
                        retry_per=args.retry,
                        auth_headers=auth_headers,
                        parallel=not args.serial,
+                        encoding=getattr(args, "encoding", None),
                    )
                except Exception as e:
                    page_errors += 1
@@ -1862,6 +1967,7 @@ def _run_single_query(query: str, args, instance_urls: list,
                    emit_progress("page_fail", query=query, page=page_no,
                                  error=str(e), error_code=classify_error(e))
                    continue
+                performed_live_query = True
                if ttl_seconds > 0:
                    cache_module.put(page_params, page_results, ttl_seconds)
                    emit_progress("cache_store", query=query, ttl=args.cache_ttl, page=page_no)
@@ -1886,7 +1992,7 @@ def _run_single_query(query: str, args, instance_urls: list,

    # v2.1.1：检测实例侧引擎挂起/限流（仅在实时查询后提示，cache hit 时
    # unresponsive_engines 信息可能已过期）
-    if cached is None:
+    if performed_live_query:
        _warn_unresponsive_engines(results, query,
                                   result_count=len(results.get("results", [])))

@@ -1975,14 +2081,54 @@ def _get_output_schema():

    Used by ``--dump-schema`` so AI agents can programmatically discover the
    output structure without parsing prose documentation.
+
+    v2.3.0: fetched.items 字段补全（与 fetch_page 实际输出对齐——
+    final_url/status/error_code/waf_type/fallback_used/title/latency 等），
+    并新增 ``batch`` 与 ``research`` 两个属性描述对应模式的输出 shape。
+    顶层 ``properties`` 仍以单查询为主，batch/research 为单独子树。
    """
-    return {
-        "$schema": "https://json-schema.org/draft/2020-12/schema",
-        "title": "SearXNG CLI Search Result",
-        "schema_version": SCHEMA_VERSION,
-        "description": "Output schema for 'python search.py --format json' (single query). "
-                       "Batch mode (--queries-file) wraps results in "
-                       '{"schema_version, queries:[]}.',
+    # 单查询结果条目（results[] 的元素）——batch/research 复用
+    result_item = {
+        "type": "object",
+        "properties": {
+            "title": {"type": "string"},
+            "url": {"type": "string", "format": "uri"},
+            "engine": {"type": "string", "description": "Source engine name."},
+            "score": {"type": ["number", "null"]},
+            "published_date": {"type": ["string", "null"]},
+            "content": {"type": "string", "description": "Snippet/summary text."},
+        },
+        "required": ["title", "url"],
+    }
+    # --fetch N 时的 fetched[] 元素（与 search.fetch_page 返回对齐）
+    fetched_item = {
+        "type": "object",
+        "properties": {
+            "url": {"type": "string"},
+            "final_url": {"type": ["string", "null"],
+                          "description": "URL after redirects / Wayback."},
+            "status": {"type": "string", "enum": ["ok", "error"]},
+            "content_type": {"type": ["string", "null"]},
+            "text": {"type": "string", "description": "Extracted page text."},
+            "text_length": {"type": "integer"},
+            "truncated": {"type": "boolean"},
+            "title": {"type": ["string", "null"],
+                      "description": "Page <title>, when available."},
+            "latency": {"type": ["number", "null"],
+                        "description": "Fetch latency in seconds."},
+            "user_agent_used": {"type": ["string", "null"]},
+            "error": {"type": ["string", "null"]},
+            "error_code": {"type": ["string", "null"]},
+            "waf_type": {"type": ["string", "null"]},
+            "fallback_used": {"type": ["string", "null"],
+                              "description": "'wayback' when the Wayback "
+                                             "Machine recovered the page."},
+            "anti_bot_detected": {"type": "boolean"},
+        },
+        "required": ["url", "status"],
+    }
+    # 单查询输出
+    single_query = {
        "type": "object",
        "properties": {
            "schema_version": {
@@ -1999,18 +2145,7 @@ def _get_output_schema():
            "results": {
                "type": "array",
                "description": "Search result items, ordered by relevance (score desc).",
-                "items": {
-                    "type": "object",
-                    "properties": {
-                        "title": {"type": "string"},
-                        "url": {"type": "string", "format": "uri"},
-                        "engine": {"type": "string", "description": "Source engine name."},
-                        "score": {"type": ["number", "null"]},
-                        "published_date": {"type": ["string", "null"]},
-                        "content": {"type": "string", "description": "Snippet/summary text."},
-                    },
-                    "required": ["title", "url"],
-                },
+                "items": result_item,
            },
            "unresponsive_engines": {
                "type": "array",
@@ -2022,19 +2157,16 @@ def _get_output_schema():
                "items": {"type": "string"},
                "description": "Related query suggestions from the instance.",
            },
+            "answers": {
+                "type": "array",
+                "items": {"type": "string"},
+                "description": "Direct answers from the instance.",
+            },
            "fetched": {
                "type": "array",
                "description": "Present only when --fetch N is used. Page content "
                               "for the top N results.",
-                "items": {
-                    "type": "object",
-                    "properties": {
-                        "url": {"type": "string"},
-                        "text": {"type": "string"},
-                        "text_length": {"type": "integer"},
-                        "error": {"type": "string"},
-                    },
-                },
+                "items": fetched_item,
            },
            "fetched_source": {
                "type": "string",
@@ -2042,9 +2174,100 @@ def _get_output_schema():
                "description": "Present only when --fetch is used. Indicates whether "
                               "search results came from JSON API or HTML fallback.",
            },
+            "pages_fetched": {
+                "type": ["integer", "null"],
+                "description": "Present only when --pages N > 1. Pages that "
+                               "succeeded (before cross-page merge).",
+            },
        },
        "required": ["query", "results"],
    }
+    # --queries-file 批量输出
+    batch_schema = {
+        "type": "object",
+        "description": "Batch mode (--queries-file) output shape.",
+        "properties": {
+            "schema_version": {"type": "string", "const": SCHEMA_VERSION},
+            "queries": {
+                "type": "array",
+                "description": "One entry per query, in file order. "
+                               "Entry: {query, status: ok|error, results} on "
+                               "success or {query, status: error, error, "
+                               "error_code} on failure.",
+                "items": {"type": "object",
+                          "required": ["query", "status"],
+                          "properties": {
+                              "query": {"type": "string"},
+                              "status": {"type": "string", "enum": ["ok", "error"]},
+                              "results": single_query,
+                              "error": {"type": "string"},
+                              "error_code": {"type": "string"},
+                          }},
+            },
+        },
+        "required": ["schema_version", "queries"],
+    }
+    # --research 研究模式输出
+    research_schema = {
+        "type": "object",
+        "description": "Research mode (--research TOPIC) output shape.",
+        "properties": {
+            "schema_version": {"type": "string", "const": SCHEMA_VERSION},
+            "research_topic": {"type": "string"},
+            "research_queries": {
+                "type": "array",
+                "description": "Expanded per-angle queries (deterministic rules).",
+                "items": {"type": "object",
+                          "properties": {
+                              "angle": {"type": "string"},
+                              "query": {"type": "string"},
+                          }},
+            },
+            "queries": {"type": "array",
+                        "description": "Per-angle results. Entry: {query, angle, "
+                                       "status: ok|error, results|error, error_code}.",
+                        "items": {"type": "object",
+                                  "properties": {
+                                      "query": {"type": "string"},
+                                      "angle": {"type": "string"},
+                                      "status": {"type": "string",
+                                                 "enum": ["ok", "error"]},
+                                      "results": single_query,
+                                      "error": {"type": "string"},
+                                      "error_code": {"type": "string"},
+                                  }}},
+            "merged_results": {
+                "type": "object",
+                "description": "Cross-angle merged + deduplicated result set "
+                               "(same shape as single-query 'results').",
+                "properties": {
+                    "query": {"type": "string"},
+                    "results": {"type": "array", "items": result_item},
+                },
+            },
+        },
+        "required": ["schema_version", "research_topic", "queries",
+                     "merged_results"],
+    }
+    return {
+        "$schema": "https://json-schema.org/draft/2020-12/schema",
+        "title": "SearXNG CLI Search Result",
+        "schema_version": SCHEMA_VERSION,
+        "description": "Output schema for 'python search.py --format json'. "
+                       "Top-level 'properties' describe single-query output; "
+                       "the 'batch' and 'research' subtrees describe "
+                       "--queries-file and --research output respectively. "
+                       "v2.3.0: fetched.items fields now match fetch_page "
+                       "output (title/latency/waf_type/error_code etc.).",
+        "type": "object",
+        "properties": single_query["properties"],
+        "required": single_query["required"],
+        "defs": {
+            "single_query": single_query,
+            "batch": batch_schema,
+            "research": research_schema,
+        },
+    }


 def _emit_error(message: str, args, query: str = None, exit_code: int = 1,
@@ -2087,13 +2310,30 @@ def _emit_error(message: str, args, query: str = None, exit_code: int = 1,
 def _read_queries_file(path: str) -> list:
    """Read queries from a file: one per line, skip blanks and ``#`` comments.

+    v2.3.0: 编码自动检测。先按 UTF-8 读取；若解码失败（Windows 下 GBK 等
+    非 UTF-8 文件常见），回退 GBK，再失败回退 utf-8 errors=replace——
+    绝不因编码问题让批量任务整体失败。
+
    Raises :class:`RuntimeError` if the file cannot be read, so the caller
    can route it through :func:`_emit_error`.
    """
+    raw = None
    try:
-        text = Path(path).read_text(encoding="utf-8")
+        raw = Path(path).read_bytes()
    except OSError as e:
        raise RuntimeError(f"cannot read queries file '{path}': {e}")
+
+    text = None
+    for enc in ("utf-8", "gbk"):
+        try:
+            text = raw.decode(enc)
+            break
+        except (UnicodeDecodeError, LookupError):
+            continue
+    if text is None:
+        # 最后兜底：UTF-8 + 替换符，保证任务可继续
+        text = raw.decode("utf-8", errors="replace")
+
    queries = []
    for line in text.splitlines():
        line = line.strip()
@@ -2260,7 +2500,7 @@ def expand_research_queries(topic: str, custom_angles: list = None) -> list:

 # ----- Mode handlers (v2.2.0: 从 main() 提取，降低单函数复杂度) -----
 # main() 只负责参数解析和分发，四条执行路径各自独立函数，便于维护和测试。
-# 纯提取重构，行为与 v2.1.1 完全一致，539 测试兜底验证。
+# 纯提取重构，行为与 v2.1.1 完全一致，544 测试兜底验证。


 def _handle_verify(args, instance_urls: list, auth_headers: dict) -> None:
@@ -2340,10 +2580,8 @@ def _handle_research(args, instance_urls: list, auth_headers: dict,
            "merged_results": merged,
        }, indent=2, ensure_ascii=False)
    elif args.format == "csv":
-        import csv as csv_mod
-        import io
        out = io.StringIO()
-        writer = csv_mod.writer(out, lineterminator="\n")
+        writer = csv.writer(out, lineterminator="\n")
        writer.writerow(["angle", "query", "title", "url", "engine",
                         "score", "published_date", "content"])
        for br in batch:
@@ -2412,11 +2650,33 @@ def _handle_research(args, instance_urls: list, auth_headers: dict,
    sys.exit(0)


+def _run_single_query_wrapper(query: str, args, instance_urls: list,
+                              auth_headers: dict, ttl_seconds: int):
+    """并发批量用的 _run_single_query 包装（v2.3.0）。
+
+    移除不可跨线程共享的参数：
+      * ``args.fetch`` 置 0 —— --fetch 的 ThreadPoolExecutor 在查询线程内
+        创建，worker 再经 ThreadPoolExecutor 二次并发会超过线程安全上限；
+        并发批量模式用 ``--queries-file`` 不适合内嵌抓取。
+      * ``args.output`` 置 None —— 输出写入统一交给 _handle_batch。
+    其余参数原样透传（行为与串行路径一致）。
+    """
+    import copy as _copy
+    qargs = _copy.copy(args)
+    qargs.fetch = 0
+    qargs.output = None
+    return _run_single_query(query, qargs, instance_urls, auth_headers,
+                             ttl_seconds)
+
+
 def _handle_batch(args, instance_urls: list, auth_headers: dict,
                  ttl_seconds: int) -> None:
-    """批量模式：从文件读取多个查询，串行执行，输出合并结果。
+    """批量模式：从文件读取多个查询，串行或并发执行，输出合并结果。

    v2.2.0 从 main() 提取为独立函数（纯重构，行为不变）。
+    v2.3.0 新增 --parallel-queries N：并发执行（受 AdaptiveThrottle
+    约束），输出保持文件顺序；并发模式下 --fetch 被禁用（见
+    _run_single_query_wrapper），日志压缩为每查询一行。
    """
    try:
        queries = _read_queries_file(args.queries_file)
@@ -2426,34 +2686,89 @@ def _handle_batch(args, instance_urls: list, auth_headers: dict,
        _emit_error(f"no queries found in '{args.queries_file}'", args,
                    error_code=E_INPUT)

-    logger.info(f"Running {len(queries)} queries from {args.queries_file}...")
-    batch = []
-    any_with_results = False
-    error_count = 0
-    for i, q in enumerate(queries, 1):
-        logger.info(f"\n[{i}/{len(queries)}] {q}")
-        results, err, err_code = _run_single_query(q, args, instance_urls,
-                                                    auth_headers, ttl_seconds)
-        if err:
-            error_count += 1
-            logger.error(f"  [ERROR] {err}")
-            entry = {"query": q, "status": "error", "error": err}
-            if err_code:
-                entry["error_code"] = err_code
-            batch.append(entry)
-        else:
-            if len(results.get("results", [])) > 0:
-                any_with_results = True
-            batch.append({"query": q, "status": "ok", "results": results})
+    parallel = getattr(args, "parallel_queries", 0) or 0
+    if parallel < 1:
+        parallel = 1
+    if parallel > 8:
+        parallel = 8
+
+    if parallel > 1:
+        logger.info(f"Running {len(queries)} queries from {args.queries_file} "
+                    f"in parallel ({parallel} workers)...")
+        batch = [None] * len(queries)
+        any_with_results = [False]
+        error_count = [0]
+
+        def _run(idx_q):
+            idx, q = idx_q
+            logger.info(f"[{idx+1}/{len(queries)}] {q}")
+            results, err, err_code = _run_single_query_wrapper(
+                q, args, instance_urls, auth_headers, ttl_seconds)
+            return idx, q, results, err, err_code
+
+        throttle = AdaptiveThrottle(0.0, parallel)
+
+        def _job(idx_q):
+            # 全局暂停（429）+ 并发槽位门控（退避降并发时快速拒绝）
+            throttle.wait_if_paused()
+            if not throttle.acquire_slot():
+                logger.info(f"  [THROTTLE] concurrency cap ({throttle.concurrency}) "
+                            f"reached, skipping query {idx_q[0]+1}")
+                return idx_q[0], idx_q[1], None, "Throttled: concurrency limit", E_RATE_LIMIT
+            try:
+                return _run(idx_q)
+            finally:
+                throttle.release_slot()
+
+        with ThreadPoolExecutor(max_workers=parallel) as ex:
+            futures = [ex.submit(_job, item) for item in enumerate(queries)]
+            for fut in as_completed(futures):
+                idx, q, results, err, err_code = fut.result()
+                if err:
+                    error_count[0] += 1
+                    entry = {"query": q, "status": "error", "error": err}
+                    if err_code:
+                        entry["error_code"] = err_code
+                    batch[idx] = entry
+                else:
+                    if len(results.get("results", [])) > 0:
+                        any_with_results[0] = True
+                    batch[idx] = {"query": q, "status": "ok", "results": results}
+
+        # 日志：并发模式下错误信息以单行输出（串行模式按序打印多行）
+        if error_count[0]:
+            logger.warning(f"Parallel batch: {error_count[0]} errors, "
+                           f"{len(queries) - error_count[0]} ok")
+
+        error_count = error_count[0]
+        any_with_results = any_with_results[0]
+    else:
+        logger.info(f"Running {len(queries)} queries from {args.queries_file}...")
+        batch = []
+        any_with_results = False
+        error_count = 0
+        for i, q in enumerate(queries, 1):
+            logger.info(f"\n[{i}/{len(queries)}] {q}")
+            results, err, err_code = _run_single_query(q, args, instance_urls,
+                                                        auth_headers, ttl_seconds)
+            if err:
+                error_count += 1
+                logger.error(f"  [ERROR] {err}")
+                entry = {"query": q, "status": "error", "error": err}
+                if err_code:
+                    entry["error_code"] = err_code
+                batch.append(entry)
+            else:
+                if len(results.get("results", [])) > 0:
+                    any_with_results = True
+                batch.append({"query": q, "status": "ok", "results": results})

    if args.format == "json":
        output = json.dumps({"schema_version": SCHEMA_VERSION, "queries": batch},
                            indent=2, ensure_ascii=False)
    elif args.format == "csv":
-        import csv as csv_mod
-        import io
        out = io.StringIO()
-        writer = csv_mod.writer(out, lineterminator="\n")
+        writer = csv.writer(out, lineterminator="\n")
        writer.writerow(["query", "title", "url", "engine", "score",
                         "published_date", "content"])
        for br in batch:
@@ -2627,6 +2942,12 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
                        help="Comma-separated categories (e.g. general,images,news)")
    parser.add_argument("--language", "-l", default=config.get("language"),
                        help="Language code (e.g. en, zh-CN, de)")
+    parser.add_argument("--encoding", default=config.get("encoding"),
+                        help="Force charset for HTML-fallback decoding "
+                             "(e.g. gbk, shift_jis). v2.2.2: auto-detected "
+                             "from the HTTP header / HTML meta when omitted; "
+                             "this flag overrides auto-detection for "
+                             "misconfigured instances.")
    parser.add_argument("--pageno", "-p", type=int, default=1,
                        help="Page number (default: 1)")
    parser.add_argument("--pages", type=int, default=1,
@@ -2769,6 +3090,13 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
                             "starting with '#' are skipped) and run them in sequence. "
                             "Results are emitted as a JSON array (or one brief block per query). "
                             "Overrides --query when set.")
+    parser.add_argument("--parallel-queries", type=int, default=0, metavar="N",
+                        help="v2.3.0: run batch queries concurrently with N workers "
+                             "(1-8, capped at 8; default 0 = sequential). Output order "
+                             "is preserved. Concurrency is gated by AdaptiveThrottle — "
+                             "on repeated failures workers are throttled, not spammed. "
+                             "When enabled, --fetch is disabled (nested parallel fetch "
+                             "is unsafe) and per-query logs are compressed.")
    parser.add_argument("--research", default=None, metavar="TOPIC",
                        help="v2.1.0 Research mode: given a topic, auto-expand into 5 "
                             "multi-angle queries (overview/profile/background/works/review) "
@@ -2803,8 +3131,7 @@ Use --config FILE to load a non-default config file (overrides the auto-discover

    # v2.2.0：生成 request_id 并重新配置 logging（带 log_format + request_id）
    # request_id 贯穿所有日志、进度事件和错误输出，便于 batch 模式追溯。
-    import os as _os
-    request_id = _os.urandom(4).hex()
+    request_id = os.urandom(4).hex()
    setup_logging(verbose=args.verbose, quiet=args.quiet,
                  log_format=getattr(args, "log_format", "text"),
                  request_id=request_id)