#!/usr/bin/env python3
"""Execute searches via a user-supplied SearXNG instance's JSON API.

Calls GET/POST /search?q=...&format=json on the chosen instance.
Auto-retries on failure with exponential backoff + tries next instance.
Falls back to HTML scraping if the instance blocks JSON output.

Instance URLs are REQUIRED (see --instance / SEARXNG_INSTANCE / config file).
"""

import argparse
import json
import logging
import os
import random
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
from concurrent.futures import ThreadPoolExecutor, as_completed
from html.parser import HTMLParser
from pathlib import Path

# Allow running standalone from any working directory
sys.path.insert(0, str(Path(__file__).resolve().parent))
from _config import SCHEMA_VERSION, USER_AGENT, VERSION
from common import (
    RETRYABLE_STATUS,
    RETRY_BACKOFF_BASE,
    MAX_RETRIES,
    RECOVERY_HINTS,
    apply_proxy,
    build_auth_headers,
    classify_error,
    emit_progress,
    force_utf8_stdout,
    resolve_auth_basic,
    resolve_auth_bearer,
    set_progress_enabled,
    setup_logging,
    E_CONFIG,
    E_AUTH,
    E_NETWORK,
    E_RATE_LIMIT,
    E_PARSE,
    E_EMPTY,
    E_INPUT,
    E_INTERNAL,
)
from fetch import extract_text, fetch_url
import cache as cache_module

logger = logging.getLogger("searxng.search")


# ----- Auth helpers -----

def _merge_headers(*header_dicts: dict) -> dict:
    """Merge multiple header dicts, with later dicts overriding earlier ones."""
    result = {}
    for d in header_dicts:
        if d:
            result.update(d)
    return result


# ----- HTML fallback: result extractor -----

class SearXNGHTMLParser(HTMLParser):
    """Extract search results + metadata from SearXNG's simple theme HTML.

    Matches:
      <article class="result result-default category-general">
        <a href="..." class="url_header"> ... </a>
        <h3><a href="...">Title</a></h3>
        <p class="content"> Snippet... </p>
        <time datetime="2024-01-15T10:30:00">...</time>
      </article>
      <div id="suggestions"><a>...</a></div>
      <div class="answer">...</div>

    The ``published_date`` field is populated from the ``<time>`` tag's
    ``datetime`` attribute (preferred) or its text content (fallback),
    mirroring the field SearXNG's JSON API exposes.
    """

    def __init__(self):
        super().__init__()
        self.results = []
        self.suggestions = []
        self.answers = []
        self.infoboxes = []

        self._current = None
        self._in_article = False
        self._in_h3 = False
        self._in_content = False
        self._in_time = False
        self._in_suggestions = False
        self._in_answer = False
        self._text_buf = []
        self._skip_depth = 0

    def handle_starttag(self, tag, attrs):
        attrs_dict = dict(attrs)
        classes = attrs_dict.get("class", "").split()
        tag_id = attrs_dict.get("id", "")

        if self._skip_depth > 0:
            self._skip_depth += 1
            return

        # Results
        if tag == "article" and "result" in classes:
            self._in_article = True
            self._current = {"title": "", "url": "", "content": "",
                             "engine": "", "published_date": ""}

        if self._in_article:
            if tag == "a" and "url_header" in classes:
                href = attrs_dict.get("href", "")
                if href:
                    self._current["url"] = href
            elif tag == "h3":
                self._in_h3 = True
                self._text_buf = []
            elif tag == "a" and self._in_h3:
                # h3 内的 <a href> 作为 url 的 fallback：某些 SearXNG 主题
                # 不使用 url_header class，URL 仅出现在 h3 的链接中。
                # url_header 优先（上面已处理），此处仅填充空 url。
                href = attrs_dict.get("href", "")
                if href and not self._current.get("url"):
                    self._current["url"] = href
            elif tag == "p" and "content" in classes:
                self._in_content = True
                self._text_buf = []
            elif tag == "time":
                # SearXNG puts the ISO timestamp in the datetime attribute
                dt = attrs_dict.get("datetime", "").strip()
                if dt and self._current is not None:
                    self._current["published_date"] = dt
                # Still collect text as a fallback for instances that omit
                # the datetime attribute but render a human-readable date.
                self._in_time = True
                self._text_buf = []
            elif tag in ("script", "style"):
                self._skip_depth += 1

        # Suggestions: <div id="suggestions"> or class containing "suggestion"
        if tag_id == "suggestions" or "suggestion" in classes:
            self._in_suggestions = True
            self._text_buf = []
        if self._in_suggestions and tag == "a":
            self._text_buf = []

        # Answer boxes
        if "answer" in classes or tag_id == "answer":
            self._in_answer = True
            self._text_buf = []

    def handle_endtag(self, tag):
        if self._skip_depth > 0:
            self._skip_depth -= 1
            return

        if self._in_article and tag == "article":
            if self._current:
                self._current["title"] = self._current.get("title", "").strip()
                self._current["content"] = self._current.get("content", "").strip()
                if self._current.get("title") or self._current.get("url"):
                    self.results.append(self._current)
            self._in_article = False
            self._current = None
        elif self._in_h3 and tag == "h3":
            if self._current:
                self._current["title"] = " ".join(self._text_buf).strip()
            self._in_h3 = False
        elif self._in_content and tag == "p":
            if self._current:
                self._current["content"] = " ".join(self._text_buf).strip()
            self._in_content = False
        elif self._in_time and tag == "time":
            # Fallback: use text content only if the datetime attribute
            # wasn't already captured at starttag time.
            if self._current and not self._current.get("published_date"):
                text = " ".join(self._text_buf).strip()
                if text:
                    self._current["published_date"] = text
            self._in_time = False

        if self._in_suggestions and tag == "div":
            self._in_suggestions = False
        if self._in_answer and tag == "div":
            if self._text_buf:
                self.answers.append(" ".join(self._text_buf).strip())
            self._in_answer = False

    def handle_data(self, data):
        if self._skip_depth > 0:
            return
        if self._in_h3:
            self._text_buf.append(data)
        elif self._in_content:
            self._text_buf.append(data)
        elif self._in_time:
            self._text_buf.append(data)
        elif self._in_suggestions:
            stripped = data.strip()
            if stripped:
                self.suggestions.append(stripped)
        elif self._in_answer:
            self._text_buf.append(data)


def parse_html_results(html: str, query: str = "") -> dict:
    """Parse SearXNG HTML results page into structured dict."""
    parser = SearXNGHTMLParser()
    parser.feed(html)
    return {
        "query": query,
        "number_of_results": len(parser.results),
        "results": parser.results,
        "answers": parser.answers,
        "corrections": [],  # rarely in simple theme
        "suggestions": parser.suggestions,
        "infoboxes": parser.infoboxes,
        "unresponsive_engines": [],
        "_fallback": "html",
    }


# ----- Instance selection -----

def parse_instances(instance_arg: str) -> list:
    """Parse user-supplied instance URL(s).

    Accepts a single URL or a comma-separated list. Each entry is normalized
    (https:// prefix added if missing, trailing slash stripped). Multiple
    instances enable multi-instance failover.
    """
    urls = []
    for raw in instance_arg.split(","):
        u = raw.strip()
        if not u:
            continue
        if not u.startswith(("http://", "https://")):
            u = "https://" + u
        urls.append(u.rstrip("/"))
    return urls


def _normalize_csv(value: str) -> str:
    """Normalize a comma-separated list: strip each item, drop empties.

    ``"google, bing, brave"`` -> ``"google,bing,brave"`` so downstream params
    match SearXNG's exact engine name requirements.
    """
    return ",".join(p.strip() for p in value.split(",") if p.strip())


def _load_toml(path: Path) -> dict:
    """Load a TOML file with Python-version-aware backend selection.

    * Python 3.11+ → stdlib ``tomllib``
    * Python 3.8–3.10 → optional ``tomli`` backport (``pip install tomli``)
    * Neither available → raise ``RuntimeError`` so the caller can fall back
      to ``instances.txt`` gracefully.

    This is required because the project claims Python 3.8+ support but
    ``tomllib`` only entered the stdlib in 3.11.
    """
    try:
        import tomllib  # Python 3.11+
    except ModuleNotFoundError:
        try:
            import tomli as tomllib  # type: ignore[no-redef]
        except ModuleNotFoundError:
            raise RuntimeError(
                "Reading .toml config requires Python 3.11+ or 'pip install tomli'"
            )
    with open(path, "rb") as f:
        return tomllib.load(f)


def _read_instance_file(path: Path) -> list:
    """Read instance URLs from a config file.

    Supports two formats:
      * ``searxng.toml``  — ``instance = "url"`` or ``instances = ["a", "b"]``
                            under a ``[searxng]`` table (or at the top level).
                            Requires Python 3.11+ or the ``tomli`` backport.
      * ``*.txt``         — one URL per line, comma-separated lists allowed,
                            ``#`` starts a comment.
    Returns an empty list if the file cannot be parsed.
    """
    try:
        if path.suffix == ".toml":
            data = _load_toml(path)
            table = data.get("searxng", data)
            raw = table.get("instance") or table.get("instances")
            if isinstance(raw, str):
                return parse_instances(raw)
            if isinstance(raw, list):
                return [u.rstrip("/") if u.startswith(("http://", "https://"))
                        else "https://" + u.rstrip("/")
                        for u in raw if u and str(u).strip()]
            return []
        # Plain text: one URL per line (comma lists allowed), # comments
        out = []
        for line in path.read_text(encoding="utf-8").splitlines():
            line = line.strip()
            if not line or line.startswith("#"):
                continue
            out.extend(parse_instances(line))
        return out
    except RuntimeError as e:
        # tomllib 缺失（Python 3.8-3.10 未装 tomli）是可恢复的——可改用
        # instances.txt——但必须明确提示用户，而不是静默返回空列表让 main
        # 报 "no instance resolved"，让用户困惑真正的失败原因。
        msg = str(e).lower()
        if "toml" in msg and ("3.11" in msg or "tomli" in msg):
            logger.error(f"Cannot parse '{path}': {e} "
                         f"(consider 'pip install tomli' or use instances.txt)")
        else:
            logger.warning(f"Warning: cannot read instance file '{path}': {e}")
        return []
    except Exception as e:
        logger.warning(f"Warning: cannot read instance file '{path}': {e}")
        return []


def _windows_appdata_config_dir() -> Path:
    """Return the Windows APPDATA config directory, or a sentinel Path if unset.

    On Windows, the conventional per-user app config directory is
    ``%APPDATA%`` (typically ``C:\\Users\\<user>\\AppData\\Roaming``).
    On POSIX, this env var is unset and we return a sentinel
    ``Path("/__no_appdata__")`` which never exists on disk, so the caller
    can unconditionally append it to the candidate list without polluting
    Linux/macOS lookups.

    Note: ``Path("")`` resolves to ``.`` (current directory) on Windows,
    which DOES exist — so we must use an absolute sentinel path instead.
    """
    appdata = os.environ.get("APPDATA", "")
    if appdata:
        return Path(appdata) / "searxng-cli"
    # Sentinel: absolute path that never exists. Using "/" + unlikely name
    # keeps it false on both POSIX and Windows (where "/" is the drive root).
    return Path("/__no_appdata__")


def resolve_instances(cli_arg: str = None) -> list:
    """Resolve instance URLs from (in priority order):

    1. ``--instance`` CLI flag (comma-separated list)
    2. ``SEARXNG_INSTANCE`` environment variable (comma-separated list)
    3. config file search order:
       a. ``./searxng.toml``
       b. ``~/.config/searxng-cli/searxng.toml``
       c. ``%APPDATA%/searxng-cli/searxng.toml``  (Windows only)
       d. ``./instances.txt``
       e. ``~/.config/searxng-cli/instances.txt``
       f. ``%APPDATA%/searxng-cli/instances.txt``  (Windows only)

    Returns an empty list if no instance can be resolved.
    """
    if cli_arg:
        return parse_instances(cli_arg)

    env = os.environ.get("SEARXNG_INSTANCE")
    if env:
        return parse_instances(env)

    win_dir = _windows_appdata_config_dir()
    candidates = [
        Path.cwd() / "searxng.toml",
        Path.home() / ".config" / "searxng-cli" / "searxng.toml",
        win_dir / "searxng.toml",
        Path.cwd() / "instances.txt",
        Path.home() / ".config" / "searxng-cli" / "instances.txt",
        win_dir / "instances.txt",
    ]
    for p in candidates:
        if p.exists():
            urls = _read_instance_file(p)
            if urls:
                return urls
    return []


def load_config(config_path: str = None) -> dict:
    """Load the full config dict from ``searxng.toml`` (if present).

    If ``config_path`` is given (from ``--config``), only that file is
    consulted. Otherwise the default search order applies:
      1. ``./searxng.toml``
      2. ``~/.config/searxng-cli/searxng.toml``

    Returns the ``[searxng]`` table (or top-level table if no section),
    which may contain any of these keys used as CLI defaults:

      * ``instance`` / ``instances`` — instance URL(s)
      * ``timeout``, ``max_retries`` — network tuning
      * ``engines``, ``categories``, ``language`` — search scope
      * ``safesearch``, ``time_range``, ``method`` — search behavior
      * ``format`` — output format (json/brief/urls/csv)
      * ``sort_by`` — result sort key
      * ``proxy`` — proxy URL
      * ``cache_ttl`` — cache TTL in minutes

    Returns an empty dict if no config file exists or it cannot be parsed.
    Only ``.toml`` files are consulted for full config; ``.txt`` files
    only carry instance URLs (handled by :func:`resolve_instances`).
    """
    if config_path:
        p = Path(config_path)
        if not p.exists():
            logger.warning(f"Warning: config file '{p}' not found")
            return {}
        try:
            data = _load_toml(p)
            return data.get("searxng", data)
        except Exception as e:
            logger.warning(f"Warning: cannot read config '{p}': {e}")
            return {}
    win_dir = _windows_appdata_config_dir()
    candidates = [
        Path.cwd() / "searxng.toml",
        Path.home() / ".config" / "searxng-cli" / "searxng.toml",
        win_dir / "searxng.toml",
    ]
    for p in candidates:
        if p.exists():
            try:
                data = _load_toml(p)
                return data.get("searxng", data)
            except Exception as e:
                logger.warning(f"Warning: cannot read config '{p}': {e}")
                return {}
    return {}


def _cfg_int(config: dict, key: str, default: int) -> int:
    """Read an int from config, tolerating str/int forms and bad values.

    argparse's ``type=int`` only converts *command-line* strings; it does NOT
    convert a ``default`` value that came in as a string from TOML. So a
    ``timeout = "15"`` in ``searxng.toml`` would leak through as a str and
    break later numeric comparisons. This helper normalizes that.
    """
    if key not in config:
        return default
    try:
        return int(config[key])
    except (TypeError, ValueError):
        return default


# ----- Retry logic -----

def _retry_with_backoff(fn, max_retries: int = MAX_RETRIES, base_delay: float = RETRY_BACKOFF_BASE):
    """Call fn with exponential backoff + jitter on transient failures."""
    last_error = None
    for attempt in range(max_retries + 1):
        try:
            return fn()
        except urllib.error.HTTPError as e:
            if e.code in RETRYABLE_STATUS:  # 403 (UA block) + 429 + 5xx
                last_error = e
                if attempt < max_retries:
                    delay = base_delay * (2 ** attempt) + random.uniform(0, 1)
                    logger.info(f"  HTTP {e.code}, retrying in {delay:.1f}s... (attempt {attempt+1}/{max_retries})")
                    time.sleep(delay)
                    continue
            raise
        except (urllib.error.URLError, OSError) as e:
            last_error = e
            if attempt < max_retries:
                delay = base_delay * (2 ** attempt) + random.uniform(0, 1)
                logger.info(f"  Connection error ({e}), retrying in {delay:.1f}s...")
                time.sleep(delay)
                continue
            raise
    raise last_error


# ----- Search execution -----

def search_json(instance: str, params: dict, method: str = "GET",
                timeout: int = 15, auth_headers: dict = None) -> dict:
    """Execute search via JSON API. Returns None if JSON unsupported."""
    query_string = urllib.parse.urlencode(params)
    headers = _merge_headers({"User-Agent": USER_AGENT}, auth_headers)

    if method.upper() == "POST":
        data = query_string.encode("utf-8")
        req = urllib.request.Request(f"{instance}/search", data=data, headers=headers, method="POST")
    else:
        req = urllib.request.Request(f"{instance}/search?{query_string}", headers=headers)

    try:
        with urllib.request.urlopen(req, timeout=timeout) as resp:
            raw = resp.read().decode("utf-8")
            if raw.strip().startswith("{") or raw.strip().startswith("["):
                return json.loads(raw)
            # Got HTML — JSON unsupported
            return None
    except urllib.error.HTTPError as e:
        # 404 = JSON endpoint truly absent → fall back to HTML scraping
        if e.code == 404:
            return None
        # 403 / 401 = auth or IP issue → raise so it isn't silently masked
        # by an HTML fallback that would just 403 again. Also surfaces
        # instances that disable the JSON format via a 403 (rare but seen).
        raise


def search_html(instance: str, params: dict, timeout: int = 15,
                auth_headers: dict = None) -> dict:
    """Execute search via HTML scraping fallback."""
    html_params = {k: v for k, v in params.items() if k != "format"}
    query_string = urllib.parse.urlencode(html_params)
    url = f"{instance}/search?{query_string}"

    headers = _merge_headers({"User-Agent": USER_AGENT}, auth_headers)
    req = urllib.request.Request(url, headers=headers)
    try:
        with urllib.request.urlopen(req, timeout=timeout) as resp:
            html = resp.read().decode("utf-8")
            return parse_html_results(html, query=params.get("q", ""))
    except Exception as e:
        raise RuntimeError(f"HTML search failed for {instance}: {e}")


def search_single(instance: str, params: dict, method: str = "GET",
                  timeout: int = 15, auth_headers: dict = None) -> dict:
    """Execute one search attempt, preferring JSON with HTML fallback."""
    result = search_json(instance, params, method=method, timeout=timeout,
                         auth_headers=auth_headers)
    if result is not None:
        return result
    logger.warning(f"Warning: {instance} does not support format=json, falling back to HTML parsing")
    return search_html(instance, params, timeout=timeout, auth_headers=auth_headers)


def search_multi(instance_urls: list, params: dict, method: str = "GET",
                 timeout: int = 15, retry_per: int = None,
                 auth_headers: dict = None, parallel: bool = True) -> dict:
    """Search across multiple instances, failing over on error.

    With parallel=True (default, multi-instance only): every instance is
    queried concurrently. The first *successful* result in the user's
    original instance order is returned — this keeps output deterministic
    (same input list always yields the same source) while drastically
    speeding up failover when an early instance is down or slow.

    With parallel=False (or a single instance): strictly sequential, one
    request at a time, trying the next instance only after the current fails.
    """
    if retry_per is None:
        retry_per = MAX_RETRIES
    instance_urls = [u.rstrip("/") for u in instance_urls]

    # Single instance, or explicit serial mode: straightforward sequential path
    if len(instance_urls) <= 1 or not parallel:
        last_error = None
        for instance in instance_urls:
            logger.info(f"Trying {instance}...")
            emit_progress("instance_try", url=instance, attempt=1)
            start = time.time()
            try:
                def _do():
                    return search_single(instance, params, method=method,
                                         timeout=timeout, auth_headers=auth_headers)
                result = _retry_with_backoff(_do, max_retries=retry_per)
                emit_progress("instance_ok", url=instance,
                              latency=round(time.time() - start, 3),
                              results=len(result.get("results", [])) if result else 0)
                return result
            except Exception as e:
                last_error = e
                logger.info(f"  Failed: {e}")
                emit_progress("instance_fail", url=instance,
                              error=str(e), error_code=classify_error(e))
                continue
        raise RuntimeError(f"All {len(instance_urls)} instances failed. Last error: {last_error}")

    # Parallel path: one search per instance concurrently (each with its own
    # backoff retries), then return the first success in original order.
    logger.info(f"Trying {len(instance_urls)} instances in parallel...")

    def _task(instance: str):
        emit_progress("instance_try", url=instance, attempt=1)
        start = time.time()
        def _do():
            return search_single(instance, params, method=method,
                                 timeout=timeout, auth_headers=auth_headers)
        try:
            result = _retry_with_backoff(_do, max_retries=retry_per)
            emit_progress("instance_ok", url=instance,
                          latency=round(time.time() - start, 3),
                          results=len(result.get("results", [])) if result else 0)
            return instance, result
        except Exception as e:
            emit_progress("instance_fail", url=instance,
                          error=str(e), error_code=classify_error(e))
            return instance, e

    results_by_url = {}
    last_parallel_error = None
    with ThreadPoolExecutor(max_workers=min(len(instance_urls), 8)) as ex:
        futures = {ex.submit(_task, u): u for u in instance_urls}
        for fut in as_completed(futures):
            u = futures[fut]
            try:
                inst, res = fut.result()
            except Exception:
                continue
            if isinstance(res, Exception):
                # 保留最后一个失败详情，让 classify_error 能从消息中提取
                # 真实错误类型（401/403→E_AUTH，429→E_RATE_LIMIT 等），
                # 而不是一律误判为 E_NETWORK。
                last_parallel_error = res
                logger.info(f"  Failed {u}: {res}")
            else:
                results_by_url[u] = res

    # Deterministic: return first successful in the user-supplied order
    for u in instance_urls:
        if u in results_by_url:
            return results_by_url[u]

    raise RuntimeError(f"All {len(instance_urls)} instances failed (parallel). "
                       f"Last error: {last_parallel_error}")


# ----- Output formatting -----

# ----- Instance health verification (E5) -----

def _probe_config_endpoint(instance: str, timeout: int,
                           auth_headers: dict = None) -> dict:
    """Probe ``/config``; return engines list and categories if reachable.

    SearXNG's ``/config`` endpoint exposes the instance's engine list,
    categories, and other settings as JSON. Some instances disable it; in
    that case we return ``reachable=False`` with an empty engine list.
    """
    headers = _merge_headers({"User-Agent": USER_AGENT}, auth_headers)
    req = urllib.request.Request(f"{instance}/config", headers=headers)
    try:
        with urllib.request.urlopen(req, timeout=timeout) as resp:
            raw = resp.read().decode("utf-8")
            data = json.loads(raw)
            engines = [e.get("name") for e in data.get("engines", [])
                       if isinstance(e, dict) and e.get("name")]
            cats = data.get("categories", {})
            if isinstance(cats, dict):
                categories = list(cats.keys())
            elif isinstance(cats, list):
                categories = [c for c in cats if isinstance(c, str)]
            else:
                categories = []
            return {"reachable": True, "engines": engines,
                    "categories": categories, "error": None}
    except urllib.error.HTTPError as e:
        return {"reachable": False, "engines": [], "categories": [],
                "error": f"HTTP {e.code}"}
    except Exception as e:
        return {"reachable": False, "engines": [], "categories": [],
                "error": str(e)[:80]}


def verify_instances(instance_urls: list, timeout: int = 15,
                     auth_headers: dict = None, concurrency: int = 5) -> list:
    """Health-check user-supplied instances: reachability, JSON support, latency,
    POST support, /config endpoint, engine list, and auth status.

    Replaces the removed public-instance discovery: you bring your own
    instances, and ``--verify`` tells you which are alive, which support the
    JSON API, which accept POST, what engines they expose, and how fast
    each responds.

    Returns a list of dicts (one per instance, in input order), each with:
      url, reachable (bool), json_supported (bool|None), post_supported (bool|None),
      config_endpoint (bool|None), engines (list[str]), latency (float|None),
      result_count (int|None), auth_status (str), error (str|None)
    """
    instance_urls = [u.rstrip("/") for u in instance_urls]
    has_auth = bool(auth_headers)

    def _check(u: str) -> dict:
        start = time.time()
        # 1. GET /search?q=test&format=json — the primary reachability probe
        try:
            data = search_json(u, {"q": "test", "format": "json"}, "GET",
                               timeout=timeout, auth_headers=auth_headers)
            latency = round(time.time() - start, 3)
            auth_status = "ok" if has_auth else "n/a"

            if data is None:
                # Instance returned HTML — JSON format disabled but reachable
                base = {"url": u, "reachable": True, "json_supported": False,
                        "latency": latency, "result_count": None,
                        "auth_status": auth_status,
                        "error": "returned HTML (format=json unsupported)"}
            else:
                base = {"url": u, "reachable": True, "json_supported": True,
                        "latency": latency,
                        "result_count": len(data.get("results", [])),
                        "auth_status": auth_status, "error": None}

            # 2. POST probe — only meaningful when JSON works
            if data is not None:
                try:
                    post_data = search_json(u, {"q": "test", "format": "json"}, "POST",
                                            timeout=timeout, auth_headers=auth_headers)
                    base["post_supported"] = post_data is not None
                except urllib.error.HTTPError as e:
                    base["post_supported"] = False
                    base["error"] = (base.get("error") or "") + f" POST: HTTP {e.code}"
                except Exception as e:
                    base["post_supported"] = False
                    base["error"] = (base.get("error") or "") + f" POST: {str(e)[:60]}"
            else:
                base["post_supported"] = None

            # 3. /config probe — engine list (independent of JSON search)
            cfg = _probe_config_endpoint(u, timeout, auth_headers)
            base["config_endpoint"] = cfg["reachable"]
            base["engines"] = cfg["engines"]
            if cfg["error"] and not cfg["reachable"]:
                # /config down is non-fatal; note it but don't overwrite search error
                base["error"] = (base.get("error") or "") + f" /config: {cfg['error']}"

            return base
        except urllib.error.HTTPError as e:
            # Distinguish auth rejection (401/403) from other HTTP errors so
            # users know whether their token is wrong or the instance is down.
            auth_status = ("rejected" if has_auth and e.code in (401, 403)
                           else ("ok" if has_auth else "n/a"))
            # 5xx 是服务器错误，实例虽然响应了但不可用，应视为不可达。
            # 400 可能只是请求格式问题，实例本身在线，仍算可达。
            is_5xx = 500 <= e.code < 600
            return {"url": u, "reachable": e.code not in (401, 403, 404) and not is_5xx,
                    "json_supported": False, "post_supported": None,
                    "config_endpoint": None, "engines": [],
                    "latency": round(time.time() - start, 3),
                    "result_count": None, "auth_status": auth_status,
                    "error": f"HTTP {e.code}"}
        except Exception as e:
            return {"url": u, "reachable": False, "json_supported": False,
                    "post_supported": None, "config_endpoint": None,
                    "engines": [], "latency": None, "result_count": None,
                    "auth_status": "unknown" if has_auth else "n/a",
                    "error": str(e)[:120]}

    out = {}
    with ThreadPoolExecutor(max_workers=min(concurrency, len(instance_urls))) as ex:
        futures = {ex.submit(_check, u): u for u in instance_urls}
        for fut in as_completed(futures):
            u = futures[fut]
            try:
                out[u] = fut.result()
            except Exception as e:
                out[u] = {"url": u, "reachable": False, "json_supported": False,
                          "post_supported": None, "config_endpoint": None,
                          "engines": [], "latency": None, "result_count": None,
                          "auth_status": "unknown" if has_auth else "n/a",
                          "error": str(e)[:120]}
    # Preserve input order
    return [out[u] for u in instance_urls]


def _print_verify_report(report: list, as_json: bool):
    """Print a health-check report (human table or JSON).

    The table shows one line per instance with the most important signals
    (reach / JSON / POST / latency / result count / engine count / auth).
    The full engine list per instance is only included in the JSON output
    to keep the table readable.
    """
    if as_json:
        print(json.dumps(report, indent=2, ensure_ascii=False))
        return
    header = (f"{'URL':<38} {'REACH':<6} {'JSON':<5} {'POST':<5} "
              f"{'LAT':<8} {'RES':<5} {'ENGS':<5} {'AUTH':<9} ERROR")
    lines = [header, "-" * len(header)]
    for r in report:
        reach = "yes" if r["reachable"] else "NO"
        js = "-" if r["json_supported"] is None else ("yes" if r["json_supported"] else "no")
        post = "-" if r.get("post_supported") is None else ("yes" if r["post_supported"] else "no")
        lat = f"{r['latency']}s" if r["latency"] is not None else "-"
        rc = "-" if r["result_count"] is None else str(r["result_count"])
        engs = str(len(r.get("engines", []))) if r.get("engines") is not None else "-"
        auth = r.get("auth_status", "-")
        err = (r["error"] or "").replace("\n", " ")[:40]
        lines.append(f"{r['url'][:37]:<38} {reach:<6} {js:<5} {post:<5} "
                     f"{lat:<8} {rc:<5} {engs:<5} {auth:<9} {err}")
    ok = sum(1 for r in report if r["reachable"])
    lines.append("-" * len(header))
    lines.append(f"{ok}/{len(report)} instances reachable")
    print("\n".join(lines))


# ----- Auto-fetch result pages -----
# NOTE: HTTP transport (retry, charset, UA fallback, size limit) is delegated
# to fetch.fetch_url — this module only post-processes its result. This
# eliminates ~70 lines of duplicated HTTP code that previously drifted
# between search.fetch_page and fetch.fetch_url.

def fetch_page(url: str, timeout: int = 10, auth_headers: dict = None,
               max_retries: int = 3, max_size: int = None) -> dict:
    """Fetch a single page; returns metadata dict with 'status'='ok' or 'error'.

    Thin wrapper around :func:`fetch.fetch_url` that adds:
      * CAPTCHA / bot-block detection (marks result as error)
      * automatic text extraction via :func:`fetch.extract_text`
      * dict-shaped return suitable for the auto-fetch feature

    All HTTP transport concerns (retry, charset, UA fallback, size limit)
    are handled by ``fetch_url``.
    """
    try:
        result = fetch_url(
            url, timeout=timeout, auth_headers=auth_headers,
            max_retries=max_retries, max_size=max_size,
            allow_redirects=True,
        )
    except Exception as e:
        msg = str(e) if str(e) else e.__class__.__name__
        return {
            "url": url, "status": "error", "error": msg,
            "text": "", "text_length": 0, "truncated": False,
        }

    content = result.content
    content_type = result.content_type
    final_url = result.final_url

    is_html = ("html" in content_type.lower() or
               content.strip().startswith("<!") or
               content.strip().startswith("<htm"))

    # Detect CAPTCHA / bot-block pages (don't retry — fetch_url already
    # exhausted UA fallback inside its retry loop).
    if is_html and _is_blocked_page(content):
        return {
            "url": url, "final_url": final_url, "status": "error",
            "error": "Bot protection detected (CAPTCHA / challenge page)",
            "text": "", "text_length": 0, "truncated": False,
        }

    text = extract_text(content) if is_html else content
    return {
        "url": url,
        "final_url": final_url,
        "status": "ok",
        "content_type": content_type,
        "text": text,
        "text_length": len(text),
        "truncated": result.truncated,
        "truncated_at": max_size if result.truncated else None,
        "user_agent_used": result.user_agent,
    }


def _is_blocked_page(content: str) -> bool:
    """Quick heuristic to detect bot-protection pages."""
    lower = content[:2000].lower()
    indicators = [
        "captcha", "challenge", "verify you are human",
        "checking your browser", "making sure you're not a bot",
        "cf-browser-verification", "anubis_challenge",
        "please enable javascript", "enable javascript to continue",
        "just a moment", "ddos protection",
    ]
    return any(ind in lower for ind in indicators)


def fetch_top_results(results: dict, count: int, timeout: int = 10,
                       concurrency: int = 5, auth_headers: dict = None,
                       max_retries: int = 3, max_size: int = None,
                       request_delay: float = 0.3) -> list:
    """Fetch full text of top N result pages concurrently.

    Features:
    - Retries transient errors with exponential backoff
    - Falls back to browser User-Agent if blocked
    - Small delay between requests to avoid rate limits
    - 5MB size limit per page
    """
    urls = []
    seen = set()
    for r in results.get("results", []):
        url = r.get("url", "")
        if url and url not in seen:
            urls.append(url)
            seen.add(url)
        if len(urls) >= count:
            break

    if not urls:
        return []

    logger.info(f"\nFetching {len(urls)} result pages (timeout={timeout}s, retries={max_retries})...")
    fetched = []
    ok_count = [0]
    err_count = [0]

    def _fetch_one(u: str) -> dict:
        """Fetch one URL with optional delay to avoid rate limiting."""
        if request_delay > 0:
            time.sleep(request_delay * random.uniform(0.5, 1.5))
        result = fetch_page(u, timeout=timeout, auth_headers=auth_headers,
                            max_retries=max_retries, max_size=max_size)
        if result["status"] == "ok":
            ok_count[0] += 1
            trunc = ", TRUNCATED" if result.get("truncated") else ""
            ua_note = ""
            if result.get("user_agent_used") != USER_AGENT:
                ua_note = " [fallback UA]"
            logger.info(f"  [OK] {u[:55]} ({result['text_length']:,} chars{trunc}{ua_note})")
        else:
            err_count[0] += 1
            logger.error(f"  [ERR] {u[:55]} ({result.get('error', 'unknown')})")
        return result

    with ThreadPoolExecutor(max_workers=min(concurrency, len(urls))) as ex:
        future_map = {ex.submit(_fetch_one, u): u for u in urls}
        for future in as_completed(future_map):
            try:
                result = future.result()
                fetched.append(result)
            except Exception as e:
                u = future_map[future]
                fetched.append({"url": u, "status": "error", "error": str(e),
                                "text": "", "text_length": 0, "truncated": False})
                logger.error(f"  [ERR] {u[:55]} (thread error: {e})")

    # Reorder to match original result order
    url_order = {u: i for i, u in enumerate(urls)}
    fetched.sort(key=lambda f: url_order.get(f["url"], 999))

    logger.info(f"Fetched: {ok_count[0]} ok, {err_count[0]} errors")
    return fetched


# ----- Output formatting -----

def deduplicate_results(results: dict) -> dict:
    """Remove duplicate results by normalized URL, keeping first occurrence.

    Different engines often return the same URL. The first occurrence keeps
    its engine/score metadata; subsequent duplicates are dropped.

    URL normalization:
      * lowercase scheme + host
      * strip fragment (``#...``)
      * drop common tracking query params (``utm_*``, ``gclid``, ``fbclid``,
        ``mc_*``, ``ref``, ``ref_*``)
      * sort remaining params so ``?b=2&a=1`` matches ``?a=1&b=2``

    Results without a URL are kept as-is (never deduped). Mutates
    ``results['results']`` in place and returns ``results`` for chaining.
    """
    if not results.get("results"):
        return results

    # 跟踪参数前缀。裸 "ref" 过于宽泛（会误删 reference/refcode 等正常参数），
    # 收紧为 "ref_" 只匹配 ref_source/ref_campaign 等跟踪参数。
    TRACKING_PREFIXES = ("utm_", "gclid", "fbclid", "mc_", "ref_")

    def _normalize_url(url: str) -> str:
        try:
            p = urllib.parse.urlparse(url)
            scheme = p.scheme.lower()
            netloc = p.netloc.lower()
            qs = urllib.parse.parse_qsl(p.query, keep_blank_values=False)
            qs = [(k, v) for k, v in qs
                  if not any(k.lower().startswith(pref) for pref in TRACKING_PREFIXES)]
            qs.sort()
            query = urllib.parse.urlencode(qs)
            return urllib.parse.urlunparse((scheme, netloc, p.path, p.params,
                                            query, ""))
        except Exception:
            return url

    seen = set()
    deduped = []
    for r in results["results"]:
        url = r.get("url", "")
        key = _normalize_url(url) if url else ""
        if key and key in seen:
            continue
        if key:
            seen.add(key)
        deduped.append(r)

    before = len(results["results"])
    results["results"] = deduped
    if before != len(deduped):
        logger.info(f"Dedup: {before} -> {len(deduped)} results (by normalized URL)")
    return results


def sort_results(results: dict, sort_by: str) -> dict:
    """Sort results by the given key.

    * ``score``  — descending (highest first); entries without score keep
                   their relative order at the end.
    * ``date``   — descending (newest first) by ``published_date`` (ISO
                   strings sort lexicographically); entries without a date
                   keep their relative order at the end.
    * ``engine`` — ascending (alphabetical by engine name).
    * ``none``   — no-op (preserve original order).

    Mutates ``results['results']`` in place and returns ``results``.
    """
    if not results.get("results") or sort_by == "none":
        return results

    rs = results["results"]
    if sort_by == "score":
        with_score = [r for r in rs if r.get("score") is not None]
        without = [r for r in rs if r.get("score") is None]
        with_score.sort(key=lambda r: r["score"], reverse=True)
        results["results"] = with_score + without
    elif sort_by == "date":
        with_date = [r for r in rs if r.get("published_date")]
        without = [r for r in rs if not r.get("published_date")]
        with_date.sort(key=lambda r: r["published_date"], reverse=True)
        results["results"] = with_date + without
    elif sort_by == "engine":
        rs.sort(key=lambda r: r.get("engine") or "")
        results["results"] = rs
    return results


def filter_results_by_domain(results: dict, include_domains: list = None,
                              exclude_domains: list = None) -> dict:
    """Filter search results by domain (allowlist or blocklist).

    Modifies ``results['results']`` in place and returns ``results`` for
    chaining. Matching is case-insensitive and ignores a leading ``www.``
    so ``example.com`` matches both ``example.com`` and ``www.example.com``.

    ``include_domains`` (allowlist) takes precedence: if non-empty, only
    results whose domain matches are kept. ``exclude_domains`` (blocklist)
    then drops any remaining matches. Empty inputs are no-ops.
    """
    if not results.get("results"):
        return results
    if not include_domains and not exclude_domains:
        return results

    def _get_domain(url: str) -> str:
        try:
            return urllib.parse.urlparse(url).netloc.lower()
        except Exception:
            return ""

    def _normalize(d: str) -> str:
        return d.lower().lstrip(".").lstrip("www.")

    include_set = {_normalize(d) for d in include_domains} if include_domains else None
    exclude_set = {_normalize(d) for d in exclude_domains} if exclude_domains else None

    filtered = []
    for r in results["results"]:
        domain = _get_domain(r.get("url", ""))
        # Strip leading www. for matching
        match_domain = domain[4:] if domain.startswith("www.") else domain

        if include_set is not None and match_domain not in include_set:
            continue
        if exclude_set is not None and match_domain in exclude_set:
            continue
        filtered.append(r)

    results["results"] = filtered
    return results


def format_brief(results: dict, snippet_len: int = 0) -> str:
    """Format results as brief text. snippet_len=0 means no truncation."""
    lines = []
    for i, r in enumerate(results.get("results", []), 1):
        title = r.get("title", "No title")
        url = r.get("url", "")
        content = r.get("content", "")
        lines.append(f"{i}. {title}")
        lines.append(f"   {url}")
        if content:
            if snippet_len > 0:
                content = content[:snippet_len]
            lines.append(f"   {content}")
        lines.append("")

    # Include suggestions if present
    sugs = results.get("suggestions", [])
    if sugs:
        lines.append(f"Suggestions: {', '.join(sugs)}")

    # Include answers if present
    for ans in results.get("answers", []):
        lines.append(f"Answer: {ans}")

    return "\n".join(lines)


def format_urls(results: dict) -> str:
    """Format results as plain URL list."""
    return "\n".join(r.get("url", "") for r in results.get("results", []) if r.get("url"))


def _format_results(results: dict, args) -> str:
    """Format a results dict into the output string selected by args.format.

    Pulled out of :func:`main` so the batch runner (--queries-file) can reuse
    the exact same formatting for each per-query block.
    """
    if args.format == "json":
        results["schema_version"] = SCHEMA_VERSION
        return json.dumps(results, indent=2, ensure_ascii=False)
    if args.format == "urls":
        return format_urls(results)
    if args.format == "csv":
        import csv as csv_mod
        import io
        out = io.StringIO()
        writer = csv_mod.writer(out, lineterminator="\n")
        writer.writerow(["title", "url", "engine", "score",
                         "published_date", "content"])
        for r in results.get("results", []):
            writer.writerow([
                r.get("title", ""),
                r.get("url", ""),
                r.get("engine", ""),
                r.get("score", "") if r.get("score") is not None else "",
                r.get("published_date", ""),
                r.get("content", ""),
            ])
        return out.getvalue().rstrip()
    # brief (also the safe fallthrough)
    output = format_brief(results, snippet_len=args.snippet_len)
    if args.fetch > 0 and results.get("fetched"):
        output += "\n\n" + "=" * 60 + "\n"
        output += f"FETCHED PAGES ({len(results['fetched'])} pages)\n"
        output += "=" * 60 + "\n"
        for f in results["fetched"]:
            output += f"\n--- {f['url']} ---\n"
            if f["status"] == "ok":
                output += f"{f['text']}\n"
            else:
                output += f"[ERROR: {f.get('error', 'unknown')}]\n"
    return output


def _build_params(query: str, args) -> dict:
    """Build the SearXNG API params dict for one query.

    Centralized so the single-query path and the --queries-file batch path
    construct identical params (and thus share cache entries).
    """
    params = {"q": query, "format": "json"}
    if args.categories:
        params["categories"] = _normalize_csv(args.categories)
    if args.language:
        params["language"] = args.language
    if args.pageno != 1:
        params["pageno"] = str(args.pageno)
    if args.time_range and args.time_range != "none":
        params["time_range"] = args.time_range
    if args.safesearch is not None:
        params["safesearch"] = str(args.safesearch)
    if args.engines:
        params["engines"] = _normalize_csv(args.engines)
    return params


def _run_single_query(query: str, args, instance_urls: list,
                      auth_headers: dict, ttl_seconds: int):
    """Run one query end-to-end: search → limit → domain-filter → fetch.

    Returns ``(results_dict, error_str, error_code)``。成功时后两者为 None。
    ``error_code`` 是结构化错误码（E_NETWORK/E_AUTH 等），让 AI Agent
    程序化判断错误类型。Cache hits 跳过网络。后处理（limit/filter/fetch）
    总是执行，保证 batch 调用方看到与单查询一致的形状。
    """
    params = _build_params(query, args)

    emit_progress("start", query=query, instances=len(instance_urls))

    cached = cache_module.get(params, ttl_seconds) if ttl_seconds > 0 else None
    if cached is not None:
        logger.info(f"[cache hit] q={query!r} TTL={args.cache_ttl}min, skipping network")
        emit_progress("cache_hit", query=query, ttl=args.cache_ttl)
        results = cached
    else:
        try:
            results = search_multi(
                instance_urls, params,
                method=args.method,
                timeout=args.timeout,
                retry_per=args.retry,
                auth_headers=auth_headers,
                parallel=not args.serial,
            )
        except Exception as e:
            err_code = classify_error(e)
            emit_progress("error", error=str(e), error_code=err_code, query=query)
            return None, str(e), err_code
        if ttl_seconds > 0:
            cache_module.put(params, results, ttl_seconds)
            logger.info(f"[cache stored] q={query!r} TTL={args.cache_ttl}min")
            emit_progress("cache_store", query=query, ttl=args.cache_ttl)

    # Dedup (default on; --no-dedup disables) then sort, both BEFORE limit
    # so --max-results keeps the highest-scoring / newest items.
    if not args.no_dedup:
        deduplicate_results(results)
    sort_results(results, args.sort_by)

    if args.max_results and "results" in results:
        results["results"] = results["results"][:args.max_results]

    if args.include_domain or args.exclude_domain:
        include_list = ([d.strip() for d in args.include_domain.split(",") if d.strip()]
                        if args.include_domain else None)
        exclude_list = ([d.strip() for d in args.exclude_domain.split(",") if d.strip()]
                        if args.exclude_domain else None)
        before = len(results.get("results", []))
        filter_results_by_domain(results, include_domains=include_list,
                                 exclude_domains=exclude_list)
        after = len(results.get("results", []))
        logger.info(f"Domain filter: {before} -> {after} results")

    if args.fetch > 0 and results.get("results"):
        emit_progress("fetch_start", count=args.fetch)
        fetched = fetch_top_results(
            results, args.fetch,
            timeout=args.fetch_timeout,
            auth_headers=auth_headers,
            max_retries=args.fetch_retries,
            max_size=args.max_size,
        )
        # Emit fetch_ok / fetch_fail events
        for f in fetched:
            if f.get("status") == "ok":
                emit_progress("fetch_ok", url=f.get("url", ""),
                              chars=f.get("text_length", 0))
            else:
                emit_progress("fetch_fail", url=f.get("url", ""),
                              error=f.get("error", "unknown"))
        results["fetched"] = fetched
        results["fetched_source"] = results.get("_fallback", "json")

    result_count = len(results.get("results", []))
    emit_progress("done", results=result_count, query=query)
    # 清理内部 _fallback 字段，避免泄漏到 JSON 输出。
    # fetched_source（对 AI 有用的公开字段）已在 --fetch 路径中设置。
    results.pop("_fallback", None)
    return results, None, None


def _get_output_schema():
    """Return the JSON Schema describing --format json output.

    Used by ``--dump-schema`` so AI agents can programmatically discover the
    output structure without parsing prose documentation.
    """
    return {
        "$schema": "https://json-schema.org/draft/2020-12/schema",
        "title": "SearXNG CLI Search Result",
        "schema_version": SCHEMA_VERSION,
        "description": "Output schema for 'python search.py --format json' (single query). "
                       "Batch mode (--queries-file) wraps results in "
                       '{"schema_version, queries:[]}.',
        "type": "object",
        "properties": {
            "schema_version": {
                "type": "string",
                "const": SCHEMA_VERSION,
                "description": "Output schema version. Bump on breaking field changes.",
            },
            "query": {"type": "string", "description": "The search query string."},
            "number_of_results": {
                "type": "integer",
                "description": "Total matches reported by SearXNG (JSON path) or "
                               "count of parsed results (HTML fallback path).",
            },
            "results": {
                "type": "array",
                "description": "Search result items, ordered by relevance (score desc).",
                "items": {
                    "type": "object",
                    "properties": {
                        "title": {"type": "string"},
                        "url": {"type": "string", "format": "uri"},
                        "engine": {"type": "string", "description": "Source engine name."},
                        "score": {"type": ["number", "null"]},
                        "published_date": {"type": ["string", "null"]},
                        "content": {"type": "string", "description": "Snippet/summary text."},
                    },
                    "required": ["title", "url"],
                },
            },
            "unresponsive_engines": {
                "type": "array",
                "items": {"type": "string"},
                "description": "Engines that failed to respond.",
            },
            "suggestions": {
                "type": "array",
                "items": {"type": "string"},
                "description": "Related query suggestions from the instance.",
            },
            "fetched": {
                "type": "array",
                "description": "Present only when --fetch N is used. Page content "
                               "for the top N results.",
                "items": {
                    "type": "object",
                    "properties": {
                        "url": {"type": "string"},
                        "text": {"type": "string"},
                        "text_length": {"type": "integer"},
                        "error": {"type": "string"},
                    },
                },
            },
            "fetched_source": {
                "type": "string",
                "enum": ["json", "html"],
                "description": "Present only when --fetch is used. Indicates whether "
                               "search results came from JSON API or HTML fallback.",
            },
        },
        "required": ["query", "results"],
    }


def _emit_error(message: str, args, query: str = None, exit_code: int = 1,
                error_code: str = None):
    """Emit an error and exit.

    In ``--format json`` mode the error is printed to **stdout** as a
    structured JSON object so agents piping stdout can parse it. All other
    formats print to stderr (keeping stdout clean for data) and exit.

    The JSON shape is::

        {"error": "...", "exit_code": N, "error_code": "E_*",
         "recovery_hint": "...", "query": "..."}

    ``error_code`` 是结构化错误码（E_CONFIG/E_AUTH/E_NETWORK 等），让 AI
    Agent 程序化判断错误类型并采取恢复策略。``recovery_hint`` 给出可操作的
    恢复建议，让 AI 能自决策下一步动作。``query`` 仅在提供时包含。
    """
    if getattr(args, "format", None) == "json":
        payload = {"error": message, "exit_code": exit_code}
        if error_code:
            payload["error_code"] = error_code
            hint = RECOVERY_HINTS.get(error_code)
            if hint:
                payload["recovery_hint"] = hint
        if query:
            payload["query"] = query
        print(json.dumps(payload, indent=2, ensure_ascii=False))
    else:
        prefix = f"[query: {query}] " if query else ""
        code_prefix = f"[{error_code}] " if error_code else ""
        hint_suffix = ""
        if error_code and error_code in RECOVERY_HINTS:
            hint_suffix = f"\n  Hint: {RECOVERY_HINTS[error_code]}"
        logger.error(f"{prefix}{code_prefix}Error: {message}{hint_suffix}")
    sys.exit(exit_code)


def _read_queries_file(path: str) -> list:
    """Read queries from a file: one per line, skip blanks and ``#`` comments.

    Raises :class:`RuntimeError` if the file cannot be read, so the caller
    can route it through :func:`_emit_error`.
    """
    try:
        text = Path(path).read_text(encoding="utf-8")
    except OSError as e:
        raise RuntimeError(f"cannot read queries file '{path}': {e}")
    queries = []
    for line in text.splitlines():
        line = line.strip()
        if not line or line.startswith("#"):
            continue
        queries.append(line)
    return queries


# ----- Main -----

def main():
    # Phase 1: pre-scan for --config / --verbose / --quiet so logging is
    # configured BEFORE config loading (which may emit warnings).
    pre = argparse.ArgumentParser(add_help=False)
    pre.add_argument("--config", default=None)
    pre.add_argument("--verbose", "-v", action="store_true", default=False)
    pre.add_argument("--quiet", action="store_true", default=False)
    pre_args, _ = pre.parse_known_args()

    setup_logging(verbose=pre_args.verbose, quiet=pre_args.quiet)
    force_utf8_stdout()  # Windows: prevent GBK crash on non-ASCII chars

    # Load config defaults from --config file, else ./searxng.toml or
    # ~/.config/searxng-cli/searxng.toml. Every CLI flag below can be
    # pre-set here; explicit flags still win.
    config = load_config(pre_args.config)

    parser = argparse.ArgumentParser(
        description="Search via a user-supplied SearXNG instance",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog=f"""searxng-cli v{VERSION}

Examples:
  %(prog)s -q "python asyncio" -i https://my-searxng.example.com
  %(prog)s -q "machine learning" -i https://a.example.com,https://b.example.com --format brief
  %(prog)s -q "climate change" -i https://s.example.com --time-range month --language en
  %(prog)s -q "open source" -i https://s.example.com --format urls > result_urls.txt
  %(prog)s -q "ai news" -i https://s.example.com --engines google,duckduckgo --method POST
  %(prog)s --verify -i https://a.example.com,https://b.example.com   # health-check, no search
  %(prog)s -q "latest news" -i https://s.example.com --time-range none   # disable time filter
  %(prog)s -q "test" --config ./my-config.toml -i https://s.example.com  # explicit config file

Instance URLs are resolved from (in priority order):
  1. -i/--instance (comma-separated for parallel failover)
  2. SEARXNG_INSTANCE environment variable
  3. ./searxng.toml or ~/.config/searxng-cli/searxng.toml  (instance = "..." or instances = [...])
     ./instances.txt or ~/.config/searxng-cli/instances.txt  (one URL per line)
At least one instance is REQUIRED (public-instance discovery has been removed).
Use --verify to health-check your instances (reachability / JSON support / latency).

searxng.toml may ALSO set defaults for most flags below (engines, categories, language,
safesearch, time_range, method, format, sort_by, timeout, max_retries, proxy, cache_ttl,
fetch, fetch_timeout, fetch_retries, max_size). Explicit CLI flags always override config values.
Use --config FILE to load a non-default config file (overrides the auto-discovered one).
        """,
    )
    parser.add_argument("--config", default=None, metavar="FILE",
                        help="Path to a searxng.toml config file. Overrides the default "
                             "auto-discovery (./searxng.toml -> ~/.config/searxng-cli/searxng.toml). "
                             "Must be the first flag if you want its values to set defaults "
                             "for other flags.")
    parser.add_argument("--verbose", "-v", action="store_true", default=False,
                        help="Verbose output: show debug-level diagnostics (HTTP request URLs, "
                             "response codes, cache keys, etc.) on stderr")
    parser.add_argument("--quiet", action="store_true", default=False,
                        help="Quiet output: suppress progress messages and retry notices on stderr; "
                             "only warnings and errors are shown. (No short flag: -q is --query)")
    parser.add_argument("--query", "-q", required=False, default=None,
                        help="Search query (required unless --verify / --queries-file is used)")
    parser.add_argument("--instance", "-i", required=False, default=None,
                        help="SearXNG instance URL(s), comma-separated for failover. "
                             "Optional if SEARXNG_INSTANCE env var or a config file is set.")
    parser.add_argument("--categories", "-c", default=config.get("categories"),
                        help="Comma-separated categories (e.g. general,images,news)")
    parser.add_argument("--language", "-l", default=config.get("language"),
                        help="Language code (e.g. en, zh-CN, de)")
    parser.add_argument("--pageno", "-p", type=int, default=1,
                        help="Page number (default: 1)")
    parser.add_argument("--time-range", "-t", choices=["day", "month", "year", "none"],
                        default=config.get("time_range", "year"),
                        help="Time range filter (default: year; 'none' disables filtering)")
    parser.add_argument("--safesearch", "-s", type=int, choices=[0, 1, 2],
                        default=_cfg_int(config, "safesearch", 0),
                        help="Safe search: 0=off, 1=moderate, 2=strict (default: 0=off)")
    parser.add_argument("--engines",
                        default=config.get("engines", "google,bing,brave,duckduckgo,startpage,wikipedia,wikidata"),
                        help="Comma-separated engine names "
                             "(default: google,bing,brave,duckduckgo,startpage,wikipedia,wikidata)")
    parser.add_argument("--method", choices=["GET", "POST"],
                        default=config.get("method", "GET"),
                        help="HTTP method (default: GET)")
    parser.add_argument("--max-results", type=int, default=None,
                        help="Limit number of results (applied AFTER dedup+sort, "
                             "so the highest-scoring/newest items are kept)")
    parser.add_argument("--sort-by", choices=["score", "date", "engine", "none"],
                        default=config.get("sort_by", "score"),
                        help="Sort results (default: score descending; 'none' preserves "
                             "instance order). Applied after dedup, before --max-results. "
                             "HTML-fallback results have no score and keep their order.")
    parser.add_argument("--no-dedup", action="store_true",
                        help="Disable cross-engine deduplication (by default, duplicate "
                             "URLs — same page ignoring tracking params/fragment — are "
                             "collapsed, keeping the first occurrence's engine/score)")
    parser.add_argument("--format", "-f", choices=["json", "brief", "urls", "csv"],
                        default=config.get("format", "json"),
                        help="Output format (default: json). 'csv' exports "
                             "title,url,engine,score,published_date,content.")
    parser.add_argument("--snippet-len", type=int, default=0,
                        help="Snippet length in brief format (default: 0 = full, no truncation)")
    parser.add_argument("--fetch", type=int, default=_cfg_int(config, "fetch", 0), metavar="N",
                        help="After search, auto-fetch full text of top N result pages")
    parser.add_argument("--fetch-timeout", type=int, default=_cfg_int(config, "fetch_timeout", 10),
                        help="Timeout per page fetch in seconds (default: 10)")
    parser.add_argument("--fetch-retries", type=int, default=_cfg_int(config, "fetch_retries", 3),
                        help="Max retries per page fetch (default: 3)")
    parser.add_argument("--max-size", type=int, default=_cfg_int(config, "max_size", None),
                        metavar="BYTES",
                        help="Max page size in bytes (default: unlimited). Set to 5242880 for 5MB cap.")
    parser.add_argument("--auth-bearer", default=None, metavar="TOKEN",
                        help="Authorization: Bearer <TOKEN> for private instances")
    parser.add_argument("--auth-bearer-file", default=None, metavar="FILE",
                        help="Read Bearer token from a file (first non-empty, non-# line). "
                             "Avoids leaving tokens in shell history.")
    parser.add_argument("--auth-basic", default=None, metavar="USER:PASS",
                        help="Authorization: Basic base64(user:pass) for private instances")
    parser.add_argument("--auth-basic-file", default=None, metavar="FILE",
                        help="Read basic auth 'user:pass' from a file (first non-empty, non-# line). "
                             "Avoids leaving passwords in shell history. "
                             "Env var SEARXNG_BASIC_AUTH is also honored.")
    parser.add_argument("--output", "-o", default=None,
                        help="Save to file instead of stdout")
    parser.add_argument("--stream", action="store_true",
                        help="Stream results as JSON Lines (one JSON object per line) to stdout. "
                             "Each line is a {\"type\": \"result\", \"result\": {...}} event. "
                             "Ends with {\"type\": \"done\", \"count\": N}. "
                             "AI Agent can process results incrementally without waiting for full output. "
                             "Only valid with --format json.")
    parser.add_argument("--progress", action="store_true",
                        help="Emit structured progress events as JSON Lines to stderr. "
                             "Events: start, instance_try, instance_ok, instance_fail, "
                             "cache_hit, cache_store, fetch_start, fetch_ok, fetch_fail, done. "
                             "AI Agent can track execution progress programmatically.")
    parser.add_argument("--timeout", type=int, default=_cfg_int(config, "timeout", 15),
                        help="Request timeout in seconds (default: 15)")
    parser.add_argument("--retry", type=int, default=_cfg_int(config, "max_retries", None),
                        help="Max retries per instance (default: 3)")
    parser.add_argument("--fail-fast", action="store_true",
                        help="Exit on first instance failure (no multi-instance retry)")
    parser.add_argument("--serial", action="store_true",
                        help="Disable parallel multi-instance probing; search instances one at a time")
    parser.add_argument("--proxy", default=config.get("proxy"), metavar="URL",
                        help="HTTP/HTTPS proxy URL (e.g. http://corp-proxy:8080). "
                             "Applies to both search and fetch requests. "
                             "Respects existing HTTP_PROXY/HTTPS_PROXY env vars when omitted.")
    parser.add_argument("--include-domain", default=None, metavar="DOMAINS",
                        help="Comma-separated allowlist; only results from these domains are kept "
                             "(e.g. 'example.com,wikipedia.org'). Applied after search.")
    parser.add_argument("--exclude-domain", default=None, metavar="DOMAINS",
                        help="Comma-separated blocklist; results from these domains are dropped "
                             "(e.g. 'pinterest.com,quora.com'). Applied after search.")
    parser.add_argument("--queries-file", default=None, metavar="FILE",
                        help="Read queries from a file (one per line; blank lines and lines "
                             "starting with '#' are skipped) and run them in sequence. "
                             "Results are emitted as a JSON array (or one brief block per query). "
                             "Overrides --query when set.")
    parser.add_argument("--verify", action="store_true",
                        help="Health-check mode: verify instances (reachability/JSON/latency) and exit without searching")
    parser.add_argument("--cache-ttl", type=int, default=_cfg_int(config, "cache_ttl", 0),
                        metavar="MINUTES",
                        help="Cache search results for N minutes (default: 0 = disabled). "
                             "Identical queries within the TTL skip the network entirely. "
                             "Cache lives at $SEARXNG_CACHE_DIR or ~/.cache/searxng-cli/cache.db")
    parser.add_argument("--clear-cache", action="store_true",
                        help="Delete all cached entries and exit (no search performed)")
    parser.add_argument("--cache-stats", action="store_true",
                        help="Print cache statistics (entry count, age, size, path) and exit")
    parser.add_argument("--dump-schema", action="store_true",
                        help="Print the JSON Schema for --format json output and exit. "
                             "Lets AI agents programmatically discover field names and types.")
    parser.add_argument("--version", action="version", version=f"searxng-cli v{VERSION}")

    args = parser.parse_args()

    # --dump-schema：输出 JSON Schema 到 stdout 并退出，AI Agent 可程序化发现字段
    if getattr(args, "dump_schema", False):
        print(json.dumps(_get_output_schema(), indent=2, ensure_ascii=False))
        sys.exit(0)

    # 启用 --progress 进度事件（JSON Lines 到 stderr）
    set_progress_enabled(getattr(args, "progress", False))

    # --stream 只在单查询 + --format json 下有效。batch 模式输出 JSON 数组，
    # 非 json 格式无 JSON Lines 语义；两种组合都显式报错 E_INPUT，避免静默失效
    # 让 AI Agent 误以为流式输出已生效。
    if getattr(args, "stream", False):
        if args.queries_file:
            _emit_error("--stream cannot be used with --queries-file: batch mode "
                        "emits a JSON array, not JSON Lines. Drop --stream for "
                        "batch output, or use a single --query with --stream.",
                        args, error_code=E_INPUT)
        if args.format != "json":
            _emit_error(f"--stream requires --format json (current: {args.format}). "
                        "JSON Lines streaming only produces valid output with json format.",
                        args, error_code=E_INPUT)

    # --query is required unless we're doing a non-search operation.
    # --queries-file is an alternative to --query for batch mode.
    if (not args.verify and not args.query and not args.queries_file
            and not args.clear_cache and not args.cache_stats):
        parser.error("--query is required (or use --verify / --queries-file / "
                     "--clear-cache / --cache-stats)")

    # Apply proxy early so every HTTP path (search, verify, fetch) honors it.
    # Setting env vars is enough: urllib reads them via getproxies() and
    # requests honors them via trust_env=True (its default).
    if args.proxy:
        apply_proxy(args.proxy)
        logger.info(f"Proxy: {args.proxy}")

    # --clear-cache / --cache-stats 不需要实例，在实例解析之前处理并退出。
    # 避免无 -i 时报 E_CONFIG "no instance resolved" 让 AI 困惑。
    if args.clear_cache:
        removed = cache_module.clear()
        logger.info(f"Cleared {removed} cache entr{'y' if removed == 1 else 'ies'}.")
        sys.exit(0)

    if args.cache_stats:
        s = cache_module.stats()
        if args.format == "json":
            # JSON 模式：结构化数据走 stdout，AI Agent 可管道解析
            print(json.dumps(s, indent=2, ensure_ascii=False))
        else:
            # 非 JSON 模式：人类可读的状态信息走 stderr，保持 stdout 纯净，
            # 避免 AI Agent 用 --format json 解析 stdout 时被非 JSON 污染。
            print(f"Cache path:  {s.get('path', '?')}", file=sys.stderr)
            print(f"Entries:     {s.get('entries', 0)}", file=sys.stderr)
            size = s.get("size_bytes", 0)
            print(f"Size:        {size:,} bytes ({size / 1024:.1f} KB)", file=sys.stderr)
            if s.get("oldest_created_at"):
                print(f"Oldest:      {time.ctime(s['oldest_created_at'])}", file=sys.stderr)
            if s.get("newest_created_at"):
                print(f"Newest:      {time.ctime(s['newest_created_at'])}", file=sys.stderr)
            if s.get("error"):
                print(f"Error:       {s['error']}", file=sys.stderr)
        sys.exit(0)

    # Resolve instance(s): -i > SEARXNG_INSTANCE env > config file
    instance_urls = resolve_instances(args.instance)
    # Fallback: if --config was used, instance may be in the config dict
    # but not in the default search paths that resolve_instances checks.
    if not instance_urls and config.get("instance"):
        instance_urls = parse_instances(config["instance"])
    elif not instance_urls and config.get("instances"):
        raw = config["instances"]
        if isinstance(raw, str):
            instance_urls = parse_instances(raw)
        elif isinstance(raw, list):
            instance_urls = [u if u.startswith(("http://", "https://"))
                             else "https://" + u for u in raw if u and str(u).strip()]
    if not instance_urls:
        # Python 3.8-3.10 无 tomllib 时，.toml 配置文件无法读取。检查这种
        # 情况并在错误信息中附加提示，让 AI Agent 能给出可操作的恢复建议。
        toml_hint = ""
        try:
            import tomllib  # Python 3.11+
        except ModuleNotFoundError:
            try:
                import tomli  # type: ignore[import-not-found]
            except ModuleNotFoundError:
                toml_candidates = [
                    Path.cwd() / "searxng.toml",
                    Path.home() / ".config" / "searxng-cli" / "searxng.toml",
                ]
                if any(p.exists() for p in toml_candidates):
                    toml_hint = (" (hint: a searxng.toml file exists but cannot be "
                                 "read on Python < 3.11 without the 'tomli' package. "
                                 "Run 'pip install tomli' or use instances.txt instead.)")
        _emit_error("no SearXNG instance resolved. Provide -i/--instance, set the "
                    "SEARXNG_INSTANCE environment variable, or create a searxng.toml / "
                    "instances.txt config file." + toml_hint, args, error_code=E_CONFIG)

    # Build auth headers if provided (needed by both verify and search).
    # Credentials may come from CLI flag, file, config file, or env var
    # (in priority order) to avoid leaking secrets via shell history or `ps`.
    try:
        bearer_token = resolve_auth_bearer(
            args.auth_bearer, args.auth_bearer_file,
            config_value=config.get("auth_bearer"))
        basic_auth = resolve_auth_basic(
            args.auth_basic, args.auth_basic_file,
            config_value=config.get("auth_basic"))
    except RuntimeError as e:
        _emit_error(str(e), args, error_code=E_AUTH)
    auth_headers = build_auth_headers(
        bearer_token=bearer_token,
        basic_auth=basic_auth,
    )
    if auth_headers:
        auth_type = "Bearer" if bearer_token else "Basic"
        logger.info(f"Auth: {auth_type} ***")

    # Health-check mode: report instance status and exit (no search performed)
    if args.verify:
        logger.info(f"Verifying {len(instance_urls)} instance(s)...")
        report = verify_instances(instance_urls, timeout=args.timeout,
                                  auth_headers=auth_headers)
        _print_verify_report(report, as_json=(args.format == "json"))
        sys.exit(0)

    if args.fail_fast:
        instance_urls = instance_urls[:1]

    logger.info(f"Instances to try: {len(instance_urls)}")

    # Cache TTL in seconds (CLI takes minutes for ergonomics)
    ttl_seconds = args.cache_ttl * 60 if args.cache_ttl > 0 else 0

    # ----- Batch mode: --queries-file -----
    # Reads one query per line (blank/# lines skipped) and runs them in
    # sequence. Output is a JSON array (json format) or concatenated blocks
    # (brief/urls). A failed query is recorded but does not abort the batch.
    if args.queries_file:
        try:
            queries = _read_queries_file(args.queries_file)
        except RuntimeError as e:
            _emit_error(str(e), args, error_code=E_INPUT)
        if not queries:
            _emit_error(f"no queries found in '{args.queries_file}'", args,
                        error_code=E_INPUT)

        logger.info(f"Running {len(queries)} queries from {args.queries_file}...")
        batch = []
        any_with_results = False
        error_count = 0
        for i, q in enumerate(queries, 1):
            logger.info(f"\n[{i}/{len(queries)}] {q}")
            results, err, err_code = _run_single_query(q, args, instance_urls,
                                                        auth_headers, ttl_seconds)
            if err:
                error_count += 1
                logger.error(f"  [ERROR] {err}")
                entry = {"query": q, "status": "error", "error": err}
                if err_code:
                    entry["error_code"] = err_code
                batch.append(entry)
            else:
                if len(results.get("results", [])) > 0:
                    any_with_results = True
                batch.append({"query": q, "status": "ok", "results": results})

        if args.format == "json":
            output = json.dumps({"schema_version": SCHEMA_VERSION, "queries": batch},
                                indent=2, ensure_ascii=False)
        elif args.format == "csv":
            import csv as csv_mod
            import io
            out = io.StringIO()
            writer = csv_mod.writer(out, lineterminator="\n")
            writer.writerow(["query", "title", "url", "engine", "score",
                             "published_date", "content"])
            for br in batch:
                q = br["query"]
                if "results" in br:
                    for r in br["results"].get("results", []):
                        writer.writerow([
                            q,
                            r.get("title", ""),
                            r.get("url", ""),
                            r.get("engine", ""),
                            r.get("score", "") if r.get("score") is not None else "",
                            r.get("published_date", ""),
                            r.get("content", ""),
                        ])
                else:
                    writer.writerow([q, "", "", "", "", "",
                                     f"[ERROR: {br['error']}]"])
            output = out.getvalue().rstrip()
        elif args.format == "urls":
            parts = []
            for br in batch:
                parts.append(f"# {br['query']}")
                if "results" in br:
                    parts.append(format_urls(br["results"]))
                else:
                    parts.append(f"# [ERROR: {br['error']}]")
            output = "\n".join(parts)
        else:  # brief
            parts = []
            for br in batch:
                parts.append("=" * 60)
                parts.append(f"QUERY: {br['query']}")
                parts.append("=" * 60)
                if "results" in br:
                    parts.append(_format_results(br["results"], args))
                else:
                    parts.append(f"[ERROR: {br['error']}]")
                parts.append("")
            output = "\n".join(parts)

        if args.output:
            with open(args.output, "w", encoding="utf-8") as f:
                f.write(output)
            logger.info(f"Saved results to {args.output}")
        else:
            print(output)
        # Exit code semantics — aligned with single-query mode so AI agents
        # can use one consistent rule:
        #   1 = all queries errored (fatal)
        #   2 = no query returned any results (empty), though at least one
        #       searched successfully without error
        #   0 = at least one query returned results
        if error_count == len(queries):
            sys.exit(1)
        if not any_with_results:
            sys.exit(2)
        sys.exit(0)

    # ----- Single query mode -----
    results, err, err_code = _run_single_query(args.query, args, instance_urls,
                                                auth_headers, ttl_seconds)

    # --stream: JSON Lines 流式输出，每条结果一行，AI 可增量处理。
    # stream 模式下所有输出（包括错误）都是单行 JSON，保持 JSON Lines 格式一致性。
    # 非 stream 模式的错误走 _emit_error（多行 JSON 或 stderr 文本）。
    if getattr(args, "stream", False) and args.format == "json":
        if err:
            error_event = {"type": "error", "error": err, "query": args.query}
            if err_code:
                error_event["error_code"] = err_code
                hint = RECOVERY_HINTS.get(err_code)
                if hint:
                    error_event["recovery_hint"] = hint
            print(json.dumps(error_event, ensure_ascii=False), flush=True)
            sys.exit(1)
        for r in results.get("results", []):
            print(json.dumps({"type": "result", "result": r},
                             ensure_ascii=False), flush=True)
        print(json.dumps({"type": "done",
                          "schema_version": SCHEMA_VERSION,
                          "count": len(results.get("results", [])),
                          "query": args.query}, ensure_ascii=False), flush=True)
        if not results.get("results"):
            sys.exit(2)
        sys.exit(0)

    if err:
        _emit_error(err, args, query=args.query, error_code=err_code)

    output = _format_results(results, args)
    if args.output:
        with open(args.output, "w", encoding="utf-8") as f:
            f.write(output)
        logger.info(f"Saved results to {args.output}")
    else:
        print(output)

    if not results.get("results"):
        logger.warning("Warning: no results returned")
        sys.exit(2)


if __name__ == "__main__":
    main()