#!/usr/bin/env python3
"""Execute searches via a user-supplied SearXNG instance's JSON API.

Calls GET/POST /search?q=...&format=json on the chosen instance.
Auto-retries on failure with exponential backoff + tries next instance.
Falls back to HTML scraping if the instance blocks JSON output.

Instance URLs are REQUIRED (see --instance / SEARXNG_INSTANCE / config file).
"""

import argparse
import json
import logging
import os
import random
import re
import sys
import threading
import time
import urllib.error
import urllib.parse
import urllib.request
from concurrent.futures import ThreadPoolExecutor, as_completed
from html.parser import HTMLParser
from pathlib import Path

# Allow running standalone from any working directory
sys.path.insert(0, str(Path(__file__).resolve().parent))
from _config import SCHEMA_VERSION, USER_AGENT, VERSION
from common import (
    RETRYABLE_STATUS,
    RETRY_BACKOFF_BASE,
    MAX_RETRIES,
    RECOVERY_HINTS,
    apply_proxy,
    build_auth_headers,
    build_wayback_url,
    classify_error,
    emit_progress,
    force_utf8_stdout,
    is_hard_blocked_domain,
    resolve_auth_basic,
    resolve_auth_bearer,
    set_progress_enabled,
    setup_logging,
    should_try_wayback,
    E_CONFIG,
    E_AUTH,
    E_NETWORK,
    E_RATE_LIMIT,
    E_PARSE,
    E_EMPTY,
    E_INPUT,
    E_INTERNAL,
)
from fetch import extract_text, fetch_url
import cache as cache_module

logger = logging.getLogger("searxng.search")


# ----- Auth helpers -----

def _merge_headers(*header_dicts: dict) -> dict:
    """Merge multiple header dicts, with later dicts overriding earlier ones."""
    result = {}
    for d in header_dicts:
        if d:
            result.update(d)
    return result


# ----- HTML fallback: result extractor -----

class SearXNGHTMLParser(HTMLParser):
    """Extract search results + metadata from SearXNG's simple theme HTML.

    Matches:
      <article class="result result-default category-general">
        <a href="..." class="url_header"> ... </a>
        <h3><a href="...">Title</a></h3>
        <p class="content"> Snippet... </p>
        <time datetime="2024-01-15T10:30:00">...</time>
      </article>
      <div id="suggestions"><a>...</a></div>
      <div class="answer">...</div>

    The ``published_date`` field is populated from the ``<time>`` tag's
    ``datetime`` attribute (preferred) or its text content (fallback),
    mirroring the field SearXNG's JSON API exposes.
    """

    def __init__(self):
        super().__init__()
        self.results = []
        self.suggestions = []
        self.answers = []
        self.infoboxes = []

        self._current = None
        self._in_article = False
        self._in_h3 = False
        self._in_content = False
        self._in_time = False
        self._in_suggestions = False
        self._in_answer = False
        self._text_buf = []
        self._skip_depth = 0

    def handle_starttag(self, tag, attrs):
        attrs_dict = dict(attrs)
        classes = attrs_dict.get("class", "").split()
        tag_id = attrs_dict.get("id", "")

        if self._skip_depth > 0:
            self._skip_depth += 1
            return

        # Results
        if tag == "article" and "result" in classes:
            self._in_article = True
            self._current = {"title": "", "url": "", "content": "",
                             "engine": "", "published_date": ""}

        if self._in_article:
            if tag == "a" and "url_header" in classes:
                href = attrs_dict.get("href", "")
                if href:
                    self._current["url"] = href
            elif tag == "h3":
                self._in_h3 = True
                self._text_buf = []
            elif tag == "a" and self._in_h3:
                # h3 内的 <a href> 作为 url 的 fallback：某些 SearXNG 主题
                # 不使用 url_header class，URL 仅出现在 h3 的链接中。
                # url_header 优先（上面已处理），此处仅填充空 url。
                href = attrs_dict.get("href", "")
                if href and not self._current.get("url"):
                    self._current["url"] = href
            elif tag == "p" and "content" in classes:
                self._in_content = True
                self._text_buf = []
            elif tag == "time":
                # SearXNG puts the ISO timestamp in the datetime attribute
                dt = attrs_dict.get("datetime", "").strip()
                if dt and self._current is not None:
                    self._current["published_date"] = dt
                # Still collect text as a fallback for instances that omit
                # the datetime attribute but render a human-readable date.
                self._in_time = True
                self._text_buf = []
            elif tag in ("script", "style"):
                self._skip_depth += 1

        # Suggestions: <div id="suggestions"> or class containing "suggestion"
        if tag_id == "suggestions" or "suggestion" in classes:
            self._in_suggestions = True
            self._text_buf = []
        if self._in_suggestions and tag == "a":
            self._text_buf = []

        # Answer boxes
        if "answer" in classes or tag_id == "answer":
            self._in_answer = True
            self._text_buf = []

    def handle_endtag(self, tag):
        if self._skip_depth > 0:
            self._skip_depth -= 1
            return

        if self._in_article and tag == "article":
            if self._current:
                self._current["title"] = self._current.get("title", "").strip()
                self._current["content"] = self._current.get("content", "").strip()
                if self._current.get("title") or self._current.get("url"):
                    self.results.append(self._current)
            self._in_article = False
            self._current = None
        elif self._in_h3 and tag == "h3":
            if self._current:
                self._current["title"] = " ".join(self._text_buf).strip()
            self._in_h3 = False
        elif self._in_content and tag == "p":
            if self._current:
                self._current["content"] = " ".join(self._text_buf).strip()
            self._in_content = False
        elif self._in_time and tag == "time":
            # Fallback: use text content only if the datetime attribute
            # wasn't already captured at starttag time.
            if self._current and not self._current.get("published_date"):
                text = " ".join(self._text_buf).strip()
                if text:
                    self._current["published_date"] = text
            self._in_time = False

        if self._in_suggestions and tag == "div":
            self._in_suggestions = False
        if self._in_answer and tag == "div":
            if self._text_buf:
                self.answers.append(" ".join(self._text_buf).strip())
            self._in_answer = False

    def handle_data(self, data):
        if self._skip_depth > 0:
            return
        if self._in_h3:
            self._text_buf.append(data)
        elif self._in_content:
            self._text_buf.append(data)
        elif self._in_time:
            self._text_buf.append(data)
        elif self._in_suggestions:
            stripped = data.strip()
            if stripped:
                self.suggestions.append(stripped)
        elif self._in_answer:
            self._text_buf.append(data)


def parse_html_results(html: str, query: str = "") -> dict:
    """Parse SearXNG HTML results page into structured dict."""
    parser = SearXNGHTMLParser()
    parser.feed(html)
    return {
        "query": query,
        "number_of_results": len(parser.results),
        "results": parser.results,
        "answers": parser.answers,
        "corrections": [],  # rarely in simple theme
        "suggestions": parser.suggestions,
        "infoboxes": parser.infoboxes,
        "unresponsive_engines": [],
        "_fallback": "html",
    }


# ----- Instance selection -----

def parse_instances(instance_arg: str) -> list:
    """Parse user-supplied instance URL(s).

    Accepts a single URL or a comma-separated list. Each entry is normalized
    (https:// prefix added if missing, trailing slash stripped). Multiple
    instances enable multi-instance failover.
    """
    urls = []
    for raw in instance_arg.split(","):
        u = raw.strip()
        if not u:
            continue
        if not u.startswith(("http://", "https://")):
            u = "https://" + u
        urls.append(u.rstrip("/"))
    return urls


def _normalize_csv(value: str) -> str:
    """Normalize a comma-separated list: strip each item, drop empties.

    ``"google, bing, brave"`` -> ``"google,bing,brave"`` so downstream params
    match SearXNG's exact engine name requirements.
    """
    return ",".join(p.strip() for p in value.split(",") if p.strip())


def _load_toml(path: Path) -> dict:
    """Load a TOML file with Python-version-aware backend selection.

    * Python 3.11+ → stdlib ``tomllib``
    * Python 3.8–3.10 → optional ``tomli`` backport (``pip install tomli``)
    * Neither available → raise ``RuntimeError`` so the caller can fall back
      to ``instances.txt`` gracefully.

    This is required because the project claims Python 3.8+ support but
    ``tomllib`` only entered the stdlib in 3.11.
    """
    try:
        import tomllib  # Python 3.11+
    except ModuleNotFoundError:
        try:
            import tomli as tomllib  # type: ignore[no-redef]
        except ModuleNotFoundError:
            raise RuntimeError(
                "Reading .toml config requires Python 3.11+ or 'pip install tomli'"
            )
    with open(path, "rb") as f:
        return tomllib.load(f)


def _read_instance_file(path: Path) -> list:
    """Read instance URLs from a config file.

    Supports two formats:
      * ``searxng.toml``  — ``instance = "url"`` or ``instances = ["a", "b"]``
                            under a ``[searxng]`` table (or at the top level).
                            Requires Python 3.11+ or the ``tomli`` backport.
      * ``*.txt``         — one URL per line, comma-separated lists allowed,
                            ``#`` starts a comment.
    Returns an empty list if the file cannot be parsed.
    """
    try:
        if path.suffix == ".toml":
            data = _load_toml(path)
            table = data.get("searxng", data)
            raw = table.get("instance") or table.get("instances")
            if isinstance(raw, str):
                return parse_instances(raw)
            if isinstance(raw, list):
                return [u.rstrip("/") if u.startswith(("http://", "https://"))
                        else "https://" + u.rstrip("/")
                        for u in raw if u and str(u).strip()]
            return []
        # Plain text: one URL per line (comma lists allowed), # comments
        out = []
        for line in path.read_text(encoding="utf-8").splitlines():
            line = line.strip()
            if not line or line.startswith("#"):
                continue
            out.extend(parse_instances(line))
        return out
    except RuntimeError as e:
        # tomllib 缺失（Python 3.8-3.10 未装 tomli）是可恢复的——可改用
        # instances.txt——但必须明确提示用户，而不是静默返回空列表让 main
        # 报 "no instance resolved"，让用户困惑真正的失败原因。
        msg = str(e).lower()
        if "toml" in msg and ("3.11" in msg or "tomli" in msg):
            logger.error(f"Cannot parse '{path}': {e} "
                         f"(consider 'pip install tomli' or use instances.txt)")
        else:
            logger.warning(f"Warning: cannot read instance file '{path}': {e}")
        return []
    except Exception as e:
        logger.warning(f"Warning: cannot read instance file '{path}': {e}")
        return []


def _windows_appdata_config_dir() -> Path:
    """Return the Windows APPDATA config directory, or a sentinel Path if unset.

    On Windows, the conventional per-user app config directory is
    ``%APPDATA%`` (typically ``C:\\Users\\<user>\\AppData\\Roaming``).
    On POSIX, this env var is unset and we return a sentinel
    ``Path("/__no_appdata__")`` which never exists on disk, so the caller
    can unconditionally append it to the candidate list without polluting
    Linux/macOS lookups.

    Note: ``Path("")`` resolves to ``.`` (current directory) on Windows,
    which DOES exist — so we must use an absolute sentinel path instead.
    """
    appdata = os.environ.get("APPDATA", "")
    if appdata:
        return Path(appdata) / "searxng-cli"
    # Sentinel: absolute path that never exists. Using "/" + unlikely name
    # keeps it false on both POSIX and Windows (where "/" is the drive root).
    return Path("/__no_appdata__")


def resolve_instances(cli_arg: str = None) -> list:
    """Resolve instance URLs from (in priority order):

    1. ``--instance`` CLI flag (comma-separated list)
    2. ``SEARXNG_INSTANCE`` environment variable (comma-separated list)
    3. config file search order:
       a. ``./searxng.toml``
       b. ``~/.config/searxng-cli/searxng.toml``
       c. ``%APPDATA%/searxng-cli/searxng.toml``  (Windows only)
       d. ``./instances.txt``
       e. ``~/.config/searxng-cli/instances.txt``
       f. ``%APPDATA%/searxng-cli/instances.txt``  (Windows only)

    Returns an empty list if no instance can be resolved.
    """
    if cli_arg:
        return parse_instances(cli_arg)

    env = os.environ.get("SEARXNG_INSTANCE")
    if env:
        return parse_instances(env)

    win_dir = _windows_appdata_config_dir()
    candidates = [
        Path.cwd() / "searxng.toml",
        Path.home() / ".config" / "searxng-cli" / "searxng.toml",
        win_dir / "searxng.toml",
        Path.cwd() / "instances.txt",
        Path.home() / ".config" / "searxng-cli" / "instances.txt",
        win_dir / "instances.txt",
    ]
    for p in candidates:
        if p.exists():
            urls = _read_instance_file(p)
            if urls:
                return urls
    return []


def load_config(config_path: str = None) -> dict:
    """Load the full config dict from ``searxng.toml`` (if present).

    If ``config_path`` is given (from ``--config``), only that file is
    consulted. Otherwise the default search order applies:
      1. ``./searxng.toml``
      2. ``~/.config/searxng-cli/searxng.toml``

    Returns the ``[searxng]`` table (or top-level table if no section),
    which may contain any of these keys used as CLI defaults:

      * ``instance`` / ``instances`` — instance URL(s)
      * ``timeout``, ``max_retries`` — network tuning
      * ``engines``, ``categories``, ``language`` — search scope
      * ``safesearch``, ``time_range``, ``method`` — search behavior
      * ``format`` — output format (json/brief/urls/csv)
      * ``sort_by`` — result sort key
      * ``proxy`` — proxy URL
      * ``cache_ttl`` — cache TTL in minutes

    Returns an empty dict if no config file exists or it cannot be parsed.
    Only ``.toml`` files are consulted for full config; ``.txt`` files
    only carry instance URLs (handled by :func:`resolve_instances`).
    """
    if config_path:
        p = Path(config_path)
        if not p.exists():
            logger.warning(f"Warning: config file '{p}' not found")
            return {}
        try:
            data = _load_toml(p)
            return data.get("searxng", data)
        except Exception as e:
            logger.warning(f"Warning: cannot read config '{p}': {e}")
            return {}
    win_dir = _windows_appdata_config_dir()
    candidates = [
        Path.cwd() / "searxng.toml",
        Path.home() / ".config" / "searxng-cli" / "searxng.toml",
        win_dir / "searxng.toml",
    ]
    for p in candidates:
        if p.exists():
            try:
                data = _load_toml(p)
                return data.get("searxng", data)
            except Exception as e:
                logger.warning(f"Warning: cannot read config '{p}': {e}")
                return {}
    return {}


def _cfg_int(config: dict, key: str, default: int) -> int:
    """Read an int from config, tolerating str/int forms and bad values.

    argparse's ``type=int`` only converts *command-line* strings; it does NOT
    convert a ``default`` value that came in as a string from TOML. So a
    ``timeout = "15"`` in ``searxng.toml`` would leak through as a str and
    break later numeric comparisons. This helper normalizes that.
    """
    if key not in config:
        return default
    try:
        return int(config[key])
    except (TypeError, ValueError):
        return default


def _cfg_float(config: dict, key: str, default: float) -> float:
    """Read a float from config, tolerating str/int/float forms. See _cfg_int."""
    if key not in config:
        return default
    try:
        return float(config[key])
    except (TypeError, ValueError):
        return default


# ----- Retry logic -----

def _retry_with_backoff(fn, max_retries: int = MAX_RETRIES, base_delay: float = RETRY_BACKOFF_BASE):
    """Call fn with exponential backoff + jitter on transient failures."""
    last_error = None
    for attempt in range(max_retries + 1):
        try:
            return fn()
        except urllib.error.HTTPError as e:
            if e.code in RETRYABLE_STATUS:  # 403 (UA block) + 429 + 5xx
                last_error = e
                if attempt < max_retries:
                    delay = base_delay * (2 ** attempt) + random.uniform(0, 1)
                    logger.info(f"  HTTP {e.code}, retrying in {delay:.1f}s... (attempt {attempt+1}/{max_retries})")
                    time.sleep(delay)
                    continue
            raise
        except (urllib.error.URLError, OSError) as e:
            last_error = e
            if attempt < max_retries:
                delay = base_delay * (2 ** attempt) + random.uniform(0, 1)
                logger.info(f"  Connection error ({e}), retrying in {delay:.1f}s...")
                time.sleep(delay)
                continue
            raise
    raise last_error


# ----- Search execution -----

def search_json(instance: str, params: dict, method: str = "GET",
                timeout: int = 15, auth_headers: dict = None) -> dict:
    """Execute search via JSON API. Returns None if JSON unsupported."""
    query_string = urllib.parse.urlencode(params)
    headers = _merge_headers({"User-Agent": USER_AGENT}, auth_headers)

    if method.upper() == "POST":
        data = query_string.encode("utf-8")
        req = urllib.request.Request(f"{instance}/search", data=data, headers=headers, method="POST")
    else:
        req = urllib.request.Request(f"{instance}/search?{query_string}", headers=headers)

    try:
        with urllib.request.urlopen(req, timeout=timeout) as resp:
            raw = resp.read().decode("utf-8")
            if raw.strip().startswith("{") or raw.strip().startswith("["):
                return json.loads(raw)
            # Got HTML — JSON unsupported
            return None
    except urllib.error.HTTPError as e:
        # 404 = JSON endpoint truly absent → fall back to HTML scraping
        if e.code == 404:
            return None
        # 403 / 401 = auth or IP issue → raise so it isn't silently masked
        # by an HTML fallback that would just 403 again. Also surfaces
        # instances that disable the JSON format via a 403 (rare but seen).
        raise


def search_html(instance: str, params: dict, timeout: int = 15,
                auth_headers: dict = None) -> dict:
    """Execute search via HTML scraping fallback."""
    html_params = {k: v for k, v in params.items() if k != "format"}
    query_string = urllib.parse.urlencode(html_params)
    url = f"{instance}/search?{query_string}"

    headers = _merge_headers({"User-Agent": USER_AGENT}, auth_headers)
    req = urllib.request.Request(url, headers=headers)
    try:
        with urllib.request.urlopen(req, timeout=timeout) as resp:
            html = resp.read().decode("utf-8")
            return parse_html_results(html, query=params.get("q", ""))
    except Exception as e:
        raise RuntimeError(f"HTML search failed for {instance}: {e}")


def search_single(instance: str, params: dict, method: str = "GET",
                  timeout: int = 15, auth_headers: dict = None) -> dict:
    """Execute one search attempt, preferring JSON with HTML fallback."""
    result = search_json(instance, params, method=method, timeout=timeout,
                         auth_headers=auth_headers)
    if result is not None:
        return result
    logger.warning(f"Warning: {instance} does not support format=json, falling back to HTML parsing")
    return search_html(instance, params, timeout=timeout, auth_headers=auth_headers)


def search_multi(instance_urls: list, params: dict, method: str = "GET",
                 timeout: int = 15, retry_per: int = None,
                 auth_headers: dict = None, parallel: bool = True) -> dict:
    """Search across multiple instances, failing over on error.

    With parallel=True (default, multi-instance only): every instance is
    queried concurrently. The first *successful* result in the user's
    original instance order is returned — this keeps output deterministic
    (same input list always yields the same source) while drastically
    speeding up failover when an early instance is down or slow.

    With parallel=False (or a single instance): strictly sequential, one
    request at a time, trying the next instance only after the current fails.
    """
    if retry_per is None:
        retry_per = MAX_RETRIES
    instance_urls = [u.rstrip("/") for u in instance_urls]

    # Single instance, or explicit serial mode: straightforward sequential path
    if len(instance_urls) <= 1 or not parallel:
        last_error = None
        for instance in instance_urls:
            logger.info(f"Trying {instance}...")
            emit_progress("instance_try", url=instance, attempt=1)
            start = time.time()
            try:
                def _do():
                    return search_single(instance, params, method=method,
                                         timeout=timeout, auth_headers=auth_headers)
                result = _retry_with_backoff(_do, max_retries=retry_per)
                emit_progress("instance_ok", url=instance,
                              latency=round(time.time() - start, 3),
                              results=len(result.get("results", [])) if result else 0)
                return result
            except Exception as e:
                last_error = e
                logger.info(f"  Failed: {e}")
                emit_progress("instance_fail", url=instance,
                              error=str(e), error_code=classify_error(e))
                continue
        raise RuntimeError(f"All {len(instance_urls)} instances failed. Last error: {last_error}")

    # Parallel path: one search per instance concurrently (each with its own
    # backoff retries), then return the first success in original order.
    logger.info(f"Trying {len(instance_urls)} instances in parallel...")

    def _task(instance: str):
        emit_progress("instance_try", url=instance, attempt=1)
        start = time.time()
        def _do():
            return search_single(instance, params, method=method,
                                 timeout=timeout, auth_headers=auth_headers)
        try:
            result = _retry_with_backoff(_do, max_retries=retry_per)
            emit_progress("instance_ok", url=instance,
                          latency=round(time.time() - start, 3),
                          results=len(result.get("results", [])) if result else 0)
            return instance, result
        except Exception as e:
            emit_progress("instance_fail", url=instance,
                          error=str(e), error_code=classify_error(e))
            return instance, e

    results_by_url = {}
    last_parallel_error = None
    with ThreadPoolExecutor(max_workers=min(len(instance_urls), 8)) as ex:
        futures = {ex.submit(_task, u): u for u in instance_urls}
        for fut in as_completed(futures):
            u = futures[fut]
            try:
                inst, res = fut.result()
            except Exception:
                continue
            if isinstance(res, Exception):
                # 保留最后一个失败详情，让 classify_error 能从消息中提取
                # 真实错误类型（401/403→E_AUTH，429→E_RATE_LIMIT 等），
                # 而不是一律误判为 E_NETWORK。
                last_parallel_error = res
                logger.info(f"  Failed {u}: {res}")
            else:
                results_by_url[u] = res

    # Deterministic: return first successful in the user-supplied order
    for u in instance_urls:
        if u in results_by_url:
            return results_by_url[u]

    raise RuntimeError(f"All {len(instance_urls)} instances failed (parallel). "
                       f"Last error: {last_parallel_error}")


# ----- Output formatting -----

# ----- Instance health verification (E5) -----

def _probe_config_endpoint(instance: str, timeout: int,
                           auth_headers: dict = None) -> dict:
    """Probe ``/config``; return engines list and categories if reachable.

    SearXNG's ``/config`` endpoint exposes the instance's engine list,
    categories, and other settings as JSON. Some instances disable it; in
    that case we return ``reachable=False`` with an empty engine list.
    """
    headers = _merge_headers({"User-Agent": USER_AGENT}, auth_headers)
    req = urllib.request.Request(f"{instance}/config", headers=headers)
    try:
        with urllib.request.urlopen(req, timeout=timeout) as resp:
            raw = resp.read().decode("utf-8")
            data = json.loads(raw)
            engines = [e.get("name") for e in data.get("engines", [])
                       if isinstance(e, dict) and e.get("name")]
            cats = data.get("categories", {})
            if isinstance(cats, dict):
                categories = list(cats.keys())
            elif isinstance(cats, list):
                categories = [c for c in cats if isinstance(c, str)]
            else:
                categories = []
            return {"reachable": True, "engines": engines,
                    "categories": categories, "error": None}
    except urllib.error.HTTPError as e:
        return {"reachable": False, "engines": [], "categories": [],
                "error": f"HTTP {e.code}"}
    except Exception as e:
        return {"reachable": False, "engines": [], "categories": [],
                "error": str(e)[:80]}


def verify_instances(instance_urls: list, timeout: int = 15,
                     auth_headers: dict = None, concurrency: int = 5) -> list:
    """Health-check user-supplied instances: reachability, JSON support, latency,
    POST support, /config endpoint, engine list, and auth status.

    Replaces the removed public-instance discovery: you bring your own
    instances, and ``--verify`` tells you which are alive, which support the
    JSON API, which accept POST, what engines they expose, and how fast
    each responds.

    Returns a list of dicts (one per instance, in input order), each with:
      url, reachable (bool), json_supported (bool|None), post_supported (bool|None),
      config_endpoint (bool|None), engines (list[str]), latency (float|None),
      result_count (int|None), auth_status (str), error (str|None)
    """
    instance_urls = [u.rstrip("/") for u in instance_urls]
    has_auth = bool(auth_headers)

    def _check(u: str) -> dict:
        start = time.time()
        # 1. GET /search?q=test&format=json — the primary reachability probe
        try:
            data = search_json(u, {"q": "test", "format": "json"}, "GET",
                               timeout=timeout, auth_headers=auth_headers)
            latency = round(time.time() - start, 3)
            auth_status = "ok" if has_auth else "n/a"

            if data is None:
                # Instance returned HTML — JSON format disabled but reachable
                base = {"url": u, "reachable": True, "json_supported": False,
                        "latency": latency, "result_count": None,
                        "auth_status": auth_status,
                        "error": "returned HTML (format=json unsupported)"}
            else:
                base = {"url": u, "reachable": True, "json_supported": True,
                        "latency": latency,
                        "result_count": len(data.get("results", [])),
                        "auth_status": auth_status, "error": None}

            # 2. POST probe — only meaningful when JSON works
            if data is not None:
                try:
                    post_data = search_json(u, {"q": "test", "format": "json"}, "POST",
                                            timeout=timeout, auth_headers=auth_headers)
                    base["post_supported"] = post_data is not None
                except urllib.error.HTTPError as e:
                    base["post_supported"] = False
                    base["error"] = (base.get("error") or "") + f" POST: HTTP {e.code}"
                except Exception as e:
                    base["post_supported"] = False
                    base["error"] = (base.get("error") or "") + f" POST: {str(e)[:60]}"
            else:
                base["post_supported"] = None

            # 3. /config probe — engine list (independent of JSON search)
            cfg = _probe_config_endpoint(u, timeout, auth_headers)
            base["config_endpoint"] = cfg["reachable"]
            base["engines"] = cfg["engines"]
            if cfg["error"] and not cfg["reachable"]:
                # /config down is non-fatal; note it but don't overwrite search error
                base["error"] = (base.get("error") or "") + f" /config: {cfg['error']}"

            return base
        except urllib.error.HTTPError as e:
            # Distinguish auth rejection (401/403) from other HTTP errors so
            # users know whether their token is wrong or the instance is down.
            auth_status = ("rejected" if has_auth and e.code in (401, 403)
                           else ("ok" if has_auth else "n/a"))
            # 5xx 是服务器错误，实例虽然响应了但不可用，应视为不可达。
            # 400 可能只是请求格式问题，实例本身在线，仍算可达。
            is_5xx = 500 <= e.code < 600
            return {"url": u, "reachable": e.code not in (401, 403, 404) and not is_5xx,
                    "json_supported": False, "post_supported": None,
                    "config_endpoint": None, "engines": [],
                    "latency": round(time.time() - start, 3),
                    "result_count": None, "auth_status": auth_status,
                    "error": f"HTTP {e.code}"}
        except Exception as e:
            return {"url": u, "reachable": False, "json_supported": False,
                    "post_supported": None, "config_endpoint": None,
                    "engines": [], "latency": None, "result_count": None,
                    "auth_status": "unknown" if has_auth else "n/a",
                    "error": str(e)[:120]}

    out = {}
    with ThreadPoolExecutor(max_workers=min(concurrency, len(instance_urls))) as ex:
        futures = {ex.submit(_check, u): u for u in instance_urls}
        for fut in as_completed(futures):
            u = futures[fut]
            try:
                out[u] = fut.result()
            except Exception as e:
                out[u] = {"url": u, "reachable": False, "json_supported": False,
                          "post_supported": None, "config_endpoint": None,
                          "engines": [], "latency": None, "result_count": None,
                          "auth_status": "unknown" if has_auth else "n/a",
                          "error": str(e)[:120]}
    # Preserve input order
    return [out[u] for u in instance_urls]


def _print_verify_report(report: list, as_json: bool):
    """Print a health-check report (human table or JSON).

    The table shows one line per instance with the most important signals
    (reach / JSON / POST / latency / result count / engine count / auth).
    The full engine list per instance is only included in the JSON output
    to keep the table readable.
    """
    if as_json:
        print(json.dumps(report, indent=2, ensure_ascii=False))
        return
    header = (f"{'URL':<38} {'REACH':<6} {'JSON':<5} {'POST':<5} "
              f"{'LAT':<8} {'RES':<5} {'ENGS':<5} {'AUTH':<9} ERROR")
    lines = [header, "-" * len(header)]
    for r in report:
        reach = "yes" if r["reachable"] else "NO"
        js = "-" if r["json_supported"] is None else ("yes" if r["json_supported"] else "no")
        post = "-" if r.get("post_supported") is None else ("yes" if r["post_supported"] else "no")
        lat = f"{r['latency']}s" if r["latency"] is not None else "-"
        rc = "-" if r["result_count"] is None else str(r["result_count"])
        engs = str(len(r.get("engines", []))) if r.get("engines") is not None else "-"
        auth = r.get("auth_status", "-")
        err = (r["error"] or "").replace("\n", " ")[:40]
        lines.append(f"{r['url'][:37]:<38} {reach:<6} {js:<5} {post:<5} "
                     f"{lat:<8} {rc:<5} {engs:<5} {auth:<9} {err}")
    ok = sum(1 for r in report if r["reachable"])
    lines.append("-" * len(header))
    lines.append(f"{ok}/{len(report)} instances reachable")
    print("\n".join(lines))


# ----- Auto-fetch result pages -----
# NOTE: HTTP transport (retry, charset, UA fallback, size limit) is delegated
# to fetch.fetch_url — this module only post-processes its result. This
# eliminates ~70 lines of duplicated HTTP code that previously drifted
# between search.fetch_page and fetch.fetch_url.

def fetch_page(url: str, timeout: int = 10, auth_headers: dict = None,
               max_retries: int = 3, max_size: int = None,
               referer: str = None,
               fallback_enabled: bool = True) -> dict:
    """Fetch a single page; returns metadata dict with 'status'='ok' or 'error'.

    Thin wrapper around :func:`fetch.fetch_url` that adds:
      * CAPTCHA / bot-block detection with WAF fingerprinting (v2.0.0)
      * Wayback Machine fallback on 404/403/timeout (v2.0.0, default on)
      * automatic text extraction via :func:`fetch.extract_text`
      * dict-shaped return suitable for the auto-fetch feature

    All HTTP transport concerns (retry, charset, UA fallback, size limit,
    browser headers, Retry-After compliance) are handled by ``fetch_url``.

    v2.0.0 新字段：
      * ``anti_bot_detected`` (bool): 是否检测到反爬页面
      * ``waf_type`` (str|None): WAF 类型（cloudflare/imperva/perimeterx/
        datadome/akamai/generic），仅当 anti_bot_detected=True 时有值
      * ``fallback_used`` (str|None): 兜底方式（"wayback"），仅当走兜底时有值
    """
    # 主抓取
    result = None
    error_msg = None
    try:
        result = fetch_url(
            url, timeout=timeout, auth_headers=auth_headers,
            max_retries=max_retries, max_size=max_size,
            allow_redirects=True, referer=referer,
        )
    except Exception as e:
        error_msg = str(e) if str(e) else e.__class__.__name__

    # 反爬检测（v2.0.0 增强：全文档扫描 + WAF 指纹库）
    # 必须在 Wayback 兜底判断之前执行：Cloudflare 质询页常返回 HTTP 200，
    # 此时 result 非 None 但内容是反爬页，必须识别出来才能触发兜底。
    anti_bot_detected = False
    waf_type = None
    if result is not None:
        content = result.content
        content_type = result.content_type or ""
        is_html = ("html" in content_type.lower() or
                   content.strip().startswith("<!") or
                   content.strip().startswith("<htm"))
        if is_html:
            waf_type = _detect_anti_bot(content)
            if waf_type:
                anti_bot_detected = True

    # Wayback 兜底触发条件（v2.0.1 修复）：
    #   1. 主抓取抛异常且错误暗示 404/403/超时（_should_try_fallback）
    #   2. 主抓取"成功"但被反爬拦截（anti_bot_detected=True）
    # 原 v2.0.0 bug：仅条件 1 触发兜底，条件 2 漏网（Cloudflare 200 质询页）。
    fallback_used = None
    need_fallback = False
    if fallback_enabled:
        if anti_bot_detected:
            need_fallback = True
        elif result is None and _should_try_fallback(result, error_msg, url):
            need_fallback = True

    if need_fallback:
        wb_result = _try_wayback_fallback(url, timeout=timeout,
                                          auth_headers=auth_headers,
                                          max_retries=max_retries,
                                          max_size=max_size)
        if wb_result is not None:
            result = wb_result
            error_msg = None
            fallback_used = "wayback"
            # Wayback 结果重新做反爬检测（防御性：Wayback 快照极少是反爬页）
            anti_bot_detected = False
            waf_type = None
            content = result.content
            content_type = result.content_type or ""
            is_html = ("html" in content_type.lower() or
                       content.strip().startswith("<!") or
                       content.strip().startswith("<htm"))
            if is_html:
                waf_type = _detect_anti_bot(content)
                if waf_type:
                    anti_bot_detected = True

    # 仍然失败
    if result is None:
        return {
            "url": url, "status": "error",
            "error": error_msg or "unknown error",
            "text": "", "text_length": 0, "truncated": False,
            "anti_bot_detected": False, "waf_type": None,
            "fallback_used": None,
        }

    # 反爬仍被检测到（Wayback 也无能为力或兜底被禁用）
    if anti_bot_detected:
        return {
            "url": url, "final_url": result.final_url, "status": "error",
            "error": f"Bot protection detected ({waf_type})",
            "text": "", "text_length": 0, "truncated": False,
            "anti_bot_detected": True, "waf_type": waf_type,
            "fallback_used": fallback_used,
        }

    text = extract_text(content) if is_html else content
    return {
        "url": url,
        "final_url": result.final_url,
        "status": "ok",
        "content_type": content_type,
        "text": text,
        "text_length": len(text),
        "truncated": result.truncated,
        "truncated_at": max_size if result.truncated else None,
        "user_agent_used": result.user_agent,
        "anti_bot_detected": False,
        "waf_type": None,
        "fallback_used": fallback_used,
    }


def _should_try_fallback(result, error_msg: str, url: str = None) -> bool:
    """判断是否应触发 Wayback 兜底。

    v2.1.0 改为调用 common.should_try_wayback 共享逻辑 + 被墙站点检测。

    触发条件：
      1. 主抓取抛异常且错误信息暗示 404/403/超时（common.should_try_wayback）
      2. 主抓取成功但被反爬拦截（理论上 fetch_page 已处理，此处防御性）
      3. v2.1.0: URL 属于被墙/强反爬站点（is_hard_blocked_domain）

    不触发：
      * 用户禁用兜底（调用方控制，不进入此函数）
      * 错误是 DNS 失败（Wayback 也访问不到）
    """
    if result is not None:
        # 主抓取成功，无需兜底
        return False
    # v2.1.0: 被墙站点直接触发兜底（不等错误信息判断）
    if url and is_hard_blocked_domain(url):
        return True
    return should_try_wayback(error_msg)


def _try_wayback_fallback(url: str, timeout: int = 10,
                          auth_headers: dict = None,
                          max_retries: int = 2,
                          max_size: int = None):
    """尝试从 Wayback Machine 获取页面快照。

    v2.1.0 改为使用 common.build_wayback_url 共享逻辑。

    使用 ``https://web.archive.org/web/2/<url>`` 端点，``2`` 表示
    "最新可用快照"。Wayback 会 302 重定向到具体时间戳快照。

    返回 FetchResult 或 None（失败时）。独立超时（10s），不阻塞主流程。
    """
    wayback_url = build_wayback_url(url)
    wb_timeout = min(timeout, 10)  # Wayback 自身可能慢，限制最大 10s
    try:
        logger.info(f"  [FALLBACK] Trying Wayback Machine for {url[:55]}")
        result = fetch_url(
            wayback_url, timeout=wb_timeout, auth_headers=None,
            max_retries=max_retries, max_size=max_size,
            allow_redirects=True,
        )
        # Wayback 包装页也算成功——它返回的是原始页面内容
        return result
    except Exception as e:
        logger.info(f"  [FALLBACK] Wayback failed for {url[:55]}: {e}")
        return None


# ----- 反爬检测（v2.0.1 收窄误判 + title 精准检测）-----
# WAF 指纹库：每项 = (waf_type, [指示词])
#
# v2.0.1 修复：v2.0.0 用裸公司名（cloudflare/akamai）和宽泛词（captcha/
# challenge/dd-）做全文匹配，导致正常文章（如引用 Cloudflare 文档、讨论
# "coding challenges" 的文章、Wayback 归档正文）被误判为反爬页，Wayback
# 兜底返回的有效内容也被丢弃。收窄原则：
#   1. 专用指纹只用 WAF 厂商的技术标识符（cookie 名/HTTP header 名/JS 变量名）
#      ——这些不会出现在文章正文里
#   2. 通用文案用完整短语而非单词（如 "please complete the captcha" 而非
#      "captcha"），避免正常内容误判
#   3. 增加 <title> 标签检测——反爬页 title 是特征文案，最精准
WAF_FINGERPRINTS = [
    ("cloudflare", [
        # Cloudflare 技术标识符（cookie/header/JS 变量名，不会出现在正文）
        "cf-ray", "cf-chl-bypass", "cf-mitigated",
        "cf-browser-verification", "cf-error-details", "cf-error-code",
        # 质询页特征文案（足够具体，正常内容不会完整出现）
        "just a moment", "checking your browser before accessing",
        "attention required! | cloudflare",
        "enable javascript and cookies to continue",
    ]),
    ("imperva", [
        # Incapsula cookie/技术标识符
        "incap_ses", "visid_incap", "incap_ses_",
        "incapsula incident id", "request unsuccessful. incapsula",
        "visit denied by incapsula",
    ]),
    ("perimeterx", [
        # PerimeterX 专有标识符（_px 太短会匹配 CSS 类名，已移除）
        "px-captcha", "pxhd", "pxcts", "pxcookie",
        "_pxff", "_pxhd",
        "press & hold to confirm you are a human",
    ]),
    ("datadome", [
        # DataDome 专有标识符（dd- 太宽泛会匹配 dd-class 等，已移除）
        "datadome", "data-dome",
        "protected by datadome", "datadome-bot-protect",
    ]),
    ("akamai", [
        # Akamai Bot Manager cookie/标识符（akamai 裸名会匹配正文引用，已移除）
        "bm_sz", "_abck", "akamaighost", "akamai-bot-manager",
        "ak_bmsc",
    ]),
    # 通用反爬指示词（v2.0.1 收窄：完整短语而非单词，避免正文误判）
    ("generic", [
        "verify you are human", "verify that you are human",
        "making sure you're not a bot", "are you a robot",
        "robot or human", "human verification",
        "please complete the captcha", "complete the security check",
        "please enable javascript to continue",
        "enable javascript to continue",
        "ddos protection by", "access denied - sucuri",
        "you have been blocked", "unusual traffic from your computer",
        "pardon our interruption", "we'll be right back",
        "bot protection", "anti-bot protection",
        "anubis_challenge", "miserere",  # Anubis 反爬系统（拦截 AI 爬虫）
    ]),
]

# <title> 标签检测：反爬页 title 通常是特征文案，比全文扫描更精准。
# key = title 中的特征子串（小写），value = 对应 WAF 类型
_TITLE_ANTI_BOT_SIGNATURES = {
    "just a moment": "cloudflare",
    "attention required": "cloudflare",
    "access denied": "generic",
    "are you a robot": "generic",
    "robot check": "generic",
    "human verification": "generic",
    "please verify you are human": "generic",
    "security check": "generic",
    "verify you are human": "generic",
}

_TITLE_RE = re.compile(r"<title[^>]*>(.*?)</title>", re.IGNORECASE | re.DOTALL)


def _detect_anti_bot(content: str) -> str:
    """检测反爬页面，返回 WAF 类型或 None。

    v2.0.1 改进：
      * 增加 <title> 标签精准检测（反爬页 title 是特征文案，误判率极低）
      * 收窄 WAF 指纹库关键词（移除裸公司名和宽泛词，改用技术标识符 + 完整短语）
      * 保留全文档扫描（v2.0.0 改进，大页面反爬页可能在前 2000 字之外）

    检测顺序：title 标签（最精准）→ 全文档指纹扫描（技术标识符 + 文案短语）。

    性能：全文档 lower() 一次 + title regex 一次，对 5MB 页面约 5ms，可接受。
    """
    if not content:
        return None
    # 1. <title> 标签检测（最精准，误判率极低）
    title_match = _TITLE_RE.search(content)
    if title_match:
        title_lower = title_match.group(1).strip().lower()
        for sig, waf_type in _TITLE_ANTI_BOT_SIGNATURES.items():
            if sig in title_lower:
                return waf_type
    # 2. 全文档指纹扫描（收窄后的技术标识符 + 完整文案）
    lower = content.lower()
    for waf_type, indicators in WAF_FINGERPRINTS:
        for ind in indicators:
            if ind in lower:
                return waf_type
    return None


def _is_blocked_page(content: str) -> bool:
    """[已废弃] 快速检测反爬页面。保留向后兼容，内部调用 _detect_anti_bot。

    v2.0.0 起请使用 _detect_anti_bot() 获取具体 WAF 类型。
    """
    return _detect_anti_bot(content) is not None


class AdaptiveThrottle:
    """自适应限流状态机（v2.0.0）。

    在 fetch_top_results 的并发抓取过程中，根据成功/失败反馈动态调整：
      * 连续 >=3 次失败 → request_delay 翻倍，concurrency 减半
      * 连续 >=5 次成功 → 逐步恢复原参数
      * 收到 429 → 标记全局暂停 N 秒（N 来自 Retry-After 或默认 30s），
        所有线程在下次请求前等待

    线程安全：所有方法加锁。状态由 fetch_top_results 的 _fetch_one 回调驱动。
    """

    def __init__(self, initial_delay: float, initial_concurrency: int):
        self._lock = threading.Lock()
        self._delay = initial_delay
        self._initial_delay = initial_delay
        self._concurrency = initial_concurrency
        self._initial_concurrency = initial_concurrency
        self._consecutive_failures = 0
        self._consecutive_successes = 0
        self._global_pause_until = 0.0  # time.monotonic() 时间戳

    @property
    def delay(self) -> float:
        with self._lock:
            return self._delay

    @property
    def concurrency(self) -> int:
        with self._lock:
            return self._concurrency

    def report_success(self) -> None:
        with self._lock:
            self._consecutive_failures = 0
            self._consecutive_successes += 1
            # 连续 5 次成功 → 逐步恢复
            if self._consecutive_successes >= 5:
                self._consecutive_successes = 0
                self._delay = max(self._initial_delay, self._delay / 2)
                if self._concurrency < self._initial_concurrency:
                    self._concurrency = min(self._initial_concurrency,
                                            self._concurrency * 2)

    def report_failure(self, error_msg: str = "") -> None:
        with self._lock:
            self._consecutive_successes = 0
            self._consecutive_failures += 1
            # 429 → 全局暂停（调用方会从 error_msg 提取秒数，这里只标记）
            if "429" in error_msg.lower():
                self._global_pause_until = time.monotonic() + 30.0
            # 连续 3 次失败 → 退避 + 降并发
            if self._consecutive_failures >= 3:
                self._consecutive_failures = 0
                self._delay = min(self._delay * 2, 10.0)  # 上限 10s
                self._concurrency = max(1, self._concurrency // 2)

    def wait_if_paused(self) -> None:
        """如果处于全局暂停期，阻塞等待直到解除。请求前调用。"""
        with self._lock:
            remaining = self._global_pause_until - time.monotonic()
        if remaining > 0:
            logger.info(f"  [THROTTLE] Global pause: waiting {remaining:.1f}s (429)")
            time.sleep(remaining)

    def stats(self) -> dict:
        """返回当前状态快照，供 --fetch-report 使用。"""
        with self._lock:
            return {
                "current_delay": round(self._delay, 3),
                "current_concurrency": self._concurrency,
                "consecutive_failures": self._consecutive_failures,
                "consecutive_successes": self._consecutive_successes,
                "global_paused": time.monotonic() < self._global_pause_until,
            }


def fetch_top_results(results: dict, count: int, timeout: int = 10,
                       concurrency: int = 5, auth_headers: dict = None,
                       max_retries: int = 3, max_size: int = None,
                       request_delay: float = 0.3,
                       referer: str = None,
                       fallback_enabled: bool = True,
                       throttle: "AdaptiveThrottle" = None) -> list:
    """Fetch full text of top N result pages concurrently.

    Features:
    - Retries transient errors with exponential backoff (in fetch_url)
    - v2.0.0 自适应限流：连续失败自动降并发+加延迟，429 全局暂停
    - v2.0.0 Wayback 兜底：404/403/超时自动尝试 Wayback Machine
    - v2.0.0 反爬检测：WAF 指纹库识别 Cloudflare/Imperva/PerimeterX 等
    - Falls back to browser User-Agent if blocked (in fetch_url)
    - Small delay between requests to avoid rate limits

    Args:
        referer: Referer URL（v2.0.0，通常设为 SearXNG 实例 URL）
        fallback_enabled: 是否启用 Wayback 兜底（默认 True）
        throttle: 外部传入的 AdaptiveThrottle 实例（可选；不传则内部创建）
    """
    urls = []
    seen = set()
    for r in results.get("results", []):
        url = r.get("url", "")
        if url and url not in seen:
            urls.append(url)
            seen.add(url)
        if len(urls) >= count:
            break

    if not urls:
        return []

    # 自适应限流器（外部未传入则创建）
    if throttle is None:
        throttle = AdaptiveThrottle(request_delay, concurrency)

    logger.info(f"\nFetching {len(urls)} result pages "
                f"(timeout={timeout}s, retries={max_retries}, "
                f"delay={throttle.delay}s, concurrency={throttle.concurrency})...")
    fetched = []
    ok_count = [0]
    err_count = [0]
    anti_bot_count = [0]
    fallback_count = [0]

    def _fetch_one(u: str) -> dict:
        """Fetch one URL with adaptive throttling to avoid rate limiting."""
        # 全局暂停检查（429 触发）
        throttle.wait_if_paused()
        # 自适应延迟
        d = throttle.delay
        if d > 0:
            time.sleep(d * random.uniform(0.5, 1.5))
        result = fetch_page(u, timeout=timeout, auth_headers=auth_headers,
                            max_retries=max_retries, max_size=max_size,
                            referer=referer, fallback_enabled=fallback_enabled)
        if result["status"] == "ok":
            ok_count[0] += 1
            throttle.report_success()
            trunc = ", TRUNCATED" if result.get("truncated") else ""
            ua_note = ""
            if result.get("user_agent_used") != USER_AGENT:
                ua_note = " [fallback UA]"
            fb_note = " [wayback]" if result.get("fallback_used") else ""
            if fb_note:
                fallback_count[0] += 1
            logger.info(f"  [OK] {u[:55]} ({result['text_length']:,} chars"
                        f"{trunc}{ua_note}{fb_note})")
        else:
            err_count[0] += 1
            throttle.report_failure(result.get("error", ""))
            # 统计反爬拦截
            if result.get("anti_bot_detected"):
                anti_bot_count[0] += 1
            logger.error(f"  [ERR] {u[:55]} ({result.get('error', 'unknown')})")
        return result

    with ThreadPoolExecutor(max_workers=min(throttle.concurrency, len(urls))) as ex:
        future_map = {ex.submit(_fetch_one, u): u for u in urls}
        for future in as_completed(future_map):
            try:
                result = future.result()
                fetched.append(result)
            except Exception as e:
                u = future_map[future]
                fetched.append({"url": u, "status": "error", "error": str(e),
                                "text": "", "text_length": 0, "truncated": False,
                                "anti_bot_detected": False, "waf_type": None,
                                "fallback_used": None})
                logger.error(f"  [ERR] {u[:55]} (thread error: {e})")

    # Reorder to match original result order
    url_order = {u: i for i, u in enumerate(urls)}
    fetched.sort(key=lambda f: url_order.get(f["url"], 999))

    logger.info(f"Fetched: {ok_count[0]} ok, {err_count[0]} errors"
                f" (anti-bot: {anti_bot_count[0]}, wayback: {fallback_count[0]})")
    return fetched


def _emit_fetch_report(fetched: list, throttle: "AdaptiveThrottle") -> None:
    """v2.0.0: 输出结构化抓取报告到 stderr。

    让 AI Agent 可程序化分析抓取过程：哪些 URL 被反爬拦截、用了什么兜底、
    自适应限流如何调整。格式为人类可读的表格 + JSON 摘要。
    """
    import sys as _sys
    out = _sys.stderr
    lines = []
    lines.append("\n" + "=" * 72)
    lines.append("FETCH REPORT (v2.0.0)")
    lines.append("=" * 72)

    # Per-URL 表
    header = f"{'URL':<45} {'Status':<8} {'WAF':<12} {'Fallback':<10} {'Chars':>10}"
    lines.append(header)
    lines.append("-" * len(header))
    for f in fetched:
        url = f.get("url", "")[:44]
        status = "OK" if f.get("status") == "ok" else "ERR"
        waf = f.get("waf_type") or "-"
        fb = f.get("fallback_used") or "-"
        chars = f.get("text_length", 0)
        lines.append(f"{url:<45} {status:<8} {waf:<12} {fb:<10} {chars:>10,}")

    # 统计摘要
    total = len(fetched)
    ok = sum(1 for f in fetched if f.get("status") == "ok")
    err = total - ok
    anti_bot = sum(1 for f in fetched if f.get("anti_bot_detected"))
    wayback = sum(1 for f in fetched if f.get("fallback_used") == "wayback")
    lines.append("-" * len(header))
    lines.append(f"Total: {total} | OK: {ok} | Error: {err} | "
                 f"Anti-bot blocked: {anti_bot} | Wayback recovered: {wayback}")

    # 自适应限流状态
    s = throttle.stats()
    lines.append(f"Throttle: delay={s['current_delay']}s "
                 f"concurrency={s['current_concurrency']} "
                 f"paused={s['global_paused']} "
                 f"consec_fail={s['consecutive_failures']} "
                 f"consec_ok={s['consecutive_successes']}")

    # JSON 摘要（一行，便于 Agent 解析）
    import json as _json
    summary = {
        "total": total, "ok": ok, "error": err,
        "anti_bot_blocked": anti_bot, "wayback_recovered": wayback,
        "throttle": s,
    }
    lines.append("JSON: " + _json.dumps(summary, ensure_ascii=False))
    lines.append("=" * 72 + "\n")
    print("\n".join(lines), file=out)


# ----- Output formatting -----

def deduplicate_results(results: dict) -> dict:
    """Remove duplicate results by normalized URL, keeping first occurrence.

    Different engines often return the same URL. The first occurrence keeps
    its engine/score metadata; subsequent duplicates are dropped.

    URL normalization:
      * lowercase scheme + host
      * strip fragment (``#...``)
      * drop common tracking query params (``utm_*``, ``gclid``, ``fbclid``,
        ``mc_*``, ``ref``, ``ref_*``)
      * sort remaining params so ``?b=2&a=1`` matches ``?a=1&b=2``

    Results without a URL are kept as-is (never deduped). Mutates
    ``results['results']`` in place and returns ``results`` for chaining.
    """
    if not results.get("results"):
        return results

    # 跟踪参数前缀。裸 "ref" 过于宽泛（会误删 reference/refcode 等正常参数），
    # 收紧为 "ref_" 只匹配 ref_source/ref_campaign 等跟踪参数。
    TRACKING_PREFIXES = ("utm_", "gclid", "fbclid", "mc_", "ref_")

    def _normalize_url(url: str) -> str:
        try:
            p = urllib.parse.urlparse(url)
            scheme = p.scheme.lower()
            netloc = p.netloc.lower()
            qs = urllib.parse.parse_qsl(p.query, keep_blank_values=False)
            qs = [(k, v) for k, v in qs
                  if not any(k.lower().startswith(pref) for pref in TRACKING_PREFIXES)]
            qs.sort()
            query = urllib.parse.urlencode(qs)
            return urllib.parse.urlunparse((scheme, netloc, p.path, p.params,
                                            query, ""))
        except Exception:
            return url

    seen = set()
    deduped = []
    for r in results["results"]:
        url = r.get("url", "")
        key = _normalize_url(url) if url else ""
        if key and key in seen:
            continue
        if key:
            seen.add(key)
        deduped.append(r)

    before = len(results["results"])
    results["results"] = deduped
    if before != len(deduped):
        logger.info(f"Dedup: {before} -> {len(deduped)} results (by normalized URL)")
    return results


def sort_results(results: dict, sort_by: str) -> dict:
    """Sort results by the given key.

    * ``score``  — descending (highest first); entries without score keep
                   their relative order at the end.
    * ``date``   — descending (newest first) by ``published_date`` (ISO
                   strings sort lexicographically); entries without a date
                   keep their relative order at the end.
    * ``engine`` — ascending (alphabetical by engine name).
    * ``none``   — no-op (preserve original order).

    Mutates ``results['results']`` in place and returns ``results``.
    """
    if not results.get("results") or sort_by == "none":
        return results

    rs = results["results"]
    if sort_by == "score":
        with_score = [r for r in rs if r.get("score") is not None]
        without = [r for r in rs if r.get("score") is None]
        with_score.sort(key=lambda r: r["score"], reverse=True)
        results["results"] = with_score + without
    elif sort_by == "date":
        with_date = [r for r in rs if r.get("published_date")]
        without = [r for r in rs if not r.get("published_date")]
        with_date.sort(key=lambda r: r["published_date"], reverse=True)
        results["results"] = with_date + without
    elif sort_by == "engine":
        rs.sort(key=lambda r: r.get("engine") or "")
        results["results"] = rs
    return results


def filter_results_by_domain(results: dict, include_domains: list = None,
                              exclude_domains: list = None) -> dict:
    """Filter search results by domain (allowlist or blocklist).

    Modifies ``results['results']`` in place and returns ``results`` for
    chaining. Matching is case-insensitive and ignores a leading ``www.``
    so ``example.com`` matches both ``example.com`` and ``www.example.com``.

    ``include_domains`` (allowlist) takes precedence: if non-empty, only
    results whose domain matches are kept. ``exclude_domains`` (blocklist)
    then drops any remaining matches. Empty inputs are no-ops.
    """
    if not results.get("results"):
        return results
    if not include_domains and not exclude_domains:
        return results

    def _get_domain(url: str) -> str:
        try:
            return urllib.parse.urlparse(url).netloc.lower()
        except Exception:
            return ""

    def _normalize(d: str) -> str:
        return d.lower().lstrip(".").lstrip("www.")

    include_set = {_normalize(d) for d in include_domains} if include_domains else None
    exclude_set = {_normalize(d) for d in exclude_domains} if exclude_domains else None

    filtered = []
    for r in results["results"]:
        domain = _get_domain(r.get("url", ""))
        # Strip leading www. for matching
        match_domain = domain[4:] if domain.startswith("www.") else domain

        if include_set is not None and match_domain not in include_set:
            continue
        if exclude_set is not None and match_domain in exclude_set:
            continue
        filtered.append(r)

    results["results"] = filtered
    return results


def format_brief(results: dict, snippet_len: int = 0) -> str:
    """Format results as brief text. snippet_len=0 means no truncation."""
    lines = []
    for i, r in enumerate(results.get("results", []), 1):
        title = r.get("title", "No title")
        url = r.get("url", "")
        content = r.get("content", "")
        lines.append(f"{i}. {title}")
        lines.append(f"   {url}")
        if content:
            if snippet_len > 0:
                content = content[:snippet_len]
            lines.append(f"   {content}")
        lines.append("")

    # Include suggestions if present
    sugs = results.get("suggestions", [])
    if sugs:
        lines.append(f"Suggestions: {', '.join(sugs)}")

    # Include answers if present
    for ans in results.get("answers", []):
        lines.append(f"Answer: {ans}")

    return "\n".join(lines)


def format_urls(results: dict) -> str:
    """Format results as plain URL list."""
    return "\n".join(r.get("url", "") for r in results.get("results", []) if r.get("url"))


def _format_results(results: dict, args) -> str:
    """Format a results dict into the output string selected by args.format.

    Pulled out of :func:`main` so the batch runner (--queries-file) can reuse
    the exact same formatting for each per-query block.
    """
    if args.format == "json":
        results["schema_version"] = SCHEMA_VERSION
        return json.dumps(results, indent=2, ensure_ascii=False)
    if args.format == "urls":
        return format_urls(results)
    if args.format == "csv":
        import csv as csv_mod
        import io
        out = io.StringIO()
        writer = csv_mod.writer(out, lineterminator="\n")
        writer.writerow(["title", "url", "engine", "score",
                         "published_date", "content"])
        for r in results.get("results", []):
            writer.writerow([
                r.get("title", ""),
                r.get("url", ""),
                r.get("engine", ""),
                r.get("score", "") if r.get("score") is not None else "",
                r.get("published_date", ""),
                r.get("content", ""),
            ])
        return out.getvalue().rstrip()
    # brief (also the safe fallthrough)
    output = format_brief(results, snippet_len=args.snippet_len)
    if args.fetch > 0 and results.get("fetched"):
        output += "\n\n" + "=" * 60 + "\n"
        output += f"FETCHED PAGES ({len(results['fetched'])} pages)\n"
        output += "=" * 60 + "\n"
        for f in results["fetched"]:
            output += f"\n--- {f['url']} ---\n"
            if f["status"] == "ok":
                output += f"{f['text']}\n"
            else:
                output += f"[ERROR: {f.get('error', 'unknown')}]\n"
    return output


def _build_params(query: str, args) -> dict:
    """Build the SearXNG API params dict for one query.

    Centralized so the single-query path and the --queries-file batch path
    construct identical params (and thus share cache entries).
    """
    params = {"q": query, "format": "json"}
    if args.categories:
        params["categories"] = _normalize_csv(args.categories)
    if args.language:
        params["language"] = args.language
    if args.pageno != 1:
        params["pageno"] = str(args.pageno)
    if args.time_range and args.time_range != "none":
        params["time_range"] = args.time_range
    if args.safesearch is not None:
        params["safesearch"] = str(args.safesearch)
    if args.engines:
        params["engines"] = _normalize_csv(args.engines)
    return params


def _run_single_query(query: str, args, instance_urls: list,
                      auth_headers: dict, ttl_seconds: int):
    """Run one query end-to-end: search → limit → domain-filter → fetch.

    Returns ``(results_dict, error_str, error_code)``。成功时后两者为 None。
    ``error_code`` 是结构化错误码（E_NETWORK/E_AUTH 等），让 AI Agent
    程序化判断错误类型。Cache hits 跳过网络。后处理（limit/filter/fetch）
    总是执行，保证 batch 调用方看到与单查询一致的形状。
    """
    params = _build_params(query, args)

    emit_progress("start", query=query, instances=len(instance_urls))

    cached = cache_module.get(params, ttl_seconds) if ttl_seconds > 0 else None
    if cached is not None:
        logger.info(f"[cache hit] q={query!r} TTL={args.cache_ttl}min, skipping network")
        emit_progress("cache_hit", query=query, ttl=args.cache_ttl)
        results = cached
    else:
        try:
            results = search_multi(
                instance_urls, params,
                method=args.method,
                timeout=args.timeout,
                retry_per=args.retry,
                auth_headers=auth_headers,
                parallel=not args.serial,
            )
        except Exception as e:
            err_code = classify_error(e)
            emit_progress("error", error=str(e), error_code=err_code, query=query)
            return None, str(e), err_code
        if ttl_seconds > 0:
            cache_module.put(params, results, ttl_seconds)
            logger.info(f"[cache stored] q={query!r} TTL={args.cache_ttl}min")
            emit_progress("cache_store", query=query, ttl=args.cache_ttl)

    # Dedup (default on; --no-dedup disables) then sort, both BEFORE limit
    # so --max-results keeps the highest-scoring / newest items.
    if not args.no_dedup:
        deduplicate_results(results)
    sort_results(results, args.sort_by)

    if args.max_results and "results" in results:
        results["results"] = results["results"][:args.max_results]

    if args.include_domain or args.exclude_domain:
        include_list = ([d.strip() for d in args.include_domain.split(",") if d.strip()]
                        if args.include_domain else None)
        exclude_list = ([d.strip() for d in args.exclude_domain.split(",") if d.strip()]
                        if args.exclude_domain else None)
        before = len(results.get("results", []))
        filter_results_by_domain(results, include_domains=include_list,
                                 exclude_domains=exclude_list)
        after = len(results.get("results", []))
        logger.info(f"Domain filter: {before} -> {after} results")

    if args.fetch > 0 and results.get("results"):
        emit_progress("fetch_start", count=args.fetch)
        # v2.0.0: Referer 默认设为首个实例 URL，伪装流量来自搜索引擎
        referer = getattr(args, "referer", None)
        if referer is None and instance_urls:
            referer = instance_urls[0]
        # v2.0.0: 创建共享 throttle 实例，用于 --fetch-report 输出
        request_delay = getattr(args, "request_delay", 0.3)
        fetch_throttle = AdaptiveThrottle(request_delay,
                                          min(5, args.fetch))
        fetched = fetch_top_results(
            results, args.fetch,
            timeout=args.fetch_timeout,
            auth_headers=auth_headers,
            max_retries=args.fetch_retries,
            max_size=args.max_size,
            request_delay=request_delay,
            referer=referer,
            fallback_enabled=not getattr(args, "no_fallback", False),
            throttle=fetch_throttle,
        )
        # Emit fetch_ok / fetch_fail events
        for f in fetched:
            if f.get("status") == "ok":
                emit_progress("fetch_ok", url=f.get("url", ""),
                              chars=f.get("text_length", 0),
                              fallback=f.get("fallback_used"))
            else:
                emit_progress("fetch_fail", url=f.get("url", ""),
                              error=f.get("error", "unknown"),
                              waf_type=f.get("waf_type"))
        results["fetched"] = fetched
        results["fetched_source"] = results.get("_fallback", "json")

        # v2.0.0: --fetch-report 输出到 stderr
        if getattr(args, "fetch_report", False):
            _emit_fetch_report(fetched, fetch_throttle)

    result_count = len(results.get("results", []))
    emit_progress("done", results=result_count, query=query)
    # 清理内部 _fallback 字段，避免泄漏到 JSON 输出。
    # fetched_source（对 AI 有用的公开字段）已在 --fetch 路径中设置。
    results.pop("_fallback", None)
    return results, None, None


def _get_output_schema():
    """Return the JSON Schema describing --format json output.

    Used by ``--dump-schema`` so AI agents can programmatically discover the
    output structure without parsing prose documentation.
    """
    return {
        "$schema": "https://json-schema.org/draft/2020-12/schema",
        "title": "SearXNG CLI Search Result",
        "schema_version": SCHEMA_VERSION,
        "description": "Output schema for 'python search.py --format json' (single query). "
                       "Batch mode (--queries-file) wraps results in "
                       '{"schema_version, queries:[]}.',
        "type": "object",
        "properties": {
            "schema_version": {
                "type": "string",
                "const": SCHEMA_VERSION,
                "description": "Output schema version. Bump on breaking field changes.",
            },
            "query": {"type": "string", "description": "The search query string."},
            "number_of_results": {
                "type": "integer",
                "description": "Total matches reported by SearXNG (JSON path) or "
                               "count of parsed results (HTML fallback path).",
            },
            "results": {
                "type": "array",
                "description": "Search result items, ordered by relevance (score desc).",
                "items": {
                    "type": "object",
                    "properties": {
                        "title": {"type": "string"},
                        "url": {"type": "string", "format": "uri"},
                        "engine": {"type": "string", "description": "Source engine name."},
                        "score": {"type": ["number", "null"]},
                        "published_date": {"type": ["string", "null"]},
                        "content": {"type": "string", "description": "Snippet/summary text."},
                    },
                    "required": ["title", "url"],
                },
            },
            "unresponsive_engines": {
                "type": "array",
                "items": {"type": "string"},
                "description": "Engines that failed to respond.",
            },
            "suggestions": {
                "type": "array",
                "items": {"type": "string"},
                "description": "Related query suggestions from the instance.",
            },
            "fetched": {
                "type": "array",
                "description": "Present only when --fetch N is used. Page content "
                               "for the top N results.",
                "items": {
                    "type": "object",
                    "properties": {
                        "url": {"type": "string"},
                        "text": {"type": "string"},
                        "text_length": {"type": "integer"},
                        "error": {"type": "string"},
                    },
                },
            },
            "fetched_source": {
                "type": "string",
                "enum": ["json", "html"],
                "description": "Present only when --fetch is used. Indicates whether "
                               "search results came from JSON API or HTML fallback.",
            },
        },
        "required": ["query", "results"],
    }


def _emit_error(message: str, args, query: str = None, exit_code: int = 1,
                error_code: str = None):
    """Emit an error and exit.

    In ``--format json`` mode the error is printed to **stdout** as a
    structured JSON object so agents piping stdout can parse it. All other
    formats print to stderr (keeping stdout clean for data) and exit.

    The JSON shape is::

        {"error": "...", "exit_code": N, "error_code": "E_*",
         "recovery_hint": "...", "query": "..."}

    ``error_code`` 是结构化错误码（E_CONFIG/E_AUTH/E_NETWORK 等），让 AI
    Agent 程序化判断错误类型并采取恢复策略。``recovery_hint`` 给出可操作的
    恢复建议，让 AI 能自决策下一步动作。``query`` 仅在提供时包含。
    """
    if getattr(args, "format", None) == "json":
        payload = {"error": message, "exit_code": exit_code}
        if error_code:
            payload["error_code"] = error_code
            hint = RECOVERY_HINTS.get(error_code)
            if hint:
                payload["recovery_hint"] = hint
        if query:
            payload["query"] = query
        print(json.dumps(payload, indent=2, ensure_ascii=False))
    else:
        prefix = f"[query: {query}] " if query else ""
        code_prefix = f"[{error_code}] " if error_code else ""
        hint_suffix = ""
        if error_code and error_code in RECOVERY_HINTS:
            hint_suffix = f"\n  Hint: {RECOVERY_HINTS[error_code]}"
        logger.error(f"{prefix}{code_prefix}Error: {message}{hint_suffix}")
    sys.exit(exit_code)


def _read_queries_file(path: str) -> list:
    """Read queries from a file: one per line, skip blanks and ``#`` comments.

    Raises :class:`RuntimeError` if the file cannot be read, so the caller
    can route it through :func:`_emit_error`.
    """
    try:
        text = Path(path).read_text(encoding="utf-8")
    except OSError as e:
        raise RuntimeError(f"cannot read queries file '{path}': {e}")
    queries = []
    for line in text.splitlines():
        line = line.strip()
        if not line or line.startswith("#"):
            continue
        queries.append(line)
    return queries


# ----- Research mode (v2.1.0) -----
# 给定一个主题，自动扩展多角度查询词，复用批量搜索逻辑。
# 扩展策略是确定性规则（不做 AI 判断），覆盖人物/主题/事件的通用研究维度。

# 研究角度定义：(角度标识, 后缀词)
# 顺序代表搜索优先级——基本信息优先，评价争议最后。
_RESEARCH_ANGLES = [
    ("overview",   ""),          # 主题本身：最直接的搜索
    ("profile",    "简介"),      # 基本信息：百科式介绍
    ("background", "经历"),      # 背景经历：生平/历史
    ("works",      "作品"),      # 作品成就：产出物
    ("review",     "评价"),      # 评价争议：外界看法
]


def expand_research_queries(topic: str) -> list:
    """将研究主题扩展为多角度查询词列表。

    v2.1.0 研究模式核心函数。给定一个主题（如"七森莉莉"或"Python asyncio"），
    自动生成 5 个角度的查询词，覆盖：
      1. overview   — 主题本身
      2. profile    — 基本信息（简介）
      3. background — 背景经历
      4. works      — 作品成就
      5. review     — 评价争议

    返回 [(angle, query), ...] 列表，angle 用于结果标注。

    确定性规则，不依赖 AI 判断——确保跨进程可复现，AI Agent 可预期。
    """
    topic = topic.strip()
    if not topic:
        return []
    queries = []
    for angle, suffix in _RESEARCH_ANGLES:
        query = f"{topic} {suffix}".strip()
        queries.append((angle, query))
    return queries


# ----- Main -----

def main():
    # Phase 1: pre-scan for --config / --verbose / --quiet so logging is
    # configured BEFORE config loading (which may emit warnings).
    pre = argparse.ArgumentParser(add_help=False)
    pre.add_argument("--config", default=None)
    pre.add_argument("--verbose", "-v", action="store_true", default=False)
    pre.add_argument("--quiet", action="store_true", default=False)
    pre_args, _ = pre.parse_known_args()

    setup_logging(verbose=pre_args.verbose, quiet=pre_args.quiet)
    force_utf8_stdout()  # Windows: prevent GBK crash on non-ASCII chars

    # Load config defaults from --config file, else ./searxng.toml or
    # ~/.config/searxng-cli/searxng.toml. Every CLI flag below can be
    # pre-set here; explicit flags still win.
    config = load_config(pre_args.config)

    parser = argparse.ArgumentParser(
        description="Search via a user-supplied SearXNG instance",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog=f"""searxng-cli v{VERSION}

Examples:
  %(prog)s -q "python asyncio" -i https://my-searxng.example.com
  %(prog)s -q "machine learning" -i https://a.example.com,https://b.example.com --format brief
  %(prog)s -q "climate change" -i https://s.example.com --time-range month --language en
  %(prog)s -q "open source" -i https://s.example.com --format urls > result_urls.txt
  %(prog)s -q "ai news" -i https://s.example.com --engines google,duckduckgo --method POST
  %(prog)s --verify -i https://a.example.com,https://b.example.com   # health-check, no search
  %(prog)s -q "latest news" -i https://s.example.com --time-range none   # disable time filter
  %(prog)s -q "test" --config ./my-config.toml -i https://s.example.com  # explicit config file

Instance URLs are resolved from (in priority order):
  1. -i/--instance (comma-separated for parallel failover)
  2. SEARXNG_INSTANCE environment variable
  3. ./searxng.toml or ~/.config/searxng-cli/searxng.toml  (instance = "..." or instances = [...])
     ./instances.txt or ~/.config/searxng-cli/instances.txt  (one URL per line)
At least one instance is REQUIRED (public-instance discovery has been removed).
Use --verify to health-check your instances (reachability / JSON support / latency).

searxng.toml may ALSO set defaults for most flags below (engines, categories, language,
safesearch, time_range, method, format, sort_by, timeout, max_retries, proxy, cache_ttl,
fetch, fetch_timeout, fetch_retries, max_size). Explicit CLI flags always override config values.
Use --config FILE to load a non-default config file (overrides the auto-discovered one).
        """,
    )
    parser.add_argument("--config", default=None, metavar="FILE",
                        help="Path to a searxng.toml config file. Overrides the default "
                             "auto-discovery (./searxng.toml -> ~/.config/searxng-cli/searxng.toml). "
                             "Must be the first flag if you want its values to set defaults "
                             "for other flags.")
    parser.add_argument("--verbose", "-v", action="store_true", default=False,
                        help="Verbose output: show debug-level diagnostics (HTTP request URLs, "
                             "response codes, cache keys, etc.) on stderr")
    parser.add_argument("--quiet", action="store_true", default=False,
                        help="Quiet output: suppress progress messages and retry notices on stderr; "
                             "only warnings and errors are shown. (No short flag: -q is --query)")
    parser.add_argument("--query", "-q", required=False, default=None,
                        help="Search query (required unless --verify / --queries-file is used)")
    parser.add_argument("--instance", "-i", required=False, default=None,
                        help="SearXNG instance URL(s), comma-separated for failover. "
                             "Optional if SEARXNG_INSTANCE env var or a config file is set.")
    parser.add_argument("--categories", "-c", default=config.get("categories"),
                        help="Comma-separated categories (e.g. general,images,news)")
    parser.add_argument("--language", "-l", default=config.get("language"),
                        help="Language code (e.g. en, zh-CN, de)")
    parser.add_argument("--pageno", "-p", type=int, default=1,
                        help="Page number (default: 1)")
    parser.add_argument("--time-range", "-t", choices=["day", "month", "year", "none"],
                        default=config.get("time_range", "year"),
                        help="Time range filter (default: year; 'none' disables filtering)")
    parser.add_argument("--safesearch", "-s", type=int, choices=[0, 1, 2],
                        default=_cfg_int(config, "safesearch", 0),
                        help="Safe search: 0=off, 1=moderate, 2=strict (default: 0=off)")
    parser.add_argument("--engines",
                        default=config.get("engines", "google,bing,brave,duckduckgo,startpage,wikipedia,wikidata"),
                        help="Comma-separated engine names "
                             "(default: google,bing,brave,duckduckgo,startpage,wikipedia,wikidata)")
    parser.add_argument("--method", choices=["GET", "POST"],
                        default=config.get("method", "GET"),
                        help="HTTP method (default: GET)")
    parser.add_argument("--max-results", type=int, default=None,
                        help="Limit number of results (applied AFTER dedup+sort, "
                             "so the highest-scoring/newest items are kept)")
    parser.add_argument("--sort-by", choices=["score", "date", "engine", "none"],
                        default=config.get("sort_by", "score"),
                        help="Sort results (default: score descending; 'none' preserves "
                             "instance order). Applied after dedup, before --max-results. "
                             "HTML-fallback results have no score and keep their order.")
    parser.add_argument("--no-dedup", action="store_true",
                        help="Disable cross-engine deduplication (by default, duplicate "
                             "URLs — same page ignoring tracking params/fragment — are "
                             "collapsed, keeping the first occurrence's engine/score)")
    parser.add_argument("--format", "-f", choices=["json", "brief", "urls", "csv"],
                        default=config.get("format", "json"),
                        help="Output format (default: json). 'csv' exports "
                             "title,url,engine,score,published_date,content.")
    parser.add_argument("--snippet-len", type=int, default=0,
                        help="Snippet length in brief format (default: 0 = full, no truncation)")
    parser.add_argument("--fetch", type=int, default=_cfg_int(config, "fetch", 0), metavar="N",
                        help="After search, auto-fetch full text of top N result pages")
    parser.add_argument("--fetch-timeout", type=int, default=_cfg_int(config, "fetch_timeout", 10),
                        help="Timeout per page fetch in seconds (default: 10)")
    parser.add_argument("--fetch-retries", type=int, default=_cfg_int(config, "fetch_retries", 3),
                        help="Max retries per page fetch (default: 3)")
    parser.add_argument("--fetch-report", action="store_true",
                        help="When used with --fetch, emit a structured fetch report to stderr "
                             "after completion: per-URL status, UA used, attempts, WAF type, "
                             "fallback used, and adaptive throttle stats. v2.0.0.")
    parser.add_argument("--no-fallback", action="store_true",
                        help="Disable Wayback Machine fallback for failed fetches (404/403/timeout). "
                             "By default Wayback fallback is ENABLED to maximize success rate. v2.0.0.")
    parser.add_argument("--referer", default=None, metavar="URL",
                        help="Set Referer header for fetch requests (e.g. the SearXNG instance URL). "
                             "Defaults to the instance URL when fetching result pages. v2.0.0.")
    parser.add_argument("--request-delay", type=float,
                        default=_cfg_float(config, "request_delay", 0.3),
                        metavar="SECONDS",
                        help="Delay between fetch requests to avoid rate limiting (default: 0.3s). "
                             "v2.0.0: adaptive throttling may increase this on consecutive failures.")
    parser.add_argument("--max-size", type=int, default=_cfg_int(config, "max_size", None),
                        metavar="BYTES",
                        help="Max page size in bytes (default: unlimited). Set to 5242880 for 5MB cap.")
    parser.add_argument("--auth-bearer", default=None, metavar="TOKEN",
                        help="Authorization: Bearer <TOKEN> for private instances")
    parser.add_argument("--auth-bearer-file", default=None, metavar="FILE",
                        help="Read Bearer token from a file (first non-empty, non-# line). "
                             "Avoids leaving tokens in shell history.")
    parser.add_argument("--auth-basic", default=None, metavar="USER:PASS",
                        help="Authorization: Basic base64(user:pass) for private instances")
    parser.add_argument("--auth-basic-file", default=None, metavar="FILE",
                        help="Read basic auth 'user:pass' from a file (first non-empty, non-# line). "
                             "Avoids leaving passwords in shell history. "
                             "Env var SEARXNG_BASIC_AUTH is also honored.")
    parser.add_argument("--output", "-o", default=None,
                        help="Save to file instead of stdout")
    parser.add_argument("--stream", action="store_true",
                        help="Stream results as JSON Lines (one JSON object per line) to stdout. "
                             "Each line is a {\"type\": \"result\", \"result\": {...}} event. "
                             "Ends with {\"type\": \"done\", \"count\": N}. "
                             "AI Agent can process results incrementally without waiting for full output. "
                             "Only valid with --format json.")
    parser.add_argument("--progress", action="store_true",
                        help="Emit structured progress events as JSON Lines to stderr. "
                             "Events: start, instance_try, instance_ok, instance_fail, "
                             "cache_hit, cache_store, fetch_start, fetch_ok, fetch_fail, done. "
                             "AI Agent can track execution progress programmatically.")
    parser.add_argument("--timeout", type=int, default=_cfg_int(config, "timeout", 15),
                        help="Request timeout in seconds (default: 15)")
    parser.add_argument("--retry", type=int, default=_cfg_int(config, "max_retries", None),
                        help="Max retries per instance (default: 3)")
    parser.add_argument("--fail-fast", action="store_true",
                        help="Exit on first instance failure (no multi-instance retry)")
    parser.add_argument("--serial", action="store_true",
                        help="Disable parallel multi-instance probing; search instances one at a time")
    parser.add_argument("--proxy", default=config.get("proxy"), metavar="URL",
                        help="HTTP/HTTPS proxy URL (e.g. http://corp-proxy:8080). "
                             "Applies to both search and fetch requests. "
                             "Respects existing HTTP_PROXY/HTTPS_PROXY env vars when omitted.")
    parser.add_argument("--include-domain", default=None, metavar="DOMAINS",
                        help="Comma-separated allowlist; only results from these domains are kept "
                             "(e.g. 'example.com,wikipedia.org'). Applied after search.")
    parser.add_argument("--exclude-domain", default=None, metavar="DOMAINS",
                        help="Comma-separated blocklist; results from these domains are dropped "
                             "(e.g. 'pinterest.com,quora.com'). Applied after search.")
    parser.add_argument("--queries-file", default=None, metavar="FILE",
                        help="Read queries from a file (one per line; blank lines and lines "
                             "starting with '#' are skipped) and run them in sequence. "
                             "Results are emitted as a JSON array (or one brief block per query). "
                             "Overrides --query when set.")
    parser.add_argument("--research", default=None, metavar="TOPIC",
                        help="v2.1.0 Research mode: given a topic, auto-expand into 5 "
                             "multi-angle queries (overview/profile/background/works/review) "
                             "and run them in sequence. Results are merged and deduplicated. "
                             "Output includes research_topic and research_queries metadata. "
                             "Mutually exclusive with --query and --queries-file.")
    parser.add_argument("--verify", action="store_true",
                        help="Health-check mode: verify instances (reachability/JSON/latency) and exit without searching")
    parser.add_argument("--cache-ttl", type=int, default=_cfg_int(config, "cache_ttl", 0),
                        metavar="MINUTES",
                        help="Cache search results for N minutes (default: 0 = disabled). "
                             "Identical queries within the TTL skip the network entirely. "
                             "Cache lives at $SEARXNG_CACHE_DIR or ~/.cache/searxng-cli/cache.db")
    parser.add_argument("--clear-cache", action="store_true",
                        help="Delete all cached entries and exit (no search performed)")
    parser.add_argument("--cache-stats", action="store_true",
                        help="Print cache statistics (entry count, age, size, path) and exit")
    parser.add_argument("--dump-schema", action="store_true",
                        help="Print the JSON Schema for --format json output and exit. "
                             "Lets AI agents programmatically discover field names and types.")
    parser.add_argument("--version", action="version", version=f"searxng-cli v{VERSION}")

    args = parser.parse_args()

    # --dump-schema：输出 JSON Schema 到 stdout 并退出，AI Agent 可程序化发现字段
    if getattr(args, "dump_schema", False):
        print(json.dumps(_get_output_schema(), indent=2, ensure_ascii=False))
        sys.exit(0)

    # 启用 --progress 进度事件（JSON Lines 到 stderr）
    set_progress_enabled(getattr(args, "progress", False))

    # --stream 只在单查询 + --format json 下有效。batch 模式输出 JSON 数组，
    # 非 json 格式无 JSON Lines 语义；两种组合都显式报错 E_INPUT，避免静默失效
    # 让 AI Agent 误以为流式输出已生效。
    if getattr(args, "stream", False):
        if args.queries_file:
            _emit_error("--stream cannot be used with --queries-file: batch mode "
                        "emits a JSON array, not JSON Lines. Drop --stream for "
                        "batch output, or use a single --query with --stream.",
                        args, error_code=E_INPUT)
        if args.research:
            _emit_error("--stream cannot be used with --research: research mode "
                        "emits a JSON array, not JSON Lines. Drop --stream for "
                        "research output, or use a single --query with --stream.",
                        args, error_code=E_INPUT)
        if args.format != "json":
            _emit_error(f"--stream requires --format json (current: {args.format}). "
                        "JSON Lines streaming only produces valid output with json format.",
                        args, error_code=E_INPUT)

    # --research 与 --query / --queries-file 互斥
    if args.research:
        if args.query:
            _emit_error("--research cannot be used with --query: research mode "
                        "auto-generates queries from the topic. Drop --query, "
                        "or use --research <topic> alone.",
                        args, error_code=E_INPUT)
        if args.queries_file:
            _emit_error("--research cannot be used with --queries-file: research "
                        "mode auto-generates queries. Drop --queries-file, "
                        "or use --research <topic> alone.",
                        args, error_code=E_INPUT)

    # --query is required unless we're doing a non-search operation.
    # --queries-file is an alternative to --query for batch mode.
    # --research is another alternative (v2.1.0).
    if (not args.verify and not args.query and not args.queries_file
            and not args.research
            and not args.clear_cache and not args.cache_stats):
        parser.error("--query is required (or use --verify / --queries-file / "
                     "--research / --clear-cache / --cache-stats)")

    # Apply proxy early so every HTTP path (search, verify, fetch) honors it.
    # Setting env vars is enough: urllib reads them via getproxies() and
    # requests honors them via trust_env=True (its default).
    if args.proxy:
        apply_proxy(args.proxy)
        logger.info(f"Proxy: {args.proxy}")

    # --clear-cache / --cache-stats 不需要实例，在实例解析之前处理并退出。
    # 避免无 -i 时报 E_CONFIG "no instance resolved" 让 AI 困惑。
    if args.clear_cache:
        removed = cache_module.clear()
        logger.info(f"Cleared {removed} cache entr{'y' if removed == 1 else 'ies'}.")
        sys.exit(0)

    if args.cache_stats:
        s = cache_module.stats()
        if args.format == "json":
            # JSON 模式：结构化数据走 stdout，AI Agent 可管道解析
            print(json.dumps(s, indent=2, ensure_ascii=False))
        else:
            # 非 JSON 模式：人类可读的状态信息走 stderr，保持 stdout 纯净，
            # 避免 AI Agent 用 --format json 解析 stdout 时被非 JSON 污染。
            print(f"Cache path:  {s.get('path', '?')}", file=sys.stderr)
            print(f"Entries:     {s.get('entries', 0)}", file=sys.stderr)
            size = s.get("size_bytes", 0)
            print(f"Size:        {size:,} bytes ({size / 1024:.1f} KB)", file=sys.stderr)
            if s.get("oldest_created_at"):
                print(f"Oldest:      {time.ctime(s['oldest_created_at'])}", file=sys.stderr)
            if s.get("newest_created_at"):
                print(f"Newest:      {time.ctime(s['newest_created_at'])}", file=sys.stderr)
            if s.get("error"):
                print(f"Error:       {s['error']}", file=sys.stderr)
        sys.exit(0)

    # Resolve instance(s): -i > SEARXNG_INSTANCE env > config file
    instance_urls = resolve_instances(args.instance)
    # Fallback: if --config was used, instance may be in the config dict
    # but not in the default search paths that resolve_instances checks.
    if not instance_urls and config.get("instance"):
        instance_urls = parse_instances(config["instance"])
    elif not instance_urls and config.get("instances"):
        raw = config["instances"]
        if isinstance(raw, str):
            instance_urls = parse_instances(raw)
        elif isinstance(raw, list):
            instance_urls = [u if u.startswith(("http://", "https://"))
                             else "https://" + u for u in raw if u and str(u).strip()]
    if not instance_urls:
        # Python 3.8-3.10 无 tomllib 时，.toml 配置文件无法读取。检查这种
        # 情况并在错误信息中附加提示，让 AI Agent 能给出可操作的恢复建议。
        toml_hint = ""
        try:
            import tomllib  # Python 3.11+
        except ModuleNotFoundError:
            try:
                import tomli  # type: ignore[import-not-found]
            except ModuleNotFoundError:
                toml_candidates = [
                    Path.cwd() / "searxng.toml",
                    Path.home() / ".config" / "searxng-cli" / "searxng.toml",
                ]
                if any(p.exists() for p in toml_candidates):
                    toml_hint = (" (hint: a searxng.toml file exists but cannot be "
                                 "read on Python < 3.11 without the 'tomli' package. "
                                 "Run 'pip install tomli' or use instances.txt instead.)")
        _emit_error("no SearXNG instance resolved. Provide -i/--instance, set the "
                    "SEARXNG_INSTANCE environment variable, or create a searxng.toml / "
                    "instances.txt config file." + toml_hint, args, error_code=E_CONFIG)

    # Build auth headers if provided (needed by both verify and search).
    # Credentials may come from CLI flag, file, config file, or env var
    # (in priority order) to avoid leaking secrets via shell history or `ps`.
    try:
        bearer_token = resolve_auth_bearer(
            args.auth_bearer, args.auth_bearer_file,
            config_value=config.get("auth_bearer"))
        basic_auth = resolve_auth_basic(
            args.auth_basic, args.auth_basic_file,
            config_value=config.get("auth_basic"))
    except RuntimeError as e:
        _emit_error(str(e), args, error_code=E_AUTH)
    auth_headers = build_auth_headers(
        bearer_token=bearer_token,
        basic_auth=basic_auth,
    )
    if auth_headers:
        auth_type = "Bearer" if bearer_token else "Basic"
        logger.info(f"Auth: {auth_type} ***")

    # Health-check mode: report instance status and exit (no search performed)
    if args.verify:
        logger.info(f"Verifying {len(instance_urls)} instance(s)...")
        report = verify_instances(instance_urls, timeout=args.timeout,
                                  auth_headers=auth_headers)
        _print_verify_report(report, as_json=(args.format == "json"))
        sys.exit(0)

    if args.fail_fast:
        instance_urls = instance_urls[:1]

    logger.info(f"Instances to try: {len(instance_urls)}")

    # Cache TTL in seconds (CLI takes minutes for ergonomics)
    ttl_seconds = args.cache_ttl * 60 if args.cache_ttl > 0 else 0

    # ----- Research mode: --research (v2.1.0) -----
    # 给定主题，自动扩展为 5 个多角度查询，串行搜索，输出带研究元数据的 JSON。
    # 复用批量搜索逻辑，但查询词来自 expand_research_queries 而非文件。
    if args.research:
        topic = args.research.strip()
        research_queries = expand_research_queries(topic)
        if not research_queries:
            _emit_error(f"research topic is empty after stripping", args,
                        error_code=E_INPUT)

        logger.info(f"Research mode: {len(research_queries)} angles for '{topic}'")
        batch = []
        any_with_results = False
        error_count = 0
        for i, (angle, q) in enumerate(research_queries, 1):
            logger.info(f"\n[{i}/{len(research_queries)}] [{angle}] {q}")
            results, err, err_code = _run_single_query(q, args, instance_urls,
                                                        auth_headers, ttl_seconds)
            if err:
                error_count += 1
                logger.error(f"  [ERROR] {err}")
                entry = {"query": q, "angle": angle, "status": "error",
                         "error": err}
                if err_code:
                    entry["error_code"] = err_code
                batch.append(entry)
            else:
                if len(results.get("results", [])) > 0:
                    any_with_results = True
                batch.append({"query": q, "angle": angle, "status": "ok",
                              "results": results})

        # 输出
        if args.format == "json":
            output = json.dumps({
                "schema_version": SCHEMA_VERSION,
                "research_topic": topic,
                "research_queries": [
                    {"angle": a, "query": q} for a, q in research_queries
                ],
                "queries": batch,
            }, indent=2, ensure_ascii=False)
        elif args.format == "csv":
            import csv as csv_mod
            import io
            out = io.StringIO()
            writer = csv_mod.writer(out, lineterminator="\n")
            writer.writerow(["angle", "query", "title", "url", "engine",
                             "score", "published_date", "content"])
            for br in batch:
                q = br["query"]
                angle = br.get("angle", "")
                if "results" in br:
                    for r in br["results"].get("results", []):
                        writer.writerow([
                            angle, q,
                            r.get("title", ""),
                            r.get("url", ""),
                            r.get("engine", ""),
                            r.get("score", "") if r.get("score") is not None else "",
                            r.get("published_date", ""),
                            r.get("content", ""),
                        ])
                else:
                    writer.writerow([angle, q, "", "", "", "", "",
                                     f"[ERROR: {br['error']}]"])
            output = out.getvalue().rstrip()
        elif args.format == "urls":
            parts = []
            for br in batch:
                parts.append(f"# [{br.get('angle', '?')}] {br['query']}")
                if "results" in br:
                    parts.append(format_urls(br["results"]))
                else:
                    parts.append(f"# [ERROR: {br['error']}]")
            output = "\n".join(parts)
        else:  # brief
            parts = []
            for br in batch:
                parts.append("=" * 60)
                parts.append(f"[{br.get('angle', '?')}] {br['query']}")
                parts.append("=" * 60)
                if "results" in br:
                    parts.append(format_brief(br["results"]))
                else:
                    parts.append(f"[ERROR: {br['error']}]")
                parts.append("")
            output = "\n".join(parts)

        if args.output:
            with open(args.output, "w", encoding="utf-8") as f:
                f.write(output)
            logger.info(f"Saved results to {args.output}")
        else:
            print(output)
        # 三态退出码：0=有结果, 2=全部空, 1=全部错误
        if error_count == len(research_queries):
            sys.exit(1)
        elif not any_with_results:
            sys.exit(2)
        sys.exit(0)

    # ----- Batch mode: --queries-file -----
    # Reads one query per line (blank/# lines skipped) and runs them in
    # sequence. Output is a JSON array (json format) or concatenated blocks
    # (brief/urls). A failed query is recorded but does not abort the batch.
    if args.queries_file:
        try:
            queries = _read_queries_file(args.queries_file)
        except RuntimeError as e:
            _emit_error(str(e), args, error_code=E_INPUT)
        if not queries:
            _emit_error(f"no queries found in '{args.queries_file}'", args,
                        error_code=E_INPUT)

        logger.info(f"Running {len(queries)} queries from {args.queries_file}...")
        batch = []
        any_with_results = False
        error_count = 0
        for i, q in enumerate(queries, 1):
            logger.info(f"\n[{i}/{len(queries)}] {q}")
            results, err, err_code = _run_single_query(q, args, instance_urls,
                                                        auth_headers, ttl_seconds)
            if err:
                error_count += 1
                logger.error(f"  [ERROR] {err}")
                entry = {"query": q, "status": "error", "error": err}
                if err_code:
                    entry["error_code"] = err_code
                batch.append(entry)
            else:
                if len(results.get("results", [])) > 0:
                    any_with_results = True
                batch.append({"query": q, "status": "ok", "results": results})

        if args.format == "json":
            output = json.dumps({"schema_version": SCHEMA_VERSION, "queries": batch},
                                indent=2, ensure_ascii=False)
        elif args.format == "csv":
            import csv as csv_mod
            import io
            out = io.StringIO()
            writer = csv_mod.writer(out, lineterminator="\n")
            writer.writerow(["query", "title", "url", "engine", "score",
                             "published_date", "content"])
            for br in batch:
                q = br["query"]
                if "results" in br:
                    for r in br["results"].get("results", []):
                        writer.writerow([
                            q,
                            r.get("title", ""),
                            r.get("url", ""),
                            r.get("engine", ""),
                            r.get("score", "") if r.get("score") is not None else "",
                            r.get("published_date", ""),
                            r.get("content", ""),
                        ])
                else:
                    writer.writerow([q, "", "", "", "", "",
                                     f"[ERROR: {br['error']}]"])
            output = out.getvalue().rstrip()
        elif args.format == "urls":
            parts = []
            for br in batch:
                parts.append(f"# {br['query']}")
                if "results" in br:
                    parts.append(format_urls(br["results"]))
                else:
                    parts.append(f"# [ERROR: {br['error']}]")
            output = "\n".join(parts)
        else:  # brief
            parts = []
            for br in batch:
                parts.append("=" * 60)
                parts.append(f"QUERY: {br['query']}")
                parts.append("=" * 60)
                if "results" in br:
                    parts.append(_format_results(br["results"], args))
                else:
                    parts.append(f"[ERROR: {br['error']}]")
                parts.append("")
            output = "\n".join(parts)

        if args.output:
            with open(args.output, "w", encoding="utf-8") as f:
                f.write(output)
            logger.info(f"Saved results to {args.output}")
        else:
            print(output)
        # Exit code semantics — aligned with single-query mode so AI agents
        # can use one consistent rule:
        #   1 = all queries errored (fatal)
        #   2 = no query returned any results (empty), though at least one
        #       searched successfully without error
        #   0 = at least one query returned results
        if error_count == len(queries):
            sys.exit(1)
        if not any_with_results:
            sys.exit(2)
        sys.exit(0)

    # ----- Single query mode -----
    results, err, err_code = _run_single_query(args.query, args, instance_urls,
                                                auth_headers, ttl_seconds)

    # --stream: JSON Lines 流式输出，每条结果一行，AI 可增量处理。
    # stream 模式下所有输出（包括错误）都是单行 JSON，保持 JSON Lines 格式一致性。
    # 非 stream 模式的错误走 _emit_error（多行 JSON 或 stderr 文本）。
    if getattr(args, "stream", False) and args.format == "json":
        if err:
            error_event = {"type": "error", "error": err, "query": args.query}
            if err_code:
                error_event["error_code"] = err_code
                hint = RECOVERY_HINTS.get(err_code)
                if hint:
                    error_event["recovery_hint"] = hint
            print(json.dumps(error_event, ensure_ascii=False), flush=True)
            sys.exit(1)
        for r in results.get("results", []):
            print(json.dumps({"type": "result", "result": r},
                             ensure_ascii=False), flush=True)
        print(json.dumps({"type": "done",
                          "schema_version": SCHEMA_VERSION,
                          "count": len(results.get("results", [])),
                          "query": args.query}, ensure_ascii=False), flush=True)
        if not results.get("results"):
            sys.exit(2)
        sys.exit(0)

    if err:
        _emit_error(err, args, query=args.query, error_code=err_code)

    output = _format_results(results, args)
    if args.output:
        with open(args.output, "w", encoding="utf-8") as f:
            f.write(output)
        logger.info(f"Saved results to {args.output}")
    else:
        print(output)

    if not results.get("results"):
        logger.warning("Warning: no results returned")
        sys.exit(2)


if __name__ == "__main__":
    main()