searxng-use-cli/scripts/common.py

"""Shared utilities for searxng-cli scripts.

This module centralizes code that was previously duplicated across
``search.py`` and ``fetch.py``:

* ``build_auth_headers``  — construct an Authorization header from CLI flags
* ``resolve_auth_basic``  — resolve basic-auth credentials from file/env/CLI
                            (avoids leaving passwords in shell history)
* ``detect_charset``      — guess a response's text encoding
* ``is_retryable_error``  — unified transient-error policy (urllib + requests)
* ``FALLBACK_UAS``        — browser-like User-Agents used when blocked
* retry constants         — ``RETRYABLE_STATUS``, ``RETRY_BACKOFF_BASE``, etc.
* ``setup_logging``       — shared logging configuration (--verbose/--quiet)
* ``force_utf8_stdout``   — force stdout to UTF-8 (fix Windows GBK crashes)

Centralizing the retry policy guarantees that both scripts treat 429/5xx
as retryable and connection errors as transient, eliminating the previous
inconsistency where ``search.py`` ignored 5xx.
"""

import io
import logging
import sys
import urllib.error

# Root logger for the searxng-cli package. All modules create child loggers
# via ``logging.getLogger("searxng.<module>")`` so a single setup_logging()
# call controls them all.
_LOG = logging.getLogger("searxng")


def setup_logging(verbose: bool = False, quiet: bool = False) -> None:
    """Configure the ``searxng`` logger hierarchy.

    * Default (no flags): ``INFO`` — progress messages, warnings, retry notices.
      Matches the previous ``print(..., file=sys.stderr)`` behavior so existing
      scripts and agents see no change.
    * ``--verbose`` / ``-v``: ``DEBUG`` — also shows HTTP request URLs, response
      status codes, cache keys, and other diagnostic detail.
    * ``--quiet`` / ``-q``: ``WARNING`` — suppresses progress and retry noise;
      only warnings and errors reach stderr.

    All log output goes to stderr; stdout is reserved for data (JSON/CSV/etc.).
    """
    if verbose:
        level = logging.DEBUG
    elif quiet:
        level = logging.WARNING
    else:
        level = logging.INFO

    _LOG.setLevel(level)
    # Avoid duplicate handlers if setup_logging() is called twice (e.g. tests).
    if not _LOG.handlers:
        handler = logging.StreamHandler(sys.stderr)
        handler.setFormatter(logging.Formatter("%(message)s"))
        _LOG.addHandler(handler)
    # Don't let root logger add its own handler — we own the searxng namespace.
    _LOG.propagate = False


def force_utf8_stdout() -> None:
    """Force stdout/stderr to UTF-8 to prevent Windows GBK encoding crashes.

    Windows Python defaults ``sys.stdout`` to the OEM codepage (often GBK on
    Chinese Windows). ``print()`` of any character outside that codepage
    (e.g. ``\\xa0`` nbsp, CJK punctuation from foreign pages) raises
    ``UnicodeEncodeError`` and kills the process.

    ``PYTHONIOENCODING=utf-8`` is unreliable here because Python 3.7+
    reconfigures stdout after reading that env var in some scenarios (e.g.
    when stdout has already been wrapped). The only reliable fix is to
    reconfigure the stream in-process.

    Uses ``sys.stdout.reconfigure()`` on Python 3.7+, falling back to
    wrapping ``sys.stdout.buffer`` on older versions. Both paths use
    ``errors='replace'`` so an undecodable byte never crashes the script —
    better to emit ``?`` than to lose all output.

    Safe to call multiple times; subsequent calls are no-ops once the
    encoding is already UTF-8 (or close enough — we check the lowercased
    encoding name to tolerate ``utf-8`` vs ``UTF-8`` vs ``utf8``).
    """
    for stream_name in ("stdout", "stderr"):
        stream = getattr(sys, stream_name, None)
        if stream is None:
            continue
        # Already UTF-8? Skip (covers Linux/macOS and re-invoked scripts).
        enc = getattr(stream, "encoding", "") or ""
        if enc.lower().replace("-", "") in ("utf8", "utf-8-sig"):
            continue
        # Python 3.7+ has TextIOWrapper.reconfigure()
        reconfigure = getattr(stream, "reconfigure", None)
        if reconfigure is not None:
            try:
                reconfigure(encoding="utf-8", errors="replace")
                continue
            except (ValueError, OSError):
                pass  # Fall through to the buffer-wrap path
        # Fallback: wrap the underlying buffer in a new UTF-8 stream.
        buffer = getattr(stream, "buffer", None)
        if buffer is not None:
            try:
                new_stream = io.TextIOWrapper(
                    buffer, encoding="utf-8", errors="replace", line_buffering=True,
                )
                setattr(sys, stream_name, new_stream)
            except (ValueError, AttributeError):
                # Last resort: keep the original stream. Better to risk a
                # GBK crash on exotic characters than to break stdout entirely.
                pass

# Retry settings (shared by both scripts)
MAX_RETRIES = 3
RETRY_BACKOFF_BASE = 1.5  # seconds; exponential backoff + jitter
RETRY_BACKOFF_CAP = 60.0  # 退避上限：1.5*2^N 无封顶时 N=10 达 1536s，会卡死进程

# HTTP status codes worth retrying. 403 is included so fetch_url's UA-fallback
# loop can kick in when a site blocks the default searxng-cli User-Agent
# (a fresh UA is tried on each retry attempt). search.py also retries 403 —
# it doesn't switch UAs, so true auth failures waste ~3 attempts, accepted
# as a trade-off for one shared retry policy across both scripts.
RETRYABLE_STATUS = frozenset({403, 429, 502, 503, 504})

# Browser-like UA strings for fallback when the searxng-cli UA is blocked.
#
# v2.0.0 扩充至 12 个：覆盖 Chrome/Edge/Firefox × Windows/macOS/Linux，
# 每个都是较新版本（131/130/129），避免被识别为过时浏览器。
# 顺序固定以便 get_ua_for_domain() 的 hash 选择可复现。
FALLBACK_UAS = [
    # Chrome 131 — Windows / macOS / Linux
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
    # Edge 131 — Windows / macOS
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
    # Firefox 133 — Windows / macOS / Linux
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) "
    "Gecko/20100101 Firefox/133.0",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:133.0) "
    "Gecko/20100101 Firefox/133.0",
    "Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0",
    # Chrome 130 — Windows / macOS (上一个版本，应对 131 被针对性识别)
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
    # Chrome 129 — Windows / Linux
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
]


def _ua_index_for_domain(domain: str, pool_size: int) -> int:
    """为域名确定性选择 UA 池索引。

    用 ``hashlib.sha256`` 而非内置 ``hash()``，因为后者对字符串做了
    随机化（PYTHONHASHSEED），跨进程不可复现。SHA-256 保证同一域名
    永远映射到同一索引，跨进程一致——这对调试和日志分析至关重要。
    """
    import hashlib
    h = hashlib.sha256(domain.encode("utf-8")).digest()
    # 取前 8 字节作为无符号整数，避免负数和短字符串的分布不均
    return int.from_bytes(h[:8], "big") % pool_size


# Per-domain UA 缓存：同一域名 + 同一进程 = 同一 UA，避免会话内 UA 突变
# 被反爬识别。跨进程通过 SHA-256 哈希复现，见 _ua_index_for_domain。
_domain_ua_cache: dict = {}


def get_ua_for_domain(url: str, user_agent: str = None) -> str:
    """返回适合某域名的 User-Agent。

    优先级：
      1. ``user_agent`` 显式传入（CLI --user-agent）→ 直接返回
      2. 该域名已缓存 → 返回缓存值
      3. 域名未缓存 → 用 SHA-256 hash 选一个 FALLBACK_UAS，缓存并返回

    设计理由：真实浏览器访问同一站点时 UA 永远不变。爬虫如果每次请求
    换一个 UA，反而会被反爬系统标记为可疑。确定性轮换保证同一域名
    稳定使用同一 UA，不同域名分散到不同 UA 上降低集体封禁风险。
    """
    if user_agent:
        return user_agent

    import urllib.parse as _up
    try:
        domain = _up.urlparse(url).netloc.lower()
        if not domain:
            return FALLBACK_UAS[0]
    except Exception:
        return FALLBACK_UAS[0]

    if domain in _domain_ua_cache:
        return _domain_ua_cache[domain]

    idx = _ua_index_for_domain(domain, len(FALLBACK_UAS))
    ua = FALLBACK_UAS[idx]
    _domain_ua_cache[domain] = ua
    return ua


def reset_domain_ua_cache() -> None:
    """清空 per-domain UA 缓存。测试用。"""
    _domain_ua_cache.clear()


def build_browser_headers(user_agent: str, referer: str = None,
                          accept_html: bool = True) -> dict:
    """构造完整的浏览器请求头，让请求看起来像真浏览器。

    v2.0.0 核心反爬措施：仅靠 User-Agent 已无法绕过现代 WAF，
    Cloudflare/Akamai/Imperva 都会检查 Sec-* 头和 Accept-Language。

    Args:
        user_agent: UA 字符串（应来自 get_ua_for_domain）
        referer: Referer URL（可选；从搜索结果抓取时设为实例 URL）
        accept_html: True 时 Accept 包含 text/html（页面抓取）；
                     False 时 Accept 为 application/json（API 调用）

    Returns:
        包含完整浏览器指纹的 headers dict。调用方需自行合并 auth_headers。
    """
    is_firefox = "Firefox/" in user_agent
    is_edge = "Edg/" in user_agent

    if accept_html:
        accept = ("text/html,application/xhtml+xml,application/xml;q=0.9,"
                  "image/avif,image/webp,*/*;q=0.8")
    else:
        accept = "application/json, text/plain, */*;q=0.8"

    headers = {
        "User-Agent": user_agent,
        "Accept": accept,
        "Accept-Language": "en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7",
        "Accept-Encoding": "gzip, deflate" if is_firefox else "gzip, deflate, br",
        "Connection": "keep-alive",
        "Upgrade-Insecure-Requests": "1" if accept_html else "0",
    }

    # Sec-Ch-Ua 系列仅 Chrome/Edge 发送，Firefox 不发
    if not is_firefox:
        # 从 UA 提取主版本号，构造 Sec-Ch-Ua
        import re
        m = re.search(r"Chrome/(\d+)", user_agent)
        ver = m.group(1) if m else "131"
        not_a_brand = '"Not_A Brand";v="99"' if ver != "99" else '"Not/A)Brand";v="99"'
        headers["Sec-Ch-Ua"] = f'"{not_a_brand}", "Chromium";v="{ver}", "Google Chrome";v="{ver}"'
        if is_edge:
            # Edge 的品牌标识
            headers["Sec-Ch-Ua"] = headers["Sec-Ch-Ua"].rstrip('"') + f'", "Microsoft Edge";v="{ver}"'
        headers["Sec-Ch-Ua-Mobile"] = '"?1"' if "Mobile" in user_agent else '"?0"'
        # 平台标识
        if "Windows" in user_agent:
            headers["Sec-Ch-Ua-Platform"] = '"Windows"'
        elif "Macintosh" in user_agent:
            headers["Sec-Ch-Ua-Platform"] = '"macOS"'
        elif "Linux" in user_agent:
            headers["Sec-Ch-Ua-Platform"] = '"Linux"'
        # Sec-Fetch 系列（Chrome 76+ 全量发送）
        if accept_html:
            headers["Sec-Fetch-Site"] = "none" if not referer else "cross-site"
            headers["Sec-Fetch-Mode"] = "navigate"
            headers["Sec-Fetch-User"] = "?1"
            headers["Sec-Fetch-Dest"] = "document"
        else:
            headers["Sec-Fetch-Site"] = "same-origin" if referer else "none"
            headers["Sec-Fetch-Mode"] = "cors"
            headers["Sec-Fetch-Dest"] = "empty"

    if referer:
        headers["Referer"] = referer

    return headers


def parse_retry_after(header_value: str) -> float:
    """解析 Retry-After header，返回应等待的秒数。

    HTTP 规范允许两种格式：
      1. 纯数字：秒数（最常见）
      2. HTTP date：绝对时间（如 ``Wed, 21 Oct 2026 07:28:00 GMT``）

    返回 0.0 表示无需等待或解析失败。对 HTTP date 格式，若已过期
    也返回 0.0（让调用方立即重试）。
    """
    if not header_value:
        return 0.0
    header_value = header_value.strip()

    # 格式 1：纯数字秒数
    try:
        seconds = float(header_value)
        return max(0.0, seconds)
    except ValueError:
        pass

    # 格式 2：HTTP date
    try:
        from email.utils import parsedate_to_datetime
        from datetime import datetime, timezone
        dt = parsedate_to_datetime(header_value)
        if dt is None:
            return 0.0
        # 确保 timezone-aware
        if dt.tzinfo is None:
            dt = dt.replace(tzinfo=timezone.utc)
        now = datetime.now(timezone.utc)
        delta = (dt - now).total_seconds()
        return max(0.0, delta)
    except (TypeError, ValueError, OverflowError):
        return 0.0


def compute_backoff_delay(attempt: int, base: float = RETRY_BACKOFF_BASE,
                          cap: float = RETRY_BACKOFF_CAP) -> float:
    """计算退避延迟，带封顶和抖动。

    ``base * 2^attempt + jitter``，但不超过 ``cap``。
    v2.0.0 新增封顶：原公式无上限，N=10 时达 1536s 会卡死进程。
    """
    import random
    delay = base * (2 ** attempt) + random.uniform(0, 1)
    return min(delay, cap)


def build_auth_headers(bearer_token: str = None, basic_auth: str = None) -> dict:
    """Build an Authorization header dict from CLI auth flags.

    ``bearer_token``: raw Bearer token string.
    ``basic_auth``:   ``"username:password"`` string (base64-encoded).

    If both are provided, Bearer takes precedence (more common for APIs).
    Returns a dict to merge into request headers, or an empty dict.
    """
    import base64

    headers = {}
    if bearer_token:
        headers["Authorization"] = f"Bearer {bearer_token}"
    elif basic_auth:
        encoded = base64.b64encode(basic_auth.encode("utf-8")).decode("ascii")
        headers["Authorization"] = f"Basic {encoded}"
    return headers


def _warn_file_perms(path: str) -> None:
    """Warn if a credentials file is readable by group/other (POSIX only).

    On Windows the Unix permission bits in ``st_mode`` do not reflect the
    actual ACL, so the check is skipped to avoid false alarms.
    """
    import os
    if os.name != "posix":
        return
    log = logging.getLogger("searxng.common")
    try:
        mode = os.stat(path).st_mode & 0o777
        if mode & 0o077:
            log.warning(
                f"Warning: credentials file '{path}' has permissions {oct(mode)} "
                f"(accessible by group/other); recommend 'chmod 600' for security."
            )
    except OSError:
        pass


def resolve_auth_basic(cli_value: str = None, file_path: str = None,
                       env_var: str = "SEARXNG_BASIC_AUTH",
                       config_value: str = None) -> str:
    """Resolve basic-auth credentials without leaking them via shell history.

    Priority (highest wins):
      1. ``cli_value``  — explicit ``--auth-basic "user:pass"`` (convenient
                          but leaks into shell history; discouraged)
      2. ``file_path``  — ``--auth-basic-file FILE``; first non-empty line
                          is read as ``user:pass``. Recommended for shells.
      3. ``config_value`` — ``auth_basic`` field from ``searxng.toml``.
                            Convenient for AI agents that read config once.
      4. ``env_var``    — ``SEARXNG_BASIC_AUTH`` environment variable.

    Returns ``"user:pass"`` or ``None`` if no source provides credentials.
    Raises ``RuntimeError`` if a file is specified but cannot be read.
    """
    if cli_value:
        return cli_value

    if file_path:
        try:
            from pathlib import Path
            text = Path(file_path).read_text(encoding="utf-8")
            _warn_file_perms(file_path)
            for line in text.splitlines():
                line = line.strip()
                if line and not line.startswith("#"):
                    return line
            raise RuntimeError(f"auth file '{file_path}' contains no credentials")
        except OSError as e:
            raise RuntimeError(f"cannot read auth file '{file_path}': {e}") from e

    if config_value:
        return config_value

    import os
    return os.environ.get(env_var)


def resolve_auth_bearer(cli_value: str = None, file_path: str = None,
                        env_var: str = "SEARXNG_BEARER_TOKEN",
                        config_value: str = None) -> str:
    """Resolve a Bearer token from CLI flag, file, config, or environment.

    Mirrors :func:`resolve_auth_basic` for token-style auth. Useful for
    long-lived API tokens that should not appear in shell history.

    Priority (highest wins):
      1. ``cli_value``  — explicit ``--auth-bearer "token"``
      2. ``file_path``  — ``--auth-bearer-file FILE``
      3. ``config_value`` — ``auth_bearer`` field from ``searxng.toml``
      4. ``env_var``    — ``SEARXNG_BEARER_TOKEN`` environment variable
    """
    if cli_value:
        return cli_value

    if file_path:
        try:
            from pathlib import Path
            text = Path(file_path).read_text(encoding="utf-8")
            _warn_file_perms(file_path)
            for line in text.splitlines():
                line = line.strip()
                if line and not line.startswith("#"):
                    return line
            raise RuntimeError(f"token file '{file_path}' contains no token")
        except OSError as e:
            raise RuntimeError(f"cannot read token file '{file_path}': {e}") from e

    if config_value:
        return config_value

    import os
    return os.environ.get(env_var)


def apply_proxy(proxy_url: str) -> None:
    """Configure proxy via environment variables.

    Sets ``HTTP_PROXY`` and ``HTTPS_PROXY`` so both urllib (which reads
    them via :func:`urllib.request.getproxies`) and ``requests`` (which
    honors them when ``trust_env=True``, the default) pick up the proxy
    without any changes to call sites.

    ``NO_PROXY`` is set to ``localhost,127.0.0.1,::1`` (if not already set)
    so local traffic stays direct — matters for self-hosted SearXNG on
    localhost behind a corporate proxy.

    Pass an empty string to clear the proxy env vars (rarely needed; the
    default unset state already means "no proxy").
    """
    import os
    if not proxy_url:
        return
    os.environ["HTTP_PROXY"] = proxy_url
    os.environ["HTTPS_PROXY"] = proxy_url
    # Keep local traffic direct unless the user has explicitly set NO_PROXY
    os.environ.setdefault("NO_PROXY", "localhost,127.0.0.1,::1")


def detect_charset(raw: bytes, content_type: str) -> str:
    """Detect charset from the Content-Type header, then an HTML meta tag.

    Falls back to UTF-8 (with replacement) if nothing reliable is found.
    """
    import re

    # 1. HTTP header
    if "charset=" in content_type:
        charset = content_type.split("charset=")[-1].split(";")[0].strip()
        try:
            raw.decode(charset)
            return charset
        except (UnicodeDecodeError, LookupError):
            pass

    # 2. HTML <meta charset> or <meta http-equiv>
    try:
        head = raw[:4096].decode("ascii", errors="replace")
        m = re.search(r'<meta[^>]+charset=["\']?([a-zA-Z0-9_-]+)', head, re.IGNORECASE)
        if m:
            charset = m.group(1).strip()
            try:
                raw.decode(charset)
                return charset
            except (UnicodeDecodeError, LookupError):
                pass
    except Exception:
        pass

    # 3. Fallback: UTF-8 with replacement
    return "utf-8"


def is_retryable_error(exc: BaseException) -> bool:
    """Return True if ``exc`` is a transient error worth retrying.

    Handles both the stdlib ``urllib`` errors and ``requests`` errors via
    duck-typing (so this module does not need to import ``requests``):

    * ``urllib.error.HTTPError`` → retry iff status in ``RETRYABLE_STATUS``
    * ``urllib.error.URLError`` / ``OSError`` / ``TimeoutError`` → retry
      (connection refused, DNS failure, timeout — all transient)
    * ``requests.exceptions.HTTPError`` → retry iff ``response.status_code``
      is in ``RETRYABLE_STATUS``
    * ``requests`` connection/timeout errors (no ``.response``) → retry
    """
    if isinstance(exc, urllib.error.HTTPError):
        return exc.code in RETRYABLE_STATUS
    if isinstance(exc, (urllib.error.URLError, OSError, TimeoutError)):
        return not isinstance(exc, urllib.error.HTTPError)

    # requests.exceptions.HTTPError / RequestException (duck-typed)
    resp = getattr(exc, "response", None)
    status = getattr(resp, "status_code", None)
    if status is not None:
        return status in RETRYABLE_STATUS
    if resp is None:
        # requests connection/timeout error without a response -> transient
        return True
    return False


# ----- Structured error classification -----
#
# 错误码体系：让 AI Agent 程序化地判断错误类型并采取恢复策略。
# 所有错误码以 E_ 前缀，在 --format json 模式下随 error_code 字段输出。
#
# AI 可根据 error_code 决策：
#   E_CONFIG     → 检查实例配置/环境变量，提示用户设置
#   E_AUTH       → 检查 token/凭证，提示用户重新认证
#   E_NETWORK    → 重试或切换实例/代理
#   E_RATE_LIMIT → 等待后重试，降低请求频率
#   E_PARSE      → 检查实例是否支持 JSON，尝试 HTML 回退
#   E_EMPTY      → 调整查询词或时间范围
#   E_INPUT      → 修正参数/文件路径
#   E_INTERNAL   → 报告 bug，附带完整错误信息

# 错误码常量（供 search.py / fetch.py 引用）
E_CONFIG = "E_CONFIG"
E_AUTH = "E_AUTH"
E_NETWORK = "E_NETWORK"
E_RATE_LIMIT = "E_RATE_LIMIT"
E_PARSE = "E_PARSE"
E_EMPTY = "E_EMPTY"
E_INPUT = "E_INPUT"
E_INTERNAL = "E_INTERNAL"

# 每个 E_* 配套的可操作恢复建议，让 AI Agent 能自决策下一步动作，
# 而不是盲目重试或放弃。在 _emit_error 的 JSON 输出中作为 recovery_hint 字段。
RECOVERY_HINTS = {
    E_CONFIG: "Provide -i/--instance, set SEARXNG_INSTANCE env var, or create "
              "searxng.toml/instances.txt config file.",
    E_AUTH: "Verify --auth-bearer/--auth-basic credentials or "
            "SEARXNG_BEARER_TOKEN/SEARXNG_BASIC_AUTH env vars. Check token "
            "expiry and instance access permissions.",
    E_NETWORK: "Retry with backoff, or try a different SearXNG instance. "
               "Check network connectivity, proxy settings, and instance uptime.",
    E_RATE_LIMIT: "Wait before retrying (exponential backoff). Reduce query "
                  "frequency, narrow --time-range, or distribute load across "
                  "multiple instances.",
    E_INPUT: "Check query syntax, --categories values, --time-range format, "
             "and flag combinations. Use --help for valid options.",
    E_PARSE: "The instance returned malformed data. Try a different instance, "
             "switch --method, or check if the instance version is compatible.",
    E_EMPTY: "Refine the query (more specific terms), broaden --time-range, "
             "add --categories, or increase --pageno to find more results.",
    E_INTERNAL: "This is likely a bug. Re-run with --verbose and report the "
                "full output for diagnosis.",
}


def classify_error(exc: BaseException) -> str:
    """将异常分类为结构化错误码，供 AI Agent 程序化处理。

    分类逻辑（按优先级）：
      1. 429 → E_RATE_LIMIT
      2. 401/403 → E_AUTH
      3. 4xx（非上述）→ E_INPUT（请求参数问题）
      4. 5xx / URLError / OSError / TimeoutError → E_NETWORK
      5. json.JSONDecodeError / ValueError → E_PARSE
      6. FileNotFoundError → E_INPUT
      7. RuntimeError → 尝试从消息中提取线索，否则 E_INTERNAL
      8. 其他 → E_INTERNAL
    """
    import json as _json

    # HTTP 错误（urllib 和 requests 都有 .code 或 .status_code）
    status = None
    if isinstance(exc, urllib.error.HTTPError):
        status = exc.code
    else:
        resp = getattr(exc, "response", None)
        status = getattr(resp, "status_code", None)

    if status is not None:
        if status == 429:
            return E_RATE_LIMIT
        if status in (401, 403):
            return E_AUTH
        if 400 <= status < 500:
            return E_INPUT
        if 500 <= status < 600:
            return E_NETWORK

    # 文件/输入错误（FileNotFoundError 是 OSError 子类，必须先于 OSError 检查）
    if isinstance(exc, FileNotFoundError):
        return E_INPUT

    # 连接级错误
    if isinstance(exc, (urllib.error.URLError, OSError, TimeoutError)):
        return E_NETWORK
    if isinstance(exc, ConnectionError):
        return E_NETWORK

    # 解析错误
    if isinstance(exc, (ValueError, _json.JSONDecodeError)):
        return E_PARSE

    # RuntimeError：从消息中推断（search_multi 的 "All N instances failed.
    # Last error: HTTP Error 403: Forbidden" 等）。并行路径现在会把 last_error
    # 拼进消息，让这里能提取真实错误类型，而不是一律误判 E_NETWORK。
    msg = str(exc).lower()
    if isinstance(exc, RuntimeError):
        # 先检查 auth/rate-limit 关键字（最常见，来自 last_error 详情）
        if "auth" in msg or "403" in msg or "401" in msg:
            return E_AUTH
        if "rate" in msg or "429" in msg:
            return E_RATE_LIMIT
        # 从 "http error NNN" / "http NNN" 模式中提取状态码，
        # 正确分类 "All instances failed. Last error: HTTP Error 403" 等
        import re as _re
        status_match = _re.search(r'http(?: error)? (\d{3})', msg)
        if status_match:
            status = int(status_match.group(1))
            if status == 429:
                return E_RATE_LIMIT
            if status in (401, 403):
                return E_AUTH
            if 400 <= status < 500:
                return E_INPUT
            if 500 <= status < 600:
                return E_NETWORK
        # 所有实例失败的通用模式（无更具体的 HTTP 状态码时才判为网络错误）
        if "all" in msg and "instance" in msg and "fail" in msg:
            return E_NETWORK
        if "parse" in msg or "json" in msg or "html" in msg:
            return E_PARSE
        if "not found" in msg or ("no " in msg and "instance" in msg):
            return E_CONFIG
        return E_INTERNAL

    return E_INTERNAL


# ----- Progress event emitter (for --progress flag) -----
#
# 当 --progress 启用时，search.py 会调用 emit_progress() 发射结构化事件到
# stderr（JSON Lines 格式）。AI Agent 可解析这些事件来跟踪执行进度。
#
# 事件类型：
#   {"event": "start", "query": "...", "instances": N}
#   {"event": "instance_try", "url": "...", "attempt": 1}
#   {"event": "instance_ok", "url": "...", "latency": 0.5, "results": 10}
#   {"event": "instance_fail", "url": "...", "error": "...", "error_code": "E_*"}
#   {"event": "cache_hit", "query": "...", "ttl": 30}
#   {"event": "cache_store", "query": "...", "ttl": 30}
#   {"event": "fetch_start", "count": 3}
#   {"event": "fetch_ok", "url": "...", "chars": 1234}
#   {"event": "fetch_fail", "url": "...", "error": "..."}
#   {"event": "done", "results": N, "query": "..."}
#   {"event": "error", "error": "...", "error_code": "E_*", "query": "..."}

_progress_enabled = False


def set_progress_enabled(enabled: bool) -> None:
    """全局开关：是否向 stderr 输出 JSON Lines 格式的进度事件。"""
    global _progress_enabled
    _progress_enabled = enabled


def emit_progress(event: str, **kwargs) -> None:
    """向 stderr 输出一行 JSON 格式的进度事件。

    仅当 --progress 启用时才输出。``event`` 是事件类型字符串，
    ``kwargs`` 是事件的附加字段。输出格式为 JSON Lines（每行一个 JSON 对象）。
    """
    if not _progress_enabled:
        return
    import json as _json
    payload = {"event": event}
    payload.update(kwargs)
    print(_json.dumps(payload, ensure_ascii=False), file=sys.stderr, flush=True)


# ----- Wayback Machine 兜底（v2.1.0 共享逻辑）-----
# 被 search.py 和 fetch.py 共用，避免逻辑漂移。

def should_try_wayback(error_msg: str) -> bool:
    """判断是否应触发 Wayback Machine 兜底。

    触发条件：错误信息暗示 404/403/超时/连接重置等可恢复失败。
    不触发：DNS 失败（Wayback 也访问不到）、空错误。

    纯字符串判断，无副作用，可安全用于 fetch.py 和 search.py。
    """
    if not error_msg:
        return False
    msg = error_msg.lower()
    triggers = ["404", "403", "timeout", "timed out", "connection reset",
                "connection refused", "max retries exceeded",
                "connectionreset", "connectionaborted"]
    return any(t in msg for t in triggers)


def build_wayback_url(url: str) -> str:
    """构造 Wayback Machine 最新快照 URL。

    使用 ``https://web.archive.org/web/2/<url>`` 端点，``2`` 表示
    "最新可用快照"。Wayback 会 302 重定向到具体时间戳快照。
    """
    return f"https://web.archive.org/web/2/{url}"


# ----- 被墙/强反爬站点智能回退（v2.1.0）-----
# 这些站点在中国大陆环境下常见 403/ConnectionReset，且对 UA 轮换不敏感
# （有更深层的反爬：Cookie/JS 指纹/登录墙）。命中时自动优先 Wayback 兜底。
#
# 维护原则：
#   1. 只收录"几乎必 403"的站点，避免误伤可正常抓取的站点
#   2. 每个站点都经过真实环境验证
#   3. 列表按域名匹配（子域名也算命中）

HARD_BLOCKED_DOMAINS = frozenset([
    "baike.baidu.com",      # 百度百科：强反爬 + Cookie 检测
    "zhidao.baidu.com",     # 百度知道：同上
    "tieba.baidu.com",      # 百度贴吧：同上
    "wenku.baidu.com",      # 百度文库：同上
    "zhihu.com",            # 知乎：登录墙 + 反爬
    "zhuanlan.zhihu.com",   # 知乎专栏：同上
    "mp.weixin.qq.com",     # 微信公众号：强反爬 + 登录墙
    "weibo.com",            # 微博：登录墙 + 反爬
    "m.weibo.cn",           # 微博移动版：同上
    "douban.com",           # 豆瓣：反爬 + 频率限制
    "www.douban.com",       # 豆瓣主站
    "book.douban.com",      # 豆瓣读书
    "movie.douban.com",     # 豆瓣电影
    "tieba.baidu.com",      # 百度贴吧（重复，确保子域匹配）
])

# 部分域名需要子域匹配（如 *.zhihu.com, *.weibo.com, *.douban.com）
_SUBDOMAIN_BLOCKED = frozenset([
    "zhihu.com",
    "weibo.com",
    "douban.com",
    "baidu.com",
])


def is_hard_blocked_domain(url: str) -> bool:
    """判断 URL 是否属于已知的强反爬/被墙站点。

    匹配逻辑：
      1. 精确匹配 HARD_BLOCKED_DOMAINS（如 baike.baidu.com）
      2. 子域匹配 _SUBDOMAIN_BLOCKED（如 *.zhihu.com）

    命中时调用方应：
      * 主抓取失败后立即尝试 Wayback（不等 should_try_wayback 判断）
      * 或直接跳过主抓取，优先 Wayback
    """
    if not url:
        return False
    # 提取域名
    try:
        from urllib.parse import urlparse
        host = urlparse(url).hostname or ""
    except Exception:
        host = ""
    if not host:
        return False
    host = host.lower().lstrip(".")
    # 精确匹配
    if host in HARD_BLOCKED_DOMAINS:
        return True
    # 子域匹配：xxx.zhihu.com → 匹配 zhihu.com
    parts = host.split(".")
    if len(parts) >= 2:
        # 检查最后两段（如 zhihu.com）或最后三段（如 baike.baidu.com）
        for i in range(len(parts) - 1):
            suffix = ".".join(parts[i:])
            if suffix in _SUBDOMAIN_BLOCKED or suffix in HARD_BLOCKED_DOMAINS:
                return True
    return False