Initial commit: SearXNG CLI Toolkit v1.6.0

Multi-instance failover, exponential-backoff retry, SQLite cache, batch mode, domain filter, cross-engine dedup, result sorting, CSV export, structured logging, enhanced Markdown conversion, 155 pytest tests, Gitea Actions CI

scripts/_config.py

@@ -0,0 +1,11 @@
+"""Package-level constants for searxng-cli scripts.
+
+Import in sibling scripts with:
+    from _config import VERSION, USER_AGENT
+
+Retry settings and shared HTTP utilities now live in ``common.py`` so that
+both ``search.py`` and ``fetch.py`` share one consistent implementation.
+"""
+
+VERSION = "1.6.0"
+USER_AGENT = f"searxng-cli/{VERSION}"

scripts/cache.py

@@ -0,0 +1,162 @@
+"""SQLite-backed result cache for searxng-cli.
+
+Avoids re-hitting the SearXNG instance for identical queries within a
+configurable TTL. Cache key is a SHA-256 of the normalized search params
+(query + engines + categories + language + time_range + safesearch +
+pageno + method), so different parameter combinations get separate entries.
+
+Storage location (in priority order):
+  1. ``$SEARXNG_CACHE_DIR`` env var (directory; ``cache.db`` is created inside)
+  2. ``~/.cache/searxng-cli/cache.db`` (XDG-style; on Windows this resolves
+     to ``C:\\Users\\<user>\\.cache\\searxng-cli\\cache.db``)
+
+Uses WAL journal mode for better read concurrency. Entries expire lazily
+on read; :func:`clear` removes all rows. Schema is created on first use.
+
+Design notes:
+  * Only the search result dict is cached — fetched page content is NOT,
+    because it is large and changes independently of the search result set.
+  * The cache key excludes auth headers and timeouts (transient concerns)
+    so two callers with the same query + params share an entry.
+  * All operations swallow ``sqlite3.Error`` and degrade gracefully — a
+    cache failure must never break a search.
+"""
+
+import hashlib
+import json
+import os
+import sqlite3
+import time
+from pathlib import Path
+
+DEFAULT_CACHE_DIR = Path.home() / ".cache" / "searxng-cli"
+
+
+def _cache_path() -> Path:
+    """Resolve the cache database path from env var or default location."""
+    env = os.environ.get("SEARXNG_CACHE_DIR")
+    if env:
+        return Path(env) / "cache.db"
+    return DEFAULT_CACHE_DIR / "cache.db"
+
+
+def _connect(path: Path) -> sqlite3.Connection:
+    """Open a connection with WAL mode and ensure the schema exists."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    conn = sqlite3.connect(str(path), timeout=10)
+    # WAL allows concurrent readers alongside a single writer, which matters
+    # when --fetch spawns parallel page fetches that might also touch the cache.
+    conn.execute("PRAGMA journal_mode=WAL")
+    conn.execute("PRAGMA synchronous=NORMAL")
+    conn.execute(
+        """
+        CREATE TABLE IF NOT EXISTS search_cache (
+            key TEXT PRIMARY KEY,
+            created_at REAL NOT NULL,
+            ttl_seconds INTEGER NOT NULL,
+            payload TEXT NOT NULL
+        )
+        """
+    )
+    conn.commit()
+    return conn
+
+
+def _make_key(params: dict) -> str:
+    """Build a stable cache key from search params.
+
+    Only params that affect the result set are included; transient fields
+    (auth, timeout, format) are excluded so the same logical query hits the
+    same cache entry regardless of output formatting.
+    """
+    # Whitelist the params that actually change what SearXNG returns.
+    # fmt: off
+    relevant = (
+        "q", "categories", "language", "pageno",
+        "time_range", "safesearch", "engines", "method",
+    )
+    # fmt: on
+    normalized = {k: params[k] for k in relevant if params.get(k)}
+    raw = json.dumps(normalized, sort_keys=True, ensure_ascii=False)
+    return hashlib.sha256(raw.encode("utf-8")).hexdigest()
+
+
+def get(params: dict, ttl_seconds: int):
+    """Return cached result if within TTL, else None.
+
+    ``ttl_seconds`` is the caller's current TTL setting. If the stored
+    entry was written with a longer TTL, the caller's shorter TTL wins
+    (so reducing --cache-ttl takes effect immediately without a clear).
+    """
+    if ttl_seconds <= 0:
+        return None
+    key = _make_key(params)
+    try:
+        with _connect(_cache_path()) as conn:
+            row = conn.execute(
+                "SELECT payload, created_at, ttl_seconds FROM search_cache "
+                "WHERE key = ?",
+                (key,),
+            ).fetchone()
+        if row is None:
+            return None
+        payload, created_at, stored_ttl = row
+        effective_ttl = min(ttl_seconds, stored_ttl)
+        if time.time() - created_at > effective_ttl:
+            return None
+        return json.loads(payload)
+    except sqlite3.Error:
+        return None
+    except (ValueError, json.JSONDecodeError):
+        # Corrupt payload — treat as miss
+        return None
+
+
+def put(params: dict, result: dict, ttl_seconds: int) -> None:
+    """Store a result with the given TTL. Silently no-ops on TTL<=0 or error."""
+    if ttl_seconds <= 0:
+        return
+    key = _make_key(params)
+    payload = json.dumps(result, ensure_ascii=False)
+    try:
+        with _connect(_cache_path()) as conn:
+            conn.execute(
+                "INSERT OR REPLACE INTO search_cache "
+                "(key, created_at, ttl_seconds, payload) VALUES (?, ?, ?, ?)",
+                (key, time.time(), ttl_seconds, payload),
+            )
+            conn.commit()
+    except sqlite3.Error:
+        pass
+
+
+def clear() -> int:
+    """Remove all cache entries. Returns count deleted, or 0 on error."""
+    try:
+        with _connect(_cache_path()) as conn:
+            cur = conn.execute("DELETE FROM search_cache")
+            conn.commit()
+            return cur.rowcount
+    except sqlite3.Error:
+        return 0
+
+
+def stats() -> dict:
+    """Return cache statistics (entry count, age range, path)."""
+    path = _cache_path()
+    try:
+        with _connect(path) as conn:
+            row = conn.execute(
+                "SELECT COUNT(*), MIN(created_at), MAX(created_at) "
+                "FROM search_cache"
+            ).fetchone()
+        count, oldest, newest = row
+        return {
+            "entries": count or 0,
+            "oldest_created_at": oldest,
+            "newest_created_at": newest,
+            "path": str(path),
+            "size_bytes": path.stat().st_size if path.exists() else 0,
+        }
+    except sqlite3.Error as e:
+        return {"entries": 0, "error": str(e), "path": str(path), "size_bytes": 0}

scripts/common.py

@@ -0,0 +1,261 @@
+"""Shared utilities for searxng-cli scripts.
+
+This module centralizes code that was previously duplicated across
+``search.py`` and ``fetch.py``:
+
+* ``build_auth_headers``  — construct an Authorization header from CLI flags
+* ``resolve_auth_basic``  — resolve basic-auth credentials from file/env/CLI
+                            (avoids leaving passwords in shell history)
+* ``detect_charset``      — guess a response's text encoding
+* ``is_retryable_error``  — unified transient-error policy (urllib + requests)
+* ``FALLBACK_UAS``        — browser-like User-Agents used when blocked
+* retry constants         — ``RETRYABLE_STATUS``, ``RETRY_BACKOFF_BASE``, etc.
+* ``setup_logging``       — shared logging configuration (--verbose/--quiet)
+
+Centralizing the retry policy guarantees that both scripts treat 429/5xx
+as retryable and connection errors as transient, eliminating the previous
+inconsistency where ``search.py`` ignored 5xx.
+"""
+
+import logging
+import sys
+import urllib.error
+
+# Root logger for the searxng-cli package. All modules create child loggers
+# via ``logging.getLogger("searxng.<module>")`` so a single setup_logging()
+# call controls them all.
+_LOG = logging.getLogger("searxng")
+
+
+def setup_logging(verbose: bool = False, quiet: bool = False) -> None:
+    """Configure the ``searxng`` logger hierarchy.
+
+    * Default (no flags): ``INFO`` — progress messages, warnings, retry notices.
+      Matches the previous ``print(..., file=sys.stderr)`` behavior so existing
+      scripts and agents see no change.
+    * ``--verbose`` / ``-v``: ``DEBUG`` — also shows HTTP request URLs, response
+      status codes, cache keys, and other diagnostic detail.
+    * ``--quiet`` / ``-q``: ``WARNING`` — suppresses progress and retry noise;
+      only warnings and errors reach stderr.
+
+    All log output goes to stderr; stdout is reserved for data (JSON/CSV/etc.).
+    """
+    if verbose:
+        level = logging.DEBUG
+    elif quiet:
+        level = logging.WARNING
+    else:
+        level = logging.INFO
+
+    _LOG.setLevel(level)
+    # Avoid duplicate handlers if setup_logging() is called twice (e.g. tests).
+    if not _LOG.handlers:
+        handler = logging.StreamHandler(sys.stderr)
+        handler.setFormatter(logging.Formatter("%(message)s"))
+        _LOG.addHandler(handler)
+    # Don't let root logger add its own handler — we own the searxng namespace.
+    _LOG.propagate = False
+
+# Retry settings (shared by both scripts)
+MAX_RETRIES = 3
+RETRY_BACKOFF_BASE = 1.5  # seconds; exponential backoff + jitter
+
+# HTTP status codes that are worth retrying (rate limit + gateway errors)
+RETRYABLE_STATUS = frozenset({429, 502, 503, 504})
+
+# Browser-like UA strings for fallback when the searxng-cli UA is blocked
+FALLBACK_UAS = [
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+]
+
+
+def build_auth_headers(bearer_token: str = None, basic_auth: str = None) -> dict:
+    """Build an Authorization header dict from CLI auth flags.
+
+    ``bearer_token``: raw Bearer token string.
+    ``basic_auth``:   ``"username:password"`` string (base64-encoded).
+
+    If both are provided, Bearer takes precedence (more common for APIs).
+    Returns a dict to merge into request headers, or an empty dict.
+    """
+    import base64
+
+    headers = {}
+    if bearer_token:
+        headers["Authorization"] = f"Bearer {bearer_token}"
+    elif basic_auth:
+        encoded = base64.b64encode(basic_auth.encode("utf-8")).decode("ascii")
+        headers["Authorization"] = f"Basic {encoded}"
+    return headers
+
+
+def _warn_file_perms(path: str) -> None:
+    """Warn if a credentials file is readable by group/other (POSIX only).
+
+    On Windows the Unix permission bits in ``st_mode`` do not reflect the
+    actual ACL, so the check is skipped to avoid false alarms.
+    """
+    import os
+    if os.name != "posix":
+        return
+    log = logging.getLogger("searxng.common")
+    try:
+        mode = os.stat(path).st_mode & 0o777
+        if mode & 0o077:
+            log.warning(
+                f"Warning: credentials file '{path}' has permissions {oct(mode)} "
+                f"(accessible by group/other); recommend 'chmod 600' for security."
+            )
+    except OSError:
+        pass
+
+
+def resolve_auth_basic(cli_value: str = None, file_path: str = None,
+                       env_var: str = "SEARXNG_BASIC_AUTH") -> str:
+    """Resolve basic-auth credentials without leaking them via shell history.
+
+    Priority (highest wins):
+      1. ``cli_value``  — explicit ``--auth-basic "user:pass"`` (convenient
+                          but leaks into shell history; discouraged)
+      2. ``file_path``  — ``--auth-basic-file FILE``; first non-empty line
+                          is read as ``user:pass``. Recommended for shells.
+      3. ``env_var``    — ``SEARXNG_BASIC_AUTH`` environment variable.
+
+    Returns ``"user:pass"`` or ``None`` if no source provides credentials.
+    Raises ``RuntimeError`` if a file is specified but cannot be read.
+    """
+    if cli_value:
+        return cli_value
+
+    if file_path:
+        try:
+            from pathlib import Path
+            text = Path(file_path).read_text(encoding="utf-8")
+            _warn_file_perms(file_path)
+            for line in text.splitlines():
+                line = line.strip()
+                if line and not line.startswith("#"):
+                    return line
+            raise RuntimeError(f"auth file '{file_path}' contains no credentials")
+        except OSError as e:
+            raise RuntimeError(f"cannot read auth file '{file_path}': {e}") from e
+
+    import os
+    return os.environ.get(env_var)
+
+
+def resolve_auth_bearer(cli_value: str = None, file_path: str = None,
+                        env_var: str = "SEARXNG_BEARER_TOKEN") -> str:
+    """Resolve a Bearer token from CLI flag, file, or environment variable.
+
+    Mirrors :func:`resolve_auth_basic` for token-style auth. Useful for
+    long-lived API tokens that should not appear in shell history.
+    """
+    if cli_value:
+        return cli_value
+
+    if file_path:
+        try:
+            from pathlib import Path
+            text = Path(file_path).read_text(encoding="utf-8")
+            _warn_file_perms(file_path)
+            for line in text.splitlines():
+                line = line.strip()
+                if line and not line.startswith("#"):
+                    return line
+            raise RuntimeError(f"token file '{file_path}' contains no token")
+        except OSError as e:
+            raise RuntimeError(f"cannot read token file '{file_path}': {e}") from e
+
+    import os
+    return os.environ.get(env_var)
+
+
+def apply_proxy(proxy_url: str) -> None:
+    """Configure proxy via environment variables.
+
+    Sets ``HTTP_PROXY`` and ``HTTPS_PROXY`` so both urllib (which reads
+    them via :func:`urllib.request.getproxies`) and ``requests`` (which
+    honors them when ``trust_env=True``, the default) pick up the proxy
+    without any changes to call sites.
+
+    ``NO_PROXY`` is set to ``localhost,127.0.0.1,::1`` (if not already set)
+    so local traffic stays direct — matters for self-hosted SearXNG on
+    localhost behind a corporate proxy.
+
+    Pass an empty string to clear the proxy env vars (rarely needed; the
+    default unset state already means "no proxy").
+    """
+    import os
+    if not proxy_url:
+        return
+    os.environ["HTTP_PROXY"] = proxy_url
+    os.environ["HTTPS_PROXY"] = proxy_url
+    # Keep local traffic direct unless the user has explicitly set NO_PROXY
+    os.environ.setdefault("NO_PROXY", "localhost,127.0.0.1,::1")
+
+
+def detect_charset(raw: bytes, content_type: str) -> str:
+    """Detect charset from the Content-Type header, then an HTML meta tag.
+
+    Falls back to UTF-8 (with replacement) if nothing reliable is found.
+    """
+    import re
+
+    # 1. HTTP header
+    if "charset=" in content_type:
+        charset = content_type.split("charset=")[-1].split(";")[0].strip()
+        try:
+            raw.decode(charset)
+            return charset
+        except (UnicodeDecodeError, LookupError):
+            pass
+
+    # 2. HTML <meta charset> or <meta http-equiv>
+    try:
+        head = raw[:4096].decode("ascii", errors="replace")
+        m = re.search(r'<meta[^>]+charset=["\']?([a-zA-Z0-9_-]+)', head, re.IGNORECASE)
+        if m:
+            charset = m.group(1).strip()
+            try:
+                raw.decode(charset)
+                return charset
+            except (UnicodeDecodeError, LookupError):
+                pass
+    except Exception:
+        pass
+
+    # 3. Fallback: UTF-8 with replacement
+    return "utf-8"
+
+
+def is_retryable_error(exc: BaseException) -> bool:
+    """Return True if ``exc`` is a transient error worth retrying.
+
+    Handles both the stdlib ``urllib`` errors and ``requests`` errors via
+    duck-typing (so this module does not need to import ``requests``):
+
+    * ``urllib.error.HTTPError`` → retry iff status in ``RETRYABLE_STATUS``
+    * ``urllib.error.URLError`` / ``OSError`` / ``TimeoutError`` → retry
+      (connection refused, DNS failure, timeout — all transient)
+    * ``requests.exceptions.HTTPError`` → retry iff ``response.status_code``
+      is in ``RETRYABLE_STATUS``
+    * ``requests`` connection/timeout errors (no ``.response``) → retry
+    """
+    if isinstance(exc, urllib.error.HTTPError):
+        return exc.code in RETRYABLE_STATUS
+    if isinstance(exc, (urllib.error.URLError, OSError, TimeoutError)):
+        return not isinstance(exc, urllib.error.HTTPError)
+
+    # requests.exceptions.HTTPError / RequestException (duck-typed)
+    resp = getattr(exc, "response", None)
+    status = getattr(resp, "status_code", None)
+    if status is not None:
+        return status in RETRYABLE_STATUS
+    if resp is None:
+        # requests connection/timeout error without a response -> transient
+        return True
+    return False

scripts/fetch.py

@@ -0,0 +1,738 @@
+#!/usr/bin/env python3
+"""Fetch a web page and extract readable content.
+
+Downloads page content via HTTP GET and extracts clean, readable text.
+Strips navigation, ads, scripts, and other boilerplate using heuristic rules.
+
+Dependencies: Python 3.8+ stdlib. Install `requests` and `beautifulsoup4`
+for improved extraction quality (optional, falls back to stdlib).
+"""
+
+import argparse
+import logging
+import random
+import re
+import sys
+import time
+import urllib.error
+import urllib.request
+from collections import namedtuple
+from html.parser import HTMLParser
+from pathlib import Path
+
+# Allow running standalone from any working directory
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from _config import USER_AGENT, VERSION
+from common import (
+    FALLBACK_UAS,
+    RETRYABLE_STATUS,
+    RETRY_BACKOFF_BASE,
+    apply_proxy,
+    build_auth_headers,
+    detect_charset,
+    is_retryable_error,
+    resolve_auth_basic,
+    resolve_auth_bearer,
+    setup_logging,
+)
+
+logger = logging.getLogger("searxng.fetch")
+
+
+# ----- Auth helpers -----
+# build_auth_headers is imported from common.py
+
+# ----- stdlib HTML-to-text extractor -----
+
+class TextExtractor(HTMLParser):
+    """Extract visible text from HTML, skipping non-content elements."""
+
+    SKIP_TAGS = {"script", "style", "nav", "footer", "header",
+                 "noscript", "iframe", "svg", "canvas", "template"}
+    BLOCK_TAGS = {"p", "div", "article", "section", "li", "h1", "h2", "h3",
+                  "h4", "h5", "h6", "blockquote", "pre", "table", "tr",
+                  "br", "hr", "main", "aside", "form", "fieldset"}
+
+    def __init__(self):
+        super().__init__()
+        self._skip_depth = 0
+        self._lines = []
+        self._current_line = []
+        self._block_pending = False
+
+    def handle_starttag(self, tag, attrs):
+        tag_lower = tag.lower()
+        if tag_lower in self.SKIP_TAGS:
+            self._skip_depth += 1
+        elif tag_lower in self.BLOCK_TAGS:
+            self._flush_line()
+            self._block_pending = True
+
+    def handle_endtag(self, tag):
+        tag_lower = tag.lower()
+        if tag_lower in self.SKIP_TAGS and self._skip_depth > 0:
+            self._skip_depth -= 1
+        elif tag_lower in self.BLOCK_TAGS:
+            self._flush_line()
+
+    def handle_data(self, data):
+        if self._skip_depth > 0:
+            return
+        text = data.strip()
+        if text:
+            self._current_line.append(text)
+            self._block_pending = False
+
+    def _flush_line(self):
+        if self._current_line:
+            self._lines.append(" ".join(self._current_line))
+            self._current_line = []
+        if self._block_pending:
+            self._lines.append("")
+            self._block_pending = False
+
+    def get_text(self) -> str:
+        self._flush_line()
+        text = "\n".join(self._lines)
+        text = re.sub(r"\n{3,}", "\n\n", text)
+        return text.strip()
+
+
+# ----- Enhanced extraction with BeautifulSoup (optional) -----
+
+_HAS_BS4 = False
+_HAS_REQUESTS = False
+
+try:
+    import requests as _requests
+    _HAS_REQUESTS = True
+except ImportError:
+    pass
+
+try:
+    from bs4 import BeautifulSoup as _BeautifulSoup
+    _HAS_BS4 = True
+except ImportError:
+    pass
+
+
+def extract_with_stdlib(html_content: str) -> str:
+    """Extract text using stdlib HTMLParser."""
+    extractor = TextExtractor()
+    extractor.feed(html_content)
+    return extractor.get_text()
+
+
+def extract_with_bs4(html_content: str) -> str:
+    """Extract text using BeautifulSoup for better quality."""
+    soup = _BeautifulSoup(html_content, "html.parser")
+
+    for tag in soup(["script", "style", "nav", "footer", "header",
+                     "noscript", "iframe", "svg", "canvas"]):
+        tag.decompose()
+
+    main = (soup.find("article") or
+            soup.find("main") or
+            soup.find(role="main") or
+            soup.find("div", class_=re.compile(r"content|article|post|entry")) or
+            soup.body)
+
+    if main is None:
+        main = soup
+
+    text = main.get_text(separator="\n", strip=True)
+    lines = [line.strip() for line in text.split("\n") if line.strip()]
+    text = "\n".join(lines)
+    text = re.sub(r"\n{3,}", "\n\n", text)
+    return text
+
+
+def extract_text(html_content: str) -> str:
+    """Extract readable text from HTML, preferring bs4 if available."""
+    if _HAS_BS4:
+        try:
+            return extract_with_bs4(html_content)
+        except Exception:
+            pass
+    return extract_with_stdlib(html_content)
+
+
+# ----- Tree-based HTML-to-Markdown converter (robust) -----
+
+class MarkdownConverter(HTMLParser):
+    """Convert HTML to Markdown using a tag-stack approach.
+
+    Handles nested tags in <a> elements correctly unlike regex. Supports:
+      * GFM tables (``<table>`` → ``| a | b |`` with separator row)
+      * fenced code blocks (``<pre>`` → triple-backtick fences)
+      * inline code (``<code>`` → backticks)
+      * blockquotes (``<blockquote>`` → ``> `` prefix per line)
+      * headings, lists, images, emphasis, links
+    """
+
+    def __init__(self):
+        super().__init__()
+        self._output = []
+        self._skip_depth = 0
+        self._list_stack = []  # list of [type, counter] for ol numbering
+        self._pending_indent = ""  # preserved leading indent for li/dd
+        self._block_buffer = ""
+        self._link_href = None
+        self._link_text = []
+        self._in_link = False
+        self._in_pre = False
+        self._pre_content = []
+        self._heading_level = 0
+        self._block_empty = True
+        # Blockquote support
+        self._in_blockquote = False
+        # Table support (GFM)
+        self._in_table = False
+        self._table_rows = []  # list of (cells_list, is_header)
+        self._current_row = None
+        self._current_cell = None
+        self._in_cell = False
+
+    def _append_inline(self, text: str):
+        """Append inline markup, routing to cell/link/block buffer.
+
+        In cell context, fragments go to ``_current_cell`` so table cell text
+        accumulates correctly. In link context, they go to ``_link_text``
+        and are collapsed at ``</a>`` time. Otherwise they append to the
+        block buffer; whitespace is normalized at ``_flush_block`` time.
+        """
+        if self._in_cell:
+            self._current_cell.append(text)
+        elif self._in_link:
+            self._link_text.append(text)
+        else:
+            self._block_buffer += text
+            self._block_empty = False
+
+    def _flush_block(self):
+        t = self._block_buffer
+        # Normalize whitespace: collapse runs of spaces/tabs/newlines
+        t = re.sub(r'\s+', ' ', t).strip()
+        if t:
+            if self._in_blockquote:
+                # Prefix each line with "> " for markdown blockquote syntax
+                t = "\n".join(
+                    ("> " + line) if line.strip() else ">"
+                    for line in t.split("\n")
+                )
+            if self._pending_indent:
+                self._output.append(self._pending_indent + t)
+            else:
+                self._output.append(t)
+        self._block_buffer = ""
+        self._block_empty = True
+        self._pending_indent = ""
+
+    def _emit_table(self):
+        """Emit a GFM table from accumulated rows.
+
+        The first row becomes the header; a ``| --- | --- |`` separator row
+        follows; remaining rows become the body. Cells are padded to the
+        header width so the table renders correctly in strict GFM parsers.
+        """
+        if not self._table_rows:
+            return
+        header_row, _ = self._table_rows[0]
+        body_rows = self._table_rows[1:]
+        if not header_row:
+            return
+        ncols = len(header_row)
+        self._output.append("| " + " | ".join(header_row) + " |")
+        self._output.append("| " + " | ".join("---" for _ in range(ncols)) + " |")
+        for row, _ in body_rows:
+            # Pad short rows; truncate long rows to header width
+            while len(row) < ncols:
+                row.append("")
+            self._output.append("| " + " | ".join(row[:ncols]) + " |")
+        self._output.append("")
+
+    def handle_starttag(self, tag, attrs):
+        tag_lower = tag.lower()
+        attrs_dict = dict(attrs)
+
+        if tag_lower in ("script", "style", "nav", "footer", "header",
+                          "noscript", "iframe", "svg", "canvas", "template"):
+            self._skip_depth += 1
+            return
+
+        if self._skip_depth > 0:
+            self._skip_depth += 1
+            return
+
+        # Table structural elements are handled up-front so cell content
+        # routing (via _in_cell) takes effect before any other tag handler.
+        if tag_lower == "table":
+            self._flush_block()
+            self._in_table = True
+            self._table_rows = []
+            return
+        if self._in_table:
+            if tag_lower == "tr":
+                self._current_row = []
+                return
+            elif tag_lower in ("th", "td"):
+                self._in_cell = True
+                self._current_cell = []
+                return
+            elif tag_lower in ("thead", "tbody", "tfoot"):
+                return  # container only — rows/cells drive the output
+            # Other tags inside cells (a/strong/em/code/br) fall through
+            # to normal handling; _append_inline routes them to _current_cell.
+
+        if tag_lower in ("p", "div", "section"):
+            self._flush_block()
+        elif tag_lower == "blockquote":
+            self._flush_block()
+            self._in_blockquote = True
+        elif tag_lower == "br":
+            self._append_inline("\n")
+        elif tag_lower in ("h1", "h2", "h3", "h4", "h5", "h6"):
+            self._flush_block()
+            self._heading_level = int(tag_lower[1])
+        elif tag_lower == "pre":
+            self._flush_block()
+            self._in_pre = True
+            self._pre_content = []
+        elif tag_lower == "code":
+            # Inline code. <pre><code> is handled by the pre path, which
+            # captures raw text verbatim — so only emit backticks when
+            # we're NOT inside a pre block.
+            if not self._in_pre:
+                self._append_inline("`")
+        elif tag_lower in ("ul", "ol"):
+            self._list_stack.append([tag_lower, 0])
+        elif tag_lower == "li":
+            self._flush_block()
+            depth = max(0, len(self._list_stack) - 1)
+            self._pending_indent = "  " * depth
+            if self._list_stack and self._list_stack[-1][0] == "ol":
+                self._list_stack[-1][1] += 1
+                marker = f"{self._list_stack[-1][1]}. "
+            else:
+                marker = "- "
+            self._block_buffer = marker
+            self._block_empty = False
+        elif tag_lower == "dt":
+            self._flush_block()
+            self._block_buffer = "**"
+            self._block_empty = False
+        elif tag_lower == "dd":
+            self._flush_block()
+            self._pending_indent = "  "
+            self._block_buffer = ""
+            self._block_empty = True
+        elif tag_lower == "a":
+            self._in_link = True
+            self._link_href = attrs_dict.get("href", "")
+            self._link_text = []
+        elif tag_lower in ("strong", "b"):
+            self._append_inline("**")
+        elif tag_lower in ("em", "i"):
+            self._append_inline("*")
+        elif tag_lower == "img":
+            alt = attrs_dict.get("alt", "")
+            src = attrs_dict.get("src", "")
+            if alt or src:
+                self._append_inline(f"![{alt}]({src})")
+
+    def handle_endtag(self, tag):
+        tag_lower = tag.lower()
+        if self._skip_depth > 0:
+            self._skip_depth -= 1
+            return
+
+        if tag_lower == "pre":
+            self._in_pre = False
+            code = "\n".join(self._pre_content)
+            self._output.append(f"```\n{code}\n```")
+            self._pre_content = []
+            self._block_empty = True
+        elif tag_lower in ("h1", "h2", "h3", "h4", "h5", "h6"):
+            prefix = "#" * self._heading_level
+            self._output.append(f"\n{prefix} {self._block_buffer.strip()}\n")
+            self._block_buffer = ""
+            self._heading_level = 0
+            self._block_empty = True
+        elif tag_lower in ("th", "td"):
+            # Collapse cell content to a single line (newlines from <br>
+            # would break the GFM table row). Pipe chars are escaped to
+            # avoid prematurely terminating cells.
+            cell_text = " ".join(self._current_cell).strip()
+            cell_text = re.sub(r'\s+', ' ', cell_text)
+            cell_text = cell_text.replace("|", "\\|")
+            if self._current_row is not None:
+                self._current_row.append(cell_text)
+            self._in_cell = False
+            self._current_cell = None
+        elif tag_lower == "tr":
+            if self._current_row is not None:
+                is_header = False  # GFM doesn't distinguish; first row is header
+                self._table_rows.append((self._current_row, is_header))
+            self._current_row = None
+        elif tag_lower == "table":
+            self._emit_table()
+            self._in_table = False
+            self._table_rows = []
+            self._current_row = None
+            self._current_cell = None
+            self._in_cell = False
+        elif tag_lower == "blockquote":
+            self._flush_block()
+            self._in_blockquote = False
+            self._output.append("")
+        elif tag_lower in ("p", "div", "section"):
+            self._flush_block()
+            self._output.append("")
+        elif tag_lower in ("ul", "ol"):
+            if self._list_stack:
+                self._list_stack.pop()
+            self._output.append("")
+        elif tag_lower == "li":
+            self._flush_block()
+        elif tag_lower == "dt":
+            self._block_buffer = self._block_buffer.rstrip() + "**"
+            self._flush_block()
+        elif tag_lower == "dd":
+            self._flush_block()
+        elif tag_lower == "a":
+            if self._in_link:
+                link_text = " ".join("".join(self._link_text).split())
+                if link_text and self._link_href:
+                    rendered = f"[{link_text}]({self._link_href})"
+                elif self._link_href:
+                    rendered = f"<{self._link_href}>"
+                else:
+                    rendered = ""
+                if self._in_cell:
+                    self._current_cell.append(rendered)
+                else:
+                    self._block_buffer += rendered
+                    self._block_empty = False
+                self._in_link = False
+                self._link_href = None
+                self._link_text = []
+        elif tag_lower == "code":
+            if not self._in_pre:
+                self._append_inline("`")
+        elif tag_lower in ("strong", "b"):
+            self._append_inline("**")
+        elif tag_lower in ("em", "i"):
+            self._append_inline("*")
+
+    def handle_data(self, data):
+        if self._skip_depth > 0:
+            return
+        if self._in_pre:
+            self._pre_content.append(data)
+        elif self._in_cell:
+            self._current_cell.append(data)
+        elif self._in_link:
+            self._link_text.append(data)
+        else:
+            # Don't strip — let _flush_block normalize whitespace
+            if data.strip():  # only skip purely whitespace nodes
+                self._block_buffer += data
+                self._block_empty = False
+
+    def get_markdown(self) -> str:
+        self._flush_block()
+        # Emit a table if one was left open (malformed HTML)
+        if self._in_table and self._table_rows:
+            self._emit_table()
+        text = "\n".join(self._output)
+        text = re.sub(r"\n{3,}", "\n\n", text)
+        return text.strip()
+
+
+def html_to_markdown(html_content: str) -> str:
+    """Convert HTML to Markdown using tree-based parser."""
+    # Quick strip of scripts/styles first
+    html_content = re.sub(r'<script[^>]*>.*?</script>', '', html_content,
+                          flags=re.DOTALL | re.IGNORECASE)
+    html_content = re.sub(r'<style[^>]*>.*?</style>', '', html_content,
+                          flags=re.DOTALL | re.IGNORECASE)
+
+    converter = MarkdownConverter()
+    converter.feed(html_content)
+    return converter.get_markdown()
+
+
+# ----- HTTP Fetch -----
+# RETRY_BACKOFF_BASE, FALLBACK_UAS, detect_charset and is_retryable_error are
+# imported from common.py (shared with search.py for a consistent retry policy).
+
+
+class _NoRedirectHandler(urllib.request.HTTPRedirectHandler):
+    """urllib handler that returns the 3xx response instead of following it."""
+
+    def http_error_302(self, req, fp, code, msg, headers):
+        return fp
+
+    http_error_301 = http_error_303 = http_error_307 = http_error_308 = http_error_302
+
+
+FetchResult = namedtuple(
+    "FetchResult",
+    ["content", "content_type", "final_url", "truncated", "user_agent"],
+)
+
+
+def fetch_url(url: str, timeout: int = 15, user_agent: str = None,
+              encoding: str = None, auth_headers: dict = None,
+              max_retries: int = 3, max_size: int = None,
+              allow_redirects: bool = True) -> "FetchResult":
+    """Fetch a URL with retry, encoding detection, UA fallback, and optional size limit.
+
+    Returns a :class:`FetchResult` namedtuple with fields:
+      ``content`` (str), ``content_type`` (str), ``final_url`` (str),
+      ``truncated`` (bool — True if ``max_size`` cut the response short),
+      ``user_agent`` (str — the UA string that succeeded; useful for logging
+      whether a fallback UA was needed).
+
+    max_size=None means unlimited (full page). Set to e.g. 5242880 for a 5MB cap.
+    allow_redirects=False stops the client from following HTTP 3xx redirects.
+    """
+    if user_agent is None:
+        user_agent = USER_AGENT
+
+    last_error = None
+    user_agents = [user_agent] + FALLBACK_UAS
+
+    for attempt in range(max_retries + 1):
+        ua = user_agents[min(attempt, len(user_agents) - 1)]
+        headers = {"User-Agent": ua}
+        if auth_headers:
+            headers.update(auth_headers)
+
+        try:
+            if _HAS_REQUESTS:
+                resp = _requests.get(url, timeout=timeout, headers=headers,
+                                     allow_redirects=allow_redirects, stream=True)
+                resp.raise_for_status()
+
+                # Read: unlimited if max_size is None, chunked with limit otherwise
+                if max_size is None:
+                    raw = resp.content
+                    truncated = False
+                else:
+                    chunks = []
+                    total = 0
+                    for chunk in resp.iter_content(chunk_size=65536, decode_unicode=False):
+                        if chunk:
+                            chunks.append(chunk)
+                            total += len(chunk)
+                            if total > max_size:
+                                break
+                    raw = b"".join(chunks)
+                    truncated = total > max_size
+
+                if encoding:
+                    content = raw.decode(encoding)
+                else:
+                    charset = detect_charset(raw, resp.headers.get("Content-Type", ""))
+                    try:
+                        content = raw.decode(charset)
+                    except (UnicodeDecodeError, LookupError):
+                        content = raw.decode("utf-8", errors="replace")
+
+                return FetchResult(content, resp.headers.get("Content-Type", ""),
+                                   resp.url, truncated, ua)
+
+            # stdlib fallback
+            req = urllib.request.Request(url, headers=headers)
+            if allow_redirects:
+                _opener = urllib.request.urlopen(req, timeout=timeout)
+            else:
+                _opener = urllib.request.build_opener(_NoRedirectHandler).open(
+                    req, timeout=timeout)
+            with _opener as resp:
+                if max_size is None:
+                    raw = resp.read()
+                    truncated = False
+                else:
+                    chunks = []
+                    total = 0
+                    while True:
+                        chunk = resp.read(65536)
+                        if not chunk:
+                            break
+                        chunks.append(chunk)
+                        total += len(chunk)
+                        if total > max_size:
+                            break
+                    raw = b"".join(chunks)
+                    truncated = total > max_size
+                content_type = resp.headers.get("Content-Type", "")
+                final_url = resp.geturl()
+
+                if encoding:
+                    charset = encoding
+                else:
+                    charset = detect_charset(raw, content_type)
+                try:
+                    content = raw.decode(charset)
+                except (UnicodeDecodeError, LookupError):
+                    content = raw.decode("utf-8", errors="replace")
+
+                return FetchResult(content, content_type, final_url, truncated, ua)
+
+        except urllib.error.HTTPError as e:
+            last_error = e
+            if is_retryable_error(e) and attempt < max_retries:
+                delay = RETRY_BACKOFF_BASE * (2 ** attempt) + random.uniform(0, 1)
+                logger.info(f"  Fetch retry {attempt+1}/{max_retries} ({e}) in {delay:.1f}s")
+                time.sleep(delay)
+                continue
+            raise RuntimeError(f"HTTP {e.code} for {url}")
+        except (urllib.error.URLError, OSError, TimeoutError) as e:
+            last_error = e
+            if attempt < max_retries:
+                delay = RETRY_BACKOFF_BASE * (2 ** attempt) + random.uniform(0, 1)
+                logger.info(f"  Fetch retry {attempt+1}/{max_retries} ({e}) in {delay:.1f}s")
+                time.sleep(delay)
+                continue
+            raise RuntimeError(f"Request failed for {url}: {e}")
+        except Exception as e:
+            # requests backend: retry only on connection errors (no response)
+            # or transient 429/5xx; do NOT retry permanent errors like 404.
+            if _HAS_REQUESTS and isinstance(e, _requests.exceptions.RequestException):
+                last_error = e
+                status = getattr(getattr(e, "response", None), "status_code", None)
+                if (status is None or status in RETRYABLE_STATUS) and attempt < max_retries:
+                    delay = RETRY_BACKOFF_BASE * (2 ** attempt) + random.uniform(0, 1)
+                    logger.info(f"  Fetch retry {attempt+1}/{max_retries} ({e}) in {delay:.1f}s")
+                    time.sleep(delay)
+                    continue
+            raise RuntimeError(f"Request failed for {url}: {e}")
+
+    raise RuntimeError(f"All {max_retries+1} attempts failed for {url}: {last_error}")
+
+
+# ----- Main -----
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Fetch a web page and extract readable content",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog=f"""searxng-cli v{VERSION}
+
+Examples:
+  %(prog)s -u https://example.com                  extract clean text
+  %(prog)s -u https://example.com -e html          raw HTML
+  %(prog)s -u https://example.com -e markdown      markdown conversion
+  %(prog)s -u https://example.com -o page.txt      save to file
+  %(prog)s -u https://example.cn -e text --encoding gbk   force charset
+        """,
+    )
+    parser.add_argument("--url", "-u", required=True, help="URL to fetch")
+    parser.add_argument("--extract", "-e", choices=["text", "html", "markdown"],
+                        default="text", help="Extraction mode (default: text)")
+    parser.add_argument("--timeout", "-t", type=int, default=15,
+                        help="Request timeout in seconds (default: 15)")
+    parser.add_argument("--retries", type=int, default=3,
+                        help="Max retries on transient errors (default: 3)")
+    parser.add_argument("--max-size", type=int, default=None, metavar="BYTES",
+                        help="Max page size in bytes (default: unlimited). Set to 5242880 for 5MB cap.")
+    parser.add_argument("--user-agent", default=None,
+                        help="Custom User-Agent header")
+    parser.add_argument("--encoding", default=None,
+                        help="Force charset for decoding (e.g. gbk, shift_jis)")
+    parser.add_argument("--no-redirect", action="store_true",
+                        help="Do not follow HTTP redirects")
+    parser.add_argument("--proxy", default=None, metavar="URL",
+                        help="HTTP/HTTPS proxy URL (e.g. http://corp-proxy:8080). "
+                             "Respects existing HTTP_PROXY/HTTPS_PROXY env vars when omitted.")
+    parser.add_argument("--output", "-o", default=None,
+                        help="Save to file instead of stdout")
+    parser.add_argument("--auth-bearer", default=None, metavar="TOKEN",
+                        help="Authorization: Bearer <TOKEN> for authenticated endpoints")
+    parser.add_argument("--auth-bearer-file", default=None, metavar="FILE",
+                        help="Read Bearer token from a file (first non-empty, non-# line). "
+                             "Avoids leaving tokens in shell history.")
+    parser.add_argument("--auth-basic", default=None, metavar="USER:PASS",
+                        help="Authorization: Basic base64(user:pass) for authenticated endpoints")
+    parser.add_argument("--auth-basic-file", default=None, metavar="FILE",
+                        help="Read basic auth 'user:pass' from a file (first non-empty, non-# line). "
+                             "Avoids leaving passwords in shell history. "
+                             "Env var SEARXNG_BASIC_AUTH is also honored.")
+    parser.add_argument("--verbose", "-v", action="store_true", default=False,
+                        help="Verbose output: show debug-level diagnostics on stderr")
+    parser.add_argument("--quiet", action="store_true", default=False,
+                        help="Quiet output: suppress progress messages on stderr; "
+                             "only warnings and errors are shown")
+    parser.add_argument("--version", action="version", version=f"searxng-cli v{VERSION}")
+
+    args = parser.parse_args()
+    setup_logging(verbose=args.verbose, quiet=args.quiet)
+
+    if not args.url.startswith(("http://", "https://")):
+        logger.error("Error: URL must start with http:// or https://")
+        sys.exit(1)
+
+    # Apply proxy via env vars so both urllib and requests honor it.
+    if args.proxy:
+        apply_proxy(args.proxy)
+        logger.info(f"Proxy: {args.proxy}")
+
+    try:
+        bearer_token = resolve_auth_bearer(args.auth_bearer, args.auth_bearer_file)
+        basic_auth = resolve_auth_basic(args.auth_basic, args.auth_basic_file)
+    except RuntimeError as e:
+        logger.error(f"Error: {e}")
+        sys.exit(1)
+    auth_headers = build_auth_headers(
+        bearer_token=bearer_token,
+        basic_auth=basic_auth,
+    )
+    if auth_headers:
+        auth_type = "Bearer" if bearer_token else "Basic"
+        logger.info(f"Auth: {auth_type} ***")
+
+    try:
+        result = fetch_url(
+            args.url, timeout=args.timeout, user_agent=args.user_agent,
+            encoding=args.encoding, auth_headers=auth_headers,
+            max_retries=args.retries, max_size=args.max_size,
+            allow_redirects=not args.no_redirect,
+        )
+        content, content_type, final_url = (
+            result.content, result.content_type, result.final_url,
+        )
+    except Exception as e:
+        logger.error(f"Error: {e}")
+        sys.exit(1)
+
+    if final_url != args.url:
+        logger.info(f"Redirected to: {final_url}")
+
+    is_html = ("html" in content_type.lower() or
+               content.strip().startswith("<!") or
+               content.strip().startswith("<htm"))
+
+    if args.extract == "html":
+        output = content
+    elif args.extract == "markdown":
+        output = html_to_markdown(content) if is_html else content
+    else:  # text
+        output = extract_text(content) if is_html else content
+
+    # Quality check (threshold: 500 chars)
+    if args.extract == "text" and len(output.strip()) < 500 and is_html:
+        logger.warning(f"Warning: extracted text is very short ({len(output.strip())} chars). "
+                       "The page may be JS-heavy or use anti-bot protection.")
+
+    if args.output:
+        with open(args.output, "w", encoding="utf-8") as f:
+            f.write(output)
+        logger.info(f"Saved {len(output)} chars to {args.output}")
+    else:
+        print(output)
+
+
+if __name__ == "__main__":
+    main()