Initial commit: SearXNG CLI Toolkit v1.6.0

Multi-instance failover, exponential-backoff retry, SQLite cache, batch mode, domain filter, cross-engine dedup, result sorting, CSV export, structured logging, enhanced Markdown conversion, 155 pytest tests, Gitea Actions CI

scripts/cache.py

@@ -0,0 +1,162 @@
+"""SQLite-backed result cache for searxng-cli.
+
+Avoids re-hitting the SearXNG instance for identical queries within a
+configurable TTL. Cache key is a SHA-256 of the normalized search params
+(query + engines + categories + language + time_range + safesearch +
+pageno + method), so different parameter combinations get separate entries.
+
+Storage location (in priority order):
+  1. ``$SEARXNG_CACHE_DIR`` env var (directory; ``cache.db`` is created inside)
+  2. ``~/.cache/searxng-cli/cache.db`` (XDG-style; on Windows this resolves
+     to ``C:\\Users\\<user>\\.cache\\searxng-cli\\cache.db``)
+
+Uses WAL journal mode for better read concurrency. Entries expire lazily
+on read; :func:`clear` removes all rows. Schema is created on first use.
+
+Design notes:
+  * Only the search result dict is cached — fetched page content is NOT,
+    because it is large and changes independently of the search result set.
+  * The cache key excludes auth headers and timeouts (transient concerns)
+    so two callers with the same query + params share an entry.
+  * All operations swallow ``sqlite3.Error`` and degrade gracefully — a
+    cache failure must never break a search.
+"""
+
+import hashlib
+import json
+import os
+import sqlite3
+import time
+from pathlib import Path
+
+DEFAULT_CACHE_DIR = Path.home() / ".cache" / "searxng-cli"
+
+
+def _cache_path() -> Path:
+    """Resolve the cache database path from env var or default location."""
+    env = os.environ.get("SEARXNG_CACHE_DIR")
+    if env:
+        return Path(env) / "cache.db"
+    return DEFAULT_CACHE_DIR / "cache.db"
+
+
+def _connect(path: Path) -> sqlite3.Connection:
+    """Open a connection with WAL mode and ensure the schema exists."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    conn = sqlite3.connect(str(path), timeout=10)
+    # WAL allows concurrent readers alongside a single writer, which matters
+    # when --fetch spawns parallel page fetches that might also touch the cache.
+    conn.execute("PRAGMA journal_mode=WAL")
+    conn.execute("PRAGMA synchronous=NORMAL")
+    conn.execute(
+        """
+        CREATE TABLE IF NOT EXISTS search_cache (
+            key TEXT PRIMARY KEY,
+            created_at REAL NOT NULL,
+            ttl_seconds INTEGER NOT NULL,
+            payload TEXT NOT NULL
+        )
+        """
+    )
+    conn.commit()
+    return conn
+
+
+def _make_key(params: dict) -> str:
+    """Build a stable cache key from search params.
+
+    Only params that affect the result set are included; transient fields
+    (auth, timeout, format) are excluded so the same logical query hits the
+    same cache entry regardless of output formatting.
+    """
+    # Whitelist the params that actually change what SearXNG returns.
+    # fmt: off
+    relevant = (
+        "q", "categories", "language", "pageno",
+        "time_range", "safesearch", "engines", "method",
+    )
+    # fmt: on
+    normalized = {k: params[k] for k in relevant if params.get(k)}
+    raw = json.dumps(normalized, sort_keys=True, ensure_ascii=False)
+    return hashlib.sha256(raw.encode("utf-8")).hexdigest()
+
+
+def get(params: dict, ttl_seconds: int):
+    """Return cached result if within TTL, else None.
+
+    ``ttl_seconds`` is the caller's current TTL setting. If the stored
+    entry was written with a longer TTL, the caller's shorter TTL wins
+    (so reducing --cache-ttl takes effect immediately without a clear).
+    """
+    if ttl_seconds <= 0:
+        return None
+    key = _make_key(params)
+    try:
+        with _connect(_cache_path()) as conn:
+            row = conn.execute(
+                "SELECT payload, created_at, ttl_seconds FROM search_cache "
+                "WHERE key = ?",
+                (key,),
+            ).fetchone()
+        if row is None:
+            return None
+        payload, created_at, stored_ttl = row
+        effective_ttl = min(ttl_seconds, stored_ttl)
+        if time.time() - created_at > effective_ttl:
+            return None
+        return json.loads(payload)
+    except sqlite3.Error:
+        return None
+    except (ValueError, json.JSONDecodeError):
+        # Corrupt payload — treat as miss
+        return None
+
+
+def put(params: dict, result: dict, ttl_seconds: int) -> None:
+    """Store a result with the given TTL. Silently no-ops on TTL<=0 or error."""
+    if ttl_seconds <= 0:
+        return
+    key = _make_key(params)
+    payload = json.dumps(result, ensure_ascii=False)
+    try:
+        with _connect(_cache_path()) as conn:
+            conn.execute(
+                "INSERT OR REPLACE INTO search_cache "
+                "(key, created_at, ttl_seconds, payload) VALUES (?, ?, ?, ?)",
+                (key, time.time(), ttl_seconds, payload),
+            )
+            conn.commit()
+    except sqlite3.Error:
+        pass
+
+
+def clear() -> int:
+    """Remove all cache entries. Returns count deleted, or 0 on error."""
+    try:
+        with _connect(_cache_path()) as conn:
+            cur = conn.execute("DELETE FROM search_cache")
+            conn.commit()
+            return cur.rowcount
+    except sqlite3.Error:
+        return 0
+
+
+def stats() -> dict:
+    """Return cache statistics (entry count, age range, path)."""
+    path = _cache_path()
+    try:
+        with _connect(path) as conn:
+            row = conn.execute(
+                "SELECT COUNT(*), MIN(created_at), MAX(created_at) "
+                "FROM search_cache"
+            ).fetchone()
+        count, oldest, newest = row
+        return {
+            "entries": count or 0,
+            "oldest_created_at": oldest,
+            "newest_created_at": newest,
+            "path": str(path),
+            "size_bytes": path.stat().st_size if path.exists() else 0,
+        }
+    except sqlite3.Error as e:
+        return {"entries": 0, "error": str(e), "path": str(path), "size_bytes": 0}