searxng-use-cli/scripts/cache.py

"""SQLite-backed result cache for searxng-cli.

Avoids re-hitting the SearXNG instance for identical queries within a
configurable TTL. Cache key is a SHA-256 of the normalized search params
(query + engines + categories + language + time_range + safesearch +
pageno + method), so different parameter combinations get separate entries.

Storage location (in priority order):
  1. ``$SEARXNG_CACHE_DIR`` env var (directory; ``cache.db`` is created inside)
  2. ``~/.cache/searxng-cli/cache.db`` (XDG-style; on Windows this resolves
     to ``C:\\Users\\<user>\\.cache\\searxng-cli\\cache.db``)

Uses WAL journal mode for better read concurrency. Entries expire lazily
on read; :func:`clear` removes all rows. Schema is created on first use.

Design notes:
  * Only the search result dict is cached — fetched page content is NOT,
    because it is large and changes independently of the search result set.
  * The cache key excludes auth headers and timeouts (transient concerns)
    so two callers with the same query + params share an entry.
  * All operations swallow ``sqlite3.Error`` and degrade gracefully — a
    cache failure must never break a search.
"""

import contextlib
import hashlib
import json
import os
import sqlite3
import time
from pathlib import Path

DEFAULT_CACHE_DIR = Path.home() / ".cache" / "searxng-cli"


def _cache_path() -> Path:
    """Resolve the cache database path from env var or default location."""
    env = os.environ.get("SEARXNG_CACHE_DIR")
    if env:
        return Path(env) / "cache.db"
    return DEFAULT_CACHE_DIR / "cache.db"


def _connect(path: Path):
    """Open a connection with WAL mode and ensure the schema exists.

    Returns a :class:`contextlib.closing` wrapper so ``with _connect(...) as conn:``
    closes the connection on exit. ``sqlite3.Connection.__exit__`` only commits /
    rolls back the transaction — it does **not** call ``close()``, which leaks
    file descriptors across many cache operations (especially under
    ``--queries-file`` + ``--cache-ttl`` batch runs).
    """
    path.parent.mkdir(parents=True, exist_ok=True)
    conn = sqlite3.connect(str(path), timeout=10)
    # WAL allows concurrent readers alongside a single writer, which matters
    # when --fetch spawns parallel page fetches that might also touch the cache.
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA synchronous=NORMAL")
    conn.execute(
        """
        CREATE TABLE IF NOT EXISTS search_cache (
            key TEXT PRIMARY KEY,
            created_at REAL NOT NULL,
            ttl_seconds INTEGER NOT NULL,
            payload TEXT NOT NULL
        )
        """
    )
    conn.commit()
    return contextlib.closing(conn)


def _make_key(params: dict) -> str:
    """Build a stable cache key from search params.

    Only params that affect the result set are included; transient fields
    (auth, timeout, format) are excluded so the same logical query hits the
    same cache entry regardless of output formatting.
    """
    # Whitelist the params that actually change what SearXNG returns.
    # fmt: off
    relevant = (
        "q", "categories", "language", "pageno",
        "time_range", "safesearch", "engines", "method",
    )
    # fmt: on
    normalized = {k: params[k] for k in relevant if params.get(k)}
    raw = json.dumps(normalized, sort_keys=True, ensure_ascii=False)
    return hashlib.sha256(raw.encode("utf-8")).hexdigest()


def get(params: dict, ttl_seconds: int):
    """Return cached result if within TTL, else None.

    ``ttl_seconds`` is the caller's current TTL setting. If the stored
    entry was written with a longer TTL, the caller's shorter TTL wins
    (so reducing --cache-ttl takes effect immediately without a clear).
    """
    if ttl_seconds <= 0:
        return None
    key = _make_key(params)
    try:
        with _connect(_cache_path()) as conn:
            row = conn.execute(
                "SELECT payload, created_at, ttl_seconds FROM search_cache "
                "WHERE key = ?",
                (key,),
            ).fetchone()
        if row is None:
            return None
        payload, created_at, stored_ttl = row
        effective_ttl = min(ttl_seconds, stored_ttl)
        if time.time() - created_at > effective_ttl:
            return None
        return json.loads(payload)
    except sqlite3.Error:
        return None
    except (ValueError, json.JSONDecodeError):
        # Corrupt payload — treat as miss
        return None


def put(params: dict, result: dict, ttl_seconds: int) -> None:
    """Store a result with the given TTL. Silently no-ops on TTL<=0 or error."""
    if ttl_seconds <= 0:
        return
    key = _make_key(params)
    payload = json.dumps(result, ensure_ascii=False)
    try:
        with _connect(_cache_path()) as conn:
            conn.execute(
                "INSERT OR REPLACE INTO search_cache "
                "(key, created_at, ttl_seconds, payload) VALUES (?, ?, ?, ?)",
                (key, time.time(), ttl_seconds, payload),
            )
            conn.commit()
    except sqlite3.Error:
        pass


def clear() -> int:
    """Remove all cache entries. Returns count deleted, or 0 on error."""
    try:
        with _connect(_cache_path()) as conn:
            cur = conn.execute("DELETE FROM search_cache")
            conn.commit()
            return cur.rowcount
    except sqlite3.Error:
        return 0


def stats() -> dict:
    """Return cache statistics (entry count, age range, path)."""
    path = _cache_path()
    try:
        with _connect(path) as conn:
            row = conn.execute(
                "SELECT COUNT(*), MIN(created_at), MAX(created_at) "
                "FROM search_cache"
            ).fetchone()
        count, oldest, newest = row
        return {
            "entries": count or 0,
            "oldest_created_at": oldest,
            "newest_created_at": newest,
            "path": str(path),
            "size_bytes": path.stat().st_size if path.exists() else 0,
        }
    except sqlite3.Error as e:
        return {"entries": 0, "error": str(e), "path": str(path), "size_bytes": 0}