Initial commit: SearXNG CLI Toolkit v1.6.0

Multi-instance failover, exponential-backoff retry, SQLite cache, batch mode, domain filter, cross-engine dedup, result sorting, CSV export, structured logging, enhanced Markdown conversion, 155 pytest tests, Gitea Actions CI

tests/conftest.py

@@ -0,0 +1,28 @@
+"""Shared fixtures and path setup for the searxng-cli test suite.
+
+This conftest is loaded by pytest before any test module is imported, so
+the ``sys.path`` insertion below makes the scripts/ directory importable
+as top-level modules (``search``, ``fetch``, ``common``, ``cache``, ``_config``)
+without requiring a package install.
+"""
+import sys
+from pathlib import Path
+
+# Make scripts/ importable as top-level modules.
+SCRIPTS_DIR = Path(__file__).resolve().parent.parent / "scripts"
+if str(SCRIPTS_DIR) not in sys.path:
+    sys.path.insert(0, str(SCRIPTS_DIR))
+
+import pytest
+
+
+@pytest.fixture
+def isolated_cache(monkeypatch, tmp_path):
+    """Redirect the SQLite cache to a per-test temp directory.
+
+    ``cache._cache_path()`` reads ``$SEARXNG_CACHE_DIR``; pointing it at a
+    fresh ``tmp_path`` keeps each test hermetic and prevents cross-test
+    contamination from leftover entries.
+    """
+    monkeypatch.setenv("SEARXNG_CACHE_DIR", str(tmp_path))
+    return tmp_path

tests/test_cache.py

@@ -0,0 +1,144 @@
+"""Tests for scripts/cache.py — SQLite-backed result cache.
+
+Covers: cache key stability, key whitelist (format/auth excluded),
+get/put roundtrip, TTL expiry, caller-shorter-TTL override, clear, stats,
+and the ttl<=0 disable behavior.
+"""
+import sqlite3
+import time
+
+import cache
+
+
+# ----- _make_key -----
+
+def test_make_key_stable():
+    p = {"q": "test", "engines": "google,bing", "format": "json"}
+    k1 = cache._make_key(p)
+    k2 = cache._make_key(p)
+    assert k1 == k2
+    assert len(k1) == 64  # SHA-256 hex digest length
+
+
+def test_make_key_excludes_format():
+    """format is transient (output shape, not result set) — must not affect key."""
+    p1 = {"q": "test", "format": "json"}
+    p2 = {"q": "test", "format": "brief"}
+    assert cache._make_key(p1) == cache._make_key(p2)
+
+
+def test_make_key_differs_on_query():
+    assert cache._make_key({"q": "python"}) != cache._make_key({"q": "rust"})
+
+
+def test_make_key_differs_on_engines():
+    assert cache._make_key({"q": "x", "engines": "google"}) != \
+           cache._make_key({"q": "x", "engines": "bing"})
+
+
+def test_make_key_differs_on_time_range():
+    assert cache._make_key({"q": "x", "time_range": "day"}) != \
+           cache._make_key({"q": "x", "time_range": "year"})
+
+
+# ----- get / put -----
+
+def test_put_get_roundtrip(isolated_cache):
+    params = {"q": "hello", "format": "json"}
+    result = {"results": [{"title": "Hi", "url": "https://example.com"}],
+              "number_of_results": 1}
+    cache.put(params, result, ttl_seconds=60)
+    got = cache.get(params, ttl_seconds=60)
+    assert got is not None
+    assert got["results"][0]["title"] == "Hi"
+
+
+def test_get_miss_when_empty(isolated_cache):
+    assert cache.get({"q": "nope"}, ttl_seconds=60) is None
+
+
+def test_get_expired(isolated_cache):
+    """Entry older than TTL is a miss."""
+    params = {"q": "old", "format": "json"}
+    cache.put(params, {"results": []}, ttl_seconds=60)
+    # Backdate the entry so it's past TTL
+    with cache._connect(cache._cache_path()) as conn:
+        conn.execute(
+            "UPDATE search_cache SET created_at = ? WHERE key = ?",
+            (time.time() - 120, cache._make_key(params)),
+        )
+        conn.commit()
+    assert cache.get(params, ttl_seconds=60) is None
+
+
+def test_get_caller_shorter_ttl_expires(isolated_cache):
+    """Caller's shorter TTL overrides a longer stored TTL (immediate effect)."""
+    params = {"q": "ttl-test", "format": "json"}
+    cache.put(params, {"results": []}, ttl_seconds=3600)
+    with cache._connect(cache._cache_path()) as conn:
+        conn.execute(
+            "UPDATE search_cache SET created_at = ? WHERE key = ?",
+            (time.time() - 120, cache._make_key(params)),
+        )
+        conn.commit()
+    # stored TTL=3600 (still fresh by stored clock) but caller asks 60s -> expired
+    assert cache.get(params, ttl_seconds=60) is None
+
+
+def test_get_caller_longer_ttl_uses_stored(isolated_cache):
+    """Caller's longer TTL does NOT resurrect an expired-by-stored entry."""
+    params = {"q": "ttl2", "format": "json"}
+    cache.put(params, {"results": []}, ttl_seconds=60)
+    with cache._connect(cache._cache_path()) as conn:
+        conn.execute(
+            "UPDATE search_cache SET created_at = ? WHERE key = ?",
+            (time.time() - 120, cache._make_key(params)),
+        )
+        conn.commit()
+    # stored TTL=60 (expired), caller asks 3600 -> effective = min(3600,60)=60 -> expired
+    assert cache.get(params, ttl_seconds=3600) is None
+
+
+# ----- clear -----
+
+def test_clear_removes_entries(isolated_cache):
+    cache.put({"q": "x"}, {"results": []}, ttl_seconds=60)
+    removed = cache.clear()
+    assert removed == 1
+    assert cache.get({"q": "x"}, ttl_seconds=60) is None
+
+
+def test_clear_empty_returns_zero(isolated_cache):
+    assert cache.clear() == 0
+
+
+# ----- stats -----
+
+def test_stats_reports_entries(isolated_cache):
+    cache.put({"q": "stats-test"}, {"results": []}, ttl_seconds=60)
+    s = cache.stats()
+    assert s["entries"] == 1
+    assert s["size_bytes"] > 0
+    assert "path" in s
+    assert s.get("oldest_created_at") is not None
+    assert s.get("newest_created_at") is not None
+
+
+def test_stats_empty(isolated_cache):
+    s = cache.stats()
+    assert s["entries"] == 0
+
+
+# ----- ttl<=0 disable -----
+
+def test_get_ttl_zero_is_disabled(isolated_cache):
+    """ttl_seconds=0 means caching off -> always miss even if stored."""
+    cache.put({"q": "disabled"}, {"results": []}, ttl_seconds=60)
+    assert cache.get({"q": "disabled"}, ttl_seconds=0) is None
+
+
+def test_put_ttl_zero_is_noop(isolated_cache):
+    """put with ttl<=0 should not store anything."""
+    cache.put({"q": "noop"}, {"results": []}, ttl_seconds=0)
+    assert cache.get({"q": "noop"}, ttl_seconds=60) is None
+    assert cache.stats()["entries"] == 0

tests/test_common.py

@@ -0,0 +1,188 @@
+"""Tests for scripts/common.py — auth, charset, retry policy, proxy.
+
+Covers: build_auth_headers (Bearer/Basic/precedence), resolve_auth_* priority
+(CLI > file > env, file parsing, empty-file errors), detect_charset
+(header/meta/fallback), is_retryable_error (429/5xx yes, 404/403 no,
+connection errors yes), and apply_proxy (env vars + NO_PROXY preservation).
+"""
+import base64
+import os
+import urllib.error
+
+import pytest
+from common import (
+    RETRYABLE_STATUS,
+    apply_proxy,
+    build_auth_headers,
+    detect_charset,
+    is_retryable_error,
+    resolve_auth_basic,
+    resolve_auth_bearer,
+)
+
+
+# ----- build_auth_headers -----
+
+def test_build_auth_bearer():
+    assert build_auth_headers(bearer_token="tok123") == \
+        {"Authorization": "Bearer tok123"}
+
+
+def test_build_auth_basic():
+    h = build_auth_headers(basic_auth="user:pass")
+    assert h["Authorization"].startswith("Basic ")
+    decoded = base64.b64decode(h["Authorization"].split(" ", 1)[1]).decode()
+    assert decoded == "user:pass"
+
+
+def test_build_auth_bearer_wins_over_basic():
+    """Bearer takes precedence when both are provided."""
+    h = build_auth_headers(bearer_token="tok", basic_auth="u:p")
+    assert h == {"Authorization": "Bearer tok"}
+
+
+def test_build_auth_none_returns_empty():
+    assert build_auth_headers() == {}
+
+
+# ----- resolve_auth_basic -----
+
+def test_resolve_auth_basic_cli_wins():
+    assert resolve_auth_basic(cli_value="cliu:clip") == "cliu:clip"
+
+
+def test_resolve_auth_basic_file(tmp_path):
+    f = tmp_path / "auth.txt"
+    f.write_text("# header\nu:secret\n", encoding="utf-8")
+    assert resolve_auth_basic(file_path=str(f)) == "u:secret"
+
+
+def test_resolve_auth_basic_env(monkeypatch):
+    monkeypatch.setenv("SEARXNG_BASIC_AUTH", "envu:envp")
+    assert resolve_auth_basic() == "envu:envp"
+
+
+def test_resolve_auth_basic_priority_cli_over_file(tmp_path):
+    f = tmp_path / "auth.txt"
+    f.write_text("fileu:filep", encoding="utf-8")
+    assert resolve_auth_basic(cli_value="cliu:clip", file_path=str(f)) == "cliu:clip"
+
+
+def test_resolve_auth_basic_empty_file_raises(tmp_path):
+    f = tmp_path / "empty.txt"
+    f.write_text("# only comment\n", encoding="utf-8")
+    with pytest.raises(RuntimeError):
+        resolve_auth_basic(file_path=str(f))
+
+
+def test_resolve_auth_basic_missing_file_raises():
+    with pytest.raises(RuntimeError):
+        resolve_auth_basic(file_path="nonexistent_file.txt")
+
+
+# ----- resolve_auth_bearer -----
+
+def test_resolve_auth_bearer_cli():
+    assert resolve_auth_bearer(cli_value="tok") == "tok"
+
+
+def test_resolve_auth_bearer_env(monkeypatch):
+    monkeypatch.setenv("SEARXNG_BEARER_TOKEN", "envtok")
+    assert resolve_auth_bearer() == "envtok"
+
+
+def test_resolve_auth_bearer_file(tmp_path):
+    f = tmp_path / "tok.txt"
+    f.write_text("# header\ntoken123\n", encoding="utf-8")
+    assert resolve_auth_bearer(file_path=str(f)) == "token123"
+
+
+# ----- detect_charset -----
+
+def test_detect_charset_from_header():
+    assert detect_charset(b"hello", "text/html; charset=utf-8") == "utf-8"
+
+
+def test_detect_charset_from_meta_tag():
+    html = b'<html><head><meta charset="gbk"></head><body>hello</body></html>'
+    assert detect_charset(html, "text/html") == "gbk"
+
+
+def test_detect_charset_fallback_utf8():
+    assert detect_charset(b"plain", "text/html") == "utf-8"
+
+
+def test_detect_charset_invalid_header_falls_back():
+    """An invalid charset in the header should not raise; falls back to utf-8."""
+    assert detect_charset(b"hello", "text/html; charset=nonexistent_encoding") == "utf-8"
+
+
+# ----- is_retryable_error -----
+
+def _http_error(code):
+    return urllib.error.HTTPError("http://x", code, "msg", {}, None)
+
+
+def test_retryable_429():
+    assert is_retryable_error(_http_error(429)) is True
+
+
+def test_retryable_502():
+    assert is_retryable_error(_http_error(502)) is True
+
+
+def test_retryable_503():
+    assert is_retryable_error(_http_error(503)) is True
+
+
+def test_retryable_504():
+    assert is_retryable_error(_http_error(504)) is True
+
+
+def test_non_retryable_404():
+    assert is_retryable_error(_http_error(404)) is False
+
+
+def test_non_retryable_403():
+    assert is_retryable_error(_http_error(403)) is False
+
+
+def test_non_retryable_200():
+    assert is_retryable_error(_http_error(200)) is False
+
+
+def test_retryable_url_error():
+    assert is_retryable_error(urllib.error.URLError("refused")) is True
+
+
+def test_retryable_os_error():
+    assert is_retryable_error(OSError("timeout")) is True
+
+
+def test_retryable_status_set_contents():
+    assert RETRYABLE_STATUS == frozenset({429, 502, 503, 504})
+
+
+# ----- apply_proxy -----
+
+def test_apply_proxy_sets_env_vars(monkeypatch):
+    monkeypatch.delenv("HTTP_PROXY", raising=False)
+    monkeypatch.delenv("HTTPS_PROXY", raising=False)
+    monkeypatch.delenv("NO_PROXY", raising=False)
+    apply_proxy("http://corp:8080")
+    assert os.environ["HTTP_PROXY"] == "http://corp:8080"
+    assert os.environ["HTTPS_PROXY"] == "http://corp:8080"
+    assert os.environ["NO_PROXY"] == "localhost,127.0.0.1,::1"
+
+
+def test_apply_proxy_empty_is_noop(monkeypatch):
+    monkeypatch.delenv("HTTP_PROXY", raising=False)
+    apply_proxy("")
+    assert "HTTP_PROXY" not in os.environ
+
+
+def test_apply_proxy_preserves_existing_no_proxy(monkeypatch):
+    """setdefault must NOT overwrite a user-set NO_PROXY."""
+    monkeypatch.setenv("NO_PROXY", "custom.host")
+    apply_proxy("http://corp:8080")
+    assert os.environ["NO_PROXY"] == "custom.host"

tests/test_fetch_markdown.py

@@ -0,0 +1,192 @@
+"""Tests for scripts/fetch.py — MarkdownConverter and text extraction.
+
+Covers: headings, links (incl. nested bold inside <a>), fenced code blocks,
+inline code, blockquotes, unordered lists, GFM tables (incl. pipe escaping),
+emphasis, image tags, script/style stripping, and the stdlib text extractor.
+"""
+from fetch import extract_with_stdlib, html_to_markdown
+
+
+# ----- Basic structure -----
+
+def test_markdown_basic_paragraph():
+    md = html_to_markdown("<p>Hello world</p>")
+    assert "Hello world" in md
+
+
+def test_markdown_h1():
+    md = html_to_markdown("<h1>Title</h1>")
+    assert "# Title" in md
+
+
+def test_markdown_h3():
+    md = html_to_markdown("<h3>Subtitle</h3>")
+    assert "### Subtitle" in md
+
+
+def test_markdown_empty_input():
+    assert html_to_markdown("").strip() == ""
+
+
+# ----- Links -----
+
+def test_markdown_simple_link():
+    md = html_to_markdown('<a href="https://example.com">click</a>')
+    assert "[click](https://example.com)" in md
+
+
+def test_markdown_link_with_nested_bold():
+    """Bold text inside a link must survive — the tree-based parser handles
+    this where a regex approach would break."""
+    md = html_to_markdown('<a href="https://x.com"><b>bold link</b></a>')
+    assert "https://x.com" in md
+    assert "bold link" in md
+
+
+# ----- Code -----
+
+def test_markdown_fenced_code_block():
+    md = html_to_markdown("<pre>print('hi')</pre>")
+    assert "```" in md
+    assert "print('hi')" in md
+
+
+def test_markdown_inline_code():
+    md = html_to_markdown("<p>use <code>pip</code> to install</p>")
+    assert "`pip`" in md
+
+
+def test_markdown_pre_code_not_double_fenced():
+    """<pre><code>...</code></pre> should render as one fence, not backticks-in-fence."""
+    md = html_to_markdown("<pre><code>x = 1</code></pre>")
+    assert "```\nx = 1\n```" in md
+    assert "`x = 1`" not in md  # no inline backticks around the content
+
+
+# ----- Blockquote / lists -----
+
+def test_markdown_blockquote():
+    md = html_to_markdown("<blockquote>quoted text</blockquote>")
+    assert "> quoted text" in md
+
+
+def test_markdown_unordered_list():
+    md = html_to_markdown("<ul><li>one</li><li>two</li></ul>")
+    assert "- one" in md
+    assert "- two" in md
+
+
+# ----- Tables (GFM) -----
+
+def test_markdown_table_basic():
+    html = "<table><tr><th>A</th><th>B</th></tr><tr><td>1</td><td>2</td></tr></table>"
+    md = html_to_markdown(html)
+    assert "| A | B |" in md
+    assert "| --- | --- |" in md
+    assert "| 1 | 2 |" in md
+
+
+def test_markdown_table_pipe_escaped():
+    """Pipe chars inside cells must be escaped so they don't break the row."""
+    html = "<table><tr><th>Col</th></tr><tr><td>a|b</td></tr></table>"
+    md = html_to_markdown(html)
+    assert "a\\|b" in md
+
+
+# ----- Emphasis / images -----
+
+def test_markdown_strong_emphasis():
+    md = html_to_markdown("<p><strong>bold</strong></p>")
+    assert "**bold**" in md
+
+
+def test_markdown_em_italic():
+    md = html_to_markdown("<p><em>italic</em></p>")
+    assert "*italic*" in md
+
+
+def test_markdown_image():
+    md = html_to_markdown('<img src="http://x.com/a.png" alt="pic">')
+    assert "![pic](http://x.com/a.png)" in md
+
+
+# ---- Stripping non-content -----
+
+def test_markdown_strips_script():
+    md = html_to_markdown("<script>alert(1)</script><p>visible</p>")
+    assert "alert" not in md
+    assert "visible" in md
+
+
+def test_markdown_strips_style():
+    md = html_to_markdown("<style>body{color:red}</style><p>visible</p>")
+    assert "color" not in md
+    assert "visible" in md
+
+
+# ----- Stdlib text extractor -----
+
+def test_extract_with_stdlib_basic():
+    html = "<html><body><p>Hello</p><script>x</script></body></html>"
+    text = extract_with_stdlib(html)
+    assert "Hello" in text
+    assert "x" not in text  # script content excluded
+
+
+def test_extract_with_stdlib_skips_nav():
+    html = "<nav>menu</nav><article>content</article>"
+    text = extract_with_stdlib(html)
+    assert "content" in text
+    assert "menu" not in text
+
+
+# ----- Ordered / nested lists -----
+
+def test_markdown_ordered_list_numbered():
+    md = html_to_markdown("<ol><li>first</li><li>second</li><li>third</li></ol>")
+    assert "1. first" in md
+    assert "2. second" in md
+    assert "3. third" in md
+
+
+def test_markdown_nested_ordered_list():
+    html = "<ol><li>outer1<ol><li>inner1</li><li>inner2</li></ol></li><li>outer2</li></ol>"
+    md = html_to_markdown(html)
+    assert "1. outer1" in md
+    assert "  1. inner1" in md
+    assert "  2. inner2" in md
+    assert "2. outer2" in md
+
+
+def test_markdown_nested_unordered_list():
+    html = "<ul><li>outer<ul><li>inner</li></ul></li></ul>"
+    md = html_to_markdown(html)
+    assert "- outer" in md
+    assert "  - inner" in md
+
+
+def test_markdown_mixed_nested_list():
+    """<ul> containing <ol> — markers must match list type at each depth."""
+    html = "<ul><li>item<ol><li>sub1</li><li>sub2</li></ol></li></ul>"
+    md = html_to_markdown(html)
+    assert "- item" in md
+    assert "  1. sub1" in md
+    assert "  2. sub2" in md
+
+
+# ----- Definition lists (<dl>/<dt>/<dd>) -----
+
+def test_markdown_definition_list():
+    html = "<dl><dt>Term</dt><dd>Definition</dd></dl>"
+    md = html_to_markdown(html)
+    assert "**Term**" in md
+    assert "  Definition" in md
+
+
+def test_markdown_definition_list_multiple():
+    html = "<dl><dt>T1</dt><dd>D1</dd><dt>T2</dt><dd>D2</dd></dl>"
+    md = html_to_markdown(html)
+    assert "**T1**" in md
+    assert "**T2**" in md
+    assert "  D1" in md
+    assert "  D2" in md

tests/test_integration.py

@@ -0,0 +1,133 @@
+"""Integration tests — mock urllib to test search_multi / fetch_url end-to-end.
+
+Covers: search_json success/HTML-fallback/404/403, search_multi serial
+failover + all-fail, fetch_url stdlib-path success.
+
+No real network calls are made; ``urllib.request.urlopen`` is patched.
+"""
+import json
+import urllib.error
+from unittest.mock import patch, MagicMock
+
+from search import search_json, search_multi
+import fetch as fetch_mod
+from fetch import fetch_url
+
+
+def _mock_urlopen(data: bytes, content_type="application/json"):
+    """Build a MagicMock that quacks like an urlopen context manager."""
+    resp = MagicMock()
+    resp.read.return_value = data
+    resp.headers = {"Content-Type": content_type}
+    resp.__enter__.return_value = resp
+    resp.__exit__.return_value = None
+    return resp
+
+
+# ----- search_json -----
+
+def test_search_json_success():
+    payload = json.dumps({"results": [{"title": "hi", "url": "https://x.com"}]})
+    with patch("urllib.request.urlopen",
+               return_value=_mock_urlopen(payload.encode())):
+        r = search_json("https://s.example.com", {"q": "test", "format": "json"})
+    assert r is not None
+    assert r["results"][0]["title"] == "hi"
+
+
+def test_search_json_html_returns_none():
+    """HTML response (not JSON) → return None (JSON unsupported)."""
+    with patch("urllib.request.urlopen",
+               return_value=_mock_urlopen(b"<html>not json</html>")):
+        r = search_json("https://s.example.com", {"q": "test", "format": "json"})
+    assert r is None
+
+
+def test_search_json_404_returns_none():
+    """404 = JSON endpoint absent → None (triggers HTML fallback upstream)."""
+    err = urllib.error.HTTPError("url", 404, "Not Found", {}, None)
+    with patch("urllib.request.urlopen", side_effect=err):
+        r = search_json("https://s.example.com", {"q": "test", "format": "json"})
+    assert r is None
+
+
+def test_search_json_403_raises():
+    """403 = auth/IP issue → must raise, not silently fall back to HTML."""
+    err = urllib.error.HTTPError("url", 403, "Forbidden", {}, None)
+    with patch("urllib.request.urlopen", side_effect=err):
+        try:
+            search_json("https://s.example.com", {"q": "test", "format": "json"})
+            assert False, "should have raised"
+        except urllib.error.HTTPError:
+            pass  # expected
+
+
+# ----- search_multi (serial failover) -----
+
+def test_search_multi_serial_failover():
+    """First instance fails, second succeeds → return second's results."""
+    payload = json.dumps({"results": [{"title": "from-b", "url": "https://b.com"}]})
+    err = urllib.error.URLError("connection refused")
+    with patch("urllib.request.urlopen",
+               side_effect=[err, _mock_urlopen(payload.encode())]):
+        r = search_multi(
+            ["https://a.example.com", "https://b.example.com"],
+            {"q": "test", "format": "json"},
+            parallel=False, retry_per=0,
+        )
+    assert r["results"][0]["title"] == "from-b"
+
+
+def test_search_multi_all_fail_raises():
+    """All instances fail → RuntimeError."""
+    err = urllib.error.URLError("connection refused")
+    with patch("urllib.request.urlopen", side_effect=err):
+        try:
+            search_multi(
+                ["https://a.example.com", "https://b.example.com"],
+                {"q": "test", "format": "json"},
+                parallel=False, retry_per=0,
+            )
+            assert False, "should have raised"
+        except RuntimeError as e:
+            assert "All 2 instances failed" in str(e)
+
+
+def test_search_multi_single_instance_success():
+    """Single instance, serial mode, success → return results."""
+    payload = json.dumps({"results": [{"title": "ok", "url": "https://a.com"}]})
+    with patch("urllib.request.urlopen",
+               return_value=_mock_urlopen(payload.encode())):
+        r = search_multi(
+            ["https://a.example.com"],
+            {"q": "test", "format": "json"},
+            parallel=False, retry_per=0,
+        )
+    assert r["results"][0]["title"] == "ok"
+
+
+# ----- fetch_url (stdlib path) -----
+
+def test_fetch_url_stdlib_success():
+    """fetch_url with stdlib path returns content + content_type."""
+    html = b"<html><body><p>Hello</p></body></html>"
+    resp = _mock_urlopen(html, content_type="text/html; charset=utf-8")
+    with patch("urllib.request.urlopen", return_value=resp), \
+         patch.object(fetch_mod, "_HAS_REQUESTS", False):
+        result = fetch_url("https://example.com", max_retries=0)
+    assert "Hello" in result.content
+    assert "text/html" in result.content_type
+
+
+def test_fetch_url_stdlib_max_size_truncates():
+    """max_size sets truncated=True when response exceeds the cap.
+
+    Note: mock's read() ignores the size arg, so content length is not
+    accurately capped here — we only verify the truncated flag is set.
+    """
+    html = b"<html>" + b"x" * 200 + b"</html>"
+    resp = _mock_urlopen(html, content_type="text/html")
+    with patch("urllib.request.urlopen", return_value=resp), \
+         patch.object(fetch_mod, "_HAS_REQUESTS", False):
+        result = fetch_url("https://example.com", max_retries=0, max_size=50)
+    assert result.truncated is True

tests/test_logging.py

@@ -0,0 +1,120 @@
+"""Tests for the shared logging configuration (setup_logging).
+
+Verifies:
+  * Default level is INFO (matches previous print-to-stderr behavior).
+  * --verbose sets DEBUG.
+  * --quiet sets WARNING.
+  * All log output goes to stderr, never stdout.
+  * Repeated setup calls don't stack duplicate handlers.
+  * Child loggers (searxng.search, searxng.fetch, searxng.common) inherit
+    the root searxng logger's level.
+"""
+import logging
+import sys
+
+from common import setup_logging
+
+
+_ROOT = logging.getLogger("searxng")
+
+
+def _reset_logger():
+    """Clear the searxng logger so each test starts fresh."""
+    _ROOT.handlers.clear()
+    _ROOT.setLevel(logging.NOTSET)
+
+
+def test_setup_logging_default_info():
+    _reset_logger()
+    setup_logging()
+    assert _ROOT.level == logging.INFO
+
+
+def test_setup_logging_verbose_debug():
+    _reset_logger()
+    setup_logging(verbose=True)
+    assert _ROOT.level == logging.DEBUG
+
+
+def test_setup_logging_quiet_warning():
+    _reset_logger()
+    setup_logging(quiet=True)
+    assert _ROOT.level == logging.WARNING
+
+
+def test_setup_logging_verbose_overrides_quiet():
+    """If both --verbose and --quiet are passed, verbose wins (checked first)."""
+    _reset_logger()
+    setup_logging(verbose=True, quiet=True)
+    assert _ROOT.level == logging.DEBUG
+
+
+def test_setup_logging_no_duplicate_handlers():
+    _reset_logger()
+    setup_logging()
+    setup_logging()
+    setup_logging()
+    assert len(_ROOT.handlers) == 1
+
+
+def test_setup_logging_output_to_stderr(capsys):
+    """Log messages must go to stderr, never stdout."""
+    _reset_logger()
+    setup_logging()
+    log = logging.getLogger("searxng.search")
+    log.info("test message")
+    captured = capsys.readouterr()
+    assert "test message" in captured.err
+    assert captured.out == ""
+
+
+def test_child_logger_inherits_level():
+    """Child loggers (searxng.search, searxng.fetch, searxng.common) must
+    see the level set on the root searxng logger."""
+    _reset_logger()
+    setup_logging(verbose=True)
+    for name in ("searxng.search", "searxng.fetch", "searxng.common"):
+        child = logging.getLogger(name)
+        assert child.getEffectiveLevel() == logging.DEBUG
+
+
+def test_quiet_suppresses_info(capsys):
+    """In quiet mode, INFO messages are NOT written to stderr."""
+    _reset_logger()
+    setup_logging(quiet=True)
+    log = logging.getLogger("searxng.search")
+    log.info("this should be hidden")
+    log.warning("this should be visible")
+    captured = capsys.readouterr()
+    assert "this should be hidden" not in captured.err
+    assert "this should be visible" in captured.err
+
+
+def test_verbose_shows_debug(capsys):
+    """In verbose mode, DEBUG messages ARE written to stderr."""
+    _reset_logger()
+    setup_logging(verbose=True)
+    log = logging.getLogger("searxng.fetch")
+    log.debug("debug detail")
+    captured = capsys.readouterr()
+    assert "debug detail" in captured.err
+
+
+def test_default_hides_debug(capsys):
+    """In default (INFO) mode, DEBUG messages are NOT written to stderr."""
+    _reset_logger()
+    setup_logging()
+    log = logging.getLogger("searxng.search")
+    log.debug("hidden debug")
+    log.info("visible info")
+    captured = capsys.readouterr()
+    assert "hidden debug" not in captured.err
+    assert "visible info" in captured.err
+
+
+def test_propagate_disabled():
+    """The searxng logger must not propagate to the root logger (avoids
+    duplicate output via the root handler)."""
+    _reset_logger()
+    setup_logging()
+    assert _ROOT.propagate is False

tests/test_search.py

@@ -0,0 +1,496 @@
+"""Tests for scripts/search.py — pure-logic functions (no network).
+
+Covers: parse_instances (URL normalization, lists, whitespace),
+_normalize_csv, filter_results_by_domain (include/exclude, www, case,
+precedence), _read_queries_file (comments, blanks, missing file),
+_cfg_int (str/int/absent/bad), _build_params (param construction,
+time_range=none exclusion), and _merge_headers.
+"""
+import pytest
+from search import (
+    _build_params,
+    _cfg_int,
+    _format_results,
+    _merge_headers,
+    _normalize_csv,
+    _read_queries_file,
+    deduplicate_results,
+    filter_results_by_domain,
+    load_config,
+    parse_instances,
+    sort_results,
+)
+
+
+# ----- parse_instances -----
+
+def test_parse_instances_single():
+    assert parse_instances("https://example.com") == ["https://example.com"]
+
+
+def test_parse_instances_adds_https_prefix():
+    assert parse_instances("example.com") == ["https://example.com"]
+
+
+def test_parse_instances_strips_trailing_slash():
+    assert parse_instances("https://example.com/") == ["https://example.com"]
+
+
+def test_parse_instances_multiple_comma():
+    assert parse_instances("a.com,b.com") == ["https://a.com", "https://b.com"]
+
+
+def test_parse_instances_handles_whitespace():
+    assert parse_instances(" a.com , b.com ") == ["https://a.com", "https://b.com"]
+
+
+def test_parse_instances_empty_string():
+    assert parse_instances("") == []
+
+
+def test_parse_instances_preserves_http():
+    assert parse_instances("http://localhost:8080") == ["http://localhost:8080"]
+
+
+def test_parse_instances_skips_empty_entries():
+    assert parse_instances("a.com,,b.com,") == ["https://a.com", "https://b.com"]
+
+
+# ----- _normalize_csv -----
+
+def test_normalize_csv_strips_spaces():
+    assert _normalize_csv("google, bing, brave") == "google,bing,brave"
+
+
+def test_normalize_csv_drops_empty_parts():
+    assert _normalize_csv("google,,bing,") == "google,bing"
+
+
+def test_normalize_csv_empty_input():
+    assert _normalize_csv("") == ""
+
+
+# ----- filter_results_by_domain -----
+
+def _results(*urls):
+    return {"results": [{"url": u, "title": u} for u in urls]}
+
+
+def test_filter_no_args_returns_unchanged():
+    r = _results("https://a.com/1", "https://b.com/2")
+    out = filter_results_by_domain(dict(r))
+    assert len(out["results"]) == 2
+
+
+def test_filter_include_allowlist():
+    r = _results("https://a.com/1", "https://b.com/2", "https://c.com/3")
+    out = filter_results_by_domain(dict(r), include_domains=["a.com"])
+    assert len(out["results"]) == 1
+    assert out["results"][0]["url"] == "https://a.com/1"
+
+
+def test_filter_include_www_normalized():
+    """www. prefix is stripped for matching, so 'a.com' matches 'www.a.com'."""
+    r = _results("https://www.a.com/1", "https://b.com/2")
+    out = filter_results_by_domain(dict(r), include_domains=["a.com"])
+    assert len(out["results"]) == 1
+    assert out["results"][0]["url"] == "https://www.a.com/1"
+
+
+def test_filter_exclude_blocklist():
+    r = _results("https://a.com/1", "https://b.com/2")
+    out = filter_results_by_domain(dict(r), exclude_domains=["b.com"])
+    assert len(out["results"]) == 1
+    assert out["results"][0]["url"] == "https://a.com/1"
+
+
+def test_filter_exclude_overrides_include():
+    """When a domain is in BOTH lists, exclude wins (result dropped).
+
+    Rationale: include filters first (allowlist), then exclude filters the
+    survivors (blocklist). A domain listed in both is kept by include then
+    removed by exclude — exclude is the more explicit "do not want" intent.
+    """
+    r = _results("https://a.com/1")
+    out = filter_results_by_domain(dict(r), include_domains=["a.com"],
+                                   exclude_domains=["a.com"])
+    assert len(out["results"]) == 0
+
+
+def test_filter_case_insensitive():
+    r = _results("https://A.COM/1")
+    out = filter_results_by_domain(dict(r), include_domains=["a.com"])
+    assert len(out["results"]) == 1
+
+
+def test_filter_empty_results_list():
+    out = filter_results_by_domain({"results": []}, include_domains=["a.com"])
+    assert out["results"] == []
+
+
+def test_filter_no_results_key():
+    """Missing 'results' key should not raise."""
+    out = filter_results_by_domain({}, include_domains=["a.com"])
+    assert out == {}
+
+
+def test_filter_multiple_include():
+    r = _results("https://a.com/1", "https://b.com/2", "https://c.com/3")
+    out = filter_results_by_domain(dict(r), include_domains=["a.com", "c.com"])
+    assert len(out["results"]) == 2
+
+
+# ----- _read_queries_file -----
+
+def test_read_queries_file_basic(tmp_path):
+    f = tmp_path / "queries.txt"
+    f.write_text("query one\n# comment\n\nquery two\n", encoding="utf-8")
+    assert _read_queries_file(str(f)) == ["query one", "query two"]
+
+
+def test_read_queries_file_missing_raises():
+    with pytest.raises(RuntimeError):
+        _read_queries_file("nonexistent_file.txt")
+
+
+def test_read_queries_file_all_comments(tmp_path):
+    f = tmp_path / "empty.txt"
+    f.write_text("# only comments\n# another\n", encoding="utf-8")
+    assert _read_queries_file(str(f)) == []
+
+
+def test_read_queries_file_strips_whitespace(tmp_path):
+    f = tmp_path / "q.txt"
+    f.write_text("  spaced query  \n", encoding="utf-8")
+    assert _read_queries_file(str(f)) == ["spaced query"]
+
+
+# ----- _cfg_int -----
+
+def test_cfg_int_present_int_value():
+    assert _cfg_int({"timeout": 15}, "timeout", 30) == 15
+
+
+def test_cfg_int_present_str_value():
+    """TOML may carry the value as a string; _cfg_int must coerce."""
+    assert _cfg_int({"timeout": "15"}, "timeout", 30) == 15
+
+
+def test_cfg_int_absent_returns_default():
+    assert _cfg_int({}, "timeout", 30) == 30
+
+
+def test_cfg_int_bad_value_returns_default():
+    assert _cfg_int({"timeout": "abc"}, "timeout", 30) == 30
+
+
+def test_cfg_int_none_default():
+    """Defaults may be None (e.g. --retry); absent key must return None."""
+    assert _cfg_int({}, "max_retries", None) is None
+
+
+# ----- _build_params -----
+
+class _Args:
+    """Minimal argparse.Namespace stand-in for _build_params tests."""
+
+    def __init__(self, **overrides):
+        self.categories = None
+        self.language = None
+        self.pageno = 1
+        self.time_range = "year"
+        self.safesearch = 0
+        self.engines = "google,bing"
+        for k, v in overrides.items():
+            setattr(self, k, v)
+
+
+def test_build_params_minimal():
+    p = _build_params("hello", _Args())
+    assert p["q"] == "hello"
+    assert p["format"] == "json"
+    assert "categories" not in p
+    assert "language" not in p
+    assert p["engines"] == "google,bing"
+
+
+def test_build_params_normalizes_categories():
+    p = _build_params("x", _Args(categories="general, news"))
+    assert p["categories"] == "general,news"
+
+
+def test_build_params_time_range_none_excluded():
+    p = _build_params("x", _Args(time_range="none"))
+    assert "time_range" not in p
+
+
+def test_build_params_pageno_as_string():
+    p = _build_params("x", _Args(pageno=3))
+    assert p["pageno"] == "3"
+
+
+def test_build_params_safesearch_as_string():
+    p = _build_params("x", _Args(safesearch=1))
+    assert p["safesearch"] == "1"
+
+
+# ----- _merge_headers -----
+
+def test_merge_headers_basic():
+    assert _merge_headers({"a": 1}, {"b": 2}) == {"a": 1, "b": 2}
+
+
+def test_merge_headers_later_overrides():
+    assert _merge_headers({"a": 1}, {"a": 2}) == {"a": 2}
+
+
+def test_merge_headers_skips_none_dicts():
+    assert _merge_headers(None, {"a": 1}, None) == {"a": 1}
+
+
+def test_merge_headers_all_none():
+    assert _merge_headers(None, None) == {}
+
+
+# ----- deduplicate_results -----
+
+def test_dedup_removes_exact_duplicate_url():
+    r = {"results": [
+        {"url": "https://a.com/1", "engine": "google", "score": 1.0},
+        {"url": "https://a.com/1", "engine": "bing", "score": 0.5},
+    ]}
+    out = deduplicate_results(r)
+    assert len(out["results"]) == 1
+    assert out["results"][0]["engine"] == "google"  # first kept
+
+
+def test_dedup_strips_tracking_params():
+    """utm_*, gclid, fbclid, etc. are stripped before comparison."""
+    r = {"results": [
+        {"url": "https://a.com/page?utm_source=x&id=1"},
+        {"url": "https://a.com/page?id=1&utm_medium=y"},
+    ]}
+    out = deduplicate_results(r)
+    assert len(out["results"]) == 1
+
+
+def test_dedup_strips_fragment():
+    r = {"results": [
+        {"url": "https://a.com/page#section1"},
+        {"url": "https://a.com/page#section2"},
+    ]}
+    out = deduplicate_results(r)
+    assert len(out["results"]) == 1
+
+
+def test_dedup_normalizes_scheme_host_case():
+    r = {"results": [
+        {"url": "HTTPS://Example.COM/path"},
+        {"url": "https://example.com/path"},
+    ]}
+    out = deduplicate_results(r)
+    assert len(out["results"]) == 1
+
+
+def test_dedup_normalizes_param_order():
+    r = {"results": [
+        {"url": "https://a.com/p?a=1&b=2"},
+        {"url": "https://a.com/p?b=2&a=1"},
+    ]}
+    out = deduplicate_results(r)
+    assert len(out["results"]) == 1
+
+
+def test_dedup_keeps_different_urls():
+    r = {"results": [
+        {"url": "https://a.com/1"},
+        {"url": "https://a.com/2"},
+    ]}
+    out = deduplicate_results(r)
+    assert len(out["results"]) == 2
+
+
+def test_dedup_keeps_url_less_results():
+    """Results without a URL are never deduped (kept as-is)."""
+    r = {"results": [
+        {"title": "no url"},
+        {"title": "also no url"},
+    ]}
+    out = deduplicate_results(r)
+    assert len(out["results"]) == 2
+
+
+def test_dedup_empty_results():
+    out = deduplicate_results({"results": []})
+    assert out["results"] == []
+
+
+def test_dedup_no_results_key():
+    out = deduplicate_results({})
+    assert out == {}
+
+
+# ----- sort_results -----
+
+def test_sort_by_score_descending():
+    r = {"results": [
+        {"url": "https://a.com", "score": 0.5},
+        {"url": "https://b.com", "score": 2.0},
+        {"url": "https://c.com", "score": 1.0},
+    ]}
+    out = sort_results(r, "score")
+    assert [x["url"] for x in out["results"]] == [
+        "https://b.com", "https://c.com", "https://a.com"
+    ]
+
+
+def test_sort_by_score_none_at_end():
+    """Entries without score keep relative order at the end."""
+    r = {"results": [
+        {"url": "https://a.com", "score": 1.0},
+        {"url": "https://b.com"},  # no score
+        {"url": "https://c.com", "score": 3.0},
+        {"url": "https://d.com"},  # no score
+    ]}
+    out = sort_results(r, "score")
+    assert out["results"][0]["url"] == "https://c.com"
+    assert out["results"][1]["url"] == "https://a.com"
+    # no-score entries keep relative order: b before d
+    assert out["results"][2]["url"] == "https://b.com"
+    assert out["results"][3]["url"] == "https://d.com"
+
+
+def test_sort_by_date_descending():
+    r = {"results": [
+        {"url": "https://a.com", "published_date": "2024-01-01"},
+        {"url": "https://b.com", "published_date": "2024-06-15"},
+        {"url": "https://c.com", "published_date": "2024-03-10"},
+    ]}
+    out = sort_results(r, "date")
+    assert [x["url"] for x in out["results"]] == [
+        "https://b.com", "https://c.com", "https://a.com"
+    ]
+
+
+def test_sort_by_date_none_at_end():
+    r = {"results": [
+        {"url": "https://a.com", "published_date": "2024-01-01"},
+        {"url": "https://b.com"},  # no date
+    ]}
+    out = sort_results(r, "date")
+    assert out["results"][0]["url"] == "https://a.com"
+    assert out["results"][1]["url"] == "https://b.com"
+
+
+def test_sort_by_engine_ascending():
+    r = {"results": [
+        {"url": "https://a.com", "engine": "duckduckgo"},
+        {"url": "https://b.com", "engine": "bing"},
+        {"url": "https://c.com", "engine": "google"},
+    ]}
+    out = sort_results(r, "engine")
+    assert [x["engine"] for x in out["results"]] == ["bing", "duckduckgo", "google"]
+
+
+def test_sort_by_none_preserves_order():
+    r = {"results": [
+        {"url": "https://a.com", "score": 0.5},
+        {"url": "https://b.com", "score": 2.0},
+    ]}
+    out = sort_results(r, "none")
+    assert [x["url"] for x in out["results"]] == ["https://a.com", "https://b.com"]
+
+
+def test_sort_empty_results():
+    out = sort_results({"results": []}, "score")
+    assert out["results"] == []
+
+
+def test_sort_no_results_key():
+    out = sort_results({}, "score")
+    assert out == {}
+
+
+# ----- _format_results (csv) -----
+
+def test_format_csv_basic():
+    results = {"results": [
+        {"title": "T1", "url": "https://a.com", "engine": "google", "score": 1.0,
+         "published_date": "2024-01-01", "content": "snippet one"},
+    ]}
+    args = _Args(format="csv", snippet_len=0, fetch=0)
+    out = _format_results(results, args)
+    assert "title,url,engine,score,published_date,content" in out
+    assert "T1" in out
+    assert "https://a.com" in out
+    assert "google" in out
+    assert "1.0" in out
+    assert "snippet one" in out
+
+
+def test_format_csv_multiple_rows():
+    results = {"results": [
+        {"title": "A", "url": "https://a.com", "engine": "google", "score": 2.0,
+         "published_date": "", "content": "ca"},
+        {"title": "B", "url": "https://b.com", "engine": "bing", "score": 1.0,
+         "published_date": "2024-06-01", "content": "cb"},
+    ]}
+    args = _Args(format="csv", snippet_len=0, fetch=0)
+    out = _format_results(results, args)
+    lines = out.strip().split("\n")
+    assert len(lines) == 3  # header + 2 rows
+    assert lines[0].startswith("title,url")
+
+
+def test_format_csv_empty_results():
+    results = {"results": []}
+    args = _Args(format="csv", snippet_len=0, fetch=0)
+    out = _format_results(results, args)
+    assert "title,url,engine,score,published_date,content" in out
+
+
+def test_format_csv_missing_fields():
+    """Results with missing fields → empty string in CSV, no crash."""
+    results = {"results": [
+        {"title": "Only Title"},  # no url, engine, score, etc.
+    ]}
+    args = _Args(format="csv", snippet_len=0, fetch=0)
+    out = _format_results(results, args)
+    assert "Only Title" in out
+
+
+def test_format_csv_comma_in_content_escaped():
+    """Commas in content are properly quoted by the csv module."""
+    results = {"results": [
+        {"title": "T", "url": "https://a.com", "engine": "g", "score": 1.0,
+         "published_date": "", "content": "has, comma"},
+    ]}
+    args = _Args(format="csv", snippet_len=0, fetch=0)
+    out = _format_results(results, args)
+    assert '"has, comma"' in out
+
+
+# ----- load_config (--config FILE) -----
+
+def test_load_config_explicit_path(tmp_path):
+    f = tmp_path / "test.toml"
+    f.write_text(
+        '[searxng]\ninstance = "https://x.example.com"\ntimeout = 20\nformat = "brief"\n',
+        encoding="utf-8")
+    cfg = load_config(str(f))
+    assert cfg["instance"] == "https://x.example.com"
+    assert cfg["timeout"] == 20
+    assert cfg["format"] == "brief"
+
+
+def test_load_config_nonexistent_returns_empty():
+    cfg = load_config("/nonexistent/path/config.toml")
+    assert cfg == {}
+
+
+def test_load_config_top_level_table(tmp_path):
+    """Config without [searxng] section — top-level keys used directly."""
+    f = tmp_path / "flat.toml"
+    f.write_text('instance = "https://flat.example.com"\n', encoding="utf-8")
+    cfg = load_config(str(f))
+    assert cfg["instance"] == "https://flat.example.com"