or id="answer" populates answers list."""
+ html = '
The answer is 42
'
+ parser = SearXNGHTMLParser()
+ parser.feed(html)
+ assert len(parser.answers) == 1
+ assert "42" in parser.answers[0]
+
+
+def test_parser_answer_by_id():
+ html = '
42
'
+ parser = SearXNGHTMLParser()
+ parser.feed(html)
+ assert len(parser.answers) == 1
+
+
+# ----- parse_html_results -----
+
+def test_parse_html_results_shape():
+ """parse_html_results returns a dict with all expected keys."""
+ html = """
+
+ c
+
+
a1
+ """
+ r = parse_html_results(html, query="test")
+ assert r["query"] == "test"
+ assert len(r["results"]) == 1
+ assert r["suggestions"] == ["s1"]
+ assert r["answers"] == ["a1"]
+ assert r["corrections"] == []
+ assert r["infoboxes"] == []
+ assert r["unresponsive_engines"] == []
+ assert r["_fallback"] == "html"
+ assert r["number_of_results"] == 1
+
+
+def test_parse_html_results_empty():
+ """Empty HTML yields an empty-but-well-shaped result."""
+ r = parse_html_results("", query="q")
+ assert r["results"] == []
+ assert r["query"] == "q"
+
+
+# ----- search_html -----
+
+def _mock_urlopen(data: bytes, content_type="text/html"):
+ resp = MagicMock()
+ resp.read.return_value = data
+ resp.headers = {"Content-Type": content_type}
+ resp.__enter__.return_value = resp
+ resp.__exit__.return_value = None
+ return resp
+
+
+def test_search_html_success():
+ """search_html fetches and parses an HTML results page."""
+ html = """
+
+ snippet
+ """
+ with patch("urllib.request.urlopen",
+ return_value=_mock_urlopen(html.encode())):
+ r = search_html("https://s.example.com", {"q": "test"})
+ assert r["results"][0]["title"] == "R"
+ assert r["results"][0]["url"] == "https://r.com"
+
+
+def test_search_html_strips_format_param():
+ """search_html must not pass format=json to the HTML endpoint."""
+ captured_req = {}
+
+ def _capture(req, timeout=None):
+ captured_req["url"] = req.full_url
+ return _mock_urlopen(b"
")
+
+ with patch("urllib.request.urlopen", side_effect=_capture):
+ search_html("https://s.example.com", {"q": "test", "format": "json"})
+ assert "format=json" not in captured_req["url"]
+ assert "q=test" in captured_req["url"]
+
+
+def test_search_html_error_raises_runtime():
+ """Network errors are wrapped in RuntimeError with context."""
+ err = urllib.error.URLError("connection refused")
+ with patch("urllib.request.urlopen", side_effect=err):
+ try:
+ search_html("https://s.example.com", {"q": "test"})
+ assert False, "should raise"
+ except RuntimeError as e:
+ assert "HTML search failed" in str(e)
+
+
+# ----- search_single -----
+
+def test_search_single_prefers_json():
+ """When JSON works, search_single returns JSON results directly."""
+ import json as json_mod
+ payload = json_mod.dumps({"results": [{"title": "json", "url": "https://j.com"}]})
+ with patch("urllib.request.urlopen",
+ return_value=_mock_urlopen(payload.encode(),
+ content_type="application/json")):
+ r = search_single("https://s.example.com", {"q": "t", "format": "json"})
+ assert r["results"][0]["title"] == "json"
+
+
+def test_search_single_falls_back_to_html():
+ """When JSON returns None, search_single falls back to HTML parsing."""
+ html = """
+
+ """
+ # First call returns HTML (JSON unsupported), second call (HTML search) also HTML
+ with patch("urllib.request.urlopen",
+ return_value=_mock_urlopen(html.encode())):
+ r = search_single("https://s.example.com", {"q": "t", "format": "json"})
+ assert r["results"][0]["title"] == "html"
+ assert r.get("_fallback") == "html"
+
+
+def test_search_single_html_fallback_marks_fallback():
+ """HTML fallback path sets _fallback='html' in the result."""
+ html = '
'
+ with patch("urllib.request.urlopen",
+ return_value=_mock_urlopen(html.encode())):
+ r = search_single("https://s.example.com", {"q": "t", "format": "json"})
+ assert r["_fallback"] == "html"
diff --git a/tests/test_instance_resolution.py b/tests/test_instance_resolution.py
new file mode 100644
index 0000000..1443bb8
--- /dev/null
+++ b/tests/test_instance_resolution.py
@@ -0,0 +1,193 @@
+"""Tests for scripts/search.py — instance resolution chain.
+
+Covers:
+ * ``_load_toml`` — TOML file loading (Python 3.11+ tomllib / 3.8-3.10 tomli)
+ * ``_read_instance_file`` — config file parsing (.toml single/list/table,
+ .txt per-line / comments / comma-separated / blanks, missing file,
+ corrupted toml)
+ * ``resolve_instances`` — priority chain (CLI > env > config file > empty)
+ and config-file auto-discovery (cwd/home, .toml before .txt)
+
+Tests are hermetic: cwd and home are redirected to ``tmp_path`` via
+monkeypatch so the real user environment never interferes.
+"""
+import pytest
+from pathlib import Path
+
+from search import _load_toml, _read_instance_file, resolve_instances
+
+
+# ----- _load_toml -----
+
+def test_load_toml_valid_file(tmp_path):
+ f = tmp_path / "test.toml"
+ f.write_text('key = "value"\nnumber = 42\n', encoding="utf-8")
+ data = _load_toml(f)
+ assert data["key"] == "value"
+ assert data["number"] == 42
+
+
+def test_load_toml_returns_dict(tmp_path):
+ f = tmp_path / "test.toml"
+ f.write_text('instance = "https://x.example.com"\n', encoding="utf-8")
+ data = _load_toml(f)
+ assert isinstance(data, dict)
+
+
+# ----- _read_instance_file: .toml -----
+
+def test_read_toml_single_instance(tmp_path):
+ """Top-level instance = "url" → parse_instances single URL."""
+ f = tmp_path / "searxng.toml"
+ f.write_text('instance = "https://x.example.com"\n', encoding="utf-8")
+ assert _read_instance_file(f) == ["https://x.example.com"]
+
+
+def test_read_toml_instances_list(tmp_path):
+ """Top-level instances = ["a", "b"] → normalized list."""
+ f = tmp_path / "searxng.toml"
+ f.write_text('instances = ["a.com", "b.com"]\n', encoding="utf-8")
+ assert _read_instance_file(f) == ["https://a.com", "https://b.com"]
+
+
+def test_read_toml_searxng_table_instance(tmp_path):
+ """Instance under [searxng] table is read correctly."""
+ f = tmp_path / "searxng.toml"
+ f.write_text('[searxng]\ninstance = "https://t.example.com"\n',
+ encoding="utf-8")
+ assert _read_instance_file(f) == ["https://t.example.com"]
+
+
+def test_read_corrupted_toml_returns_empty(tmp_path):
+ """Corrupted (unparseable) toml → empty list, no exception raised."""
+ f = tmp_path / "bad.toml"
+ f.write_text("[searxng\ninstance = broken\n", encoding="utf-8")
+ assert _read_instance_file(f) == []
+
+
+# ----- _read_instance_file: .txt -----
+
+def test_read_txt_one_url_per_line(tmp_path):
+ f = tmp_path / "instances.txt"
+ f.write_text("https://a.com\nhttps://b.com\n", encoding="utf-8")
+ assert _read_instance_file(f) == ["https://a.com", "https://b.com"]
+
+
+def test_read_txt_skips_comments(tmp_path):
+ f = tmp_path / "instances.txt"
+ f.write_text("# comment\nhttps://a.com\n# another\nhttps://b.com\n",
+ encoding="utf-8")
+ assert _read_instance_file(f) == ["https://a.com", "https://b.com"]
+
+
+def test_read_txt_comma_separated(tmp_path):
+ """Comma-separated URLs on one line are split."""
+ f = tmp_path / "instances.txt"
+ f.write_text("a.com,b.com\n", encoding="utf-8")
+ assert _read_instance_file(f) == ["https://a.com", "https://b.com"]
+
+
+def test_read_txt_skips_blank_lines(tmp_path):
+ f = tmp_path / "instances.txt"
+ f.write_text("https://a.com\n\n\nhttps://b.com\n", encoding="utf-8")
+ assert _read_instance_file(f) == ["https://a.com", "https://b.com"]
+
+
+# ----- _read_instance_file: edge cases -----
+
+def test_read_nonexistent_file_returns_empty(tmp_path):
+ """Missing file → empty list (FileNotFoundError caught internally)."""
+ f = tmp_path / "nonexistent.toml"
+ assert _read_instance_file(f) == []
+
+
+# ----- resolve_instances: priority chain -----
+
+def test_resolve_cli_arg_takes_priority_over_env(monkeypatch, tmp_path):
+ """CLI arg wins even when SEARXNG_INSTANCE env var is set."""
+ monkeypatch.setenv("SEARXNG_INSTANCE", "https://env.example.com")
+ assert resolve_instances("https://cli.example.com") == ["https://cli.example.com"]
+
+
+def test_resolve_env_takes_priority_over_config_file(monkeypatch, tmp_path):
+ """Env var wins over a present config file."""
+ monkeypatch.chdir(tmp_path)
+ monkeypatch.setattr(Path, "home", lambda: tmp_path)
+ (tmp_path / "searxng.toml").write_text(
+ 'instance = "https://file.example.com"\n', encoding="utf-8")
+ monkeypatch.setenv("SEARXNG_INSTANCE", "https://env.example.com")
+ assert resolve_instances(None) == ["https://env.example.com"]
+
+
+def test_resolve_no_source_returns_empty(monkeypatch, tmp_path):
+ """No CLI, no env, no config file → empty list."""
+ monkeypatch.chdir(tmp_path)
+ monkeypatch.setattr(Path, "home", lambda: tmp_path)
+ monkeypatch.delenv("SEARXNG_INSTANCE", raising=False)
+ assert resolve_instances(None) == []
+
+
+def test_resolve_cli_none_falls_back_to_env(monkeypatch, tmp_path):
+ """cli_arg=None → use SEARXNG_INSTANCE env var."""
+ monkeypatch.chdir(tmp_path)
+ monkeypatch.setattr(Path, "home", lambda: tmp_path)
+ monkeypatch.setenv("SEARXNG_INSTANCE", "https://env.example.com")
+ assert resolve_instances(None) == ["https://env.example.com"]
+
+
+def test_resolve_no_env_falls_back_to_config_file(monkeypatch, tmp_path):
+ """No CLI, no env → fall back to config file."""
+ monkeypatch.chdir(tmp_path)
+ monkeypatch.setattr(Path, "home", lambda: tmp_path)
+ monkeypatch.delenv("SEARXNG_INSTANCE", raising=False)
+ (tmp_path / "searxng.toml").write_text(
+ 'instance = "https://file.example.com"\n', encoding="utf-8")
+ assert resolve_instances(None) == ["https://file.example.com"]
+
+
+# ----- resolve_instances: config file discovery -----
+
+def test_resolve_discovers_cwd_searxng_toml(monkeypatch, tmp_path):
+ """./searxng.toml is auto-discovered."""
+ monkeypatch.chdir(tmp_path)
+ monkeypatch.setattr(Path, "home", lambda: tmp_path)
+ monkeypatch.delenv("SEARXNG_INSTANCE", raising=False)
+ (tmp_path / "searxng.toml").write_text(
+ 'instance = "https://cwd.example.com"\n', encoding="utf-8")
+ assert resolve_instances(None) == ["https://cwd.example.com"]
+
+
+def test_resolve_discovers_home_searxng_toml(monkeypatch, tmp_path):
+ """~/.config/searxng-cli/searxng.toml is auto-discovered."""
+ monkeypatch.chdir(tmp_path) # cwd has no config file
+ home = tmp_path / "fake_home"
+ cfg_dir = home / ".config" / "searxng-cli"
+ cfg_dir.mkdir(parents=True)
+ (cfg_dir / "searxng.toml").write_text(
+ 'instance = "https://home.example.com"\n', encoding="utf-8")
+ monkeypatch.setattr(Path, "home", lambda: home)
+ monkeypatch.delenv("SEARXNG_INSTANCE", raising=False)
+ assert resolve_instances(None) == ["https://home.example.com"]
+
+
+def test_resolve_discovers_cwd_instances_txt(monkeypatch, tmp_path):
+ """./instances.txt is auto-discovered when no .toml present."""
+ monkeypatch.chdir(tmp_path)
+ monkeypatch.setattr(Path, "home", lambda: tmp_path)
+ monkeypatch.delenv("SEARXNG_INSTANCE", raising=False)
+ (tmp_path / "instances.txt").write_text("https://txt.example.com\n",
+ encoding="utf-8")
+ assert resolve_instances(None) == ["https://txt.example.com"]
+
+
+def test_resolve_prefers_toml_over_txt(monkeypatch, tmp_path):
+ """When both ./searxng.toml and ./instances.txt exist, .toml wins."""
+ monkeypatch.chdir(tmp_path)
+ monkeypatch.setattr(Path, "home", lambda: tmp_path)
+ monkeypatch.delenv("SEARXNG_INSTANCE", raising=False)
+ (tmp_path / "searxng.toml").write_text(
+ 'instance = "https://toml.example.com"\n', encoding="utf-8")
+ (tmp_path / "instances.txt").write_text("https://txt.example.com\n",
+ encoding="utf-8")
+ result = resolve_instances(None)
+ assert result == ["https://toml.example.com"]
diff --git a/tests/test_output_format.py b/tests/test_output_format.py
new file mode 100644
index 0000000..f72d99f
--- /dev/null
+++ b/tests/test_output_format.py
@@ -0,0 +1,187 @@
+"""Tests for scripts/search.py — output formatting functions.
+
+Pure-function tests (no network, no mocking) covering:
+ * format_brief — title/url/content rendering, snippet truncation,
+ suggestions, answers, empty input
+ * format_urls — basic list, skip-empty-url, empty input
+ * _format_results — json/urls/brief/csv dispatch + fetched-pages section
+"""
+import json
+import types
+
+from search import format_brief, format_urls, _format_results
+
+
+# ----- format_brief -----
+
+def test_format_brief_basic():
+ """Basic brief output includes title, URL, and content lines."""
+ results = {"results": [
+ {"title": "Hello World", "url": "https://example.com/1",
+ "content": "A short snippet."},
+ ]}
+ out = format_brief(results)
+ assert "1. Hello World" in out
+ assert "https://example.com/1" in out
+ assert "A short snippet." in out
+
+
+def test_format_brief_no_content():
+ """When content is empty/missing, no content line is emitted."""
+ results = {"results": [
+ {"title": "No Snippet", "url": "https://example.com/2"},
+ ]}
+ out = format_brief(results)
+ assert "1. No Snippet" in out
+ assert "https://example.com/2" in out
+ # Only title + url lines plus a trailing blank line — no content line.
+ lines = out.split("\n")
+ assert len(lines) == 3
+ assert lines[2] == ""
+
+
+def test_format_brief_snippet_truncation():
+ """snippet_len > 0 truncates content to that many characters."""
+ long_text = "abcdefghijklmnopqrstuvwxyz"
+ results = {"results": [
+ {"title": "T", "url": "https://example.com", "content": long_text},
+ ]}
+ out = format_brief(results, snippet_len=5)
+ assert "abcde" in out
+ assert "abcdef" not in out # truncated beyond 5 chars
+
+
+def test_format_brief_with_suggestions():
+ """Suggestions list is rendered as a comma-separated line."""
+ results = {
+ "results": [{"title": "T", "url": "https://example.com", "content": "c"}],
+ "suggestions": ["python asyncio", "python requests"],
+ }
+ out = format_brief(results)
+ assert "Suggestions: python asyncio, python requests" in out
+
+
+def test_format_brief_with_answers():
+ """Each answer is emitted on its own 'Answer:' line."""
+ results = {
+ "results": [{"title": "T", "url": "https://example.com", "content": "c"}],
+ "answers": ["42", "the answer"],
+ }
+ out = format_brief(results)
+ assert "Answer: 42" in out
+ assert "Answer: the answer" in out
+
+
+def test_format_brief_empty_results():
+ """Empty results list (or missing key) yields an empty string."""
+ assert format_brief({"results": []}) == ""
+ assert format_brief({}) == ""
+
+
+# ----- format_urls -----
+
+def test_format_urls_basic():
+ """Each result URL appears on its own line."""
+ results = {"results": [
+ {"url": "https://a.com/1"},
+ {"url": "https://b.com/2"},
+ ]}
+ out = format_urls(results)
+ assert out == "https://a.com/1\nhttps://b.com/2"
+
+
+def test_format_urls_skips_empty():
+ """Results with empty/missing URLs are skipped."""
+ results = {"results": [
+ {"url": "https://a.com/1"},
+ {"url": ""},
+ {"title": "no url here"},
+ {"url": "https://b.com/2"},
+ ]}
+ out = format_urls(results)
+ assert out == "https://a.com/1\nhttps://b.com/2"
+
+
+def test_format_urls_empty_results():
+ """Empty results yield an empty string."""
+ assert format_urls({"results": []}) == ""
+ assert format_urls({}) == ""
+
+
+# ----- _format_results -----
+
+def test_format_results_json():
+ """json format emits valid, parseable JSON mirroring the input."""
+ results = {"results": [
+ {"title": "T", "url": "https://example.com", "content": "c"},
+ ], "suggestions": ["x"]}
+ args = types.SimpleNamespace(format="json")
+ out = _format_results(results, args)
+ parsed = json.loads(out)
+ assert parsed == results
+
+
+def test_format_results_brief():
+ """brief format dispatches to format_brief."""
+ results = {"results": [
+ {"title": "T", "url": "https://example.com", "content": "snippet"},
+ ]}
+ args = types.SimpleNamespace(format="brief", snippet_len=0, fetch=0)
+ out = _format_results(results, args)
+ assert "1. T" in out
+ assert "https://example.com" in out
+ assert "snippet" in out
+
+
+def test_format_results_urls():
+ """urls format dispatches to format_urls."""
+ results = {"results": [
+ {"url": "https://a.com/1"},
+ {"url": "https://b.com/2"},
+ ]}
+ args = types.SimpleNamespace(format="urls")
+ out = _format_results(results, args)
+ assert out == "https://a.com/1\nhttps://b.com/2"
+
+
+def test_format_results_csv_smoke():
+ """csv format emits the header row + one row per result (smoke test)."""
+ results = {"results": [
+ {"title": "T1", "url": "https://a.com", "engine": "google",
+ "score": 1.0, "published_date": "2024-01-01", "content": "snip"},
+ ]}
+ args = types.SimpleNamespace(format="csv", snippet_len=0, fetch=0)
+ out = _format_results(results, args)
+ assert "title,url,engine,score,published_date,content" in out
+ assert "T1" in out
+ assert "https://a.com" in out
+
+
+def test_format_results_brief_with_fetched():
+ """brief + args.fetch>0 + results['fetched'] appends a fetched section."""
+ results = {
+ "results": [{"title": "T", "url": "https://example.com", "content": "c"}],
+ "fetched": [
+ {"url": "https://example.com", "status": "ok",
+ "text": "Full page content here."},
+ ],
+ }
+ args = types.SimpleNamespace(format="brief", snippet_len=0, fetch=3)
+ out = _format_results(results, args)
+ assert "FETCHED PAGES (1 pages)" in out
+ assert "--- https://example.com ---" in out
+ assert "Full page content here." in out
+
+
+def test_format_results_brief_with_fetched_error():
+ """Fetched entries with status != 'ok' render an [ERROR: ...] line."""
+ results = {
+ "results": [{"title": "T", "url": "https://example.com", "content": "c"}],
+ "fetched": [
+ {"url": "https://broken.example", "status": "error", "error": "timeout"},
+ ],
+ }
+ args = types.SimpleNamespace(format="brief", snippet_len=0, fetch=3)
+ out = _format_results(results, args)
+ assert "FETCHED PAGES (1 pages)" in out
+ assert "[ERROR: timeout]" in out
diff --git a/tests/test_parallel_search.py b/tests/test_parallel_search.py
new file mode 100644
index 0000000..3f9aba4
--- /dev/null
+++ b/tests/test_parallel_search.py
@@ -0,0 +1,276 @@
+"""Parallel-mode tests for search_multi (parallel=True branch).
+
+The serial path (parallel=False / single instance) is covered by
+``test_integration.py``. This module focuses on the ThreadPoolExecutor
+branch: result ordering by user input, failover, all-fail, auth-header
+forwarding, retry_per=0 semantics, and exception capture.
+
+Thread-safety note: ``urllib.request.urlopen`` is patched with a
+URL-dispatching *function* (not a ``side_effect`` list). A function with
+no mutable shared state is safe to call concurrently from multiple
+worker threads, whereas a list-based ``side_effect`` would race on the
+shared iterator. ``MagicMock.call_count`` is itself thread-safe.
+"""
+import json
+import logging
+import time
+import urllib.error
+from unittest.mock import MagicMock, patch
+
+from search import search_multi
+
+
+def _mock_urlopen(data: bytes, content_type="application/json"):
+ """Build a MagicMock that quacks like an urlopen context manager."""
+ resp = MagicMock()
+ resp.read.return_value = data
+ resp.headers = {"Content-Type": content_type}
+ resp.__enter__.return_value = resp
+ resp.__exit__.return_value = None
+ return resp
+
+
+def _url_router(routing=None, default=None):
+ """Build a thread-safe side_effect that dispatches by request URL.
+
+ ``routing`` maps a URL substring to either a response object
+ (returned) or a ``BaseException`` (raised). Entries are checked in
+ insertion order (dicts are ordered on Python 3.7+). ``default`` is
+ used when no key matches; if it is an exception it is raised,
+ otherwise returned. An ``AssertionError`` is raised when nothing
+ matches and no default is set — this makes unexpected calls loud
+ rather than silently returning a MagicMock.
+ """
+ routing = routing or {}
+
+ def _side_effect(req, *args, **kwargs):
+ url = getattr(req, "full_url", str(req))
+ for key, resp in routing.items():
+ if key in url:
+ if isinstance(resp, BaseException):
+ raise resp
+ return resp
+ if default is not None:
+ if isinstance(default, BaseException):
+ raise default
+ return default
+ raise AssertionError(
+ f"unexpected urlopen for {url!r} (routing={list(routing)})"
+ )
+ return _side_effect
+
+
+def _payload(title: str, url: str = "https://x.com") -> bytes:
+ return json.dumps({"results": [{"title": title, "url": url}]}).encode()
+
+
+# ------------------------------------------------------------------
+# parallel=True: success paths
+# ------------------------------------------------------------------
+
+def test_parallel_both_success_returns_first_in_order():
+ """Both instances succeed → return the first one in user-supplied order."""
+ router = _url_router({
+ "a.example.com": _mock_urlopen(_payload("from-a", "https://a.com")),
+ "b.example.com": _mock_urlopen(_payload("from-b", "https://b.com")),
+ })
+ with patch("urllib.request.urlopen", side_effect=router):
+ r = search_multi(
+ ["https://a.example.com", "https://b.example.com"],
+ {"q": "test", "format": "json"},
+ parallel=True, retry_per=0,
+ )
+ assert r["results"][0]["title"] == "from-a"
+
+
+def test_parallel_first_fails_second_succeeds():
+ """First instance fails, second succeeds → return second's results."""
+ router = _url_router({
+ "a.example.com": urllib.error.URLError("connection refused"),
+ "b.example.com": _mock_urlopen(_payload("from-b", "https://b.com")),
+ })
+ with patch("urllib.request.urlopen", side_effect=router):
+ r = search_multi(
+ ["https://a.example.com", "https://b.example.com"],
+ {"q": "test", "format": "json"},
+ parallel=True, retry_per=0,
+ )
+ assert r["results"][0]["title"] == "from-b"
+
+
+def test_parallel_all_fail_raises():
+ """All instances fail → RuntimeError carrying the '(parallel)' marker."""
+ err = urllib.error.URLError("down")
+ with patch("urllib.request.urlopen", side_effect=_url_router(default=err)):
+ try:
+ search_multi(
+ ["https://a.example.com", "https://b.example.com"],
+ {"q": "test", "format": "json"},
+ parallel=True, retry_per=0,
+ )
+ assert False, "should have raised RuntimeError"
+ except RuntimeError as e:
+ assert "(parallel)" in str(e)
+ assert "2" in str(e)
+
+
+def test_parallel_three_middle_succeeds():
+ """Three instances, only the middle one succeeds → return middle."""
+ router = _url_router({
+ "a.example.com": urllib.error.URLError("down"),
+ "b.example.com": _mock_urlopen(_payload("middle-b", "https://b.com")),
+ "c.example.com": urllib.error.URLError("down"),
+ })
+ with patch("urllib.request.urlopen", side_effect=router):
+ r = search_multi(
+ ["https://a.example.com", "https://b.example.com",
+ "https://c.example.com"],
+ {"q": "test", "format": "json"},
+ parallel=True, retry_per=0,
+ )
+ assert r["results"][0]["title"] == "middle-b"
+
+
+def test_parallel_returns_user_order_not_completion_order():
+ """Result follows user input order, NOT completion order.
+
+ Instance 'a' is deliberately slowed down so 'b' finishes first, yet
+ 'a' (listed first) must still be the returned result when both
+ succeed. This guards the ``for u in instance_urls`` deterministic
+ selection at the end of the parallel branch.
+ """
+ resp_a = _mock_urlopen(_payload("slow-a", "https://a.com"))
+ resp_b = _mock_urlopen(_payload("fast-b", "https://b.com"))
+
+ def _side_effect(req, *args, **kwargs):
+ url = getattr(req, "full_url", str(req))
+ if "a.example.com" in url:
+ time.sleep(0.15) # 'a' finishes after 'b'
+ return resp_a
+ if "b.example.com" in url:
+ return resp_b
+ raise AssertionError(f"unexpected urlopen for {url!r}")
+
+ with patch("urllib.request.urlopen", side_effect=_side_effect):
+ r = search_multi(
+ ["https://a.example.com", "https://b.example.com"],
+ {"q": "test", "format": "json"},
+ parallel=True, retry_per=0,
+ )
+ assert r["results"][0]["title"] == "slow-a"
+
+
+# ------------------------------------------------------------------
+# parallel=True: serial-path fallback conditions
+# ------------------------------------------------------------------
+
+def test_parallel_single_instance_uses_serial_path():
+ """A single instance (len <= 1) takes the serial path even with
+ parallel=True. Distinguishable by error message: serial path says
+ 'Last error', parallel path says '(parallel)'.
+ """
+ err = urllib.error.URLError("down")
+ with patch("urllib.request.urlopen", side_effect=err):
+ try:
+ search_multi(
+ ["https://a.example.com"],
+ {"q": "test", "format": "json"},
+ parallel=True, retry_per=0,
+ )
+ assert False, "should have raised RuntimeError"
+ except RuntimeError as e:
+ assert "Last error" in str(e)
+ assert "(parallel)" not in str(e)
+
+
+def test_parallel_false_explicit_serial_path():
+ """parallel=False explicitly forces the serial path for >1 instances."""
+ err = urllib.error.URLError("down")
+ with patch("urllib.request.urlopen", side_effect=err):
+ try:
+ search_multi(
+ ["https://a.example.com", "https://b.example.com"],
+ {"q": "test", "format": "json"},
+ parallel=False, retry_per=0,
+ )
+ assert False, "should have raised RuntimeError"
+ except RuntimeError as e:
+ assert "Last error" in str(e)
+ assert "(parallel)" not in str(e)
+
+
+# ------------------------------------------------------------------
+# parallel=True: header forwarding & retry semantics
+# ------------------------------------------------------------------
+
+def test_parallel_auth_headers_passed():
+ """auth_headers are forwarded to every instance request."""
+ captured = [] # list.append is atomic under CPython's GIL
+ payload = _payload("ok", "https://x.com")
+
+ def _side_effect(req, *args, **kwargs):
+ captured.append(req.headers)
+ return _mock_urlopen(payload)
+
+ with patch("urllib.request.urlopen", side_effect=_side_effect):
+ r = search_multi(
+ ["https://a.example.com", "https://b.example.com"],
+ {"q": "test", "format": "json"},
+ parallel=True, retry_per=0,
+ auth_headers={"Authorization": "Bearer secret-token"},
+ )
+ assert r["results"][0]["title"] == "ok"
+ # Both instances were called exactly once (success path, no HTML fallback)
+ assert len(captured) == 2
+ for hdrs in captured:
+ assert hdrs.get("Authorization") == "Bearer secret-token"
+
+
+def test_parallel_retry_per_zero_no_retry():
+ """retry_per=0 → exactly one attempt per instance, no backoff retries.
+
+ Three failing instances must produce exactly 3 urlopen calls total.
+ MagicMock.call_count is thread-safe, so concurrent increments are
+ observed correctly after the executor joins.
+ """
+ err = urllib.error.URLError("down")
+ mock_open = MagicMock(side_effect=err)
+ with patch("urllib.request.urlopen", mock_open):
+ try:
+ search_multi(
+ ["https://a.example.com", "https://b.example.com",
+ "https://c.example.com"],
+ {"q": "test", "format": "json"},
+ parallel=True, retry_per=0,
+ )
+ except RuntimeError:
+ pass # expected: all failed
+ # 3 instances × 1 attempt (retry_per=0) = 3 calls, no retries
+ assert mock_open.call_count == 3
+
+
+def test_parallel_exception_caught_and_logged(caplog):
+ """A raising instance is caught and logged; the survivor's result wins.
+
+ Instance 'a' raises HTTPError 403 (not retryable, re-raised by
+ _retry_with_backoff). The parallel _task wrapper must catch it,
+ record it as a failure, and let instance 'b' succeed.
+ """
+ router = _url_router({
+ "a.example.com": urllib.error.HTTPError(
+ "url", 403, "Forbidden", {}, None),
+ "b.example.com": _mock_urlopen(_payload("survivor-b", "https://b.com")),
+ })
+ with patch("urllib.request.urlopen", side_effect=router), \
+ caplog.at_level(logging.INFO, logger="searxng.search"):
+ r = search_multi(
+ ["https://a.example.com", "https://b.example.com"],
+ {"q": "test", "format": "json"},
+ parallel=True, retry_per=0,
+ )
+ assert r["results"][0]["title"] == "survivor-b"
+ # The failure was logged (line: logger.info(f" Failed {u}: {res}"))
+ assert any(
+ "Failed" in rec.message and "a.example.com" in rec.message
+ for rec in caplog.records
+ ), f"expected a 'Failed ... a.example.com' log line, got: {[r.message for r in caplog.records]}"
diff --git a/tests/test_stream_progress.py b/tests/test_stream_progress.py
new file mode 100644
index 0000000..5975690
--- /dev/null
+++ b/tests/test_stream_progress.py
@@ -0,0 +1,207 @@
+"""Tests for --stream (JSON Lines output) and --progress (progress events).
+
+Covers: stream output format (result/done/error events), progress event
+emission (start/cache_hit/cache_store/done/error/fetch_*), progress
+disabled by default, stream only works with --format json.
+"""
+import json
+import sys
+from types import SimpleNamespace
+from unittest.mock import patch
+import pytest
+
+import common
+from common import emit_progress, set_progress_enabled
+import search as search_mod
+from search import _run_single_query, main
+
+
+# ----- emit_progress: default disabled -----
+
+def test_progress_disabled_by_default(capsys):
+ """Without --progress, emit_progress is a no-op."""
+ set_progress_enabled(False)
+ emit_progress("start", query="test", instances=1)
+ out, err = capsys.readouterr()
+ assert out == ""
+ assert err == ""
+
+
+def test_progress_enabled_emits_json(capsys):
+ """With --progress, emit_progress outputs JSON Lines to stderr."""
+ set_progress_enabled(True)
+ try:
+ emit_progress("start", query="test", instances=2)
+ _, err = capsys.readouterr()
+ data = json.loads(err.strip())
+ assert data["event"] == "start"
+ assert data["query"] == "test"
+ assert data["instances"] == 2
+ finally:
+ set_progress_enabled(False)
+
+
+def test_progress_multiple_events(capsys):
+ """Multiple events produce multiple JSON Lines."""
+ set_progress_enabled(True)
+ try:
+ emit_progress("start", query="q", instances=1)
+ emit_progress("cache_hit", query="q", ttl=30)
+ emit_progress("done", results=5, query="q")
+ _, err = capsys.readouterr()
+ lines = [l for l in err.strip().split("\n") if l]
+ assert len(lines) == 3
+ events = [json.loads(l)["event"] for l in lines]
+ assert events == ["start", "cache_hit", "done"]
+ finally:
+ set_progress_enabled(False)
+
+
+def test_progress_event_with_error_code(capsys):
+ """Error events include error_code field."""
+ set_progress_enabled(True)
+ try:
+ emit_progress("error", error="timeout", error_code="E_NETWORK",
+ query="test")
+ _, err = capsys.readouterr()
+ data = json.loads(err.strip())
+ assert data["error_code"] == "E_NETWORK"
+ finally:
+ set_progress_enabled(False)
+
+
+# ----- _run_single_query: progress events -----
+
+def _make_args(**overrides):
+ """Construct a minimal args object for _run_single_query."""
+ defaults = dict(
+ query="test", format="json", method="GET", timeout=15, retry=0,
+ serial=False, no_dedup=False, sort_by="none", max_results=None,
+ include_domain=None, exclude_domain=None, fetch=0, fetch_timeout=10,
+ fetch_retries=3, max_size=None, cache_ttl=0, snippet_len=0,
+ categories=None, language=None, pageno=1, time_range="year",
+ safesearch=0, engines="google,bing",
+ )
+ defaults.update(overrides)
+ return SimpleNamespace(**defaults)
+
+
+def test_run_single_query_emits_start_and_done(capsys):
+ """_run_single_query emits start and done events when progress is enabled."""
+ set_progress_enabled(True)
+ try:
+ mock_results = {"results": [{"title": "t", "url": "https://x.com"}]}
+ with patch.object(search_mod, "search_multi", return_value=mock_results):
+ _run_single_query("test", _make_args(),
+ ["https://x.example.com"], {}, 0)
+ _, err = capsys.readouterr()
+ lines = [json.loads(l) for l in err.strip().split("\n") if l]
+ events = [e["event"] for e in lines]
+ assert "start" in events
+ assert "done" in events
+ done_event = next(e for e in lines if e["event"] == "done")
+ assert done_event["results"] == 1
+ finally:
+ set_progress_enabled(False)
+
+
+def test_run_single_query_emits_cache_hit(capsys, isolated_cache):
+ """Cache hit emits cache_hit event."""
+ set_progress_enabled(True)
+ try:
+ mock_results = {"results": [{"title": "t", "url": "https://x.com"}]}
+ with patch.object(search_mod, "search_multi", return_value=mock_results):
+ # First call: cache miss, stores result
+ _run_single_query("test", _make_args(cache_ttl=30),
+ ["https://x.example.com"], {}, 1800)
+ capsys.readouterr() # clear
+ # Second call: cache hit
+ _run_single_query("test", _make_args(cache_ttl=30),
+ ["https://x.example.com"], {}, 1800)
+ _, err = capsys.readouterr()
+ lines = [json.loads(l) for l in err.strip().split("\n") if l]
+ events = [e["event"] for e in lines]
+ assert "cache_hit" in events
+ finally:
+ set_progress_enabled(False)
+
+
+def test_run_single_query_emits_error_on_failure(capsys):
+ """Search failure emits error event with error_code."""
+ set_progress_enabled(True)
+ try:
+ import urllib.error
+ err = urllib.error.HTTPError("url", 403, "Forbidden", {}, None)
+ with patch.object(search_mod, "search_multi", side_effect=err):
+ _run_single_query("test", _make_args(),
+ ["https://x.example.com"], {}, 0)
+ _, err_out = capsys.readouterr()
+ lines = [json.loads(l) for l in err_out.strip().split("\n") if l]
+ error_events = [e for e in lines if e["event"] == "error"]
+ assert len(error_events) == 1
+ assert error_events[0]["error_code"] == "E_AUTH"
+ finally:
+ set_progress_enabled(False)
+
+
+# ----- --stream: JSON Lines output -----
+
+def test_stream_outputs_json_lines(capsys):
+ """--stream outputs each result as a JSON Line + a done event."""
+ mock_results = {"results": [
+ {"title": "first", "url": "https://a.com"},
+ {"title": "second", "url": "https://b.com"},
+ ]}
+ with patch.object(search_mod, "search_multi", return_value=mock_results):
+ with pytest.raises(SystemExit) as exc_info:
+ main.__wrapped__ if hasattr(main, "__wrapped__") else None
+ # Call main with --stream
+ with patch.object(sys, "argv", ["search.py", "-q", "test",
+ "-i", "https://x.example.com",
+ "--stream", "--format", "json"]):
+ # Mock the early argparse for logging
+ with patch.object(search_mod, "setup_logging"):
+ main()
+ assert exc_info.value.code == 0
+ out, _ = capsys.readouterr()
+ lines = [json.loads(l) for l in out.strip().split("\n") if l]
+ # Should have 2 result events + 1 done event
+ result_events = [e for e in lines if e["type"] == "result"]
+ done_events = [e for e in lines if e["type"] == "done"]
+ assert len(result_events) == 2
+ assert len(done_events) == 1
+ assert done_events[0]["count"] == 2
+ assert result_events[0]["result"]["title"] == "first"
+
+
+def test_stream_empty_results_exit_2(capsys):
+ """--stream with empty results exits with code 2 and emits done with count=0."""
+ mock_results = {"results": []}
+ with patch.object(search_mod, "search_multi", return_value=mock_results):
+ with pytest.raises(SystemExit) as exc_info:
+ with patch.object(sys, "argv", ["search.py", "-q", "test",
+ "-i", "https://x.example.com",
+ "--stream", "--format", "json"]):
+ with patch.object(search_mod, "setup_logging"):
+ main()
+ assert exc_info.value.code == 2
+ out, _ = capsys.readouterr()
+ lines = [json.loads(l) for l in out.strip().split("\n") if l]
+ done_events = [e for e in lines if e["type"] == "done"]
+ assert len(done_events) == 1
+ assert done_events[0]["count"] == 0
+
+
+def test_stream_result_event_shape():
+ """Each result event has type=result and result=
."""
+ # Unit test the stream output logic directly
+ results = [{"title": "t", "url": "https://x.com"}]
+ lines = []
+ for r in results:
+ lines.append(json.dumps({"type": "result", "result": r}))
+ lines.append(json.dumps({"type": "done", "count": len(results)}))
+ parsed = [json.loads(l) for l in lines]
+ assert parsed[0]["type"] == "result"
+ assert parsed[0]["result"]["url"] == "https://x.com"
+ assert parsed[-1]["type"] == "done"
+ assert parsed[-1]["count"] == 1
diff --git a/tests/test_verify.py b/tests/test_verify.py
new file mode 100644
index 0000000..703b91f
--- /dev/null
+++ b/tests/test_verify.py
@@ -0,0 +1,214 @@
+"""Tests for the --verify instance health-check feature.
+
+Covers: _probe_config_endpoint (reachable/disabled/error), verify_instances
+(reachability, JSON support, POST probe, latency, auth status, error
+classification), _print_verify_report (JSON and table output).
+"""
+import json
+import urllib.error
+from unittest.mock import patch, MagicMock
+
+from search import _probe_config_endpoint, verify_instances, _print_verify_report
+
+
+def _mock_urlopen(data: bytes, content_type="application/json"):
+ """Build a MagicMock that quacks like an urlopen context manager."""
+ resp = MagicMock()
+ resp.read.return_value = data
+ resp.headers = {"Content-Type": content_type}
+ resp.__enter__.return_value = resp
+ resp.__exit__.return_value = None
+ return resp
+
+
+# ----- _probe_config_endpoint -----
+
+def test_probe_config_reachable():
+ """A working /config endpoint returns engines and categories."""
+ payload = json.dumps({
+ "engines": [{"name": "google"}, {"name": "bing"}, {"name": "wikipedia"}],
+ "categories": {"general": [], "news": [], "images": []},
+ })
+ with patch("urllib.request.urlopen",
+ return_value=_mock_urlopen(payload.encode())):
+ r = _probe_config_endpoint("https://s.example.com", timeout=10)
+ assert r["reachable"] is True
+ assert r["engines"] == ["google", "bing", "wikipedia"]
+ assert set(r["categories"]) == {"general", "news", "images"}
+ assert r["error"] is None
+
+
+def test_probe_config_categories_as_list():
+ """Some instances return categories as a list instead of a dict."""
+ payload = json.dumps({
+ "engines": [{"name": "google"}],
+ "categories": ["general", "news"],
+ })
+ with patch("urllib.request.urlopen",
+ return_value=_mock_urlopen(payload.encode())):
+ r = _probe_config_endpoint("https://s.example.com", timeout=10)
+ assert r["reachable"] is True
+ assert r["categories"] == ["general", "news"]
+
+
+def test_probe_config_http_error():
+ """/config returns 403 → reachable=False with HTTP code in error."""
+ err = urllib.error.HTTPError("url", 403, "Forbidden", {}, None)
+ with patch("urllib.request.urlopen", side_effect=err):
+ r = _probe_config_endpoint("https://s.example.com", timeout=10)
+ assert r["reachable"] is False
+ assert "403" in r["error"]
+ assert r["engines"] == []
+
+
+def test_probe_config_connection_error():
+ """Connection error → reachable=False with error message."""
+ err = urllib.error.URLError("connection refused")
+ with patch("urllib.request.urlopen", side_effect=err):
+ r = _probe_config_endpoint("https://s.example.com", timeout=10)
+ assert r["reachable"] is False
+ assert r["engines"] == []
+ assert "connection refused" in r["error"]
+
+
+# ----- verify_instances -----
+
+def _mock_json_search_response(results=None):
+ """Mock a successful JSON search response."""
+ payload = json.dumps({"results": results or [{"title": "t", "url": "https://x.com"}]})
+ return _mock_urlopen(payload.encode(), content_type="application/json")
+
+
+def _mock_html_search_response():
+ """Mock an HTML search response (JSON disabled)."""
+ return _mock_urlopen(b"not json", content_type="text/html")
+
+
+def test_verify_reachable_json_supported():
+ """Instance returns JSON → reachable + json_supported + post probe."""
+ with patch("urllib.request.urlopen",
+ return_value=_mock_json_search_response()):
+ # /config also needs to respond
+ config_payload = json.dumps({"engines": [{"name": "google"}], "categories": {}})
+ config_resp = _mock_urlopen(config_payload.encode())
+ with patch("urllib.request.urlopen",
+ return_value=_mock_json_search_response()):
+ report = verify_instances(["https://s.example.com"])
+ r = report[0]
+ assert r["url"] == "https://s.example.com"
+ assert r["reachable"] is True
+ assert r["json_supported"] is True
+
+
+def test_verify_html_only_instance():
+ """Instance returns HTML (no JSON) → reachable but json_supported=False."""
+ with patch("urllib.request.urlopen",
+ return_value=_mock_html_search_response()):
+ report = verify_instances(["https://s.example.com"])
+ r = report[0]
+ assert r["reachable"] is True
+ assert r["json_supported"] is False
+
+
+def test_verify_auth_rejected():
+ """401/403 with auth → auth_status='rejected'."""
+ err = urllib.error.HTTPError("url", 403, "Forbidden", {}, None)
+ auth_headers = {"Authorization": "Bearer token123"}
+ with patch("urllib.request.urlopen", side_effect=err):
+ report = verify_instances(["https://s.example.com"],
+ auth_headers=auth_headers)
+ r = report[0]
+ assert r["auth_status"] == "rejected"
+
+
+def test_verify_no_auth_returns_na():
+ """Without auth headers, auth_status is 'n/a'."""
+ err = urllib.error.URLError("timeout")
+ with patch("urllib.request.urlopen", side_effect=err):
+ report = verify_instances(["https://s.example.com"])
+ r = report[0]
+ assert r["auth_status"] == "n/a"
+
+
+def test_verify_connection_error():
+ """Connection error → reachable=False, latency=None (no response received)."""
+ err = urllib.error.URLError("connection refused")
+ with patch("urllib.request.urlopen", side_effect=err):
+ report = verify_instances(["https://s.example.com"])
+ r = report[0]
+ assert r["reachable"] is False
+ assert r["json_supported"] is False
+ assert r["latency"] is None # no response → no latency
+ assert "connection refused" in r["error"]
+
+
+def test_verify_preserves_input_order():
+ """Multiple instances: report preserves the original input order."""
+ urls = ["https://a.example.com", "https://b.example.com", "https://c.example.com"]
+ with patch("urllib.request.urlopen",
+ return_value=_mock_json_search_response()):
+ report = verify_instances(urls)
+ assert [r["url"] for r in report] == urls
+
+
+def test_verify_has_latency():
+ """Latency is a positive float for reachable instances."""
+ with patch("urllib.request.urlopen",
+ return_value=_mock_json_search_response()):
+ report = verify_instances(["https://s.example.com"])
+ assert report[0]["latency"] is not None
+ assert report[0]["latency"] >= 0
+
+
+def test_verify_has_result_count():
+ """Result count from the test query is recorded."""
+ with patch("urllib.request.urlopen",
+ return_value=_mock_json_search_response(
+ [{"title": "a"}, {"title": "b"}, {"title": "c"}])):
+ report = verify_instances(["https://s.example.com"])
+ assert report[0]["result_count"] == 3
+
+
+# ----- _print_verify_report -----
+
+def test_print_report_json(capsys):
+ """JSON output is valid JSON with the full report."""
+ report = [
+ {"url": "https://a.com", "reachable": True, "json_supported": True,
+ "post_supported": True, "latency": 0.5, "result_count": 10,
+ "engines": ["google"], "auth_status": "n/a", "error": None,
+ "config_endpoint": True},
+ ]
+ _print_verify_report(report, as_json=True)
+ out, _ = capsys.readouterr()
+ data = json.loads(out)
+ assert data[0]["url"] == "https://a.com"
+ assert data[0]["reachable"] is True
+
+
+def test_print_report_table(capsys):
+ """Table output includes the header and a summary line."""
+ report = [
+ {"url": "https://a.com", "reachable": True, "json_supported": True,
+ "post_supported": True, "latency": 0.5, "result_count": 10,
+ "engines": ["google", "bing"], "auth_status": "n/a", "error": None,
+ "config_endpoint": True},
+ {"url": "https://b.com", "reachable": False, "json_supported": False,
+ "post_supported": None, "latency": None, "result_count": None,
+ "engines": [], "auth_status": "n/a", "error": "timeout",
+ "config_endpoint": None},
+ ]
+ _print_verify_report(report, as_json=False)
+ out, _ = capsys.readouterr()
+ assert "URL" in out
+ assert "REACH" in out
+ assert "https://a.com" in out
+ assert "https://b.com" in out
+ assert "1/2 instances reachable" in out
+
+
+def test_print_report_empty(capsys):
+ """Empty report produces a table with 0/0 summary."""
+ _print_verify_report([], as_json=False)
+ out, _ = capsys.readouterr()
+ assert "0/0 instances reachable" in out