"""Parallel-mode tests for search_multi (parallel=True branch). The serial path (parallel=False / single instance) is covered by ``test_integration.py``. This module focuses on the ThreadPoolExecutor branch: result ordering by user input, failover, all-fail, auth-header forwarding, retry_per=0 semantics, and exception capture. Thread-safety note: ``urllib.request.urlopen`` is patched with a URL-dispatching *function* (not a ``side_effect`` list). A function with no mutable shared state is safe to call concurrently from multiple worker threads, whereas a list-based ``side_effect`` would race on the shared iterator. ``MagicMock.call_count`` is itself thread-safe. """ import json import logging import time import urllib.error from unittest.mock import MagicMock, patch from search import search_multi def _mock_urlopen(data: bytes, content_type="application/json"): """Build a MagicMock that quacks like an urlopen context manager.""" resp = MagicMock() resp.read.return_value = data resp.headers = {"Content-Type": content_type} resp.__enter__.return_value = resp resp.__exit__.return_value = None return resp def _url_router(routing=None, default=None): """Build a thread-safe side_effect that dispatches by request URL. ``routing`` maps a URL substring to either a response object (returned) or a ``BaseException`` (raised). Entries are checked in insertion order (dicts are ordered on Python 3.7+). ``default`` is used when no key matches; if it is an exception it is raised, otherwise returned. An ``AssertionError`` is raised when nothing matches and no default is set — this makes unexpected calls loud rather than silently returning a MagicMock. """ routing = routing or {} def _side_effect(req, *args, **kwargs): url = getattr(req, "full_url", str(req)) for key, resp in routing.items(): if key in url: if isinstance(resp, BaseException): raise resp return resp if default is not None: if isinstance(default, BaseException): raise default return default raise AssertionError( f"unexpected urlopen for {url!r} (routing={list(routing)})" ) return _side_effect def _payload(title: str, url: str = "https://x.com") -> bytes: return json.dumps({"results": [{"title": title, "url": url}]}).encode() # ------------------------------------------------------------------ # parallel=True: success paths # ------------------------------------------------------------------ def test_parallel_both_success_returns_first_in_order(): """Both instances succeed → return the first one in user-supplied order.""" router = _url_router({ "a.example.com": _mock_urlopen(_payload("from-a", "https://a.com")), "b.example.com": _mock_urlopen(_payload("from-b", "https://b.com")), }) with patch("urllib.request.urlopen", side_effect=router): r = search_multi( ["https://a.example.com", "https://b.example.com"], {"q": "test", "format": "json"}, parallel=True, retry_per=0, ) assert r["results"][0]["title"] == "from-a" def test_parallel_first_fails_second_succeeds(): """First instance fails, second succeeds → return second's results.""" router = _url_router({ "a.example.com": urllib.error.URLError("connection refused"), "b.example.com": _mock_urlopen(_payload("from-b", "https://b.com")), }) with patch("urllib.request.urlopen", side_effect=router): r = search_multi( ["https://a.example.com", "https://b.example.com"], {"q": "test", "format": "json"}, parallel=True, retry_per=0, ) assert r["results"][0]["title"] == "from-b" def test_parallel_all_fail_raises(): """All instances fail → RuntimeError carrying the '(parallel)' marker.""" err = urllib.error.URLError("down") with patch("urllib.request.urlopen", side_effect=_url_router(default=err)): try: search_multi( ["https://a.example.com", "https://b.example.com"], {"q": "test", "format": "json"}, parallel=True, retry_per=0, ) assert False, "should have raised RuntimeError" except RuntimeError as e: assert "(parallel)" in str(e) assert "2" in str(e) def test_parallel_three_middle_succeeds(): """Three instances, only the middle one succeeds → return middle.""" router = _url_router({ "a.example.com": urllib.error.URLError("down"), "b.example.com": _mock_urlopen(_payload("middle-b", "https://b.com")), "c.example.com": urllib.error.URLError("down"), }) with patch("urllib.request.urlopen", side_effect=router): r = search_multi( ["https://a.example.com", "https://b.example.com", "https://c.example.com"], {"q": "test", "format": "json"}, parallel=True, retry_per=0, ) assert r["results"][0]["title"] == "middle-b" def test_parallel_returns_user_order_not_completion_order(): """Result follows user input order, NOT completion order. Instance 'a' is deliberately slowed down so 'b' finishes first, yet 'a' (listed first) must still be the returned result when both succeed. This guards the ``for u in instance_urls`` deterministic selection at the end of the parallel branch. """ resp_a = _mock_urlopen(_payload("slow-a", "https://a.com")) resp_b = _mock_urlopen(_payload("fast-b", "https://b.com")) def _side_effect(req, *args, **kwargs): url = getattr(req, "full_url", str(req)) if "a.example.com" in url: time.sleep(0.15) # 'a' finishes after 'b' return resp_a if "b.example.com" in url: return resp_b raise AssertionError(f"unexpected urlopen for {url!r}") with patch("urllib.request.urlopen", side_effect=_side_effect): r = search_multi( ["https://a.example.com", "https://b.example.com"], {"q": "test", "format": "json"}, parallel=True, retry_per=0, ) assert r["results"][0]["title"] == "slow-a" # ------------------------------------------------------------------ # parallel=True: serial-path fallback conditions # ------------------------------------------------------------------ def test_parallel_single_instance_uses_serial_path(): """A single instance (len <= 1) takes the serial path even with parallel=True. Distinguishable by error message: serial path says 'Last error', parallel path says '(parallel)'. """ err = urllib.error.URLError("down") with patch("urllib.request.urlopen", side_effect=err): try: search_multi( ["https://a.example.com"], {"q": "test", "format": "json"}, parallel=True, retry_per=0, ) assert False, "should have raised RuntimeError" except RuntimeError as e: assert "Last error" in str(e) assert "(parallel)" not in str(e) def test_parallel_false_explicit_serial_path(): """parallel=False explicitly forces the serial path for >1 instances.""" err = urllib.error.URLError("down") with patch("urllib.request.urlopen", side_effect=err): try: search_multi( ["https://a.example.com", "https://b.example.com"], {"q": "test", "format": "json"}, parallel=False, retry_per=0, ) assert False, "should have raised RuntimeError" except RuntimeError as e: assert "Last error" in str(e) assert "(parallel)" not in str(e) # ------------------------------------------------------------------ # parallel=True: header forwarding & retry semantics # ------------------------------------------------------------------ def test_parallel_auth_headers_passed(): """auth_headers are forwarded to every instance request.""" captured = [] # list.append is atomic under CPython's GIL payload = _payload("ok", "https://x.com") def _side_effect(req, *args, **kwargs): captured.append(req.headers) return _mock_urlopen(payload) with patch("urllib.request.urlopen", side_effect=_side_effect): r = search_multi( ["https://a.example.com", "https://b.example.com"], {"q": "test", "format": "json"}, parallel=True, retry_per=0, auth_headers={"Authorization": "Bearer secret-token"}, ) assert r["results"][0]["title"] == "ok" # Both instances were called exactly once (success path, no HTML fallback) assert len(captured) == 2 for hdrs in captured: assert hdrs.get("Authorization") == "Bearer secret-token" def test_parallel_retry_per_zero_no_retry(): """retry_per=0 → exactly one attempt per instance, no backoff retries. Three failing instances must produce exactly 3 urlopen calls total. MagicMock.call_count is thread-safe, so concurrent increments are observed correctly after the executor joins. """ err = urllib.error.URLError("down") mock_open = MagicMock(side_effect=err) with patch("urllib.request.urlopen", mock_open): try: search_multi( ["https://a.example.com", "https://b.example.com", "https://c.example.com"], {"q": "test", "format": "json"}, parallel=True, retry_per=0, ) except RuntimeError: pass # expected: all failed # 3 instances × 1 attempt (retry_per=0) = 3 calls, no retries assert mock_open.call_count == 3 def test_parallel_exception_caught_and_logged(caplog): """A raising instance is caught and logged; the survivor's result wins. Instance 'a' raises HTTPError 403 (not retryable, re-raised by _retry_with_backoff). The parallel _task wrapper must catch it, record it as a failure, and let instance 'b' succeed. """ router = _url_router({ "a.example.com": urllib.error.HTTPError( "url", 403, "Forbidden", {}, None), "b.example.com": _mock_urlopen(_payload("survivor-b", "https://b.com")), }) with patch("urllib.request.urlopen", side_effect=router), \ caplog.at_level(logging.INFO, logger="searxng.search"): r = search_multi( ["https://a.example.com", "https://b.example.com"], {"q": "test", "format": "json"}, parallel=True, retry_per=0, ) assert r["results"][0]["title"] == "survivor-b" # The failure was logged (line: logger.info(f" Failed {u}: {res}")) assert any( "Failed" in rec.message and "a.example.com" in rec.message for rec in caplog.records ), f"expected a 'Failed ... a.example.com' log line, got: {[r.message for r in caplog.records]}"