thzxx
f983a9377e
核心新增（面向 AI Agent 程序化使用）： - 结构化错误码体系：E_CONFIG/E_AUTH/E_NETWORK/E_RATE_LIMIT/E_PARSE/E_EMPTY/E_INPUT/E_INTERNAL classify_error() 自动分类异常，JSON 错误输出含 error_code 字段 - JSON Lines 流式输出 (--stream)：每条结果独立一行，AI 可增量处理 - 进度事件 (--progress)：JSON Lines 事件流到 stderr（start/cache_hit/fetch_ok/done 等） 测试补全（+154 例，覆盖全部高风险盲区）： - HTML 回退搜索路径 (19) - --fetch 自动抓取 (21) - --verify 健康检查 (15) - 输出格式化 (15) - 实例解析链 (20) - 并行多实例搜索 (10) - CLI 入口与端到端 (17) - 错误码分类 (27) - 流式输出与进度事件 (10) 源码改进： - search.py: h3 内 a 标签 href 作为 url fallback，提升 SearXNG 主题兼容性 - common.py: 新增 classify_error/emit_progress/set_progress_enabled 文档同步：SKILL.md 新增 AI Agent Integration Guide 章节，README.md 更新参数与错误码表
277 lines
11 KiB
Python
277 lines
11 KiB
Python
"""Parallel-mode tests for search_multi (parallel=True branch).
|
||
|
||
The serial path (parallel=False / single instance) is covered by
|
||
``test_integration.py``. This module focuses on the ThreadPoolExecutor
|
||
branch: result ordering by user input, failover, all-fail, auth-header
|
||
forwarding, retry_per=0 semantics, and exception capture.
|
||
|
||
Thread-safety note: ``urllib.request.urlopen`` is patched with a
|
||
URL-dispatching *function* (not a ``side_effect`` list). A function with
|
||
no mutable shared state is safe to call concurrently from multiple
|
||
worker threads, whereas a list-based ``side_effect`` would race on the
|
||
shared iterator. ``MagicMock.call_count`` is itself thread-safe.
|
||
"""
|
||
import json
|
||
import logging
|
||
import time
|
||
import urllib.error
|
||
from unittest.mock import MagicMock, patch
|
||
|
||
from search import search_multi
|
||
|
||
|
||
def _mock_urlopen(data: bytes, content_type="application/json"):
|
||
"""Build a MagicMock that quacks like an urlopen context manager."""
|
||
resp = MagicMock()
|
||
resp.read.return_value = data
|
||
resp.headers = {"Content-Type": content_type}
|
||
resp.__enter__.return_value = resp
|
||
resp.__exit__.return_value = None
|
||
return resp
|
||
|
||
|
||
def _url_router(routing=None, default=None):
|
||
"""Build a thread-safe side_effect that dispatches by request URL.
|
||
|
||
``routing`` maps a URL substring to either a response object
|
||
(returned) or a ``BaseException`` (raised). Entries are checked in
|
||
insertion order (dicts are ordered on Python 3.7+). ``default`` is
|
||
used when no key matches; if it is an exception it is raised,
|
||
otherwise returned. An ``AssertionError`` is raised when nothing
|
||
matches and no default is set — this makes unexpected calls loud
|
||
rather than silently returning a MagicMock.
|
||
"""
|
||
routing = routing or {}
|
||
|
||
def _side_effect(req, *args, **kwargs):
|
||
url = getattr(req, "full_url", str(req))
|
||
for key, resp in routing.items():
|
||
if key in url:
|
||
if isinstance(resp, BaseException):
|
||
raise resp
|
||
return resp
|
||
if default is not None:
|
||
if isinstance(default, BaseException):
|
||
raise default
|
||
return default
|
||
raise AssertionError(
|
||
f"unexpected urlopen for {url!r} (routing={list(routing)})"
|
||
)
|
||
return _side_effect
|
||
|
||
|
||
def _payload(title: str, url: str = "https://x.com") -> bytes:
|
||
return json.dumps({"results": [{"title": title, "url": url}]}).encode()
|
||
|
||
|
||
# ------------------------------------------------------------------
|
||
# parallel=True: success paths
|
||
# ------------------------------------------------------------------
|
||
|
||
def test_parallel_both_success_returns_first_in_order():
|
||
"""Both instances succeed → return the first one in user-supplied order."""
|
||
router = _url_router({
|
||
"a.example.com": _mock_urlopen(_payload("from-a", "https://a.com")),
|
||
"b.example.com": _mock_urlopen(_payload("from-b", "https://b.com")),
|
||
})
|
||
with patch("urllib.request.urlopen", side_effect=router):
|
||
r = search_multi(
|
||
["https://a.example.com", "https://b.example.com"],
|
||
{"q": "test", "format": "json"},
|
||
parallel=True, retry_per=0,
|
||
)
|
||
assert r["results"][0]["title"] == "from-a"
|
||
|
||
|
||
def test_parallel_first_fails_second_succeeds():
|
||
"""First instance fails, second succeeds → return second's results."""
|
||
router = _url_router({
|
||
"a.example.com": urllib.error.URLError("connection refused"),
|
||
"b.example.com": _mock_urlopen(_payload("from-b", "https://b.com")),
|
||
})
|
||
with patch("urllib.request.urlopen", side_effect=router):
|
||
r = search_multi(
|
||
["https://a.example.com", "https://b.example.com"],
|
||
{"q": "test", "format": "json"},
|
||
parallel=True, retry_per=0,
|
||
)
|
||
assert r["results"][0]["title"] == "from-b"
|
||
|
||
|
||
def test_parallel_all_fail_raises():
|
||
"""All instances fail → RuntimeError carrying the '(parallel)' marker."""
|
||
err = urllib.error.URLError("down")
|
||
with patch("urllib.request.urlopen", side_effect=_url_router(default=err)):
|
||
try:
|
||
search_multi(
|
||
["https://a.example.com", "https://b.example.com"],
|
||
{"q": "test", "format": "json"},
|
||
parallel=True, retry_per=0,
|
||
)
|
||
assert False, "should have raised RuntimeError"
|
||
except RuntimeError as e:
|
||
assert "(parallel)" in str(e)
|
||
assert "2" in str(e)
|
||
|
||
|
||
def test_parallel_three_middle_succeeds():
|
||
"""Three instances, only the middle one succeeds → return middle."""
|
||
router = _url_router({
|
||
"a.example.com": urllib.error.URLError("down"),
|
||
"b.example.com": _mock_urlopen(_payload("middle-b", "https://b.com")),
|
||
"c.example.com": urllib.error.URLError("down"),
|
||
})
|
||
with patch("urllib.request.urlopen", side_effect=router):
|
||
r = search_multi(
|
||
["https://a.example.com", "https://b.example.com",
|
||
"https://c.example.com"],
|
||
{"q": "test", "format": "json"},
|
||
parallel=True, retry_per=0,
|
||
)
|
||
assert r["results"][0]["title"] == "middle-b"
|
||
|
||
|
||
def test_parallel_returns_user_order_not_completion_order():
|
||
"""Result follows user input order, NOT completion order.
|
||
|
||
Instance 'a' is deliberately slowed down so 'b' finishes first, yet
|
||
'a' (listed first) must still be the returned result when both
|
||
succeed. This guards the ``for u in instance_urls`` deterministic
|
||
selection at the end of the parallel branch.
|
||
"""
|
||
resp_a = _mock_urlopen(_payload("slow-a", "https://a.com"))
|
||
resp_b = _mock_urlopen(_payload("fast-b", "https://b.com"))
|
||
|
||
def _side_effect(req, *args, **kwargs):
|
||
url = getattr(req, "full_url", str(req))
|
||
if "a.example.com" in url:
|
||
time.sleep(0.15) # 'a' finishes after 'b'
|
||
return resp_a
|
||
if "b.example.com" in url:
|
||
return resp_b
|
||
raise AssertionError(f"unexpected urlopen for {url!r}")
|
||
|
||
with patch("urllib.request.urlopen", side_effect=_side_effect):
|
||
r = search_multi(
|
||
["https://a.example.com", "https://b.example.com"],
|
||
{"q": "test", "format": "json"},
|
||
parallel=True, retry_per=0,
|
||
)
|
||
assert r["results"][0]["title"] == "slow-a"
|
||
|
||
|
||
# ------------------------------------------------------------------
|
||
# parallel=True: serial-path fallback conditions
|
||
# ------------------------------------------------------------------
|
||
|
||
def test_parallel_single_instance_uses_serial_path():
|
||
"""A single instance (len <= 1) takes the serial path even with
|
||
parallel=True. Distinguishable by error message: serial path says
|
||
'Last error', parallel path says '(parallel)'.
|
||
"""
|
||
err = urllib.error.URLError("down")
|
||
with patch("urllib.request.urlopen", side_effect=err):
|
||
try:
|
||
search_multi(
|
||
["https://a.example.com"],
|
||
{"q": "test", "format": "json"},
|
||
parallel=True, retry_per=0,
|
||
)
|
||
assert False, "should have raised RuntimeError"
|
||
except RuntimeError as e:
|
||
assert "Last error" in str(e)
|
||
assert "(parallel)" not in str(e)
|
||
|
||
|
||
def test_parallel_false_explicit_serial_path():
|
||
"""parallel=False explicitly forces the serial path for >1 instances."""
|
||
err = urllib.error.URLError("down")
|
||
with patch("urllib.request.urlopen", side_effect=err):
|
||
try:
|
||
search_multi(
|
||
["https://a.example.com", "https://b.example.com"],
|
||
{"q": "test", "format": "json"},
|
||
parallel=False, retry_per=0,
|
||
)
|
||
assert False, "should have raised RuntimeError"
|
||
except RuntimeError as e:
|
||
assert "Last error" in str(e)
|
||
assert "(parallel)" not in str(e)
|
||
|
||
|
||
# ------------------------------------------------------------------
|
||
# parallel=True: header forwarding & retry semantics
|
||
# ------------------------------------------------------------------
|
||
|
||
def test_parallel_auth_headers_passed():
|
||
"""auth_headers are forwarded to every instance request."""
|
||
captured = [] # list.append is atomic under CPython's GIL
|
||
payload = _payload("ok", "https://x.com")
|
||
|
||
def _side_effect(req, *args, **kwargs):
|
||
captured.append(req.headers)
|
||
return _mock_urlopen(payload)
|
||
|
||
with patch("urllib.request.urlopen", side_effect=_side_effect):
|
||
r = search_multi(
|
||
["https://a.example.com", "https://b.example.com"],
|
||
{"q": "test", "format": "json"},
|
||
parallel=True, retry_per=0,
|
||
auth_headers={"Authorization": "Bearer secret-token"},
|
||
)
|
||
assert r["results"][0]["title"] == "ok"
|
||
# Both instances were called exactly once (success path, no HTML fallback)
|
||
assert len(captured) == 2
|
||
for hdrs in captured:
|
||
assert hdrs.get("Authorization") == "Bearer secret-token"
|
||
|
||
|
||
def test_parallel_retry_per_zero_no_retry():
|
||
"""retry_per=0 → exactly one attempt per instance, no backoff retries.
|
||
|
||
Three failing instances must produce exactly 3 urlopen calls total.
|
||
MagicMock.call_count is thread-safe, so concurrent increments are
|
||
observed correctly after the executor joins.
|
||
"""
|
||
err = urllib.error.URLError("down")
|
||
mock_open = MagicMock(side_effect=err)
|
||
with patch("urllib.request.urlopen", mock_open):
|
||
try:
|
||
search_multi(
|
||
["https://a.example.com", "https://b.example.com",
|
||
"https://c.example.com"],
|
||
{"q": "test", "format": "json"},
|
||
parallel=True, retry_per=0,
|
||
)
|
||
except RuntimeError:
|
||
pass # expected: all failed
|
||
# 3 instances × 1 attempt (retry_per=0) = 3 calls, no retries
|
||
assert mock_open.call_count == 3
|
||
|
||
|
||
def test_parallel_exception_caught_and_logged(caplog):
|
||
"""A raising instance is caught and logged; the survivor's result wins.
|
||
|
||
Instance 'a' raises HTTPError 403 (not retryable, re-raised by
|
||
_retry_with_backoff). The parallel _task wrapper must catch it,
|
||
record it as a failure, and let instance 'b' succeed.
|
||
"""
|
||
router = _url_router({
|
||
"a.example.com": urllib.error.HTTPError(
|
||
"url", 403, "Forbidden", {}, None),
|
||
"b.example.com": _mock_urlopen(_payload("survivor-b", "https://b.com")),
|
||
})
|
||
with patch("urllib.request.urlopen", side_effect=router), \
|
||
caplog.at_level(logging.INFO, logger="searxng.search"):
|
||
r = search_multi(
|
||
["https://a.example.com", "https://b.example.com"],
|
||
{"q": "test", "format": "json"},
|
||
parallel=True, retry_per=0,
|
||
)
|
||
assert r["results"][0]["title"] == "survivor-b"
|
||
# The failure was logged (line: logger.info(f" Failed {u}: {res}"))
|
||
assert any(
|
||
"Failed" in rec.message and "a.example.com" in rec.message
|
||
for rec in caplog.records
|
||
), f"expected a 'Failed ... a.example.com' log line, got: {[r.message for r in caplog.records]}"
|