feat(v1.7.0): AI 友好度增强 + 测试补全 (155→309)
核心新增（面向 AI Agent 程序化使用）： - 结构化错误码体系：E_CONFIG/E_AUTH/E_NETWORK/E_RATE_LIMIT/E_PARSE/E_EMPTY/E_INPUT/E_INTERNAL classify_error() 自动分类异常，JSON 错误输出含 error_code 字段 - JSON Lines 流式输出 (--stream)：每条结果独立一行，AI 可增量处理 - 进度事件 (--progress)：JSON Lines 事件流到 stderr（start/cache_hit/fetch_ok/done 等） 测试补全（+154 例，覆盖全部高风险盲区）： - HTML 回退搜索路径 (19) - --fetch 自动抓取 (21) - --verify 健康检查 (15) - 输出格式化 (15) - 实例解析链 (20) - 并行多实例搜索 (10) - CLI 入口与端到端 (17) - 错误码分类 (27) - 流式输出与进度事件 (10) 源码改进： - search.py: h3 内 a 标签 href 作为 url fallback，提升 SearXNG 主题兼容性 - common.py: 新增 classify_error/emit_progress/set_progress_enabled 文档同步：SKILL.md 新增 AI Agent Integration Guide 章节，README.md 更新参数与错误码表
This commit is contained in:
@@ -0,0 +1,276 @@
|
||||
"""Parallel-mode tests for search_multi (parallel=True branch).
|
||||
|
||||
The serial path (parallel=False / single instance) is covered by
|
||||
``test_integration.py``. This module focuses on the ThreadPoolExecutor
|
||||
branch: result ordering by user input, failover, all-fail, auth-header
|
||||
forwarding, retry_per=0 semantics, and exception capture.
|
||||
|
||||
Thread-safety note: ``urllib.request.urlopen`` is patched with a
|
||||
URL-dispatching *function* (not a ``side_effect`` list). A function with
|
||||
no mutable shared state is safe to call concurrently from multiple
|
||||
worker threads, whereas a list-based ``side_effect`` would race on the
|
||||
shared iterator. ``MagicMock.call_count`` is itself thread-safe.
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
import urllib.error
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from search import search_multi
|
||||
|
||||
|
||||
def _mock_urlopen(data: bytes, content_type="application/json"):
|
||||
"""Build a MagicMock that quacks like an urlopen context manager."""
|
||||
resp = MagicMock()
|
||||
resp.read.return_value = data
|
||||
resp.headers = {"Content-Type": content_type}
|
||||
resp.__enter__.return_value = resp
|
||||
resp.__exit__.return_value = None
|
||||
return resp
|
||||
|
||||
|
||||
def _url_router(routing=None, default=None):
|
||||
"""Build a thread-safe side_effect that dispatches by request URL.
|
||||
|
||||
``routing`` maps a URL substring to either a response object
|
||||
(returned) or a ``BaseException`` (raised). Entries are checked in
|
||||
insertion order (dicts are ordered on Python 3.7+). ``default`` is
|
||||
used when no key matches; if it is an exception it is raised,
|
||||
otherwise returned. An ``AssertionError`` is raised when nothing
|
||||
matches and no default is set — this makes unexpected calls loud
|
||||
rather than silently returning a MagicMock.
|
||||
"""
|
||||
routing = routing or {}
|
||||
|
||||
def _side_effect(req, *args, **kwargs):
|
||||
url = getattr(req, "full_url", str(req))
|
||||
for key, resp in routing.items():
|
||||
if key in url:
|
||||
if isinstance(resp, BaseException):
|
||||
raise resp
|
||||
return resp
|
||||
if default is not None:
|
||||
if isinstance(default, BaseException):
|
||||
raise default
|
||||
return default
|
||||
raise AssertionError(
|
||||
f"unexpected urlopen for {url!r} (routing={list(routing)})"
|
||||
)
|
||||
return _side_effect
|
||||
|
||||
|
||||
def _payload(title: str, url: str = "https://x.com") -> bytes:
|
||||
return json.dumps({"results": [{"title": title, "url": url}]}).encode()
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# parallel=True: success paths
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def test_parallel_both_success_returns_first_in_order():
|
||||
"""Both instances succeed → return the first one in user-supplied order."""
|
||||
router = _url_router({
|
||||
"a.example.com": _mock_urlopen(_payload("from-a", "https://a.com")),
|
||||
"b.example.com": _mock_urlopen(_payload("from-b", "https://b.com")),
|
||||
})
|
||||
with patch("urllib.request.urlopen", side_effect=router):
|
||||
r = search_multi(
|
||||
["https://a.example.com", "https://b.example.com"],
|
||||
{"q": "test", "format": "json"},
|
||||
parallel=True, retry_per=0,
|
||||
)
|
||||
assert r["results"][0]["title"] == "from-a"
|
||||
|
||||
|
||||
def test_parallel_first_fails_second_succeeds():
|
||||
"""First instance fails, second succeeds → return second's results."""
|
||||
router = _url_router({
|
||||
"a.example.com": urllib.error.URLError("connection refused"),
|
||||
"b.example.com": _mock_urlopen(_payload("from-b", "https://b.com")),
|
||||
})
|
||||
with patch("urllib.request.urlopen", side_effect=router):
|
||||
r = search_multi(
|
||||
["https://a.example.com", "https://b.example.com"],
|
||||
{"q": "test", "format": "json"},
|
||||
parallel=True, retry_per=0,
|
||||
)
|
||||
assert r["results"][0]["title"] == "from-b"
|
||||
|
||||
|
||||
def test_parallel_all_fail_raises():
|
||||
"""All instances fail → RuntimeError carrying the '(parallel)' marker."""
|
||||
err = urllib.error.URLError("down")
|
||||
with patch("urllib.request.urlopen", side_effect=_url_router(default=err)):
|
||||
try:
|
||||
search_multi(
|
||||
["https://a.example.com", "https://b.example.com"],
|
||||
{"q": "test", "format": "json"},
|
||||
parallel=True, retry_per=0,
|
||||
)
|
||||
assert False, "should have raised RuntimeError"
|
||||
except RuntimeError as e:
|
||||
assert "(parallel)" in str(e)
|
||||
assert "2" in str(e)
|
||||
|
||||
|
||||
def test_parallel_three_middle_succeeds():
|
||||
"""Three instances, only the middle one succeeds → return middle."""
|
||||
router = _url_router({
|
||||
"a.example.com": urllib.error.URLError("down"),
|
||||
"b.example.com": _mock_urlopen(_payload("middle-b", "https://b.com")),
|
||||
"c.example.com": urllib.error.URLError("down"),
|
||||
})
|
||||
with patch("urllib.request.urlopen", side_effect=router):
|
||||
r = search_multi(
|
||||
["https://a.example.com", "https://b.example.com",
|
||||
"https://c.example.com"],
|
||||
{"q": "test", "format": "json"},
|
||||
parallel=True, retry_per=0,
|
||||
)
|
||||
assert r["results"][0]["title"] == "middle-b"
|
||||
|
||||
|
||||
def test_parallel_returns_user_order_not_completion_order():
|
||||
"""Result follows user input order, NOT completion order.
|
||||
|
||||
Instance 'a' is deliberately slowed down so 'b' finishes first, yet
|
||||
'a' (listed first) must still be the returned result when both
|
||||
succeed. This guards the ``for u in instance_urls`` deterministic
|
||||
selection at the end of the parallel branch.
|
||||
"""
|
||||
resp_a = _mock_urlopen(_payload("slow-a", "https://a.com"))
|
||||
resp_b = _mock_urlopen(_payload("fast-b", "https://b.com"))
|
||||
|
||||
def _side_effect(req, *args, **kwargs):
|
||||
url = getattr(req, "full_url", str(req))
|
||||
if "a.example.com" in url:
|
||||
time.sleep(0.15) # 'a' finishes after 'b'
|
||||
return resp_a
|
||||
if "b.example.com" in url:
|
||||
return resp_b
|
||||
raise AssertionError(f"unexpected urlopen for {url!r}")
|
||||
|
||||
with patch("urllib.request.urlopen", side_effect=_side_effect):
|
||||
r = search_multi(
|
||||
["https://a.example.com", "https://b.example.com"],
|
||||
{"q": "test", "format": "json"},
|
||||
parallel=True, retry_per=0,
|
||||
)
|
||||
assert r["results"][0]["title"] == "slow-a"
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# parallel=True: serial-path fallback conditions
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def test_parallel_single_instance_uses_serial_path():
|
||||
"""A single instance (len <= 1) takes the serial path even with
|
||||
parallel=True. Distinguishable by error message: serial path says
|
||||
'Last error', parallel path says '(parallel)'.
|
||||
"""
|
||||
err = urllib.error.URLError("down")
|
||||
with patch("urllib.request.urlopen", side_effect=err):
|
||||
try:
|
||||
search_multi(
|
||||
["https://a.example.com"],
|
||||
{"q": "test", "format": "json"},
|
||||
parallel=True, retry_per=0,
|
||||
)
|
||||
assert False, "should have raised RuntimeError"
|
||||
except RuntimeError as e:
|
||||
assert "Last error" in str(e)
|
||||
assert "(parallel)" not in str(e)
|
||||
|
||||
|
||||
def test_parallel_false_explicit_serial_path():
|
||||
"""parallel=False explicitly forces the serial path for >1 instances."""
|
||||
err = urllib.error.URLError("down")
|
||||
with patch("urllib.request.urlopen", side_effect=err):
|
||||
try:
|
||||
search_multi(
|
||||
["https://a.example.com", "https://b.example.com"],
|
||||
{"q": "test", "format": "json"},
|
||||
parallel=False, retry_per=0,
|
||||
)
|
||||
assert False, "should have raised RuntimeError"
|
||||
except RuntimeError as e:
|
||||
assert "Last error" in str(e)
|
||||
assert "(parallel)" not in str(e)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# parallel=True: header forwarding & retry semantics
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def test_parallel_auth_headers_passed():
|
||||
"""auth_headers are forwarded to every instance request."""
|
||||
captured = [] # list.append is atomic under CPython's GIL
|
||||
payload = _payload("ok", "https://x.com")
|
||||
|
||||
def _side_effect(req, *args, **kwargs):
|
||||
captured.append(req.headers)
|
||||
return _mock_urlopen(payload)
|
||||
|
||||
with patch("urllib.request.urlopen", side_effect=_side_effect):
|
||||
r = search_multi(
|
||||
["https://a.example.com", "https://b.example.com"],
|
||||
{"q": "test", "format": "json"},
|
||||
parallel=True, retry_per=0,
|
||||
auth_headers={"Authorization": "Bearer secret-token"},
|
||||
)
|
||||
assert r["results"][0]["title"] == "ok"
|
||||
# Both instances were called exactly once (success path, no HTML fallback)
|
||||
assert len(captured) == 2
|
||||
for hdrs in captured:
|
||||
assert hdrs.get("Authorization") == "Bearer secret-token"
|
||||
|
||||
|
||||
def test_parallel_retry_per_zero_no_retry():
|
||||
"""retry_per=0 → exactly one attempt per instance, no backoff retries.
|
||||
|
||||
Three failing instances must produce exactly 3 urlopen calls total.
|
||||
MagicMock.call_count is thread-safe, so concurrent increments are
|
||||
observed correctly after the executor joins.
|
||||
"""
|
||||
err = urllib.error.URLError("down")
|
||||
mock_open = MagicMock(side_effect=err)
|
||||
with patch("urllib.request.urlopen", mock_open):
|
||||
try:
|
||||
search_multi(
|
||||
["https://a.example.com", "https://b.example.com",
|
||||
"https://c.example.com"],
|
||||
{"q": "test", "format": "json"},
|
||||
parallel=True, retry_per=0,
|
||||
)
|
||||
except RuntimeError:
|
||||
pass # expected: all failed
|
||||
# 3 instances × 1 attempt (retry_per=0) = 3 calls, no retries
|
||||
assert mock_open.call_count == 3
|
||||
|
||||
|
||||
def test_parallel_exception_caught_and_logged(caplog):
|
||||
"""A raising instance is caught and logged; the survivor's result wins.
|
||||
|
||||
Instance 'a' raises HTTPError 403 (not retryable, re-raised by
|
||||
_retry_with_backoff). The parallel _task wrapper must catch it,
|
||||
record it as a failure, and let instance 'b' succeed.
|
||||
"""
|
||||
router = _url_router({
|
||||
"a.example.com": urllib.error.HTTPError(
|
||||
"url", 403, "Forbidden", {}, None),
|
||||
"b.example.com": _mock_urlopen(_payload("survivor-b", "https://b.com")),
|
||||
})
|
||||
with patch("urllib.request.urlopen", side_effect=router), \
|
||||
caplog.at_level(logging.INFO, logger="searxng.search"):
|
||||
r = search_multi(
|
||||
["https://a.example.com", "https://b.example.com"],
|
||||
{"q": "test", "format": "json"},
|
||||
parallel=True, retry_per=0,
|
||||
)
|
||||
assert r["results"][0]["title"] == "survivor-b"
|
||||
# The failure was logged (line: logger.info(f" Failed {u}: {res}"))
|
||||
assert any(
|
||||
"Failed" in rec.message and "a.example.com" in rec.message
|
||||
for rec in caplog.records
|
||||
), f"expected a 'Failed ... a.example.com' log line, got: {[r.message for r in caplog.records]}"
|
||||
Reference in New Issue
Block a user