feat(v2.3.0): fetch 结构化 JSON 契约 + 并发批量 + 真实并发门控

迭代 1 — 正确性修复: - 修复 --pages N 多页聚合的 unresponsive-engine 警告误判: 原用循环末次 cached 变量判断, 缓存命中时警告被错误跳过/误触发; 改用独立 performed_live_query 标记 - UA 池单一来源: 删除 common.py 手工副本 _FALLBACK_UAS_BUILTIN, FALLBACK_UAS 直接引用 _config.UA_POOL, 消除双份漂移 - search_html 解码修复: 硬编码 utf-8 改为 detect_charset(header/meta 自动检测), 新增 --encoding 强制覆盖, 贯穿 search_multi 全链 - AdaptiveThrottle 真实并发门控: acquire_slot()/release_slot() 槽位机制, 退避降并发后新请求被快速拒绝(E_RATE_LIMIT), 实现持久降并发而非名义降并发迭代 2 — fetch JSON 契约 + 批量并发: - fetch.py --format json: 成功 {status,url,final_url,content_type,extract, truncated,text_length,user_agent}; 失败 {status,error,error_code, status_code,url}, 对齐 search.py 错误码体系 - fetch_page 采集 title + latency, 填充 --fetch-report json 空字段 - --queries-file --parallel-queries N (1-8): 并发批量, 输出保序, 受 AdaptiveThrottle 门控; 并发模式禁用 --fetch(嵌套并行不安全) - queries 文件编码自动检测 (UTF-8 → GBK 回退) 迭代 3 — 工程化: - 新增 pyproject.toml (searxng-search/searxng-fetch 入口点) - 收敛 20+ 处函数内冗余导入 - --dump-schema 扩展: fetched.items 补全 15 字段, 新增 defs.batch/research - 新增 17 个测试 (tests/test_v230_features.py), 全量 561 测试通过 - 文档同步 (SKILL.md/README.md, 版本号 2.3.0)
2026-08-05 20:14:07 +08:00
parent 6cedba9042
commit 4df521dc9d
8 changed files with 1006 additions and 226 deletions
@@ -0,0 +1,366 @@
+"""Tests for v2.3.0 changes.
+
+Covers:
+  * multi-page (--pages N) live-query tracking — unresponsive-engine
+    warnings fire when ANY page did a live query (regression for the
+    ``cached``-variable pollution fix)
+  * search_html charset auto-detection + --encoding override
+  * UA pool single-source (common.FALLBACK_UAS is _config.UA_POOL)
+  * AdaptiveThrottle acquire_slot/release_slot real concurrency gating
+    (incl. persistent throttling after backoff reduces concurrency)
+  * fetch_page title/latency fields (consumed by --fetch-report json)
+  * fetch.py _emit_fetch_result JSON output shapes (success + error)
+  * _read_queries_file GBK fallback
+  * --parallel-queries concurrent batch (output order preserved)
+"""
+import json
+import logging
+import sys
+import time
+import urllib.error
+from types import SimpleNamespace
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+import common as common_mod
+import _config
+from fetch import FetchResult, _emit_fetch_result
+import search as search_mod
+from search import (
+    AdaptiveThrottle,
+    _read_queries_file,
+    _run_single_query,
+    _build_params,
+    fetch_page,
+    fetch_top_results,
+    search_html,
+)
+import cache as cache_module
+
+
+def _make_args(**overrides):
+    """args object matching the argparse.Namespace shape main() produces."""
+    base = dict(
+        query="test", format="json", method="GET", timeout=15, retry=0,
+        serial=False, no_dedup=False, sort_by="none", max_results=None,
+        include_domain=None, exclude_domain=None, fetch=0, fetch_timeout=10,
+        fetch_retries=3, max_size=None, cache_ttl=0, snippet_len=0,
+        categories=None, language=None, pageno=1, pages=1, time_range="year",
+        safesearch=0, engines="google,bing",
+        referer=None, no_fallback=False, fetch_report=False,
+        request_delay=0.3, throttle_failure_threshold=3,
+        throttle_pause_seconds=30, throttle_max_delay=10,
+        similarity_dedup=False, similarity_threshold=0.85,
+        encoding=None, output=None, parallel_queries=0,
+    )
+    base.update(overrides)
+    return SimpleNamespace(**base)
+
+
+# ===== Multi-page live-query tracking (cached-var pollution fix) =====
+
+def test_multi_page_live_query_warns_unresponsive(isolated_cache, caplog):
+    """pages=2: page1 live + page2 cached hit -> warning still fires.
+
+    Regression: the old code used the last loop iteration's ``cached``
+    variable, so a cache hit on page 2 suppressed the unresponsive-engine
+    warning even though page 1 hit the network live.
+    """
+    args = _make_args(pages=2, cache_ttl=30)
+    # Pre-fill page 2 cache so page 2 is a cache hit.
+    page2_params = dict(_build_params("test", args))
+    page2_params["pageno"] = "2"
+    cache_module.put(page2_params, {"results": []}, 300)
+
+    live = {"results": [{"url": "https://a.com/1", "title": "A"}],
+            "unresponsive_engines": [["brave", "Suspended: too many requests"]]}
+
+    def fake_multi(urls, params, **kw):
+        assert params.get("pageno") is None  # only page 1 hits the network
+        return live
+
+    with patch.object(search_mod, "search_multi", side_effect=fake_multi):
+        with caplog.at_level(logging.WARNING, logger="searxng.search"):
+            results, err, _ = _run_single_query(
+                "test", args, ["https://x.example.com"], {}, 300)
+    assert err is None
+    assert any("unresponsive" in r.getMessage()
+               for r in caplog.records), "warning should fire after a live query"
+    assert results["pages_fetched"] == 2
+
+
+def test_multi_page_all_cache_hit_no_warning(isolated_cache, caplog):
+    """pages=2: both pages cached -> no network, no warning."""
+    args = _make_args(pages=2, cache_ttl=30)
+    cached = {"results": [{"url": "https://c.com/1", "title": "C"}]}
+    for page_no in (1, 2):
+        p = dict(_build_params("test", args))
+        if page_no != 1:
+            p["pageno"] = str(page_no)
+        cache_module.put(p, cached, 300)
+
+    with patch.object(search_mod, "search_multi",
+                      side_effect=AssertionError("must not hit network")):
+        with caplog.at_level(logging.WARNING, logger="searxng.search"):
+            results, err, _ = _run_single_query(
+                "test", args, ["https://x.example.com"], {}, 300)
+    assert err is None
+    assert not any("unresponsive" in r.getMessage()
+                   for r in caplog.records)
+
+
+# ===== search_html charset detection =====
+
+def test_search_html_detects_gbk_charset():
+    """GBK HTML detected via <meta charset> (header has no charset)."""
+    html = ('<html><head><meta charset="gbk"></head><body>'
+            '<article class="result"><h3><a href="https://x.com">中文标题</a>'
+            '</h3></article></body></html>')
+    raw = html.encode("gbk")
+    resp = MagicMock()
+    resp.read.return_value = raw
+    resp.headers = {"Content-Type": "text/html"}  # no charset in header
+    resp.__enter__.return_value = resp
+    with patch("urllib.request.urlopen", return_value=resp):
+        r = search_html("https://s.example.com", {"q": "test"})
+    assert r["results"][0]["title"] == "中文标题"
+
+
+def test_search_html_encoding_override():
+    """--encoding gbk forces the charset even with wrong header."""
+    html = '<article class="result"><h3><a href="https://x.com">标题</a></h3></article>'
+    raw = html.encode("gbk")
+    resp = MagicMock()
+    resp.read.return_value = raw
+    resp.headers = {"Content-Type": "text/html; charset=utf-8"}  # lies
+    resp.__enter__.return_value = resp
+    with patch("urllib.request.urlopen", return_value=resp):
+        r = search_html("https://s.example.com", {"q": "test"}, encoding="gbk")
+    assert r["results"][0]["title"] == "标题"
+
+
+# ===== UA pool single source =====
+
+def test_ua_pool_is_single_source():
+    """common.FALLBACK_UAS must be the _config.UA_POOL object (no duplicate)."""
+    assert common_mod.FALLBACK_UAS is _config.UA_POOL
+    assert len(_config.UA_POOL) >= 12
+
+
+# ===== AdaptiveThrottle slot gating =====
+
+def test_throttle_acquire_release_slot_basic():
+    """acquire_slot respects the concurrency cap; release frees it."""
+    t = AdaptiveThrottle(0.0, 1)
+    assert t.acquire_slot() is True
+    # Second acquisition must fail while one slot is in flight.
+    assert t.acquire_slot(timeout=0.05) is False
+    t.release_slot()
+    assert t.acquire_slot(timeout=0.05) is True
+    t.release_slot()
+
+
+def test_throttle_slot_gates_on_reduced_concurrency():
+    """After backoff halves concurrency, in-flight requests gate new ones.
+
+    Regression: ThreadPoolExecutor size is fixed at creation; the semaphore
+    + in-flight check must persist the reduced concurrency. With
+    initial=2, two slots in flight, then backoff -> concurrency=1: a new
+    acquisition must be rejected even though a semaphore slot is free.
+    """
+    t = AdaptiveThrottle(0.0, 2)
+    assert t.acquire_slot() is True
+    assert t.acquire_slot() is True
+    # 3 consecutive failures -> concurrency 2 -> 1
+    for _ in range(3):
+        t.report_failure()
+    assert t.concurrency == 1
+    # Release one in-flight slot: semaphore has room, but in_flight (1)
+    # now exceeds the reduced concurrency target (1) -> reject.
+    t.release_slot()
+    assert t.acquire_slot(timeout=0.05) is False
+    t.release_slot()  # cleanup the remaining acquired slot
+
+
+def test_fetch_top_results_reports_throttled_skips():
+    """fetch_top_results marks requests gated by the concurrency cap.
+
+    Semantics covered deterministically by the acquire_slot unit tests;
+    this integration test asserts the end-to-end shape: a URL skipped by
+    the gate appears with status=error + error_code=E_RATE_LIMIT. We
+    simulate the gate directly by stubbing acquire_slot.
+    """
+    throttle = AdaptiveThrottle(0.0, 5)
+    real_acquire = throttle.acquire_slot
+
+    def _stub_acquire(timeout=0.05):
+        # First call wins the slot; every subsequent call is gated.
+        return False
+
+    results = {"results": [{"url": "https://a.com"},
+                           {"url": "https://b.com"}]}
+
+    def _fake_fetch(url, **kwargs):
+        return {"url": url, "status": "ok", "text": "x", "text_length": 1,
+                "truncated": False}
+
+    with patch.object(search_mod, "fetch_page", side_effect=_fake_fetch):
+        with patch.object(throttle, "acquire_slot", side_effect=_stub_acquire):
+            out = fetch_top_results(results, 2, request_delay=0.0,
+                                    throttle=throttle)
+    assert len(out) == 2
+    for f in out:
+        assert f["status"] == "error"
+        assert f["error_code"] == "E_RATE_LIMIT"
+        assert "Throttled" in f["error"]
+
+
+# ===== fetch_page title/latency =====
+
+def _mock_fetch_result(content, content_type="text/html",
+                       final_url="https://example.com", ua="TestUA"):
+    return FetchResult(content=content, content_type=content_type,
+                       final_url=final_url, truncated=False, user_agent=ua)
+
+
+def test_fetch_page_collects_title_and_latency():
+    html = "<html><head><title>Page Title</title></head>" \
+           "<body><article>Hello world</article></body></html>"
+    with patch.object(search_mod, "fetch_url",
+                      return_value=_mock_fetch_result(html)):
+        r = fetch_page("https://example.com", fallback_enabled=False)
+    assert r["status"] == "ok"
+    assert r["title"] == "Page Title"
+    assert r["latency"] is not None and r["latency"] >= 0
+
+
+def test_fetch_page_title_none_for_non_html():
+    with patch.object(search_mod, "fetch_url",
+                      return_value=_mock_fetch_result(
+                          "plain", content_type="text/plain")):
+        r = fetch_page("https://example.com/x.txt", fallback_enabled=False)
+    assert r["status"] == "ok"
+    assert r["title"] is None
+    assert r["latency"] is not None
+
+
+# ===== fetch.py --format json output =====
+
+def _json_args(**overrides):
+    base = dict(url="https://x.example.com", extract="text", format="json",
+                output=None)
+    base.update(overrides)
+    return SimpleNamespace(**base)
+
+
+def test_emit_fetch_result_json_success(capsys):
+    _emit_fetch_result(_json_args(), "hello world", "https://x.example.com",
+                       "https://x.example.com/", "text/html", False,
+                       user_agent="TestUA")
+    out, _ = capsys.readouterr()
+    data = json.loads(out)
+    assert data["status"] == "ok"
+    assert data["url"] == "https://x.example.com"
+    assert data["final_url"] == "https://x.example.com/"
+    assert data["extract"] == "text"
+    assert data["text_length"] == 11
+    assert data["truncated"] is False
+    assert data["user_agent"] == "TestUA"
+    assert "content_type" in data
+
+
+def test_emit_fetch_result_json_error(capsys):
+    _emit_fetch_result(_json_args(), "", "https://x.example.com",
+                       "https://x.example.com", "", False,
+                       error="HTTP 404", error_code="E_NETWORK",
+                       status_code=404)
+    out, _ = capsys.readouterr()
+    data = json.loads(out)
+    assert data["status"] == "error"
+    assert data["error_code"] == "E_NETWORK"
+    assert data["status_code"] == 404
+    assert data["url"] == "https://x.example.com"
+
+
+def test_emit_fetch_result_text_mode_unchanged(capsys):
+    """--format text keeps raw content on stdout (v2.2.x behavior)."""
+    _emit_fetch_result(_json_args(format="text"), "raw content",
+                       "https://x.example.com", "https://x.example.com",
+                       "text/html", False)
+    out, _ = capsys.readouterr()
+    assert out == "raw content\n"
+
+
+# ===== _read_queries_file encoding fallback =====
+
+def test_read_queries_file_gbk_fallback(tmp_path):
+    """GBK-encoded queries file decodes via the utf-8 -> gbk fallback."""
+    p = tmp_path / "queries.txt"
+    p.write_bytes("中文查询一\n中文查询二\n".encode("gbk"))
+    qs = _read_queries_file(str(p))
+    assert qs == ["中文查询一", "中文查询二"]
+
+
+def test_read_queries_file_utf8_preferred(tmp_path):
+    p = tmp_path / "queries.txt"
+    p.write_text("q1\n# comment\nq2\n", encoding="utf-8")
+    assert _read_queries_file(str(p)) == ["q1", "q2"]
+
+
+# ===== --parallel-queries concurrent batch =====
+
+def _batch_argv(queries_file_path, parallel):
+    return ["search.py", "-i", "https://x.example.com",
+            "--queries-file", str(queries_file_path),
+            "--parallel-queries", str(parallel),
+            "--format", "json", "--retry", "0"]
+
+
+def test_parallel_batch_preserves_order(tmp_path, capsys):
+    """Concurrent batch output keeps the file order (deterministic)."""
+    qf = tmp_path / "queries.txt"
+    qf.write_text("q1\nq2\nq3\nq4\n", encoding="utf-8")
+
+    def fake_multi(urls, params, **kw):
+        q = params["q"]
+        time.sleep(0.01 * int(q[-1]))  # q4 slowest, q1 fastest
+        return {"results": [{"url": f"https://{q}.com", "title": q}]}
+
+    with patch.object(search_mod, "search_multi", side_effect=fake_multi):
+        with pytest.raises(SystemExit) as exc_info:
+            with patch.object(sys, "argv", _batch_argv(qf, 2)):
+                with patch.object(search_mod, "setup_logging"):
+                    search_mod.main()
+    assert exc_info.value.code == 0
+    out, _ = capsys.readouterr()
+    data = json.loads(out)
+    queries = [e["query"] for e in data["queries"]]
+    assert queries == ["q1", "q2", "q3", "q4"]  # order preserved
+    assert all(e["status"] == "ok" for e in data["queries"])
+
+
+def test_parallel_batch_records_errors(tmp_path, capsys):
+    """Concurrent batch keeps per-query error entries without aborting."""
+    qf = tmp_path / "queries.txt"
+    qf.write_text("ok1\nbad1\nok2\n", encoding="utf-8")
+    err = urllib.error.URLError("connection refused")
+
+    def fake_multi(urls, params, **kw):
+        if params["q"].startswith("bad"):
+            raise err
+        return {"results": [{"url": f"https://{params['q']}.com"}]}
+
+    with patch.object(search_mod, "search_multi", side_effect=fake_multi):
+        with pytest.raises(SystemExit) as exc_info:
+            with patch.object(sys, "argv", _batch_argv(qf, 3)):
+                with patch.object(search_mod, "setup_logging"):
+                    search_mod.main()
+    assert exc_info.value.code == 0  # partial success
+    out, _ = capsys.readouterr()
+    data = json.loads(out)
+    statuses = {e["query"]: e for e in data["queries"]}
+    assert statuses["bad1"]["status"] == "error"
+    assert statuses["bad1"]["error_code"] == "E_NETWORK"
+    assert statuses["ok1"]["status"] == "ok"
+    assert statuses["ok2"]["status"] == "ok"