searxng-use-cli/tests/test_v230_features.py

"""Tests for v2.3.0 changes.

Covers:
  * multi-page (--pages N) live-query tracking — unresponsive-engine
    warnings fire when ANY page did a live query (regression for the
    ``cached``-variable pollution fix)
  * search_html charset auto-detection + --encoding override
  * UA pool single-source (common.FALLBACK_UAS is _config.UA_POOL)
  * AdaptiveThrottle acquire_slot/release_slot real concurrency gating
    (incl. persistent throttling after backoff reduces concurrency)
  * fetch_page title/latency fields (consumed by --fetch-report json)
  * fetch.py _emit_fetch_result JSON output shapes (success + error)
  * _read_queries_file GBK fallback
  * --parallel-queries concurrent batch (output order preserved)
"""
import json
import logging
import sys
import time
import urllib.error
from types import SimpleNamespace
from unittest.mock import MagicMock, patch

import pytest

import common as common_mod
import _config
from fetch import FetchResult, _emit_fetch_result
import search as search_mod
from search import (
    AdaptiveThrottle,
    _read_queries_file,
    _run_single_query,
    _build_params,
    fetch_page,
    fetch_top_results,
    search_html,
)
import cache as cache_module


def _make_args(**overrides):
    """args object matching the argparse.Namespace shape main() produces."""
    base = dict(
        query="test", format="json", method="GET", timeout=15, retry=0,
        serial=False, no_dedup=False, sort_by="none", max_results=None,
        include_domain=None, exclude_domain=None, fetch=0, fetch_timeout=10,
        fetch_retries=3, max_size=None, cache_ttl=0, snippet_len=0,
        categories=None, language=None, pageno=1, pages=1, time_range="year",
        safesearch=0, engines="google,bing",
        referer=None, no_fallback=False, fetch_report=False,
        request_delay=0.3, throttle_failure_threshold=3,
        throttle_pause_seconds=30, throttle_max_delay=10,
        similarity_dedup=False, similarity_threshold=0.85,
        encoding=None, output=None, parallel_queries=0,
    )
    base.update(overrides)
    return SimpleNamespace(**base)


# ===== Multi-page live-query tracking (cached-var pollution fix) =====

def test_multi_page_live_query_warns_unresponsive(isolated_cache, caplog):
    """pages=2: page1 live + page2 cached hit -> warning still fires.

    Regression: the old code used the last loop iteration's ``cached``
    variable, so a cache hit on page 2 suppressed the unresponsive-engine
    warning even though page 1 hit the network live.
    """
    args = _make_args(pages=2, cache_ttl=30)
    # Pre-fill page 2 cache so page 2 is a cache hit.
    page2_params = dict(_build_params("test", args))
    page2_params["pageno"] = "2"
    cache_module.put(page2_params, {"results": []}, 300)

    live = {"results": [{"url": "https://a.com/1", "title": "A"}],
            "unresponsive_engines": [["brave", "Suspended: too many requests"]]}

    def fake_multi(urls, params, **kw):
        assert params.get("pageno") is None  # only page 1 hits the network
        return live

    with patch.object(search_mod, "search_multi", side_effect=fake_multi):
        with caplog.at_level(logging.WARNING, logger="searxng.search"):
            results, err, _ = _run_single_query(
                "test", args, ["https://x.example.com"], {}, 300)
    assert err is None
    assert any("unresponsive" in r.getMessage()
               for r in caplog.records), "warning should fire after a live query"
    assert results["pages_fetched"] == 2


def test_multi_page_all_cache_hit_no_warning(isolated_cache, caplog):
    """pages=2: both pages cached -> no network, no warning."""
    args = _make_args(pages=2, cache_ttl=30)
    cached = {"results": [{"url": "https://c.com/1", "title": "C"}]}
    for page_no in (1, 2):
        p = dict(_build_params("test", args))
        if page_no != 1:
            p["pageno"] = str(page_no)
        cache_module.put(p, cached, 300)

    with patch.object(search_mod, "search_multi",
                      side_effect=AssertionError("must not hit network")):
        with caplog.at_level(logging.WARNING, logger="searxng.search"):
            results, err, _ = _run_single_query(
                "test", args, ["https://x.example.com"], {}, 300)
    assert err is None
    assert not any("unresponsive" in r.getMessage()
                   for r in caplog.records)


# ===== search_html charset detection =====

def test_search_html_detects_gbk_charset():
    """GBK HTML detected via <meta charset> (header has no charset)."""
    html = ('<html><head><meta charset="gbk"></head><body>'
            '<article class="result"><h3><a href="https://x.com">中文标题</a>'
            '</h3></article></body></html>')
    raw = html.encode("gbk")
    resp = MagicMock()
    resp.read.return_value = raw
    resp.headers = {"Content-Type": "text/html"}  # no charset in header
    resp.__enter__.return_value = resp
    with patch("urllib.request.urlopen", return_value=resp):
        r = search_html("https://s.example.com", {"q": "test"})
    assert r["results"][0]["title"] == "中文标题"


def test_search_html_encoding_override():
    """--encoding gbk forces the charset even with wrong header."""
    html = '<article class="result"><h3><a href="https://x.com">标题</a></h3></article>'
    raw = html.encode("gbk")
    resp = MagicMock()
    resp.read.return_value = raw
    resp.headers = {"Content-Type": "text/html; charset=utf-8"}  # lies
    resp.__enter__.return_value = resp
    with patch("urllib.request.urlopen", return_value=resp):
        r = search_html("https://s.example.com", {"q": "test"}, encoding="gbk")
    assert r["results"][0]["title"] == "标题"


# ===== UA pool single source =====

def test_ua_pool_is_single_source():
    """common.FALLBACK_UAS must be the _config.UA_POOL object (no duplicate)."""
    assert common_mod.FALLBACK_UAS is _config.UA_POOL
    assert len(_config.UA_POOL) >= 12


# ===== AdaptiveThrottle slot gating =====

def test_throttle_acquire_release_slot_basic():
    """acquire_slot respects the concurrency cap; release frees it."""
    t = AdaptiveThrottle(0.0, 1)
    assert t.acquire_slot() is True
    # Second acquisition must fail while one slot is in flight.
    assert t.acquire_slot(timeout=0.05) is False
    t.release_slot()
    assert t.acquire_slot(timeout=0.05) is True
    t.release_slot()


def test_throttle_slot_gates_on_reduced_concurrency():
    """After backoff halves concurrency, in-flight requests gate new ones.

    Regression: ThreadPoolExecutor size is fixed at creation; the semaphore
    + in-flight check must persist the reduced concurrency. With
    initial=2, two slots in flight, then backoff -> concurrency=1: a new
    acquisition must be rejected even though a semaphore slot is free.
    """
    t = AdaptiveThrottle(0.0, 2)
    assert t.acquire_slot() is True
    assert t.acquire_slot() is True
    # 3 consecutive failures -> concurrency 2 -> 1
    for _ in range(3):
        t.report_failure()
    assert t.concurrency == 1
    # Release one in-flight slot: semaphore has room, but in_flight (1)
    # now exceeds the reduced concurrency target (1) -> reject.
    t.release_slot()
    assert t.acquire_slot(timeout=0.05) is False
    t.release_slot()  # cleanup the remaining acquired slot


def test_fetch_top_results_reports_throttled_skips():
    """fetch_top_results marks requests gated by the concurrency cap.

    Semantics covered deterministically by the acquire_slot unit tests;
    this integration test asserts the end-to-end shape: a URL skipped by
    the gate appears with status=error + error_code=E_RATE_LIMIT. We
    simulate the gate directly by stubbing acquire_slot.
    """
    throttle = AdaptiveThrottle(0.0, 5)
    real_acquire = throttle.acquire_slot

    def _stub_acquire(timeout=0.05):
        # First call wins the slot; every subsequent call is gated.
        return False

    results = {"results": [{"url": "https://a.com"},
                           {"url": "https://b.com"}]}

    def _fake_fetch(url, **kwargs):
        return {"url": url, "status": "ok", "text": "x", "text_length": 1,
                "truncated": False}

    with patch.object(search_mod, "fetch_page", side_effect=_fake_fetch):
        with patch.object(throttle, "acquire_slot", side_effect=_stub_acquire):
            out = fetch_top_results(results, 2, request_delay=0.0,
                                    throttle=throttle)
    assert len(out) == 2
    for f in out:
        assert f["status"] == "error"
        assert f["error_code"] == "E_RATE_LIMIT"
        assert "Throttled" in f["error"]


# ===== fetch_page title/latency =====

def _mock_fetch_result(content, content_type="text/html",
                       final_url="https://example.com", ua="TestUA"):
    return FetchResult(content=content, content_type=content_type,
                       final_url=final_url, truncated=False, user_agent=ua)


def test_fetch_page_collects_title_and_latency():
    html = "<html><head><title>Page Title</title></head>" \
           "<body><article>Hello world</article></body></html>"
    with patch.object(search_mod, "fetch_url",
                      return_value=_mock_fetch_result(html)):
        r = fetch_page("https://example.com", fallback_enabled=False)
    assert r["status"] == "ok"
    assert r["title"] == "Page Title"
    assert r["latency"] is not None and r["latency"] >= 0


def test_fetch_page_title_none_for_non_html():
    with patch.object(search_mod, "fetch_url",
                      return_value=_mock_fetch_result(
                          "plain", content_type="text/plain")):
        r = fetch_page("https://example.com/x.txt", fallback_enabled=False)
    assert r["status"] == "ok"
    assert r["title"] is None
    assert r["latency"] is not None


# ===== fetch.py --format json output =====

def _json_args(**overrides):
    base = dict(url="https://x.example.com", extract="text", format="json",
                output=None)
    base.update(overrides)
    return SimpleNamespace(**base)


def test_emit_fetch_result_json_success(capsys):
    _emit_fetch_result(_json_args(), "hello world", "https://x.example.com",
                       "https://x.example.com/", "text/html", False,
                       user_agent="TestUA")
    out, _ = capsys.readouterr()
    data = json.loads(out)
    assert data["status"] == "ok"
    assert data["url"] == "https://x.example.com"
    assert data["final_url"] == "https://x.example.com/"
    assert data["extract"] == "text"
    assert data["text_length"] == 11
    assert data["truncated"] is False
    assert data["user_agent"] == "TestUA"
    assert "content_type" in data


def test_emit_fetch_result_json_error(capsys):
    _emit_fetch_result(_json_args(), "", "https://x.example.com",
                       "https://x.example.com", "", False,
                       error="HTTP 404", error_code="E_NETWORK",
                       status_code=404)
    out, _ = capsys.readouterr()
    data = json.loads(out)
    assert data["status"] == "error"
    assert data["error_code"] == "E_NETWORK"
    assert data["status_code"] == 404
    assert data["url"] == "https://x.example.com"


def test_emit_fetch_result_text_mode_unchanged(capsys):
    """--format text keeps raw content on stdout (v2.2.x behavior)."""
    _emit_fetch_result(_json_args(format="text"), "raw content",
                       "https://x.example.com", "https://x.example.com",
                       "text/html", False)
    out, _ = capsys.readouterr()
    assert out == "raw content\n"


# ===== _read_queries_file encoding fallback =====

def test_read_queries_file_gbk_fallback(tmp_path):
    """GBK-encoded queries file decodes via the utf-8 -> gbk fallback."""
    p = tmp_path / "queries.txt"
    p.write_bytes("中文查询一\n中文查询二\n".encode("gbk"))
    qs = _read_queries_file(str(p))
    assert qs == ["中文查询一", "中文查询二"]


def test_read_queries_file_utf8_preferred(tmp_path):
    p = tmp_path / "queries.txt"
    p.write_text("q1\n# comment\nq2\n", encoding="utf-8")
    assert _read_queries_file(str(p)) == ["q1", "q2"]


# ===== --parallel-queries concurrent batch =====

def _batch_argv(queries_file_path, parallel):
    return ["search.py", "-i", "https://x.example.com",
            "--queries-file", str(queries_file_path),
            "--parallel-queries", str(parallel),
            "--format", "json", "--retry", "0"]


def test_parallel_batch_preserves_order(tmp_path, capsys):
    """Concurrent batch output keeps the file order (deterministic)."""
    qf = tmp_path / "queries.txt"
    qf.write_text("q1\nq2\nq3\nq4\n", encoding="utf-8")

    def fake_multi(urls, params, **kw):
        q = params["q"]
        time.sleep(0.01 * int(q[-1]))  # q4 slowest, q1 fastest
        return {"results": [{"url": f"https://{q}.com", "title": q}]}

    with patch.object(search_mod, "search_multi", side_effect=fake_multi):
        with pytest.raises(SystemExit) as exc_info:
            with patch.object(sys, "argv", _batch_argv(qf, 2)):
                with patch.object(search_mod, "setup_logging"):
                    search_mod.main()
    assert exc_info.value.code == 0
    out, _ = capsys.readouterr()
    data = json.loads(out)
    queries = [e["query"] for e in data["queries"]]
    assert queries == ["q1", "q2", "q3", "q4"]  # order preserved
    assert all(e["status"] == "ok" for e in data["queries"])


def test_parallel_batch_records_errors(tmp_path, capsys):
    """Concurrent batch keeps per-query error entries without aborting."""
    qf = tmp_path / "queries.txt"
    qf.write_text("ok1\nbad1\nok2\n", encoding="utf-8")
    err = urllib.error.URLError("connection refused")

    def fake_multi(urls, params, **kw):
        if params["q"].startswith("bad"):
            raise err
        return {"results": [{"url": f"https://{params['q']}.com"}]}

    with patch.object(search_mod, "search_multi", side_effect=fake_multi):
        with pytest.raises(SystemExit) as exc_info:
            with patch.object(sys, "argv", _batch_argv(qf, 3)):
                with patch.object(search_mod, "setup_logging"):
                    search_mod.main()
    assert exc_info.value.code == 0  # partial success
    out, _ = capsys.readouterr()
    data = json.loads(out)
    statuses = {e["query"]: e for e in data["queries"]}
    assert statuses["bad1"]["status"] == "error"
    assert statuses["bad1"]["error_code"] == "E_NETWORK"
    assert statuses["ok1"]["status"] == "ok"
    assert statuses["ok2"]["status"] == "ok"