searxng-use-cli/tests/test_html_fallback.py

"""Tests for the HTML fallback search path.

Covers: SearXNGHTMLParser (result/suggestion/answer extraction, <time> tag,
script/style skipping), parse_html_results (shape), search_html (HTTP +
error), search_single (JSON→HTML fallback ordering).
"""
import urllib.error
from unittest.mock import patch, MagicMock

from search import (
    SearXNGHTMLParser,
    parse_html_results,
    search_html,
    search_single,
)


# ----- SearXNGHTMLParser: result extraction -----

def test_parser_single_result():
    """A complete <article class="result"> with title/url/content."""
    html = """
    <article class="result result-default category-general">
      <h3><a href="https://example.com/page">Example Title</a></h3>
      <p class="content">Snippet text here</p>
    </article>
    """
    parser = SearXNGHTMLParser()
    parser.feed(html)
    assert len(parser.results) == 1
    r = parser.results[0]
    assert r["title"] == "Example Title"
    assert r["url"] == "https://example.com/page"
    assert r["content"] == "Snippet text here"


def test_parser_url_header_class():
    """<a class="url_header"> populates the url field."""
    html = """
    <article class="result">
      <a href="https://hdr.example.com" class="url_header">link</a>
      <h3><a href="https://title.example.com">Title</a></h3>
    </article>
    """
    parser = SearXNGHTMLParser()
    parser.feed(html)
    assert len(parser.results) == 1
    # url_header takes precedence over h3's href
    assert parser.results[0]["url"] == "https://hdr.example.com"


def test_parser_time_datetime_attr():
    """<time datetime="..."> populates published_date from the attribute."""
    html = """
    <article class="result">
      <h3><a href="https://x.com">T</a></h3>
      <time datetime="2024-01-15T10:30:00">Jan 15, 2024</time>
    </article>
    """
    parser = SearXNGHTMLParser()
    parser.feed(html)
    assert parser.results[0]["published_date"] == "2024-01-15T10:30:00"


def test_parser_time_text_fallback():
    """When datetime attr is absent, text content is used as published_date."""
    html = """
    <article class="result">
      <h3><a href="https://x.com">T</a></h3>
      <time>3 days ago</time>
    </article>
    """
    parser = SearXNGHTMLParser()
    parser.feed(html)
    assert parser.results[0]["published_date"] == "3 days ago"


def test_parser_skips_script_and_style():
    """<script> and <style> content must not leak into results."""
    html = """
    <article class="result">
      <h3><a href="https://x.com">T</a></h3>
      <script>var evil = "ignore me";</script>
      <style>.css { color: red; }</style>
      <p class="content">real content</p>
    </article>
    """
    parser = SearXNGHTMLParser()
    parser.feed(html)
    assert len(parser.results) == 1
    assert "evil" not in parser.results[0]["content"]
    assert "css" not in parser.results[0]["content"]
    assert parser.results[0]["content"] == "real content"


def test_parser_multiple_results():
    html = """
    <article class="result"><h3><a href="https://a.com">A</a></h3></article>
    <article class="result"><h3><a href="https://b.com">B</a></h3></article>
    <article class="result"><h3><a href="https://c.com">C</a></h3></article>
    """
    parser = SearXNGHTMLParser()
    parser.feed(html)
    assert len(parser.results) == 3
    assert [r["title"] for r in parser.results] == ["A", "B", "C"]


def test_parser_empty_article_skipped():
    """An <article> with neither title nor url is dropped."""
    html = '<article class="result"><p class="content">no title or url</p></article>'
    parser = SearXNGHTMLParser()
    parser.feed(html)
    assert len(parser.results) == 0


def test_parser_suggestions():
    """<div id="suggestions"> with <a> tags populates suggestions list."""
    html = """
    <div id="suggestions">
      <a>first suggestion</a>
      <a>second suggestion</a>
    </div>
    """
    parser = SearXNGHTMLParser()
    parser.feed(html)
    assert "first suggestion" in parser.suggestions
    assert "second suggestion" in parser.suggestions


def test_parser_suggestions_class():
    """class="suggestion" also triggers suggestion capture."""
    html = '<div class="suggestion"><a>via class</a></div>'
    parser = SearXNGHTMLParser()
    parser.feed(html)
    assert "via class" in parser.suggestions


def test_parser_answer_box():
    """<div class="answer"> or id="answer" populates answers list."""
    html = '<div class="answer">The answer is 42</div>'
    parser = SearXNGHTMLParser()
    parser.feed(html)
    assert len(parser.answers) == 1
    assert "42" in parser.answers[0]


def test_parser_answer_by_id():
    html = '<div id="answer">42</div>'
    parser = SearXNGHTMLParser()
    parser.feed(html)
    assert len(parser.answers) == 1


# ----- parse_html_results -----

def test_parse_html_results_shape():
    """parse_html_results returns a dict with all expected keys."""
    html = """
    <article class="result"><h3><a href="https://x.com">T</a></h3>
    <p class="content">c</p></article>
    <div id="suggestions"><a>s1</a></div>
    <div class="answer">a1</div>
    """
    r = parse_html_results(html, query="test")
    assert r["query"] == "test"
    assert len(r["results"]) == 1
    assert r["suggestions"] == ["s1"]
    assert r["answers"] == ["a1"]
    assert r["corrections"] == []
    assert r["infoboxes"] == []
    assert r["unresponsive_engines"] == []
    assert r["_fallback"] == "html"
    assert r["number_of_results"] == 1


def test_parse_html_results_empty():
    """Empty HTML yields an empty-but-well-shaped result."""
    r = parse_html_results("", query="q")
    assert r["results"] == []
    assert r["query"] == "q"


# ----- search_html -----

def _mock_urlopen(data: bytes, content_type="text/html"):
    resp = MagicMock()
    resp.read.return_value = data
    resp.headers = {"Content-Type": content_type}
    resp.__enter__.return_value = resp
    resp.__exit__.return_value = None
    return resp


def test_search_html_success():
    """search_html fetches and parses an HTML results page."""
    html = """
    <article class="result"><h3><a href="https://r.com">R</a></h3>
    <p class="content">snippet</p></article>
    """
    with patch("urllib.request.urlopen",
               return_value=_mock_urlopen(html.encode())):
        r = search_html("https://s.example.com", {"q": "test"})
    assert r["results"][0]["title"] == "R"
    assert r["results"][0]["url"] == "https://r.com"


def test_search_html_strips_format_param():
    """search_html must not pass format=json to the HTML endpoint."""
    captured_req = {}

    def _capture(req, timeout=None):
        captured_req["url"] = req.full_url
        return _mock_urlopen(b"<article class='result'></article>")

    with patch("urllib.request.urlopen", side_effect=_capture):
        search_html("https://s.example.com", {"q": "test", "format": "json"})
    assert "format=json" not in captured_req["url"]
    assert "q=test" in captured_req["url"]


def test_search_html_error_raises_runtime():
    """Network errors are wrapped in RuntimeError with context."""
    err = urllib.error.URLError("connection refused")
    with patch("urllib.request.urlopen", side_effect=err):
        try:
            search_html("https://s.example.com", {"q": "test"})
            assert False, "should raise"
        except RuntimeError as e:
            assert "HTML search failed" in str(e)


# ----- search_single -----

def test_search_single_prefers_json():
    """When JSON works, search_single returns JSON results directly."""
    import json as json_mod
    payload = json_mod.dumps({"results": [{"title": "json", "url": "https://j.com"}]})
    with patch("urllib.request.urlopen",
               return_value=_mock_urlopen(payload.encode(),
                                          content_type="application/json")):
        r = search_single("https://s.example.com", {"q": "t", "format": "json"})
    assert r["results"][0]["title"] == "json"


def test_search_single_falls_back_to_html():
    """When JSON returns None, search_single falls back to HTML parsing."""
    html = """
    <article class="result"><h3><a href="https://h.com">html</a></h3></article>
    """
    # First call returns HTML (JSON unsupported), second call (HTML search) also HTML
    with patch("urllib.request.urlopen",
               return_value=_mock_urlopen(html.encode())):
        r = search_single("https://s.example.com", {"q": "t", "format": "json"})
    assert r["results"][0]["title"] == "html"
    assert r.get("_fallback") == "html"


def test_search_single_html_fallback_marks_fallback():
    """HTML fallback path sets _fallback='html' in the result."""
    html = '<article class="result"><h3><a href="https://x.com">x</a></h3></article>'
    with patch("urllib.request.urlopen",
               return_value=_mock_urlopen(html.encode())):
        r = search_single("https://s.example.com", {"q": "t", "format": "json"})
    assert r["_fallback"] == "html"