thzxx
f983a9377e
核心新增（面向 AI Agent 程序化使用）： - 结构化错误码体系：E_CONFIG/E_AUTH/E_NETWORK/E_RATE_LIMIT/E_PARSE/E_EMPTY/E_INPUT/E_INTERNAL classify_error() 自动分类异常，JSON 错误输出含 error_code 字段 - JSON Lines 流式输出 (--stream)：每条结果独立一行，AI 可增量处理 - 进度事件 (--progress)：JSON Lines 事件流到 stderr（start/cache_hit/fetch_ok/done 等） 测试补全（+154 例，覆盖全部高风险盲区）： - HTML 回退搜索路径 (19) - --fetch 自动抓取 (21) - --verify 健康检查 (15) - 输出格式化 (15) - 实例解析链 (20) - 并行多实例搜索 (10) - CLI 入口与端到端 (17) - 错误码分类 (27) - 流式输出与进度事件 (10) 源码改进： - search.py: h3 内 a 标签 href 作为 url fallback，提升 SearXNG 主题兼容性 - common.py: 新增 classify_error/emit_progress/set_progress_enabled 文档同步：SKILL.md 新增 AI Agent Integration Guide 章节，README.md 更新参数与错误码表
265 lines
8.8 KiB
Python
265 lines
8.8 KiB
Python
"""Tests for the HTML fallback search path.
|
|
|
|
Covers: SearXNGHTMLParser (result/suggestion/answer extraction, <time> tag,
|
|
script/style skipping), parse_html_results (shape), search_html (HTTP +
|
|
error), search_single (JSON→HTML fallback ordering).
|
|
"""
|
|
import urllib.error
|
|
from unittest.mock import patch, MagicMock
|
|
|
|
from search import (
|
|
SearXNGHTMLParser,
|
|
parse_html_results,
|
|
search_html,
|
|
search_single,
|
|
)
|
|
|
|
|
|
# ----- SearXNGHTMLParser: result extraction -----
|
|
|
|
def test_parser_single_result():
|
|
"""A complete <article class="result"> with title/url/content."""
|
|
html = """
|
|
<article class="result result-default category-general">
|
|
<h3><a href="https://example.com/page">Example Title</a></h3>
|
|
<p class="content">Snippet text here</p>
|
|
</article>
|
|
"""
|
|
parser = SearXNGHTMLParser()
|
|
parser.feed(html)
|
|
assert len(parser.results) == 1
|
|
r = parser.results[0]
|
|
assert r["title"] == "Example Title"
|
|
assert r["url"] == "https://example.com/page"
|
|
assert r["content"] == "Snippet text here"
|
|
|
|
|
|
def test_parser_url_header_class():
|
|
"""<a class="url_header"> populates the url field."""
|
|
html = """
|
|
<article class="result">
|
|
<a href="https://hdr.example.com" class="url_header">link</a>
|
|
<h3><a href="https://title.example.com">Title</a></h3>
|
|
</article>
|
|
"""
|
|
parser = SearXNGHTMLParser()
|
|
parser.feed(html)
|
|
assert len(parser.results) == 1
|
|
# url_header takes precedence over h3's href
|
|
assert parser.results[0]["url"] == "https://hdr.example.com"
|
|
|
|
|
|
def test_parser_time_datetime_attr():
|
|
"""<time datetime="..."> populates published_date from the attribute."""
|
|
html = """
|
|
<article class="result">
|
|
<h3><a href="https://x.com">T</a></h3>
|
|
<time datetime="2024-01-15T10:30:00">Jan 15, 2024</time>
|
|
</article>
|
|
"""
|
|
parser = SearXNGHTMLParser()
|
|
parser.feed(html)
|
|
assert parser.results[0]["published_date"] == "2024-01-15T10:30:00"
|
|
|
|
|
|
def test_parser_time_text_fallback():
|
|
"""When datetime attr is absent, text content is used as published_date."""
|
|
html = """
|
|
<article class="result">
|
|
<h3><a href="https://x.com">T</a></h3>
|
|
<time>3 days ago</time>
|
|
</article>
|
|
"""
|
|
parser = SearXNGHTMLParser()
|
|
parser.feed(html)
|
|
assert parser.results[0]["published_date"] == "3 days ago"
|
|
|
|
|
|
def test_parser_skips_script_and_style():
|
|
"""<script> and <style> content must not leak into results."""
|
|
html = """
|
|
<article class="result">
|
|
<h3><a href="https://x.com">T</a></h3>
|
|
<script>var evil = "ignore me";</script>
|
|
<style>.css { color: red; }</style>
|
|
<p class="content">real content</p>
|
|
</article>
|
|
"""
|
|
parser = SearXNGHTMLParser()
|
|
parser.feed(html)
|
|
assert len(parser.results) == 1
|
|
assert "evil" not in parser.results[0]["content"]
|
|
assert "css" not in parser.results[0]["content"]
|
|
assert parser.results[0]["content"] == "real content"
|
|
|
|
|
|
def test_parser_multiple_results():
|
|
html = """
|
|
<article class="result"><h3><a href="https://a.com">A</a></h3></article>
|
|
<article class="result"><h3><a href="https://b.com">B</a></h3></article>
|
|
<article class="result"><h3><a href="https://c.com">C</a></h3></article>
|
|
"""
|
|
parser = SearXNGHTMLParser()
|
|
parser.feed(html)
|
|
assert len(parser.results) == 3
|
|
assert [r["title"] for r in parser.results] == ["A", "B", "C"]
|
|
|
|
|
|
def test_parser_empty_article_skipped():
|
|
"""An <article> with neither title nor url is dropped."""
|
|
html = '<article class="result"><p class="content">no title or url</p></article>'
|
|
parser = SearXNGHTMLParser()
|
|
parser.feed(html)
|
|
assert len(parser.results) == 0
|
|
|
|
|
|
def test_parser_suggestions():
|
|
"""<div id="suggestions"> with <a> tags populates suggestions list."""
|
|
html = """
|
|
<div id="suggestions">
|
|
<a>first suggestion</a>
|
|
<a>second suggestion</a>
|
|
</div>
|
|
"""
|
|
parser = SearXNGHTMLParser()
|
|
parser.feed(html)
|
|
assert "first suggestion" in parser.suggestions
|
|
assert "second suggestion" in parser.suggestions
|
|
|
|
|
|
def test_parser_suggestions_class():
|
|
"""class="suggestion" also triggers suggestion capture."""
|
|
html = '<div class="suggestion"><a>via class</a></div>'
|
|
parser = SearXNGHTMLParser()
|
|
parser.feed(html)
|
|
assert "via class" in parser.suggestions
|
|
|
|
|
|
def test_parser_answer_box():
|
|
"""<div class="answer"> or id="answer" populates answers list."""
|
|
html = '<div class="answer">The answer is 42</div>'
|
|
parser = SearXNGHTMLParser()
|
|
parser.feed(html)
|
|
assert len(parser.answers) == 1
|
|
assert "42" in parser.answers[0]
|
|
|
|
|
|
def test_parser_answer_by_id():
|
|
html = '<div id="answer">42</div>'
|
|
parser = SearXNGHTMLParser()
|
|
parser.feed(html)
|
|
assert len(parser.answers) == 1
|
|
|
|
|
|
# ----- parse_html_results -----
|
|
|
|
def test_parse_html_results_shape():
|
|
"""parse_html_results returns a dict with all expected keys."""
|
|
html = """
|
|
<article class="result"><h3><a href="https://x.com">T</a></h3>
|
|
<p class="content">c</p></article>
|
|
<div id="suggestions"><a>s1</a></div>
|
|
<div class="answer">a1</div>
|
|
"""
|
|
r = parse_html_results(html, query="test")
|
|
assert r["query"] == "test"
|
|
assert len(r["results"]) == 1
|
|
assert r["suggestions"] == ["s1"]
|
|
assert r["answers"] == ["a1"]
|
|
assert r["corrections"] == []
|
|
assert r["infoboxes"] == []
|
|
assert r["unresponsive_engines"] == []
|
|
assert r["_fallback"] == "html"
|
|
assert r["number_of_results"] == 1
|
|
|
|
|
|
def test_parse_html_results_empty():
|
|
"""Empty HTML yields an empty-but-well-shaped result."""
|
|
r = parse_html_results("", query="q")
|
|
assert r["results"] == []
|
|
assert r["query"] == "q"
|
|
|
|
|
|
# ----- search_html -----
|
|
|
|
def _mock_urlopen(data: bytes, content_type="text/html"):
|
|
resp = MagicMock()
|
|
resp.read.return_value = data
|
|
resp.headers = {"Content-Type": content_type}
|
|
resp.__enter__.return_value = resp
|
|
resp.__exit__.return_value = None
|
|
return resp
|
|
|
|
|
|
def test_search_html_success():
|
|
"""search_html fetches and parses an HTML results page."""
|
|
html = """
|
|
<article class="result"><h3><a href="https://r.com">R</a></h3>
|
|
<p class="content">snippet</p></article>
|
|
"""
|
|
with patch("urllib.request.urlopen",
|
|
return_value=_mock_urlopen(html.encode())):
|
|
r = search_html("https://s.example.com", {"q": "test"})
|
|
assert r["results"][0]["title"] == "R"
|
|
assert r["results"][0]["url"] == "https://r.com"
|
|
|
|
|
|
def test_search_html_strips_format_param():
|
|
"""search_html must not pass format=json to the HTML endpoint."""
|
|
captured_req = {}
|
|
|
|
def _capture(req, timeout=None):
|
|
captured_req["url"] = req.full_url
|
|
return _mock_urlopen(b"<article class='result'></article>")
|
|
|
|
with patch("urllib.request.urlopen", side_effect=_capture):
|
|
search_html("https://s.example.com", {"q": "test", "format": "json"})
|
|
assert "format=json" not in captured_req["url"]
|
|
assert "q=test" in captured_req["url"]
|
|
|
|
|
|
def test_search_html_error_raises_runtime():
|
|
"""Network errors are wrapped in RuntimeError with context."""
|
|
err = urllib.error.URLError("connection refused")
|
|
with patch("urllib.request.urlopen", side_effect=err):
|
|
try:
|
|
search_html("https://s.example.com", {"q": "test"})
|
|
assert False, "should raise"
|
|
except RuntimeError as e:
|
|
assert "HTML search failed" in str(e)
|
|
|
|
|
|
# ----- search_single -----
|
|
|
|
def test_search_single_prefers_json():
|
|
"""When JSON works, search_single returns JSON results directly."""
|
|
import json as json_mod
|
|
payload = json_mod.dumps({"results": [{"title": "json", "url": "https://j.com"}]})
|
|
with patch("urllib.request.urlopen",
|
|
return_value=_mock_urlopen(payload.encode(),
|
|
content_type="application/json")):
|
|
r = search_single("https://s.example.com", {"q": "t", "format": "json"})
|
|
assert r["results"][0]["title"] == "json"
|
|
|
|
|
|
def test_search_single_falls_back_to_html():
|
|
"""When JSON returns None, search_single falls back to HTML parsing."""
|
|
html = """
|
|
<article class="result"><h3><a href="https://h.com">html</a></h3></article>
|
|
"""
|
|
# First call returns HTML (JSON unsupported), second call (HTML search) also HTML
|
|
with patch("urllib.request.urlopen",
|
|
return_value=_mock_urlopen(html.encode())):
|
|
r = search_single("https://s.example.com", {"q": "t", "format": "json"})
|
|
assert r["results"][0]["title"] == "html"
|
|
assert r.get("_fallback") == "html"
|
|
|
|
|
|
def test_search_single_html_fallback_marks_fallback():
|
|
"""HTML fallback path sets _fallback='html' in the result."""
|
|
html = '<article class="result"><h3><a href="https://x.com">x</a></h3></article>'
|
|
with patch("urllib.request.urlopen",
|
|
return_value=_mock_urlopen(html.encode())):
|
|
r = search_single("https://s.example.com", {"q": "t", "format": "json"})
|
|
assert r["_fallback"] == "html"
|