Initial commit: SearXNG CLI Toolkit v1.6.0

Multi-instance failover, exponential-backoff retry, SQLite cache, batch mode, domain filter, cross-engine dedup, result sorting, CSV export, structured logging, enhanced Markdown conversion, 155 pytest tests, Gitea Actions CI
2026-08-01 17:02:34 +08:00
commit 0468dd4e9d
17 changed files with 4791 additions and 0 deletions
@@ -0,0 +1,192 @@
+"""Tests for scripts/fetch.py — MarkdownConverter and text extraction.
+
+Covers: headings, links (incl. nested bold inside <a>), fenced code blocks,
+inline code, blockquotes, unordered lists, GFM tables (incl. pipe escaping),
+emphasis, image tags, script/style stripping, and the stdlib text extractor.
+"""
+from fetch import extract_with_stdlib, html_to_markdown
+
+
+# ----- Basic structure -----
+
+def test_markdown_basic_paragraph():
+    md = html_to_markdown("<p>Hello world</p>")
+    assert "Hello world" in md
+
+
+def test_markdown_h1():
+    md = html_to_markdown("<h1>Title</h1>")
+    assert "# Title" in md
+
+
+def test_markdown_h3():
+    md = html_to_markdown("<h3>Subtitle</h3>")
+    assert "### Subtitle" in md
+
+
+def test_markdown_empty_input():
+    assert html_to_markdown("").strip() == ""
+
+
+# ----- Links -----
+
+def test_markdown_simple_link():
+    md = html_to_markdown('<a href="https://example.com">click</a>')
+    assert "[click](https://example.com)" in md
+
+
+def test_markdown_link_with_nested_bold():
+    """Bold text inside a link must survive — the tree-based parser handles
+    this where a regex approach would break."""
+    md = html_to_markdown('<a href="https://x.com"><b>bold link</b></a>')
+    assert "https://x.com" in md
+    assert "bold link" in md
+
+
+# ----- Code -----
+
+def test_markdown_fenced_code_block():
+    md = html_to_markdown("<pre>print('hi')</pre>")
+    assert "```" in md
+    assert "print('hi')" in md
+
+
+def test_markdown_inline_code():
+    md = html_to_markdown("<p>use <code>pip</code> to install</p>")
+    assert "`pip`" in md
+
+
+def test_markdown_pre_code_not_double_fenced():
+    """<pre><code>...</code></pre> should render as one fence, not backticks-in-fence."""
+    md = html_to_markdown("<pre><code>x = 1</code></pre>")
+    assert "```\nx = 1\n```" in md
+    assert "`x = 1`" not in md  # no inline backticks around the content
+
+
+# ----- Blockquote / lists -----
+
+def test_markdown_blockquote():
+    md = html_to_markdown("<blockquote>quoted text</blockquote>")
+    assert "> quoted text" in md
+
+
+def test_markdown_unordered_list():
+    md = html_to_markdown("<ul><li>one</li><li>two</li></ul>")
+    assert "- one" in md
+    assert "- two" in md
+
+
+# ----- Tables (GFM) -----
+
+def test_markdown_table_basic():
+    html = "<table><tr><th>A</th><th>B</th></tr><tr><td>1</td><td>2</td></tr></table>"
+    md = html_to_markdown(html)
+    assert "| A | B |" in md
+    assert "| --- | --- |" in md
+    assert "| 1 | 2 |" in md
+
+
+def test_markdown_table_pipe_escaped():
+    """Pipe chars inside cells must be escaped so they don't break the row."""
+    html = "<table><tr><th>Col</th></tr><tr><td>a|b</td></tr></table>"
+    md = html_to_markdown(html)
+    assert "a\\|b" in md
+
+
+# ----- Emphasis / images -----
+
+def test_markdown_strong_emphasis():
+    md = html_to_markdown("<p><strong>bold</strong></p>")
+    assert "**bold**" in md
+
+
+def test_markdown_em_italic():
+    md = html_to_markdown("<p><em>italic</em></p>")
+    assert "*italic*" in md
+
+
+def test_markdown_image():
+    md = html_to_markdown('<img src="http://x.com/a.png" alt="pic">')
+    assert "![pic](http://x.com/a.png)" in md
+
+
+# ---- Stripping non-content -----
+
+def test_markdown_strips_script():
+    md = html_to_markdown("<script>alert(1)</script><p>visible</p>")
+    assert "alert" not in md
+    assert "visible" in md
+
+
+def test_markdown_strips_style():
+    md = html_to_markdown("<style>body{color:red}</style><p>visible</p>")
+    assert "color" not in md
+    assert "visible" in md
+
+
+# ----- Stdlib text extractor -----
+
+def test_extract_with_stdlib_basic():
+    html = "<html><body><p>Hello</p><script>x</script></body></html>"
+    text = extract_with_stdlib(html)
+    assert "Hello" in text
+    assert "x" not in text  # script content excluded
+
+
+def test_extract_with_stdlib_skips_nav():
+    html = "<nav>menu</nav><article>content</article>"
+    text = extract_with_stdlib(html)
+    assert "content" in text
+    assert "menu" not in text
+
+
+# ----- Ordered / nested lists -----
+
+def test_markdown_ordered_list_numbered():
+    md = html_to_markdown("<ol><li>first</li><li>second</li><li>third</li></ol>")
+    assert "1. first" in md
+    assert "2. second" in md
+    assert "3. third" in md
+
+
+def test_markdown_nested_ordered_list():
+    html = "<ol><li>outer1<ol><li>inner1</li><li>inner2</li></ol></li><li>outer2</li></ol>"
+    md = html_to_markdown(html)
+    assert "1. outer1" in md
+    assert "  1. inner1" in md
+    assert "  2. inner2" in md
+    assert "2. outer2" in md
+
+
+def test_markdown_nested_unordered_list():
+    html = "<ul><li>outer<ul><li>inner</li></ul></li></ul>"
+    md = html_to_markdown(html)
+    assert "- outer" in md
+    assert "  - inner" in md
+
+
+def test_markdown_mixed_nested_list():
+    """<ul> containing <ol> — markers must match list type at each depth."""
+    html = "<ul><li>item<ol><li>sub1</li><li>sub2</li></ol></li></ul>"
+    md = html_to_markdown(html)
+    assert "- item" in md
+    assert "  1. sub1" in md
+    assert "  2. sub2" in md
+
+
+# ----- Definition lists (<dl>/<dt>/<dd>) -----
+
+def test_markdown_definition_list():
+    html = "<dl><dt>Term</dt><dd>Definition</dd></dl>"
+    md = html_to_markdown(html)
+    assert "**Term**" in md
+    assert "  Definition" in md
+
+
+def test_markdown_definition_list_multiple():
+    html = "<dl><dt>T1</dt><dd>D1</dd><dt>T2</dt><dd>D2</dd></dl>"
+    md = html_to_markdown(html)
+    assert "**T1**" in md
+    assert "**T2**" in md
+    assert "  D1" in md
+    assert "  D2" in md