Metona Team
0468dd4e9d
Multi-instance failover, exponential-backoff retry, SQLite cache, batch mode, domain filter, cross-engine dedup, result sorting, CSV export, structured logging, enhanced Markdown conversion, 155 pytest tests, Gitea Actions CI
193 lines
5.3 KiB
Python
193 lines
5.3 KiB
Python
"""Tests for scripts/fetch.py — MarkdownConverter and text extraction.
|
|
|
|
Covers: headings, links (incl. nested bold inside <a>), fenced code blocks,
|
|
inline code, blockquotes, unordered lists, GFM tables (incl. pipe escaping),
|
|
emphasis, image tags, script/style stripping, and the stdlib text extractor.
|
|
"""
|
|
from fetch import extract_with_stdlib, html_to_markdown
|
|
|
|
|
|
# ----- Basic structure -----
|
|
|
|
def test_markdown_basic_paragraph():
|
|
md = html_to_markdown("<p>Hello world</p>")
|
|
assert "Hello world" in md
|
|
|
|
|
|
def test_markdown_h1():
|
|
md = html_to_markdown("<h1>Title</h1>")
|
|
assert "# Title" in md
|
|
|
|
|
|
def test_markdown_h3():
|
|
md = html_to_markdown("<h3>Subtitle</h3>")
|
|
assert "### Subtitle" in md
|
|
|
|
|
|
def test_markdown_empty_input():
|
|
assert html_to_markdown("").strip() == ""
|
|
|
|
|
|
# ----- Links -----
|
|
|
|
def test_markdown_simple_link():
|
|
md = html_to_markdown('<a href="https://example.com">click</a>')
|
|
assert "[click](https://example.com)" in md
|
|
|
|
|
|
def test_markdown_link_with_nested_bold():
|
|
"""Bold text inside a link must survive — the tree-based parser handles
|
|
this where a regex approach would break."""
|
|
md = html_to_markdown('<a href="https://x.com"><b>bold link</b></a>')
|
|
assert "https://x.com" in md
|
|
assert "bold link" in md
|
|
|
|
|
|
# ----- Code -----
|
|
|
|
def test_markdown_fenced_code_block():
|
|
md = html_to_markdown("<pre>print('hi')</pre>")
|
|
assert "```" in md
|
|
assert "print('hi')" in md
|
|
|
|
|
|
def test_markdown_inline_code():
|
|
md = html_to_markdown("<p>use <code>pip</code> to install</p>")
|
|
assert "`pip`" in md
|
|
|
|
|
|
def test_markdown_pre_code_not_double_fenced():
|
|
"""<pre><code>...</code></pre> should render as one fence, not backticks-in-fence."""
|
|
md = html_to_markdown("<pre><code>x = 1</code></pre>")
|
|
assert "```\nx = 1\n```" in md
|
|
assert "`x = 1`" not in md # no inline backticks around the content
|
|
|
|
|
|
# ----- Blockquote / lists -----
|
|
|
|
def test_markdown_blockquote():
|
|
md = html_to_markdown("<blockquote>quoted text</blockquote>")
|
|
assert "> quoted text" in md
|
|
|
|
|
|
def test_markdown_unordered_list():
|
|
md = html_to_markdown("<ul><li>one</li><li>two</li></ul>")
|
|
assert "- one" in md
|
|
assert "- two" in md
|
|
|
|
|
|
# ----- Tables (GFM) -----
|
|
|
|
def test_markdown_table_basic():
|
|
html = "<table><tr><th>A</th><th>B</th></tr><tr><td>1</td><td>2</td></tr></table>"
|
|
md = html_to_markdown(html)
|
|
assert "| A | B |" in md
|
|
assert "| --- | --- |" in md
|
|
assert "| 1 | 2 |" in md
|
|
|
|
|
|
def test_markdown_table_pipe_escaped():
|
|
"""Pipe chars inside cells must be escaped so they don't break the row."""
|
|
html = "<table><tr><th>Col</th></tr><tr><td>a|b</td></tr></table>"
|
|
md = html_to_markdown(html)
|
|
assert "a\\|b" in md
|
|
|
|
|
|
# ----- Emphasis / images -----
|
|
|
|
def test_markdown_strong_emphasis():
|
|
md = html_to_markdown("<p><strong>bold</strong></p>")
|
|
assert "**bold**" in md
|
|
|
|
|
|
def test_markdown_em_italic():
|
|
md = html_to_markdown("<p><em>italic</em></p>")
|
|
assert "*italic*" in md
|
|
|
|
|
|
def test_markdown_image():
|
|
md = html_to_markdown('<img src="http://x.com/a.png" alt="pic">')
|
|
assert "" in md
|
|
|
|
|
|
# ---- Stripping non-content -----
|
|
|
|
def test_markdown_strips_script():
|
|
md = html_to_markdown("<script>alert(1)</script><p>visible</p>")
|
|
assert "alert" not in md
|
|
assert "visible" in md
|
|
|
|
|
|
def test_markdown_strips_style():
|
|
md = html_to_markdown("<style>body{color:red}</style><p>visible</p>")
|
|
assert "color" not in md
|
|
assert "visible" in md
|
|
|
|
|
|
# ----- Stdlib text extractor -----
|
|
|
|
def test_extract_with_stdlib_basic():
|
|
html = "<html><body><p>Hello</p><script>x</script></body></html>"
|
|
text = extract_with_stdlib(html)
|
|
assert "Hello" in text
|
|
assert "x" not in text # script content excluded
|
|
|
|
|
|
def test_extract_with_stdlib_skips_nav():
|
|
html = "<nav>menu</nav><article>content</article>"
|
|
text = extract_with_stdlib(html)
|
|
assert "content" in text
|
|
assert "menu" not in text
|
|
|
|
|
|
# ----- Ordered / nested lists -----
|
|
|
|
def test_markdown_ordered_list_numbered():
|
|
md = html_to_markdown("<ol><li>first</li><li>second</li><li>third</li></ol>")
|
|
assert "1. first" in md
|
|
assert "2. second" in md
|
|
assert "3. third" in md
|
|
|
|
|
|
def test_markdown_nested_ordered_list():
|
|
html = "<ol><li>outer1<ol><li>inner1</li><li>inner2</li></ol></li><li>outer2</li></ol>"
|
|
md = html_to_markdown(html)
|
|
assert "1. outer1" in md
|
|
assert " 1. inner1" in md
|
|
assert " 2. inner2" in md
|
|
assert "2. outer2" in md
|
|
|
|
|
|
def test_markdown_nested_unordered_list():
|
|
html = "<ul><li>outer<ul><li>inner</li></ul></li></ul>"
|
|
md = html_to_markdown(html)
|
|
assert "- outer" in md
|
|
assert " - inner" in md
|
|
|
|
|
|
def test_markdown_mixed_nested_list():
|
|
"""<ul> containing <ol> — markers must match list type at each depth."""
|
|
html = "<ul><li>item<ol><li>sub1</li><li>sub2</li></ol></li></ul>"
|
|
md = html_to_markdown(html)
|
|
assert "- item" in md
|
|
assert " 1. sub1" in md
|
|
assert " 2. sub2" in md
|
|
|
|
|
|
# ----- Definition lists (<dl>/<dt>/<dd>) -----
|
|
|
|
def test_markdown_definition_list():
|
|
html = "<dl><dt>Term</dt><dd>Definition</dd></dl>"
|
|
md = html_to_markdown(html)
|
|
assert "**Term**" in md
|
|
assert " Definition" in md
|
|
|
|
|
|
def test_markdown_definition_list_multiple():
|
|
html = "<dl><dt>T1</dt><dd>D1</dd><dt>T2</dt><dd>D2</dd></dl>"
|
|
md = html_to_markdown(html)
|
|
assert "**T1**" in md
|
|
assert "**T2**" in md
|
|
assert " D1" in md
|
|
assert " D2" in md
|