`` → ``| a | b |`` with separator row)
* fenced code blocks (```` → triple-backtick fences)
* inline code (```` → backticks)
* blockquotes (```` → ``> `` prefix per line)
* headings, lists, images, emphasis, links
"""
def __init__(self):
super().__init__()
self._output = []
self._skip_depth = 0
self._list_stack = [] # list of [type, counter] for ol numbering
self._pending_indent = "" # preserved leading indent for li/dd
self._block_buffer = ""
self._link_href = None
self._link_text = []
self._in_link = False
self._in_pre = False
self._pre_content = []
self._heading_level = 0
self._block_empty = True
# Blockquote support
self._in_blockquote = False
# Table support (GFM)
self._in_table = False
self._table_rows = [] # list of (cells_list, is_header)
self._current_row = None
self._current_cell = None
self._in_cell = False
def _append_inline(self, text: str):
"""Append inline markup, routing to cell/link/block buffer.
In cell context, fragments go to ``_current_cell`` so table cell text
accumulates correctly. In link context, they go to ``_link_text``
and are collapsed at ```` time. Otherwise they append to the
block buffer; whitespace is normalized at ``_flush_block`` time.
"""
if self._in_cell:
self._current_cell.append(text)
elif self._in_link:
self._link_text.append(text)
else:
self._block_buffer += text
self._block_empty = False
def _flush_block(self):
t = self._block_buffer
# Normalize whitespace: collapse runs of spaces/tabs/newlines
t = re.sub(r'\s+', ' ', t).strip()
if t:
if self._in_blockquote:
# Prefix each line with "> " for markdown blockquote syntax
t = "\n".join(
("> " + line) if line.strip() else ">"
for line in t.split("\n")
)
if self._pending_indent:
self._output.append(self._pending_indent + t)
else:
self._output.append(t)
self._block_buffer = ""
self._block_empty = True
self._pending_indent = ""
def _emit_table(self):
"""Emit a GFM table from accumulated rows.
The first row becomes the header; a ``| --- | --- |`` separator row
follows; remaining rows become the body. Cells are padded to the
header width so the table renders correctly in strict GFM parsers.
"""
if not self._table_rows:
return
header_row, _ = self._table_rows[0]
body_rows = self._table_rows[1:]
if not header_row:
return
ncols = len(header_row)
self._output.append("| " + " | ".join(header_row) + " |")
self._output.append("| " + " | ".join("---" for _ in range(ncols)) + " |")
for row, _ in body_rows:
# Pad short rows; truncate long rows to header width
while len(row) < ncols:
row.append("")
self._output.append("| " + " | ".join(row[:ncols]) + " |")
self._output.append("")
def handle_starttag(self, tag, attrs):
tag_lower = tag.lower()
attrs_dict = dict(attrs)
if tag_lower in ("script", "style", "nav", "footer", "header",
"noscript", "iframe", "svg", "canvas", "template"):
self._skip_depth += 1
return
if self._skip_depth > 0:
self._skip_depth += 1
return
# Table structural elements are handled up-front so cell content
# routing (via _in_cell) takes effect before any other tag handler.
if tag_lower == "table":
self._flush_block()
self._in_table = True
self._table_rows = []
return
if self._in_table:
if tag_lower == "tr":
self._current_row = []
return
elif tag_lower in ("th", "td"):
self._in_cell = True
self._current_cell = []
return
elif tag_lower in ("thead", "tbody", "tfoot"):
return # container only — rows/cells drive the output
# Other tags inside cells (a/strong/em/code/br) fall through
# to normal handling; _append_inline routes them to _current_cell.
if tag_lower in ("p", "div", "section"):
self._flush_block()
elif tag_lower == "blockquote":
self._flush_block()
self._in_blockquote = True
elif tag_lower == "br":
self._append_inline("\n")
elif tag_lower in ("h1", "h2", "h3", "h4", "h5", "h6"):
self._flush_block()
self._heading_level = int(tag_lower[1])
elif tag_lower == "pre":
self._flush_block()
self._in_pre = True
self._pre_content = []
elif tag_lower == "code":
# Inline code. is handled by the pre path, which
# captures raw text verbatim — so only emit backticks when
# we're NOT inside a pre block.
if not self._in_pre:
self._append_inline("`")
elif tag_lower in ("ul", "ol"):
self._list_stack.append([tag_lower, 0])
elif tag_lower == "li":
self._flush_block()
depth = max(0, len(self._list_stack) - 1)
self._pending_indent = " " * depth
if self._list_stack and self._list_stack[-1][0] == "ol":
self._list_stack[-1][1] += 1
marker = f"{self._list_stack[-1][1]}. "
else:
marker = "- "
self._block_buffer = marker
self._block_empty = False
elif tag_lower == "dt":
self._flush_block()
self._block_buffer = "**"
self._block_empty = False
elif tag_lower == "dd":
self._flush_block()
self._pending_indent = " "
self._block_buffer = ""
self._block_empty = True
elif tag_lower == "a":
self._in_link = True
self._link_href = attrs_dict.get("href", "")
self._link_text = []
elif tag_lower in ("strong", "b"):
self._append_inline("**")
elif tag_lower in ("em", "i"):
self._append_inline("*")
elif tag_lower == "img":
alt = attrs_dict.get("alt", "")
src = attrs_dict.get("src", "")
if alt or src:
self._append_inline(f"")
def handle_endtag(self, tag):
tag_lower = tag.lower()
if self._skip_depth > 0:
self._skip_depth -= 1
return
if tag_lower == "pre":
self._in_pre = False
code = "\n".join(self._pre_content)
self._output.append(f"```\n{code}\n```")
self._pre_content = []
self._block_empty = True
elif tag_lower in ("h1", "h2", "h3", "h4", "h5", "h6"):
prefix = "#" * self._heading_level
self._output.append(f"\n{prefix} {self._block_buffer.strip()}\n")
self._block_buffer = ""
self._heading_level = 0
self._block_empty = True
elif tag_lower in ("th", "td"):
# Collapse cell content to a single line (newlines from
# would break the GFM table row). Pipe chars are escaped to
# avoid prematurely terminating cells.
cell_text = " ".join(self._current_cell).strip()
cell_text = re.sub(r'\s+', ' ', cell_text)
cell_text = cell_text.replace("|", "\\|")
if self._current_row is not None:
self._current_row.append(cell_text)
self._in_cell = False
self._current_cell = None
elif tag_lower == "tr":
if self._current_row is not None:
is_header = False # GFM doesn't distinguish; first row is header
self._table_rows.append((self._current_row, is_header))
self._current_row = None
elif tag_lower == "table":
self._emit_table()
self._in_table = False
self._table_rows = []
self._current_row = None
self._current_cell = None
self._in_cell = False
elif tag_lower == "blockquote":
self._flush_block()
self._in_blockquote = False
self._output.append("")
elif tag_lower in ("p", "div", "section"):
self._flush_block()
self._output.append("")
elif tag_lower in ("ul", "ol"):
if self._list_stack:
self._list_stack.pop()
self._output.append("")
elif tag_lower == "li":
self._flush_block()
elif tag_lower == "dt":
self._block_buffer = self._block_buffer.rstrip() + "**"
self._flush_block()
elif tag_lower == "dd":
self._flush_block()
elif tag_lower == "a":
if self._in_link:
link_text = " ".join("".join(self._link_text).split())
if link_text and self._link_href:
rendered = f"[{link_text}]({self._link_href})"
elif self._link_href:
rendered = f"<{self._link_href}>"
else:
rendered = ""
if self._in_cell:
self._current_cell.append(rendered)
else:
self._block_buffer += rendered
self._block_empty = False
self._in_link = False
self._link_href = None
self._link_text = []
elif tag_lower == "code":
if not self._in_pre:
self._append_inline("`")
elif tag_lower in ("strong", "b"):
self._append_inline("**")
elif tag_lower in ("em", "i"):
self._append_inline("*")
def handle_data(self, data):
if self._skip_depth > 0:
return
if self._in_pre:
self._pre_content.append(data)
elif self._in_cell:
self._current_cell.append(data)
elif self._in_link:
self._link_text.append(data)
else:
# Don't strip — let _flush_block normalize whitespace
if data.strip(): # only skip purely whitespace nodes
self._block_buffer += data
self._block_empty = False
def get_markdown(self) -> str:
self._flush_block()
# Emit a table if one was left open (malformed HTML)
if self._in_table and self._table_rows:
self._emit_table()
text = "\n".join(self._output)
text = re.sub(r"\n{3,}", "\n\n", text)
return text.strip()
def html_to_markdown(html_content: str) -> str:
"""Convert HTML to Markdown using tree-based parser."""
# Quick strip of scripts/styles first
html_content = re.sub(r']*>.*?', '', html_content,
flags=re.DOTALL | re.IGNORECASE)
html_content = re.sub(r']*>.*?', '', html_content,
flags=re.DOTALL | re.IGNORECASE)
converter = MarkdownConverter()
converter.feed(html_content)
return converter.get_markdown()
# ----- HTTP Fetch -----
# RETRY_BACKOFF_BASE, FALLBACK_UAS, detect_charset and is_retryable_error are
# imported from common.py (shared with search.py for a consistent retry policy).
class _NoRedirectHandler(urllib.request.HTTPRedirectHandler):
"""urllib handler that returns the 3xx response instead of following it."""
def http_error_302(self, req, fp, code, msg, headers):
return fp
http_error_301 = http_error_303 = http_error_307 = http_error_308 = http_error_302
FetchResult = namedtuple(
"FetchResult",
["content", "content_type", "final_url", "truncated", "user_agent"],
)
def fetch_url(url: str, timeout: int = 15, user_agent: str = None,
encoding: str = None, auth_headers: dict = None,
max_retries: int = 3, max_size: int = None,
allow_redirects: bool = True) -> "FetchResult":
"""Fetch a URL with retry, encoding detection, UA fallback, and optional size limit.
Returns a :class:`FetchResult` namedtuple with fields:
``content`` (str), ``content_type`` (str), ``final_url`` (str),
``truncated`` (bool — True if ``max_size`` cut the response short),
``user_agent`` (str — the UA string that succeeded; useful for logging
whether a fallback UA was needed).
max_size=None means unlimited (full page). Set to e.g. 5242880 for a 5MB cap.
allow_redirects=False stops the client from following HTTP 3xx redirects.
"""
if user_agent is None:
user_agent = USER_AGENT
last_error = None
user_agents = [user_agent] + FALLBACK_UAS
for attempt in range(max_retries + 1):
ua = user_agents[min(attempt, len(user_agents) - 1)]
headers = {"User-Agent": ua}
if auth_headers:
headers.update(auth_headers)
try:
if _HAS_REQUESTS:
resp = _requests.get(url, timeout=timeout, headers=headers,
allow_redirects=allow_redirects, stream=True)
resp.raise_for_status()
# Read: unlimited if max_size is None, chunked with limit otherwise
if max_size is None:
raw = resp.content
truncated = False
else:
chunks = []
total = 0
for chunk in resp.iter_content(chunk_size=65536, decode_unicode=False):
if chunk:
chunks.append(chunk)
total += len(chunk)
if total > max_size:
break
raw = b"".join(chunks)
truncated = total > max_size
if encoding:
content = raw.decode(encoding)
else:
charset = detect_charset(raw, resp.headers.get("Content-Type", ""))
try:
content = raw.decode(charset)
except (UnicodeDecodeError, LookupError):
content = raw.decode("utf-8", errors="replace")
return FetchResult(content, resp.headers.get("Content-Type", ""),
resp.url, truncated, ua)
# stdlib fallback
req = urllib.request.Request(url, headers=headers)
if allow_redirects:
_opener = urllib.request.urlopen(req, timeout=timeout)
else:
_opener = urllib.request.build_opener(_NoRedirectHandler).open(
req, timeout=timeout)
with _opener as resp:
if max_size is None:
raw = resp.read()
truncated = False
else:
chunks = []
total = 0
while True:
chunk = resp.read(65536)
if not chunk:
break
chunks.append(chunk)
total += len(chunk)
if total > max_size:
break
raw = b"".join(chunks)
truncated = total > max_size
content_type = resp.headers.get("Content-Type", "")
final_url = resp.geturl()
if encoding:
charset = encoding
else:
charset = detect_charset(raw, content_type)
try:
content = raw.decode(charset)
except (UnicodeDecodeError, LookupError):
content = raw.decode("utf-8", errors="replace")
return FetchResult(content, content_type, final_url, truncated, ua)
except urllib.error.HTTPError as e:
last_error = e
if is_retryable_error(e) and attempt < max_retries:
delay = RETRY_BACKOFF_BASE * (2 ** attempt) + random.uniform(0, 1)
logger.info(f" Fetch retry {attempt+1}/{max_retries} ({e}) in {delay:.1f}s")
time.sleep(delay)
continue
raise RuntimeError(f"HTTP {e.code} for {url}")
except (urllib.error.URLError, OSError, TimeoutError) as e:
last_error = e
if attempt < max_retries:
delay = RETRY_BACKOFF_BASE * (2 ** attempt) + random.uniform(0, 1)
logger.info(f" Fetch retry {attempt+1}/{max_retries} ({e}) in {delay:.1f}s")
time.sleep(delay)
continue
raise RuntimeError(f"Request failed for {url}: {e}")
except Exception as e:
# requests backend: retry only on connection errors (no response)
# or transient 429/5xx; do NOT retry permanent errors like 404.
if _HAS_REQUESTS and isinstance(e, _requests.exceptions.RequestException):
last_error = e
status = getattr(getattr(e, "response", None), "status_code", None)
if (status is None or status in RETRYABLE_STATUS) and attempt < max_retries:
delay = RETRY_BACKOFF_BASE * (2 ** attempt) + random.uniform(0, 1)
logger.info(f" Fetch retry {attempt+1}/{max_retries} ({e}) in {delay:.1f}s")
time.sleep(delay)
continue
raise RuntimeError(f"Request failed for {url}: {e}")
raise RuntimeError(f"All {max_retries+1} attempts failed for {url}: {last_error}")
# ----- Main -----
def main():
parser = argparse.ArgumentParser(
description="Fetch a web page and extract readable content",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog=f"""searxng-cli v{VERSION}
Examples:
%(prog)s -u https://example.com extract clean text
%(prog)s -u https://example.com -e html raw HTML
%(prog)s -u https://example.com -e markdown markdown conversion
%(prog)s -u https://example.com -o page.txt save to file
%(prog)s -u https://example.cn -e text --encoding gbk force charset
""",
)
parser.add_argument("--url", "-u", required=True, help="URL to fetch")
parser.add_argument("--extract", "-e", choices=["text", "html", "markdown"],
default="text", help="Extraction mode (default: text)")
parser.add_argument("--timeout", "-t", type=int, default=15,
help="Request timeout in seconds (default: 15)")
parser.add_argument("--retries", type=int, default=3,
help="Max retries on transient errors (default: 3)")
parser.add_argument("--max-size", type=int, default=None, metavar="BYTES",
help="Max page size in bytes (default: unlimited). Set to 5242880 for 5MB cap.")
parser.add_argument("--user-agent", default=None,
help="Custom User-Agent header")
parser.add_argument("--encoding", default=None,
help="Force charset for decoding (e.g. gbk, shift_jis)")
parser.add_argument("--no-redirect", action="store_true",
help="Do not follow HTTP redirects")
parser.add_argument("--proxy", default=None, metavar="URL",
help="HTTP/HTTPS proxy URL (e.g. http://corp-proxy:8080). "
"Respects existing HTTP_PROXY/HTTPS_PROXY env vars when omitted.")
parser.add_argument("--output", "-o", default=None,
help="Save to file instead of stdout")
parser.add_argument("--auth-bearer", default=None, metavar="TOKEN",
help="Authorization: Bearer for authenticated endpoints")
parser.add_argument("--auth-bearer-file", default=None, metavar="FILE",
help="Read Bearer token from a file (first non-empty, non-# line). "
"Avoids leaving tokens in shell history.")
parser.add_argument("--auth-basic", default=None, metavar="USER:PASS",
help="Authorization: Basic base64(user:pass) for authenticated endpoints")
parser.add_argument("--auth-basic-file", default=None, metavar="FILE",
help="Read basic auth 'user:pass' from a file (first non-empty, non-# line). "
"Avoids leaving passwords in shell history. "
"Env var SEARXNG_BASIC_AUTH is also honored.")
parser.add_argument("--verbose", "-v", action="store_true", default=False,
help="Verbose output: show debug-level diagnostics on stderr")
parser.add_argument("--quiet", action="store_true", default=False,
help="Quiet output: suppress progress messages on stderr; "
"only warnings and errors are shown")
parser.add_argument("--version", action="version", version=f"searxng-cli v{VERSION}")
args = parser.parse_args()
setup_logging(verbose=args.verbose, quiet=args.quiet)
if not args.url.startswith(("http://", "https://")):
logger.error("Error: URL must start with http:// or https://")
sys.exit(1)
# Apply proxy via env vars so both urllib and requests honor it.
if args.proxy:
apply_proxy(args.proxy)
logger.info(f"Proxy: {args.proxy}")
try:
bearer_token = resolve_auth_bearer(args.auth_bearer, args.auth_bearer_file)
basic_auth = resolve_auth_basic(args.auth_basic, args.auth_basic_file)
except RuntimeError as e:
logger.error(f"Error: {e}")
sys.exit(1)
auth_headers = build_auth_headers(
bearer_token=bearer_token,
basic_auth=basic_auth,
)
if auth_headers:
auth_type = "Bearer" if bearer_token else "Basic"
logger.info(f"Auth: {auth_type} ***")
try:
result = fetch_url(
args.url, timeout=args.timeout, user_agent=args.user_agent,
encoding=args.encoding, auth_headers=auth_headers,
max_retries=args.retries, max_size=args.max_size,
allow_redirects=not args.no_redirect,
)
content, content_type, final_url = (
result.content, result.content_type, result.final_url,
)
except Exception as e:
logger.error(f"Error: {e}")
sys.exit(1)
if final_url != args.url:
logger.info(f"Redirected to: {final_url}")
is_html = ("html" in content_type.lower() or
content.strip().startswith("