thzxx
d9a08716bc
Bug 1: Wayback 兜底未触发 (search.py fetch_page) - 根因: Cloudflare JS 质询页常返回 HTTP 200 (非 403), 主抓取 result 非 None - _should_try_fallback 在 result 非 None 时直接返回 False, 跳过兜底 - 反爬检测在兜底判断之后执行, 错过兜底入口 - 修复: 反爬检测提前到兜底判断之前, anti_bot_detected=True 也触发 Wayback - Wayback 结果重新做反爬检测 (防御性) Bug 2: WAF 指纹库误判正常内容 (search.py WAF_FINGERPRINTS) - 根因: 裸公司名 (cloudflare/akamai) 和宽泛词 (captcha/challenge/dd-) 做全文匹配 - DataCamp 文章引用 cloudflare.com 文档链接 -> 误判为 cloudflare WAF - 'coding challenges' 正常内容 -> 误判为 generic 反爬 - Wayback 归档正文被误判, 兜底返回的有效内容被丢弃 - 修复: 移除裸公司名和宽泛词, 改用技术标识符 (cf-ray/incap_ses/bm_sz 等) - 通用文案用完整短语 (please complete the captcha) 替代单词 - 增加 <title> 标签精准检测 (反爬页 title 是特征文案, 误判率极低) - 新增 Anubis 反爬系统检测 (anubis_challenge/miserere) 真实测试验证 (search.metona.cn 实例): - v2.0.0: --fetch 3 全部失败 (3 ERR: cloudflare/generic, Wayback 未触发) - v2.0.1: --fetch 3 全部成功 (3 OK: 38100/93442/1945 chars, UA 轮换绕过 Cloudflare) 测试: 458 个全部通过 (新增 7 个测试覆盖修复行为)
2287 lines
100 KiB
Python
2287 lines
100 KiB
Python
#!/usr/bin/env python3
|
||
"""Execute searches via a user-supplied SearXNG instance's JSON API.
|
||
|
||
Calls GET/POST /search?q=...&format=json on the chosen instance.
|
||
Auto-retries on failure with exponential backoff + tries next instance.
|
||
Falls back to HTML scraping if the instance blocks JSON output.
|
||
|
||
Instance URLs are REQUIRED (see --instance / SEARXNG_INSTANCE / config file).
|
||
"""
|
||
|
||
import argparse
|
||
import json
|
||
import logging
|
||
import os
|
||
import random
|
||
import re
|
||
import sys
|
||
import threading
|
||
import time
|
||
import urllib.error
|
||
import urllib.parse
|
||
import urllib.request
|
||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||
from html.parser import HTMLParser
|
||
from pathlib import Path
|
||
|
||
# Allow running standalone from any working directory
|
||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||
from _config import SCHEMA_VERSION, USER_AGENT, VERSION
|
||
from common import (
|
||
RETRYABLE_STATUS,
|
||
RETRY_BACKOFF_BASE,
|
||
MAX_RETRIES,
|
||
RECOVERY_HINTS,
|
||
apply_proxy,
|
||
build_auth_headers,
|
||
classify_error,
|
||
emit_progress,
|
||
force_utf8_stdout,
|
||
resolve_auth_basic,
|
||
resolve_auth_bearer,
|
||
set_progress_enabled,
|
||
setup_logging,
|
||
E_CONFIG,
|
||
E_AUTH,
|
||
E_NETWORK,
|
||
E_RATE_LIMIT,
|
||
E_PARSE,
|
||
E_EMPTY,
|
||
E_INPUT,
|
||
E_INTERNAL,
|
||
)
|
||
from fetch import extract_text, fetch_url
|
||
import cache as cache_module
|
||
|
||
logger = logging.getLogger("searxng.search")
|
||
|
||
|
||
# ----- Auth helpers -----
|
||
|
||
def _merge_headers(*header_dicts: dict) -> dict:
|
||
"""Merge multiple header dicts, with later dicts overriding earlier ones."""
|
||
result = {}
|
||
for d in header_dicts:
|
||
if d:
|
||
result.update(d)
|
||
return result
|
||
|
||
|
||
# ----- HTML fallback: result extractor -----
|
||
|
||
class SearXNGHTMLParser(HTMLParser):
|
||
"""Extract search results + metadata from SearXNG's simple theme HTML.
|
||
|
||
Matches:
|
||
<article class="result result-default category-general">
|
||
<a href="..." class="url_header"> ... </a>
|
||
<h3><a href="...">Title</a></h3>
|
||
<p class="content"> Snippet... </p>
|
||
<time datetime="2024-01-15T10:30:00">...</time>
|
||
</article>
|
||
<div id="suggestions"><a>...</a></div>
|
||
<div class="answer">...</div>
|
||
|
||
The ``published_date`` field is populated from the ``<time>`` tag's
|
||
``datetime`` attribute (preferred) or its text content (fallback),
|
||
mirroring the field SearXNG's JSON API exposes.
|
||
"""
|
||
|
||
def __init__(self):
|
||
super().__init__()
|
||
self.results = []
|
||
self.suggestions = []
|
||
self.answers = []
|
||
self.infoboxes = []
|
||
|
||
self._current = None
|
||
self._in_article = False
|
||
self._in_h3 = False
|
||
self._in_content = False
|
||
self._in_time = False
|
||
self._in_suggestions = False
|
||
self._in_answer = False
|
||
self._text_buf = []
|
||
self._skip_depth = 0
|
||
|
||
def handle_starttag(self, tag, attrs):
|
||
attrs_dict = dict(attrs)
|
||
classes = attrs_dict.get("class", "").split()
|
||
tag_id = attrs_dict.get("id", "")
|
||
|
||
if self._skip_depth > 0:
|
||
self._skip_depth += 1
|
||
return
|
||
|
||
# Results
|
||
if tag == "article" and "result" in classes:
|
||
self._in_article = True
|
||
self._current = {"title": "", "url": "", "content": "",
|
||
"engine": "", "published_date": ""}
|
||
|
||
if self._in_article:
|
||
if tag == "a" and "url_header" in classes:
|
||
href = attrs_dict.get("href", "")
|
||
if href:
|
||
self._current["url"] = href
|
||
elif tag == "h3":
|
||
self._in_h3 = True
|
||
self._text_buf = []
|
||
elif tag == "a" and self._in_h3:
|
||
# h3 内的 <a href> 作为 url 的 fallback：某些 SearXNG 主题
|
||
# 不使用 url_header class，URL 仅出现在 h3 的链接中。
|
||
# url_header 优先（上面已处理），此处仅填充空 url。
|
||
href = attrs_dict.get("href", "")
|
||
if href and not self._current.get("url"):
|
||
self._current["url"] = href
|
||
elif tag == "p" and "content" in classes:
|
||
self._in_content = True
|
||
self._text_buf = []
|
||
elif tag == "time":
|
||
# SearXNG puts the ISO timestamp in the datetime attribute
|
||
dt = attrs_dict.get("datetime", "").strip()
|
||
if dt and self._current is not None:
|
||
self._current["published_date"] = dt
|
||
# Still collect text as a fallback for instances that omit
|
||
# the datetime attribute but render a human-readable date.
|
||
self._in_time = True
|
||
self._text_buf = []
|
||
elif tag in ("script", "style"):
|
||
self._skip_depth += 1
|
||
|
||
# Suggestions: <div id="suggestions"> or class containing "suggestion"
|
||
if tag_id == "suggestions" or "suggestion" in classes:
|
||
self._in_suggestions = True
|
||
self._text_buf = []
|
||
if self._in_suggestions and tag == "a":
|
||
self._text_buf = []
|
||
|
||
# Answer boxes
|
||
if "answer" in classes or tag_id == "answer":
|
||
self._in_answer = True
|
||
self._text_buf = []
|
||
|
||
def handle_endtag(self, tag):
|
||
if self._skip_depth > 0:
|
||
self._skip_depth -= 1
|
||
return
|
||
|
||
if self._in_article and tag == "article":
|
||
if self._current:
|
||
self._current["title"] = self._current.get("title", "").strip()
|
||
self._current["content"] = self._current.get("content", "").strip()
|
||
if self._current.get("title") or self._current.get("url"):
|
||
self.results.append(self._current)
|
||
self._in_article = False
|
||
self._current = None
|
||
elif self._in_h3 and tag == "h3":
|
||
if self._current:
|
||
self._current["title"] = " ".join(self._text_buf).strip()
|
||
self._in_h3 = False
|
||
elif self._in_content and tag == "p":
|
||
if self._current:
|
||
self._current["content"] = " ".join(self._text_buf).strip()
|
||
self._in_content = False
|
||
elif self._in_time and tag == "time":
|
||
# Fallback: use text content only if the datetime attribute
|
||
# wasn't already captured at starttag time.
|
||
if self._current and not self._current.get("published_date"):
|
||
text = " ".join(self._text_buf).strip()
|
||
if text:
|
||
self._current["published_date"] = text
|
||
self._in_time = False
|
||
|
||
if self._in_suggestions and tag == "div":
|
||
self._in_suggestions = False
|
||
if self._in_answer and tag == "div":
|
||
if self._text_buf:
|
||
self.answers.append(" ".join(self._text_buf).strip())
|
||
self._in_answer = False
|
||
|
||
def handle_data(self, data):
|
||
if self._skip_depth > 0:
|
||
return
|
||
if self._in_h3:
|
||
self._text_buf.append(data)
|
||
elif self._in_content:
|
||
self._text_buf.append(data)
|
||
elif self._in_time:
|
||
self._text_buf.append(data)
|
||
elif self._in_suggestions:
|
||
stripped = data.strip()
|
||
if stripped:
|
||
self.suggestions.append(stripped)
|
||
elif self._in_answer:
|
||
self._text_buf.append(data)
|
||
|
||
|
||
def parse_html_results(html: str, query: str = "") -> dict:
|
||
"""Parse SearXNG HTML results page into structured dict."""
|
||
parser = SearXNGHTMLParser()
|
||
parser.feed(html)
|
||
return {
|
||
"query": query,
|
||
"number_of_results": len(parser.results),
|
||
"results": parser.results,
|
||
"answers": parser.answers,
|
||
"corrections": [], # rarely in simple theme
|
||
"suggestions": parser.suggestions,
|
||
"infoboxes": parser.infoboxes,
|
||
"unresponsive_engines": [],
|
||
"_fallback": "html",
|
||
}
|
||
|
||
|
||
# ----- Instance selection -----
|
||
|
||
def parse_instances(instance_arg: str) -> list:
|
||
"""Parse user-supplied instance URL(s).
|
||
|
||
Accepts a single URL or a comma-separated list. Each entry is normalized
|
||
(https:// prefix added if missing, trailing slash stripped). Multiple
|
||
instances enable multi-instance failover.
|
||
"""
|
||
urls = []
|
||
for raw in instance_arg.split(","):
|
||
u = raw.strip()
|
||
if not u:
|
||
continue
|
||
if not u.startswith(("http://", "https://")):
|
||
u = "https://" + u
|
||
urls.append(u.rstrip("/"))
|
||
return urls
|
||
|
||
|
||
def _normalize_csv(value: str) -> str:
|
||
"""Normalize a comma-separated list: strip each item, drop empties.
|
||
|
||
``"google, bing, brave"`` -> ``"google,bing,brave"`` so downstream params
|
||
match SearXNG's exact engine name requirements.
|
||
"""
|
||
return ",".join(p.strip() for p in value.split(",") if p.strip())
|
||
|
||
|
||
def _load_toml(path: Path) -> dict:
|
||
"""Load a TOML file with Python-version-aware backend selection.
|
||
|
||
* Python 3.11+ → stdlib ``tomllib``
|
||
* Python 3.8–3.10 → optional ``tomli`` backport (``pip install tomli``)
|
||
* Neither available → raise ``RuntimeError`` so the caller can fall back
|
||
to ``instances.txt`` gracefully.
|
||
|
||
This is required because the project claims Python 3.8+ support but
|
||
``tomllib`` only entered the stdlib in 3.11.
|
||
"""
|
||
try:
|
||
import tomllib # Python 3.11+
|
||
except ModuleNotFoundError:
|
||
try:
|
||
import tomli as tomllib # type: ignore[no-redef]
|
||
except ModuleNotFoundError:
|
||
raise RuntimeError(
|
||
"Reading .toml config requires Python 3.11+ or 'pip install tomli'"
|
||
)
|
||
with open(path, "rb") as f:
|
||
return tomllib.load(f)
|
||
|
||
|
||
def _read_instance_file(path: Path) -> list:
|
||
"""Read instance URLs from a config file.
|
||
|
||
Supports two formats:
|
||
* ``searxng.toml`` — ``instance = "url"`` or ``instances = ["a", "b"]``
|
||
under a ``[searxng]`` table (or at the top level).
|
||
Requires Python 3.11+ or the ``tomli`` backport.
|
||
* ``*.txt`` — one URL per line, comma-separated lists allowed,
|
||
``#`` starts a comment.
|
||
Returns an empty list if the file cannot be parsed.
|
||
"""
|
||
try:
|
||
if path.suffix == ".toml":
|
||
data = _load_toml(path)
|
||
table = data.get("searxng", data)
|
||
raw = table.get("instance") or table.get("instances")
|
||
if isinstance(raw, str):
|
||
return parse_instances(raw)
|
||
if isinstance(raw, list):
|
||
return [u.rstrip("/") if u.startswith(("http://", "https://"))
|
||
else "https://" + u.rstrip("/")
|
||
for u in raw if u and str(u).strip()]
|
||
return []
|
||
# Plain text: one URL per line (comma lists allowed), # comments
|
||
out = []
|
||
for line in path.read_text(encoding="utf-8").splitlines():
|
||
line = line.strip()
|
||
if not line or line.startswith("#"):
|
||
continue
|
||
out.extend(parse_instances(line))
|
||
return out
|
||
except RuntimeError as e:
|
||
# tomllib 缺失（Python 3.8-3.10 未装 tomli）是可恢复的——可改用
|
||
# instances.txt——但必须明确提示用户，而不是静默返回空列表让 main
|
||
# 报 "no instance resolved"，让用户困惑真正的失败原因。
|
||
msg = str(e).lower()
|
||
if "toml" in msg and ("3.11" in msg or "tomli" in msg):
|
||
logger.error(f"Cannot parse '{path}': {e} "
|
||
f"(consider 'pip install tomli' or use instances.txt)")
|
||
else:
|
||
logger.warning(f"Warning: cannot read instance file '{path}': {e}")
|
||
return []
|
||
except Exception as e:
|
||
logger.warning(f"Warning: cannot read instance file '{path}': {e}")
|
||
return []
|
||
|
||
|
||
def _windows_appdata_config_dir() -> Path:
|
||
"""Return the Windows APPDATA config directory, or a sentinel Path if unset.
|
||
|
||
On Windows, the conventional per-user app config directory is
|
||
``%APPDATA%`` (typically ``C:\\Users\\<user>\\AppData\\Roaming``).
|
||
On POSIX, this env var is unset and we return a sentinel
|
||
``Path("/__no_appdata__")`` which never exists on disk, so the caller
|
||
can unconditionally append it to the candidate list without polluting
|
||
Linux/macOS lookups.
|
||
|
||
Note: ``Path("")`` resolves to ``.`` (current directory) on Windows,
|
||
which DOES exist — so we must use an absolute sentinel path instead.
|
||
"""
|
||
appdata = os.environ.get("APPDATA", "")
|
||
if appdata:
|
||
return Path(appdata) / "searxng-cli"
|
||
# Sentinel: absolute path that never exists. Using "/" + unlikely name
|
||
# keeps it false on both POSIX and Windows (where "/" is the drive root).
|
||
return Path("/__no_appdata__")
|
||
|
||
|
||
def resolve_instances(cli_arg: str = None) -> list:
|
||
"""Resolve instance URLs from (in priority order):
|
||
|
||
1. ``--instance`` CLI flag (comma-separated list)
|
||
2. ``SEARXNG_INSTANCE`` environment variable (comma-separated list)
|
||
3. config file search order:
|
||
a. ``./searxng.toml``
|
||
b. ``~/.config/searxng-cli/searxng.toml``
|
||
c. ``%APPDATA%/searxng-cli/searxng.toml`` (Windows only)
|
||
d. ``./instances.txt``
|
||
e. ``~/.config/searxng-cli/instances.txt``
|
||
f. ``%APPDATA%/searxng-cli/instances.txt`` (Windows only)
|
||
|
||
Returns an empty list if no instance can be resolved.
|
||
"""
|
||
if cli_arg:
|
||
return parse_instances(cli_arg)
|
||
|
||
env = os.environ.get("SEARXNG_INSTANCE")
|
||
if env:
|
||
return parse_instances(env)
|
||
|
||
win_dir = _windows_appdata_config_dir()
|
||
candidates = [
|
||
Path.cwd() / "searxng.toml",
|
||
Path.home() / ".config" / "searxng-cli" / "searxng.toml",
|
||
win_dir / "searxng.toml",
|
||
Path.cwd() / "instances.txt",
|
||
Path.home() / ".config" / "searxng-cli" / "instances.txt",
|
||
win_dir / "instances.txt",
|
||
]
|
||
for p in candidates:
|
||
if p.exists():
|
||
urls = _read_instance_file(p)
|
||
if urls:
|
||
return urls
|
||
return []
|
||
|
||
|
||
def load_config(config_path: str = None) -> dict:
|
||
"""Load the full config dict from ``searxng.toml`` (if present).
|
||
|
||
If ``config_path`` is given (from ``--config``), only that file is
|
||
consulted. Otherwise the default search order applies:
|
||
1. ``./searxng.toml``
|
||
2. ``~/.config/searxng-cli/searxng.toml``
|
||
|
||
Returns the ``[searxng]`` table (or top-level table if no section),
|
||
which may contain any of these keys used as CLI defaults:
|
||
|
||
* ``instance`` / ``instances`` — instance URL(s)
|
||
* ``timeout``, ``max_retries`` — network tuning
|
||
* ``engines``, ``categories``, ``language`` — search scope
|
||
* ``safesearch``, ``time_range``, ``method`` — search behavior
|
||
* ``format`` — output format (json/brief/urls/csv)
|
||
* ``sort_by`` — result sort key
|
||
* ``proxy`` — proxy URL
|
||
* ``cache_ttl`` — cache TTL in minutes
|
||
|
||
Returns an empty dict if no config file exists or it cannot be parsed.
|
||
Only ``.toml`` files are consulted for full config; ``.txt`` files
|
||
only carry instance URLs (handled by :func:`resolve_instances`).
|
||
"""
|
||
if config_path:
|
||
p = Path(config_path)
|
||
if not p.exists():
|
||
logger.warning(f"Warning: config file '{p}' not found")
|
||
return {}
|
||
try:
|
||
data = _load_toml(p)
|
||
return data.get("searxng", data)
|
||
except Exception as e:
|
||
logger.warning(f"Warning: cannot read config '{p}': {e}")
|
||
return {}
|
||
win_dir = _windows_appdata_config_dir()
|
||
candidates = [
|
||
Path.cwd() / "searxng.toml",
|
||
Path.home() / ".config" / "searxng-cli" / "searxng.toml",
|
||
win_dir / "searxng.toml",
|
||
]
|
||
for p in candidates:
|
||
if p.exists():
|
||
try:
|
||
data = _load_toml(p)
|
||
return data.get("searxng", data)
|
||
except Exception as e:
|
||
logger.warning(f"Warning: cannot read config '{p}': {e}")
|
||
return {}
|
||
return {}
|
||
|
||
|
||
def _cfg_int(config: dict, key: str, default: int) -> int:
|
||
"""Read an int from config, tolerating str/int forms and bad values.
|
||
|
||
argparse's ``type=int`` only converts *command-line* strings; it does NOT
|
||
convert a ``default`` value that came in as a string from TOML. So a
|
||
``timeout = "15"`` in ``searxng.toml`` would leak through as a str and
|
||
break later numeric comparisons. This helper normalizes that.
|
||
"""
|
||
if key not in config:
|
||
return default
|
||
try:
|
||
return int(config[key])
|
||
except (TypeError, ValueError):
|
||
return default
|
||
|
||
|
||
def _cfg_float(config: dict, key: str, default: float) -> float:
|
||
"""Read a float from config, tolerating str/int/float forms. See _cfg_int."""
|
||
if key not in config:
|
||
return default
|
||
try:
|
||
return float(config[key])
|
||
except (TypeError, ValueError):
|
||
return default
|
||
|
||
|
||
# ----- Retry logic -----
|
||
|
||
def _retry_with_backoff(fn, max_retries: int = MAX_RETRIES, base_delay: float = RETRY_BACKOFF_BASE):
|
||
"""Call fn with exponential backoff + jitter on transient failures."""
|
||
last_error = None
|
||
for attempt in range(max_retries + 1):
|
||
try:
|
||
return fn()
|
||
except urllib.error.HTTPError as e:
|
||
if e.code in RETRYABLE_STATUS: # 403 (UA block) + 429 + 5xx
|
||
last_error = e
|
||
if attempt < max_retries:
|
||
delay = base_delay * (2 ** attempt) + random.uniform(0, 1)
|
||
logger.info(f" HTTP {e.code}, retrying in {delay:.1f}s... (attempt {attempt+1}/{max_retries})")
|
||
time.sleep(delay)
|
||
continue
|
||
raise
|
||
except (urllib.error.URLError, OSError) as e:
|
||
last_error = e
|
||
if attempt < max_retries:
|
||
delay = base_delay * (2 ** attempt) + random.uniform(0, 1)
|
||
logger.info(f" Connection error ({e}), retrying in {delay:.1f}s...")
|
||
time.sleep(delay)
|
||
continue
|
||
raise
|
||
raise last_error
|
||
|
||
|
||
# ----- Search execution -----
|
||
|
||
def search_json(instance: str, params: dict, method: str = "GET",
|
||
timeout: int = 15, auth_headers: dict = None) -> dict:
|
||
"""Execute search via JSON API. Returns None if JSON unsupported."""
|
||
query_string = urllib.parse.urlencode(params)
|
||
headers = _merge_headers({"User-Agent": USER_AGENT}, auth_headers)
|
||
|
||
if method.upper() == "POST":
|
||
data = query_string.encode("utf-8")
|
||
req = urllib.request.Request(f"{instance}/search", data=data, headers=headers, method="POST")
|
||
else:
|
||
req = urllib.request.Request(f"{instance}/search?{query_string}", headers=headers)
|
||
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||
raw = resp.read().decode("utf-8")
|
||
if raw.strip().startswith("{") or raw.strip().startswith("["):
|
||
return json.loads(raw)
|
||
# Got HTML — JSON unsupported
|
||
return None
|
||
except urllib.error.HTTPError as e:
|
||
# 404 = JSON endpoint truly absent → fall back to HTML scraping
|
||
if e.code == 404:
|
||
return None
|
||
# 403 / 401 = auth or IP issue → raise so it isn't silently masked
|
||
# by an HTML fallback that would just 403 again. Also surfaces
|
||
# instances that disable the JSON format via a 403 (rare but seen).
|
||
raise
|
||
|
||
|
||
def search_html(instance: str, params: dict, timeout: int = 15,
|
||
auth_headers: dict = None) -> dict:
|
||
"""Execute search via HTML scraping fallback."""
|
||
html_params = {k: v for k, v in params.items() if k != "format"}
|
||
query_string = urllib.parse.urlencode(html_params)
|
||
url = f"{instance}/search?{query_string}"
|
||
|
||
headers = _merge_headers({"User-Agent": USER_AGENT}, auth_headers)
|
||
req = urllib.request.Request(url, headers=headers)
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||
html = resp.read().decode("utf-8")
|
||
return parse_html_results(html, query=params.get("q", ""))
|
||
except Exception as e:
|
||
raise RuntimeError(f"HTML search failed for {instance}: {e}")
|
||
|
||
|
||
def search_single(instance: str, params: dict, method: str = "GET",
|
||
timeout: int = 15, auth_headers: dict = None) -> dict:
|
||
"""Execute one search attempt, preferring JSON with HTML fallback."""
|
||
result = search_json(instance, params, method=method, timeout=timeout,
|
||
auth_headers=auth_headers)
|
||
if result is not None:
|
||
return result
|
||
logger.warning(f"Warning: {instance} does not support format=json, falling back to HTML parsing")
|
||
return search_html(instance, params, timeout=timeout, auth_headers=auth_headers)
|
||
|
||
|
||
def search_multi(instance_urls: list, params: dict, method: str = "GET",
|
||
timeout: int = 15, retry_per: int = None,
|
||
auth_headers: dict = None, parallel: bool = True) -> dict:
|
||
"""Search across multiple instances, failing over on error.
|
||
|
||
With parallel=True (default, multi-instance only): every instance is
|
||
queried concurrently. The first *successful* result in the user's
|
||
original instance order is returned — this keeps output deterministic
|
||
(same input list always yields the same source) while drastically
|
||
speeding up failover when an early instance is down or slow.
|
||
|
||
With parallel=False (or a single instance): strictly sequential, one
|
||
request at a time, trying the next instance only after the current fails.
|
||
"""
|
||
if retry_per is None:
|
||
retry_per = MAX_RETRIES
|
||
instance_urls = [u.rstrip("/") for u in instance_urls]
|
||
|
||
# Single instance, or explicit serial mode: straightforward sequential path
|
||
if len(instance_urls) <= 1 or not parallel:
|
||
last_error = None
|
||
for instance in instance_urls:
|
||
logger.info(f"Trying {instance}...")
|
||
emit_progress("instance_try", url=instance, attempt=1)
|
||
start = time.time()
|
||
try:
|
||
def _do():
|
||
return search_single(instance, params, method=method,
|
||
timeout=timeout, auth_headers=auth_headers)
|
||
result = _retry_with_backoff(_do, max_retries=retry_per)
|
||
emit_progress("instance_ok", url=instance,
|
||
latency=round(time.time() - start, 3),
|
||
results=len(result.get("results", [])) if result else 0)
|
||
return result
|
||
except Exception as e:
|
||
last_error = e
|
||
logger.info(f" Failed: {e}")
|
||
emit_progress("instance_fail", url=instance,
|
||
error=str(e), error_code=classify_error(e))
|
||
continue
|
||
raise RuntimeError(f"All {len(instance_urls)} instances failed. Last error: {last_error}")
|
||
|
||
# Parallel path: one search per instance concurrently (each with its own
|
||
# backoff retries), then return the first success in original order.
|
||
logger.info(f"Trying {len(instance_urls)} instances in parallel...")
|
||
|
||
def _task(instance: str):
|
||
emit_progress("instance_try", url=instance, attempt=1)
|
||
start = time.time()
|
||
def _do():
|
||
return search_single(instance, params, method=method,
|
||
timeout=timeout, auth_headers=auth_headers)
|
||
try:
|
||
result = _retry_with_backoff(_do, max_retries=retry_per)
|
||
emit_progress("instance_ok", url=instance,
|
||
latency=round(time.time() - start, 3),
|
||
results=len(result.get("results", [])) if result else 0)
|
||
return instance, result
|
||
except Exception as e:
|
||
emit_progress("instance_fail", url=instance,
|
||
error=str(e), error_code=classify_error(e))
|
||
return instance, e
|
||
|
||
results_by_url = {}
|
||
last_parallel_error = None
|
||
with ThreadPoolExecutor(max_workers=min(len(instance_urls), 8)) as ex:
|
||
futures = {ex.submit(_task, u): u for u in instance_urls}
|
||
for fut in as_completed(futures):
|
||
u = futures[fut]
|
||
try:
|
||
inst, res = fut.result()
|
||
except Exception:
|
||
continue
|
||
if isinstance(res, Exception):
|
||
# 保留最后一个失败详情，让 classify_error 能从消息中提取
|
||
# 真实错误类型（401/403→E_AUTH，429→E_RATE_LIMIT 等），
|
||
# 而不是一律误判为 E_NETWORK。
|
||
last_parallel_error = res
|
||
logger.info(f" Failed {u}: {res}")
|
||
else:
|
||
results_by_url[u] = res
|
||
|
||
# Deterministic: return first successful in the user-supplied order
|
||
for u in instance_urls:
|
||
if u in results_by_url:
|
||
return results_by_url[u]
|
||
|
||
raise RuntimeError(f"All {len(instance_urls)} instances failed (parallel). "
|
||
f"Last error: {last_parallel_error}")
|
||
|
||
|
||
# ----- Output formatting -----
|
||
|
||
# ----- Instance health verification (E5) -----
|
||
|
||
def _probe_config_endpoint(instance: str, timeout: int,
|
||
auth_headers: dict = None) -> dict:
|
||
"""Probe ``/config``; return engines list and categories if reachable.
|
||
|
||
SearXNG's ``/config`` endpoint exposes the instance's engine list,
|
||
categories, and other settings as JSON. Some instances disable it; in
|
||
that case we return ``reachable=False`` with an empty engine list.
|
||
"""
|
||
headers = _merge_headers({"User-Agent": USER_AGENT}, auth_headers)
|
||
req = urllib.request.Request(f"{instance}/config", headers=headers)
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||
raw = resp.read().decode("utf-8")
|
||
data = json.loads(raw)
|
||
engines = [e.get("name") for e in data.get("engines", [])
|
||
if isinstance(e, dict) and e.get("name")]
|
||
cats = data.get("categories", {})
|
||
if isinstance(cats, dict):
|
||
categories = list(cats.keys())
|
||
elif isinstance(cats, list):
|
||
categories = [c for c in cats if isinstance(c, str)]
|
||
else:
|
||
categories = []
|
||
return {"reachable": True, "engines": engines,
|
||
"categories": categories, "error": None}
|
||
except urllib.error.HTTPError as e:
|
||
return {"reachable": False, "engines": [], "categories": [],
|
||
"error": f"HTTP {e.code}"}
|
||
except Exception as e:
|
||
return {"reachable": False, "engines": [], "categories": [],
|
||
"error": str(e)[:80]}
|
||
|
||
|
||
def verify_instances(instance_urls: list, timeout: int = 15,
|
||
auth_headers: dict = None, concurrency: int = 5) -> list:
|
||
"""Health-check user-supplied instances: reachability, JSON support, latency,
|
||
POST support, /config endpoint, engine list, and auth status.
|
||
|
||
Replaces the removed public-instance discovery: you bring your own
|
||
instances, and ``--verify`` tells you which are alive, which support the
|
||
JSON API, which accept POST, what engines they expose, and how fast
|
||
each responds.
|
||
|
||
Returns a list of dicts (one per instance, in input order), each with:
|
||
url, reachable (bool), json_supported (bool|None), post_supported (bool|None),
|
||
config_endpoint (bool|None), engines (list[str]), latency (float|None),
|
||
result_count (int|None), auth_status (str), error (str|None)
|
||
"""
|
||
instance_urls = [u.rstrip("/") for u in instance_urls]
|
||
has_auth = bool(auth_headers)
|
||
|
||
def _check(u: str) -> dict:
|
||
start = time.time()
|
||
# 1. GET /search?q=test&format=json — the primary reachability probe
|
||
try:
|
||
data = search_json(u, {"q": "test", "format": "json"}, "GET",
|
||
timeout=timeout, auth_headers=auth_headers)
|
||
latency = round(time.time() - start, 3)
|
||
auth_status = "ok" if has_auth else "n/a"
|
||
|
||
if data is None:
|
||
# Instance returned HTML — JSON format disabled but reachable
|
||
base = {"url": u, "reachable": True, "json_supported": False,
|
||
"latency": latency, "result_count": None,
|
||
"auth_status": auth_status,
|
||
"error": "returned HTML (format=json unsupported)"}
|
||
else:
|
||
base = {"url": u, "reachable": True, "json_supported": True,
|
||
"latency": latency,
|
||
"result_count": len(data.get("results", [])),
|
||
"auth_status": auth_status, "error": None}
|
||
|
||
# 2. POST probe — only meaningful when JSON works
|
||
if data is not None:
|
||
try:
|
||
post_data = search_json(u, {"q": "test", "format": "json"}, "POST",
|
||
timeout=timeout, auth_headers=auth_headers)
|
||
base["post_supported"] = post_data is not None
|
||
except urllib.error.HTTPError as e:
|
||
base["post_supported"] = False
|
||
base["error"] = (base.get("error") or "") + f" POST: HTTP {e.code}"
|
||
except Exception as e:
|
||
base["post_supported"] = False
|
||
base["error"] = (base.get("error") or "") + f" POST: {str(e)[:60]}"
|
||
else:
|
||
base["post_supported"] = None
|
||
|
||
# 3. /config probe — engine list (independent of JSON search)
|
||
cfg = _probe_config_endpoint(u, timeout, auth_headers)
|
||
base["config_endpoint"] = cfg["reachable"]
|
||
base["engines"] = cfg["engines"]
|
||
if cfg["error"] and not cfg["reachable"]:
|
||
# /config down is non-fatal; note it but don't overwrite search error
|
||
base["error"] = (base.get("error") or "") + f" /config: {cfg['error']}"
|
||
|
||
return base
|
||
except urllib.error.HTTPError as e:
|
||
# Distinguish auth rejection (401/403) from other HTTP errors so
|
||
# users know whether their token is wrong or the instance is down.
|
||
auth_status = ("rejected" if has_auth and e.code in (401, 403)
|
||
else ("ok" if has_auth else "n/a"))
|
||
# 5xx 是服务器错误，实例虽然响应了但不可用，应视为不可达。
|
||
# 400 可能只是请求格式问题，实例本身在线，仍算可达。
|
||
is_5xx = 500 <= e.code < 600
|
||
return {"url": u, "reachable": e.code not in (401, 403, 404) and not is_5xx,
|
||
"json_supported": False, "post_supported": None,
|
||
"config_endpoint": None, "engines": [],
|
||
"latency": round(time.time() - start, 3),
|
||
"result_count": None, "auth_status": auth_status,
|
||
"error": f"HTTP {e.code}"}
|
||
except Exception as e:
|
||
return {"url": u, "reachable": False, "json_supported": False,
|
||
"post_supported": None, "config_endpoint": None,
|
||
"engines": [], "latency": None, "result_count": None,
|
||
"auth_status": "unknown" if has_auth else "n/a",
|
||
"error": str(e)[:120]}
|
||
|
||
out = {}
|
||
with ThreadPoolExecutor(max_workers=min(concurrency, len(instance_urls))) as ex:
|
||
futures = {ex.submit(_check, u): u for u in instance_urls}
|
||
for fut in as_completed(futures):
|
||
u = futures[fut]
|
||
try:
|
||
out[u] = fut.result()
|
||
except Exception as e:
|
||
out[u] = {"url": u, "reachable": False, "json_supported": False,
|
||
"post_supported": None, "config_endpoint": None,
|
||
"engines": [], "latency": None, "result_count": None,
|
||
"auth_status": "unknown" if has_auth else "n/a",
|
||
"error": str(e)[:120]}
|
||
# Preserve input order
|
||
return [out[u] for u in instance_urls]
|
||
|
||
|
||
def _print_verify_report(report: list, as_json: bool):
|
||
"""Print a health-check report (human table or JSON).
|
||
|
||
The table shows one line per instance with the most important signals
|
||
(reach / JSON / POST / latency / result count / engine count / auth).
|
||
The full engine list per instance is only included in the JSON output
|
||
to keep the table readable.
|
||
"""
|
||
if as_json:
|
||
print(json.dumps(report, indent=2, ensure_ascii=False))
|
||
return
|
||
header = (f"{'URL':<38} {'REACH':<6} {'JSON':<5} {'POST':<5} "
|
||
f"{'LAT':<8} {'RES':<5} {'ENGS':<5} {'AUTH':<9} ERROR")
|
||
lines = [header, "-" * len(header)]
|
||
for r in report:
|
||
reach = "yes" if r["reachable"] else "NO"
|
||
js = "-" if r["json_supported"] is None else ("yes" if r["json_supported"] else "no")
|
||
post = "-" if r.get("post_supported") is None else ("yes" if r["post_supported"] else "no")
|
||
lat = f"{r['latency']}s" if r["latency"] is not None else "-"
|
||
rc = "-" if r["result_count"] is None else str(r["result_count"])
|
||
engs = str(len(r.get("engines", []))) if r.get("engines") is not None else "-"
|
||
auth = r.get("auth_status", "-")
|
||
err = (r["error"] or "").replace("\n", " ")[:40]
|
||
lines.append(f"{r['url'][:37]:<38} {reach:<6} {js:<5} {post:<5} "
|
||
f"{lat:<8} {rc:<5} {engs:<5} {auth:<9} {err}")
|
||
ok = sum(1 for r in report if r["reachable"])
|
||
lines.append("-" * len(header))
|
||
lines.append(f"{ok}/{len(report)} instances reachable")
|
||
print("\n".join(lines))
|
||
|
||
|
||
# ----- Auto-fetch result pages -----
|
||
# NOTE: HTTP transport (retry, charset, UA fallback, size limit) is delegated
|
||
# to fetch.fetch_url — this module only post-processes its result. This
|
||
# eliminates ~70 lines of duplicated HTTP code that previously drifted
|
||
# between search.fetch_page and fetch.fetch_url.
|
||
|
||
def fetch_page(url: str, timeout: int = 10, auth_headers: dict = None,
|
||
max_retries: int = 3, max_size: int = None,
|
||
referer: str = None,
|
||
fallback_enabled: bool = True) -> dict:
|
||
"""Fetch a single page; returns metadata dict with 'status'='ok' or 'error'.
|
||
|
||
Thin wrapper around :func:`fetch.fetch_url` that adds:
|
||
* CAPTCHA / bot-block detection with WAF fingerprinting (v2.0.0)
|
||
* Wayback Machine fallback on 404/403/timeout (v2.0.0, default on)
|
||
* automatic text extraction via :func:`fetch.extract_text`
|
||
* dict-shaped return suitable for the auto-fetch feature
|
||
|
||
All HTTP transport concerns (retry, charset, UA fallback, size limit,
|
||
browser headers, Retry-After compliance) are handled by ``fetch_url``.
|
||
|
||
v2.0.0 新字段：
|
||
* ``anti_bot_detected`` (bool): 是否检测到反爬页面
|
||
* ``waf_type`` (str|None): WAF 类型（cloudflare/imperva/perimeterx/
|
||
datadome/akamai/generic），仅当 anti_bot_detected=True 时有值
|
||
* ``fallback_used`` (str|None): 兜底方式（"wayback"），仅当走兜底时有值
|
||
"""
|
||
# 主抓取
|
||
result = None
|
||
error_msg = None
|
||
try:
|
||
result = fetch_url(
|
||
url, timeout=timeout, auth_headers=auth_headers,
|
||
max_retries=max_retries, max_size=max_size,
|
||
allow_redirects=True, referer=referer,
|
||
)
|
||
except Exception as e:
|
||
error_msg = str(e) if str(e) else e.__class__.__name__
|
||
|
||
# 反爬检测（v2.0.0 增强：全文档扫描 + WAF 指纹库）
|
||
# 必须在 Wayback 兜底判断之前执行：Cloudflare 质询页常返回 HTTP 200，
|
||
# 此时 result 非 None 但内容是反爬页，必须识别出来才能触发兜底。
|
||
anti_bot_detected = False
|
||
waf_type = None
|
||
if result is not None:
|
||
content = result.content
|
||
content_type = result.content_type or ""
|
||
is_html = ("html" in content_type.lower() or
|
||
content.strip().startswith("<!") or
|
||
content.strip().startswith("<htm"))
|
||
if is_html:
|
||
waf_type = _detect_anti_bot(content)
|
||
if waf_type:
|
||
anti_bot_detected = True
|
||
|
||
# Wayback 兜底触发条件（v2.0.1 修复）：
|
||
# 1. 主抓取抛异常且错误暗示 404/403/超时（_should_try_fallback）
|
||
# 2. 主抓取"成功"但被反爬拦截（anti_bot_detected=True）
|
||
# 原 v2.0.0 bug：仅条件 1 触发兜底，条件 2 漏网（Cloudflare 200 质询页）。
|
||
fallback_used = None
|
||
need_fallback = False
|
||
if fallback_enabled:
|
||
if anti_bot_detected:
|
||
need_fallback = True
|
||
elif result is None and _should_try_fallback(result, error_msg):
|
||
need_fallback = True
|
||
|
||
if need_fallback:
|
||
wb_result = _try_wayback_fallback(url, timeout=timeout,
|
||
auth_headers=auth_headers,
|
||
max_retries=max_retries,
|
||
max_size=max_size)
|
||
if wb_result is not None:
|
||
result = wb_result
|
||
error_msg = None
|
||
fallback_used = "wayback"
|
||
# Wayback 结果重新做反爬检测（防御性：Wayback 快照极少是反爬页）
|
||
anti_bot_detected = False
|
||
waf_type = None
|
||
content = result.content
|
||
content_type = result.content_type or ""
|
||
is_html = ("html" in content_type.lower() or
|
||
content.strip().startswith("<!") or
|
||
content.strip().startswith("<htm"))
|
||
if is_html:
|
||
waf_type = _detect_anti_bot(content)
|
||
if waf_type:
|
||
anti_bot_detected = True
|
||
|
||
# 仍然失败
|
||
if result is None:
|
||
return {
|
||
"url": url, "status": "error",
|
||
"error": error_msg or "unknown error",
|
||
"text": "", "text_length": 0, "truncated": False,
|
||
"anti_bot_detected": False, "waf_type": None,
|
||
"fallback_used": None,
|
||
}
|
||
|
||
# 反爬仍被检测到（Wayback 也无能为力或兜底被禁用）
|
||
if anti_bot_detected:
|
||
return {
|
||
"url": url, "final_url": result.final_url, "status": "error",
|
||
"error": f"Bot protection detected ({waf_type})",
|
||
"text": "", "text_length": 0, "truncated": False,
|
||
"anti_bot_detected": True, "waf_type": waf_type,
|
||
"fallback_used": fallback_used,
|
||
}
|
||
|
||
text = extract_text(content) if is_html else content
|
||
return {
|
||
"url": url,
|
||
"final_url": result.final_url,
|
||
"status": "ok",
|
||
"content_type": content_type,
|
||
"text": text,
|
||
"text_length": len(text),
|
||
"truncated": result.truncated,
|
||
"truncated_at": max_size if result.truncated else None,
|
||
"user_agent_used": result.user_agent,
|
||
"anti_bot_detected": False,
|
||
"waf_type": None,
|
||
"fallback_used": fallback_used,
|
||
}
|
||
|
||
|
||
def _should_try_fallback(result, error_msg: str) -> bool:
|
||
"""判断是否应触发 Wayback 兜底。
|
||
|
||
触发条件：
|
||
1. 主抓取抛异常且错误信息暗示 404/403/超时
|
||
2. 主抓取成功但被反爬拦截（理论上 fetch_page 已处理，此处防御性）
|
||
|
||
不触发：
|
||
* 用户禁用兜底（调用方控制，不进入此函数）
|
||
* 错误是 DNS 失败（Wayback 也访问不到）
|
||
"""
|
||
if result is not None:
|
||
# 主抓取成功，无需兜底
|
||
return False
|
||
if not error_msg:
|
||
return False
|
||
msg = error_msg.lower()
|
||
# 404/403/超时/连接重置 → 尝试 Wayback
|
||
triggers = ["404", "403", "timeout", "timed out", "connection reset",
|
||
"connection refused", "max retries exceeded"]
|
||
if any(t in msg for t in triggers):
|
||
return True
|
||
return False
|
||
|
||
|
||
def _try_wayback_fallback(url: str, timeout: int = 10,
|
||
auth_headers: dict = None,
|
||
max_retries: int = 2,
|
||
max_size: int = None):
|
||
"""尝试从 Wayback Machine 获取页面快照。
|
||
|
||
使用 ``https://web.archive.org/web/2/<url>`` 端点，``2`` 表示
|
||
"最新可用快照"。Wayback 会 302 重定向到具体时间戳快照。
|
||
|
||
返回 FetchResult 或 None（失败时）。独立超时（10s），不阻塞主流程。
|
||
"""
|
||
wayback_url = f"https://web.archive.org/web/2/{url}"
|
||
wb_timeout = min(timeout, 10) # Wayback 自身可能慢，限制最大 10s
|
||
try:
|
||
logger.info(f" [FALLBACK] Trying Wayback Machine for {url[:55]}")
|
||
result = fetch_url(
|
||
wayback_url, timeout=wb_timeout, auth_headers=None,
|
||
max_retries=max_retries, max_size=max_size,
|
||
allow_redirects=True,
|
||
)
|
||
# Wayback 包装页也算成功——它返回的是原始页面内容
|
||
return result
|
||
except Exception as e:
|
||
logger.info(f" [FALLBACK] Wayback failed for {url[:55]}: {e}")
|
||
return None
|
||
|
||
|
||
# ----- 反爬检测（v2.0.1 收窄误判 + title 精准检测）-----
|
||
# WAF 指纹库：每项 = (waf_type, [指示词])
|
||
#
|
||
# v2.0.1 修复：v2.0.0 用裸公司名（cloudflare/akamai）和宽泛词（captcha/
|
||
# challenge/dd-）做全文匹配，导致正常文章（如引用 Cloudflare 文档、讨论
|
||
# "coding challenges" 的文章、Wayback 归档正文）被误判为反爬页，Wayback
|
||
# 兜底返回的有效内容也被丢弃。收窄原则：
|
||
# 1. 专用指纹只用 WAF 厂商的技术标识符（cookie 名/HTTP header 名/JS 变量名）
|
||
# ——这些不会出现在文章正文里
|
||
# 2. 通用文案用完整短语而非单词（如 "please complete the captcha" 而非
|
||
# "captcha"），避免正常内容误判
|
||
# 3. 增加 <title> 标签检测——反爬页 title 是特征文案，最精准
|
||
WAF_FINGERPRINTS = [
|
||
("cloudflare", [
|
||
# Cloudflare 技术标识符（cookie/header/JS 变量名，不会出现在正文）
|
||
"cf-ray", "cf-chl-bypass", "cf-mitigated",
|
||
"cf-browser-verification", "cf-error-details", "cf-error-code",
|
||
# 质询页特征文案（足够具体，正常内容不会完整出现）
|
||
"just a moment", "checking your browser before accessing",
|
||
"attention required! | cloudflare",
|
||
"enable javascript and cookies to continue",
|
||
]),
|
||
("imperva", [
|
||
# Incapsula cookie/技术标识符
|
||
"incap_ses", "visid_incap", "incap_ses_",
|
||
"incapsula incident id", "request unsuccessful. incapsula",
|
||
"visit denied by incapsula",
|
||
]),
|
||
("perimeterx", [
|
||
# PerimeterX 专有标识符（_px 太短会匹配 CSS 类名，已移除）
|
||
"px-captcha", "pxhd", "pxcts", "pxcookie",
|
||
"_pxff", "_pxhd",
|
||
"press & hold to confirm you are a human",
|
||
]),
|
||
("datadome", [
|
||
# DataDome 专有标识符（dd- 太宽泛会匹配 dd-class 等，已移除）
|
||
"datadome", "data-dome",
|
||
"protected by datadome", "datadome-bot-protect",
|
||
]),
|
||
("akamai", [
|
||
# Akamai Bot Manager cookie/标识符（akamai 裸名会匹配正文引用，已移除）
|
||
"bm_sz", "_abck", "akamaighost", "akamai-bot-manager",
|
||
"ak_bmsc",
|
||
]),
|
||
# 通用反爬指示词（v2.0.1 收窄：完整短语而非单词，避免正文误判）
|
||
("generic", [
|
||
"verify you are human", "verify that you are human",
|
||
"making sure you're not a bot", "are you a robot",
|
||
"robot or human", "human verification",
|
||
"please complete the captcha", "complete the security check",
|
||
"please enable javascript to continue",
|
||
"enable javascript to continue",
|
||
"ddos protection by", "access denied - sucuri",
|
||
"you have been blocked", "unusual traffic from your computer",
|
||
"pardon our interruption", "we'll be right back",
|
||
"bot protection", "anti-bot protection",
|
||
"anubis_challenge", "miserere", # Anubis 反爬系统（拦截 AI 爬虫）
|
||
]),
|
||
]
|
||
|
||
# <title> 标签检测：反爬页 title 通常是特征文案，比全文扫描更精准。
|
||
# key = title 中的特征子串（小写），value = 对应 WAF 类型
|
||
_TITLE_ANTI_BOT_SIGNATURES = {
|
||
"just a moment": "cloudflare",
|
||
"attention required": "cloudflare",
|
||
"access denied": "generic",
|
||
"are you a robot": "generic",
|
||
"robot check": "generic",
|
||
"human verification": "generic",
|
||
"please verify you are human": "generic",
|
||
"security check": "generic",
|
||
"verify you are human": "generic",
|
||
}
|
||
|
||
_TITLE_RE = re.compile(r"<title[^>]*>(.*?)</title>", re.IGNORECASE | re.DOTALL)
|
||
|
||
|
||
def _detect_anti_bot(content: str) -> str:
|
||
"""检测反爬页面，返回 WAF 类型或 None。
|
||
|
||
v2.0.1 改进：
|
||
* 增加 <title> 标签精准检测（反爬页 title 是特征文案，误判率极低）
|
||
* 收窄 WAF 指纹库关键词（移除裸公司名和宽泛词，改用技术标识符 + 完整短语）
|
||
* 保留全文档扫描（v2.0.0 改进，大页面反爬页可能在前 2000 字之外）
|
||
|
||
检测顺序：title 标签（最精准）→ 全文档指纹扫描（技术标识符 + 文案短语）。
|
||
|
||
性能：全文档 lower() 一次 + title regex 一次，对 5MB 页面约 5ms，可接受。
|
||
"""
|
||
if not content:
|
||
return None
|
||
# 1. <title> 标签检测（最精准，误判率极低）
|
||
title_match = _TITLE_RE.search(content)
|
||
if title_match:
|
||
title_lower = title_match.group(1).strip().lower()
|
||
for sig, waf_type in _TITLE_ANTI_BOT_SIGNATURES.items():
|
||
if sig in title_lower:
|
||
return waf_type
|
||
# 2. 全文档指纹扫描（收窄后的技术标识符 + 完整文案）
|
||
lower = content.lower()
|
||
for waf_type, indicators in WAF_FINGERPRINTS:
|
||
for ind in indicators:
|
||
if ind in lower:
|
||
return waf_type
|
||
return None
|
||
|
||
|
||
def _is_blocked_page(content: str) -> bool:
|
||
"""[已废弃] 快速检测反爬页面。保留向后兼容，内部调用 _detect_anti_bot。
|
||
|
||
v2.0.0 起请使用 _detect_anti_bot() 获取具体 WAF 类型。
|
||
"""
|
||
return _detect_anti_bot(content) is not None
|
||
|
||
|
||
class AdaptiveThrottle:
|
||
"""自适应限流状态机（v2.0.0）。
|
||
|
||
在 fetch_top_results 的并发抓取过程中，根据成功/失败反馈动态调整：
|
||
* 连续 >=3 次失败 → request_delay 翻倍，concurrency 减半
|
||
* 连续 >=5 次成功 → 逐步恢复原参数
|
||
* 收到 429 → 标记全局暂停 N 秒（N 来自 Retry-After 或默认 30s），
|
||
所有线程在下次请求前等待
|
||
|
||
线程安全：所有方法加锁。状态由 fetch_top_results 的 _fetch_one 回调驱动。
|
||
"""
|
||
|
||
def __init__(self, initial_delay: float, initial_concurrency: int):
|
||
self._lock = threading.Lock()
|
||
self._delay = initial_delay
|
||
self._initial_delay = initial_delay
|
||
self._concurrency = initial_concurrency
|
||
self._initial_concurrency = initial_concurrency
|
||
self._consecutive_failures = 0
|
||
self._consecutive_successes = 0
|
||
self._global_pause_until = 0.0 # time.monotonic() 时间戳
|
||
|
||
@property
|
||
def delay(self) -> float:
|
||
with self._lock:
|
||
return self._delay
|
||
|
||
@property
|
||
def concurrency(self) -> int:
|
||
with self._lock:
|
||
return self._concurrency
|
||
|
||
def report_success(self) -> None:
|
||
with self._lock:
|
||
self._consecutive_failures = 0
|
||
self._consecutive_successes += 1
|
||
# 连续 5 次成功 → 逐步恢复
|
||
if self._consecutive_successes >= 5:
|
||
self._consecutive_successes = 0
|
||
self._delay = max(self._initial_delay, self._delay / 2)
|
||
if self._concurrency < self._initial_concurrency:
|
||
self._concurrency = min(self._initial_concurrency,
|
||
self._concurrency * 2)
|
||
|
||
def report_failure(self, error_msg: str = "") -> None:
|
||
with self._lock:
|
||
self._consecutive_successes = 0
|
||
self._consecutive_failures += 1
|
||
# 429 → 全局暂停（调用方会从 error_msg 提取秒数，这里只标记）
|
||
if "429" in error_msg.lower():
|
||
self._global_pause_until = time.monotonic() + 30.0
|
||
# 连续 3 次失败 → 退避 + 降并发
|
||
if self._consecutive_failures >= 3:
|
||
self._consecutive_failures = 0
|
||
self._delay = min(self._delay * 2, 10.0) # 上限 10s
|
||
self._concurrency = max(1, self._concurrency // 2)
|
||
|
||
def wait_if_paused(self) -> None:
|
||
"""如果处于全局暂停期，阻塞等待直到解除。请求前调用。"""
|
||
with self._lock:
|
||
remaining = self._global_pause_until - time.monotonic()
|
||
if remaining > 0:
|
||
logger.info(f" [THROTTLE] Global pause: waiting {remaining:.1f}s (429)")
|
||
time.sleep(remaining)
|
||
|
||
def stats(self) -> dict:
|
||
"""返回当前状态快照，供 --fetch-report 使用。"""
|
||
with self._lock:
|
||
return {
|
||
"current_delay": round(self._delay, 3),
|
||
"current_concurrency": self._concurrency,
|
||
"consecutive_failures": self._consecutive_failures,
|
||
"consecutive_successes": self._consecutive_successes,
|
||
"global_paused": time.monotonic() < self._global_pause_until,
|
||
}
|
||
|
||
|
||
def fetch_top_results(results: dict, count: int, timeout: int = 10,
|
||
concurrency: int = 5, auth_headers: dict = None,
|
||
max_retries: int = 3, max_size: int = None,
|
||
request_delay: float = 0.3,
|
||
referer: str = None,
|
||
fallback_enabled: bool = True,
|
||
throttle: "AdaptiveThrottle" = None) -> list:
|
||
"""Fetch full text of top N result pages concurrently.
|
||
|
||
Features:
|
||
- Retries transient errors with exponential backoff (in fetch_url)
|
||
- v2.0.0 自适应限流：连续失败自动降并发+加延迟，429 全局暂停
|
||
- v2.0.0 Wayback 兜底：404/403/超时自动尝试 Wayback Machine
|
||
- v2.0.0 反爬检测：WAF 指纹库识别 Cloudflare/Imperva/PerimeterX 等
|
||
- Falls back to browser User-Agent if blocked (in fetch_url)
|
||
- Small delay between requests to avoid rate limits
|
||
|
||
Args:
|
||
referer: Referer URL（v2.0.0，通常设为 SearXNG 实例 URL）
|
||
fallback_enabled: 是否启用 Wayback 兜底（默认 True）
|
||
throttle: 外部传入的 AdaptiveThrottle 实例（可选；不传则内部创建）
|
||
"""
|
||
urls = []
|
||
seen = set()
|
||
for r in results.get("results", []):
|
||
url = r.get("url", "")
|
||
if url and url not in seen:
|
||
urls.append(url)
|
||
seen.add(url)
|
||
if len(urls) >= count:
|
||
break
|
||
|
||
if not urls:
|
||
return []
|
||
|
||
# 自适应限流器（外部未传入则创建）
|
||
if throttle is None:
|
||
throttle = AdaptiveThrottle(request_delay, concurrency)
|
||
|
||
logger.info(f"\nFetching {len(urls)} result pages "
|
||
f"(timeout={timeout}s, retries={max_retries}, "
|
||
f"delay={throttle.delay}s, concurrency={throttle.concurrency})...")
|
||
fetched = []
|
||
ok_count = [0]
|
||
err_count = [0]
|
||
anti_bot_count = [0]
|
||
fallback_count = [0]
|
||
|
||
def _fetch_one(u: str) -> dict:
|
||
"""Fetch one URL with adaptive throttling to avoid rate limiting."""
|
||
# 全局暂停检查（429 触发）
|
||
throttle.wait_if_paused()
|
||
# 自适应延迟
|
||
d = throttle.delay
|
||
if d > 0:
|
||
time.sleep(d * random.uniform(0.5, 1.5))
|
||
result = fetch_page(u, timeout=timeout, auth_headers=auth_headers,
|
||
max_retries=max_retries, max_size=max_size,
|
||
referer=referer, fallback_enabled=fallback_enabled)
|
||
if result["status"] == "ok":
|
||
ok_count[0] += 1
|
||
throttle.report_success()
|
||
trunc = ", TRUNCATED" if result.get("truncated") else ""
|
||
ua_note = ""
|
||
if result.get("user_agent_used") != USER_AGENT:
|
||
ua_note = " [fallback UA]"
|
||
fb_note = " [wayback]" if result.get("fallback_used") else ""
|
||
if fb_note:
|
||
fallback_count[0] += 1
|
||
logger.info(f" [OK] {u[:55]} ({result['text_length']:,} chars"
|
||
f"{trunc}{ua_note}{fb_note})")
|
||
else:
|
||
err_count[0] += 1
|
||
throttle.report_failure(result.get("error", ""))
|
||
# 统计反爬拦截
|
||
if result.get("anti_bot_detected"):
|
||
anti_bot_count[0] += 1
|
||
logger.error(f" [ERR] {u[:55]} ({result.get('error', 'unknown')})")
|
||
return result
|
||
|
||
with ThreadPoolExecutor(max_workers=min(throttle.concurrency, len(urls))) as ex:
|
||
future_map = {ex.submit(_fetch_one, u): u for u in urls}
|
||
for future in as_completed(future_map):
|
||
try:
|
||
result = future.result()
|
||
fetched.append(result)
|
||
except Exception as e:
|
||
u = future_map[future]
|
||
fetched.append({"url": u, "status": "error", "error": str(e),
|
||
"text": "", "text_length": 0, "truncated": False,
|
||
"anti_bot_detected": False, "waf_type": None,
|
||
"fallback_used": None})
|
||
logger.error(f" [ERR] {u[:55]} (thread error: {e})")
|
||
|
||
# Reorder to match original result order
|
||
url_order = {u: i for i, u in enumerate(urls)}
|
||
fetched.sort(key=lambda f: url_order.get(f["url"], 999))
|
||
|
||
logger.info(f"Fetched: {ok_count[0]} ok, {err_count[0]} errors"
|
||
f" (anti-bot: {anti_bot_count[0]}, wayback: {fallback_count[0]})")
|
||
return fetched
|
||
|
||
|
||
def _emit_fetch_report(fetched: list, throttle: "AdaptiveThrottle") -> None:
|
||
"""v2.0.0: 输出结构化抓取报告到 stderr。
|
||
|
||
让 AI Agent 可程序化分析抓取过程：哪些 URL 被反爬拦截、用了什么兜底、
|
||
自适应限流如何调整。格式为人类可读的表格 + JSON 摘要。
|
||
"""
|
||
import sys as _sys
|
||
out = _sys.stderr
|
||
lines = []
|
||
lines.append("\n" + "=" * 72)
|
||
lines.append("FETCH REPORT (v2.0.0)")
|
||
lines.append("=" * 72)
|
||
|
||
# Per-URL 表
|
||
header = f"{'URL':<45} {'Status':<8} {'WAF':<12} {'Fallback':<10} {'Chars':>10}"
|
||
lines.append(header)
|
||
lines.append("-" * len(header))
|
||
for f in fetched:
|
||
url = f.get("url", "")[:44]
|
||
status = "OK" if f.get("status") == "ok" else "ERR"
|
||
waf = f.get("waf_type") or "-"
|
||
fb = f.get("fallback_used") or "-"
|
||
chars = f.get("text_length", 0)
|
||
lines.append(f"{url:<45} {status:<8} {waf:<12} {fb:<10} {chars:>10,}")
|
||
|
||
# 统计摘要
|
||
total = len(fetched)
|
||
ok = sum(1 for f in fetched if f.get("status") == "ok")
|
||
err = total - ok
|
||
anti_bot = sum(1 for f in fetched if f.get("anti_bot_detected"))
|
||
wayback = sum(1 for f in fetched if f.get("fallback_used") == "wayback")
|
||
lines.append("-" * len(header))
|
||
lines.append(f"Total: {total} | OK: {ok} | Error: {err} | "
|
||
f"Anti-bot blocked: {anti_bot} | Wayback recovered: {wayback}")
|
||
|
||
# 自适应限流状态
|
||
s = throttle.stats()
|
||
lines.append(f"Throttle: delay={s['current_delay']}s "
|
||
f"concurrency={s['current_concurrency']} "
|
||
f"paused={s['global_paused']} "
|
||
f"consec_fail={s['consecutive_failures']} "
|
||
f"consec_ok={s['consecutive_successes']}")
|
||
|
||
# JSON 摘要（一行，便于 Agent 解析）
|
||
import json as _json
|
||
summary = {
|
||
"total": total, "ok": ok, "error": err,
|
||
"anti_bot_blocked": anti_bot, "wayback_recovered": wayback,
|
||
"throttle": s,
|
||
}
|
||
lines.append("JSON: " + _json.dumps(summary, ensure_ascii=False))
|
||
lines.append("=" * 72 + "\n")
|
||
print("\n".join(lines), file=out)
|
||
|
||
|
||
# ----- Output formatting -----
|
||
|
||
def deduplicate_results(results: dict) -> dict:
|
||
"""Remove duplicate results by normalized URL, keeping first occurrence.
|
||
|
||
Different engines often return the same URL. The first occurrence keeps
|
||
its engine/score metadata; subsequent duplicates are dropped.
|
||
|
||
URL normalization:
|
||
* lowercase scheme + host
|
||
* strip fragment (``#...``)
|
||
* drop common tracking query params (``utm_*``, ``gclid``, ``fbclid``,
|
||
``mc_*``, ``ref``, ``ref_*``)
|
||
* sort remaining params so ``?b=2&a=1`` matches ``?a=1&b=2``
|
||
|
||
Results without a URL are kept as-is (never deduped). Mutates
|
||
``results['results']`` in place and returns ``results`` for chaining.
|
||
"""
|
||
if not results.get("results"):
|
||
return results
|
||
|
||
# 跟踪参数前缀。裸 "ref" 过于宽泛（会误删 reference/refcode 等正常参数），
|
||
# 收紧为 "ref_" 只匹配 ref_source/ref_campaign 等跟踪参数。
|
||
TRACKING_PREFIXES = ("utm_", "gclid", "fbclid", "mc_", "ref_")
|
||
|
||
def _normalize_url(url: str) -> str:
|
||
try:
|
||
p = urllib.parse.urlparse(url)
|
||
scheme = p.scheme.lower()
|
||
netloc = p.netloc.lower()
|
||
qs = urllib.parse.parse_qsl(p.query, keep_blank_values=False)
|
||
qs = [(k, v) for k, v in qs
|
||
if not any(k.lower().startswith(pref) for pref in TRACKING_PREFIXES)]
|
||
qs.sort()
|
||
query = urllib.parse.urlencode(qs)
|
||
return urllib.parse.urlunparse((scheme, netloc, p.path, p.params,
|
||
query, ""))
|
||
except Exception:
|
||
return url
|
||
|
||
seen = set()
|
||
deduped = []
|
||
for r in results["results"]:
|
||
url = r.get("url", "")
|
||
key = _normalize_url(url) if url else ""
|
||
if key and key in seen:
|
||
continue
|
||
if key:
|
||
seen.add(key)
|
||
deduped.append(r)
|
||
|
||
before = len(results["results"])
|
||
results["results"] = deduped
|
||
if before != len(deduped):
|
||
logger.info(f"Dedup: {before} -> {len(deduped)} results (by normalized URL)")
|
||
return results
|
||
|
||
|
||
def sort_results(results: dict, sort_by: str) -> dict:
|
||
"""Sort results by the given key.
|
||
|
||
* ``score`` — descending (highest first); entries without score keep
|
||
their relative order at the end.
|
||
* ``date`` — descending (newest first) by ``published_date`` (ISO
|
||
strings sort lexicographically); entries without a date
|
||
keep their relative order at the end.
|
||
* ``engine`` — ascending (alphabetical by engine name).
|
||
* ``none`` — no-op (preserve original order).
|
||
|
||
Mutates ``results['results']`` in place and returns ``results``.
|
||
"""
|
||
if not results.get("results") or sort_by == "none":
|
||
return results
|
||
|
||
rs = results["results"]
|
||
if sort_by == "score":
|
||
with_score = [r for r in rs if r.get("score") is not None]
|
||
without = [r for r in rs if r.get("score") is None]
|
||
with_score.sort(key=lambda r: r["score"], reverse=True)
|
||
results["results"] = with_score + without
|
||
elif sort_by == "date":
|
||
with_date = [r for r in rs if r.get("published_date")]
|
||
without = [r for r in rs if not r.get("published_date")]
|
||
with_date.sort(key=lambda r: r["published_date"], reverse=True)
|
||
results["results"] = with_date + without
|
||
elif sort_by == "engine":
|
||
rs.sort(key=lambda r: r.get("engine") or "")
|
||
results["results"] = rs
|
||
return results
|
||
|
||
|
||
def filter_results_by_domain(results: dict, include_domains: list = None,
|
||
exclude_domains: list = None) -> dict:
|
||
"""Filter search results by domain (allowlist or blocklist).
|
||
|
||
Modifies ``results['results']`` in place and returns ``results`` for
|
||
chaining. Matching is case-insensitive and ignores a leading ``www.``
|
||
so ``example.com`` matches both ``example.com`` and ``www.example.com``.
|
||
|
||
``include_domains`` (allowlist) takes precedence: if non-empty, only
|
||
results whose domain matches are kept. ``exclude_domains`` (blocklist)
|
||
then drops any remaining matches. Empty inputs are no-ops.
|
||
"""
|
||
if not results.get("results"):
|
||
return results
|
||
if not include_domains and not exclude_domains:
|
||
return results
|
||
|
||
def _get_domain(url: str) -> str:
|
||
try:
|
||
return urllib.parse.urlparse(url).netloc.lower()
|
||
except Exception:
|
||
return ""
|
||
|
||
def _normalize(d: str) -> str:
|
||
return d.lower().lstrip(".").lstrip("www.")
|
||
|
||
include_set = {_normalize(d) for d in include_domains} if include_domains else None
|
||
exclude_set = {_normalize(d) for d in exclude_domains} if exclude_domains else None
|
||
|
||
filtered = []
|
||
for r in results["results"]:
|
||
domain = _get_domain(r.get("url", ""))
|
||
# Strip leading www. for matching
|
||
match_domain = domain[4:] if domain.startswith("www.") else domain
|
||
|
||
if include_set is not None and match_domain not in include_set:
|
||
continue
|
||
if exclude_set is not None and match_domain in exclude_set:
|
||
continue
|
||
filtered.append(r)
|
||
|
||
results["results"] = filtered
|
||
return results
|
||
|
||
|
||
def format_brief(results: dict, snippet_len: int = 0) -> str:
|
||
"""Format results as brief text. snippet_len=0 means no truncation."""
|
||
lines = []
|
||
for i, r in enumerate(results.get("results", []), 1):
|
||
title = r.get("title", "No title")
|
||
url = r.get("url", "")
|
||
content = r.get("content", "")
|
||
lines.append(f"{i}. {title}")
|
||
lines.append(f" {url}")
|
||
if content:
|
||
if snippet_len > 0:
|
||
content = content[:snippet_len]
|
||
lines.append(f" {content}")
|
||
lines.append("")
|
||
|
||
# Include suggestions if present
|
||
sugs = results.get("suggestions", [])
|
||
if sugs:
|
||
lines.append(f"Suggestions: {', '.join(sugs)}")
|
||
|
||
# Include answers if present
|
||
for ans in results.get("answers", []):
|
||
lines.append(f"Answer: {ans}")
|
||
|
||
return "\n".join(lines)
|
||
|
||
|
||
def format_urls(results: dict) -> str:
|
||
"""Format results as plain URL list."""
|
||
return "\n".join(r.get("url", "") for r in results.get("results", []) if r.get("url"))
|
||
|
||
|
||
def _format_results(results: dict, args) -> str:
|
||
"""Format a results dict into the output string selected by args.format.
|
||
|
||
Pulled out of :func:`main` so the batch runner (--queries-file) can reuse
|
||
the exact same formatting for each per-query block.
|
||
"""
|
||
if args.format == "json":
|
||
results["schema_version"] = SCHEMA_VERSION
|
||
return json.dumps(results, indent=2, ensure_ascii=False)
|
||
if args.format == "urls":
|
||
return format_urls(results)
|
||
if args.format == "csv":
|
||
import csv as csv_mod
|
||
import io
|
||
out = io.StringIO()
|
||
writer = csv_mod.writer(out, lineterminator="\n")
|
||
writer.writerow(["title", "url", "engine", "score",
|
||
"published_date", "content"])
|
||
for r in results.get("results", []):
|
||
writer.writerow([
|
||
r.get("title", ""),
|
||
r.get("url", ""),
|
||
r.get("engine", ""),
|
||
r.get("score", "") if r.get("score") is not None else "",
|
||
r.get("published_date", ""),
|
||
r.get("content", ""),
|
||
])
|
||
return out.getvalue().rstrip()
|
||
# brief (also the safe fallthrough)
|
||
output = format_brief(results, snippet_len=args.snippet_len)
|
||
if args.fetch > 0 and results.get("fetched"):
|
||
output += "\n\n" + "=" * 60 + "\n"
|
||
output += f"FETCHED PAGES ({len(results['fetched'])} pages)\n"
|
||
output += "=" * 60 + "\n"
|
||
for f in results["fetched"]:
|
||
output += f"\n--- {f['url']} ---\n"
|
||
if f["status"] == "ok":
|
||
output += f"{f['text']}\n"
|
||
else:
|
||
output += f"[ERROR: {f.get('error', 'unknown')}]\n"
|
||
return output
|
||
|
||
|
||
def _build_params(query: str, args) -> dict:
|
||
"""Build the SearXNG API params dict for one query.
|
||
|
||
Centralized so the single-query path and the --queries-file batch path
|
||
construct identical params (and thus share cache entries).
|
||
"""
|
||
params = {"q": query, "format": "json"}
|
||
if args.categories:
|
||
params["categories"] = _normalize_csv(args.categories)
|
||
if args.language:
|
||
params["language"] = args.language
|
||
if args.pageno != 1:
|
||
params["pageno"] = str(args.pageno)
|
||
if args.time_range and args.time_range != "none":
|
||
params["time_range"] = args.time_range
|
||
if args.safesearch is not None:
|
||
params["safesearch"] = str(args.safesearch)
|
||
if args.engines:
|
||
params["engines"] = _normalize_csv(args.engines)
|
||
return params
|
||
|
||
|
||
def _run_single_query(query: str, args, instance_urls: list,
|
||
auth_headers: dict, ttl_seconds: int):
|
||
"""Run one query end-to-end: search → limit → domain-filter → fetch.
|
||
|
||
Returns ``(results_dict, error_str, error_code)``。成功时后两者为 None。
|
||
``error_code`` 是结构化错误码（E_NETWORK/E_AUTH 等），让 AI Agent
|
||
程序化判断错误类型。Cache hits 跳过网络。后处理（limit/filter/fetch）
|
||
总是执行，保证 batch 调用方看到与单查询一致的形状。
|
||
"""
|
||
params = _build_params(query, args)
|
||
|
||
emit_progress("start", query=query, instances=len(instance_urls))
|
||
|
||
cached = cache_module.get(params, ttl_seconds) if ttl_seconds > 0 else None
|
||
if cached is not None:
|
||
logger.info(f"[cache hit] q={query!r} TTL={args.cache_ttl}min, skipping network")
|
||
emit_progress("cache_hit", query=query, ttl=args.cache_ttl)
|
||
results = cached
|
||
else:
|
||
try:
|
||
results = search_multi(
|
||
instance_urls, params,
|
||
method=args.method,
|
||
timeout=args.timeout,
|
||
retry_per=args.retry,
|
||
auth_headers=auth_headers,
|
||
parallel=not args.serial,
|
||
)
|
||
except Exception as e:
|
||
err_code = classify_error(e)
|
||
emit_progress("error", error=str(e), error_code=err_code, query=query)
|
||
return None, str(e), err_code
|
||
if ttl_seconds > 0:
|
||
cache_module.put(params, results, ttl_seconds)
|
||
logger.info(f"[cache stored] q={query!r} TTL={args.cache_ttl}min")
|
||
emit_progress("cache_store", query=query, ttl=args.cache_ttl)
|
||
|
||
# Dedup (default on; --no-dedup disables) then sort, both BEFORE limit
|
||
# so --max-results keeps the highest-scoring / newest items.
|
||
if not args.no_dedup:
|
||
deduplicate_results(results)
|
||
sort_results(results, args.sort_by)
|
||
|
||
if args.max_results and "results" in results:
|
||
results["results"] = results["results"][:args.max_results]
|
||
|
||
if args.include_domain or args.exclude_domain:
|
||
include_list = ([d.strip() for d in args.include_domain.split(",") if d.strip()]
|
||
if args.include_domain else None)
|
||
exclude_list = ([d.strip() for d in args.exclude_domain.split(",") if d.strip()]
|
||
if args.exclude_domain else None)
|
||
before = len(results.get("results", []))
|
||
filter_results_by_domain(results, include_domains=include_list,
|
||
exclude_domains=exclude_list)
|
||
after = len(results.get("results", []))
|
||
logger.info(f"Domain filter: {before} -> {after} results")
|
||
|
||
if args.fetch > 0 and results.get("results"):
|
||
emit_progress("fetch_start", count=args.fetch)
|
||
# v2.0.0: Referer 默认设为首个实例 URL，伪装流量来自搜索引擎
|
||
referer = getattr(args, "referer", None)
|
||
if referer is None and instance_urls:
|
||
referer = instance_urls[0]
|
||
# v2.0.0: 创建共享 throttle 实例，用于 --fetch-report 输出
|
||
request_delay = getattr(args, "request_delay", 0.3)
|
||
fetch_throttle = AdaptiveThrottle(request_delay,
|
||
min(5, args.fetch))
|
||
fetched = fetch_top_results(
|
||
results, args.fetch,
|
||
timeout=args.fetch_timeout,
|
||
auth_headers=auth_headers,
|
||
max_retries=args.fetch_retries,
|
||
max_size=args.max_size,
|
||
request_delay=request_delay,
|
||
referer=referer,
|
||
fallback_enabled=not getattr(args, "no_fallback", False),
|
||
throttle=fetch_throttle,
|
||
)
|
||
# Emit fetch_ok / fetch_fail events
|
||
for f in fetched:
|
||
if f.get("status") == "ok":
|
||
emit_progress("fetch_ok", url=f.get("url", ""),
|
||
chars=f.get("text_length", 0),
|
||
fallback=f.get("fallback_used"))
|
||
else:
|
||
emit_progress("fetch_fail", url=f.get("url", ""),
|
||
error=f.get("error", "unknown"),
|
||
waf_type=f.get("waf_type"))
|
||
results["fetched"] = fetched
|
||
results["fetched_source"] = results.get("_fallback", "json")
|
||
|
||
# v2.0.0: --fetch-report 输出到 stderr
|
||
if getattr(args, "fetch_report", False):
|
||
_emit_fetch_report(fetched, fetch_throttle)
|
||
|
||
result_count = len(results.get("results", []))
|
||
emit_progress("done", results=result_count, query=query)
|
||
# 清理内部 _fallback 字段，避免泄漏到 JSON 输出。
|
||
# fetched_source（对 AI 有用的公开字段）已在 --fetch 路径中设置。
|
||
results.pop("_fallback", None)
|
||
return results, None, None
|
||
|
||
|
||
def _get_output_schema():
|
||
"""Return the JSON Schema describing --format json output.
|
||
|
||
Used by ``--dump-schema`` so AI agents can programmatically discover the
|
||
output structure without parsing prose documentation.
|
||
"""
|
||
return {
|
||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||
"title": "SearXNG CLI Search Result",
|
||
"schema_version": SCHEMA_VERSION,
|
||
"description": "Output schema for 'python search.py --format json' (single query). "
|
||
"Batch mode (--queries-file) wraps results in "
|
||
'{"schema_version, queries:[]}.',
|
||
"type": "object",
|
||
"properties": {
|
||
"schema_version": {
|
||
"type": "string",
|
||
"const": SCHEMA_VERSION,
|
||
"description": "Output schema version. Bump on breaking field changes.",
|
||
},
|
||
"query": {"type": "string", "description": "The search query string."},
|
||
"number_of_results": {
|
||
"type": "integer",
|
||
"description": "Total matches reported by SearXNG (JSON path) or "
|
||
"count of parsed results (HTML fallback path).",
|
||
},
|
||
"results": {
|
||
"type": "array",
|
||
"description": "Search result items, ordered by relevance (score desc).",
|
||
"items": {
|
||
"type": "object",
|
||
"properties": {
|
||
"title": {"type": "string"},
|
||
"url": {"type": "string", "format": "uri"},
|
||
"engine": {"type": "string", "description": "Source engine name."},
|
||
"score": {"type": ["number", "null"]},
|
||
"published_date": {"type": ["string", "null"]},
|
||
"content": {"type": "string", "description": "Snippet/summary text."},
|
||
},
|
||
"required": ["title", "url"],
|
||
},
|
||
},
|
||
"unresponsive_engines": {
|
||
"type": "array",
|
||
"items": {"type": "string"},
|
||
"description": "Engines that failed to respond.",
|
||
},
|
||
"suggestions": {
|
||
"type": "array",
|
||
"items": {"type": "string"},
|
||
"description": "Related query suggestions from the instance.",
|
||
},
|
||
"fetched": {
|
||
"type": "array",
|
||
"description": "Present only when --fetch N is used. Page content "
|
||
"for the top N results.",
|
||
"items": {
|
||
"type": "object",
|
||
"properties": {
|
||
"url": {"type": "string"},
|
||
"text": {"type": "string"},
|
||
"text_length": {"type": "integer"},
|
||
"error": {"type": "string"},
|
||
},
|
||
},
|
||
},
|
||
"fetched_source": {
|
||
"type": "string",
|
||
"enum": ["json", "html"],
|
||
"description": "Present only when --fetch is used. Indicates whether "
|
||
"search results came from JSON API or HTML fallback.",
|
||
},
|
||
},
|
||
"required": ["query", "results"],
|
||
}
|
||
|
||
|
||
def _emit_error(message: str, args, query: str = None, exit_code: int = 1,
|
||
error_code: str = None):
|
||
"""Emit an error and exit.
|
||
|
||
In ``--format json`` mode the error is printed to **stdout** as a
|
||
structured JSON object so agents piping stdout can parse it. All other
|
||
formats print to stderr (keeping stdout clean for data) and exit.
|
||
|
||
The JSON shape is::
|
||
|
||
{"error": "...", "exit_code": N, "error_code": "E_*",
|
||
"recovery_hint": "...", "query": "..."}
|
||
|
||
``error_code`` 是结构化错误码（E_CONFIG/E_AUTH/E_NETWORK 等），让 AI
|
||
Agent 程序化判断错误类型并采取恢复策略。``recovery_hint`` 给出可操作的
|
||
恢复建议，让 AI 能自决策下一步动作。``query`` 仅在提供时包含。
|
||
"""
|
||
if getattr(args, "format", None) == "json":
|
||
payload = {"error": message, "exit_code": exit_code}
|
||
if error_code:
|
||
payload["error_code"] = error_code
|
||
hint = RECOVERY_HINTS.get(error_code)
|
||
if hint:
|
||
payload["recovery_hint"] = hint
|
||
if query:
|
||
payload["query"] = query
|
||
print(json.dumps(payload, indent=2, ensure_ascii=False))
|
||
else:
|
||
prefix = f"[query: {query}] " if query else ""
|
||
code_prefix = f"[{error_code}] " if error_code else ""
|
||
hint_suffix = ""
|
||
if error_code and error_code in RECOVERY_HINTS:
|
||
hint_suffix = f"\n Hint: {RECOVERY_HINTS[error_code]}"
|
||
logger.error(f"{prefix}{code_prefix}Error: {message}{hint_suffix}")
|
||
sys.exit(exit_code)
|
||
|
||
|
||
def _read_queries_file(path: str) -> list:
|
||
"""Read queries from a file: one per line, skip blanks and ``#`` comments.
|
||
|
||
Raises :class:`RuntimeError` if the file cannot be read, so the caller
|
||
can route it through :func:`_emit_error`.
|
||
"""
|
||
try:
|
||
text = Path(path).read_text(encoding="utf-8")
|
||
except OSError as e:
|
||
raise RuntimeError(f"cannot read queries file '{path}': {e}")
|
||
queries = []
|
||
for line in text.splitlines():
|
||
line = line.strip()
|
||
if not line or line.startswith("#"):
|
||
continue
|
||
queries.append(line)
|
||
return queries
|
||
|
||
|
||
# ----- Main -----
|
||
|
||
def main():
|
||
# Phase 1: pre-scan for --config / --verbose / --quiet so logging is
|
||
# configured BEFORE config loading (which may emit warnings).
|
||
pre = argparse.ArgumentParser(add_help=False)
|
||
pre.add_argument("--config", default=None)
|
||
pre.add_argument("--verbose", "-v", action="store_true", default=False)
|
||
pre.add_argument("--quiet", action="store_true", default=False)
|
||
pre_args, _ = pre.parse_known_args()
|
||
|
||
setup_logging(verbose=pre_args.verbose, quiet=pre_args.quiet)
|
||
force_utf8_stdout() # Windows: prevent GBK crash on non-ASCII chars
|
||
|
||
# Load config defaults from --config file, else ./searxng.toml or
|
||
# ~/.config/searxng-cli/searxng.toml. Every CLI flag below can be
|
||
# pre-set here; explicit flags still win.
|
||
config = load_config(pre_args.config)
|
||
|
||
parser = argparse.ArgumentParser(
|
||
description="Search via a user-supplied SearXNG instance",
|
||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||
epilog=f"""searxng-cli v{VERSION}
|
||
|
||
Examples:
|
||
%(prog)s -q "python asyncio" -i https://my-searxng.example.com
|
||
%(prog)s -q "machine learning" -i https://a.example.com,https://b.example.com --format brief
|
||
%(prog)s -q "climate change" -i https://s.example.com --time-range month --language en
|
||
%(prog)s -q "open source" -i https://s.example.com --format urls > result_urls.txt
|
||
%(prog)s -q "ai news" -i https://s.example.com --engines google,duckduckgo --method POST
|
||
%(prog)s --verify -i https://a.example.com,https://b.example.com # health-check, no search
|
||
%(prog)s -q "latest news" -i https://s.example.com --time-range none # disable time filter
|
||
%(prog)s -q "test" --config ./my-config.toml -i https://s.example.com # explicit config file
|
||
|
||
Instance URLs are resolved from (in priority order):
|
||
1. -i/--instance (comma-separated for parallel failover)
|
||
2. SEARXNG_INSTANCE environment variable
|
||
3. ./searxng.toml or ~/.config/searxng-cli/searxng.toml (instance = "..." or instances = [...])
|
||
./instances.txt or ~/.config/searxng-cli/instances.txt (one URL per line)
|
||
At least one instance is REQUIRED (public-instance discovery has been removed).
|
||
Use --verify to health-check your instances (reachability / JSON support / latency).
|
||
|
||
searxng.toml may ALSO set defaults for most flags below (engines, categories, language,
|
||
safesearch, time_range, method, format, sort_by, timeout, max_retries, proxy, cache_ttl,
|
||
fetch, fetch_timeout, fetch_retries, max_size). Explicit CLI flags always override config values.
|
||
Use --config FILE to load a non-default config file (overrides the auto-discovered one).
|
||
""",
|
||
)
|
||
parser.add_argument("--config", default=None, metavar="FILE",
|
||
help="Path to a searxng.toml config file. Overrides the default "
|
||
"auto-discovery (./searxng.toml -> ~/.config/searxng-cli/searxng.toml). "
|
||
"Must be the first flag if you want its values to set defaults "
|
||
"for other flags.")
|
||
parser.add_argument("--verbose", "-v", action="store_true", default=False,
|
||
help="Verbose output: show debug-level diagnostics (HTTP request URLs, "
|
||
"response codes, cache keys, etc.) on stderr")
|
||
parser.add_argument("--quiet", action="store_true", default=False,
|
||
help="Quiet output: suppress progress messages and retry notices on stderr; "
|
||
"only warnings and errors are shown. (No short flag: -q is --query)")
|
||
parser.add_argument("--query", "-q", required=False, default=None,
|
||
help="Search query (required unless --verify / --queries-file is used)")
|
||
parser.add_argument("--instance", "-i", required=False, default=None,
|
||
help="SearXNG instance URL(s), comma-separated for failover. "
|
||
"Optional if SEARXNG_INSTANCE env var or a config file is set.")
|
||
parser.add_argument("--categories", "-c", default=config.get("categories"),
|
||
help="Comma-separated categories (e.g. general,images,news)")
|
||
parser.add_argument("--language", "-l", default=config.get("language"),
|
||
help="Language code (e.g. en, zh-CN, de)")
|
||
parser.add_argument("--pageno", "-p", type=int, default=1,
|
||
help="Page number (default: 1)")
|
||
parser.add_argument("--time-range", "-t", choices=["day", "month", "year", "none"],
|
||
default=config.get("time_range", "year"),
|
||
help="Time range filter (default: year; 'none' disables filtering)")
|
||
parser.add_argument("--safesearch", "-s", type=int, choices=[0, 1, 2],
|
||
default=_cfg_int(config, "safesearch", 0),
|
||
help="Safe search: 0=off, 1=moderate, 2=strict (default: 0=off)")
|
||
parser.add_argument("--engines",
|
||
default=config.get("engines", "google,bing,brave,duckduckgo,startpage,wikipedia,wikidata"),
|
||
help="Comma-separated engine names "
|
||
"(default: google,bing,brave,duckduckgo,startpage,wikipedia,wikidata)")
|
||
parser.add_argument("--method", choices=["GET", "POST"],
|
||
default=config.get("method", "GET"),
|
||
help="HTTP method (default: GET)")
|
||
parser.add_argument("--max-results", type=int, default=None,
|
||
help="Limit number of results (applied AFTER dedup+sort, "
|
||
"so the highest-scoring/newest items are kept)")
|
||
parser.add_argument("--sort-by", choices=["score", "date", "engine", "none"],
|
||
default=config.get("sort_by", "score"),
|
||
help="Sort results (default: score descending; 'none' preserves "
|
||
"instance order). Applied after dedup, before --max-results. "
|
||
"HTML-fallback results have no score and keep their order.")
|
||
parser.add_argument("--no-dedup", action="store_true",
|
||
help="Disable cross-engine deduplication (by default, duplicate "
|
||
"URLs — same page ignoring tracking params/fragment — are "
|
||
"collapsed, keeping the first occurrence's engine/score)")
|
||
parser.add_argument("--format", "-f", choices=["json", "brief", "urls", "csv"],
|
||
default=config.get("format", "json"),
|
||
help="Output format (default: json). 'csv' exports "
|
||
"title,url,engine,score,published_date,content.")
|
||
parser.add_argument("--snippet-len", type=int, default=0,
|
||
help="Snippet length in brief format (default: 0 = full, no truncation)")
|
||
parser.add_argument("--fetch", type=int, default=_cfg_int(config, "fetch", 0), metavar="N",
|
||
help="After search, auto-fetch full text of top N result pages")
|
||
parser.add_argument("--fetch-timeout", type=int, default=_cfg_int(config, "fetch_timeout", 10),
|
||
help="Timeout per page fetch in seconds (default: 10)")
|
||
parser.add_argument("--fetch-retries", type=int, default=_cfg_int(config, "fetch_retries", 3),
|
||
help="Max retries per page fetch (default: 3)")
|
||
parser.add_argument("--fetch-report", action="store_true",
|
||
help="When used with --fetch, emit a structured fetch report to stderr "
|
||
"after completion: per-URL status, UA used, attempts, WAF type, "
|
||
"fallback used, and adaptive throttle stats. v2.0.0.")
|
||
parser.add_argument("--no-fallback", action="store_true",
|
||
help="Disable Wayback Machine fallback for failed fetches (404/403/timeout). "
|
||
"By default Wayback fallback is ENABLED to maximize success rate. v2.0.0.")
|
||
parser.add_argument("--referer", default=None, metavar="URL",
|
||
help="Set Referer header for fetch requests (e.g. the SearXNG instance URL). "
|
||
"Defaults to the instance URL when fetching result pages. v2.0.0.")
|
||
parser.add_argument("--request-delay", type=float,
|
||
default=_cfg_float(config, "request_delay", 0.3),
|
||
metavar="SECONDS",
|
||
help="Delay between fetch requests to avoid rate limiting (default: 0.3s). "
|
||
"v2.0.0: adaptive throttling may increase this on consecutive failures.")
|
||
parser.add_argument("--max-size", type=int, default=_cfg_int(config, "max_size", None),
|
||
metavar="BYTES",
|
||
help="Max page size in bytes (default: unlimited). Set to 5242880 for 5MB cap.")
|
||
parser.add_argument("--auth-bearer", default=None, metavar="TOKEN",
|
||
help="Authorization: Bearer <TOKEN> for private instances")
|
||
parser.add_argument("--auth-bearer-file", default=None, metavar="FILE",
|
||
help="Read Bearer token from a file (first non-empty, non-# line). "
|
||
"Avoids leaving tokens in shell history.")
|
||
parser.add_argument("--auth-basic", default=None, metavar="USER:PASS",
|
||
help="Authorization: Basic base64(user:pass) for private instances")
|
||
parser.add_argument("--auth-basic-file", default=None, metavar="FILE",
|
||
help="Read basic auth 'user:pass' from a file (first non-empty, non-# line). "
|
||
"Avoids leaving passwords in shell history. "
|
||
"Env var SEARXNG_BASIC_AUTH is also honored.")
|
||
parser.add_argument("--output", "-o", default=None,
|
||
help="Save to file instead of stdout")
|
||
parser.add_argument("--stream", action="store_true",
|
||
help="Stream results as JSON Lines (one JSON object per line) to stdout. "
|
||
"Each line is a {\"type\": \"result\", \"result\": {...}} event. "
|
||
"Ends with {\"type\": \"done\", \"count\": N}. "
|
||
"AI Agent can process results incrementally without waiting for full output. "
|
||
"Only valid with --format json.")
|
||
parser.add_argument("--progress", action="store_true",
|
||
help="Emit structured progress events as JSON Lines to stderr. "
|
||
"Events: start, instance_try, instance_ok, instance_fail, "
|
||
"cache_hit, cache_store, fetch_start, fetch_ok, fetch_fail, done. "
|
||
"AI Agent can track execution progress programmatically.")
|
||
parser.add_argument("--timeout", type=int, default=_cfg_int(config, "timeout", 15),
|
||
help="Request timeout in seconds (default: 15)")
|
||
parser.add_argument("--retry", type=int, default=_cfg_int(config, "max_retries", None),
|
||
help="Max retries per instance (default: 3)")
|
||
parser.add_argument("--fail-fast", action="store_true",
|
||
help="Exit on first instance failure (no multi-instance retry)")
|
||
parser.add_argument("--serial", action="store_true",
|
||
help="Disable parallel multi-instance probing; search instances one at a time")
|
||
parser.add_argument("--proxy", default=config.get("proxy"), metavar="URL",
|
||
help="HTTP/HTTPS proxy URL (e.g. http://corp-proxy:8080). "
|
||
"Applies to both search and fetch requests. "
|
||
"Respects existing HTTP_PROXY/HTTPS_PROXY env vars when omitted.")
|
||
parser.add_argument("--include-domain", default=None, metavar="DOMAINS",
|
||
help="Comma-separated allowlist; only results from these domains are kept "
|
||
"(e.g. 'example.com,wikipedia.org'). Applied after search.")
|
||
parser.add_argument("--exclude-domain", default=None, metavar="DOMAINS",
|
||
help="Comma-separated blocklist; results from these domains are dropped "
|
||
"(e.g. 'pinterest.com,quora.com'). Applied after search.")
|
||
parser.add_argument("--queries-file", default=None, metavar="FILE",
|
||
help="Read queries from a file (one per line; blank lines and lines "
|
||
"starting with '#' are skipped) and run them in sequence. "
|
||
"Results are emitted as a JSON array (or one brief block per query). "
|
||
"Overrides --query when set.")
|
||
parser.add_argument("--verify", action="store_true",
|
||
help="Health-check mode: verify instances (reachability/JSON/latency) and exit without searching")
|
||
parser.add_argument("--cache-ttl", type=int, default=_cfg_int(config, "cache_ttl", 0),
|
||
metavar="MINUTES",
|
||
help="Cache search results for N minutes (default: 0 = disabled). "
|
||
"Identical queries within the TTL skip the network entirely. "
|
||
"Cache lives at $SEARXNG_CACHE_DIR or ~/.cache/searxng-cli/cache.db")
|
||
parser.add_argument("--clear-cache", action="store_true",
|
||
help="Delete all cached entries and exit (no search performed)")
|
||
parser.add_argument("--cache-stats", action="store_true",
|
||
help="Print cache statistics (entry count, age, size, path) and exit")
|
||
parser.add_argument("--dump-schema", action="store_true",
|
||
help="Print the JSON Schema for --format json output and exit. "
|
||
"Lets AI agents programmatically discover field names and types.")
|
||
parser.add_argument("--version", action="version", version=f"searxng-cli v{VERSION}")
|
||
|
||
args = parser.parse_args()
|
||
|
||
# --dump-schema：输出 JSON Schema 到 stdout 并退出，AI Agent 可程序化发现字段
|
||
if getattr(args, "dump_schema", False):
|
||
print(json.dumps(_get_output_schema(), indent=2, ensure_ascii=False))
|
||
sys.exit(0)
|
||
|
||
# 启用 --progress 进度事件（JSON Lines 到 stderr）
|
||
set_progress_enabled(getattr(args, "progress", False))
|
||
|
||
# --stream 只在单查询 + --format json 下有效。batch 模式输出 JSON 数组，
|
||
# 非 json 格式无 JSON Lines 语义；两种组合都显式报错 E_INPUT，避免静默失效
|
||
# 让 AI Agent 误以为流式输出已生效。
|
||
if getattr(args, "stream", False):
|
||
if args.queries_file:
|
||
_emit_error("--stream cannot be used with --queries-file: batch mode "
|
||
"emits a JSON array, not JSON Lines. Drop --stream for "
|
||
"batch output, or use a single --query with --stream.",
|
||
args, error_code=E_INPUT)
|
||
if args.format != "json":
|
||
_emit_error(f"--stream requires --format json (current: {args.format}). "
|
||
"JSON Lines streaming only produces valid output with json format.",
|
||
args, error_code=E_INPUT)
|
||
|
||
# --query is required unless we're doing a non-search operation.
|
||
# --queries-file is an alternative to --query for batch mode.
|
||
if (not args.verify and not args.query and not args.queries_file
|
||
and not args.clear_cache and not args.cache_stats):
|
||
parser.error("--query is required (or use --verify / --queries-file / "
|
||
"--clear-cache / --cache-stats)")
|
||
|
||
# Apply proxy early so every HTTP path (search, verify, fetch) honors it.
|
||
# Setting env vars is enough: urllib reads them via getproxies() and
|
||
# requests honors them via trust_env=True (its default).
|
||
if args.proxy:
|
||
apply_proxy(args.proxy)
|
||
logger.info(f"Proxy: {args.proxy}")
|
||
|
||
# --clear-cache / --cache-stats 不需要实例，在实例解析之前处理并退出。
|
||
# 避免无 -i 时报 E_CONFIG "no instance resolved" 让 AI 困惑。
|
||
if args.clear_cache:
|
||
removed = cache_module.clear()
|
||
logger.info(f"Cleared {removed} cache entr{'y' if removed == 1 else 'ies'}.")
|
||
sys.exit(0)
|
||
|
||
if args.cache_stats:
|
||
s = cache_module.stats()
|
||
if args.format == "json":
|
||
# JSON 模式：结构化数据走 stdout，AI Agent 可管道解析
|
||
print(json.dumps(s, indent=2, ensure_ascii=False))
|
||
else:
|
||
# 非 JSON 模式：人类可读的状态信息走 stderr，保持 stdout 纯净，
|
||
# 避免 AI Agent 用 --format json 解析 stdout 时被非 JSON 污染。
|
||
print(f"Cache path: {s.get('path', '?')}", file=sys.stderr)
|
||
print(f"Entries: {s.get('entries', 0)}", file=sys.stderr)
|
||
size = s.get("size_bytes", 0)
|
||
print(f"Size: {size:,} bytes ({size / 1024:.1f} KB)", file=sys.stderr)
|
||
if s.get("oldest_created_at"):
|
||
print(f"Oldest: {time.ctime(s['oldest_created_at'])}", file=sys.stderr)
|
||
if s.get("newest_created_at"):
|
||
print(f"Newest: {time.ctime(s['newest_created_at'])}", file=sys.stderr)
|
||
if s.get("error"):
|
||
print(f"Error: {s['error']}", file=sys.stderr)
|
||
sys.exit(0)
|
||
|
||
# Resolve instance(s): -i > SEARXNG_INSTANCE env > config file
|
||
instance_urls = resolve_instances(args.instance)
|
||
# Fallback: if --config was used, instance may be in the config dict
|
||
# but not in the default search paths that resolve_instances checks.
|
||
if not instance_urls and config.get("instance"):
|
||
instance_urls = parse_instances(config["instance"])
|
||
elif not instance_urls and config.get("instances"):
|
||
raw = config["instances"]
|
||
if isinstance(raw, str):
|
||
instance_urls = parse_instances(raw)
|
||
elif isinstance(raw, list):
|
||
instance_urls = [u if u.startswith(("http://", "https://"))
|
||
else "https://" + u for u in raw if u and str(u).strip()]
|
||
if not instance_urls:
|
||
# Python 3.8-3.10 无 tomllib 时，.toml 配置文件无法读取。检查这种
|
||
# 情况并在错误信息中附加提示，让 AI Agent 能给出可操作的恢复建议。
|
||
toml_hint = ""
|
||
try:
|
||
import tomllib # Python 3.11+
|
||
except ModuleNotFoundError:
|
||
try:
|
||
import tomli # type: ignore[import-not-found]
|
||
except ModuleNotFoundError:
|
||
toml_candidates = [
|
||
Path.cwd() / "searxng.toml",
|
||
Path.home() / ".config" / "searxng-cli" / "searxng.toml",
|
||
]
|
||
if any(p.exists() for p in toml_candidates):
|
||
toml_hint = (" (hint: a searxng.toml file exists but cannot be "
|
||
"read on Python < 3.11 without the 'tomli' package. "
|
||
"Run 'pip install tomli' or use instances.txt instead.)")
|
||
_emit_error("no SearXNG instance resolved. Provide -i/--instance, set the "
|
||
"SEARXNG_INSTANCE environment variable, or create a searxng.toml / "
|
||
"instances.txt config file." + toml_hint, args, error_code=E_CONFIG)
|
||
|
||
# Build auth headers if provided (needed by both verify and search).
|
||
# Credentials may come from CLI flag, file, config file, or env var
|
||
# (in priority order) to avoid leaking secrets via shell history or `ps`.
|
||
try:
|
||
bearer_token = resolve_auth_bearer(
|
||
args.auth_bearer, args.auth_bearer_file,
|
||
config_value=config.get("auth_bearer"))
|
||
basic_auth = resolve_auth_basic(
|
||
args.auth_basic, args.auth_basic_file,
|
||
config_value=config.get("auth_basic"))
|
||
except RuntimeError as e:
|
||
_emit_error(str(e), args, error_code=E_AUTH)
|
||
auth_headers = build_auth_headers(
|
||
bearer_token=bearer_token,
|
||
basic_auth=basic_auth,
|
||
)
|
||
if auth_headers:
|
||
auth_type = "Bearer" if bearer_token else "Basic"
|
||
logger.info(f"Auth: {auth_type} ***")
|
||
|
||
# Health-check mode: report instance status and exit (no search performed)
|
||
if args.verify:
|
||
logger.info(f"Verifying {len(instance_urls)} instance(s)...")
|
||
report = verify_instances(instance_urls, timeout=args.timeout,
|
||
auth_headers=auth_headers)
|
||
_print_verify_report(report, as_json=(args.format == "json"))
|
||
sys.exit(0)
|
||
|
||
if args.fail_fast:
|
||
instance_urls = instance_urls[:1]
|
||
|
||
logger.info(f"Instances to try: {len(instance_urls)}")
|
||
|
||
# Cache TTL in seconds (CLI takes minutes for ergonomics)
|
||
ttl_seconds = args.cache_ttl * 60 if args.cache_ttl > 0 else 0
|
||
|
||
# ----- Batch mode: --queries-file -----
|
||
# Reads one query per line (blank/# lines skipped) and runs them in
|
||
# sequence. Output is a JSON array (json format) or concatenated blocks
|
||
# (brief/urls). A failed query is recorded but does not abort the batch.
|
||
if args.queries_file:
|
||
try:
|
||
queries = _read_queries_file(args.queries_file)
|
||
except RuntimeError as e:
|
||
_emit_error(str(e), args, error_code=E_INPUT)
|
||
if not queries:
|
||
_emit_error(f"no queries found in '{args.queries_file}'", args,
|
||
error_code=E_INPUT)
|
||
|
||
logger.info(f"Running {len(queries)} queries from {args.queries_file}...")
|
||
batch = []
|
||
any_with_results = False
|
||
error_count = 0
|
||
for i, q in enumerate(queries, 1):
|
||
logger.info(f"\n[{i}/{len(queries)}] {q}")
|
||
results, err, err_code = _run_single_query(q, args, instance_urls,
|
||
auth_headers, ttl_seconds)
|
||
if err:
|
||
error_count += 1
|
||
logger.error(f" [ERROR] {err}")
|
||
entry = {"query": q, "status": "error", "error": err}
|
||
if err_code:
|
||
entry["error_code"] = err_code
|
||
batch.append(entry)
|
||
else:
|
||
if len(results.get("results", [])) > 0:
|
||
any_with_results = True
|
||
batch.append({"query": q, "status": "ok", "results": results})
|
||
|
||
if args.format == "json":
|
||
output = json.dumps({"schema_version": SCHEMA_VERSION, "queries": batch},
|
||
indent=2, ensure_ascii=False)
|
||
elif args.format == "csv":
|
||
import csv as csv_mod
|
||
import io
|
||
out = io.StringIO()
|
||
writer = csv_mod.writer(out, lineterminator="\n")
|
||
writer.writerow(["query", "title", "url", "engine", "score",
|
||
"published_date", "content"])
|
||
for br in batch:
|
||
q = br["query"]
|
||
if "results" in br:
|
||
for r in br["results"].get("results", []):
|
||
writer.writerow([
|
||
q,
|
||
r.get("title", ""),
|
||
r.get("url", ""),
|
||
r.get("engine", ""),
|
||
r.get("score", "") if r.get("score") is not None else "",
|
||
r.get("published_date", ""),
|
||
r.get("content", ""),
|
||
])
|
||
else:
|
||
writer.writerow([q, "", "", "", "", "",
|
||
f"[ERROR: {br['error']}]"])
|
||
output = out.getvalue().rstrip()
|
||
elif args.format == "urls":
|
||
parts = []
|
||
for br in batch:
|
||
parts.append(f"# {br['query']}")
|
||
if "results" in br:
|
||
parts.append(format_urls(br["results"]))
|
||
else:
|
||
parts.append(f"# [ERROR: {br['error']}]")
|
||
output = "\n".join(parts)
|
||
else: # brief
|
||
parts = []
|
||
for br in batch:
|
||
parts.append("=" * 60)
|
||
parts.append(f"QUERY: {br['query']}")
|
||
parts.append("=" * 60)
|
||
if "results" in br:
|
||
parts.append(_format_results(br["results"], args))
|
||
else:
|
||
parts.append(f"[ERROR: {br['error']}]")
|
||
parts.append("")
|
||
output = "\n".join(parts)
|
||
|
||
if args.output:
|
||
with open(args.output, "w", encoding="utf-8") as f:
|
||
f.write(output)
|
||
logger.info(f"Saved results to {args.output}")
|
||
else:
|
||
print(output)
|
||
# Exit code semantics — aligned with single-query mode so AI agents
|
||
# can use one consistent rule:
|
||
# 1 = all queries errored (fatal)
|
||
# 2 = no query returned any results (empty), though at least one
|
||
# searched successfully without error
|
||
# 0 = at least one query returned results
|
||
if error_count == len(queries):
|
||
sys.exit(1)
|
||
if not any_with_results:
|
||
sys.exit(2)
|
||
sys.exit(0)
|
||
|
||
# ----- Single query mode -----
|
||
results, err, err_code = _run_single_query(args.query, args, instance_urls,
|
||
auth_headers, ttl_seconds)
|
||
|
||
# --stream: JSON Lines 流式输出，每条结果一行，AI 可增量处理。
|
||
# stream 模式下所有输出（包括错误）都是单行 JSON，保持 JSON Lines 格式一致性。
|
||
# 非 stream 模式的错误走 _emit_error（多行 JSON 或 stderr 文本）。
|
||
if getattr(args, "stream", False) and args.format == "json":
|
||
if err:
|
||
error_event = {"type": "error", "error": err, "query": args.query}
|
||
if err_code:
|
||
error_event["error_code"] = err_code
|
||
hint = RECOVERY_HINTS.get(err_code)
|
||
if hint:
|
||
error_event["recovery_hint"] = hint
|
||
print(json.dumps(error_event, ensure_ascii=False), flush=True)
|
||
sys.exit(1)
|
||
for r in results.get("results", []):
|
||
print(json.dumps({"type": "result", "result": r},
|
||
ensure_ascii=False), flush=True)
|
||
print(json.dumps({"type": "done",
|
||
"schema_version": SCHEMA_VERSION,
|
||
"count": len(results.get("results", [])),
|
||
"query": args.query}, ensure_ascii=False), flush=True)
|
||
if not results.get("results"):
|
||
sys.exit(2)
|
||
sys.exit(0)
|
||
|
||
if err:
|
||
_emit_error(err, args, query=args.query, error_code=err_code)
|
||
|
||
output = _format_results(results, args)
|
||
if args.output:
|
||
with open(args.output, "w", encoding="utf-8") as f:
|
||
f.write(output)
|
||
logger.info(f"Saved results to {args.output}")
|
||
else:
|
||
print(output)
|
||
|
||
if not results.get("results"):
|
||
logger.warning("Warning: no results returned")
|
||
sys.exit(2)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|