MetonaTeam/searxng-use-cli
2
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(v2.5.0): Sec-Ch-Ua 头修复 + 结构化提取 + token 预算 + 正文去重 + 连接复用

Browse Source 
正确性修复:
- 修复 Sec-Ch-Ua 构造 bug: 原实现产出 ""Not_A Brand";v="99"" 双重引号
  畸形头(Chrome/Edge 两路径), 严格校验的 WAF 会忽略; 改为品牌数组拼接
- search 403 快速失败: 实例级 403 不再退避重试(~10.5s 空等), 立即 failover
- AdaptiveThrottle: --throttle-failure-threshold 0 现为真正禁用语义
- number_of_results 缺失时用 len(results) 兜底(JSON/HTML 路径契约对齐)
- 版本对齐: pyproject.toml 与 _config.py 同步 2.5.0

AI 代理体验:
- fetch.py --extract json: 结构化骨架(title/meta/headings/links/images)
- fetch.py --max-chars N: 提取后语义级截断(区别于 --max-size 字节截断)
- search --fetch-total-chars N: --fetch 全局字符预算, 耗尽后 status=skipped
- --dedup-fetched-content: 抓取正文 SimHash 去重, status=duplicate
- --progress 新增 angle_start/ok/fail + fetch_skip/fetch_duplicate 事件
- fetch.py 补齐 --log-format json + --dump-schema
- CSV 媒体列自适应(images/videos 类别自动追加媒体字段列)
- --dry-run 批量模式打印实际查询列表
- search 连接复用: requests 可用时走模块级 Session(连接池)

工程治理:
- 新增 scripts/release_check.py 发布一致性检查(版本/错误码表漂移)
- 新增 tests/test_v250_features.py 46+4 个回归测试(全量 622 通过)
- tests/conftest.py: autouse fixture 强制 stdlib 路径(本机有 requests 时
  既有 urllib mock 测试不失效)
This commit is contained in:
thzxx
2026-08-07 15:19:17 +08:00
parent 471818074d
commit 10c01a3abd
10 changed files with 1746 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/fetch.py
+252 -7
View File
@@ -600,6 +600,145 @@ def html_to_markdown(html_content: str) -> str:
return converter.get_markdown()
# ----- Structured extraction (--extract json, v2.5.0) -----
# 输出页面的结构化骨架（title/meta/headings/links/images），AI Agent 可以
# 只看骨架就判断页面是否有用，无需拉全文——显著节省 token。
# 元素数量上限：防止超大页面（5MB HTML 有数千链接）撑爆输出
_MAX_HEADINGS = 50
_MAX_LINKS = 100
_MAX_IMAGES = 50
class _StructureExtractor(HTMLParser):
"""stdlib 结构化提取器（bs4 不可用时的回退）。
与 bs4 路径输出同一 shape：title / meta_description / headings / links /
images。heading 文本带层级前缀（"h2: 内容"），便于 AI 直接理解结构。
"""
def __init__(self):
super().__init__()
self.title = ""
self.meta_description = ""
self.headings = []
self.links = []
self.images = []
self._in_title = False
self._current_heading = None
self._heading_text = []
def handle_starttag(self, tag, attrs):
tag = tag.lower()
attrs_dict = dict(attrs)
if tag == "title":
self._in_title = True
elif tag == "meta":
name = (attrs_dict.get("name") or "").lower()
if name == "description" and attrs_dict.get("content"):
self.meta_description = attrs_dict["content"].strip()[:500]
elif tag in ("h1", "h2", "h3", "h4", "h5", "h6"):
self._current_heading = tag
self._heading_text = []
elif tag == "a" and attrs_dict.get("href"):
href = attrs_dict["href"].strip()
if href and href not in self.links and len(self.links) < _MAX_LINKS:
self.links.append(href)
elif tag == "img":
# data-src 兜底：懒加载站点的 src 常为空
src = attrs_dict.get("src") or attrs_dict.get("data-src") or ""
if src and src not in self.images and len(self.images) < _MAX_IMAGES:
self.images.append(src)
def handle_endtag(self, tag):
tag = tag.lower()
if tag == "title":
self._in_title = False
elif tag in ("h1", "h2", "h3", "h4", "h5", "h6") and self._current_heading:
text = " ".join(self._heading_text).strip()
if text and len(self.headings) < _MAX_HEADINGS:
self.headings.append(f"{self._current_heading}: {text}"[:200])
self._current_heading = None
def handle_data(self, data):
if self._in_title:
self.title += data
elif self._current_heading:
self._heading_text.append(data)
def _extract_structure_stdlib(html_content: str) -> dict:
"""stdlib HTMLParser 路径的结构化提取。"""
ex = _StructureExtractor()
try:
ex.feed(html_content)
except Exception:
pass # 解析失败时返回部分结果，绝不因提取失败而丢弃页面
return {
"title": ex.title.strip()[:300],
"meta_description": ex.meta_description[:500],
"headings": ex.headings,
"links": ex.links,
"images": ex.images,
}
def _extract_structure_bs4(html_content: str) -> dict:
"""bs4 路径的结构化提取（质量更高，优先使用）。"""
soup = _BeautifulSoup(html_content, "html.parser")
title = soup.title.get_text(strip=True)[:300] if soup.title else ""
meta_description = ""
m = soup.find("meta", attrs={"name": lambda v: v and v.lower() == "description"})
if m and m.get("content"):
meta_description = m["content"].strip()[:500]
headings = []
for tag in soup.find_all(["h1", "h2", "h3", "h4", "h5", "h6"]):
t = tag.get_text(" ", strip=True)
if t:
headings.append(f"{tag.name}: {t}"[:200])
if len(headings) >= _MAX_HEADINGS:
break
links, images = [], []
for a in soup.find_all("a", href=True):
href = a["href"].strip()
if href and href not in links:
links.append(href)
if len(links) >= _MAX_LINKS:
break
for img in soup.find_all("img"):
src = img.get("src") or img.get("data-src") or ""
if src and src not in images:
images.append(src)
if len(images) >= _MAX_IMAGES:
break
return {
"title": title,
"meta_description": meta_description,
"headings": headings,
"links": links,
"images": images,
}
def extract_structure(html_content: str) -> dict:
"""从 HTML 提取结构化骨架（--extract json 用）。
bs4 优先；stdlib HTMLParser 回退（两路径输出同一 shape）。
非 HTML 内容返回空结构（调用方判断 ``is_html`` 后再调用）。
"""
if _HAS_BS4:
try:
return _extract_structure_bs4(html_content)
except Exception:
pass
return _extract_structure_stdlib(html_content)
# ----- HTTP Fetch -----
# RETRY_BACKOFF_BASE, FALLBACK_UAS, detect_charset and is_retryable_error are
# imported from common.py (shared with search.py for a consistent retry policy).
@@ -1136,11 +1275,59 @@ def fetch_url(url: str, timeout=15, user_agent: str = None,
# ----- Main -----
def _get_output_schema() -> dict:
"""返回 fetch.py ``--format json`` 输出的 JSON Schema（v2.5.0）。
AI Agent 可程序化发现字段名与类型，无需解析散文文档。
成功 shape 的 ``extract`` 字段枚举四种模式；``--extract json`` 时
额外包含结构化骨架字段（title/meta_description/headings/links/images）。
"""
return {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "SearXNG CLI Fetch Result",
"description": "Output schema for 'python fetch.py --format json'. "
"With --extract json (v2.5.0) the success payload "
"includes structured skeleton fields "
"(title/meta_description/headings/links/images) "
"alongside the extracted text.",
"type": "object",
"properties": {
"status": {"type": "string", "enum": ["ok", "error"]},
"url": {"type": "string", "format": "uri"},
"final_url": {"type": "string",
"description": "URL after redirects / Wayback."},
"content_type": {"type": "string"},
"extract": {"type": "string",
"enum": ["text", "html", "markdown", "json"]},
"truncated": {"type": "boolean",
"description": "True when --max-size cut raw bytes or "
"--max-chars cut extracted text."},
"text_length": {"type": "integer"},
"user_agent": {"type": "string"},
"title": {"type": "string",
"description": "Page <title> (extract=json only)."},
"meta_description": {"type": "string",
"description": "Meta description (extract=json only)."},
"headings": {"type": "array", "items": {"type": "string"},
"description": "h1-h6 headings, prefixed with tag "
"(extract=json only)."},
"links": {"type": "array", "items": {"type": "string"},
"description": "Up to 100 unique hrefs (extract=json only)."},
"images": {"type": "array", "items": {"type": "string"},
"description": "Up to 50 image srcs (extract=json only)."},
"error": {"type": "string"},
"error_code": {"type": "string"},
"status_code": {"type": "integer"},
},
"required": ["status", "url"],
}
def _emit_fetch_result(args, output: str, url: str, final_url: str,
content_type: str, truncated: bool,
user_agent: str = None,
error: str = None, error_code: str = None,
status_code: int = None) -> None:
status_code: int = None,
structure: dict = None) -> None:
"""输出抓取结果到 stdout / --output 文件。
v2.3.0: ``--format json`` 提供结构化 JSON 契约，AI Agent 可程序化
@@ -1155,6 +1342,10 @@ def _emit_fetch_result(args, output: str, url: str, final_url: str,
失败 shape::
{"status": "error", "url", "error", "error_code", "status_code"}
v2.5.0: ``structure`` 参数携带 ``--extract json`` 的结构化骨架字段
（title/meta_description/headings/links/images），合并进 json 成功
payload。text 模式忽略该参数。
"""
if args.format == "json":
if error:
@@ -1174,6 +1365,8 @@ def _emit_fetch_result(args, output: str, url: str, final_url: str,
"text_length": len(output),
"user_agent": user_agent,
}
if structure:
payload.update(structure)
text = json.dumps(payload, indent=2, ensure_ascii=False)
else:
text = output
@@ -1200,9 +1393,16 @@ Examples:
%(prog)s -u https://example.com --format json structured JSON output
""",
)
parser.add_argument("--url", "-u", required=True, help="URL to fetch")
parser.add_argument("--extract", "-e", choices=["text", "html", "markdown"],
default="text", help="Extraction mode (default: text)")
parser.add_argument("--url", "-u", required=False,
help="URL to fetch (required unless --dump-schema)")
parser.add_argument("--extract", "-e",
choices=["text", "html", "markdown", "json"],
default="text",
help="Extraction mode (default: text). 'json' (v2.5.0) "
"outputs a structured JSON object with the page's "
"text plus its skeleton: title, meta_description, "
"headings, links, images — lets agents judge a page "
"without reading the full body. Implies --format json.")
parser.add_argument("--format", "-f", choices=["text", "json"], default="text",
help="Output format (default: text). 'json' emits a structured "
"JSON object {status, url, final_url, content_type, extract, "
@@ -1215,6 +1415,11 @@ Examples:
help="Max retries on transient errors (default: 3)")
parser.add_argument("--max-size", type=int, default=None, metavar="BYTES",
help="Max page size in bytes (default: unlimited). Set to 5242880 for 5MB cap.")
parser.add_argument("--max-chars", type=int, default=None, metavar="N",
help="v2.5.0: truncate EXTRACTED text to N chars "
"(semantic cut, after extraction; sets truncated=true "
"in json output). Distinct from --max-size which caps "
"raw response bytes. 0 = no limit (default).")
parser.add_argument("--user-agent", default=None,
help="Custom User-Agent header")
parser.add_argument("--encoding", default=None,
@@ -1251,16 +1456,43 @@ Examples:
parser.add_argument("--quiet", action="store_true", default=False,
help="Quiet output: suppress progress messages on stderr; "
"only warnings and errors are shown")
parser.add_argument("--log-format", choices=["text", "json"], default="text",
help="v2.5.0: log output format (default: text). 'json' "
"emits one JSON object per line for programmatic "
"parsing by AI Agents.")
parser.add_argument("--dump-schema", action="store_true",
help="v2.5.0: print the JSON Schema for --format json "
"output to stdout and exit. Lets AI agents discover "
"field names and types programmatically.")
parser.add_argument("--version", action="version", version=f"searxng-cli v{VERSION}")
args = parser.parse_args()
setup_logging(verbose=args.verbose, quiet=args.quiet)
setup_logging(verbose=args.verbose, quiet=args.quiet,
log_format=args.log_format)
force_utf8_stdout() # Windows: prevent GBK crash on non-ASCII chars
# --dump-schema：输出 JSON Schema 到 stdout 并退出，AI Agent 可程序化
# 发现字段（无需 --url）。必须在 URL 校验之前处理。
if args.dump_schema:
print(json.dumps(_get_output_schema(), indent=2, ensure_ascii=False))
sys.exit(0)
# --url 手动校验（required=False 是为了让 --dump-schema 单独可用）
if not args.url:
logger.error("Error: --url is required")
sys.exit(1)
if not args.url.startswith(("http://", "https://")):
logger.error("Error: URL must start with http:// or https://")
sys.exit(1)
# --extract json 隐含 --format json（结构化输出必须走 json 通道）。
# 用户显式传 --format text 时以 json 为准并提示，避免静默产出无意义文本。
if args.extract == "json" and args.format != "json":
logger.info("--extract json implies --format json; ignoring --format "
f"{args.format}")
args.format = "json"
# Apply proxy via env vars so both urllib and requests honor it.
if args.proxy:
apply_proxy(args.proxy)
@@ -1298,6 +1530,7 @@ Examples:
fatal_error = None
fatal_error_code = None
fatal_status_code = None
structure = None # v2.5.0: --extract json 的结构化骨架
try:
result = fetch_url(
@@ -1395,20 +1628,32 @@ Examples:
content.strip().startswith("<!") or
content.strip().startswith("<htm"))
if args.extract == "html":
if args.extract == "json":
# v2.5.0：结构化骨架 + 正文。骨架字段由 _emit_fetch_result 合并进
# json payload；output 保持为正文文本（供 text_length / --max-chars）。
structure = extract_structure(content) if is_html else {}
output = extract_text(content) if is_html else content
elif args.extract == "html":
output = content
elif args.extract == "markdown":
output = html_to_markdown(content) if is_html else content
else: # text
output = extract_text(content) if is_html else content
# v2.5.0: --max-chars 语义级截断（提取后按字符截断，区别于 --max-size
# 的原始字节截断）。截断后标记 truncated=true（json 契约可感知）。
if args.max_chars and args.max_chars > 0 and output and len(output) > args.max_chars:
output = output[:args.max_chars]
truncated = True
logger.info(f"Truncated extracted text to {args.max_chars} chars")
# Quality check (threshold: 500 chars)
if args.extract == "text" and len(output.strip()) < 500 and is_html:
logger.warning(f"Warning: extracted text is very short ({len(output.strip())} chars). "
"The page may be JS-heavy or use anti-bot protection.")
_emit_fetch_result(args, output, args.url, final_url, content_type,
truncated, user_agent)
truncated, user_agent, structure=structure)
if __name__ == "__main__":