feat(v2.3.0): fetch 结构化 JSON 契约 + 并发批量 + 真实并发门控
迭代 1 — 正确性修复:
- 修复 --pages N 多页聚合的 unresponsive-engine 警告误判: 原用循环末次
cached 变量判断, 缓存命中时警告被错误跳过/误触发; 改用独立
performed_live_query 标记
- UA 池单一来源: 删除 common.py 手工副本 _FALLBACK_UAS_BUILTIN,
FALLBACK_UAS 直接引用 _config.UA_POOL, 消除双份漂移
- search_html 解码修复: 硬编码 utf-8 改为 detect_charset(header/meta
自动检测), 新增 --encoding 强制覆盖, 贯穿 search_multi 全链
- AdaptiveThrottle 真实并发门控: acquire_slot()/release_slot() 槽位机制,
退避降并发后新请求被快速拒绝(E_RATE_LIMIT), 实现持久降并发而非名义降并发
迭代 2 — fetch JSON 契约 + 批量并发:
- fetch.py --format json: 成功 {status,url,final_url,content_type,extract,
truncated,text_length,user_agent}; 失败 {status,error,error_code,
status_code,url}, 对齐 search.py 错误码体系
- fetch_page 采集 title + latency, 填充 --fetch-report json 空字段
- --queries-file --parallel-queries N (1-8): 并发批量, 输出保序, 受
AdaptiveThrottle 门控; 并发模式禁用 --fetch(嵌套并行不安全)
- queries 文件编码自动检测 (UTF-8 → GBK 回退)
迭代 3 — 工程化:
- 新增 pyproject.toml (searxng-search/searxng-fetch 入口点)
- 收敛 20+ 处函数内冗余导入
- --dump-schema 扩展: fetched.items 补全 15 字段, 新增 defs.batch/research
- 新增 17 个测试 (tests/test_v230_features.py), 全量 561 测试通过
- 文档同步 (SKILL.md/README.md, 版本号 2.3.0)
This commit is contained in:
@@ -0,0 +1,42 @@
|
||||
[build-system]
|
||||
requires = ["setuptools>=61.0"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "searxng-use-cli"
|
||||
version = "2.3.0"
|
||||
description = "Privacy-respecting SearXNG search + web-fetch CLI toolkit for AI agents"
|
||||
readme = "README.md"
|
||||
license = { text = "MIT" }
|
||||
requires-python = ">=3.8"
|
||||
keywords = ["searxng", "search", "web-scraping", "ai-agent", "cli", "privacy"]
|
||||
classifiers = [
|
||||
"Environment :: Console",
|
||||
"Intended Audience :: Developers",
|
||||
"License :: OSI Approved :: MIT License",
|
||||
"Operating System :: OS Independent",
|
||||
"Programming Language :: Python :: 3",
|
||||
"Programming Language :: Python :: 3.8",
|
||||
"Programming Language :: Python :: 3.9",
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Topic :: Internet :: WWW/HTTP :: Indexing/Search",
|
||||
"Topic :: Software Development :: Libraries :: Python Modules",
|
||||
]
|
||||
dependencies = []
|
||||
|
||||
[project.optional-dependencies]
|
||||
fetch = ["requests", "beautifulsoup4"]
|
||||
toml = ["tomli; python_version < '3.11'"]
|
||||
all = ["requests", "beautifulsoup4", "tomli; python_version < '3.11'"]
|
||||
test = ["pytest", "requests", "beautifulsoup4"]
|
||||
|
||||
[project.scripts]
|
||||
searxng-search = "scripts.search:main"
|
||||
searxng-fetch = "scripts.fetch:main"
|
||||
|
||||
[tool.setuptools]
|
||||
packages = ["scripts"]
|
||||
|
||||
[tool.setuptools.package-data]
|
||||
scripts = ["*.py"]
|
||||
Reference in New Issue
Block a user