feat(v2.5.0): Sec-Ch-Ua 头修复 + 结构化提取 + token 预算 + 正文去重 + 连接复用
正确性修复: - 修复 Sec-Ch-Ua 构造 bug: 原实现产出 ""Not_A Brand";v="99"" 双重引号 畸形头(Chrome/Edge 两路径), 严格校验的 WAF 会忽略; 改为品牌数组拼接 - search 403 快速失败: 实例级 403 不再退避重试(~10.5s 空等), 立即 failover - AdaptiveThrottle: --throttle-failure-threshold 0 现为真正禁用语义 - number_of_results 缺失时用 len(results) 兜底(JSON/HTML 路径契约对齐) - 版本对齐: pyproject.toml 与 _config.py 同步 2.5.0 AI 代理体验: - fetch.py --extract json: 结构化骨架(title/meta/headings/links/images) - fetch.py --max-chars N: 提取后语义级截断(区别于 --max-size 字节截断) - search --fetch-total-chars N: --fetch 全局字符预算, 耗尽后 status=skipped - --dedup-fetched-content: 抓取正文 SimHash 去重, status=duplicate - --progress 新增 angle_start/ok/fail + fetch_skip/fetch_duplicate 事件 - fetch.py 补齐 --log-format json + --dump-schema - CSV 媒体列自适应(images/videos 类别自动追加媒体字段列) - --dry-run 批量模式打印实际查询列表 - search 连接复用: requests 可用时走模块级 Session(连接池) 工程治理: - 新增 scripts/release_check.py 发布一致性检查(版本/错误码表漂移) - 新增 tests/test_v250_features.py 46+4 个回归测试(全量 622 通过) - tests/conftest.py: autouse fixture 强制 stdlib 路径(本机有 requests 时 既有 urllib mock 测试不失效)
This commit is contained in:
+1
-1
@@ -7,7 +7,7 @@ Retry settings and shared HTTP utilities now live in ``common.py`` so that
|
||||
both ``search.py`` and ``fetch.py`` share one consistent implementation.
|
||||
"""
|
||||
|
||||
VERSION = "2.4.0"
|
||||
VERSION = "2.5.0"
|
||||
SCHEMA_VERSION = "1.0"
|
||||
USER_AGENT = f"searxng-cli/{VERSION}"
|
||||
|
||||
|
||||
+38
-5
@@ -329,14 +329,25 @@ def build_browser_headers(user_agent: str, referer: str = None,
|
||||
|
||||
# Sec-Ch-Ua 系列仅 Chrome/Edge 发送，Firefox 不发
|
||||
if not is_firefox:
|
||||
# 从 UA 提取主版本号，构造 Sec-Ch-Ua
|
||||
# 从 UA 提取主版本号，构造 Sec-Ch-Ua。
|
||||
# v2.5.0 修复：原实现把已含引号的 not_a_brand 再包进 f-string 引号，
|
||||
# 生成 ""Not_A Brand";v="99"" 的畸形头——Chrome/Edge 两条路径都中招，
|
||||
# 严格校验 Sec-CH-UA 的 WAF（Cloudflare 等会校验与 UA 一致性）会直接
|
||||
# 忽略或判定不一致。现改为品牌数组统一拼接，产出合法格式：
|
||||
# Chrome: "Not_A Brand";v="99", "Chromium";v="138", "Google Chrome";v="138"
|
||||
# Edge: 上述 + , "Microsoft Edge";v="138"
|
||||
m = re.search(r"Chrome/(\d+)", user_agent)
|
||||
ver = m.group(1) if m else "131"
|
||||
not_a_brand = '"Not_A Brand";v="99"' if ver != "99" else '"Not/A)Brand";v="99"'
|
||||
headers["Sec-Ch-Ua"] = f'"{not_a_brand}", "Chromium";v="{ver}", "Google Chrome";v="{ver}"'
|
||||
not_a_brand = "Not_A Brand" if ver != "99" else "Not/A)Brand"
|
||||
brands = [
|
||||
f'"{not_a_brand}";v="99"',
|
||||
f'"Chromium";v="{ver}"',
|
||||
f'"Google Chrome";v="{ver}"',
|
||||
]
|
||||
if is_edge:
|
||||
# Edge 的品牌标识
|
||||
headers["Sec-Ch-Ua"] = headers["Sec-Ch-Ua"].rstrip('"') + f'", "Microsoft Edge";v="{ver}"'
|
||||
# Edge 额外声明 Microsoft Edge 品牌
|
||||
brands.append(f'"Microsoft Edge";v="{ver}"')
|
||||
headers["Sec-Ch-Ua"] = ", ".join(brands)
|
||||
headers["Sec-Ch-Ua-Mobile"] = '"?1"' if "Mobile" in user_agent else '"?0"'
|
||||
# 平台标识
|
||||
if "Windows" in user_agent:
|
||||
@@ -1054,3 +1065,25 @@ def is_similar(result_a: dict, result_b: dict, threshold: float = 0.85) -> bool:
|
||||
max_distance = max(0, min(3, int(round((1.0 - threshold) / 0.05))))
|
||||
return _hamming_distance(hash_a, hash_b) <= max_distance
|
||||
|
||||
|
||||
def texts_are_similar(text_a: str, text_b: str, threshold: float = 0.85) -> bool:
|
||||
"""判断两段正文是否近似重复（v2.5.0，供抓取正文去重用）。
|
||||
|
||||
镜像/转载站点的正文与原文高度相似但 URL 不同——URL 级去重无法合并，
|
||||
需要按正文指纹判定。复用与 :func:`is_similar` 相同的
|
||||
SimHash + 汉明距离逻辑，但针对**正文**而非标题：
|
||||
|
||||
* 归一化（小写/去标点/CJK 单字分词）后取前 1000 字符做指纹窗口——
|
||||
镜像页前 1000 字符通常一致，长文全量计算只会稀释指纹且增加开销
|
||||
* 空文本（len<窗口）返回 False，避免短错误文本误判为重复
|
||||
* threshold → 汉明距离阈值映射与 is_similar 一致（0.85→3 ... 1.0→0）
|
||||
"""
|
||||
norm_a = _normalize_title(text_a)[:1000]
|
||||
norm_b = _normalize_title(text_b)[:1000]
|
||||
if len(norm_a) < 5 or len(norm_b) < 5:
|
||||
return False
|
||||
hash_a = _simhash(norm_a)
|
||||
hash_b = _simhash(norm_b)
|
||||
max_distance = max(0, min(3, int(round((1.0 - threshold) / 0.05))))
|
||||
return _hamming_distance(hash_a, hash_b) <= max_distance
|
||||
|
||||
|
||||
+252
-7
@@ -600,6 +600,145 @@ def html_to_markdown(html_content: str) -> str:
|
||||
return converter.get_markdown()
|
||||
|
||||
|
||||
# ----- Structured extraction (--extract json, v2.5.0) -----
|
||||
# 输出页面的结构化骨架（title/meta/headings/links/images），AI Agent 可以
|
||||
# 只看骨架就判断页面是否有用，无需拉全文——显著节省 token。
|
||||
|
||||
# 元素数量上限：防止超大页面（5MB HTML 有数千链接）撑爆输出
|
||||
_MAX_HEADINGS = 50
|
||||
_MAX_LINKS = 100
|
||||
_MAX_IMAGES = 50
|
||||
|
||||
|
||||
class _StructureExtractor(HTMLParser):
|
||||
"""stdlib 结构化提取器（bs4 不可用时的回退）。
|
||||
|
||||
与 bs4 路径输出同一 shape：title / meta_description / headings / links /
|
||||
images。heading 文本带层级前缀（"h2: 内容"），便于 AI 直接理解结构。
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.title = ""
|
||||
self.meta_description = ""
|
||||
self.headings = []
|
||||
self.links = []
|
||||
self.images = []
|
||||
self._in_title = False
|
||||
self._current_heading = None
|
||||
self._heading_text = []
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
tag = tag.lower()
|
||||
attrs_dict = dict(attrs)
|
||||
if tag == "title":
|
||||
self._in_title = True
|
||||
elif tag == "meta":
|
||||
name = (attrs_dict.get("name") or "").lower()
|
||||
if name == "description" and attrs_dict.get("content"):
|
||||
self.meta_description = attrs_dict["content"].strip()[:500]
|
||||
elif tag in ("h1", "h2", "h3", "h4", "h5", "h6"):
|
||||
self._current_heading = tag
|
||||
self._heading_text = []
|
||||
elif tag == "a" and attrs_dict.get("href"):
|
||||
href = attrs_dict["href"].strip()
|
||||
if href and href not in self.links and len(self.links) < _MAX_LINKS:
|
||||
self.links.append(href)
|
||||
elif tag == "img":
|
||||
# data-src 兜底：懒加载站点的 src 常为空
|
||||
src = attrs_dict.get("src") or attrs_dict.get("data-src") or ""
|
||||
if src and src not in self.images and len(self.images) < _MAX_IMAGES:
|
||||
self.images.append(src)
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
tag = tag.lower()
|
||||
if tag == "title":
|
||||
self._in_title = False
|
||||
elif tag in ("h1", "h2", "h3", "h4", "h5", "h6") and self._current_heading:
|
||||
text = " ".join(self._heading_text).strip()
|
||||
if text and len(self.headings) < _MAX_HEADINGS:
|
||||
self.headings.append(f"{self._current_heading}: {text}"[:200])
|
||||
self._current_heading = None
|
||||
|
||||
def handle_data(self, data):
|
||||
if self._in_title:
|
||||
self.title += data
|
||||
elif self._current_heading:
|
||||
self._heading_text.append(data)
|
||||
|
||||
|
||||
def _extract_structure_stdlib(html_content: str) -> dict:
|
||||
"""stdlib HTMLParser 路径的结构化提取。"""
|
||||
ex = _StructureExtractor()
|
||||
try:
|
||||
ex.feed(html_content)
|
||||
except Exception:
|
||||
pass # 解析失败时返回部分结果，绝不因提取失败而丢弃页面
|
||||
return {
|
||||
"title": ex.title.strip()[:300],
|
||||
"meta_description": ex.meta_description[:500],
|
||||
"headings": ex.headings,
|
||||
"links": ex.links,
|
||||
"images": ex.images,
|
||||
}
|
||||
|
||||
|
||||
def _extract_structure_bs4(html_content: str) -> dict:
|
||||
"""bs4 路径的结构化提取（质量更高，优先使用）。"""
|
||||
soup = _BeautifulSoup(html_content, "html.parser")
|
||||
|
||||
title = soup.title.get_text(strip=True)[:300] if soup.title else ""
|
||||
|
||||
meta_description = ""
|
||||
m = soup.find("meta", attrs={"name": lambda v: v and v.lower() == "description"})
|
||||
if m and m.get("content"):
|
||||
meta_description = m["content"].strip()[:500]
|
||||
|
||||
headings = []
|
||||
for tag in soup.find_all(["h1", "h2", "h3", "h4", "h5", "h6"]):
|
||||
t = tag.get_text(" ", strip=True)
|
||||
if t:
|
||||
headings.append(f"{tag.name}: {t}"[:200])
|
||||
if len(headings) >= _MAX_HEADINGS:
|
||||
break
|
||||
|
||||
links, images = [], []
|
||||
for a in soup.find_all("a", href=True):
|
||||
href = a["href"].strip()
|
||||
if href and href not in links:
|
||||
links.append(href)
|
||||
if len(links) >= _MAX_LINKS:
|
||||
break
|
||||
for img in soup.find_all("img"):
|
||||
src = img.get("src") or img.get("data-src") or ""
|
||||
if src and src not in images:
|
||||
images.append(src)
|
||||
if len(images) >= _MAX_IMAGES:
|
||||
break
|
||||
|
||||
return {
|
||||
"title": title,
|
||||
"meta_description": meta_description,
|
||||
"headings": headings,
|
||||
"links": links,
|
||||
"images": images,
|
||||
}
|
||||
|
||||
|
||||
def extract_structure(html_content: str) -> dict:
|
||||
"""从 HTML 提取结构化骨架（--extract json 用）。
|
||||
|
||||
bs4 优先；stdlib HTMLParser 回退（两路径输出同一 shape）。
|
||||
非 HTML 内容返回空结构（调用方判断 ``is_html`` 后再调用）。
|
||||
"""
|
||||
if _HAS_BS4:
|
||||
try:
|
||||
return _extract_structure_bs4(html_content)
|
||||
except Exception:
|
||||
pass
|
||||
return _extract_structure_stdlib(html_content)
|
||||
|
||||
|
||||
# ----- HTTP Fetch -----
|
||||
# RETRY_BACKOFF_BASE, FALLBACK_UAS, detect_charset and is_retryable_error are
|
||||
# imported from common.py (shared with search.py for a consistent retry policy).
|
||||
@@ -1136,11 +1275,59 @@ def fetch_url(url: str, timeout=15, user_agent: str = None,
|
||||
|
||||
# ----- Main -----
|
||||
|
||||
def _get_output_schema() -> dict:
|
||||
"""返回 fetch.py ``--format json`` 输出的 JSON Schema（v2.5.0）。
|
||||
|
||||
AI Agent 可程序化发现字段名与类型，无需解析散文文档。
|
||||
成功 shape 的 ``extract`` 字段枚举四种模式；``--extract json`` 时
|
||||
额外包含结构化骨架字段（title/meta_description/headings/links/images）。
|
||||
"""
|
||||
return {
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": "SearXNG CLI Fetch Result",
|
||||
"description": "Output schema for 'python fetch.py --format json'. "
|
||||
"With --extract json (v2.5.0) the success payload "
|
||||
"includes structured skeleton fields "
|
||||
"(title/meta_description/headings/links/images) "
|
||||
"alongside the extracted text.",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"status": {"type": "string", "enum": ["ok", "error"]},
|
||||
"url": {"type": "string", "format": "uri"},
|
||||
"final_url": {"type": "string",
|
||||
"description": "URL after redirects / Wayback."},
|
||||
"content_type": {"type": "string"},
|
||||
"extract": {"type": "string",
|
||||
"enum": ["text", "html", "markdown", "json"]},
|
||||
"truncated": {"type": "boolean",
|
||||
"description": "True when --max-size cut raw bytes or "
|
||||
"--max-chars cut extracted text."},
|
||||
"text_length": {"type": "integer"},
|
||||
"user_agent": {"type": "string"},
|
||||
"title": {"type": "string",
|
||||
"description": "Page <title> (extract=json only)."},
|
||||
"meta_description": {"type": "string",
|
||||
"description": "Meta description (extract=json only)."},
|
||||
"headings": {"type": "array", "items": {"type": "string"},
|
||||
"description": "h1-h6 headings, prefixed with tag "
|
||||
"(extract=json only)."},
|
||||
"links": {"type": "array", "items": {"type": "string"},
|
||||
"description": "Up to 100 unique hrefs (extract=json only)."},
|
||||
"images": {"type": "array", "items": {"type": "string"},
|
||||
"description": "Up to 50 image srcs (extract=json only)."},
|
||||
"error": {"type": "string"},
|
||||
"error_code": {"type": "string"},
|
||||
"status_code": {"type": "integer"},
|
||||
},
|
||||
"required": ["status", "url"],
|
||||
}
|
||||
|
||||
def _emit_fetch_result(args, output: str, url: str, final_url: str,
|
||||
content_type: str, truncated: bool,
|
||||
user_agent: str = None,
|
||||
error: str = None, error_code: str = None,
|
||||
status_code: int = None) -> None:
|
||||
status_code: int = None,
|
||||
structure: dict = None) -> None:
|
||||
"""输出抓取结果到 stdout / --output 文件。
|
||||
|
||||
v2.3.0: ``--format json`` 提供结构化 JSON 契约，AI Agent 可程序化
|
||||
@@ -1155,6 +1342,10 @@ def _emit_fetch_result(args, output: str, url: str, final_url: str,
|
||||
失败 shape::
|
||||
|
||||
{"status": "error", "url", "error", "error_code", "status_code"}
|
||||
|
||||
v2.5.0: ``structure`` 参数携带 ``--extract json`` 的结构化骨架字段
|
||||
（title/meta_description/headings/links/images），合并进 json 成功
|
||||
payload。text 模式忽略该参数。
|
||||
"""
|
||||
if args.format == "json":
|
||||
if error:
|
||||
@@ -1174,6 +1365,8 @@ def _emit_fetch_result(args, output: str, url: str, final_url: str,
|
||||
"text_length": len(output),
|
||||
"user_agent": user_agent,
|
||||
}
|
||||
if structure:
|
||||
payload.update(structure)
|
||||
text = json.dumps(payload, indent=2, ensure_ascii=False)
|
||||
else:
|
||||
text = output
|
||||
@@ -1200,9 +1393,16 @@ Examples:
|
||||
%(prog)s -u https://example.com --format json structured JSON output
|
||||
""",
|
||||
)
|
||||
parser.add_argument("--url", "-u", required=True, help="URL to fetch")
|
||||
parser.add_argument("--extract", "-e", choices=["text", "html", "markdown"],
|
||||
default="text", help="Extraction mode (default: text)")
|
||||
parser.add_argument("--url", "-u", required=False,
|
||||
help="URL to fetch (required unless --dump-schema)")
|
||||
parser.add_argument("--extract", "-e",
|
||||
choices=["text", "html", "markdown", "json"],
|
||||
default="text",
|
||||
help="Extraction mode (default: text). 'json' (v2.5.0) "
|
||||
"outputs a structured JSON object with the page's "
|
||||
"text plus its skeleton: title, meta_description, "
|
||||
"headings, links, images — lets agents judge a page "
|
||||
"without reading the full body. Implies --format json.")
|
||||
parser.add_argument("--format", "-f", choices=["text", "json"], default="text",
|
||||
help="Output format (default: text). 'json' emits a structured "
|
||||
"JSON object {status, url, final_url, content_type, extract, "
|
||||
@@ -1215,6 +1415,11 @@ Examples:
|
||||
help="Max retries on transient errors (default: 3)")
|
||||
parser.add_argument("--max-size", type=int, default=None, metavar="BYTES",
|
||||
help="Max page size in bytes (default: unlimited). Set to 5242880 for 5MB cap.")
|
||||
parser.add_argument("--max-chars", type=int, default=None, metavar="N",
|
||||
help="v2.5.0: truncate EXTRACTED text to N chars "
|
||||
"(semantic cut, after extraction; sets truncated=true "
|
||||
"in json output). Distinct from --max-size which caps "
|
||||
"raw response bytes. 0 = no limit (default).")
|
||||
parser.add_argument("--user-agent", default=None,
|
||||
help="Custom User-Agent header")
|
||||
parser.add_argument("--encoding", default=None,
|
||||
@@ -1251,16 +1456,43 @@ Examples:
|
||||
parser.add_argument("--quiet", action="store_true", default=False,
|
||||
help="Quiet output: suppress progress messages on stderr; "
|
||||
"only warnings and errors are shown")
|
||||
parser.add_argument("--log-format", choices=["text", "json"], default="text",
|
||||
help="v2.5.0: log output format (default: text). 'json' "
|
||||
"emits one JSON object per line for programmatic "
|
||||
"parsing by AI Agents.")
|
||||
parser.add_argument("--dump-schema", action="store_true",
|
||||
help="v2.5.0: print the JSON Schema for --format json "
|
||||
"output to stdout and exit. Lets AI agents discover "
|
||||
"field names and types programmatically.")
|
||||
parser.add_argument("--version", action="version", version=f"searxng-cli v{VERSION}")
|
||||
|
||||
args = parser.parse_args()
|
||||
setup_logging(verbose=args.verbose, quiet=args.quiet)
|
||||
setup_logging(verbose=args.verbose, quiet=args.quiet,
|
||||
log_format=args.log_format)
|
||||
force_utf8_stdout() # Windows: prevent GBK crash on non-ASCII chars
|
||||
|
||||
# --dump-schema：输出 JSON Schema 到 stdout 并退出，AI Agent 可程序化
|
||||
# 发现字段（无需 --url）。必须在 URL 校验之前处理。
|
||||
if args.dump_schema:
|
||||
print(json.dumps(_get_output_schema(), indent=2, ensure_ascii=False))
|
||||
sys.exit(0)
|
||||
|
||||
# --url 手动校验（required=False 是为了让 --dump-schema 单独可用）
|
||||
if not args.url:
|
||||
logger.error("Error: --url is required")
|
||||
sys.exit(1)
|
||||
|
||||
if not args.url.startswith(("http://", "https://")):
|
||||
logger.error("Error: URL must start with http:// or https://")
|
||||
sys.exit(1)
|
||||
|
||||
# --extract json 隐含 --format json（结构化输出必须走 json 通道）。
|
||||
# 用户显式传 --format text 时以 json 为准并提示，避免静默产出无意义文本。
|
||||
if args.extract == "json" and args.format != "json":
|
||||
logger.info("--extract json implies --format json; ignoring --format "
|
||||
f"{args.format}")
|
||||
args.format = "json"
|
||||
|
||||
# Apply proxy via env vars so both urllib and requests honor it.
|
||||
if args.proxy:
|
||||
apply_proxy(args.proxy)
|
||||
@@ -1298,6 +1530,7 @@ Examples:
|
||||
fatal_error = None
|
||||
fatal_error_code = None
|
||||
fatal_status_code = None
|
||||
structure = None # v2.5.0: --extract json 的结构化骨架
|
||||
|
||||
try:
|
||||
result = fetch_url(
|
||||
@@ -1395,20 +1628,32 @@ Examples:
|
||||
content.strip().startswith("<!") or
|
||||
content.strip().startswith("<htm"))
|
||||
|
||||
if args.extract == "html":
|
||||
if args.extract == "json":
|
||||
# v2.5.0：结构化骨架 + 正文。骨架字段由 _emit_fetch_result 合并进
|
||||
# json payload；output 保持为正文文本（供 text_length / --max-chars）。
|
||||
structure = extract_structure(content) if is_html else {}
|
||||
output = extract_text(content) if is_html else content
|
||||
elif args.extract == "html":
|
||||
output = content
|
||||
elif args.extract == "markdown":
|
||||
output = html_to_markdown(content) if is_html else content
|
||||
else: # text
|
||||
output = extract_text(content) if is_html else content
|
||||
|
||||
# v2.5.0: --max-chars 语义级截断（提取后按字符截断，区别于 --max-size
|
||||
# 的原始字节截断）。截断后标记 truncated=true（json 契约可感知）。
|
||||
if args.max_chars and args.max_chars > 0 and output and len(output) > args.max_chars:
|
||||
output = output[:args.max_chars]
|
||||
truncated = True
|
||||
logger.info(f"Truncated extracted text to {args.max_chars} chars")
|
||||
|
||||
# Quality check (threshold: 500 chars)
|
||||
if args.extract == "text" and len(output.strip()) < 500 and is_html:
|
||||
logger.warning(f"Warning: extracted text is very short ({len(output.strip())} chars). "
|
||||
"The page may be JS-heavy or use anti-bot protection.")
|
||||
|
||||
_emit_fetch_result(args, output, args.url, final_url, content_type,
|
||||
truncated, user_agent)
|
||||
truncated, user_agent, structure=structure)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
#!/usr/bin/env python3
|
||||
"""发布前一致性检查（v2.5.0）。
|
||||
|
||||
检查版本号与错误码表在多个文件间是否漂移——历史上 pyproject.toml 与
|
||||
``_config.VERSION`` 曾不同步（v2.4.0 发布时前者停在 2.3.0）。发布前运行
|
||||
本脚本即可一次性发现：
|
||||
|
||||
1. 版本一致性：pyproject.toml <-> _config.py（同一来源），并警告
|
||||
README.md / SKILL.md 中提到的版本号是否与最新版本一致（文档是
|
||||
手工维护的，仅警告不阻断）
|
||||
2. 错误码一致性：common.py 的 ``E_*`` 常量 vs 各脚本实际使用 vs
|
||||
RECOVERY_HINTS 覆盖 vs README/SKILL 错误码表中提及的错误码
|
||||
|
||||
用法（项目根目录）::
|
||||
|
||||
python scripts/release_check.py # 完整检查
|
||||
python scripts/release_check.py --strict # 文档警告也视为失败
|
||||
|
||||
零依赖（stdlib），退出码 0=通过 / 1=有阻断问题。
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SCRIPTS_DIR = ROOT / "scripts"
|
||||
|
||||
# 文档中可能提及版本号的锚点模式（供提示）
|
||||
_VERSION_HINT_RE = re.compile(r"\b\d+\.\d+\.\d+\b")
|
||||
|
||||
# 错误码表（README.md 与 SKILL.md 均有）应包含的 E_* 代码，从 common.py
|
||||
# 动态提取，与源码单一来源保持一致。捕获组只取标识符本身（不含 = "）。
|
||||
_ERROR_CODE_RE = re.compile(r"^\s*(E_[A-Z_]+)\s*=\s*[\"']", re.MULTILINE)
|
||||
|
||||
|
||||
def _read(path: Path) -> str:
|
||||
try:
|
||||
return path.read_text(encoding="utf-8")
|
||||
except OSError as e:
|
||||
print(f" [WARN] 无法读取 {path}: {e}")
|
||||
return ""
|
||||
|
||||
|
||||
def check_version(strict: bool) -> list:
|
||||
"""检查版本号一致性。返回错误消息列表（空 = 通过）。"""
|
||||
errors = []
|
||||
|
||||
# 1. pyproject.toml 的 version 字段
|
||||
pyproject = ROOT / "pyproject.toml"
|
||||
pp = _read(pyproject)
|
||||
m = re.search(r"^version\s*=\s*[\"']([\d.]+)[\"']", pp, re.MULTILINE)
|
||||
pp_version = m.group(1) if m else None
|
||||
|
||||
# 2. _config.py 的 VERSION（权威来源）
|
||||
import importlib.util
|
||||
spec = importlib.util.spec_from_file_location("_config", SCRIPTS_DIR / "_config.py")
|
||||
cfg = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(cfg)
|
||||
cfg_version = cfg.VERSION
|
||||
|
||||
if pp_version and pp_version != cfg_version:
|
||||
errors.append(
|
||||
f"版本漂移: pyproject.toml={pp_version} 但 _config.VERSION={cfg_version}。"
|
||||
f"请同步 pyproject.toml 或改为从 _config 动态读取。"
|
||||
)
|
||||
elif not pp_version:
|
||||
errors.append("pyproject.toml 缺少 version 字段。")
|
||||
|
||||
# 3. 文档版本提示（仅警告）
|
||||
if cfg_version:
|
||||
for doc_name in ("README.md", "SKILL.md"):
|
||||
doc = _read(ROOT / doc_name)
|
||||
if not doc:
|
||||
continue
|
||||
mentioned = set(_VERSION_HINT_RE.findall(doc))
|
||||
# 文档可能同时提到旧版本历史（changelog 里 v2.3.0 等），
|
||||
# 只要"存在"当前版本号即视为已更新
|
||||
if cfg_version not in mentioned:
|
||||
msg = (f"文档 {doc_name} 未提及当前版本 {cfg_version} "
|
||||
f"(提到: {sorted(mentioned)[:6]}...)。请更新文档。")
|
||||
if strict:
|
||||
errors.append(msg)
|
||||
else:
|
||||
print(f" [WARN] {msg}")
|
||||
return errors
|
||||
|
||||
|
||||
def check_error_codes(strict: bool) -> list:
|
||||
"""检查错误码表一致性。返回错误消息列表（空 = 通过）。"""
|
||||
errors = []
|
||||
|
||||
common_src = _read(SCRIPTS_DIR / "common.py")
|
||||
codes = _ERROR_CODE_RE.findall(common_src)
|
||||
if not codes:
|
||||
errors.append("common.py 中未找到 E_* 常量定义。")
|
||||
return errors
|
||||
codes = sorted(codes)
|
||||
|
||||
# 1. 每个 E_* 是否有 recovery_hint
|
||||
hint_keys = set(re.findall(r'^\s*"?(E_[A-Z_]+)"?\s*:\s*[\(\["\']',
|
||||
common_src, re.MULTILINE))
|
||||
missing_hints = [c for c in codes if c not in hint_keys]
|
||||
if missing_hints:
|
||||
errors.append(f"缺少 RECOVERY_HINTS 的错误码: {missing_hints}")
|
||||
|
||||
# 2. README/SKILL 错误码表是否覆盖所有 E_*（文档漂移提示）
|
||||
for doc_name in ("README.md", "SKILL.md"):
|
||||
doc = _read(ROOT / doc_name)
|
||||
if not doc:
|
||||
continue
|
||||
missing_doc = [c for c in codes if c not in doc]
|
||||
if missing_doc:
|
||||
msg = (f"文档 {doc_name} 错误码表缺少: {missing_doc}。"
|
||||
f"源码已新增这些错误码，请更新文档表格。")
|
||||
if strict:
|
||||
errors.append(msg)
|
||||
else:
|
||||
print(f" [WARN] {msg}")
|
||||
return errors
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description="searxng-cli 发布前一致性检查")
|
||||
ap.add_argument("--strict", action="store_true",
|
||||
help="文档警告也视为失败（默认仅提示）")
|
||||
args = ap.parse_args()
|
||||
|
||||
errors = []
|
||||
errors += check_version(args.strict)
|
||||
errors += check_error_codes(args.strict)
|
||||
|
||||
if errors:
|
||||
print("✗ 检查未通过:")
|
||||
for e in errors:
|
||||
print(f" - {e}")
|
||||
return 1
|
||||
print("✓ 版本号与错误码一致性检查全部通过")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+354
-38
@@ -45,6 +45,7 @@ from common import (
|
||||
force_utf8_stdout,
|
||||
is_hard_blocked_domain,
|
||||
is_similar,
|
||||
texts_are_similar,
|
||||
parse_retry_after,
|
||||
resolve_auth_basic,
|
||||
resolve_auth_bearer,
|
||||
@@ -488,13 +489,19 @@ def _retry_with_backoff(fn, max_retries: int = MAX_RETRIES, base_delay: float =
|
||||
v2.1.0 修复：复用 common.compute_backoff_delay（带 60s 封顶），
|
||||
避免高重试次数（如 --retry 10）时 1.5*2^10=1536s 卡死进程。
|
||||
同时遵守 Retry-After header（429/503），与 fetch.py 保持一致。
|
||||
|
||||
v2.5.0：403 不再重试。RETRYABLE_STATUS 含 403 是为 fetch.py 的 UA
|
||||
轮换设计的（每次重试换新 UA），但本函数仅供 search 使用——search 不
|
||||
换 UA，实例级 403 是认证/封禁问题，重试只会空等 ~10.5s 后才 failover。
|
||||
现在 403 立即 raise，由 search_multi 直接切换下一实例，classify_error
|
||||
仍正确归为 E_AUTH（实例认证失败语义不变）。
|
||||
"""
|
||||
last_error = None
|
||||
for attempt in range(max_retries + 1):
|
||||
try:
|
||||
return fn()
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code in RETRYABLE_STATUS: # 403 (UA block) + 429 + 5xx
|
||||
if e.code in RETRYABLE_STATUS and e.code != 403: # 429 + 5xx
|
||||
last_error = e
|
||||
if attempt < max_retries:
|
||||
# 429/503：遵守 Retry-After header，避免触发更严厉限流
|
||||
@@ -523,23 +530,115 @@ def _retry_with_backoff(fn, max_retries: int = MAX_RETRIES, base_delay: float =
|
||||
|
||||
# ----- Search execution -----
|
||||
|
||||
# v2.5.0 连接复用：requests 可用时复用模块级 Session（连接池 + TLS 会话
|
||||
# 恢复），--pages/批量/研究模式的多请求场景显著减少 TLS 握手开销。
|
||||
# stdlib 路径保持零依赖可用（每次新建连接，功能等价）——与 fetch.py 的
|
||||
# "requests 路径受益，stdlib 路径功能完整" 设计一致。
|
||||
_HAS_REQUESTS = False
|
||||
try:
|
||||
import requests as _requests # type: ignore
|
||||
_HAS_REQUESTS = True
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
_session = None
|
||||
|
||||
|
||||
def _get_session():
|
||||
"""获取（惰性创建）模块级 requests.Session。
|
||||
|
||||
连接池：每主机最多 10 连接，总最多 20 连接。max_retries=0 让 requests
|
||||
不做自己的重试——重试统一由 _retry_with_backoff 控制，避免双重退避。
|
||||
"""
|
||||
global _session
|
||||
if _session is None and _HAS_REQUESTS:
|
||||
_session = _requests.Session()
|
||||
adapter = _requests.adapters.HTTPAdapter(
|
||||
pool_connections=10, pool_maxsize=10, max_retries=0,
|
||||
)
|
||||
_session.mount("http://", adapter)
|
||||
_session.mount("https://", adapter)
|
||||
return _session
|
||||
|
||||
|
||||
def _reset_session() -> None:
|
||||
"""关闭并重置模块级 Session。测试用。"""
|
||||
global _session
|
||||
if _session is not None:
|
||||
try:
|
||||
_session.close()
|
||||
except Exception:
|
||||
pass
|
||||
_session = None
|
||||
|
||||
|
||||
def _finalize_json_result(data):
|
||||
"""兜底填充 SearXNG JSON 响应的可选契约字段。
|
||||
|
||||
部分实例/版本的 JSON 输出缺失 ``number_of_results``（服务端
|
||||
``search.result_number()`` 的估算值，见 --dump-schema 描述）。缺失时
|
||||
用实际结果数近似填充，保证 JSON 路径与 HTML fallback 路径
|
||||
（parse_html_results 必然填充）的输出契约一致——AI Agent 依赖该字段
|
||||
判断结果规模与是否需要翻页。
|
||||
"""
|
||||
if isinstance(data, dict) and "number_of_results" not in data:
|
||||
data["number_of_results"] = len(data.get("results", []))
|
||||
return data
|
||||
|
||||
|
||||
def search_json(instance: str, params: dict, method: str = "GET",
|
||||
timeout: int = 15, auth_headers: dict = None) -> dict:
|
||||
"""Execute search via JSON API. Returns None if JSON unsupported."""
|
||||
"""Execute search via JSON API. Returns None if JSON unsupported.
|
||||
|
||||
v2.5.0: requests 可用时走模块级 Session（连接池复用）；否则 stdlib。
|
||||
两个后端发出完全相同的 URL（query_string 拼好后原样使用），解码逻辑
|
||||
一致（resp.content 手动 decode，不用 resp.text 的自动解码），保证
|
||||
行为可预测、跨后端可复现。
|
||||
"""
|
||||
query_string = urllib.parse.urlencode(params)
|
||||
headers = _merge_headers({"User-Agent": USER_AGENT}, auth_headers)
|
||||
|
||||
if _HAS_REQUESTS:
|
||||
session = _get_session()
|
||||
try:
|
||||
if method.upper() == "POST":
|
||||
resp = session.post(f"{instance}/search", data=query_string,
|
||||
headers=headers, timeout=timeout)
|
||||
else:
|
||||
resp = session.get(f"{instance}/search?{query_string}",
|
||||
headers=headers, timeout=timeout)
|
||||
if resp.status_code == 404:
|
||||
# 404 = JSON endpoint truly absent → fall back to HTML scraping
|
||||
return None
|
||||
if resp.status_code in (401, 403):
|
||||
# auth / IP issue → raise so it isn't silently masked by an
|
||||
# HTML fallback that would just 403 again.
|
||||
resp.raise_for_status()
|
||||
raw = resp.content.decode("utf-8")
|
||||
if raw.strip().startswith("{") or raw.strip().startswith("["):
|
||||
return _finalize_json_result(json.loads(raw))
|
||||
# Got HTML — JSON unsupported
|
||||
return None
|
||||
except _requests.exceptions.HTTPError as e:
|
||||
if getattr(e, "response", None) is not None and \
|
||||
e.response.status_code == 404:
|
||||
return None
|
||||
raise
|
||||
|
||||
# stdlib path（零依赖可用）
|
||||
if method.upper() == "POST":
|
||||
data = query_string.encode("utf-8")
|
||||
req = urllib.request.Request(f"{instance}/search", data=data, headers=headers, method="POST")
|
||||
req = urllib.request.Request(f"{instance}/search", data=data,
|
||||
headers=headers, method="POST")
|
||||
else:
|
||||
req = urllib.request.Request(f"{instance}/search?{query_string}", headers=headers)
|
||||
req = urllib.request.Request(f"{instance}/search?{query_string}",
|
||||
headers=headers)
|
||||
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
raw = resp.read().decode("utf-8")
|
||||
if raw.strip().startswith("{") or raw.strip().startswith("["):
|
||||
return json.loads(raw)
|
||||
return _finalize_json_result(json.loads(raw))
|
||||
# Got HTML — JSON unsupported
|
||||
return None
|
||||
except urllib.error.HTTPError as e:
|
||||
@@ -559,31 +658,42 @@ def search_html(instance: str, params: dict, timeout: int = 15,
|
||||
v2.2.2：解码改用 common.detect_charset（此前硬编码 utf-8，GBK/Shift-JIS
|
||||
等非 UTF-8 实例的页面会整体乱码）。``encoding`` 为显式覆盖（来自
|
||||
``--language`` 无关的 CLI ``--encoding``），优先级最高。
|
||||
v2.5.0：requests 可用时走模块级 Session（与 search_json 一致）。
|
||||
"""
|
||||
html_params = {k: v for k, v in params.items() if k != "format"}
|
||||
query_string = urllib.parse.urlencode(html_params)
|
||||
url = f"{instance}/search?{query_string}"
|
||||
|
||||
headers = _merge_headers({"User-Agent": USER_AGENT}, auth_headers)
|
||||
req = urllib.request.Request(url, headers=headers)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
raw = resp.read()
|
||||
|
||||
if _HAS_REQUESTS:
|
||||
try:
|
||||
resp = _get_session().get(url, headers=headers, timeout=timeout)
|
||||
raw = resp.content
|
||||
content_type = resp.headers.get("Content-Type", "")
|
||||
if encoding:
|
||||
try:
|
||||
html = raw.decode(encoding)
|
||||
except (UnicodeDecodeError, LookupError):
|
||||
html = raw.decode("utf-8", errors="replace")
|
||||
else:
|
||||
charset = detect_charset(raw, content_type)
|
||||
try:
|
||||
html = raw.decode(charset)
|
||||
except (UnicodeDecodeError, LookupError):
|
||||
html = raw.decode("utf-8", errors="replace")
|
||||
return parse_html_results(html, query=params.get("q", ""))
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"HTML search failed for {instance}: {e}")
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"HTML search failed for {instance}: {e}")
|
||||
else:
|
||||
req = urllib.request.Request(url, headers=headers)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
raw = resp.read()
|
||||
content_type = resp.headers.get("Content-Type", "")
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"HTML search failed for {instance}: {e}")
|
||||
|
||||
if encoding:
|
||||
try:
|
||||
html = raw.decode(encoding)
|
||||
except (UnicodeDecodeError, LookupError):
|
||||
html = raw.decode("utf-8", errors="replace")
|
||||
else:
|
||||
charset = detect_charset(raw, content_type)
|
||||
try:
|
||||
html = raw.decode(charset)
|
||||
except (UnicodeDecodeError, LookupError):
|
||||
html = raw.decode("utf-8", errors="replace")
|
||||
return parse_html_results(html, query=params.get("q", ""))
|
||||
|
||||
|
||||
def search_single(instance: str, params: dict, method: str = "GET",
|
||||
@@ -1272,8 +1382,13 @@ class AdaptiveThrottle:
|
||||
"rate limit" in error_msg.lower())
|
||||
if is_rate_limit:
|
||||
self._global_pause_until = time.monotonic() + self._pause_seconds
|
||||
# 连续失败达阈值 → 退避 + 降并发
|
||||
if self._consecutive_failures >= self._failure_threshold:
|
||||
# 连续失败达阈值 → 退避 + 降并发。
|
||||
# v2.5.0：threshold <= 0 表示"禁用自适应退避"——只统计失败次数
|
||||
# （供 stats()/报告展示），不再触发翻倍延迟与降并发。原实现
|
||||
# threshold=0 时 ``0 >= 0`` 恒真，首次失败即退避，与 CLI 帮助
|
||||
# 声称的 "0 disables throttling" 相反。
|
||||
if self._failure_threshold > 0 and \
|
||||
self._consecutive_failures >= self._failure_threshold:
|
||||
self._consecutive_failures = 0
|
||||
self._delay = min(self._delay * 2, self._max_delay)
|
||||
self._concurrency = max(1, self._concurrency // 2)
|
||||
@@ -1334,7 +1449,8 @@ def fetch_top_results(results: dict, count: int, timeout: int = 10,
|
||||
request_delay: float = 0.3,
|
||||
referer: str = None,
|
||||
fallback_enabled: bool = True,
|
||||
throttle: "AdaptiveThrottle" = None) -> list:
|
||||
throttle: "AdaptiveThrottle" = None,
|
||||
total_chars: int = 0) -> list:
|
||||
"""Fetch full text of top N result pages concurrently.
|
||||
|
||||
Features:
|
||||
@@ -1345,10 +1461,17 @@ def fetch_top_results(results: dict, count: int, timeout: int = 10,
|
||||
- Falls back to browser User-Agent if blocked (in fetch_url)
|
||||
- Small delay between requests to avoid rate limits
|
||||
|
||||
v2.5.0 全局字符预算（``total_chars``）：
|
||||
按原始结果顺序从顶部开始分配字符额度——前一页实际消费后剩余的预算
|
||||
留给下一页（顺序填满，保证最重要的结果拿到完整正文）。预算耗尽后
|
||||
剩余 URL 返回 ``status="skipped"``（不发请求），让 AI Agent 明确知道
|
||||
是预算跳过而非抓取失败。默认 0 = 不限制。
|
||||
|
||||
Args:
|
||||
referer: Referer URL（v2.0.0，通常设为 SearXNG 实例 URL）
|
||||
fallback_enabled: 是否启用 Wayback 兜底（默认 True）
|
||||
throttle: 外部传入的 AdaptiveThrottle 实例（可选；不传则内部创建）
|
||||
total_chars: 全局字符预算（v2.5.0；0 = 不限）
|
||||
"""
|
||||
urls = []
|
||||
seen = set()
|
||||
@@ -1370,16 +1493,57 @@ def fetch_top_results(results: dict, count: int, timeout: int = 10,
|
||||
logger.info(f"\nFetching {len(urls)} result pages "
|
||||
f"(timeout={timeout}s, retries={max_retries}, "
|
||||
f"delay={throttle.delay}s, concurrency={throttle.concurrency})...")
|
||||
if total_chars and total_chars > 0:
|
||||
logger.info(f"Char budget: {total_chars:,} total (--fetch-total-chars)")
|
||||
fetched = []
|
||||
ok_count = [0]
|
||||
err_count = [0]
|
||||
skipped_count = [0]
|
||||
anti_bot_count = [0]
|
||||
fallback_count = [0]
|
||||
|
||||
# v2.5.0 全局预算的线程安全计数器：_budget_remaining[0] 是剩余可用字符。
|
||||
# _budget_enabled[0] 区分"未启用预算"（total_chars<=0，无限）与"已耗尽"
|
||||
# （remaining=0）——前者放行所有 URL，后者跳过。_consume_budget 只在
|
||||
# 抓取成功后按实际 text_length 扣减，未用掉的额度自动留给下一页。
|
||||
_budget_enabled = [total_chars is not None and total_chars > 0]
|
||||
_budget_remaining = [total_chars if _budget_enabled[0] else 0]
|
||||
_budget_lock = threading.Lock()
|
||||
|
||||
def _take_budget() -> "tuple":
|
||||
"""分配本 URL 的字符上限。返回 (cap, allowed)。
|
||||
|
||||
cap 为 None 表示预算未启用（无限）。allowed=False 表示预算已耗尽，
|
||||
调用方应跳过本 URL（status="skipped"，不发请求）。
|
||||
"""
|
||||
if not _budget_enabled[0]:
|
||||
return None, True
|
||||
with _budget_lock:
|
||||
remaining = _budget_remaining[0]
|
||||
if remaining <= 0:
|
||||
return 0, False
|
||||
return remaining, True
|
||||
|
||||
def _consume_budget(used: int) -> None:
|
||||
"""抓取成功后按实际消费的字符数扣减预算。"""
|
||||
with _budget_lock:
|
||||
_budget_remaining[0] = max(0, _budget_remaining[0] - used)
|
||||
|
||||
def _fetch_one(u: str) -> dict:
|
||||
"""Fetch one URL with adaptive throttling to avoid rate limiting."""
|
||||
# 全局暂停检查（429 触发）
|
||||
throttle.wait_if_paused()
|
||||
# v2.5.0 预算检查放在取槽位之前：预算耗尽时连并发槽位都不占用
|
||||
cap, allowed = _take_budget()
|
||||
if not allowed:
|
||||
skipped_count[0] += 1
|
||||
logger.info(f" [BUDGET] char budget exhausted, skipping {u[:55]}")
|
||||
return {"url": u, "status": "skipped",
|
||||
"error": "char budget exhausted (--fetch-total-chars)",
|
||||
"text": "", "text_length": 0, "truncated": False,
|
||||
"anti_bot_detected": False, "waf_type": None,
|
||||
"fallback_used": None,
|
||||
"title": None, "latency": None}
|
||||
# v2.2.2：真实并发门控。退避降并发后，超出当前并发目标的新请求
|
||||
# 在这里被拒绝（不占用 fetch_page），实现持久降并发而非名义降并发。
|
||||
if not throttle.acquire_slot():
|
||||
@@ -1403,6 +1567,15 @@ def fetch_top_results(results: dict, count: int, timeout: int = 10,
|
||||
if result["status"] == "ok":
|
||||
ok_count[0] += 1
|
||||
throttle.report_success()
|
||||
# v2.5.0 字符预算：按分配到的上限截断正文（提取后语义截断），
|
||||
# 再按实际消费扣减预算。cap 为 None 表示未启用预算。
|
||||
if cap:
|
||||
text = result.get("text", "")
|
||||
if len(text) > cap:
|
||||
result["text"] = text[:cap]
|
||||
result["text_length"] = cap
|
||||
result["truncated"] = True
|
||||
_consume_budget(result["text_length"])
|
||||
trunc = ", TRUNCATED" if result.get("truncated") else ""
|
||||
ua_note = ""
|
||||
if result.get("user_agent_used") != USER_AGENT:
|
||||
@@ -1445,10 +1618,50 @@ def fetch_top_results(results: dict, count: int, timeout: int = 10,
|
||||
fetched.sort(key=lambda f: url_order.get(f["url"], 999))
|
||||
|
||||
logger.info(f"Fetched: {ok_count[0]} ok, {err_count[0]} errors"
|
||||
f" (anti-bot: {anti_bot_count[0]}, wayback: {fallback_count[0]})")
|
||||
f" (anti-bot: {anti_bot_count[0]}, wayback: {fallback_count[0]}, "
|
||||
f"skipped: {skipped_count[0]})")
|
||||
return fetched
|
||||
|
||||
|
||||
def deduplicate_fetched_content(fetched: list, threshold: float = 0.85) -> int:
|
||||
"""按正文相似度去重已抓取页面（v2.5.0 --dedup-fetched-content）。
|
||||
|
||||
镜像/转载站点的正文与原文高度相似但 URL 不同——URL 级去重
|
||||
（deduplicate_results）无法合并。这里对每个成功页面（status="ok"）
|
||||
的正文计算 SimHash 指纹，与已保留的页面两两比较，后出现的近似重复
|
||||
条目标记为 ``status="duplicate"``（text 清空，保留 url/title/error 供
|
||||
AI 查看原因），避免 AI 阅读同一内容的多个副本浪费 token。
|
||||
|
||||
只对 ok 条目参与去重；error/skipped/duplicate 条目原样保留。
|
||||
``fetched`` 保持原始结果顺序（首个出现的保留，通常 score 最高）。
|
||||
返回标记为 duplicate 的条目数。
|
||||
"""
|
||||
kept = []
|
||||
dup_count = 0
|
||||
for f in fetched:
|
||||
if f.get("status") != "ok":
|
||||
kept.append(f)
|
||||
continue
|
||||
text = f.get("text", "")
|
||||
is_dup = False
|
||||
for k in kept:
|
||||
if k.get("status") != "ok":
|
||||
continue
|
||||
if texts_are_similar(text, k.get("text", ""), threshold=threshold):
|
||||
is_dup = True
|
||||
break
|
||||
if is_dup:
|
||||
dup_count += 1
|
||||
f["status"] = "duplicate"
|
||||
f["text"] = ""
|
||||
f["text_length"] = 0
|
||||
f["error"] = f"duplicate content (similar to {k.get('url', '')})"
|
||||
kept.append(f)
|
||||
else:
|
||||
kept.append(f)
|
||||
return dup_count
|
||||
|
||||
|
||||
def _emit_fetch_report(fetched: list, throttle: "AdaptiveThrottle",
|
||||
fmt: str = "text") -> None:
|
||||
"""输出结构化抓取报告到 stderr。
|
||||
@@ -1491,10 +1704,14 @@ def _emit_fetch_report_text(fetched: list, throttle: "AdaptiveThrottle") -> None
|
||||
total = len(fetched)
|
||||
ok = sum(1 for f in fetched if f.get("status") == "ok")
|
||||
err = total - ok
|
||||
skipped = sum(1 for f in fetched if f.get("status") == "skipped")
|
||||
dup = sum(1 for f in fetched if f.get("status") == "duplicate")
|
||||
err = err - skipped - dup # skipped/duplicate 不是错误，单独统计
|
||||
anti_bot = sum(1 for f in fetched if f.get("anti_bot_detected"))
|
||||
wayback = sum(1 for f in fetched if f.get("fallback_used") == "wayback")
|
||||
lines.append("-" * len(header))
|
||||
lines.append(f"Total: {total} | OK: {ok} | Error: {err} | "
|
||||
f"Skipped: {skipped} | Duplicate: {dup} | "
|
||||
f"Anti-bot blocked: {anti_bot} | Wayback recovered: {wayback}")
|
||||
|
||||
# 自适应限流状态
|
||||
@@ -1507,7 +1724,8 @@ def _emit_fetch_report_text(fetched: list, throttle: "AdaptiveThrottle") -> None
|
||||
|
||||
# JSON 摘要（一行，便于 Agent 解析）
|
||||
summary = {
|
||||
"total": total, "ok": ok, "error": err,
|
||||
"total": total, "ok": ok, "error": err, "skipped": skipped,
|
||||
"duplicate": dup,
|
||||
"anti_bot_blocked": anti_bot, "wayback_recovered": wayback,
|
||||
"throttle": s,
|
||||
}
|
||||
@@ -1551,6 +1769,9 @@ def _emit_fetch_report_json(fetched: list, throttle: "AdaptiveThrottle") -> None
|
||||
total = len(fetched)
|
||||
ok = sum(1 for f in fetched if f.get("status") == "ok")
|
||||
err = total - ok
|
||||
skipped = sum(1 for f in fetched if f.get("status") == "skipped")
|
||||
dup = sum(1 for f in fetched if f.get("status") == "duplicate")
|
||||
err = err - skipped - dup # skipped/duplicate 不是错误，单独统计
|
||||
anti_bot = sum(1 for f in fetched if f.get("anti_bot_detected"))
|
||||
wayback = sum(1 for f in fetched if f.get("fallback_used") == "wayback")
|
||||
|
||||
@@ -1559,6 +1780,8 @@ def _emit_fetch_report_json(fetched: list, throttle: "AdaptiveThrottle") -> None
|
||||
"total": total,
|
||||
"ok": ok,
|
||||
"error": err,
|
||||
"skipped": skipped,
|
||||
"duplicate": dup,
|
||||
"anti_bot_blocked": anti_bot,
|
||||
"wayback_recovered": wayback,
|
||||
"throttle": throttle.stats(),
|
||||
@@ -1782,6 +2005,28 @@ def format_urls(results: dict) -> str:
|
||||
return "\n".join(r.get("url", "") for r in results.get("results", []) if r.get("url"))
|
||||
|
||||
|
||||
# ----- Media-category CSV columns (v2.5.0) -----
|
||||
# SearXNG 的 images 类别结果（template="images.html"）带 img_src/thumbnail_src/
|
||||
# resolution/source，videos 类别（template="videos.html"）带 iframe_src/
|
||||
# thumbnail_src。--format json 已透传这些字段（原始 dict 原样序列化），但
|
||||
# CSV 固定列会丢弃。以下辅助让 CSV 在结果含媒体字段时自动追加对应列，
|
||||
# 表格分析（图片缩略图链接、视频内嵌 URL）直接可用。
|
||||
_MEDIA_COLUMNS = ["img_src", "thumbnail_src", "resolution", "iframe_src", "source"]
|
||||
|
||||
|
||||
def _detect_media_columns(results_iter) -> list:
|
||||
"""返回结果集中实际存在的媒体列子集（保持 _MEDIA_COLUMNS 顺序）。
|
||||
|
||||
仅当至少一个结果包含非空值时才列出该列，避免 CSV 出现整列空白。
|
||||
"""
|
||||
present = set()
|
||||
for r in results_iter:
|
||||
for col in _MEDIA_COLUMNS:
|
||||
if r.get(col):
|
||||
present.add(col)
|
||||
return [c for c in _MEDIA_COLUMNS if c in present]
|
||||
|
||||
|
||||
def _format_results(results: dict, args) -> str:
|
||||
"""Format a results dict into the output string selected by args.format.
|
||||
|
||||
@@ -1796,9 +2041,11 @@ def _format_results(results: dict, args) -> str:
|
||||
if args.format == "csv":
|
||||
out = io.StringIO()
|
||||
writer = csv.writer(out, lineterminator="\n")
|
||||
rs = results.get("results", [])
|
||||
media_cols = _detect_media_columns(rs)
|
||||
writer.writerow(["title", "url", "engine", "score",
|
||||
"published_date", "content"])
|
||||
for r in results.get("results", []):
|
||||
"published_date", "content"] + media_cols)
|
||||
for r in rs:
|
||||
writer.writerow([
|
||||
r.get("title", ""),
|
||||
r.get("url", ""),
|
||||
@@ -1806,7 +2053,7 @@ def _format_results(results: dict, args) -> str:
|
||||
r.get("score", "") if r.get("score") is not None else "",
|
||||
r.get("published_date", ""),
|
||||
r.get("content", ""),
|
||||
])
|
||||
] + [r.get(c, "") for c in media_cols])
|
||||
return out.getvalue().rstrip()
|
||||
# brief (also the safe fallthrough)
|
||||
output = format_brief(results, snippet_len=args.snippet_len)
|
||||
@@ -1818,6 +2065,10 @@ def _format_results(results: dict, args) -> str:
|
||||
output += f"\n--- {f['url']} ---\n"
|
||||
if f["status"] == "ok":
|
||||
output += f"{f['text']}\n"
|
||||
elif f["status"] == "skipped":
|
||||
output += f"[SKIPPED: {f.get('error', 'char budget exhausted')}]\n"
|
||||
elif f["status"] == "duplicate":
|
||||
output += f"[DUPLICATE: {f.get('error', 'similar content')}]\n"
|
||||
else:
|
||||
output += f"[ERROR: {f.get('error', 'unknown')}]\n"
|
||||
return output
|
||||
@@ -2045,13 +2296,28 @@ def _run_single_query(query: str, args, instance_urls: list,
|
||||
referer=referer,
|
||||
fallback_enabled=not getattr(args, "no_fallback", False),
|
||||
throttle=fetch_throttle,
|
||||
total_chars=getattr(args, "fetch_total_chars", 0),
|
||||
)
|
||||
# Emit fetch_ok / fetch_fail events
|
||||
# v2.5.0: 正文相似度去重（--dedup-fetched-content），复用
|
||||
# --similarity-threshold 阈值。镜像/转载页标记为 status="duplicate"。
|
||||
if getattr(args, "dedup_fetched_content", False):
|
||||
dup_n = deduplicate_fetched_content(fetched,
|
||||
threshold=args.similarity_threshold)
|
||||
if dup_n:
|
||||
logger.info(f"Fetched-content dedup: {dup_n} duplicate page(s) "
|
||||
f"dropped (threshold={args.similarity_threshold})")
|
||||
# Emit fetch_ok / fetch_fail / fetch_duplicate / fetch_skip events
|
||||
for f in fetched:
|
||||
if f.get("status") == "ok":
|
||||
emit_progress("fetch_ok", url=f.get("url", ""),
|
||||
chars=f.get("text_length", 0),
|
||||
fallback=f.get("fallback_used"))
|
||||
elif f.get("status") == "duplicate":
|
||||
emit_progress("fetch_duplicate", url=f.get("url", ""),
|
||||
error=f.get("error", "duplicate content"))
|
||||
elif f.get("status") == "skipped":
|
||||
emit_progress("fetch_skip", url=f.get("url", ""),
|
||||
error=f.get("error", "budget exhausted"))
|
||||
else:
|
||||
emit_progress("fetch_fail", url=f.get("url", ""),
|
||||
error=f.get("error", "unknown"),
|
||||
@@ -2110,7 +2376,15 @@ def _get_output_schema():
|
||||
"url": {"type": "string"},
|
||||
"final_url": {"type": ["string", "null"],
|
||||
"description": "URL after redirects / Wayback."},
|
||||
"status": {"type": "string", "enum": ["ok", "error"]},
|
||||
"status": {"type": "string",
|
||||
"enum": ["ok", "error", "skipped", "duplicate"],
|
||||
"description": "'skipped' (v2.5.0) means the page was "
|
||||
"not fetched because the "
|
||||
"--fetch-total-chars budget was "
|
||||
"exhausted. 'duplicate' (v2.5.0) means "
|
||||
"the body was near-identical to an "
|
||||
"earlier page and was dropped by "
|
||||
"--dedup-fetched-content."},
|
||||
"content_type": {"type": ["string", "null"]},
|
||||
"text": {"type": "string", "description": "Extracted page text."},
|
||||
"text_length": {"type": "integer"},
|
||||
@@ -2371,6 +2645,7 @@ def _save_config(args, path: str) -> None:
|
||||
"fetch": "fetch",
|
||||
"fetch_timeout": "fetch_timeout",
|
||||
"fetch_retries": "fetch_retries",
|
||||
"fetch_total_chars": "fetch_total_chars",
|
||||
"throttle_failure_threshold": "throttle_failure_threshold",
|
||||
"throttle_pause_seconds": "throttle_pause_seconds",
|
||||
"throttle_max_delay": "throttle_max_delay",
|
||||
@@ -2421,6 +2696,13 @@ def _dry_run_preview(args, instance_urls: list, auth_headers: dict) -> None:
|
||||
elif args.queries_file:
|
||||
preview["action"] = "batch"
|
||||
preview["queries_file"] = args.queries_file
|
||||
# v2.5.0: dry-run 批量模式打印实际查询列表（读本地文件，不发 HTTP）。
|
||||
# 读取失败时降级为仅文件名，不阻塞预览。
|
||||
try:
|
||||
qs = _read_queries_file(args.queries_file)
|
||||
preview["queries"] = qs
|
||||
except RuntimeError as e:
|
||||
preview["queries_error"] = str(e)
|
||||
elif args.verify:
|
||||
preview["action"] = "verify"
|
||||
else:
|
||||
@@ -2539,19 +2821,27 @@ def _handle_research(args, instance_urls: list, auth_headers: dict,
|
||||
error_count = 0
|
||||
for i, (angle, q) in enumerate(research_queries, 1):
|
||||
logger.info(f"\n[{i}/{len(research_queries)}] [{angle}] {q}")
|
||||
# v2.5.0: 角度级进度事件——_run_single_query 内部的事件无法区分
|
||||
# "当前第几个角度"，加 angle_* 事件让 AI Agent 能跟踪多角度进度
|
||||
emit_progress("angle_start", angle=angle, query=q,
|
||||
index=i, total=len(research_queries))
|
||||
results, err, err_code = _run_single_query(q, args, instance_urls,
|
||||
auth_headers, ttl_seconds)
|
||||
if err:
|
||||
error_count += 1
|
||||
logger.error(f" [ERROR] {err}")
|
||||
emit_progress("angle_fail", angle=angle, query=q,
|
||||
error=err, error_code=err_code)
|
||||
entry = {"query": q, "angle": angle, "status": "error",
|
||||
"error": err}
|
||||
if err_code:
|
||||
entry["error_code"] = err_code
|
||||
batch.append(entry)
|
||||
else:
|
||||
if len(results.get("results", [])) > 0:
|
||||
angle_count = len(results.get("results", []))
|
||||
if angle_count > 0:
|
||||
any_with_results = True
|
||||
emit_progress("angle_ok", angle=angle, query=q, results=angle_count)
|
||||
batch.append({"query": q, "angle": angle, "status": "ok",
|
||||
"results": results})
|
||||
|
||||
@@ -2585,8 +2875,12 @@ def _handle_research(args, instance_urls: list, auth_headers: dict,
|
||||
elif args.format == "csv":
|
||||
out = io.StringIO()
|
||||
writer = csv.writer(out, lineterminator="\n")
|
||||
# v2.5.0: 媒体列检测——研究模式内任意角度含图片/视频字段则追加
|
||||
media_cols = _detect_media_columns(
|
||||
br["results"].get("results", [])
|
||||
for br in batch if "results" in br)
|
||||
writer.writerow(["angle", "query", "title", "url", "engine",
|
||||
"score", "published_date", "content"])
|
||||
"score", "published_date", "content"] + media_cols)
|
||||
for br in batch:
|
||||
q = br["query"]
|
||||
angle = br.get("angle", "")
|
||||
@@ -2600,7 +2894,7 @@ def _handle_research(args, instance_urls: list, auth_headers: dict,
|
||||
r.get("score", "") if r.get("score") is not None else "",
|
||||
r.get("published_date", ""),
|
||||
r.get("content", ""),
|
||||
])
|
||||
] + [r.get(c, "") for c in media_cols])
|
||||
else:
|
||||
writer.writerow([angle, q, "", "", "", "", "",
|
||||
f"[ERROR: {br['error']}]"])
|
||||
@@ -2772,8 +3066,12 @@ def _handle_batch(args, instance_urls: list, auth_headers: dict,
|
||||
elif args.format == "csv":
|
||||
out = io.StringIO()
|
||||
writer = csv.writer(out, lineterminator="\n")
|
||||
# v2.5.0: 媒体列检测——批量内任意结果含图片/视频字段则追加对应列
|
||||
media_cols = _detect_media_columns(
|
||||
br["results"].get("results", [])
|
||||
for br in batch if "results" in br)
|
||||
writer.writerow(["query", "title", "url", "engine", "score",
|
||||
"published_date", "content"])
|
||||
"published_date", "content"] + media_cols)
|
||||
for br in batch:
|
||||
q = br["query"]
|
||||
if "results" in br:
|
||||
@@ -2786,7 +3084,7 @@ def _handle_batch(args, instance_urls: list, auth_headers: dict,
|
||||
r.get("score", "") if r.get("score") is not None else "",
|
||||
r.get("published_date", ""),
|
||||
r.get("content", ""),
|
||||
])
|
||||
] + [r.get(c, "") for c in media_cols])
|
||||
else:
|
||||
writer.writerow([q, "", "", "", "", "",
|
||||
f"[ERROR: {br['error']}]"])
|
||||
@@ -2992,6 +3290,14 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
|
||||
metavar="FLOAT",
|
||||
help="相似度去重阈值（默认: 0.85）。越高越严格，1.0 要求标题几乎完全相同。"
|
||||
"仅当 --similarity-dedup 启用时生效。")
|
||||
parser.add_argument("--dedup-fetched-content", action="store_true",
|
||||
help="v2.5.0: deduplicate fetched page content by "
|
||||
"similarity (SimHash). Mirror/republished pages "
|
||||
"with near-identical bodies are marked "
|
||||
"status='duplicate' (text cleared) — keeps AI "
|
||||
"from reading the same content multiple times. "
|
||||
"Uses --similarity-threshold (default 0.85). "
|
||||
"Only applies with --fetch.")
|
||||
parser.add_argument("--format", "-f", choices=["json", "brief", "urls", "csv"],
|
||||
default=config.get("format", "json"),
|
||||
help="Output format (default: json). 'csv' exports "
|
||||
@@ -3004,6 +3310,16 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
|
||||
help="Timeout per page fetch in seconds (default: 10)")
|
||||
parser.add_argument("--fetch-retries", type=int, default=_cfg_int(config, "fetch_retries", 3),
|
||||
help="Max retries per page fetch (default: 3)")
|
||||
parser.add_argument("--fetch-total-chars", type=int,
|
||||
default=_cfg_int(config, "fetch_total_chars", 0),
|
||||
metavar="N",
|
||||
help="v2.5.0: global character budget for --fetch. "
|
||||
"Chars are allocated top-down in result order — "
|
||||
"unused budget rolls over to the next page. Once "
|
||||
"exhausted, remaining URLs are marked status="
|
||||
"'skipped' (no request sent). Lets token-limited "
|
||||
"agents cap total fetched content. 0 = unlimited "
|
||||
"(default).")
|
||||
parser.add_argument("--fetch-report", dest="fetch_report",
|
||||
nargs="?", const="text", default=False, metavar="FORMAT",
|
||||
help="When used with --fetch, emit a structured fetch report to stderr "
|
||||
|
||||
Reference in New Issue
Block a user