feat(v2.2.1): 修复 Brotli 乱码 + 缓存治理 + 多页聚合 + 研究模式增强
核心修复（v2.2.1）: - 修复 Brotli 乱码 bug: build_browser_headers 智能声明 Accept-Encoding, 仅在 brotli 可用时才声明 br; fetch.py 双路径 br 解压（requests + stdlib） 此前 Chrome/Edge UA 抓取 example.com 等返回 br 的站点输出乱码 v2.2.0 新功能: - main() 拆分为 _handle_verify/_handle_research/_handle_batch/_handle_single - --cache-max-size MB: 缓存大小上限 + LRU 淘汰（默认 100MB） - --pages N: 多页聚合 + 跨页去重 - --research 跨角度合并: 新增 merged_results 字段 - --stream / --progress: JSON Lines 流式输出 + request_id 贯穿 - --dry-run / --save-config / --log-format json - --similarity-dedup / --throttle-* 参数化 - 15-UA 池 + PDF/docx 解析 + error_code 字段 文档与测试: - SKILL.md: 版本号唯一（元数据），删除版本标记干扰 - README.md: 测试数量 539 -> 544 - 544 passed (新增 5 个 Content-Encoding 解压测试)
This commit is contained in:
+299
-47
@@ -18,9 +18,12 @@ as retryable and connection errors as transient, eliminating the previous
|
||||
inconsistency where ``search.py`` ignored 5xx.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import logging
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
import urllib.error
|
||||
|
||||
# Root logger for the searxng-cli package. All modules create child loggers
|
||||
@@ -28,8 +31,27 @@ import urllib.error
|
||||
# call controls them all.
|
||||
_LOG = logging.getLogger("searxng")
|
||||
|
||||
# Brotli 解压支持检测（v2.2.1）。
|
||||
# requests 库自动解压 gzip/deflate，但**不自动解压 Brotli**（除非安装
|
||||
# brotli/brotlicffi 包）。若在 Accept-Encoding 中声明 br 而系统未安装
|
||||
# 解压器，服务器返回的 br 压缩字节会被当作文本解码 → 全页乱码。
|
||||
# 此检测用于 build_browser_headers() 智能声明 Accept-Encoding，避免
|
||||
# 声明无法兑现的 br。
|
||||
try:
|
||||
import brotli as _brotli # type: ignore
|
||||
_HAS_BROTLI = True
|
||||
except ImportError:
|
||||
try:
|
||||
import brotlicffi as _brotli # type: ignore
|
||||
_HAS_BROTLI = True
|
||||
except ImportError:
|
||||
_brotli = None
|
||||
_HAS_BROTLI = False
|
||||
|
||||
def setup_logging(verbose: bool = False, quiet: bool = False) -> None:
|
||||
|
||||
def setup_logging(verbose: bool = False, quiet: bool = False,
|
||||
log_format: str = "text",
|
||||
request_id: str = None) -> None:
|
||||
"""Configure the ``searxng`` logger hierarchy.
|
||||
|
||||
* Default (no flags): ``INFO`` — progress messages, warnings, retry notices.
|
||||
@@ -39,6 +61,9 @@ def setup_logging(verbose: bool = False, quiet: bool = False) -> None:
|
||||
status codes, cache keys, and other diagnostic detail.
|
||||
* ``--quiet`` / ``-q``: ``WARNING`` — suppresses progress and retry noise;
|
||||
only warnings and errors reach stderr.
|
||||
* ``--log-format json`` (v2.2.0): 每行一个 JSON 对象，便于 AI Agent
|
||||
程序化解析。包含 ts/level/logger/msg/request_id 字段。
|
||||
* request_id (v2.2.0): 贯穿所有日志和进度事件的请求标识符。
|
||||
|
||||
All log output goes to stderr; stdout is reserved for data (JSON/CSV/etc.).
|
||||
"""
|
||||
@@ -50,15 +75,57 @@ def setup_logging(verbose: bool = False, quiet: bool = False) -> None:
|
||||
level = logging.INFO
|
||||
|
||||
_LOG.setLevel(level)
|
||||
# 存储 request_id 到 logger 全局，供 formatter 和进度事件使用
|
||||
_LOG._request_id = request_id
|
||||
|
||||
# Avoid duplicate handlers if setup_logging() is called twice (e.g. tests).
|
||||
if not _LOG.handlers:
|
||||
handler = logging.StreamHandler(sys.stderr)
|
||||
handler.setFormatter(logging.Formatter("%(message)s"))
|
||||
if log_format == "json":
|
||||
handler.setFormatter(_JsonFormatter())
|
||||
else:
|
||||
handler.setFormatter(logging.Formatter("%(message)s"))
|
||||
_LOG.addHandler(handler)
|
||||
else:
|
||||
# 已有 handler（如测试环境），更新 formatter
|
||||
for h in _LOG.handlers:
|
||||
if log_format == "json":
|
||||
h.setFormatter(_JsonFormatter())
|
||||
else:
|
||||
h.setFormatter(logging.Formatter("%(message)s"))
|
||||
# Don't let root logger add its own handler — we own the searxng namespace.
|
||||
_LOG.propagate = False
|
||||
|
||||
|
||||
class _JsonFormatter(logging.Formatter):
|
||||
"""JSON 结构化日志 formatter（v2.2.0）。
|
||||
|
||||
每行输出一个 JSON 对象：{"ts", "level", "logger", "msg", "request_id"}。
|
||||
让 AI Agent 可程序化解析日志（统计重试次数、识别慢实例等）。
|
||||
"""
|
||||
|
||||
def format(self, record):
|
||||
import json as _json
|
||||
entry = {
|
||||
"ts": _datetime_iso(record),
|
||||
"level": record.levelname,
|
||||
"logger": record.name,
|
||||
"msg": record.getMessage(),
|
||||
}
|
||||
rid = getattr(_LOG, "_request_id", None)
|
||||
if rid:
|
||||
entry["request_id"] = rid
|
||||
if record.exc_info and record.exc_info[1]:
|
||||
entry["exception"] = type(record.exc_info[1]).__name__
|
||||
return _json.dumps(entry, ensure_ascii=False)
|
||||
|
||||
|
||||
def _datetime_iso(record):
|
||||
"""格式化日志时间戳为 ISO 8601 字符串。"""
|
||||
import datetime as _dt
|
||||
return _dt.datetime.fromtimestamp(record.created).isoformat(timespec="milliseconds")
|
||||
|
||||
|
||||
def force_utf8_stdout() -> None:
|
||||
"""Force stdout/stderr to UTF-8 to prevent Windows GBK encoding crashes.
|
||||
|
||||
@@ -124,40 +191,58 @@ RETRYABLE_STATUS = frozenset({403, 429, 502, 503, 504})
|
||||
|
||||
# Browser-like UA strings for fallback when the searxng-cli UA is blocked.
|
||||
#
|
||||
# v2.0.0 扩充至 12 个：覆盖 Chrome/Edge/Firefox × Windows/macOS/Linux，
|
||||
# 每个都是较新版本（131/130/129），避免被识别为过时浏览器。
|
||||
# v2.2.0：UA 池迁移至 _config.py 的 UA_POOL（SSOT），此处通过导入引用。
|
||||
# 若 _config.py 不可导入（如 common.py 被单独分发），回退到下方内置副本
|
||||
# _FALLBACK_UAS_BUILTIN，保证模块始终可用。两份列表需保持同步，
|
||||
# _config.UA_POOL 为唯一权威来源。
|
||||
#
|
||||
# v2.0.0 起覆盖 Chrome/Edge/Firefox × Windows/macOS/Linux；v2.2.0 升级到
|
||||
# 2026 年版本（Chrome 138-140 / Edge 138 / Firefox 140 / Safari 18）。
|
||||
# 顺序固定以便 get_ua_for_domain() 的 hash 选择可复现。
|
||||
FALLBACK_UAS = [
|
||||
# Chrome 131 — Windows / macOS / Linux
|
||||
_FALLBACK_UAS_BUILTIN = [
|
||||
# Chrome 140 — Windows / macOS / Linux
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||||
"(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||||
"(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||||
# Edge 131 — Windows / macOS
|
||||
"(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
|
||||
# Chrome 139 — Windows / macOS / Linux
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
|
||||
"(KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
|
||||
# Firefox 133 — Windows / macOS / Linux
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) "
|
||||
"Gecko/20100101 Firefox/133.0",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:133.0) "
|
||||
"Gecko/20100101 Firefox/133.0",
|
||||
"Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0",
|
||||
# Chrome 130 — Windows / macOS (上一个版本，应对 131 被针对性识别)
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
|
||||
# Chrome 129 — Windows / Linux
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
|
||||
"(KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36",
|
||||
"(KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36",
|
||||
# Chrome 138 — Windows / macOS / Linux
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
|
||||
# Edge 138 — Windows / macOS
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36 Edg/138.0.0.0",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36 Edg/138.0.0.0",
|
||||
# Firefox 140 — Windows / macOS / Linux
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:140.0) "
|
||||
"Gecko/20100101 Firefox/140.0",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:140.0) "
|
||||
"Gecko/20100101 Firefox/140.0",
|
||||
"Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0",
|
||||
# Safari 18 — macOS（WebKit 指纹，应对 Chromium 针对性拦截）
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 "
|
||||
"(KHTML, like Gecko) Version/18.0 Safari/605.1.15",
|
||||
]
|
||||
|
||||
# 从 _config.py 导入权威 UA_POOL；导入失败时回退到内置副本，保证向后兼容。
|
||||
try:
|
||||
from _config import UA_POOL as FALLBACK_UAS
|
||||
except ImportError:
|
||||
FALLBACK_UAS = _FALLBACK_UAS_BUILTIN
|
||||
|
||||
|
||||
def _ua_index_for_domain(domain: str, pool_size: int) -> int:
|
||||
"""为域名确定性选择 UA 池索引。
|
||||
@@ -176,6 +261,13 @@ def _ua_index_for_domain(domain: str, pool_size: int) -> int:
|
||||
# 被反爬识别。跨进程通过 SHA-256 哈希复现，见 _ua_index_for_domain。
|
||||
_domain_ua_cache: dict = {}
|
||||
|
||||
# 保护 _domain_ua_cache 的"检查-设置"原子性锁。
|
||||
# v2.2.0：并发场景下（如 search_multi 并行请求多实例），多个线程可能同时
|
||||
# 检查 domain not in cache 并同时写入，虽不致命但会浪费计算且可能写入不同
|
||||
# UA（因 hash 本应稳定，但极端时序下逻辑可读性问题）。用锁串行化 dict 读写。
|
||||
# 注意：锁内只做 dict 读写，绝不放网络/重计算，避免阻塞其他线程。
|
||||
_domain_ua_lock = threading.Lock()
|
||||
|
||||
|
||||
def get_ua_for_domain(url: str, user_agent: str = None) -> str:
|
||||
"""返回适合某域名的 User-Agent。
|
||||
@@ -188,6 +280,10 @@ def get_ua_for_domain(url: str, user_agent: str = None) -> str:
|
||||
设计理由：真实浏览器访问同一站点时 UA 永远不变。爬虫如果每次请求
|
||||
换一个 UA，反而会被反爬系统标记为可疑。确定性轮换保证同一域名
|
||||
稳定使用同一 UA，不同域名分散到不同 UA 上降低集体封禁风险。
|
||||
|
||||
线程安全：缓存的"检查-设置"用 ``_domain_ua_lock`` 保护。本函数无网络
|
||||
调用，但遵循"锁内只做 dict 读写"原则——hash 计算放在锁外，写入时做
|
||||
双检查（其他线程可能在此期间已写入），兼顾正确性与并发吞吐。
|
||||
"""
|
||||
if user_agent:
|
||||
return user_agent
|
||||
@@ -200,18 +296,27 @@ def get_ua_for_domain(url: str, user_agent: str = None) -> str:
|
||||
except Exception:
|
||||
return FALLBACK_UAS[0]
|
||||
|
||||
if domain in _domain_ua_cache:
|
||||
return _domain_ua_cache[domain]
|
||||
# 快速路径：锁内检查缓存命中
|
||||
with _domain_ua_lock:
|
||||
if domain in _domain_ua_cache:
|
||||
return _domain_ua_cache[domain]
|
||||
|
||||
# 缓存未命中：在锁外计算 UA（SHA-256 hash，无副作用，不阻塞其他线程）
|
||||
idx = _ua_index_for_domain(domain, len(FALLBACK_UAS))
|
||||
ua = FALLBACK_UAS[idx]
|
||||
_domain_ua_cache[domain] = ua
|
||||
|
||||
# 加锁写入；双检查避免覆盖其他线程并发写入的值
|
||||
with _domain_ua_lock:
|
||||
if domain in _domain_ua_cache:
|
||||
return _domain_ua_cache[domain]
|
||||
_domain_ua_cache[domain] = ua
|
||||
return ua
|
||||
|
||||
|
||||
def reset_domain_ua_cache() -> None:
|
||||
"""清空 per-domain UA 缓存。测试用。"""
|
||||
_domain_ua_cache.clear()
|
||||
with _domain_ua_lock:
|
||||
_domain_ua_cache.clear()
|
||||
|
||||
|
||||
def build_browser_headers(user_agent: str, referer: str = None,
|
||||
@@ -239,11 +344,22 @@ def build_browser_headers(user_agent: str, referer: str = None,
|
||||
else:
|
||||
accept = "application/json, text/plain, */*;q=0.8"
|
||||
|
||||
# v2.2.1 智能声明 Accept-Encoding：仅当本机安装了 brotli 解压器时
|
||||
# 才声明 br。否则服务器返回 br 压缩字节而 requests 无法解压 → 乱码。
|
||||
# Firefox UA 路径保持只声明 gzip/deflate（与真 Firefox 行为一致，
|
||||
# Firefox 虽支持 br 但为减少指纹差异在此工具中不声明）。
|
||||
if is_firefox:
|
||||
accept_encoding = "gzip, deflate"
|
||||
elif _HAS_BROTLI:
|
||||
accept_encoding = "gzip, deflate, br"
|
||||
else:
|
||||
accept_encoding = "gzip, deflate"
|
||||
|
||||
headers = {
|
||||
"User-Agent": user_agent,
|
||||
"Accept": accept,
|
||||
"Accept-Language": "en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7",
|
||||
"Accept-Encoding": "gzip, deflate" if is_firefox else "gzip, deflate, br",
|
||||
"Accept-Encoding": accept_encoding,
|
||||
"Connection": "keep-alive",
|
||||
"Upgrade-Insecure-Requests": "1" if accept_html else "0",
|
||||
}
|
||||
@@ -588,22 +704,17 @@ RECOVERY_HINTS = {
|
||||
}
|
||||
|
||||
|
||||
def classify_error(exc: BaseException) -> str:
|
||||
"""将异常分类为结构化错误码，供 AI Agent 程序化处理。
|
||||
def _classify_by_type_and_status(exc, _json):
|
||||
"""根据异常类型和 HTTP 状态码分类，返回错误码或 None。
|
||||
|
||||
分类逻辑（按优先级）：
|
||||
1. 429 → E_RATE_LIMIT
|
||||
2. 401/403 → E_AUTH
|
||||
3. 4xx（非上述）→ E_INPUT（请求参数问题）
|
||||
4. 5xx / URLError / OSError / TimeoutError → E_NETWORK
|
||||
5. json.JSONDecodeError / ValueError → E_PARSE
|
||||
6. FileNotFoundError → E_INPUT
|
||||
7. RuntimeError → 尝试从消息中提取线索，否则 E_INTERNAL
|
||||
8. 其他 → E_INTERNAL
|
||||
检查顺序：HTTP 状态码 → 文件错误 → 连接错误 → 解析错误。
|
||||
返回 None 表示该异常无法靠类型/状态码判定，需走字符串 fallback。
|
||||
不处理 RuntimeError 消息推断（由 :func:`classify_error` 调用方做 fallback）。
|
||||
|
||||
抽取为独立函数，便于 :func:`classify_error` 对 ``exc`` 本身和其
|
||||
``__cause__`` 复用同一套基于真实类型的判定逻辑。
|
||||
"""
|
||||
import json as _json
|
||||
|
||||
# HTTP 错误（urllib 和 requests 都有 .code 或 .status_code）
|
||||
# HTTP 错误（urllib HTTPError 用 .code，requests HTTPError 用 .response.status_code）
|
||||
status = None
|
||||
if isinstance(exc, urllib.error.HTTPError):
|
||||
status = exc.code
|
||||
@@ -635,11 +746,48 @@ def classify_error(exc: BaseException) -> str:
|
||||
if isinstance(exc, (ValueError, _json.JSONDecodeError)):
|
||||
return E_PARSE
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def classify_error(exc: BaseException) -> str:
|
||||
"""将异常分类为结构化错误码，供 AI Agent 程序化处理。
|
||||
|
||||
分类逻辑（按优先级）：
|
||||
1. 优先检查异常链 ``__cause__``：``raise X from Y`` 场景下 Y 才是真实
|
||||
错误源（如 ``raise RuntimeError(...) from HTTPError(403)``），
|
||||
用 Y 的类型/状态码分类比从 X 的消息字符串推断更可靠、不再脆弱。
|
||||
2. HTTP 状态码：429→E_RATE_LIMIT, 401/403→E_AUTH, 4xx→E_INPUT, 5xx→E_NETWORK
|
||||
3. 连接类异常（ConnectionError/TimeoutError/URLError/OSError）→ E_NETWORK
|
||||
4. 解析错误（ValueError/JSONDecodeError）→ E_PARSE
|
||||
5. FileNotFoundError → E_INPUT
|
||||
6. RuntimeError：从消息中推断（仅当 ``__cause__`` 缺失时的最后 fallback，
|
||||
兼容旧路径——search_multi 把 last_error 拼进消息）
|
||||
7. 其他 → E_INTERNAL
|
||||
"""
|
||||
import json as _json
|
||||
|
||||
# 优先检查异常链 __cause__：raise X from Y 时，Y 指向真实底层异常。
|
||||
# 旧逻辑只能从外层 RuntimeError 的消息字符串推断（脆弱，依赖 "403"/"auth"
|
||||
# 等关键字匹配），新逻辑直接从 __cause__ 的 .code/.status_code 或异常
|
||||
# 类型判定。仅检查一层 __cause__，不递归——单层已覆盖 search_multi 的
|
||||
# raise-from 模式，深层链罕见且递归有循环风险。
|
||||
cause = getattr(exc, "__cause__", None)
|
||||
if cause is not None and cause is not exc:
|
||||
code = _classify_by_type_and_status(cause, _json)
|
||||
if code is not None:
|
||||
return code
|
||||
|
||||
# 检查 exc 本身的类型和状态码
|
||||
code = _classify_by_type_and_status(exc, _json)
|
||||
if code is not None:
|
||||
return code
|
||||
|
||||
# RuntimeError：从消息中推断（search_multi 的 "All N instances failed.
|
||||
# Last error: HTTP Error 403: Forbidden" 等）。并行路径现在会把 last_error
|
||||
# 拼进消息，让这里能提取真实错误类型，而不是一律误判 E_NETWORK。
|
||||
msg = str(exc).lower()
|
||||
# 字符串匹配仅作为 __cause__ 缺失时的最后 fallback。
|
||||
if isinstance(exc, RuntimeError):
|
||||
msg = str(exc).lower()
|
||||
# 先检查 auth/rate-limit 关键字（最常见，来自 last_error 详情）
|
||||
if "auth" in msg or "403" in msg or "401" in msg:
|
||||
return E_AUTH
|
||||
@@ -703,11 +851,15 @@ def emit_progress(event: str, **kwargs) -> None:
|
||||
|
||||
仅当 --progress 启用时才输出。``event`` 是事件类型字符串，
|
||||
``kwargs`` 是事件的附加字段。输出格式为 JSON Lines（每行一个 JSON 对象）。
|
||||
v2.2.0：自动注入 request_id（如果已设置）。
|
||||
"""
|
||||
if not _progress_enabled:
|
||||
return
|
||||
import json as _json
|
||||
payload = {"event": event}
|
||||
rid = getattr(_LOG, "_request_id", None)
|
||||
if rid:
|
||||
payload["request_id"] = rid
|
||||
payload.update(kwargs)
|
||||
print(_json.dumps(payload, ensure_ascii=False), file=sys.stderr, flush=True)
|
||||
|
||||
@@ -813,3 +965,103 @@ def is_hard_blocked_domain(url: str) -> bool:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
# ----- 相似度去重（v2.x：让 AI Agent 在研究模式下获得更干净的 merged_results）-----
|
||||
# 同一内容在不同 URL/引擎下常重复出现，仅靠 URL 去重无法合并。
|
||||
# SimHash 对标题做局部敏感哈希，汉明距离小的视为近似重复。
|
||||
|
||||
def _normalize_title(title: str) -> str:
|
||||
"""归一化标题：小写、去标点、去多余空格，用于相似度比较。"""
|
||||
if not title:
|
||||
return ""
|
||||
s = title.lower()
|
||||
# 保留字母、数字、CJK 和空格，其余替换为空格
|
||||
s = re.sub(r'[^\w\s]', ' ', s)
|
||||
# \w 包含下划线，单独去掉
|
||||
s = s.replace('_', ' ')
|
||||
s = re.sub(r'\s+', ' ', s).strip()
|
||||
return s
|
||||
|
||||
|
||||
def _simhash(text: str, hash_bits: int = 64) -> int:
|
||||
"""计算文本的 SimHash 指纹。
|
||||
- 分词（按空格 + CJK 单字符）
|
||||
- 每个 token 算普通 hash，按 bit 投票
|
||||
- 返回 hash_bits 位的指纹
|
||||
"""
|
||||
if not text:
|
||||
return 0
|
||||
# 分词：按空格切分，CJK 字符再逐个拆成单字 token
|
||||
tokens = []
|
||||
for word in text.split():
|
||||
buf = []
|
||||
for ch in word:
|
||||
if '\u4e00' <= ch <= '\u9fff':
|
||||
# 遇到 CJK：先冲出缓冲区里的非 CJK 片段，再加入单字
|
||||
if buf:
|
||||
tokens.append(''.join(buf))
|
||||
buf = []
|
||||
tokens.append(ch)
|
||||
else:
|
||||
buf.append(ch)
|
||||
if buf:
|
||||
tokens.append(''.join(buf))
|
||||
if not tokens:
|
||||
return 0
|
||||
# 每个 token 算 SHA-256（跨进程可复现，避免 hash() 随机化），按 bit 投票
|
||||
v = [0] * hash_bits
|
||||
for token in tokens:
|
||||
h = hashlib.sha256(token.encode('utf-8')).digest()
|
||||
token_hash = int.from_bytes(h[:8], 'big')
|
||||
for i in range(hash_bits):
|
||||
if (token_hash >> i) & 1:
|
||||
v[i] += 1
|
||||
else:
|
||||
v[i] -= 1
|
||||
# 投票为正的位置 1
|
||||
fingerprint = 0
|
||||
for i in range(hash_bits):
|
||||
if v[i] > 0:
|
||||
fingerprint |= (1 << i)
|
||||
return fingerprint
|
||||
|
||||
|
||||
def _hamming_distance(a: int, b: int) -> int:
|
||||
"""两个整数的汉明距离。"""
|
||||
return bin(a ^ b).count('1')
|
||||
|
||||
|
||||
def _jaccard_similarity(set_a: set, set_b: set) -> float:
|
||||
"""Jaccard 相似度。"""
|
||||
if not set_a and not set_b:
|
||||
return 0.0
|
||||
union = set_a | set_b
|
||||
if not union:
|
||||
return 0.0
|
||||
return len(set_a & set_b) / len(union)
|
||||
|
||||
|
||||
def is_similar(result_a: dict, result_b: dict, threshold: float = 0.85) -> bool:
|
||||
"""判断两个搜索结果是否相似。
|
||||
- 优先用标题 SimHash（汉明距离 <= 3 视为相似，对应 64 位中约 95% 相似）
|
||||
- 标题太短（< 5 字符）时用 URL 域名 + 标题 Jaccard
|
||||
- threshold 参数控制严格程度
|
||||
"""
|
||||
title_a = _normalize_title(result_a.get("title", ""))
|
||||
title_b = _normalize_title(result_b.get("title", ""))
|
||||
# 标题太短时 SimHash 不稳定，改用 Jaccard
|
||||
if len(title_a) < 5 or len(title_b) < 5:
|
||||
import urllib.parse as _up
|
||||
domain_a = _up.urlparse(result_a.get("url", "")).netloc.lower()
|
||||
domain_b = _up.urlparse(result_b.get("url", "")).netloc.lower()
|
||||
set_a = set(title_a.split()) | {domain_a}
|
||||
set_b = set(title_b.split()) | {domain_b}
|
||||
return _jaccard_similarity(set_a, set_b) >= threshold
|
||||
# 标题足够长：用 SimHash 汉明距离
|
||||
hash_a = _simhash(title_a)
|
||||
hash_b = _simhash(title_b)
|
||||
# threshold → 汉明距离阈值映射：
|
||||
# 0.85 → 3（默认，宽松），0.90 → 2，0.95 → 1，1.0 → 0（几乎完全相同）
|
||||
max_distance = max(0, min(3, int(round((1.0 - threshold) / 0.05))))
|
||||
return _hamming_distance(hash_a, hash_b) <= max_distance
|
||||
|
||||
|
||||
Reference in New Issue
Block a user