feat(v2.5.0): Sec-Ch-Ua 头修复 + 结构化提取 + token 预算 + 正文去重 + 连接复用

正确性修复: - 修复 Sec-Ch-Ua 构造 bug: 原实现产出 ""Not_A Brand";v="99"" 双重引号畸形头(Chrome/Edge 两路径), 严格校验的 WAF 会忽略; 改为品牌数组拼接 - search 403 快速失败: 实例级 403 不再退避重试(~10.5s 空等), 立即 failover - AdaptiveThrottle: --throttle-failure-threshold 0 现为真正禁用语义 - number_of_results 缺失时用 len(results) 兜底(JSON/HTML 路径契约对齐) - 版本对齐: pyproject.toml 与 _config.py 同步 2.5.0 AI 代理体验: - fetch.py --extract json: 结构化骨架(title/meta/headings/links/images) - fetch.py --max-chars N: 提取后语义级截断(区别于 --max-size 字节截断) - search --fetch-total-chars N: --fetch 全局字符预算, 耗尽后 status=skipped - --dedup-fetched-content: 抓取正文 SimHash 去重, status=duplicate - --progress 新增 angle_start/ok/fail + fetch_skip/fetch_duplicate 事件 - fetch.py 补齐 --log-format json + --dump-schema - CSV 媒体列自适应(images/videos 类别自动追加媒体字段列) - --dry-run 批量模式打印实际查询列表 - search 连接复用: requests 可用时走模块级 Session(连接池) 工程治理: - 新增 scripts/release_check.py 发布一致性检查(版本/错误码表漂移) - 新增 tests/test_v250_features.py 46+4 个回归测试(全量 622 通过) - tests/conftest.py: autouse fixture 强制 stdlib 路径(本机有 requests 时既有 urllib mock 测试不失效)
2026-08-07 15:19:17 +08:00
parent 471818074d
commit 10c01a3abd
10 changed files with 1746 additions and 54 deletions
@@ -45,6 +45,7 @@ from common import (
    force_utf8_stdout,
    is_hard_blocked_domain,
    is_similar,
+    texts_are_similar,
    parse_retry_after,
    resolve_auth_basic,
    resolve_auth_bearer,
@@ -488,13 +489,19 @@ def _retry_with_backoff(fn, max_retries: int = MAX_RETRIES, base_delay: float =
    v2.1.0 修复：复用 common.compute_backoff_delay（带 60s 封顶），
    避免高重试次数（如 --retry 10）时 1.5*2^10=1536s 卡死进程。
    同时遵守 Retry-After header（429/503），与 fetch.py 保持一致。
+
+    v2.5.0：403 不再重试。RETRYABLE_STATUS 含 403 是为 fetch.py 的 UA
+    轮换设计的（每次重试换新 UA），但本函数仅供 search 使用——search 不
+    换 UA，实例级 403 是认证/封禁问题，重试只会空等 ~10.5s 后才 failover。
+    现在 403 立即 raise，由 search_multi 直接切换下一实例，classify_error
+    仍正确归为 E_AUTH（实例认证失败语义不变）。
    """
    last_error = None
    for attempt in range(max_retries + 1):
        try:
            return fn()
        except urllib.error.HTTPError as e:
-            if e.code in RETRYABLE_STATUS:  # 403 (UA block) + 429 + 5xx
+            if e.code in RETRYABLE_STATUS and e.code != 403:  # 429 + 5xx
                last_error = e
                if attempt < max_retries:
                    # 429/503：遵守 Retry-After header，避免触发更严厉限流
@@ -523,23 +530,115 @@ def _retry_with_backoff(fn, max_retries: int = MAX_RETRIES, base_delay: float =

 # ----- Search execution -----

+# v2.5.0 连接复用：requests 可用时复用模块级 Session（连接池 + TLS 会话
+# 恢复），--pages/批量/研究模式的多请求场景显著减少 TLS 握手开销。
+# stdlib 路径保持零依赖可用（每次新建连接，功能等价）——与 fetch.py 的
+# "requests 路径受益，stdlib 路径功能完整" 设计一致。
+_HAS_REQUESTS = False
+try:
+    import requests as _requests  # type: ignore
+    _HAS_REQUESTS = True
+except ImportError:
+    pass
+
+_session = None
+
+
+def _get_session():
+    """获取（惰性创建）模块级 requests.Session。
+
+    连接池：每主机最多 10 连接，总最多 20 连接。max_retries=0 让 requests
+    不做自己的重试——重试统一由 _retry_with_backoff 控制，避免双重退避。
+    """
+    global _session
+    if _session is None and _HAS_REQUESTS:
+        _session = _requests.Session()
+        adapter = _requests.adapters.HTTPAdapter(
+            pool_connections=10, pool_maxsize=10, max_retries=0,
+        )
+        _session.mount("http://", adapter)
+        _session.mount("https://", adapter)
+    return _session
+
+
+def _reset_session() -> None:
+    """关闭并重置模块级 Session。测试用。"""
+    global _session
+    if _session is not None:
+        try:
+            _session.close()
+        except Exception:
+            pass
+        _session = None
+
+
+def _finalize_json_result(data):
+    """兜底填充 SearXNG JSON 响应的可选契约字段。
+
+    部分实例/版本的 JSON 输出缺失 ``number_of_results``（服务端
+    ``search.result_number()`` 的估算值，见 --dump-schema 描述）。缺失时
+    用实际结果数近似填充，保证 JSON 路径与 HTML fallback 路径
+    （parse_html_results 必然填充）的输出契约一致——AI Agent 依赖该字段
+    判断结果规模与是否需要翻页。
+    """
+    if isinstance(data, dict) and "number_of_results" not in data:
+        data["number_of_results"] = len(data.get("results", []))
+    return data
+
+
 def search_json(instance: str, params: dict, method: str = "GET",
                timeout: int = 15, auth_headers: dict = None) -> dict:
-    """Execute search via JSON API. Returns None if JSON unsupported."""
+    """Execute search via JSON API. Returns None if JSON unsupported.
+
+    v2.5.0: requests 可用时走模块级 Session（连接池复用）；否则 stdlib。
+    两个后端发出完全相同的 URL（query_string 拼好后原样使用），解码逻辑
+    一致（resp.content 手动 decode，不用 resp.text 的自动解码），保证
+    行为可预测、跨后端可复现。
+    """
    query_string = urllib.parse.urlencode(params)
    headers = _merge_headers({"User-Agent": USER_AGENT}, auth_headers)

+    if _HAS_REQUESTS:
+        session = _get_session()
+        try:
+            if method.upper() == "POST":
+                resp = session.post(f"{instance}/search", data=query_string,
+                                    headers=headers, timeout=timeout)
+            else:
+                resp = session.get(f"{instance}/search?{query_string}",
+                                   headers=headers, timeout=timeout)
+            if resp.status_code == 404:
+                # 404 = JSON endpoint truly absent → fall back to HTML scraping
+                return None
+            if resp.status_code in (401, 403):
+                # auth / IP issue → raise so it isn't silently masked by an
+                # HTML fallback that would just 403 again.
+                resp.raise_for_status()
+            raw = resp.content.decode("utf-8")
+            if raw.strip().startswith("{") or raw.strip().startswith("["):
+                return _finalize_json_result(json.loads(raw))
+            # Got HTML — JSON unsupported
+            return None
+        except _requests.exceptions.HTTPError as e:
+            if getattr(e, "response", None) is not None and \
+                    e.response.status_code == 404:
+                return None
+            raise
+
+    # stdlib path（零依赖可用）
    if method.upper() == "POST":
        data = query_string.encode("utf-8")
-        req = urllib.request.Request(f"{instance}/search", data=data, headers=headers, method="POST")
+        req = urllib.request.Request(f"{instance}/search", data=data,
+                                     headers=headers, method="POST")
    else:
-        req = urllib.request.Request(f"{instance}/search?{query_string}", headers=headers)
+        req = urllib.request.Request(f"{instance}/search?{query_string}",
+                                     headers=headers)

    try:
        with urllib.request.urlopen(req, timeout=timeout) as resp:
            raw = resp.read().decode("utf-8")
            if raw.strip().startswith("{") or raw.strip().startswith("["):
-                return json.loads(raw)
+                return _finalize_json_result(json.loads(raw))
            # Got HTML — JSON unsupported
            return None
    except urllib.error.HTTPError as e:
@@ -559,31 +658,42 @@ def search_html(instance: str, params: dict, timeout: int = 15,
    v2.2.2：解码改用 common.detect_charset（此前硬编码 utf-8，GBK/Shift-JIS
    等非 UTF-8 实例的页面会整体乱码）。``encoding`` 为显式覆盖（来自
    ``--language`` 无关的 CLI ``--encoding``），优先级最高。
+    v2.5.0：requests 可用时走模块级 Session（与 search_json 一致）。
    """
    html_params = {k: v for k, v in params.items() if k != "format"}
    query_string = urllib.parse.urlencode(html_params)
    url = f"{instance}/search?{query_string}"

    headers = _merge_headers({"User-Agent": USER_AGENT}, auth_headers)
-    req = urllib.request.Request(url, headers=headers)
-    try:
-        with urllib.request.urlopen(req, timeout=timeout) as resp:
-            raw = resp.read()
+
+    if _HAS_REQUESTS:
+        try:
+            resp = _get_session().get(url, headers=headers, timeout=timeout)
+            raw = resp.content
            content_type = resp.headers.get("Content-Type", "")
-            if encoding:
-                try:
-                    html = raw.decode(encoding)
-                except (UnicodeDecodeError, LookupError):
-                    html = raw.decode("utf-8", errors="replace")
-            else:
-                charset = detect_charset(raw, content_type)
-                try:
-                    html = raw.decode(charset)
-                except (UnicodeDecodeError, LookupError):
-                    html = raw.decode("utf-8", errors="replace")
-            return parse_html_results(html, query=params.get("q", ""))
-    except Exception as e:
-        raise RuntimeError(f"HTML search failed for {instance}: {e}")
+        except Exception as e:
+            raise RuntimeError(f"HTML search failed for {instance}: {e}")
+    else:
+        req = urllib.request.Request(url, headers=headers)
+        try:
+            with urllib.request.urlopen(req, timeout=timeout) as resp:
+                raw = resp.read()
+                content_type = resp.headers.get("Content-Type", "")
+        except Exception as e:
+            raise RuntimeError(f"HTML search failed for {instance}: {e}")
+
+    if encoding:
+        try:
+            html = raw.decode(encoding)
+        except (UnicodeDecodeError, LookupError):
+            html = raw.decode("utf-8", errors="replace")
+    else:
+        charset = detect_charset(raw, content_type)
+        try:
+            html = raw.decode(charset)
+        except (UnicodeDecodeError, LookupError):
+            html = raw.decode("utf-8", errors="replace")
+    return parse_html_results(html, query=params.get("q", ""))


 def search_single(instance: str, params: dict, method: str = "GET",
@@ -1272,8 +1382,13 @@ class AdaptiveThrottle:
                             "rate limit" in error_msg.lower())
            if is_rate_limit:
                self._global_pause_until = time.monotonic() + self._pause_seconds
-            # 连续失败达阈值 → 退避 + 降并发
-            if self._consecutive_failures >= self._failure_threshold:
+            # 连续失败达阈值 → 退避 + 降并发。
+            # v2.5.0：threshold <= 0 表示"禁用自适应退避"——只统计失败次数
+            # （供 stats()/报告展示），不再触发翻倍延迟与降并发。原实现
+            # threshold=0 时 ``0 >= 0`` 恒真，首次失败即退避，与 CLI 帮助
+            # 声称的 "0 disables throttling" 相反。
+            if self._failure_threshold > 0 and \
+                    self._consecutive_failures >= self._failure_threshold:
                self._consecutive_failures = 0
                self._delay = min(self._delay * 2, self._max_delay)
                self._concurrency = max(1, self._concurrency // 2)
@@ -1334,7 +1449,8 @@ def fetch_top_results(results: dict, count: int, timeout: int = 10,
                       request_delay: float = 0.3,
                       referer: str = None,
                       fallback_enabled: bool = True,
-                       throttle: "AdaptiveThrottle" = None) -> list:
+                       throttle: "AdaptiveThrottle" = None,
+                       total_chars: int = 0) -> list:
    """Fetch full text of top N result pages concurrently.

    Features:
@@ -1345,10 +1461,17 @@ def fetch_top_results(results: dict, count: int, timeout: int = 10,
    - Falls back to browser User-Agent if blocked (in fetch_url)
    - Small delay between requests to avoid rate limits

+    v2.5.0 全局字符预算（``total_chars``）：
+    按原始结果顺序从顶部开始分配字符额度——前一页实际消费后剩余的预算
+    留给下一页（顺序填满，保证最重要的结果拿到完整正文）。预算耗尽后
+    剩余 URL 返回 ``status="skipped"``（不发请求），让 AI Agent 明确知道
+    是预算跳过而非抓取失败。默认 0 = 不限制。
+
    Args:
        referer: Referer URL（v2.0.0，通常设为 SearXNG 实例 URL）
        fallback_enabled: 是否启用 Wayback 兜底（默认 True）
        throttle: 外部传入的 AdaptiveThrottle 实例（可选；不传则内部创建）
+        total_chars: 全局字符预算（v2.5.0；0 = 不限）
    """
    urls = []
    seen = set()
@@ -1370,16 +1493,57 @@ def fetch_top_results(results: dict, count: int, timeout: int = 10,
    logger.info(f"\nFetching {len(urls)} result pages "
                f"(timeout={timeout}s, retries={max_retries}, "
                f"delay={throttle.delay}s, concurrency={throttle.concurrency})...")
+    if total_chars and total_chars > 0:
+        logger.info(f"Char budget: {total_chars:,} total (--fetch-total-chars)")
    fetched = []
    ok_count = [0]
    err_count = [0]
+    skipped_count = [0]
    anti_bot_count = [0]
    fallback_count = [0]

+    # v2.5.0 全局预算的线程安全计数器：_budget_remaining[0] 是剩余可用字符。
+    # _budget_enabled[0] 区分"未启用预算"（total_chars<=0，无限）与"已耗尽"
+    # （remaining=0）——前者放行所有 URL，后者跳过。_consume_budget 只在
+    # 抓取成功后按实际 text_length 扣减，未用掉的额度自动留给下一页。
+    _budget_enabled = [total_chars is not None and total_chars > 0]
+    _budget_remaining = [total_chars if _budget_enabled[0] else 0]
+    _budget_lock = threading.Lock()
+
+    def _take_budget() -> "tuple":
+        """分配本 URL 的字符上限。返回 (cap, allowed)。
+
+        cap 为 None 表示预算未启用（无限）。allowed=False 表示预算已耗尽，
+        调用方应跳过本 URL（status="skipped"，不发请求）。
+        """
+        if not _budget_enabled[0]:
+            return None, True
+        with _budget_lock:
+            remaining = _budget_remaining[0]
+            if remaining <= 0:
+                return 0, False
+            return remaining, True
+
+    def _consume_budget(used: int) -> None:
+        """抓取成功后按实际消费的字符数扣减预算。"""
+        with _budget_lock:
+            _budget_remaining[0] = max(0, _budget_remaining[0] - used)
+
    def _fetch_one(u: str) -> dict:
        """Fetch one URL with adaptive throttling to avoid rate limiting."""
        # 全局暂停检查（429 触发）
        throttle.wait_if_paused()
+        # v2.5.0 预算检查放在取槽位之前：预算耗尽时连并发槽位都不占用
+        cap, allowed = _take_budget()
+        if not allowed:
+            skipped_count[0] += 1
+            logger.info(f"  [BUDGET] char budget exhausted, skipping {u[:55]}")
+            return {"url": u, "status": "skipped",
+                    "error": "char budget exhausted (--fetch-total-chars)",
+                    "text": "", "text_length": 0, "truncated": False,
+                    "anti_bot_detected": False, "waf_type": None,
+                    "fallback_used": None,
+                    "title": None, "latency": None}
        # v2.2.2：真实并发门控。退避降并发后，超出当前并发目标的新请求
        # 在这里被拒绝（不占用 fetch_page），实现持久降并发而非名义降并发。
        if not throttle.acquire_slot():
@@ -1403,6 +1567,15 @@ def fetch_top_results(results: dict, count: int, timeout: int = 10,
            if result["status"] == "ok":
                ok_count[0] += 1
                throttle.report_success()
+                # v2.5.0 字符预算：按分配到的上限截断正文（提取后语义截断），
+                # 再按实际消费扣减预算。cap 为 None 表示未启用预算。
+                if cap:
+                    text = result.get("text", "")
+                    if len(text) > cap:
+                        result["text"] = text[:cap]
+                        result["text_length"] = cap
+                        result["truncated"] = True
+                    _consume_budget(result["text_length"])
                trunc = ", TRUNCATED" if result.get("truncated") else ""
                ua_note = ""
                if result.get("user_agent_used") != USER_AGENT:
@@ -1445,10 +1618,50 @@ def fetch_top_results(results: dict, count: int, timeout: int = 10,
    fetched.sort(key=lambda f: url_order.get(f["url"], 999))

    logger.info(f"Fetched: {ok_count[0]} ok, {err_count[0]} errors"
-                f" (anti-bot: {anti_bot_count[0]}, wayback: {fallback_count[0]})")
+                f" (anti-bot: {anti_bot_count[0]}, wayback: {fallback_count[0]}, "
+                f"skipped: {skipped_count[0]})")
    return fetched


+def deduplicate_fetched_content(fetched: list, threshold: float = 0.85) -> int:
+    """按正文相似度去重已抓取页面（v2.5.0 --dedup-fetched-content）。
+
+    镜像/转载站点的正文与原文高度相似但 URL 不同——URL 级去重
+    （deduplicate_results）无法合并。这里对每个成功页面（status="ok"）
+    的正文计算 SimHash 指纹，与已保留的页面两两比较，后出现的近似重复
+    条目标记为 ``status="duplicate"``（text 清空，保留 url/title/error 供
+    AI 查看原因），避免 AI 阅读同一内容的多个副本浪费 token。
+
+    只对 ok 条目参与去重；error/skipped/duplicate 条目原样保留。
+    ``fetched`` 保持原始结果顺序（首个出现的保留，通常 score 最高）。
+    返回标记为 duplicate 的条目数。
+    """
+    kept = []
+    dup_count = 0
+    for f in fetched:
+        if f.get("status") != "ok":
+            kept.append(f)
+            continue
+        text = f.get("text", "")
+        is_dup = False
+        for k in kept:
+            if k.get("status") != "ok":
+                continue
+            if texts_are_similar(text, k.get("text", ""), threshold=threshold):
+                is_dup = True
+                break
+        if is_dup:
+            dup_count += 1
+            f["status"] = "duplicate"
+            f["text"] = ""
+            f["text_length"] = 0
+            f["error"] = f"duplicate content (similar to {k.get('url', '')})"
+            kept.append(f)
+        else:
+            kept.append(f)
+    return dup_count
+
+
 def _emit_fetch_report(fetched: list, throttle: "AdaptiveThrottle",
                       fmt: str = "text") -> None:
    """输出结构化抓取报告到 stderr。
@@ -1491,10 +1704,14 @@ def _emit_fetch_report_text(fetched: list, throttle: "AdaptiveThrottle") -> None
    total = len(fetched)
    ok = sum(1 for f in fetched if f.get("status") == "ok")
    err = total - ok
+    skipped = sum(1 for f in fetched if f.get("status") == "skipped")
+    dup = sum(1 for f in fetched if f.get("status") == "duplicate")
+    err = err - skipped - dup  # skipped/duplicate 不是错误，单独统计
    anti_bot = sum(1 for f in fetched if f.get("anti_bot_detected"))
    wayback = sum(1 for f in fetched if f.get("fallback_used") == "wayback")
    lines.append("-" * len(header))
    lines.append(f"Total: {total} | OK: {ok} | Error: {err} | "
+                 f"Skipped: {skipped} | Duplicate: {dup} | "
                 f"Anti-bot blocked: {anti_bot} | Wayback recovered: {wayback}")

    # 自适应限流状态
@@ -1507,7 +1724,8 @@ def _emit_fetch_report_text(fetched: list, throttle: "AdaptiveThrottle") -> None

    # JSON 摘要（一行，便于 Agent 解析）
    summary = {
-        "total": total, "ok": ok, "error": err,
+        "total": total, "ok": ok, "error": err, "skipped": skipped,
+        "duplicate": dup,
        "anti_bot_blocked": anti_bot, "wayback_recovered": wayback,
        "throttle": s,
    }
@@ -1551,6 +1769,9 @@ def _emit_fetch_report_json(fetched: list, throttle: "AdaptiveThrottle") -> None
    total = len(fetched)
    ok = sum(1 for f in fetched if f.get("status") == "ok")
    err = total - ok
+    skipped = sum(1 for f in fetched if f.get("status") == "skipped")
+    dup = sum(1 for f in fetched if f.get("status") == "duplicate")
+    err = err - skipped - dup  # skipped/duplicate 不是错误，单独统计
    anti_bot = sum(1 for f in fetched if f.get("anti_bot_detected"))
    wayback = sum(1 for f in fetched if f.get("fallback_used") == "wayback")

@@ -1559,6 +1780,8 @@ def _emit_fetch_report_json(fetched: list, throttle: "AdaptiveThrottle") -> None
        "total": total,
        "ok": ok,
        "error": err,
+        "skipped": skipped,
+        "duplicate": dup,
        "anti_bot_blocked": anti_bot,
        "wayback_recovered": wayback,
        "throttle": throttle.stats(),
@@ -1782,6 +2005,28 @@ def format_urls(results: dict) -> str:
    return "\n".join(r.get("url", "") for r in results.get("results", []) if r.get("url"))


+# ----- Media-category CSV columns (v2.5.0) -----
+# SearXNG 的 images 类别结果（template="images.html"）带 img_src/thumbnail_src/
+# resolution/source，videos 类别（template="videos.html"）带 iframe_src/
+# thumbnail_src。--format json 已透传这些字段（原始 dict 原样序列化），但
+# CSV 固定列会丢弃。以下辅助让 CSV 在结果含媒体字段时自动追加对应列，
+# 表格分析（图片缩略图链接、视频内嵌 URL）直接可用。
+_MEDIA_COLUMNS = ["img_src", "thumbnail_src", "resolution", "iframe_src", "source"]
+
+
+def _detect_media_columns(results_iter) -> list:
+    """返回结果集中实际存在的媒体列子集（保持 _MEDIA_COLUMNS 顺序）。
+
+    仅当至少一个结果包含非空值时才列出该列，避免 CSV 出现整列空白。
+    """
+    present = set()
+    for r in results_iter:
+        for col in _MEDIA_COLUMNS:
+            if r.get(col):
+                present.add(col)
+    return [c for c in _MEDIA_COLUMNS if c in present]
+
+
 def _format_results(results: dict, args) -> str:
    """Format a results dict into the output string selected by args.format.

@@ -1796,9 +2041,11 @@ def _format_results(results: dict, args) -> str:
    if args.format == "csv":
        out = io.StringIO()
        writer = csv.writer(out, lineterminator="\n")
+        rs = results.get("results", [])
+        media_cols = _detect_media_columns(rs)
        writer.writerow(["title", "url", "engine", "score",
-                         "published_date", "content"])
-        for r in results.get("results", []):
+                         "published_date", "content"] + media_cols)
+        for r in rs:
            writer.writerow([
                r.get("title", ""),
                r.get("url", ""),
@@ -1806,7 +2053,7 @@ def _format_results(results: dict, args) -> str:
                r.get("score", "") if r.get("score") is not None else "",
                r.get("published_date", ""),
                r.get("content", ""),
-            ])
+            ] + [r.get(c, "") for c in media_cols])
        return out.getvalue().rstrip()
    # brief (also the safe fallthrough)
    output = format_brief(results, snippet_len=args.snippet_len)
@@ -1818,6 +2065,10 @@ def _format_results(results: dict, args) -> str:
            output += f"\n--- {f['url']} ---\n"
            if f["status"] == "ok":
                output += f"{f['text']}\n"
+            elif f["status"] == "skipped":
+                output += f"[SKIPPED: {f.get('error', 'char budget exhausted')}]\n"
+            elif f["status"] == "duplicate":
+                output += f"[DUPLICATE: {f.get('error', 'similar content')}]\n"
            else:
                output += f"[ERROR: {f.get('error', 'unknown')}]\n"
    return output
@@ -2045,13 +2296,28 @@ def _run_single_query(query: str, args, instance_urls: list,
            referer=referer,
            fallback_enabled=not getattr(args, "no_fallback", False),
            throttle=fetch_throttle,
+            total_chars=getattr(args, "fetch_total_chars", 0),
        )
-        # Emit fetch_ok / fetch_fail events
+        # v2.5.0: 正文相似度去重（--dedup-fetched-content），复用
+        # --similarity-threshold 阈值。镜像/转载页标记为 status="duplicate"。
+        if getattr(args, "dedup_fetched_content", False):
+            dup_n = deduplicate_fetched_content(fetched,
+                                                threshold=args.similarity_threshold)
+            if dup_n:
+                logger.info(f"Fetched-content dedup: {dup_n} duplicate page(s) "
+                            f"dropped (threshold={args.similarity_threshold})")
+        # Emit fetch_ok / fetch_fail / fetch_duplicate / fetch_skip events
        for f in fetched:
            if f.get("status") == "ok":
                emit_progress("fetch_ok", url=f.get("url", ""),
                              chars=f.get("text_length", 0),
                              fallback=f.get("fallback_used"))
+            elif f.get("status") == "duplicate":
+                emit_progress("fetch_duplicate", url=f.get("url", ""),
+                              error=f.get("error", "duplicate content"))
+            elif f.get("status") == "skipped":
+                emit_progress("fetch_skip", url=f.get("url", ""),
+                              error=f.get("error", "budget exhausted"))
            else:
                emit_progress("fetch_fail", url=f.get("url", ""),
                              error=f.get("error", "unknown"),
@@ -2110,7 +2376,15 @@ def _get_output_schema():
            "url": {"type": "string"},
            "final_url": {"type": ["string", "null"],
                          "description": "URL after redirects / Wayback."},
-            "status": {"type": "string", "enum": ["ok", "error"]},
+            "status": {"type": "string",
+                        "enum": ["ok", "error", "skipped", "duplicate"],
+                        "description": "'skipped' (v2.5.0) means the page was "
+                                       "not fetched because the "
+                                       "--fetch-total-chars budget was "
+                                       "exhausted. 'duplicate' (v2.5.0) means "
+                                       "the body was near-identical to an "
+                                       "earlier page and was dropped by "
+                                       "--dedup-fetched-content."},
            "content_type": {"type": ["string", "null"]},
            "text": {"type": "string", "description": "Extracted page text."},
            "text_length": {"type": "integer"},
@@ -2371,6 +2645,7 @@ def _save_config(args, path: str) -> None:
        "fetch": "fetch",
        "fetch_timeout": "fetch_timeout",
        "fetch_retries": "fetch_retries",
+        "fetch_total_chars": "fetch_total_chars",
        "throttle_failure_threshold": "throttle_failure_threshold",
        "throttle_pause_seconds": "throttle_pause_seconds",
        "throttle_max_delay": "throttle_max_delay",
@@ -2421,6 +2696,13 @@ def _dry_run_preview(args, instance_urls: list, auth_headers: dict) -> None:
    elif args.queries_file:
        preview["action"] = "batch"
        preview["queries_file"] = args.queries_file
+        # v2.5.0: dry-run 批量模式打印实际查询列表（读本地文件，不发 HTTP）。
+        # 读取失败时降级为仅文件名，不阻塞预览。
+        try:
+            qs = _read_queries_file(args.queries_file)
+            preview["queries"] = qs
+        except RuntimeError as e:
+            preview["queries_error"] = str(e)
    elif args.verify:
        preview["action"] = "verify"
    else:
@@ -2539,19 +2821,27 @@ def _handle_research(args, instance_urls: list, auth_headers: dict,
    error_count = 0
    for i, (angle, q) in enumerate(research_queries, 1):
        logger.info(f"\n[{i}/{len(research_queries)}] [{angle}] {q}")
+        # v2.5.0: 角度级进度事件——_run_single_query 内部的事件无法区分
+        # "当前第几个角度"，加 angle_* 事件让 AI Agent 能跟踪多角度进度
+        emit_progress("angle_start", angle=angle, query=q,
+                      index=i, total=len(research_queries))
        results, err, err_code = _run_single_query(q, args, instance_urls,
                                                    auth_headers, ttl_seconds)
        if err:
            error_count += 1
            logger.error(f"  [ERROR] {err}")
+            emit_progress("angle_fail", angle=angle, query=q,
+                          error=err, error_code=err_code)
            entry = {"query": q, "angle": angle, "status": "error",
                     "error": err}
            if err_code:
                entry["error_code"] = err_code
            batch.append(entry)
        else:
-            if len(results.get("results", [])) > 0:
+            angle_count = len(results.get("results", []))
+            if angle_count > 0:
                any_with_results = True
+            emit_progress("angle_ok", angle=angle, query=q, results=angle_count)
            batch.append({"query": q, "angle": angle, "status": "ok",
                          "results": results})

@@ -2585,8 +2875,12 @@ def _handle_research(args, instance_urls: list, auth_headers: dict,
    elif args.format == "csv":
        out = io.StringIO()
        writer = csv.writer(out, lineterminator="\n")
+        # v2.5.0: 媒体列检测——研究模式内任意角度含图片/视频字段则追加
+        media_cols = _detect_media_columns(
+            br["results"].get("results", [])
+            for br in batch if "results" in br)
        writer.writerow(["angle", "query", "title", "url", "engine",
-                         "score", "published_date", "content"])
+                         "score", "published_date", "content"] + media_cols)
        for br in batch:
            q = br["query"]
            angle = br.get("angle", "")
@@ -2600,7 +2894,7 @@ def _handle_research(args, instance_urls: list, auth_headers: dict,
                        r.get("score", "") if r.get("score") is not None else "",
                        r.get("published_date", ""),
                        r.get("content", ""),
-                    ])
+                    ] + [r.get(c, "") for c in media_cols])
            else:
                writer.writerow([angle, q, "", "", "", "", "",
                                 f"[ERROR: {br['error']}]"])
@@ -2772,8 +3066,12 @@ def _handle_batch(args, instance_urls: list, auth_headers: dict,
    elif args.format == "csv":
        out = io.StringIO()
        writer = csv.writer(out, lineterminator="\n")
+        # v2.5.0: 媒体列检测——批量内任意结果含图片/视频字段则追加对应列
+        media_cols = _detect_media_columns(
+            br["results"].get("results", [])
+            for br in batch if "results" in br)
        writer.writerow(["query", "title", "url", "engine", "score",
-                         "published_date", "content"])
+                         "published_date", "content"] + media_cols)
        for br in batch:
            q = br["query"]
            if "results" in br:
@@ -2786,7 +3084,7 @@ def _handle_batch(args, instance_urls: list, auth_headers: dict,
                        r.get("score", "") if r.get("score") is not None else "",
                        r.get("published_date", ""),
                        r.get("content", ""),
-                    ])
+                    ] + [r.get(c, "") for c in media_cols])
            else:
                writer.writerow([q, "", "", "", "", "",
                                 f"[ERROR: {br['error']}]"])
@@ -2992,6 +3290,14 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
                        metavar="FLOAT",
                        help="相似度去重阈值（默认: 0.85）。越高越严格，1.0 要求标题几乎完全相同。"
                             "仅当 --similarity-dedup 启用时生效。")
+    parser.add_argument("--dedup-fetched-content", action="store_true",
+                        help="v2.5.0: deduplicate fetched page content by "
+                             "similarity (SimHash). Mirror/republished pages "
+                             "with near-identical bodies are marked "
+                             "status='duplicate' (text cleared) — keeps AI "
+                             "from reading the same content multiple times. "
+                             "Uses --similarity-threshold (default 0.85). "
+                             "Only applies with --fetch.")
    parser.add_argument("--format", "-f", choices=["json", "brief", "urls", "csv"],
                        default=config.get("format", "json"),
                        help="Output format (default: json). 'csv' exports "
@@ -3004,6 +3310,16 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
                        help="Timeout per page fetch in seconds (default: 10)")
    parser.add_argument("--fetch-retries", type=int, default=_cfg_int(config, "fetch_retries", 3),
                        help="Max retries per page fetch (default: 3)")
+    parser.add_argument("--fetch-total-chars", type=int,
+                        default=_cfg_int(config, "fetch_total_chars", 0),
+                        metavar="N",
+                        help="v2.5.0: global character budget for --fetch. "
+                             "Chars are allocated top-down in result order — "
+                             "unused budget rolls over to the next page. Once "
+                             "exhausted, remaining URLs are marked status="
+                             "'skipped' (no request sent). Lets token-limited "
+                             "agents cap total fetched content. 0 = unlimited "
+                             "(default).")
    parser.add_argument("--fetch-report", dest="fetch_report",
                        nargs="?", const="text", default=False, metavar="FORMAT",
                        help="When used with --fetch, emit a structured fetch report to stderr "