diff --git a/scripts/fetch.py b/scripts/fetch.py index 88f68c5..57688c2 100644 --- a/scripts/fetch.py +++ b/scripts/fetch.py @@ -10,13 +10,17 @@ for improved extraction quality (optional, falls back to stdlib). import argparse import gzip +import io import logging import random import re +import subprocess import sys import time import urllib.error import urllib.request +import xml.etree.ElementTree as ET +import zipfile import zlib from collections import namedtuple from html.parser import HTMLParser @@ -867,7 +871,7 @@ def _parse_document_content(raw: bytes, content_type: str): if ct == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": return _parse_xlsx(raw) - if ct in _BINARY_CONTENT_TYPES: + if ct.startswith(("image/", "audio/", "video/")) or ct in _BINARY_CONTENT_TYPES: return (None, E_UNSUPPORTED_MEDIA, f"Unsupported binary content type: {ct}") @@ -1236,6 +1240,14 @@ Examples: content, content_type, final_url = ( result.content, result.content_type, result.final_url, ) + # 文档解析失败(PDF/DOCX/XLSX 等)时 fetch_url 不抛异常, + # 而是返回带 error_code 的 FetchResult——必须显式检查, + # 否则失败会被静默吞掉(空输出 + exit 0)。 + if result.error_code: + logger.error( + f"Error: {result.error_message or result.error_code} " + f"(error_code={result.error_code}, url={args.url})") + sys.exit(1) except Exception as e: # 诊断信息增强:从 __cause__ 链中提取 HTTP 状态码、原始异常类型, # 让 AI Agent 能程序化判断失败原因(404 vs 403 vs DNS 失败等), diff --git a/scripts/search.py b/scripts/search.py index 464ac84..9af0188 100644 --- a/scripts/search.py +++ b/scripts/search.py @@ -879,6 +879,14 @@ def fetch_page(url: str, timeout: int = 10, auth_headers: dict = None, # 而非字符串匹配("Too Many Requests" 不含 "429" 会漏判) error_code = classify_error(e) + # fetch_url 对 PDF/DOCX/XLSX 解析失败不抛异常,而是返回带 error_code + # 的 FetchResult——此处必须显式检查,否则失败会被当作成功处理 + # (空文本 + status="ok")。 + if result is not None and result.error_code: + error_msg = result.error_message or result.error_code + error_code = result.error_code + result = None + # 反爬检测(v2.0.0 增强:全文档扫描 + WAF 指纹库) # 必须在 Wayback 兜底判断之前执行:Cloudflare 质询页常返回 HTTP 200, # 此时 result 非 None 但内容是反爬页,必须识别出来才能触发兜底。