fix: 文档解析链路修复 —— 缺失导入 + 静默吞错

- fetch.py: 补全 subprocess/zipfile/xml.etree.ElementTree/io 导入
  (PDF/DOCX/XLSX 解析此前直接 NameError)
- fetch.py main(): 检查 FetchResult.error_code, 文档解析失败不再
  静默输出空内容 + exit 0, 改为明确报错并 exit 1
- search.py fetch_page(): 同样检查 error_code, 修复 --fetch 抓取
  PDF 失败被误报 status=ok 的问题
- _parse_document_content: image/audio/video/* 类型统一返回
  E_UNSUPPORTED_MEDIA (此前退化为乱码文本)
This commit is contained in:
2026-08-03 17:51:05 +08:00
parent 157219d982
commit 6cedba9042
2 changed files with 21 additions and 1 deletions
+13 -1
View File
@@ -10,13 +10,17 @@ for improved extraction quality (optional, falls back to stdlib).
import argparse import argparse
import gzip import gzip
import io
import logging import logging
import random import random
import re import re
import subprocess
import sys import sys
import time import time
import urllib.error import urllib.error
import urllib.request import urllib.request
import xml.etree.ElementTree as ET
import zipfile
import zlib import zlib
from collections import namedtuple from collections import namedtuple
from html.parser import HTMLParser from html.parser import HTMLParser
@@ -867,7 +871,7 @@ def _parse_document_content(raw: bytes, content_type: str):
if ct == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": if ct == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet":
return _parse_xlsx(raw) return _parse_xlsx(raw)
if ct in _BINARY_CONTENT_TYPES: if ct.startswith(("image/", "audio/", "video/")) or ct in _BINARY_CONTENT_TYPES:
return (None, E_UNSUPPORTED_MEDIA, return (None, E_UNSUPPORTED_MEDIA,
f"Unsupported binary content type: {ct}") f"Unsupported binary content type: {ct}")
@@ -1236,6 +1240,14 @@ Examples:
content, content_type, final_url = ( content, content_type, final_url = (
result.content, result.content_type, result.final_url, result.content, result.content_type, result.final_url,
) )
# 文档解析失败(PDF/DOCX/XLSX 等)时 fetch_url 不抛异常,
# 而是返回带 error_code 的 FetchResult——必须显式检查,
# 否则失败会被静默吞掉(空输出 + exit 0)。
if result.error_code:
logger.error(
f"Error: {result.error_message or result.error_code} "
f"(error_code={result.error_code}, url={args.url})")
sys.exit(1)
except Exception as e: except Exception as e:
# 诊断信息增强:从 __cause__ 链中提取 HTTP 状态码、原始异常类型, # 诊断信息增强:从 __cause__ 链中提取 HTTP 状态码、原始异常类型,
# 让 AI Agent 能程序化判断失败原因(404 vs 403 vs DNS 失败等), # 让 AI Agent 能程序化判断失败原因(404 vs 403 vs DNS 失败等),
+8
View File
@@ -879,6 +879,14 @@ def fetch_page(url: str, timeout: int = 10, auth_headers: dict = None,
# 而非字符串匹配("Too Many Requests" 不含 "429" 会漏判) # 而非字符串匹配("Too Many Requests" 不含 "429" 会漏判)
error_code = classify_error(e) error_code = classify_error(e)
# fetch_url 对 PDF/DOCX/XLSX 解析失败不抛异常,而是返回带 error_code
# 的 FetchResult——此处必须显式检查,否则失败会被当作成功处理
# (空文本 + status="ok")。
if result is not None and result.error_code:
error_msg = result.error_message or result.error_code
error_code = result.error_code
result = None
# 反爬检测(v2.0.0 增强:全文档扫描 + WAF 指纹库) # 反爬检测(v2.0.0 增强:全文档扫描 + WAF 指纹库)
# 必须在 Wayback 兜底判断之前执行:Cloudflare 质询页常返回 HTTP 200 # 必须在 Wayback 兜底判断之前执行:Cloudflare 质询页常返回 HTTP 200
# 此时 result 非 None 但内容是反爬页,必须识别出来才能触发兜底。 # 此时 result 非 None 但内容是反爬页,必须识别出来才能触发兜底。