fix: 文档解析链路修复 —— 缺失导入 + 静默吞错
- fetch.py: 补全 subprocess/zipfile/xml.etree.ElementTree/io 导入 (PDF/DOCX/XLSX 解析此前直接 NameError) - fetch.py main(): 检查 FetchResult.error_code, 文档解析失败不再 静默输出空内容 + exit 0, 改为明确报错并 exit 1 - search.py fetch_page(): 同样检查 error_code, 修复 --fetch 抓取 PDF 失败被误报 status=ok 的问题 - _parse_document_content: image/audio/video/* 类型统一返回 E_UNSUPPORTED_MEDIA (此前退化为乱码文本)
This commit is contained in:
+13
-1
@@ -10,13 +10,17 @@ for improved extraction quality (optional, falls back to stdlib).
|
|||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import gzip
|
import gzip
|
||||||
|
import io
|
||||||
import logging
|
import logging
|
||||||
import random
|
import random
|
||||||
import re
|
import re
|
||||||
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
import urllib.error
|
import urllib.error
|
||||||
import urllib.request
|
import urllib.request
|
||||||
|
import xml.etree.ElementTree as ET
|
||||||
|
import zipfile
|
||||||
import zlib
|
import zlib
|
||||||
from collections import namedtuple
|
from collections import namedtuple
|
||||||
from html.parser import HTMLParser
|
from html.parser import HTMLParser
|
||||||
@@ -867,7 +871,7 @@ def _parse_document_content(raw: bytes, content_type: str):
|
|||||||
if ct == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet":
|
if ct == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet":
|
||||||
return _parse_xlsx(raw)
|
return _parse_xlsx(raw)
|
||||||
|
|
||||||
if ct in _BINARY_CONTENT_TYPES:
|
if ct.startswith(("image/", "audio/", "video/")) or ct in _BINARY_CONTENT_TYPES:
|
||||||
return (None, E_UNSUPPORTED_MEDIA,
|
return (None, E_UNSUPPORTED_MEDIA,
|
||||||
f"Unsupported binary content type: {ct}")
|
f"Unsupported binary content type: {ct}")
|
||||||
|
|
||||||
@@ -1236,6 +1240,14 @@ Examples:
|
|||||||
content, content_type, final_url = (
|
content, content_type, final_url = (
|
||||||
result.content, result.content_type, result.final_url,
|
result.content, result.content_type, result.final_url,
|
||||||
)
|
)
|
||||||
|
# 文档解析失败(PDF/DOCX/XLSX 等)时 fetch_url 不抛异常,
|
||||||
|
# 而是返回带 error_code 的 FetchResult——必须显式检查,
|
||||||
|
# 否则失败会被静默吞掉(空输出 + exit 0)。
|
||||||
|
if result.error_code:
|
||||||
|
logger.error(
|
||||||
|
f"Error: {result.error_message or result.error_code} "
|
||||||
|
f"(error_code={result.error_code}, url={args.url})")
|
||||||
|
sys.exit(1)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
# 诊断信息增强:从 __cause__ 链中提取 HTTP 状态码、原始异常类型,
|
# 诊断信息增强:从 __cause__ 链中提取 HTTP 状态码、原始异常类型,
|
||||||
# 让 AI Agent 能程序化判断失败原因(404 vs 403 vs DNS 失败等),
|
# 让 AI Agent 能程序化判断失败原因(404 vs 403 vs DNS 失败等),
|
||||||
|
|||||||
@@ -879,6 +879,14 @@ def fetch_page(url: str, timeout: int = 10, auth_headers: dict = None,
|
|||||||
# 而非字符串匹配("Too Many Requests" 不含 "429" 会漏判)
|
# 而非字符串匹配("Too Many Requests" 不含 "429" 会漏判)
|
||||||
error_code = classify_error(e)
|
error_code = classify_error(e)
|
||||||
|
|
||||||
|
# fetch_url 对 PDF/DOCX/XLSX 解析失败不抛异常,而是返回带 error_code
|
||||||
|
# 的 FetchResult——此处必须显式检查,否则失败会被当作成功处理
|
||||||
|
# (空文本 + status="ok")。
|
||||||
|
if result is not None and result.error_code:
|
||||||
|
error_msg = result.error_message or result.error_code
|
||||||
|
error_code = result.error_code
|
||||||
|
result = None
|
||||||
|
|
||||||
# 反爬检测(v2.0.0 增强:全文档扫描 + WAF 指纹库)
|
# 反爬检测(v2.0.0 增强:全文档扫描 + WAF 指纹库)
|
||||||
# 必须在 Wayback 兜底判断之前执行:Cloudflare 质询页常返回 HTTP 200,
|
# 必须在 Wayback 兜底判断之前执行:Cloudflare 质询页常返回 HTTP 200,
|
||||||
# 此时 result 非 None 但内容是反爬页,必须识别出来才能触发兜底。
|
# 此时 result 非 None 但内容是反爬页,必须识别出来才能触发兜底。
|
||||||
|
|||||||
Reference in New Issue
Block a user