feat(v2.2.1): 修复 Brotli 乱码 + 缓存治理 + 多页聚合 + 研究模式增强
核心修复（v2.2.1）: - 修复 Brotli 乱码 bug: build_browser_headers 智能声明 Accept-Encoding, 仅在 brotli 可用时才声明 br; fetch.py 双路径 br 解压（requests + stdlib） 此前 Chrome/Edge UA 抓取 example.com 等返回 br 的站点输出乱码 v2.2.0 新功能: - main() 拆分为 _handle_verify/_handle_research/_handle_batch/_handle_single - --cache-max-size MB: 缓存大小上限 + LRU 淘汰（默认 100MB） - --pages N: 多页聚合 + 跨页去重 - --research 跨角度合并: 新增 merged_results 字段 - --stream / --progress: JSON Lines 流式输出 + request_id 贯穿 - --dry-run / --save-config / --log-format json - --similarity-dedup / --throttle-* 参数化 - 15-UA 池 + PDF/docx 解析 + error_code 字段 文档与测试: - SKILL.md: 版本号唯一（元数据），删除版本标记干扰 - README.md: 测试数量 539 -> 544 - 544 passed (新增 5 个 Content-Encoding 解压测试)
This commit is contained in:
+329
-32
@@ -23,6 +23,21 @@ from html.parser import HTMLParser
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
# Brotli 解压支持检测（v2.2.1）。
|
||||
# requests 自动解压 gzip/deflate，但**不自动解压 Brotli**（除非安装
|
||||
# brotli/brotlicffi）。build_browser_headers() 已根据此检测智能声明
|
||||
# Accept-Encoding，此处作为双保险：若代理/CDN 强制返回 br，仍可解压。
|
||||
try:
|
||||
import brotli as _brotli # type: ignore
|
||||
_HAS_BROTLI = True
|
||||
except ImportError:
|
||||
try:
|
||||
import brotlicffi as _brotli # type: ignore
|
||||
_HAS_BROTLI = True
|
||||
except ImportError:
|
||||
_brotli = None
|
||||
_HAS_BROTLI = False
|
||||
|
||||
# Allow running standalone from any working directory
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from _config import USER_AGENT, VERSION
|
||||
@@ -50,6 +65,11 @@ from common import (
|
||||
logger = logging.getLogger("searxng.fetch")
|
||||
|
||||
|
||||
# 错误码：不支持的媒体类型（PDF/DOCX/XLSX 解析失败或未知二进制类型）。
|
||||
# 与 common.py 中 E_CONFIG / E_AUTH / E_NETWORK 等错误码保持一致的 E_* 命名模式。
|
||||
E_UNSUPPORTED_MEDIA = "E_UNSUPPORTED_MEDIA"
|
||||
|
||||
|
||||
# ----- Auth helpers -----
|
||||
# build_auth_headers is imported from common.py
|
||||
|
||||
@@ -167,6 +187,37 @@ def extract_with_bs4(html_content: str) -> str:
|
||||
return text
|
||||
|
||||
|
||||
# CJK 字符范围：中文 \u4e00-\u9fff、日文 \u3040-\u30ff、韩文 \uac00-\ud7af
|
||||
_CJK_CHAR_RE = re.compile(
|
||||
r"[\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]"
|
||||
)
|
||||
|
||||
|
||||
def _is_cjk_text(text: str) -> bool:
|
||||
"""判断文本是否以 CJK（中文/日文/韩文）为主。
|
||||
|
||||
统计 CJK 字符占非空白字符的比例，>30% 则视为 CJK 内容。
|
||||
CJK 文本信息密度高，readability-lite 的最小字符阈值应相应降低。
|
||||
"""
|
||||
if not text:
|
||||
return False
|
||||
# 按非空白字符统计，避免大量空白/缩进拉低比例造成误判
|
||||
non_ws_len = sum(1 for ch in text if not ch.isspace())
|
||||
if non_ws_len == 0:
|
||||
return False
|
||||
cjk_count = len(_CJK_CHAR_RE.findall(text))
|
||||
return cjk_count / non_ws_len > 0.30
|
||||
|
||||
|
||||
def _min_content_length(text: str) -> int:
|
||||
"""根据文本语言返回 readability-lite 最小正文字符阈值。
|
||||
|
||||
CJK 内容（信息密度高）：100 字符
|
||||
其他语言（英文等）：200 字符
|
||||
"""
|
||||
return 100 if _is_cjk_text(text) else 200
|
||||
|
||||
|
||||
def _readability_lite(root) -> "Optional[object]":
|
||||
"""readability-lite：用文本密度算法选择最可能是正文的子元素。
|
||||
|
||||
@@ -174,7 +225,7 @@ def _readability_lite(root) -> "Optional[object]":
|
||||
1. 遍历 body 下所有 div/section/article 子节点
|
||||
2. 计算每个节点的"文本密度" = 纯文本字符数 / 标签数
|
||||
3. 排除明显是导航/侧边栏的节点（class/id 含 nav/sidebar/menu/footer）
|
||||
4. 返回文本密度最高且字符数 > 200 的节点
|
||||
4. 返回文本密度最高且字符数超过阈值的节点（CJK 100，其他 200）
|
||||
|
||||
返回 bs4 Tag 或 None（找不到合适节点时）。
|
||||
这是 extract_with_bs4 的兜底增强，不改变原有 article/main 优先级。
|
||||
@@ -204,8 +255,10 @@ def _readability_lite(root) -> "Optional[object]":
|
||||
# 计算纯文本字符数（去空白）
|
||||
text = node.get_text(separator=" ", strip=True)
|
||||
text_len = len(text)
|
||||
if text_len < 200:
|
||||
continue # 正文至少 200 字符
|
||||
# 阈值按语言动态调整：CJK 内容 100 字符，其他 200 字符
|
||||
min_len = _min_content_length(text)
|
||||
if text_len < min_len:
|
||||
continue # 正文至少 min_len 字符（CJK 100，其他 200）
|
||||
|
||||
# 计算标签数（粗略：所有后代标签）
|
||||
tag_count = len(node.find_all())
|
||||
@@ -602,10 +655,225 @@ class _NoRedirectHandler(urllib.request.HTTPRedirectHandler):
|
||||
|
||||
FetchResult = namedtuple(
|
||||
"FetchResult",
|
||||
["content", "content_type", "final_url", "truncated", "user_agent"],
|
||||
["content", "content_type", "final_url", "truncated", "user_agent",
|
||||
"error_code", "error_message"],
|
||||
# error_code / error_message 默认 None：
|
||||
# * 向后兼容——旧的 5 参数构造（位置或关键字）仍然可用
|
||||
# * 仅当 fetch_url 遇到不支持的媒体类型时才填充
|
||||
defaults=[None, None],
|
||||
)
|
||||
|
||||
|
||||
def _handle_rate_limit_status(status_code, headers, attempt, max_retries):
|
||||
"""处理 429/503 限流响应的 Retry-After，决定是否重试。
|
||||
|
||||
统一抽取自 requests 路径、stdlib 路径、requests.exceptions 路径三处
|
||||
原本重复的 Retry-After 解析 + 退避 sleep 逻辑。
|
||||
|
||||
解析 Retry-After 头（秒数或 HTTP 日期，委托给 common.parse_retry_after），
|
||||
与 compute_backoff_delay(attempt) 取较大值作为实际等待时间。
|
||||
若 ``attempt < max_retries``，sleep 后返回 ``(True, retry_after_sec)``
|
||||
表示应当重试；否则返回 ``(False, retry_after_sec)``，由调用方决定后续
|
||||
（通常会落到 raise_for_status / 抛 RuntimeError）。
|
||||
|
||||
``headers`` 兼容 dict 和 http.client.HTTPMessage（均支持 ``.get()``）；
|
||||
为 None 时按空 header 处理（retry_after=0）。
|
||||
|
||||
返回 ``(should_retry, retry_after_seconds)``。
|
||||
"""
|
||||
retry_after_raw = ""
|
||||
if headers:
|
||||
retry_after_raw = headers.get("Retry-After", "") or ""
|
||||
retry_after_sec = parse_retry_after(retry_after_raw)
|
||||
if attempt < max_retries:
|
||||
delay = max(retry_after_sec, compute_backoff_delay(attempt))
|
||||
logger.info(f" Fetch retry {attempt+1}/{max_retries} "
|
||||
f"(HTTP {status_code}, Retry-After={retry_after_sec:.1f}s) "
|
||||
f"in {delay:.1f}s")
|
||||
time.sleep(delay)
|
||||
return (True, retry_after_sec)
|
||||
return (False, retry_after_sec)
|
||||
|
||||
|
||||
# 明确的非文本二进制 MIME 类型：无法作为文本 decode，直接拒绝。
|
||||
# application/octet-stream 是通用二进制兜底类型；其余为已知归档/可执行/
|
||||
# 旧版 Office（.doc/.xls/.ppt 不在本次支持范围）等。
|
||||
_BINARY_CONTENT_TYPES = frozenset([
|
||||
"application/octet-stream",
|
||||
"application/zip",
|
||||
"application/x-gzip",
|
||||
"application/gzip",
|
||||
"application/x-rar-compressed",
|
||||
"application/x-7z-compressed",
|
||||
"application/x-tar",
|
||||
"application/x-bzip",
|
||||
"application/x-bzip2",
|
||||
"application/x-msdownload",
|
||||
"application/x-shockwave-flash",
|
||||
"application/msword", # 旧 .doc（不支持）
|
||||
"application/vnd.ms-excel", # 旧 .xls（不支持）
|
||||
"application/vnd.ms-powerpoint", # 旧 .ppt（不支持）
|
||||
"application/x-elf",
|
||||
"application/x-executable",
|
||||
])
|
||||
|
||||
|
||||
# OOXML（.docx / .xlsx）主命名空间，ElementTree 用 {ns}tag 形式匹配
|
||||
_W_NS = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}"
|
||||
_S_NS = "{http://schemas.openxmlformats.org/spreadsheetml/2006/main}"
|
||||
|
||||
|
||||
def _parse_pdf(raw: bytes):
|
||||
"""用 pdftotext（poppler-utils）从 PDF 字节流提取文本。
|
||||
|
||||
通过 subprocess 调用 ``pdftotext - -``（stdin 读、stdout 写），
|
||||
不引入新依赖。pdftotext 不存在或失败时返回 E_UNSUPPORTED_MEDIA。
|
||||
|
||||
返回 ``(content, error_code, error_message)``：
|
||||
成功 → ``(text, None, None)``；失败 → ``(None, E_UNSUPPORTED_MEDIA, msg)``。
|
||||
"""
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
["pdftotext", "-", "-"],
|
||||
input=raw,
|
||||
capture_output=True,
|
||||
timeout=30,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
return (None, E_UNSUPPORTED_MEDIA,
|
||||
"PDF parsing requires poppler-utils (pdftotext) to be installed")
|
||||
except subprocess.TimeoutExpired:
|
||||
return (None, E_UNSUPPORTED_MEDIA, "PDF parsing timed out (>30s)")
|
||||
except OSError as e:
|
||||
return (None, E_UNSUPPORTED_MEDIA, f"PDF parsing failed: {e}")
|
||||
|
||||
if proc.returncode != 0:
|
||||
stderr = proc.stderr.decode("utf-8", errors="replace").strip()
|
||||
msg = f"pdftotext exited {proc.returncode}"
|
||||
if stderr:
|
||||
msg += f": {stderr[:200]}"
|
||||
return (None, E_UNSUPPORTED_MEDIA, msg)
|
||||
|
||||
text = proc.stdout.decode("utf-8", errors="replace")
|
||||
return (text, None, None)
|
||||
|
||||
|
||||
def _parse_docx(raw: bytes):
|
||||
"""从 .docx 字节流提取文本（stdlib zipfile + ElementTree）。
|
||||
|
||||
读取 ``word/document.xml``，按段落（<w:p>）提取 <w:t> 文本，
|
||||
段落间以换行分隔。失败时返回 E_UNSUPPORTED_MEDIA。
|
||||
|
||||
返回 ``(content, error_code, error_message)``。
|
||||
"""
|
||||
try:
|
||||
with zipfile.ZipFile(io.BytesIO(raw)) as zf:
|
||||
xml_bytes = zf.read("word/document.xml")
|
||||
except (zipfile.BadZipFile, KeyError) as e:
|
||||
return (None, E_UNSUPPORTED_MEDIA, f"DOCX parsing failed: {e}")
|
||||
|
||||
try:
|
||||
root = ET.fromstring(xml_bytes)
|
||||
except ET.ParseError as e:
|
||||
return (None, E_UNSUPPORTED_MEDIA, f"DOCX XML parse failed: {e}")
|
||||
|
||||
# 遍历段落，每段内拼接所有 <w:t>，段落间换行
|
||||
lines = []
|
||||
for p in root.iter(_W_NS + "p"):
|
||||
parts = [t.text for t in p.iter(_W_NS + "t") if t.text]
|
||||
if parts:
|
||||
lines.append("".join(parts))
|
||||
return ("\n".join(lines), None, None)
|
||||
|
||||
|
||||
def _parse_xlsx(raw: bytes):
|
||||
"""从 .xlsx 字节流提取文本（stdlib zipfile + ElementTree）。
|
||||
|
||||
读取 ``xl/sharedStrings.xml``（共享字符串表）与各
|
||||
``xl/worksheets/sheetN.xml``，按行提取单元格文本，单元格以制表符
|
||||
分隔、行以换行分隔。失败时返回 E_UNSUPPORTED_MEDIA。
|
||||
|
||||
返回 ``(content, error_code, error_message)``。
|
||||
"""
|
||||
try:
|
||||
with zipfile.ZipFile(io.BytesIO(raw)) as zf:
|
||||
# 共享字符串表（可能不存在——纯数字表格）
|
||||
shared = []
|
||||
try:
|
||||
sroot = ET.fromstring(zf.read("xl/sharedStrings.xml"))
|
||||
for si in sroot.iter(_S_NS + "si"):
|
||||
parts = [t.text for t in si.iter(_S_NS + "t") if t.text]
|
||||
shared.append("".join(parts))
|
||||
except (KeyError, ET.ParseError):
|
||||
pass # 无共享字符串表，单元格均为内联值
|
||||
|
||||
sheet_names = [n for n in zf.namelist()
|
||||
if re.match(r"xl/worksheets/sheet\d+\.xml$", n)]
|
||||
lines = []
|
||||
for sheet_name in sorted(sheet_names):
|
||||
try:
|
||||
sroot = ET.fromstring(zf.read(sheet_name))
|
||||
except ET.ParseError:
|
||||
continue
|
||||
for row in sroot.iter(_S_NS + "row"):
|
||||
cells = []
|
||||
for c in row.iter(_S_NS + "c"):
|
||||
cell_type = c.get("t")
|
||||
v = c.find(_S_NS + "v")
|
||||
if v is not None and v.text is not None:
|
||||
if cell_type == "s":
|
||||
# 共享字符串索引引用
|
||||
try:
|
||||
idx = int(v.text)
|
||||
cells.append(
|
||||
shared[idx] if 0 <= idx < len(shared) else "")
|
||||
except (ValueError, IndexError):
|
||||
cells.append("")
|
||||
else:
|
||||
cells.append(v.text)
|
||||
else:
|
||||
# 内联字符串 <is><t>...</t></is>
|
||||
is_el = c.find(_S_NS + "is")
|
||||
if is_el is not None:
|
||||
parts = [t.text for t in is_el.iter(_S_NS + "t")
|
||||
if t.text]
|
||||
cells.append("".join(parts))
|
||||
if cells:
|
||||
lines.append("\t".join(cells))
|
||||
return ("\n".join(lines), None, None)
|
||||
except (zipfile.BadZipFile, ET.ParseError) as e:
|
||||
return (None, E_UNSUPPORTED_MEDIA, f"XLSX parsing failed: {e}")
|
||||
|
||||
|
||||
def _parse_document_content(raw: bytes, content_type: str):
|
||||
"""根据 Content-Type 将二进制文档解析为文本。
|
||||
|
||||
支持：PDF（需 pdftotext）、DOCX、XLSX。
|
||||
对明确的非文本二进制 MIME（application/octet-stream、zip、rar 等）返回
|
||||
E_UNSUPPORTED_MEDIA。其他类型（text/* 、application/json、HTML 等）返回
|
||||
``(None, None, None)``，由调用方走原有 decode 流程。
|
||||
|
||||
返回 ``(content, error_code, error_message)``：
|
||||
* 非文档类型 → ``(None, None, None)``：调用方继续 decode
|
||||
* 解析成功 → ``(text, None, None)``
|
||||
* 解析失败 → ``(None, E_UNSUPPORTED_MEDIA, msg)``
|
||||
"""
|
||||
ct = (content_type or "").lower().split(";")[0].strip()
|
||||
|
||||
if ct == "application/pdf":
|
||||
return _parse_pdf(raw)
|
||||
if ct == "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
|
||||
return _parse_docx(raw)
|
||||
if ct == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet":
|
||||
return _parse_xlsx(raw)
|
||||
|
||||
if ct in _BINARY_CONTENT_TYPES:
|
||||
return (None, E_UNSUPPORTED_MEDIA,
|
||||
f"Unsupported binary content type: {ct}")
|
||||
|
||||
return (None, None, None)
|
||||
|
||||
|
||||
def fetch_url(url: str, timeout=15, user_agent: str = None,
|
||||
encoding: str = None, auth_headers: dict = None,
|
||||
max_retries: int = 3, max_size: int = None,
|
||||
@@ -669,16 +937,11 @@ def fetch_url(url: str, timeout=15, user_agent: str = None,
|
||||
try:
|
||||
# 429/503：读取 Retry-After，作为最小重试延迟
|
||||
if resp.status_code in (429, 503) and attempt < max_retries:
|
||||
retry_after_raw = resp.headers.get("Retry-After", "")
|
||||
retry_after_sec = parse_retry_after(retry_after_raw)
|
||||
resp.close()
|
||||
delay = max(retry_after_sec,
|
||||
compute_backoff_delay(attempt))
|
||||
logger.info(f" Fetch retry {attempt+1}/{max_retries} "
|
||||
f"(HTTP {resp.status_code}, Retry-After={retry_after_sec:.1f}s) "
|
||||
f"in {delay:.1f}s")
|
||||
time.sleep(delay)
|
||||
continue
|
||||
should_retry, _ = _handle_rate_limit_status(
|
||||
resp.status_code, resp.headers, attempt, max_retries)
|
||||
if should_retry:
|
||||
continue
|
||||
|
||||
resp.raise_for_status()
|
||||
|
||||
@@ -698,6 +961,33 @@ def fetch_url(url: str, timeout=15, user_agent: str = None,
|
||||
raw = b"".join(chunks)
|
||||
truncated = total > max_size
|
||||
|
||||
# v2.2.1 修复：requests 自动解压 gzip/deflate，但**不自动
|
||||
# 解压 Brotli**（除非安装 brotli 包）。当服务器返回
|
||||
# Content-Encoding: br 而本机有 brotli 解压器时，手动解压；
|
||||
# 否则保留原 raw，让下游 errors="replace" 兜底（虽是乱码但
|
||||
# 不崩溃）。build_browser_headers() 已尽量避免声明 br，此处
|
||||
# 作为双保险，应对代理/CDN 强制返回 br 的边缘情况。
|
||||
content_encoding = (resp.headers.get("Content-Encoding", "")
|
||||
.lower().strip())
|
||||
if "br" in content_encoding and _HAS_BROTLI and raw:
|
||||
try:
|
||||
raw = _brotli.decompress(raw)
|
||||
except Exception as e:
|
||||
logger.debug(f" brotli decompress failed: {e}")
|
||||
|
||||
# 二进制文档解析（PDF/DOCX/XLSX）及不支持的媒体类型检测。
|
||||
# 命中时直接返回，跳过后续文本 decode 流程。
|
||||
doc_text, doc_err_code, doc_err_msg = _parse_document_content(
|
||||
raw, resp.headers.get("Content-Type", ""))
|
||||
if doc_err_code is not None:
|
||||
return FetchResult(
|
||||
"", resp.headers.get("Content-Type", ""), resp.url,
|
||||
truncated, ua, doc_err_code, doc_err_msg)
|
||||
if doc_text is not None:
|
||||
return FetchResult(
|
||||
doc_text, resp.headers.get("Content-Type", ""), resp.url,
|
||||
truncated, ua, None, None)
|
||||
|
||||
if encoding:
|
||||
content = raw.decode(encoding)
|
||||
else:
|
||||
@@ -745,11 +1035,14 @@ def fetch_url(url: str, timeout=15, user_agent: str = None,
|
||||
# requests 库会自动处理 Content-Encoding，但 stdlib 不会。
|
||||
# 此前该 bug 被沙箱伪响应掩盖（两者都产生 U+FFFD），
|
||||
# 实际在无 requests 的真实环境中会复现。
|
||||
# v2.2.1 补充：br 解压（与 requests 路径对齐）。
|
||||
content_encoding = (resp.headers.get("Content-Encoding", "")
|
||||
.lower().strip())
|
||||
if content_encoding and raw:
|
||||
try:
|
||||
if "gzip" in content_encoding:
|
||||
if "br" in content_encoding and _HAS_BROTLI:
|
||||
raw = _brotli.decompress(raw)
|
||||
elif "gzip" in content_encoding:
|
||||
raw = gzip.decompress(raw)
|
||||
elif "deflate" in content_encoding:
|
||||
# deflate 可能是 zlib 包装或裸 deflate
|
||||
@@ -761,6 +1054,19 @@ def fetch_url(url: str, timeout=15, user_agent: str = None,
|
||||
logger.debug(f" decompress failed ({content_encoding}): {e}")
|
||||
# 解压失败保留原 raw，让下游 decode 兜底
|
||||
|
||||
# 二进制文档解析（PDF/DOCX/XLSX）及不支持的媒体类型检测。
|
||||
# 命中时直接返回，跳过后续文本 decode 流程。
|
||||
doc_text, doc_err_code, doc_err_msg = _parse_document_content(
|
||||
raw, content_type)
|
||||
if doc_err_code is not None:
|
||||
return FetchResult(
|
||||
"", content_type, final_url, truncated, ua,
|
||||
doc_err_code, doc_err_msg)
|
||||
if doc_text is not None:
|
||||
return FetchResult(
|
||||
doc_text, content_type, final_url, truncated, ua,
|
||||
None, None)
|
||||
|
||||
if encoding:
|
||||
charset = encoding
|
||||
else:
|
||||
@@ -776,16 +1082,12 @@ def fetch_url(url: str, timeout=15, user_agent: str = None,
|
||||
last_error = e
|
||||
# 429/503：读取 Retry-After（stdlib 路径）
|
||||
if e.code in (429, 503) and attempt < max_retries:
|
||||
retry_after_raw = e.headers.get("Retry-After", "") if e.headers else ""
|
||||
retry_after_sec = parse_retry_after(retry_after_raw)
|
||||
# HTTPError 本身是可读的响应对象（fp 已被 urllib 消费），
|
||||
# 无需显式 close；直接进入退避。
|
||||
delay = max(retry_after_sec, compute_backoff_delay(attempt))
|
||||
logger.info(f" Fetch retry {attempt+1}/{max_retries} "
|
||||
f"(HTTP {e.code}, Retry-After={retry_after_sec:.1f}s) "
|
||||
f"in {delay:.1f}s")
|
||||
time.sleep(delay)
|
||||
continue
|
||||
should_retry, _ = _handle_rate_limit_status(
|
||||
e.code, e.headers, attempt, max_retries)
|
||||
if should_retry:
|
||||
continue
|
||||
if is_retryable_error(e) and attempt < max_retries:
|
||||
delay = compute_backoff_delay(attempt)
|
||||
logger.info(f" Fetch retry {attempt+1}/{max_retries} ({e}) in {delay:.1f}s")
|
||||
@@ -809,18 +1111,13 @@ def fetch_url(url: str, timeout=15, user_agent: str = None,
|
||||
# 429/503 with Retry-After
|
||||
if status in (429, 503) and attempt < max_retries:
|
||||
resp_obj = getattr(e, "response", None)
|
||||
retry_after_raw = ""
|
||||
if resp_obj is not None:
|
||||
retry_after_raw = resp_obj.headers.get("Retry-After", "")
|
||||
retry_after_sec = parse_retry_after(retry_after_raw)
|
||||
resp_headers = resp_obj.headers if resp_obj is not None else None
|
||||
if resp_obj is not None:
|
||||
resp_obj.close()
|
||||
delay = max(retry_after_sec, compute_backoff_delay(attempt))
|
||||
logger.info(f" Fetch retry {attempt+1}/{max_retries} "
|
||||
f"(HTTP {status}, Retry-After={retry_after_sec:.1f}s) "
|
||||
f"in {delay:.1f}s")
|
||||
time.sleep(delay)
|
||||
continue
|
||||
should_retry, _ = _handle_rate_limit_status(
|
||||
status, resp_headers, attempt, max_retries)
|
||||
if should_retry:
|
||||
continue
|
||||
if (status is None or status in RETRYABLE_STATUS) and attempt < max_retries:
|
||||
delay = compute_backoff_delay(attempt)
|
||||
logger.info(f" Fetch retry {attempt+1}/{max_retries} ({e}) in {delay:.1f}s")
|
||||
|
||||
Reference in New Issue
Block a user