feat(v2.1.0): 研究模式 + fetch.py Wayback 兜底 + 被墙站点智能回退
A. fetch.py 补齐 Wayback 兜底 (修复重大 gap) - v2.0.1 gap: fetch.py 独立调用 403 时无 Wayback 兜底 (仅 search.py --fetch 有) - AI Agent 用 fetch.py -u URL 直接抓取被墙站点时, 403 后无任何回退 - 修复: fetch.py main() 增加 Wayback 兜底逻辑 + --no-fallback flag - 共享逻辑抽取到 common.py: should_try_wayback() + build_wayback_url() B. --research 研究模式 - 给定主题自动扩展 5 个多角度查询: overview/profile/background/works/review - 确定性规则 (不依赖 AI 判断), 跨进程可复现 - 输出含 research_topic + research_queries 元数据, AI Agent 可按角度结构化汇编 - 与 --query/--queries-file 互斥, 支持所有输出格式 (json/brief/urls/csv) - 三态退出码: 0=有结果, 2=全部空, 1=全部错误 C. 被墙站点智能回退 - common.py 增加 HARD_BLOCKED_DOMAINS: 百度百科/知乎/微博/微信公众号/豆瓣等 - is_hard_blocked_domain() 精确匹配 + 子域匹配 - 命中被墙站点时: 主抓取失败后立即 Wayback (不等 should_try_wayback 判断) - search.py _should_try_fallback 增加 url 参数, 被墙站点直接触发兜底 真实测试验证 (search.metona.cn 实例): - fetch.py 百度百科兜底: 403 → Wayback 恢复 150,493 chars ✓ - --research 模式: 5 角度查询扩展 + research 元数据 + 三态退出码 ✓ - 被墙站点检测: Hard-blocked domain detected 日志 + 自动 Wayback ✓ 测试: 503 个全部通过 (新增 45 个: test_wayback_shared + test_research_mode) 来源: 另一个 AI Agent 反馈 Wikipedia/百度百科/知乎 fetch 失败, 需要多角度搜索+失败回退+被墙站点列表
This commit is contained in:
+184
-13
@@ -34,13 +34,16 @@ from common import (
|
||||
RECOVERY_HINTS,
|
||||
apply_proxy,
|
||||
build_auth_headers,
|
||||
build_wayback_url,
|
||||
classify_error,
|
||||
emit_progress,
|
||||
force_utf8_stdout,
|
||||
is_hard_blocked_domain,
|
||||
resolve_auth_basic,
|
||||
resolve_auth_bearer,
|
||||
set_progress_enabled,
|
||||
setup_logging,
|
||||
should_try_wayback,
|
||||
E_CONFIG,
|
||||
E_AUTH,
|
||||
E_NETWORK,
|
||||
@@ -881,7 +884,7 @@ def fetch_page(url: str, timeout: int = 10, auth_headers: dict = None,
|
||||
if fallback_enabled:
|
||||
if anti_bot_detected:
|
||||
need_fallback = True
|
||||
elif result is None and _should_try_fallback(result, error_msg):
|
||||
elif result is None and _should_try_fallback(result, error_msg, url):
|
||||
need_fallback = True
|
||||
|
||||
if need_fallback:
|
||||
@@ -943,12 +946,15 @@ def fetch_page(url: str, timeout: int = 10, auth_headers: dict = None,
|
||||
}
|
||||
|
||||
|
||||
def _should_try_fallback(result, error_msg: str) -> bool:
|
||||
def _should_try_fallback(result, error_msg: str, url: str = None) -> bool:
|
||||
"""判断是否应触发 Wayback 兜底。
|
||||
|
||||
v2.1.0 改为调用 common.should_try_wayback 共享逻辑 + 被墙站点检测。
|
||||
|
||||
触发条件：
|
||||
1. 主抓取抛异常且错误信息暗示 404/403/超时
|
||||
1. 主抓取抛异常且错误信息暗示 404/403/超时（common.should_try_wayback）
|
||||
2. 主抓取成功但被反爬拦截（理论上 fetch_page 已处理，此处防御性）
|
||||
3. v2.1.0: URL 属于被墙/强反爬站点（is_hard_blocked_domain）
|
||||
|
||||
不触发：
|
||||
* 用户禁用兜底（调用方控制，不进入此函数）
|
||||
@@ -957,15 +963,10 @@ def _should_try_fallback(result, error_msg: str) -> bool:
|
||||
if result is not None:
|
||||
# 主抓取成功，无需兜底
|
||||
return False
|
||||
if not error_msg:
|
||||
return False
|
||||
msg = error_msg.lower()
|
||||
# 404/403/超时/连接重置 → 尝试 Wayback
|
||||
triggers = ["404", "403", "timeout", "timed out", "connection reset",
|
||||
"connection refused", "max retries exceeded"]
|
||||
if any(t in msg for t in triggers):
|
||||
# v2.1.0: 被墙站点直接触发兜底（不等错误信息判断）
|
||||
if url and is_hard_blocked_domain(url):
|
||||
return True
|
||||
return False
|
||||
return should_try_wayback(error_msg)
|
||||
|
||||
|
||||
def _try_wayback_fallback(url: str, timeout: int = 10,
|
||||
@@ -974,12 +975,14 @@ def _try_wayback_fallback(url: str, timeout: int = 10,
|
||||
max_size: int = None):
|
||||
"""尝试从 Wayback Machine 获取页面快照。
|
||||
|
||||
v2.1.0 改为使用 common.build_wayback_url 共享逻辑。
|
||||
|
||||
使用 ``https://web.archive.org/web/2/<url>`` 端点，``2`` 表示
|
||||
"最新可用快照"。Wayback 会 302 重定向到具体时间戳快照。
|
||||
|
||||
返回 FetchResult 或 None（失败时）。独立超时（10s），不阻塞主流程。
|
||||
"""
|
||||
wayback_url = f"https://web.archive.org/web/2/{url}"
|
||||
wayback_url = build_wayback_url(url)
|
||||
wb_timeout = min(timeout, 10) # Wayback 自身可能慢，限制最大 10s
|
||||
try:
|
||||
logger.info(f" [FALLBACK] Trying Wayback Machine for {url[:55]}")
|
||||
@@ -1814,6 +1817,46 @@ def _read_queries_file(path: str) -> list:
|
||||
return queries
|
||||
|
||||
|
||||
# ----- Research mode (v2.1.0) -----
|
||||
# 给定一个主题，自动扩展多角度查询词，复用批量搜索逻辑。
|
||||
# 扩展策略是确定性规则（不做 AI 判断），覆盖人物/主题/事件的通用研究维度。
|
||||
|
||||
# 研究角度定义：(角度标识, 后缀词)
|
||||
# 顺序代表搜索优先级——基本信息优先，评价争议最后。
|
||||
_RESEARCH_ANGLES = [
|
||||
("overview", ""), # 主题本身：最直接的搜索
|
||||
("profile", "简介"), # 基本信息：百科式介绍
|
||||
("background", "经历"), # 背景经历：生平/历史
|
||||
("works", "作品"), # 作品成就：产出物
|
||||
("review", "评价"), # 评价争议：外界看法
|
||||
]
|
||||
|
||||
|
||||
def expand_research_queries(topic: str) -> list:
|
||||
"""将研究主题扩展为多角度查询词列表。
|
||||
|
||||
v2.1.0 研究模式核心函数。给定一个主题（如"七森莉莉"或"Python asyncio"），
|
||||
自动生成 5 个角度的查询词，覆盖：
|
||||
1. overview — 主题本身
|
||||
2. profile — 基本信息（简介）
|
||||
3. background — 背景经历
|
||||
4. works — 作品成就
|
||||
5. review — 评价争议
|
||||
|
||||
返回 [(angle, query), ...] 列表，angle 用于结果标注。
|
||||
|
||||
确定性规则，不依赖 AI 判断——确保跨进程可复现，AI Agent 可预期。
|
||||
"""
|
||||
topic = topic.strip()
|
||||
if not topic:
|
||||
return []
|
||||
queries = []
|
||||
for angle, suffix in _RESEARCH_ANGLES:
|
||||
query = f"{topic} {suffix}".strip()
|
||||
queries.append((angle, query))
|
||||
return queries
|
||||
|
||||
|
||||
# ----- Main -----
|
||||
|
||||
def main():
|
||||
@@ -1986,6 +2029,12 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
|
||||
"starting with '#' are skipped) and run them in sequence. "
|
||||
"Results are emitted as a JSON array (or one brief block per query). "
|
||||
"Overrides --query when set.")
|
||||
parser.add_argument("--research", default=None, metavar="TOPIC",
|
||||
help="v2.1.0 Research mode: given a topic, auto-expand into 5 "
|
||||
"multi-angle queries (overview/profile/background/works/review) "
|
||||
"and run them in sequence. Results are merged and deduplicated. "
|
||||
"Output includes research_topic and research_queries metadata. "
|
||||
"Mutually exclusive with --query and --queries-file.")
|
||||
parser.add_argument("--verify", action="store_true",
|
||||
help="Health-check mode: verify instances (reachability/JSON/latency) and exit without searching")
|
||||
parser.add_argument("--cache-ttl", type=int, default=_cfg_int(config, "cache_ttl", 0),
|
||||
@@ -2021,17 +2070,37 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
|
||||
"emits a JSON array, not JSON Lines. Drop --stream for "
|
||||
"batch output, or use a single --query with --stream.",
|
||||
args, error_code=E_INPUT)
|
||||
if args.research:
|
||||
_emit_error("--stream cannot be used with --research: research mode "
|
||||
"emits a JSON array, not JSON Lines. Drop --stream for "
|
||||
"research output, or use a single --query with --stream.",
|
||||
args, error_code=E_INPUT)
|
||||
if args.format != "json":
|
||||
_emit_error(f"--stream requires --format json (current: {args.format}). "
|
||||
"JSON Lines streaming only produces valid output with json format.",
|
||||
args, error_code=E_INPUT)
|
||||
|
||||
# --research 与 --query / --queries-file 互斥
|
||||
if args.research:
|
||||
if args.query:
|
||||
_emit_error("--research cannot be used with --query: research mode "
|
||||
"auto-generates queries from the topic. Drop --query, "
|
||||
"or use --research <topic> alone.",
|
||||
args, error_code=E_INPUT)
|
||||
if args.queries_file:
|
||||
_emit_error("--research cannot be used with --queries-file: research "
|
||||
"mode auto-generates queries. Drop --queries-file, "
|
||||
"or use --research <topic> alone.",
|
||||
args, error_code=E_INPUT)
|
||||
|
||||
# --query is required unless we're doing a non-search operation.
|
||||
# --queries-file is an alternative to --query for batch mode.
|
||||
# --research is another alternative (v2.1.0).
|
||||
if (not args.verify and not args.query and not args.queries_file
|
||||
and not args.research
|
||||
and not args.clear_cache and not args.cache_stats):
|
||||
parser.error("--query is required (or use --verify / --queries-file / "
|
||||
"--clear-cache / --cache-stats)")
|
||||
"--research / --clear-cache / --cache-stats)")
|
||||
|
||||
# Apply proxy early so every HTTP path (search, verify, fetch) honors it.
|
||||
# Setting env vars is enough: urllib reads them via getproxies() and
|
||||
@@ -2138,6 +2207,108 @@ Use --config FILE to load a non-default config file (overrides the auto-discover
|
||||
# Cache TTL in seconds (CLI takes minutes for ergonomics)
|
||||
ttl_seconds = args.cache_ttl * 60 if args.cache_ttl > 0 else 0
|
||||
|
||||
# ----- Research mode: --research (v2.1.0) -----
|
||||
# 给定主题，自动扩展为 5 个多角度查询，串行搜索，输出带研究元数据的 JSON。
|
||||
# 复用批量搜索逻辑，但查询词来自 expand_research_queries 而非文件。
|
||||
if args.research:
|
||||
topic = args.research.strip()
|
||||
research_queries = expand_research_queries(topic)
|
||||
if not research_queries:
|
||||
_emit_error(f"research topic is empty after stripping", args,
|
||||
error_code=E_INPUT)
|
||||
|
||||
logger.info(f"Research mode: {len(research_queries)} angles for '{topic}'")
|
||||
batch = []
|
||||
any_with_results = False
|
||||
error_count = 0
|
||||
for i, (angle, q) in enumerate(research_queries, 1):
|
||||
logger.info(f"\n[{i}/{len(research_queries)}] [{angle}] {q}")
|
||||
results, err, err_code = _run_single_query(q, args, instance_urls,
|
||||
auth_headers, ttl_seconds)
|
||||
if err:
|
||||
error_count += 1
|
||||
logger.error(f" [ERROR] {err}")
|
||||
entry = {"query": q, "angle": angle, "status": "error",
|
||||
"error": err}
|
||||
if err_code:
|
||||
entry["error_code"] = err_code
|
||||
batch.append(entry)
|
||||
else:
|
||||
if len(results.get("results", [])) > 0:
|
||||
any_with_results = True
|
||||
batch.append({"query": q, "angle": angle, "status": "ok",
|
||||
"results": results})
|
||||
|
||||
# 输出
|
||||
if args.format == "json":
|
||||
output = json.dumps({
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"research_topic": topic,
|
||||
"research_queries": [
|
||||
{"angle": a, "query": q} for a, q in research_queries
|
||||
],
|
||||
"queries": batch,
|
||||
}, indent=2, ensure_ascii=False)
|
||||
elif args.format == "csv":
|
||||
import csv as csv_mod
|
||||
import io
|
||||
out = io.StringIO()
|
||||
writer = csv_mod.writer(out, lineterminator="\n")
|
||||
writer.writerow(["angle", "query", "title", "url", "engine",
|
||||
"score", "published_date", "content"])
|
||||
for br in batch:
|
||||
q = br["query"]
|
||||
angle = br.get("angle", "")
|
||||
if "results" in br:
|
||||
for r in br["results"].get("results", []):
|
||||
writer.writerow([
|
||||
angle, q,
|
||||
r.get("title", ""),
|
||||
r.get("url", ""),
|
||||
r.get("engine", ""),
|
||||
r.get("score", "") if r.get("score") is not None else "",
|
||||
r.get("published_date", ""),
|
||||
r.get("content", ""),
|
||||
])
|
||||
else:
|
||||
writer.writerow([angle, q, "", "", "", "", "",
|
||||
f"[ERROR: {br['error']}]"])
|
||||
output = out.getvalue().rstrip()
|
||||
elif args.format == "urls":
|
||||
parts = []
|
||||
for br in batch:
|
||||
parts.append(f"# [{br.get('angle', '?')}] {br['query']}")
|
||||
if "results" in br:
|
||||
parts.append(format_urls(br["results"]))
|
||||
else:
|
||||
parts.append(f"# [ERROR: {br['error']}]")
|
||||
output = "\n".join(parts)
|
||||
else: # brief
|
||||
parts = []
|
||||
for br in batch:
|
||||
parts.append("=" * 60)
|
||||
parts.append(f"[{br.get('angle', '?')}] {br['query']}")
|
||||
parts.append("=" * 60)
|
||||
if "results" in br:
|
||||
parts.append(format_brief(br["results"]))
|
||||
else:
|
||||
parts.append(f"[ERROR: {br['error']}]")
|
||||
parts.append("")
|
||||
output = "\n".join(parts)
|
||||
|
||||
if args.output:
|
||||
with open(args.output, "w", encoding="utf-8") as f:
|
||||
f.write(output)
|
||||
logger.info(f"Saved results to {args.output}")
|
||||
else:
|
||||
print(output)
|
||||
# 三态退出码：0=有结果, 2=全部空, 1=全部错误
|
||||
if error_count == len(research_queries):
|
||||
sys.exit(1)
|
||||
elif not any_with_results:
|
||||
sys.exit(2)
|
||||
sys.exit(0)
|
||||
|
||||
# ----- Batch mode: --queries-file -----
|
||||
# Reads one query per line (blank/# lines skipped) and runs them in
|
||||
# sequence. Output is a JSON array (json format) or concatenated blocks
|
||||
|
||||
Reference in New Issue
Block a user