A. fetch.py 补齐 Wayback 兜底 (修复重大 gap) - v2.0.1 gap: fetch.py 独立调用 403 时无 Wayback 兜底 (仅 search.py --fetch 有) - AI Agent 用 fetch.py -u URL 直接抓取被墙站点时, 403 后无任何回退 - 修复: fetch.py main() 增加 Wayback 兜底逻辑 + --no-fallback flag - 共享逻辑抽取到 common.py: should_try_wayback() + build_wayback_url() B. --research 研究模式 - 给定主题自动扩展 5 个多角度查询: overview/profile/background/works/review - 确定性规则 (不依赖 AI 判断), 跨进程可复现 - 输出含 research_topic + research_queries 元数据, AI Agent 可按角度结构化汇编 - 与 --query/--queries-file 互斥, 支持所有输出格式 (json/brief/urls/csv) - 三态退出码: 0=有结果, 2=全部空, 1=全部错误 C. 被墙站点智能回退 - common.py 增加 HARD_BLOCKED_DOMAINS: 百度百科/知乎/微博/微信公众号/豆瓣等 - is_hard_blocked_domain() 精确匹配 + 子域匹配 - 命中被墙站点时: 主抓取失败后立即 Wayback (不等 should_try_wayback 判断) - search.py _should_try_fallback 增加 url 参数, 被墙站点直接触发兜底 真实测试验证 (search.metona.cn 实例): - fetch.py 百度百科兜底: 403 → Wayback 恢复 150,493 chars ✓ - --research 模式: 5 角度查询扩展 + research 元数据 + 三态退出码 ✓ - 被墙站点检测: Hard-blocked domain detected 日志 + 自动 Wayback ✓ 测试: 503 个全部通过 (新增 45 个: test_wayback_shared + test_research_mode) 来源: 另一个 AI Agent 反馈 Wikipedia/百度百科/知乎 fetch 失败, 需要多角度搜索+失败回退+被墙站点列表
147 lines
5.4 KiB
Python
147 lines
5.4 KiB
Python
"""Tests for v2.1.0 --research mode.
|
|
|
|
Covers:
|
|
* expand_research_queries — topic → [(angle, query), ...] expansion
|
|
* --research CLI flag — mutex with --query/--queries-file, --stream
|
|
* --research end-to-end — subprocess with real CLI (stubbed at search_multi)
|
|
"""
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
|
|
|
|
from search import expand_research_queries, _RESEARCH_ANGLES
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
|
|
|
|
|
# ===== expand_research_queries =====
|
|
|
|
class TestExpandResearchQueries:
|
|
def test_basic_topic(self):
|
|
queries = expand_research_queries("Python")
|
|
assert len(queries) == 5
|
|
# First angle is overview (no suffix)
|
|
assert queries[0] == ("overview", "Python")
|
|
# Each query starts with the topic
|
|
for angle, q in queries:
|
|
assert q.startswith("Python")
|
|
|
|
def test_chinese_topic(self):
|
|
queries = expand_research_queries("七森莉莉")
|
|
assert len(queries) == 5
|
|
assert queries[0] == ("overview", "七森莉莉")
|
|
assert queries[1] == ("profile", "七森莉莉 简介")
|
|
assert queries[2] == ("background", "七森莉莉 经历")
|
|
assert queries[3] == ("works", "七森莉莉 作品")
|
|
assert queries[4] == ("review", "七森莉莉 评价")
|
|
|
|
def test_angles_match_definition(self):
|
|
queries = expand_research_queries("test")
|
|
angles = [a for a, _ in queries]
|
|
assert angles == [a for a, _ in _RESEARCH_ANGLES]
|
|
|
|
def test_empty_topic_returns_empty(self):
|
|
assert expand_research_queries("") == []
|
|
|
|
def test_whitespace_topic_returns_empty(self):
|
|
assert expand_research_queries(" ") == []
|
|
|
|
def test_topic_is_stripped(self):
|
|
queries = expand_research_queries(" Python ")
|
|
assert queries[0] == ("overview", "Python")
|
|
|
|
def test_deterministic(self):
|
|
"""Same topic always produces same queries (no randomness)."""
|
|
q1 = expand_research_queries("Rust async")
|
|
q2 = expand_research_queries("Rust async")
|
|
assert q1 == q2
|
|
|
|
def test_multi_word_topic(self):
|
|
queries = expand_research_queries("Python asyncio tutorial")
|
|
assert queries[0] == ("overview", "Python asyncio tutorial")
|
|
assert "简介" in queries[1][1]
|
|
|
|
|
|
# ===== --research CLI mutex checks =====
|
|
|
|
def _run_cli(*args, env=None):
|
|
"""Run scripts/search.py as a real subprocess."""
|
|
full_env = {**os.environ, **(env or {})}
|
|
cmd = [sys.executable, str(PROJECT_ROOT / "scripts" / "search.py")] + list(args)
|
|
return subprocess.run(
|
|
cmd, cwd=str(PROJECT_ROOT),
|
|
capture_output=True, text=True, encoding="utf-8", timeout=30,
|
|
env=full_env,
|
|
)
|
|
|
|
|
|
class TestResearchCLIMutex:
|
|
def test_research_with_query_errors(self):
|
|
"""--research + --query → E_INPUT error."""
|
|
r = _run_cli("--research", "topic", "-q", "query", "-i", "https://x.com")
|
|
assert r.returncode != 0
|
|
# Error goes to stdout in json mode, stderr in other modes
|
|
combined = r.stdout + r.stderr
|
|
assert "--research cannot be used with --query" in combined
|
|
|
|
def test_research_with_queries_file_errors(self, tmp_path):
|
|
"""--research + --queries-file → E_INPUT error."""
|
|
qf = tmp_path / "queries.txt"
|
|
qf.write_text("test\n", encoding="utf-8")
|
|
r = _run_cli("--research", "topic", "--queries-file", str(qf),
|
|
"-i", "https://x.com")
|
|
assert r.returncode != 0
|
|
combined = r.stdout + r.stderr
|
|
assert "--research cannot be used with --queries-file" in combined
|
|
|
|
def test_research_with_stream_errors(self):
|
|
"""--research + --stream → E_INPUT error."""
|
|
r = _run_cli("--research", "topic", "--stream", "-i", "https://x.com")
|
|
assert r.returncode != 0
|
|
combined = r.stdout + r.stderr
|
|
assert "--stream cannot be used with --research" in combined
|
|
|
|
def test_research_alone_without_instance_errors(self):
|
|
"""--research without -i or config → E_CONFIG (instance required).
|
|
|
|
Uses SEARXNG_INSTANCE= (empty) + a non-existent HOME to prevent
|
|
config file auto-discovery from finding the project's searxng.toml.
|
|
"""
|
|
r = _run_cli("--research", "topic",
|
|
env={"SEARXNG_INSTANCE": "",
|
|
"USERPROFILE": "/nonexistent",
|
|
"HOME": "/nonexistent"})
|
|
assert r.returncode != 0
|
|
|
|
|
|
# ===== --research end-to-end (stubbed) =====
|
|
|
|
class TestResearchExpandOutput:
|
|
"""Verify that expand_research_queries produces the expected output shape
|
|
that the CLI relies on."""
|
|
|
|
def test_output_is_list_of_tuples(self):
|
|
queries = expand_research_queries("test")
|
|
assert isinstance(queries, list)
|
|
for item in queries:
|
|
assert isinstance(item, tuple)
|
|
assert len(item) == 2
|
|
assert isinstance(item[0], str) # angle
|
|
assert isinstance(item[1], str) # query
|
|
|
|
def test_all_angles_present(self):
|
|
queries = expand_research_queries("test")
|
|
angles = {a for a, _ in queries}
|
|
assert angles == {"overview", "profile", "background", "works", "review"}
|
|
|
|
def test_overview_has_no_suffix(self):
|
|
"""The overview angle should be just the topic itself."""
|
|
queries = expand_research_queries("my topic")
|
|
assert queries[0][1] == "my topic"
|