feat(v2.1.0): 研究模式 + fetch.py Wayback 兜底 + 被墙站点智能回退

A. fetch.py 补齐 Wayback 兜底 (修复重大 gap) - v2.0.1 gap: fetch.py 独立调用 403 时无 Wayback 兜底 (仅 search.py --fetch 有) - AI Agent 用 fetch.py -u URL 直接抓取被墙站点时, 403 后无任何回退 - 修复: fetch.py main() 增加 Wayback 兜底逻辑 + --no-fallback flag - 共享逻辑抽取到 common.py: should_try_wayback() + build_wayback_url() B. --research 研究模式 - 给定主题自动扩展 5 个多角度查询: overview/profile/background/works/review - 确定性规则 (不依赖 AI 判断), 跨进程可复现 - 输出含 research_topic + research_queries 元数据, AI Agent 可按角度结构化汇编 - 与 --query/--queries-file 互斥, 支持所有输出格式 (json/brief/urls/csv) - 三态退出码: 0=有结果, 2=全部空, 1=全部错误 C. 被墙站点智能回退 - common.py 增加 HARD_BLOCKED_DOMAINS: 百度百科/知乎/微博/微信公众号/豆瓣等 - is_hard_blocked_domain() 精确匹配 + 子域匹配 - 命中被墙站点时: 主抓取失败后立即 Wayback (不等 should_try_wayback 判断) - search.py _should_try_fallback 增加 url 参数, 被墙站点直接触发兜底真实测试验证 (search.metona.cn 实例): - fetch.py 百度百科兜底: 403 → Wayback 恢复 150,493 chars ✓ - --research 模式: 5 角度查询扩展 + research 元数据 + 三态退出码 ✓ - 被墙站点检测: Hard-blocked domain detected 日志 + 自动 Wayback ✓ 测试: 503 个全部通过 (新增 45 个: test_wayback_shared + test_research_mode) 来源: 另一个 AI Agent 反馈 Wikipedia/百度百科/知乎 fetch 失败, 需要多角度搜索+失败回退+被墙站点列表
2026-08-02 08:27:45 +08:00
parent b62095570d
commit e94cbe0783
8 changed files with 629 additions and 20 deletions
@@ -0,0 +1,146 @@
+"""Tests for v2.1.0 --research mode.
+
+Covers:
+  * expand_research_queries — topic → [(angle, query), ...] expansion
+  * --research CLI flag     — mutex with --query/--queries-file, --stream
+  * --research end-to-end   — subprocess with real CLI (stubbed at search_multi)
+"""
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts"))
+
+from search import expand_research_queries, _RESEARCH_ANGLES
+
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
+
+
+# ===== expand_research_queries =====
+
+class TestExpandResearchQueries:
+    def test_basic_topic(self):
+        queries = expand_research_queries("Python")
+        assert len(queries) == 5
+        # First angle is overview (no suffix)
+        assert queries[0] == ("overview", "Python")
+        # Each query starts with the topic
+        for angle, q in queries:
+            assert q.startswith("Python")
+
+    def test_chinese_topic(self):
+        queries = expand_research_queries("七森莉莉")
+        assert len(queries) == 5
+        assert queries[0] == ("overview", "七森莉莉")
+        assert queries[1] == ("profile", "七森莉莉 简介")
+        assert queries[2] == ("background", "七森莉莉 经历")
+        assert queries[3] == ("works", "七森莉莉 作品")
+        assert queries[4] == ("review", "七森莉莉 评价")
+
+    def test_angles_match_definition(self):
+        queries = expand_research_queries("test")
+        angles = [a for a, _ in queries]
+        assert angles == [a for a, _ in _RESEARCH_ANGLES]
+
+    def test_empty_topic_returns_empty(self):
+        assert expand_research_queries("") == []
+
+    def test_whitespace_topic_returns_empty(self):
+        assert expand_research_queries("   ") == []
+
+    def test_topic_is_stripped(self):
+        queries = expand_research_queries("  Python  ")
+        assert queries[0] == ("overview", "Python")
+
+    def test_deterministic(self):
+        """Same topic always produces same queries (no randomness)."""
+        q1 = expand_research_queries("Rust async")
+        q2 = expand_research_queries("Rust async")
+        assert q1 == q2
+
+    def test_multi_word_topic(self):
+        queries = expand_research_queries("Python asyncio tutorial")
+        assert queries[0] == ("overview", "Python asyncio tutorial")
+        assert "简介" in queries[1][1]
+
+
+# ===== --research CLI mutex checks =====
+
+def _run_cli(*args, env=None):
+    """Run scripts/search.py as a real subprocess."""
+    full_env = {**os.environ, **(env or {})}
+    cmd = [sys.executable, str(PROJECT_ROOT / "scripts" / "search.py")] + list(args)
+    return subprocess.run(
+        cmd, cwd=str(PROJECT_ROOT),
+        capture_output=True, text=True, encoding="utf-8", timeout=30,
+        env=full_env,
+    )
+
+
+class TestResearchCLIMutex:
+    def test_research_with_query_errors(self):
+        """--research + --query → E_INPUT error."""
+        r = _run_cli("--research", "topic", "-q", "query", "-i", "https://x.com")
+        assert r.returncode != 0
+        # Error goes to stdout in json mode, stderr in other modes
+        combined = r.stdout + r.stderr
+        assert "--research cannot be used with --query" in combined
+
+    def test_research_with_queries_file_errors(self, tmp_path):
+        """--research + --queries-file → E_INPUT error."""
+        qf = tmp_path / "queries.txt"
+        qf.write_text("test\n", encoding="utf-8")
+        r = _run_cli("--research", "topic", "--queries-file", str(qf),
+                     "-i", "https://x.com")
+        assert r.returncode != 0
+        combined = r.stdout + r.stderr
+        assert "--research cannot be used with --queries-file" in combined
+
+    def test_research_with_stream_errors(self):
+        """--research + --stream → E_INPUT error."""
+        r = _run_cli("--research", "topic", "--stream", "-i", "https://x.com")
+        assert r.returncode != 0
+        combined = r.stdout + r.stderr
+        assert "--stream cannot be used with --research" in combined
+
+    def test_research_alone_without_instance_errors(self):
+        """--research without -i or config → E_CONFIG (instance required).
+
+        Uses SEARXNG_INSTANCE= (empty) + a non-existent HOME to prevent
+        config file auto-discovery from finding the project's searxng.toml.
+        """
+        r = _run_cli("--research", "topic",
+                     env={"SEARXNG_INSTANCE": "",
+                          "USERPROFILE": "/nonexistent",
+                          "HOME": "/nonexistent"})
+        assert r.returncode != 0
+
+
+# ===== --research end-to-end (stubbed) =====
+
+class TestResearchExpandOutput:
+    """Verify that expand_research_queries produces the expected output shape
+    that the CLI relies on."""
+
+    def test_output_is_list_of_tuples(self):
+        queries = expand_research_queries("test")
+        assert isinstance(queries, list)
+        for item in queries:
+            assert isinstance(item, tuple)
+            assert len(item) == 2
+            assert isinstance(item[0], str)  # angle
+            assert isinstance(item[1], str)  # query
+
+    def test_all_angles_present(self):
+        queries = expand_research_queries("test")
+        angles = {a for a, _ in queries}
+        assert angles == {"overview", "profile", "background", "works", "review"}
+
+    def test_overview_has_no_suffix(self):
+        """The overview angle should be just the topic itself."""
+        queries = expand_research_queries("my topic")
+        assert queries[0][1] == "my topic"