"""Tests for v2.1.0 shared Wayback fallback + hard-blocked domain logic. Covers common.py functions: * should_try_wayback — error string → bool (should we try Wayback?) * build_wayback_url — URL → Wayback Machine URL * is_hard_blocked_domain — URL → bool (is this a known anti-bot site?) """ import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) from common import ( should_try_wayback, build_wayback_url, is_hard_blocked_domain, HARD_BLOCKED_DOMAINS, ) # ===== should_try_wayback ===== class TestShouldTryWayback: def test_403_triggers(self): assert should_try_wayback("403 Client Error: Forbidden") is True def test_404_triggers(self): assert should_try_wayback("404 Client Error: Not Found") is True def test_timeout_triggers(self): assert should_try_wayback("Connection timeout") is True def test_timed_out_triggers(self): assert should_try_wayback("Read timed out") is True def test_connection_reset_triggers(self): assert should_try_wayback("ConnectionResetError: connection reset") is True def test_connection_aborted_triggers(self): assert should_try_wayback("ConnectionAbortedError") is True def test_max_retries_triggers(self): assert should_try_wayback("Max retries exceeded with url") is True def test_dns_failure_does_not_trigger(self): """DNS failures should not trigger Wayback — Wayback can't resolve either.""" assert should_try_wayback("Name or service not known") is False def test_empty_does_not_trigger(self): assert should_try_wayback("") is False def test_none_does_not_trigger(self): assert should_try_wayback(None) is False def test_generic_error_does_not_trigger(self): assert should_try_wayback("Some random error") is False # ===== build_wayback_url ===== class TestBuildWaybackUrl: def test_basic_url(self): url = "https://example.com/page" result = build_wayback_url(url) assert result == "https://web.archive.org/web/2/https://example.com/page" def test_http_url(self): url = "http://example.com" result = build_wayback_url(url) assert result == "https://web.archive.org/web/2/http://example.com" def test_url_with_query_params(self): url = "https://example.com/search?q=test&lang=en" result = build_wayback_url(url) assert "web.archive.org/web/2/" in result assert url in result # ===== is_hard_blocked_domain ===== class TestIsHardBlockedDomain: def test_baike_baidu_com(self): assert is_hard_blocked_domain("https://baike.baidu.com/item/Python") is True def test_zhihu_com(self): assert is_hard_blocked_domain("https://zhuanlan.zhihu.com/p/123") is True def test_zhihu_com_root(self): assert is_hard_blocked_domain("https://www.zhihu.com/question/123") is True def test_weibo_com(self): assert is_hard_blocked_domain("https://weibo.com/123456") is True def test_m_weibo_cn(self): assert is_hard_blocked_domain("https://m.weibo.cn/detail/123") is True def test_mp_weixin_qq_com(self): assert is_hard_blocked_domain("https://mp.weixin.qq.com/s/abc") is True def test_douban_com(self): assert is_hard_blocked_domain("https://book.douban.com/subject/123") is True def test_zhidao_baidu_com(self): assert is_hard_blocked_domain("https://zhidao.baidu.com/question/123") is True def test_normal_site_not_blocked(self): assert is_hard_blocked_domain("https://example.com") is False def test_wikipedia_not_blocked(self): assert is_hard_blocked_domain("https://zh.wikipedia.org/wiki/Python") is False def test_github_not_blocked(self): assert is_hard_blocked_domain("https://github.com/python/cpython") is False def test_baidu_search_not_blocked(self): """www.baidu.com search page is in the hard-blocked list. v2.1.0: moved from _SUBDOMAIN_BLOCKED (baidu.com) to HARD_BLOCKED_DOMAINS (www.baidu.com) to avoid blocking pan.baidu.com / cloud.baidu.com.""" assert is_hard_blocked_domain("https://www.baidu.com/s?wd=test") is True def test_baidu_pan_not_blocked(self): """pan.baidu.com (百度网盘) should NOT be blocked after v2.1.0 fix. Previously matched by _SUBDOMAIN_BLOCKED 'baidu.com' — now only www/baike/zhidao/tieba/wenku are in the precise list.""" assert is_hard_blocked_domain("https://pan.baidu.com/s/abc123") is False def test_baidu_cloud_not_blocked(self): """cloud.baidu.com (百度智能云) should NOT be blocked after v2.1.0 fix.""" assert is_hard_blocked_domain("https://cloud.baidu.com/product/abc") is False def test_empty_url(self): assert is_hard_blocked_domain("") is False def test_none_url(self): assert is_hard_blocked_domain(None) is False def test_no_scheme(self): """URLs without scheme: urlparse won't extract hostname. This is expected — callers should provide full URLs.""" # With scheme: works assert is_hard_blocked_domain("https://baike.baidu.com/item/test") is True # Without scheme: urlparse returns no hostname — function returns False # This is acceptable: all real callers (fetch.py, search.py) pass full URLs assert is_hard_blocked_domain("baike.baidu.com/item/test") is False def test_case_insensitive(self): assert is_hard_blocked_domain("https://BAIKE.BAIDU.COM/item/test") is True assert is_hard_blocked_domain("https://ZHIHU.COM/question/123") is True