diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f7b124d --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Metona Team + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 65676eb..3682bfe 100644 --- a/README.md +++ b/README.md @@ -112,9 +112,9 @@ pip install requests beautifulsoup4 ### 配置实例 -AI Agent 无需每次传入实例 URL，支持三种配置方式（优先级从高到低）： +AI Agent 无需每次传入实例 URL 和认证信息，支持通过配置文件一次性设置： -1. **CLI 参数**：`-i https://your-instance` +1. **CLI 参数**：`-i https://your-instance`（最高优先级） 2. **环境变量**：`export SEARXNG_INSTANCE="https://your-instance"` 3. **配置文件**：`./searxng.toml` 或 `~/.config/searxng-cli/searxng.toml` @@ -122,8 +122,15 @@ AI Agent 无需每次传入实例 URL，支持三种配置方式（优先级从 # searxng.toml [searxng] instance = "https://your-searxng.example.com" +# 私有实例认证（可选，二选一）： +auth_basic = "user:password" # Basic 认证 +# auth_bearer = "sk-token-123" # Bearer Token（与 auth_basic 同时设置时 bearer 优先） ``` +**认证优先级**（从高到低）：`--auth-*` CLI 参数 > `--auth-*-file` 文件 > `searxng.toml` 配置 > 环境变量 + +> **安全提醒**：配置文件中的明文密码有泄露风险。生产环境推荐使用 `--auth-basic-file ~/.searxng_auth` 或环境变量。 + ## 参数参考 ### search.py — 网络搜索 @@ -234,7 +241,8 @@ python scripts/fetch.py -u https://example.com \ **网络** - 代理支持（`--proxy`） -- 认证（Bearer/Basic，支持文件和环境变量，避免 shell 历史泄露） +- 认证（Bearer/Basic，支持 CLI/文件/配置文件/环境变量四级优先级） +- `searxng.toml` 配置 `auth_basic` / `auth_bearer` 字段，AI Agent 一次配置即可 - 凭证文件权限警告（POSIX） **工程** @@ -243,7 +251,7 @@ python scripts/fetch.py -u https://example.com \ - 结构化错误码（`E_NETWORK` / `E_AUTH` / `E_RATE_LIMIT` 等） - JSON Lines 流式输出（`--stream`） - 进度事件（`--progress`，JSON Lines 到 stderr） -- 309 个单元+集成测试 +- 330 个单元+集成测试 ## 跨 Agent 兼容性 @@ -271,7 +279,7 @@ pip install pytest pytest -q ``` -309 个测试覆盖：缓存操作、认证解析、域名过滤、Markdown 转换、搜索逻辑、集成流程、日志配置、HTML 回退、自动抓取、健康检查、输出格式化、实例解析、并行搜索、CLI 端到端、错误码分类、流式输出、进度事件。 +330 个测试覆盖：缓存操作、认证解析、域名过滤、Markdown 转换、搜索逻辑、集成流程、日志配置、HTML 回退、自动抓取、健康检查、输出格式化、实例解析、并行搜索、CLI 端到端、错误码分类、流式输出、进度事件、配置文件认证。 ## 项目结构 diff --git a/SKILL.md b/SKILL.md index 1aa6cc5..9906799 100644 --- a/SKILL.md +++ b/SKILL.md @@ -44,7 +44,8 @@ SearXNG is a privacy-respecting metasearch engine that aggregates results from 7 **Network & auth** - Proxy support (`--proxy`) for both search and fetch (sets `HTTP_PROXY`/`HTTPS_PROXY`/`NO_PROXY`) -- Auth via CLI flag, file, or env var (`--auth-bearer` / `--auth-basic` + `*-file` variants) to avoid leaking secrets in shell history +- Auth via CLI flag, file, config file, or env var (`--auth-bearer` / `--auth-basic` + `*-file` variants) to avoid leaking secrets in shell history +- Config-file auth — `auth_basic` and `auth_bearer` fields in `searxng.toml` let AI agents set credentials once (priority: CLI > file > config > env) - Credentials-file permission warning — `--auth-*-file` warns on stderr if the file is group/other-readable (POSIX only) **Engineering** @@ -101,10 +102,14 @@ python scripts/search.py -q "python asyncio tutorial" # -i not needed # [searxng] # instance = "https://my-searxng.example.com" # # or: instances = ["https://a.example.com", "https://b.example.com"] +# # Auth (optional, for private instances): +# auth_basic = "user:password" # Basic auth +# auth_bearer = "sk-token-123" # Bearer token (auth_bearer wins if both set) # # Any flag below can also be pre-set here (engines, categories, language, # # safesearch, time_range, method, format, timeout, max_retries, proxy, # # cache_ttl, fetch, fetch_timeout, fetch_retries, max_size). # # Explicit CLI flags always override config values. +# # Auth priority: --auth-* > --auth-*-file > searxng.toml > env var # Plain list also works in ./instances.txt (one URL per line, # for comments) # 7. Cache results for 30 minutes (identical queries skip the network) @@ -442,8 +447,8 @@ usage: search.py [-h] [--query QUERY] [--instance URL] 10. **Result caching:** `--cache-ttl 30` stores results for 30 min; identical queries within the TTL skip the network entirely. Cache lives at `$SEARXNG_CACHE_DIR` or `~/.cache/searxng-cli/cache.db` (SQLite, WAL mode). `--clear-cache` / `--cache-stats` manage it without searching 11. **Batch mode:** `--queries-file FILE` reads one query per line (blank/`#` lines skipped) and runs them in sequence; output is a JSON array (or one block per query in brief/urls). A failed query is recorded but does not abort the batch 12. **Dedup + sort + domain filter:** After search (and cache), duplicate URLs are collapsed (default; `--no-dedup` disables), results are sorted (`--sort-by`; default: score descending), and then `--include-domain`/`--exclude-domain` filter by domain. Matching is case-insensitive and ignores a leading `www.`; when a domain is in both lists, exclude wins -13. **Proxy & auth:** `--proxy URL` routes both search and fetch through a proxy; `--auth-bearer` / `--auth-basic` (plus `*-file` variants and `SEARXNG_BEARER_TOKEN` / `SEARXNG_BASIC_AUTH` env vars) supply credentials without leaking them via shell history -14. **Config defaults:** `searxng.toml` may pre-set most flags (engines, categories, language, safesearch, time_range, method, format, sort_by, timeout, max_retries, proxy, cache_ttl, fetch, fetch_timeout, fetch_retries, max_size); explicit CLI flags always win +13. **Proxy & auth:** `--proxy URL` routes both search and fetch through a proxy; `--auth-bearer` / `--auth-basic` (plus `*-file` variants, `searxng.toml` `auth_basic`/`auth_bearer` fields, and `SEARXNG_BEARER_TOKEN` / `SEARXNG_BASIC_AUTH` env vars) supply credentials. Priority: CLI flag > file > config file > env var +14. **Config defaults:** `searxng.toml` may pre-set most flags (engines, categories, language, safesearch, time_range, method, format, sort_by, timeout, max_retries, proxy, cache_ttl, fetch, fetch_timeout, fetch_retries, max_size, auth_basic, auth_bearer); explicit CLI flags always win 15. **Structured errors:** in `--format json` mode, failures print a JSON object `{"error": "...", "exit_code": N, "query": "..."}` to stdout so agents can parse them **Key options:** diff --git a/scripts/common.py b/scripts/common.py index d790082..e4ff6e8 100644 --- a/scripts/common.py +++ b/scripts/common.py @@ -114,7 +114,8 @@ def _warn_file_perms(path: str) -> None: def resolve_auth_basic(cli_value: str = None, file_path: str = None, - env_var: str = "SEARXNG_BASIC_AUTH") -> str: + env_var: str = "SEARXNG_BASIC_AUTH", + config_value: str = None) -> str: """Resolve basic-auth credentials without leaking them via shell history. Priority (highest wins): @@ -122,7 +123,9 @@ def resolve_auth_basic(cli_value: str = None, file_path: str = None, but leaks into shell history; discouraged) 2. ``file_path`` — ``--auth-basic-file FILE``; first non-empty line is read as ``user:pass``. Recommended for shells. - 3. ``env_var`` — ``SEARXNG_BASIC_AUTH`` environment variable. + 3. ``config_value`` — ``auth_basic`` field from ``searxng.toml``. + Convenient for AI agents that read config once. + 4. ``env_var`` — ``SEARXNG_BASIC_AUTH`` environment variable. Returns ``"user:pass"`` or ``None`` if no source provides credentials. Raises ``RuntimeError`` if a file is specified but cannot be read. @@ -143,16 +146,26 @@ def resolve_auth_basic(cli_value: str = None, file_path: str = None, except OSError as e: raise RuntimeError(f"cannot read auth file '{file_path}': {e}") from e + if config_value: + return config_value + import os return os.environ.get(env_var) def resolve_auth_bearer(cli_value: str = None, file_path: str = None, - env_var: str = "SEARXNG_BEARER_TOKEN") -> str: - """Resolve a Bearer token from CLI flag, file, or environment variable. + env_var: str = "SEARXNG_BEARER_TOKEN", + config_value: str = None) -> str: + """Resolve a Bearer token from CLI flag, file, config, or environment. Mirrors :func:`resolve_auth_basic` for token-style auth. Useful for long-lived API tokens that should not appear in shell history. + + Priority (highest wins): + 1. ``cli_value`` — explicit ``--auth-bearer "token"`` + 2. ``file_path`` — ``--auth-bearer-file FILE`` + 3. ``config_value`` — ``auth_bearer`` field from ``searxng.toml`` + 4. ``env_var`` — ``SEARXNG_BEARER_TOKEN`` environment variable """ if cli_value: return cli_value @@ -170,6 +183,9 @@ def resolve_auth_bearer(cli_value: str = None, file_path: str = None, except OSError as e: raise RuntimeError(f"cannot read token file '{file_path}': {e}") from e + if config_value: + return config_value + import os return os.environ.get(env_var) diff --git a/scripts/search.py b/scripts/search.py index f64c2a8..9dcf1d5 100644 --- a/scripts/search.py +++ b/scripts/search.py @@ -1429,17 +1429,32 @@ Use --config FILE to load a non-default config file (overrides the auto-discover # Resolve instance(s): -i > SEARXNG_INSTANCE env > config file instance_urls = resolve_instances(args.instance) + # Fallback: if --config was used, instance may be in the config dict + # but not in the default search paths that resolve_instances checks. + if not instance_urls and config.get("instance"): + instance_urls = parse_instances(config["instance"]) + elif not instance_urls and config.get("instances"): + raw = config["instances"] + if isinstance(raw, str): + instance_urls = parse_instances(raw) + elif isinstance(raw, list): + instance_urls = [u if u.startswith(("http://", "https://")) + else "https://" + u for u in raw if u and str(u).strip()] if not instance_urls: _emit_error("no SearXNG instance resolved. Provide -i/--instance, set the " "SEARXNG_INSTANCE environment variable, or create a searxng.toml / " "instances.txt config file.", args, error_code=E_CONFIG) # Build auth headers if provided (needed by both verify and search). - # Credentials may come from CLI flag, file, or env var (in priority order) - # to avoid leaking secrets via shell history or `ps`. + # Credentials may come from CLI flag, file, config file, or env var + # (in priority order) to avoid leaking secrets via shell history or `ps`. try: - bearer_token = resolve_auth_bearer(args.auth_bearer, args.auth_bearer_file) - basic_auth = resolve_auth_basic(args.auth_basic, args.auth_basic_file) + bearer_token = resolve_auth_bearer( + args.auth_bearer, args.auth_bearer_file, + config_value=config.get("auth_bearer")) + basic_auth = resolve_auth_basic( + args.auth_basic, args.auth_basic_file, + config_value=config.get("auth_basic")) except RuntimeError as e: _emit_error(str(e), args, error_code=E_AUTH) auth_headers = build_auth_headers( diff --git a/tests/test_config_auth.py b/tests/test_config_auth.py new file mode 100644 index 0000000..67f9ebe --- /dev/null +++ b/tests/test_config_auth.py @@ -0,0 +1,257 @@ +"""Tests for auth credentials resolved from searxng.toml config file. + +Covers: resolve_auth_basic/resolve_auth_bearer with config_value parameter, +priority chain (CLI > file > config > env), and main() integration that +reads auth_basic/auth_bearer from load_config(). +""" +import os +from unittest.mock import patch +from pathlib import Path + +import pytest + +from common import resolve_auth_basic, resolve_auth_bearer +from search import load_config + + +# ----- resolve_auth_basic: config_value parameter ----- + +def test_basic_config_value_used_when_no_cli_or_file(): + """config_value is returned when CLI and file are not provided.""" + assert resolve_auth_basic(config_value="user:pass") == "user:pass" + + +def test_basic_cli_overrides_config(): + """CLI value takes precedence over config_value.""" + assert resolve_auth_basic(cli_value="cli:pass", + config_value="cfg:pass") == "cli:pass" + + +def test_basic_file_overrides_config(tmp_path): + """File takes precedence over config_value.""" + auth_file = tmp_path / "auth.txt" + auth_file.write_text("file:pass\n") + assert resolve_auth_basic(file_path=str(auth_file), + config_value="cfg:pass") == "file:pass" + + +def test_config_overrides_env(monkeypatch): + """config_value takes precedence over environment variable.""" + monkeypatch.setenv("SEARXNG_BASIC_AUTH", "env:pass") + assert resolve_auth_basic(config_value="cfg:pass") == "cfg:pass" + + +def test_env_used_when_config_is_none(monkeypatch): + """Environment variable is the fallback when config_value is None.""" + monkeypatch.setenv("SEARXNG_BASIC_AUTH", "env:pass") + assert resolve_auth_basic(config_value=None) == "env:pass" + + +def test_config_none_and_no_env_returns_none(monkeypatch): + """All sources absent → returns None.""" + monkeypatch.delenv("SEARXNG_BASIC_AUTH", raising=False) + assert resolve_auth_basic(config_value=None) is None + + +def test_config_empty_string_falls_through_to_env(monkeypatch): + """Empty string config_value is treated as absent (falsy).""" + monkeypatch.setenv("SEARXNG_BASIC_AUTH", "env:pass") + assert resolve_auth_basic(config_value="") == "env:pass" + + +# ----- resolve_auth_bearer: config_value parameter ----- + +def test_bearer_config_value_used_when_no_cli_or_file(): + """config_value is returned when CLI and file are not provided.""" + assert resolve_auth_bearer(config_value="cfg-token-123") == "cfg-token-123" + + +def test_bearer_cli_overrides_config(): + """CLI value takes precedence over config_value.""" + assert resolve_auth_bearer(cli_value="cli-token", + config_value="cfg-token") == "cli-token" + + +def test_bearer_file_overrides_config(tmp_path): + """File takes precedence over config_value.""" + token_file = tmp_path / "token.txt" + token_file.write_text("file-token\n") + assert resolve_auth_bearer(file_path=str(token_file), + config_value="cfg-token") == "file-token" + + +def test_bearer_config_overrides_env(monkeypatch): + """config_value takes precedence over environment variable.""" + monkeypatch.setenv("SEARXNG_BEARER_TOKEN", "env-token") + assert resolve_auth_bearer(config_value="cfg-token") == "cfg-token" + + +def test_bearer_env_used_when_config_is_none(monkeypatch): + """Environment variable is the fallback when config_value is None.""" + monkeypatch.setenv("SEARXNG_BEARER_TOKEN", "env-token") + assert resolve_auth_bearer(config_value=None) == "env-token" + + +def test_bearer_config_none_and_no_env_returns_none(monkeypatch): + """All sources absent → returns None.""" + monkeypatch.delenv("SEARXNG_BEARER_TOKEN", raising=False) + assert resolve_auth_bearer(config_value=None) is None + + +# ----- Full priority chain verification ----- + +def test_basic_full_priority_chain(tmp_path, monkeypatch): + """Verify full priority: CLI > file > config > env.""" + monkeypatch.setenv("SEARXNG_BASIC_AUTH", "env:pass") + auth_file = tmp_path / "auth.txt" + auth_file.write_text("file:pass\n") + + # CLI wins over all + assert resolve_auth_basic(cli_value="cli:pass", + file_path=str(auth_file), + config_value="cfg:pass") == "cli:pass" + # File wins over config and env + assert resolve_auth_basic(file_path=str(auth_file), + config_value="cfg:pass") == "file:pass" + # Config wins over env + assert resolve_auth_basic(config_value="cfg:pass") == "cfg:pass" + # Env is fallback + assert resolve_auth_basic() == "env:pass" + + +def test_bearer_full_priority_chain(tmp_path, monkeypatch): + """Verify full priority: CLI > file > config > env.""" + monkeypatch.setenv("SEARXNG_BEARER_TOKEN", "env-token") + token_file = tmp_path / "token.txt" + token_file.write_text("file-token\n") + + assert resolve_auth_bearer(cli_value="cli-token", + file_path=str(token_file), + config_value="cfg-token") == "cli-token" + assert resolve_auth_bearer(file_path=str(token_file), + config_value="cfg-token") == "file-token" + assert resolve_auth_bearer(config_value="cfg-token") == "cfg-token" + assert resolve_auth_bearer() == "env-token" + + +# ----- load_config: auth fields in toml ----- + +def test_load_config_reads_auth_basic(tmp_path): + """load_config() returns auth_basic from searxng.toml.""" + cfg = tmp_path / "searxng.toml" + cfg.write_text(""" +[searxng] +instance = "https://example.com" +auth_basic = "user:pass" +""") + config = load_config(str(cfg)) + assert config.get("auth_basic") == "user:pass" + + +def test_load_config_reads_auth_bearer(tmp_path): + """load_config() returns auth_bearer from searxng.toml.""" + cfg = tmp_path / "searxng.toml" + cfg.write_text(""" +[searxng] +instance = "https://example.com" +auth_bearer = "sk-token-123" +""") + config = load_config(str(cfg)) + assert config.get("auth_bearer") == "sk-token-123" + + +def test_load_config_reads_both_auth_fields(tmp_path): + """Both auth_basic and auth_bearer can be set in the same config.""" + cfg = tmp_path / "searxng.toml" + cfg.write_text(""" +[searxng] +instance = "https://example.com" +auth_basic = "user:pass" +auth_bearer = "sk-token" +""") + config = load_config(str(cfg)) + assert config.get("auth_basic") == "user:pass" + assert config.get("auth_bearer") == "sk-token" + + +def test_load_config_no_auth_fields(tmp_path): + """Config without auth fields returns None for both.""" + cfg = tmp_path / "searxng.toml" + cfg.write_text(""" +[searxng] +instance = "https://example.com" +""") + config = load_config(str(cfg)) + assert config.get("auth_basic") is None + assert config.get("auth_bearer") is None + + +# ----- main() integration: auth from config file ----- + +def test_main_uses_auth_from_config(tmp_path, monkeypatch, capsys): + """main() reads auth_basic from searxng.toml and uses it for search.""" + import search as search_mod + import sys as _sys + + cfg = tmp_path / "searxng.toml" + cfg.write_text(f""" +[searxng] +instance = "https://config-auth.example.com" +auth_basic = "cfguser:cfgpass" +""") + + captured_auth = {} + + def _fake_search_multi(urls, params, **kwargs): + captured_auth["headers"] = kwargs.get("auth_headers", {}) + return {"results": [{"title": "t", "url": "https://x.com"}]} + + monkeypatch.setattr(_sys, "argv", [ + "search.py", "-q", "test", "--config", str(cfg), "--format", "json", + ]) + monkeypatch.setattr(search_mod, "search_multi", _fake_search_multi) + # Ensure env vars don't interfere + monkeypatch.delenv("SEARXNG_BASIC_AUTH", raising=False) + monkeypatch.delenv("SEARXNG_BEARER_TOKEN", raising=False) + + # main() calls sys.exit(2) on empty results, sys.exit(0) on stream mode; + # on normal success it returns without exit. We just want it to not raise. + search_mod.main() + + # Verify auth headers were built from config value + assert "Authorization" in captured_auth["headers"] + assert "Basic" in captured_auth["headers"]["Authorization"] + + +def test_main_cli_auth_overrides_config(tmp_path, monkeypatch, capsys): + """CLI --auth-basic overrides auth_basic in searxng.toml.""" + import search as search_mod + import sys as _sys + + cfg = tmp_path / "searxng.toml" + cfg.write_text(""" +[searxng] +instance = "https://config-auth.example.com" +auth_basic = "cfguser:cfgpass" +""") + + captured_auth = {} + + def _fake_search_multi(urls, params, **kwargs): + captured_auth["headers"] = kwargs.get("auth_headers", {}) + return {"results": [{"title": "t", "url": "https://x.com"}]} + + monkeypatch.setattr(_sys, "argv", [ + "search.py", "-q", "test", "--config", str(cfg), + "--auth-basic", "cliuser:clipass", "--format", "json", + ]) + monkeypatch.setattr(search_mod, "search_multi", _fake_search_multi) + monkeypatch.delenv("SEARXNG_BASIC_AUTH", raising=False) + + search_mod.main() + + # CLI value should be used, not config value + import base64 + auth_header = captured_auth["headers"]["Authorization"] + decoded = base64.b64decode(auth_header.split(" ")[1]).decode() + assert decoded == "cliuser:clipass"