]*class="[^"]*t[^"]*"[^>]*>[\s\S]*?
]*href="([^"]*)"[^>]*>([\s\S]*?)`)
+ reS := regexp.MustCompile(`]*>([\s\S]*?)
`)
+
+ var out []Result
+ for _, m := range reB.FindAllStringSubmatch(html, -1) {
+ if len(out) >= n {
+ break
+ }
+ tm := reT.FindStringSubmatch(m[1])
+ if tm == nil {
+ continue
+ }
+ link := tm[1]
+ if !strings.HasPrefix(link, "http") {
+ if dm := regexp.MustCompile(`data-url="(https?://[^"]*)"`).FindStringSubmatch(m[1]); dm != nil {
+ link = dm[1]
+ }
+ }
+ title := clean(tm[2])
+ if title == "" {
+ continue
+ }
+ snip := ""
+ if sm := reS.FindStringSubmatch(m[1]); sm != nil {
+ snip = clean(sm[1])
+ }
+ out = append(out, Result{Title: title, URL: link, Snippet: snip, Engine: "baidu"})
+ }
+ return out
+}
+
+// ── 工具 ──
+
func clean(s string) string {
s = regexp.MustCompile(`<[^>]+>`).ReplaceAllString(s, "")
s = decodeEntities(s)
@@ -169,44 +273,89 @@ func clean(s string) string {
}
func decodeEntities(s string) string {
- repl := strings.NewReplacer(
+ r := strings.NewReplacer(
" ", " ", "<", "<", ">", ">", "&", "&",
""", `"`, "'", "'", "'", "'",
"…", "…", "—", "—", "–", "–",
)
- return repl.Replace(s)
+ return r.Replace(s)
+}
+
+func parallelSearch(q string, n int) []Result {
+ type engResult struct {
+ results []Result
+ }
+ ch := make(chan engResult, 4)
+ engines := []func(string, int) []Result{searchGoogle, searchBing, searchBaidu, searchDDG}
+ for _, fn := range engines {
+ go func(f func(string, int) []Result) {
+ r := f(q, n)
+ if r == nil {
+ r = []Result{}
+ }
+ ch <- engResult{r}
+ }(fn)
+ }
+ seen := map[string]bool{}
+ var out []Result
+ for range engines {
+ for _, r := range (<-ch).results {
+ key := strings.TrimRight(r.URL, "/")
+ if !seen[key] {
+ seen[key] = true
+ out = append(out, r)
+ }
+ }
+ }
+ if len(out) > n {
+ out = out[:n]
+ }
+ return out
}
// ── Handler ──
+
func handleSearch(w http.ResponseWriter, r *http.Request) {
+ if !limiter.Allow() {
+ writeJSON(w, http.StatusTooManyRequests, SearchResp{Error: "rate limited"})
+ return
+ }
+
q := strings.TrimSpace(r.URL.Query().Get("q"))
if q == "" {
- writeJSON(w, http.StatusBadRequest, SearchResponse{Success: false, Error: "missing q"})
+ writeJSON(w, http.StatusBadRequest, SearchResp{Error: "missing q"})
return
}
engine := r.URL.Query().Get("engine")
if engine == "" {
- engine = "google"
+ engine = "all"
}
n := 10
fmt.Sscanf(r.URL.Query().Get("n"), "%d", &n)
+ if n < 1 {
+ n = 10
+ }
if n > 20 {
n = 20
}
+ // 缓存
ck := fmt.Sprintf("%s|%s|%d", q, engine, n)
if v, ok := cache.Load(ck); ok {
ce := v.(cacheEntry)
if time.Now().Unix() < ce.ExpiresAt {
- writeJSON(w, http.StatusOK, SearchResponse{Success: true, Query: q, Total: len(ce.Results), Results: ce.Results})
+ writeJSON(w, http.StatusOK, SearchResp{
+ Success: true, Query: q, Engine: engine,
+ Total: len(ce.Results), Results: ce.Results, FromCache: true,
+ })
return
}
cache.Delete(ck)
}
- var results []SearchResult
+ var results []Result
switch engine {
case "google":
results = searchGoogle(q, n)
@@ -214,41 +363,32 @@ func handleSearch(w http.ResponseWriter, r *http.Request) {
results = searchBing(q, n)
case "ddg":
results = searchDDG(q, n)
+ case "baidu":
+ results = searchBaidu(q, n)
case "all":
results = parallelSearch(q, n)
default:
- writeJSON(w, http.StatusBadRequest, SearchResponse{Success: false, Error: "unknown engine: " + engine})
+ writeJSON(w, http.StatusBadRequest, SearchResp{Error: "unknown engine: " + engine + " (use google/bing/ddg/baidu/all)"})
return
}
cache.Store(ck, cacheEntry{Results: results, ExpiresAt: time.Now().Add(5 * time.Minute).Unix()})
- writeJSON(w, http.StatusOK, SearchResponse{Success: true, Query: q, Total: len(results), Results: results})
+ writeJSON(w, http.StatusOK, SearchResp{
+ Success: true, Query: q, Engine: engine,
+ Total: len(results), Results: results,
+ })
}
-func parallelSearch(q string, n int) []SearchResult {
- ch := make(chan []SearchResult, 3)
- go func() { ch <- searchGoogle(q, n) }()
- go func() { ch <- searchBing(q, n) }()
- go func() { ch <- searchDDG(q, n) }()
- seen := map[string]bool{}
- var results []SearchResult
- for i := 0; i < 3; i++ {
- for _, r := range <-ch {
- key := strings.TrimRight(r.URL, "/")
- if !seen[key] {
- seen[key] = true
- results = append(results, r)
- }
- }
- }
- if len(results) > n {
- results = results[:n]
- }
- return results
+func handleHealth(w http.ResponseWriter, r *http.Request) {
+ writeJSON(w, http.StatusOK, map[string]interface{}{
+ "status": "ok",
+ "engines": []string{"google", "bing", "ddg", "baidu", "all"},
+ "time": time.Now().UTC().Format(time.RFC3339),
+ })
}
func writeJSON(w http.ResponseWriter, status int, v interface{}) {
- w.Header().Set("Content-Type", "application/json")
+ w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.Header().Set("Access-Control-Allow-Origin", "*")
w.WriteHeader(status)
json.NewEncoder(w).Encode(v)
@@ -257,15 +397,35 @@ func writeJSON(w http.ResponseWriter, status int, v interface{}) {
func main() {
port := os.Getenv("PORT")
if port == "" {
- port = "8080"
+ port = "18080"
}
mux := http.NewServeMux()
- mux.HandleFunc("/search", handleSearch)
- mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
- writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
- })
+ mux.HandleFunc("/search", recoverWrap(handleSearch))
+ mux.HandleFunc("/health", recoverWrap(handleHealth))
- log.Printf("search-proxy :%s (google/bing/ddg/all)", port)
- log.Fatal(http.ListenAndServe(":"+port, mux))
+ srv := &http.Server{
+ Addr: ":" + port,
+ Handler: mux,
+ ReadTimeout: 10 * time.Second,
+ WriteTimeout: 15 * time.Second,
+ IdleTimeout: 60 * time.Second,
+ }
+
+ // 优雅关闭
+ go func() {
+ sigCh := make(chan os.Signal, 1)
+ signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
+ <-sigCh
+ log.Println("shutting down...")
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ srv.Shutdown(ctx)
+ }()
+
+ log.Printf("search-proxy :%s engines: google/bing/ddg/baidu/all cache:5min rate:%d/s", port, 30)
+ if err := srv.ListenAndServe(); err != http.ErrServerClosed {
+ log.Fatal(err)
+ }
+ log.Println("stopped")
}
diff --git a/search-proxy/search-proxy.service b/search-proxy/search-proxy.service
index 32df306..2f3e953 100644
--- a/search-proxy/search-proxy.service
+++ b/search-proxy/search-proxy.service
@@ -1,5 +1,5 @@
[Unit]
-Description=Search Proxy - Google/Bing/DDG API
+Description=Search Proxy - Google/Bing/DDG/Baidu API
After=network.target
[Service]
@@ -7,7 +7,7 @@ Type=simple
ExecStart=/opt/search-proxy/search-proxy
Restart=always
RestartSec=5
-Environment=PORT=8080
+Environment=PORT=18080
[Install]
WantedBy=multi-user.target