searxng-mcp-go/internal/searxng/searcher.go
xlion-bot-renovate[bot] 6e4c553a9e
chore(deps): update dependency go to v1.27.1 (#588)
* chore(deps): update dependency go to v1.27.1

* Fixed all lints; just verify passes.

Co-authored-by: xlionjuan <[email protected]>

* Fixed `.golangci.yml` paths→path; `just verify` passes.

Co-authored-by: xlionjuan <[email protected]>

* Parallelized golden & error-path tests; verify passes.

Co-authored-by: xlionjuan <[email protected]>

* Verify passes; HTTPClient/Logger nil set; exhaustruct covers Config.

Co-authored-by: xlionjuan <[email protected]>

---------

Co-authored-by: xlion-bot-renovate[bot] <218815870+xlion-bot-renovate[bot]@users.noreply.github.com>
Co-authored-by: opencode-agent[bot] <opencode-agent[bot]@users.noreply.github.com>
Co-authored-by: xlionjuan <[email protected]>
2026-09-06 23:14:09 +08:00

706 lines
20 KiB
Go

package searxng
import (
"context"
"errors"
"fmt"
"log/slog"
"net/http"
"net/url"
"sort"
"strconv"
"strings"
"sync"
"time"
)
var (
errSearcherConfigRequired = errors.New("newSearXNGSearcher: config cannot be nil")
errSearcherURLParseInternal = errors.New(
"newSearXNGSearcher: url.Parse failed after validateBaseURL passed (internal error)",
)
errRequestCreateFailed = errors.New("failed to create request")
errSearchRequestFailed = errors.New("failed to execute search request")
errErrorBodyTooLarge = errors.New("error response body exceeded maximum size limit")
errSearchEmptyResults = errors.New("search returned empty results after all retries")
errGETFallbackUsed = errors.New(
"GET fallback was used; search query parameters may have been sent in the request URL",
)
errNilFinishResponse = errors.New("finishResponse: nil http.Response")
)
const getFallbackLogRisk = "Search query parameters may be sent in upstream URLs and recorded by " +
"SearXNG, proxy, CDN, or access logs"
// SearXNGSearcher performs web searches via a SearXNG instance.
//
//nolint:containedctx // lifecycle context canceled on Close(), propagated via AfterFunc
type SearXNGSearcher struct {
client *http.Client // Configurable HTTP client
searchEndpoint *url.URL // Precomputed /search endpoint URL; cloned per request
debug bool // When true, enables verbose HTTP request/response logging
logger *slog.Logger // Logger for warnings and debug output; nil = slog.Default()
searcherCtx context.Context
searcherCancel context.CancelFunc
closeOnce sync.Once
retryStrategy *exponentialBackoffStrategy
ownsTransport bool // true if the searcher created its own transport (safe to close)
allowGETFallback bool
// retryWaitHook is a test-only hook called immediately before retryWait.
// It allows tests to synchronize at the point where the retry loop has
// entered the backoff wait so Close-during-backoff can be verified directly.
retryWaitHook func()
}
// NewSearXNGSearcher creates a new SearXNGSearcher with the given configuration.
// Returns an error if cfg is nil, cfg.Validate fails, the base URL is empty
// or invalid, or endpoint construction fails internally.
func NewSearXNGSearcher(cfg *Config, debug bool) (*SearXNGSearcher, error) {
if cfg == nil {
return nil, errSearcherConfigRequired
}
// Validate config first to catch invalid negative values before normalization
err := cfg.Validate()
if err != nil {
return nil, fmt.Errorf("newSearXNGSearcher: %w", err)
}
// Normalize config: apply safe defaults for zero values
cfg = cfg.Normalize()
baseURL := cfg.SearXNGURL
err = validateBaseURL(baseURL)
if err != nil {
return nil, fmt.Errorf("newSearXNGSearcher: %w", err)
}
// Precompute the /search endpoint URL once so per-request construction
// only has to clone the result. computeSearchEndpoint may still fail
// after validateBaseURL (defensive); surface that as an internal error.
searchEndpoint, err := computeSearchEndpoint(baseURL)
if err != nil {
return nil, fmt.Errorf("%w: %w", errSearcherURLParseInternal, err)
}
logger := slog.Default()
if cfg.Logger != nil {
logger = cfg.Logger
}
if searchEndpoint.Scheme == schemeHTTP && !isPrivateHost(searchEndpoint.Host) {
logger.Warn("Using HTTP for non-private host. " +
"Search queries may be transmitted in clear text. " +
"Search results could be intercepted and modified by a MITM attacker")
}
if cfg.AllowGETFallback {
logger.Warn("GET fallback is enabled. " + getFallbackLogRisk)
}
client, ownsTransport := buildSearchHTTPClient(cfg)
searcher := &SearXNGSearcher{
client: client,
searchEndpoint: searchEndpoint,
debug: debug,
logger: logger,
retryStrategy: newExponentialBackoffStrategy(cfg.MaxRetries, cfg.RetryDelay, cfg.MaxRetryDelay),
ownsTransport: ownsTransport,
allowGETFallback: cfg.AllowGETFallback,
}
searcher.searcherCtx, searcher.searcherCancel = context.WithCancel(context.Background())
return searcher, nil
}
// buildSearchHTTPClient wraps the configured HTTP client with the search
// redirect policy, or constructs a fresh client with the configured timeout.
// It returns the client and whether the searcher owns it (and so must close
// its idle connections on Close).
func buildSearchHTTPClient(cfg *Config) (*http.Client, bool) {
client := cfg.HTTPClient
var ownsTransport bool
if client != nil {
originalCheckRedirect := client.CheckRedirect
wrapped := *client
wrapped.CheckRedirect = func(req *http.Request, via []*http.Request) error {
err := enforceSearchRedirectPolicy(req, via)
if err != nil {
return err
}
if originalCheckRedirect != nil {
return originalCheckRedirect(req, via)
}
return nil
}
client = &wrapped
} else {
if cfg.Timeout > 0 {
client = newHTTPClient(cfg.Timeout)
ownsTransport = true
} else {
client = getDefaultHTTPClient()
// ownsTransport remains false — we do not own the shared singleton
}
}
return client, ownsTransport
}
// Close releases resources held by the searcher and cancels in-flight searches.
// Close is safe to call on searchers that use a shared default client —
// it will skip closing idle connections on transports it does not own.
func (s *SearXNGSearcher) Close() error {
s.closeOnce.Do(func() {
if s.searcherCancel != nil {
s.searcherCancel()
}
})
if s.ownsTransport && s.client != nil && s.client.Transport != nil {
if transport, ok := s.client.Transport.(*http.Transport); ok {
transport.CloseIdleConnections()
}
}
return nil
}
// Search executes the search query against SearXNG with retry support.
// Returns SearXNGError wrapping the last error if all retries are exhausted.
// Returns ValidationError if args are invalid.
func (s *SearXNGSearcher) Search(ctx context.Context, args *SearchArgs) (*SearchResponse, error) {
normalized, err := ValidateSearchArgs(args)
if err != nil {
return nil, err
}
args = normalized
// Tie search lifecycle to searcher close
searchCtx, searchCancel := s.searchContext(ctx)
defer searchCancel()
return s.searchWithRetries(searchCtx, args)
}
// searchWithRetries runs the retry loop for a single search.
//
//nolint:gocognit // retry orchestration: attempt, classification, error tracking, and wait
func (s *SearXNGSearcher) searchWithRetries(ctx context.Context, args *SearchArgs) (*SearchResponse, error) {
var lastErr error
maxRetries := s.retryStrategy.MaxRetries()
for attempt := 0; attempt <= maxRetries; attempt++ {
resp, _, err := s.doSearchAttempt(ctx, args)
ar, classifyErr := s.classifyAttempt(ctx, attempt, maxRetries, resp, err, args)
if classifyErr != nil {
return nil, classifyErr
}
if ar.outcome == OutcomeSuccess {
return ar.result, nil
}
if ar.outcome == OutcomeEmptyExhausted {
return nil, wrapSearchError(errSearchEmptyResults)
}
trackErr := attemptError(err, ar)
shouldRetry, delay := s.retryStrategy.ShouldRetry(ctx, attempt, ar.outcome)
if !shouldRetry {
if trackErr == nil && ctx.Err() != nil {
trackErr = ctx.Err()
}
direct, termErr := s.resolveTerminalOutcome(resp, args, trackErr)
if direct {
return nil, termErr
}
if termErr != nil {
lastErr = termErr
}
break
}
// Track last error for final wrapping
lastErr = trackErr
s.logDebugRetry(attempt, maxRetries+1, delay, lastErr)
closeResponseBody(resp, s.getLogger())
waitErr := s.waitForRetry(ctx, delay)
if waitErr != nil {
lastErr = waitErr
break
}
}
return nil, wrapSearchError(lastErr)
}
// attemptError returns the error to track for a non-successful attempt: the
// attempt's own error when present, otherwise the classified attempt's error
// (e.g. a body-read failure), otherwise the empty-results sentinel for an
// OutcomeEmptyRetry attempt.
func attemptError(err error, ar *attemptResult) error {
if err != nil {
return err
}
if ar.err != nil {
return ar.err
}
if ar.outcome == OutcomeEmptyRetry {
return errSearchEmptyResults
}
return nil
}
// resolveTerminalOutcome handles a final (non-retried) attempt. It returns
// whether the error must short-circuit the retry loop (a direct
// finishResponse failure) and the error to record as the last error (which
// the retry loop wraps) otherwise.
func (s *SearXNGSearcher) resolveTerminalOutcome(
resp *http.Response,
args *SearchArgs,
trackErr error,
) (bool, error) {
if trackErr != nil {
closeResponseBody(resp, s.getLogger())
return false, trackErr
}
if resp != nil && resp.StatusCode != http.StatusOK {
_, finishErr := s.finishResponse(resp, args)
return true, finishErr
}
return false, nil
}
// waitForRetry invokes the test-only retryWaitHook (used to verify
// Close-during-backoff) and then sleeps for the retry backoff delay.
func (s *SearXNGSearcher) waitForRetry(ctx context.Context, delay time.Duration) error {
if s.retryWaitHook != nil {
s.retryWaitHook()
}
return retryWait(ctx, delay)
}
// searchContext creates a context tied to the searcher lifecycle.
// The returned context is canceled when the searcher is closed.
func (s *SearXNGSearcher) searchContext(ctx context.Context) (context.Context, context.CancelFunc) {
searchCtx, cancel := context.WithCancel(ctx)
// When the searcher is closed, cancel the search context.
// AfterFunc runs f in a goroutine only when s.searcherCtx is done,
// so this is a no-op (just bookkeeping) in the happy path.
// The nil guard covers test literals that bypass constructors;
// all production paths initialize searcherCtx at construction.
if s.searcherCtx != nil {
stop := context.AfterFunc(s.searcherCtx, cancel)
return searchCtx, func() {
stop()
cancel()
}
}
return searchCtx, cancel
}
// wrapSearchError wraps the last error for the Search return value.
// Preserves SearXNGError unwrapping to avoid hiding the real status code.
// Returns the fallback error for the "should never reach here" case when err is nil.
func wrapSearchError(err error) error {
if _, ok := errors.AsType[*SearXNGError](err); ok {
return fmt.Errorf("%w: %w", errSearchRequestFailed, err)
}
if err != nil {
return NewSearXNGError(0, "", "", fmt.Errorf("%w: %w", errSearchRequestFailed, err))
}
// Should never reach here
return NewSearXNGError(0, "", "", errSearchRequestFailed)
}
// attemptResult holds the result of classifying a single search attempt.
type attemptResult struct {
outcome Outcome
result *SearchResponse
err error // underlying error when outcome requires tracking (e.g. body-read failure)
}
// classifyAttempt processes the response from a single search attempt and
// determines its Outcome. Returns an error only if finishResponse itself
// fails (parse error, non-retryable HTTP status). For all other cases, it
// returns an attemptResult whose Outcome guides the retry decision.
func (s *SearXNGSearcher) classifyAttempt(
ctx context.Context,
attempt, maxRetries int,
resp *http.Response,
err error,
args *SearchArgs,
) (*attemptResult, error) {
if err != nil {
return &attemptResult{outcome: classifyOutcome(ctx, attempt, maxRetries, resp, err, false)}, nil
}
if resp.StatusCode != http.StatusOK {
return &attemptResult{outcome: classifyOutcome(ctx, attempt, maxRetries, resp, nil, false)}, nil
}
result, finishErr := s.finishResponse(resp, args)
// finishResponse closed resp.Body via defer; nil it so the retry
// loop does not close the same body a second time.
resp.Body = nil
if finishErr != nil {
// Body-read failures (unexpected EOF, connection reset during
// read) are transient network errors — retry them. Invalid
// JSON, HTML, unexpected content type, and oversized bodies
// remain non-retryable hard failures.
if errors.Is(finishErr, errResponseReadFailed) {
return &attemptResult{outcome: OutcomeRetry, err: finishErr}, nil
}
return nil, finishErr
}
isEmpty := s.isEmptyResponse(result)
outcome := classifyOutcome(ctx, attempt, maxRetries, resp, nil, isEmpty)
return &attemptResult{outcome: outcome, result: result}, nil
}
// finishResponse handles non-OK status, JSON parsing, and body closure for a response.
//
// resp is required: passing nil triggers errNilFinishResponse. The retry
// path inside Search never reaches here with a nil response, but guarding
// the field makes the function robust against future refactors and
// self-documents the precondition.
func (s *SearXNGSearcher) finishResponse(resp *http.Response, args *SearchArgs) (*SearchResponse, error) {
if resp == nil {
return nil, errNilFinishResponse
}
defer closeResponseBody(resp, s.getLogger())
if resp.StatusCode != http.StatusOK {
return nil, s.handleNonOKResponse(resp)
}
return s.parseSearchResponse(resp, args)
}
func (s *SearXNGSearcher) doSearchAttempt(ctx context.Context, args *SearchArgs) (*http.Response, string, error) {
postReq, postBodyStr, err := s.buildSearchRequest(ctx, args)
if err != nil {
return nil, postBodyStr, err
}
s.logDebugRequest(postReq, postBodyStr)
resp, err := s.doHTTP(postReq)
s.logDebugResponse(resp, err)
shouldFallbackToGET := err == nil && resp != nil && s.allowGETFallback &&
(resp.StatusCode == http.StatusMethodNotAllowed || resp.StatusCode == http.StatusNotImplemented)
if shouldFallbackToGET {
resp, err = s.executeGETfallback(ctx, resp, postReq, postBodyStr)
}
return resp, postBodyStr, err
}
// redactedHTTPError keeps the original HTTP error available for errors.Is and
// errors.As while providing a query-redacted representation for logs and users.
type redactedHTTPError struct {
err error
message string
}
func (e *redactedHTTPError) Error() string {
return e.message
}
func (e *redactedHTTPError) Unwrap() error {
return e.err
}
// doHTTP normalizes response-body ownership and the observable representation
// of every error returned by http.Client.Do.
func (s *SearXNGSearcher) doHTTP(req *http.Request) (*http.Response, error) {
//nolint:gosec // Callers use the validated SearXNG endpoint or its derived GET fallback URL.
resp, err := s.client.Do(req)
if err == nil {
return resp, nil
}
if resp != nil {
// http.Client.Do returns both values only when CheckRedirect rejects a
// redirect. The client has already closed resp.Body, so retaining resp
// would give its body a second owner.
resp = nil
}
return resp, redactHTTPError(err)
}
func redactHTTPError(err error) error {
if err == nil {
return nil
}
redactions := make(map[string]string)
collectHTTPErrorURLRedactions(err, redactions)
if len(redactions) == 0 {
return err
}
urls := make([]string, 0, len(redactions))
for rawURL := range redactions {
urls = append(urls, rawURL)
}
// Replace longer URLs first so a URL that prefixes another target cannot
// leave the longer target's remaining query parameters visible.
sort.Slice(urls, func(i, j int) bool {
if len(urls[i]) == len(urls[j]) {
return urls[i] < urls[j]
}
return len(urls[i]) > len(urls[j])
})
originalMessage := err.Error()
message := originalMessage
for _, rawURL := range urls {
redactedURL := redactions[rawURL]
// url.Error renders URL with %q. Replace that exact representation first
// to handle escaped characters, then replace unquoted copies emitted by
// wrappers that reuse the typed URL value in their own context.
message = strings.ReplaceAll(message, strconv.Quote(rawURL), strconv.Quote(redactedURL))
message = strings.ReplaceAll(message, rawURL, redactedURL)
}
if message == originalMessage {
return err
}
return &redactedHTTPError{err: err, message: message}
}
func collectHTTPErrorURLRedactions(err error, redactions map[string]string) {
if err == nil {
return
}
// Inspect this concrete node rather than using errors.As, which would jump
// to a descendant and prevent the walker from handling every URL node.
urlErr, ok := err.(*url.Error) //nolint:errorlint // inspect the current concrete node, not a descendant
if ok && urlErr != nil && urlErr.URL != "" {
redactedURL := redactSearchURLParams(urlErr.URL)
if redactedURL != urlErr.URL {
redactions[urlErr.URL] = redactedURL
}
}
// This is structural traversal of the current node, not error matching.
// errors.As cannot distinguish the node's one-child and many-child shapes.
//nolint:errorlint // inspect the current node's concrete Unwrap shape, not a descendant
switch unwrapped := err.(type) {
case interface{ Unwrap() []error }:
for _, child := range unwrapped.Unwrap() {
collectHTTPErrorURLRedactions(child, redactions)
}
case interface{ Unwrap() error }:
collectHTTPErrorURLRedactions(unwrapped.Unwrap(), redactions)
}
}
func (s *SearXNGSearcher) isEmptyResponse(resp *SearchResponse) bool {
return len(resp.Results) == 0 &&
len(resp.Infoboxes) == 0 &&
len(resp.Answers) == 0 &&
len(resp.Suggestions) == 0
}
func (s *SearXNGSearcher) logDebugRequest(req *http.Request, body string) {
if !s.debug {
return
}
if req.Method == http.MethodGet {
s.getLogger().Debug(
"HTTP request",
"method", req.Method,
"url", redactSearchURLParams(req.URL.String()),
"Accept", req.Header.Get("Accept"),
)
return
}
bodyPreview := string(truncateBytesToValidUTF8([]byte(body), DebugBodyPreviewBytes))
s.getLogger().Debug(
"HTTP request",
"method", req.Method,
"url", req.URL.String(),
"Content-Type", req.Header.Get("Content-Type"),
"Accept", req.Header.Get("Accept"),
"body", bodyPreview,
)
}
func (s *SearXNGSearcher) logDebugResponse(resp *http.Response, err error) {
if !s.debug || err != nil || resp == nil {
return
}
s.getLogger().Debug(
"HTTP response",
"status", resp.StatusCode,
"content_type", resp.Header.Get("Content-Type"),
)
}
func (s *SearXNGSearcher) logDebugRetry(attempt, maxAttempts int, delay time.Duration, err error) {
if !s.debug {
return
}
s.getLogger().Debug(
"retrying search request",
"attempt", attempt+1,
"max_attempts", maxAttempts,
"delay", delay,
"error", err,
)
}
func (s *SearXNGSearcher) executeGETfallback(
ctx context.Context,
resp *http.Response,
postReq *http.Request,
postBodyStr string,
) (*http.Response, error) {
s.getLogger().Warn(
"GET fallback used after POST search was rejected. "+getFallbackLogRisk,
"status", resp.StatusCode,
)
if s.debug {
s.getLogger().Debug("Redirecting to GET fallback",
"status", resp.StatusCode,
"reason", "POST not supported by server")
}
// Capture the original 405/501 error before closing the response body so the
// hint (including the "set SEARXNG_ALLOW_GET_FALLBACK=1" message) is preserved
// when GET fallback also fails.
origErr := HTTPStatusError(resp.StatusCode, resp.Header.Get("Content-Type"), nil)
closeResponseBody(resp, s.getLogger())
getURL := *postReq.URL
getURL.RawQuery = postBodyStr
//nolint:gosec // GET fallback reuses the validated SearXNG base URL and only changes query parameters.
getReq, reqErr := http.NewRequestWithContext(ctx, http.MethodGet, getURL.String(), http.NoBody)
if reqErr != nil {
return nil, errors.Join(origErr, NewSearXNGError(0, "", "", fmt.Errorf("%w: %w", errRequestCreateFailed, reqErr)))
}
setBrowserHeaders(getReq)
s.logDebugRequest(getReq, "")
getResp, err := s.doHTTP(getReq)
if err != nil {
err = fmt.Errorf("%w: %w", errGETFallbackUsed, err)
err = errors.Join(origErr, err)
}
s.logDebugResponse(getResp, err)
return getResp, err
}
func redactSearchURLParams(rawURL string) string {
parsed, err := url.Parse(rawURL)
if err != nil {
return rawURL
}
if parsed.RawQuery != "" {
parsed.RawQuery = "[REDACTED]"
}
return parsed.String()
}
func (s *SearXNGSearcher) handleNonOKResponse(resp *http.Response) error {
body, truncated, readErr := readBodyWithLimit(resp.Body, MaxErrorBodySize)
if readErr != nil {
return NewSearXNGError(
resp.StatusCode, resp.Header.Get("Content-Type"), "",
fmt.Errorf("failed to read error response body: %w", readErr),
)
}
if s.debug {
errBodyPreview := string(truncateBytesToValidUTF8(body, DebugBodyPreviewBytes))
s.getLogger().Debug(
"HTTP error response body",
"status", resp.StatusCode,
"content_type", resp.Header.Get("Content-Type"),
"body_size", len(body),
"body_preview", errBodyPreview,
)
}
if truncated {
return NewSearXNGError(resp.StatusCode, resp.Header.Get("Content-Type"), buildErrorPreview(body),
fmt.Errorf("%w of %d bytes", errErrorBodyTooLarge, MaxErrorBodySize))
}
return HTTPStatusError(resp.StatusCode, resp.Header.Get("Content-Type"), body)
}
// getLogger returns the searcher's logger, falling back to slog.Default() if nil.
func (s *SearXNGSearcher) getLogger() *slog.Logger {
if s.logger != nil {
return s.logger
}
return slog.Default()
}