mirror of
https://github.com/xlionjuan/searxng-mcp-go.git
synced 2026-11-17 04:17:51 +00:00
* chore(deps): update dependency go to v1.27.1 * Fixed all lints; just verify passes. Co-authored-by: xlionjuan <[email protected]> * Fixed `.golangci.yml` paths→path; `just verify` passes. Co-authored-by: xlionjuan <[email protected]> * Parallelized golden & error-path tests; verify passes. Co-authored-by: xlionjuan <[email protected]> * Verify passes; HTTPClient/Logger nil set; exhaustruct covers Config. Co-authored-by: xlionjuan <[email protected]> --------- Co-authored-by: xlion-bot-renovate[bot] <218815870+xlion-bot-renovate[bot]@users.noreply.github.com> Co-authored-by: opencode-agent[bot] <opencode-agent[bot]@users.noreply.github.com> Co-authored-by: xlionjuan <[email protected]>
706 lines
20 KiB
Go
706 lines
20 KiB
Go
package searxng
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"net/http"
|
|
"net/url"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
var (
|
|
errSearcherConfigRequired = errors.New("newSearXNGSearcher: config cannot be nil")
|
|
errSearcherURLParseInternal = errors.New(
|
|
"newSearXNGSearcher: url.Parse failed after validateBaseURL passed (internal error)",
|
|
)
|
|
errRequestCreateFailed = errors.New("failed to create request")
|
|
errSearchRequestFailed = errors.New("failed to execute search request")
|
|
errErrorBodyTooLarge = errors.New("error response body exceeded maximum size limit")
|
|
errSearchEmptyResults = errors.New("search returned empty results after all retries")
|
|
errGETFallbackUsed = errors.New(
|
|
"GET fallback was used; search query parameters may have been sent in the request URL",
|
|
)
|
|
errNilFinishResponse = errors.New("finishResponse: nil http.Response")
|
|
)
|
|
|
|
const getFallbackLogRisk = "Search query parameters may be sent in upstream URLs and recorded by " +
|
|
"SearXNG, proxy, CDN, or access logs"
|
|
|
|
// SearXNGSearcher performs web searches via a SearXNG instance.
|
|
//
|
|
//nolint:containedctx // lifecycle context canceled on Close(), propagated via AfterFunc
|
|
type SearXNGSearcher struct {
|
|
client *http.Client // Configurable HTTP client
|
|
searchEndpoint *url.URL // Precomputed /search endpoint URL; cloned per request
|
|
debug bool // When true, enables verbose HTTP request/response logging
|
|
logger *slog.Logger // Logger for warnings and debug output; nil = slog.Default()
|
|
searcherCtx context.Context
|
|
searcherCancel context.CancelFunc
|
|
closeOnce sync.Once
|
|
retryStrategy *exponentialBackoffStrategy
|
|
ownsTransport bool // true if the searcher created its own transport (safe to close)
|
|
allowGETFallback bool
|
|
|
|
// retryWaitHook is a test-only hook called immediately before retryWait.
|
|
// It allows tests to synchronize at the point where the retry loop has
|
|
// entered the backoff wait so Close-during-backoff can be verified directly.
|
|
retryWaitHook func()
|
|
}
|
|
|
|
// NewSearXNGSearcher creates a new SearXNGSearcher with the given configuration.
|
|
// Returns an error if cfg is nil, cfg.Validate fails, the base URL is empty
|
|
// or invalid, or endpoint construction fails internally.
|
|
func NewSearXNGSearcher(cfg *Config, debug bool) (*SearXNGSearcher, error) {
|
|
if cfg == nil {
|
|
return nil, errSearcherConfigRequired
|
|
}
|
|
|
|
// Validate config first to catch invalid negative values before normalization
|
|
err := cfg.Validate()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("newSearXNGSearcher: %w", err)
|
|
}
|
|
|
|
// Normalize config: apply safe defaults for zero values
|
|
cfg = cfg.Normalize()
|
|
|
|
baseURL := cfg.SearXNGURL
|
|
|
|
err = validateBaseURL(baseURL)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("newSearXNGSearcher: %w", err)
|
|
}
|
|
|
|
// Precompute the /search endpoint URL once so per-request construction
|
|
// only has to clone the result. computeSearchEndpoint may still fail
|
|
// after validateBaseURL (defensive); surface that as an internal error.
|
|
searchEndpoint, err := computeSearchEndpoint(baseURL)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("%w: %w", errSearcherURLParseInternal, err)
|
|
}
|
|
|
|
logger := slog.Default()
|
|
if cfg.Logger != nil {
|
|
logger = cfg.Logger
|
|
}
|
|
|
|
if searchEndpoint.Scheme == schemeHTTP && !isPrivateHost(searchEndpoint.Host) {
|
|
logger.Warn("Using HTTP for non-private host. " +
|
|
"Search queries may be transmitted in clear text. " +
|
|
"Search results could be intercepted and modified by a MITM attacker")
|
|
}
|
|
|
|
if cfg.AllowGETFallback {
|
|
logger.Warn("GET fallback is enabled. " + getFallbackLogRisk)
|
|
}
|
|
|
|
client, ownsTransport := buildSearchHTTPClient(cfg)
|
|
|
|
searcher := &SearXNGSearcher{
|
|
client: client,
|
|
searchEndpoint: searchEndpoint,
|
|
debug: debug,
|
|
logger: logger,
|
|
retryStrategy: newExponentialBackoffStrategy(cfg.MaxRetries, cfg.RetryDelay, cfg.MaxRetryDelay),
|
|
ownsTransport: ownsTransport,
|
|
allowGETFallback: cfg.AllowGETFallback,
|
|
}
|
|
searcher.searcherCtx, searcher.searcherCancel = context.WithCancel(context.Background())
|
|
|
|
return searcher, nil
|
|
}
|
|
|
|
// buildSearchHTTPClient wraps the configured HTTP client with the search
|
|
// redirect policy, or constructs a fresh client with the configured timeout.
|
|
// It returns the client and whether the searcher owns it (and so must close
|
|
// its idle connections on Close).
|
|
func buildSearchHTTPClient(cfg *Config) (*http.Client, bool) {
|
|
client := cfg.HTTPClient
|
|
|
|
var ownsTransport bool
|
|
|
|
if client != nil {
|
|
originalCheckRedirect := client.CheckRedirect
|
|
wrapped := *client
|
|
wrapped.CheckRedirect = func(req *http.Request, via []*http.Request) error {
|
|
err := enforceSearchRedirectPolicy(req, via)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if originalCheckRedirect != nil {
|
|
return originalCheckRedirect(req, via)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
client = &wrapped
|
|
} else {
|
|
if cfg.Timeout > 0 {
|
|
client = newHTTPClient(cfg.Timeout)
|
|
ownsTransport = true
|
|
} else {
|
|
client = getDefaultHTTPClient()
|
|
// ownsTransport remains false — we do not own the shared singleton
|
|
}
|
|
}
|
|
|
|
return client, ownsTransport
|
|
}
|
|
|
|
// Close releases resources held by the searcher and cancels in-flight searches.
|
|
// Close is safe to call on searchers that use a shared default client —
|
|
// it will skip closing idle connections on transports it does not own.
|
|
func (s *SearXNGSearcher) Close() error {
|
|
s.closeOnce.Do(func() {
|
|
if s.searcherCancel != nil {
|
|
s.searcherCancel()
|
|
}
|
|
})
|
|
|
|
if s.ownsTransport && s.client != nil && s.client.Transport != nil {
|
|
if transport, ok := s.client.Transport.(*http.Transport); ok {
|
|
transport.CloseIdleConnections()
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Search executes the search query against SearXNG with retry support.
|
|
// Returns SearXNGError wrapping the last error if all retries are exhausted.
|
|
// Returns ValidationError if args are invalid.
|
|
func (s *SearXNGSearcher) Search(ctx context.Context, args *SearchArgs) (*SearchResponse, error) {
|
|
normalized, err := ValidateSearchArgs(args)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
args = normalized
|
|
|
|
// Tie search lifecycle to searcher close
|
|
searchCtx, searchCancel := s.searchContext(ctx)
|
|
defer searchCancel()
|
|
|
|
return s.searchWithRetries(searchCtx, args)
|
|
}
|
|
|
|
// searchWithRetries runs the retry loop for a single search.
|
|
//
|
|
//nolint:gocognit // retry orchestration: attempt, classification, error tracking, and wait
|
|
func (s *SearXNGSearcher) searchWithRetries(ctx context.Context, args *SearchArgs) (*SearchResponse, error) {
|
|
var lastErr error
|
|
|
|
maxRetries := s.retryStrategy.MaxRetries()
|
|
|
|
for attempt := 0; attempt <= maxRetries; attempt++ {
|
|
resp, _, err := s.doSearchAttempt(ctx, args)
|
|
|
|
ar, classifyErr := s.classifyAttempt(ctx, attempt, maxRetries, resp, err, args)
|
|
if classifyErr != nil {
|
|
return nil, classifyErr
|
|
}
|
|
|
|
if ar.outcome == OutcomeSuccess {
|
|
return ar.result, nil
|
|
}
|
|
|
|
if ar.outcome == OutcomeEmptyExhausted {
|
|
return nil, wrapSearchError(errSearchEmptyResults)
|
|
}
|
|
|
|
trackErr := attemptError(err, ar)
|
|
|
|
shouldRetry, delay := s.retryStrategy.ShouldRetry(ctx, attempt, ar.outcome)
|
|
|
|
if !shouldRetry {
|
|
if trackErr == nil && ctx.Err() != nil {
|
|
trackErr = ctx.Err()
|
|
}
|
|
|
|
direct, termErr := s.resolveTerminalOutcome(resp, args, trackErr)
|
|
if direct {
|
|
return nil, termErr
|
|
}
|
|
|
|
if termErr != nil {
|
|
lastErr = termErr
|
|
}
|
|
|
|
break
|
|
}
|
|
|
|
// Track last error for final wrapping
|
|
lastErr = trackErr
|
|
|
|
s.logDebugRetry(attempt, maxRetries+1, delay, lastErr)
|
|
|
|
closeResponseBody(resp, s.getLogger())
|
|
|
|
waitErr := s.waitForRetry(ctx, delay)
|
|
if waitErr != nil {
|
|
lastErr = waitErr
|
|
|
|
break
|
|
}
|
|
}
|
|
|
|
return nil, wrapSearchError(lastErr)
|
|
}
|
|
|
|
// attemptError returns the error to track for a non-successful attempt: the
|
|
// attempt's own error when present, otherwise the classified attempt's error
|
|
// (e.g. a body-read failure), otherwise the empty-results sentinel for an
|
|
// OutcomeEmptyRetry attempt.
|
|
func attemptError(err error, ar *attemptResult) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if ar.err != nil {
|
|
return ar.err
|
|
}
|
|
|
|
if ar.outcome == OutcomeEmptyRetry {
|
|
return errSearchEmptyResults
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// resolveTerminalOutcome handles a final (non-retried) attempt. It returns
|
|
// whether the error must short-circuit the retry loop (a direct
|
|
// finishResponse failure) and the error to record as the last error (which
|
|
// the retry loop wraps) otherwise.
|
|
func (s *SearXNGSearcher) resolveTerminalOutcome(
|
|
resp *http.Response,
|
|
args *SearchArgs,
|
|
trackErr error,
|
|
) (bool, error) {
|
|
if trackErr != nil {
|
|
closeResponseBody(resp, s.getLogger())
|
|
|
|
return false, trackErr
|
|
}
|
|
|
|
if resp != nil && resp.StatusCode != http.StatusOK {
|
|
_, finishErr := s.finishResponse(resp, args)
|
|
|
|
return true, finishErr
|
|
}
|
|
|
|
return false, nil
|
|
}
|
|
|
|
// waitForRetry invokes the test-only retryWaitHook (used to verify
|
|
// Close-during-backoff) and then sleeps for the retry backoff delay.
|
|
func (s *SearXNGSearcher) waitForRetry(ctx context.Context, delay time.Duration) error {
|
|
if s.retryWaitHook != nil {
|
|
s.retryWaitHook()
|
|
}
|
|
|
|
return retryWait(ctx, delay)
|
|
}
|
|
|
|
// searchContext creates a context tied to the searcher lifecycle.
|
|
// The returned context is canceled when the searcher is closed.
|
|
func (s *SearXNGSearcher) searchContext(ctx context.Context) (context.Context, context.CancelFunc) {
|
|
searchCtx, cancel := context.WithCancel(ctx)
|
|
|
|
// When the searcher is closed, cancel the search context.
|
|
// AfterFunc runs f in a goroutine only when s.searcherCtx is done,
|
|
// so this is a no-op (just bookkeeping) in the happy path.
|
|
// The nil guard covers test literals that bypass constructors;
|
|
// all production paths initialize searcherCtx at construction.
|
|
if s.searcherCtx != nil {
|
|
stop := context.AfterFunc(s.searcherCtx, cancel)
|
|
|
|
return searchCtx, func() {
|
|
stop()
|
|
cancel()
|
|
}
|
|
}
|
|
|
|
return searchCtx, cancel
|
|
}
|
|
|
|
// wrapSearchError wraps the last error for the Search return value.
|
|
// Preserves SearXNGError unwrapping to avoid hiding the real status code.
|
|
// Returns the fallback error for the "should never reach here" case when err is nil.
|
|
func wrapSearchError(err error) error {
|
|
if _, ok := errors.AsType[*SearXNGError](err); ok {
|
|
return fmt.Errorf("%w: %w", errSearchRequestFailed, err)
|
|
}
|
|
|
|
if err != nil {
|
|
return NewSearXNGError(0, "", "", fmt.Errorf("%w: %w", errSearchRequestFailed, err))
|
|
}
|
|
|
|
// Should never reach here
|
|
return NewSearXNGError(0, "", "", errSearchRequestFailed)
|
|
}
|
|
|
|
// attemptResult holds the result of classifying a single search attempt.
|
|
type attemptResult struct {
|
|
outcome Outcome
|
|
result *SearchResponse
|
|
err error // underlying error when outcome requires tracking (e.g. body-read failure)
|
|
}
|
|
|
|
// classifyAttempt processes the response from a single search attempt and
|
|
// determines its Outcome. Returns an error only if finishResponse itself
|
|
// fails (parse error, non-retryable HTTP status). For all other cases, it
|
|
// returns an attemptResult whose Outcome guides the retry decision.
|
|
func (s *SearXNGSearcher) classifyAttempt(
|
|
ctx context.Context,
|
|
attempt, maxRetries int,
|
|
resp *http.Response,
|
|
err error,
|
|
args *SearchArgs,
|
|
) (*attemptResult, error) {
|
|
if err != nil {
|
|
return &attemptResult{outcome: classifyOutcome(ctx, attempt, maxRetries, resp, err, false)}, nil
|
|
}
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
return &attemptResult{outcome: classifyOutcome(ctx, attempt, maxRetries, resp, nil, false)}, nil
|
|
}
|
|
|
|
result, finishErr := s.finishResponse(resp, args)
|
|
|
|
// finishResponse closed resp.Body via defer; nil it so the retry
|
|
// loop does not close the same body a second time.
|
|
resp.Body = nil
|
|
|
|
if finishErr != nil {
|
|
// Body-read failures (unexpected EOF, connection reset during
|
|
// read) are transient network errors — retry them. Invalid
|
|
// JSON, HTML, unexpected content type, and oversized bodies
|
|
// remain non-retryable hard failures.
|
|
if errors.Is(finishErr, errResponseReadFailed) {
|
|
return &attemptResult{outcome: OutcomeRetry, err: finishErr}, nil
|
|
}
|
|
|
|
return nil, finishErr
|
|
}
|
|
|
|
isEmpty := s.isEmptyResponse(result)
|
|
outcome := classifyOutcome(ctx, attempt, maxRetries, resp, nil, isEmpty)
|
|
|
|
return &attemptResult{outcome: outcome, result: result}, nil
|
|
}
|
|
|
|
// finishResponse handles non-OK status, JSON parsing, and body closure for a response.
|
|
//
|
|
// resp is required: passing nil triggers errNilFinishResponse. The retry
|
|
// path inside Search never reaches here with a nil response, but guarding
|
|
// the field makes the function robust against future refactors and
|
|
// self-documents the precondition.
|
|
func (s *SearXNGSearcher) finishResponse(resp *http.Response, args *SearchArgs) (*SearchResponse, error) {
|
|
if resp == nil {
|
|
return nil, errNilFinishResponse
|
|
}
|
|
|
|
defer closeResponseBody(resp, s.getLogger())
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
return nil, s.handleNonOKResponse(resp)
|
|
}
|
|
|
|
return s.parseSearchResponse(resp, args)
|
|
}
|
|
|
|
func (s *SearXNGSearcher) doSearchAttempt(ctx context.Context, args *SearchArgs) (*http.Response, string, error) {
|
|
postReq, postBodyStr, err := s.buildSearchRequest(ctx, args)
|
|
if err != nil {
|
|
return nil, postBodyStr, err
|
|
}
|
|
|
|
s.logDebugRequest(postReq, postBodyStr)
|
|
|
|
resp, err := s.doHTTP(postReq)
|
|
|
|
s.logDebugResponse(resp, err)
|
|
|
|
shouldFallbackToGET := err == nil && resp != nil && s.allowGETFallback &&
|
|
(resp.StatusCode == http.StatusMethodNotAllowed || resp.StatusCode == http.StatusNotImplemented)
|
|
if shouldFallbackToGET {
|
|
resp, err = s.executeGETfallback(ctx, resp, postReq, postBodyStr)
|
|
}
|
|
|
|
return resp, postBodyStr, err
|
|
}
|
|
|
|
// redactedHTTPError keeps the original HTTP error available for errors.Is and
|
|
// errors.As while providing a query-redacted representation for logs and users.
|
|
type redactedHTTPError struct {
|
|
err error
|
|
message string
|
|
}
|
|
|
|
func (e *redactedHTTPError) Error() string {
|
|
return e.message
|
|
}
|
|
|
|
func (e *redactedHTTPError) Unwrap() error {
|
|
return e.err
|
|
}
|
|
|
|
// doHTTP normalizes response-body ownership and the observable representation
|
|
// of every error returned by http.Client.Do.
|
|
func (s *SearXNGSearcher) doHTTP(req *http.Request) (*http.Response, error) {
|
|
//nolint:gosec // Callers use the validated SearXNG endpoint or its derived GET fallback URL.
|
|
resp, err := s.client.Do(req)
|
|
if err == nil {
|
|
return resp, nil
|
|
}
|
|
|
|
if resp != nil {
|
|
// http.Client.Do returns both values only when CheckRedirect rejects a
|
|
// redirect. The client has already closed resp.Body, so retaining resp
|
|
// would give its body a second owner.
|
|
resp = nil
|
|
}
|
|
|
|
return resp, redactHTTPError(err)
|
|
}
|
|
|
|
func redactHTTPError(err error) error {
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
|
|
redactions := make(map[string]string)
|
|
collectHTTPErrorURLRedactions(err, redactions)
|
|
|
|
if len(redactions) == 0 {
|
|
return err
|
|
}
|
|
|
|
urls := make([]string, 0, len(redactions))
|
|
for rawURL := range redactions {
|
|
urls = append(urls, rawURL)
|
|
}
|
|
|
|
// Replace longer URLs first so a URL that prefixes another target cannot
|
|
// leave the longer target's remaining query parameters visible.
|
|
sort.Slice(urls, func(i, j int) bool {
|
|
if len(urls[i]) == len(urls[j]) {
|
|
return urls[i] < urls[j]
|
|
}
|
|
|
|
return len(urls[i]) > len(urls[j])
|
|
})
|
|
|
|
originalMessage := err.Error()
|
|
message := originalMessage
|
|
|
|
for _, rawURL := range urls {
|
|
redactedURL := redactions[rawURL]
|
|
|
|
// url.Error renders URL with %q. Replace that exact representation first
|
|
// to handle escaped characters, then replace unquoted copies emitted by
|
|
// wrappers that reuse the typed URL value in their own context.
|
|
message = strings.ReplaceAll(message, strconv.Quote(rawURL), strconv.Quote(redactedURL))
|
|
message = strings.ReplaceAll(message, rawURL, redactedURL)
|
|
}
|
|
|
|
if message == originalMessage {
|
|
return err
|
|
}
|
|
|
|
return &redactedHTTPError{err: err, message: message}
|
|
}
|
|
|
|
func collectHTTPErrorURLRedactions(err error, redactions map[string]string) {
|
|
if err == nil {
|
|
return
|
|
}
|
|
|
|
// Inspect this concrete node rather than using errors.As, which would jump
|
|
// to a descendant and prevent the walker from handling every URL node.
|
|
urlErr, ok := err.(*url.Error) //nolint:errorlint // inspect the current concrete node, not a descendant
|
|
if ok && urlErr != nil && urlErr.URL != "" {
|
|
redactedURL := redactSearchURLParams(urlErr.URL)
|
|
if redactedURL != urlErr.URL {
|
|
redactions[urlErr.URL] = redactedURL
|
|
}
|
|
}
|
|
|
|
// This is structural traversal of the current node, not error matching.
|
|
// errors.As cannot distinguish the node's one-child and many-child shapes.
|
|
//nolint:errorlint // inspect the current node's concrete Unwrap shape, not a descendant
|
|
switch unwrapped := err.(type) {
|
|
case interface{ Unwrap() []error }:
|
|
for _, child := range unwrapped.Unwrap() {
|
|
collectHTTPErrorURLRedactions(child, redactions)
|
|
}
|
|
case interface{ Unwrap() error }:
|
|
collectHTTPErrorURLRedactions(unwrapped.Unwrap(), redactions)
|
|
}
|
|
}
|
|
|
|
func (s *SearXNGSearcher) isEmptyResponse(resp *SearchResponse) bool {
|
|
return len(resp.Results) == 0 &&
|
|
len(resp.Infoboxes) == 0 &&
|
|
len(resp.Answers) == 0 &&
|
|
len(resp.Suggestions) == 0
|
|
}
|
|
|
|
func (s *SearXNGSearcher) logDebugRequest(req *http.Request, body string) {
|
|
if !s.debug {
|
|
return
|
|
}
|
|
|
|
if req.Method == http.MethodGet {
|
|
s.getLogger().Debug(
|
|
"HTTP request",
|
|
"method", req.Method,
|
|
"url", redactSearchURLParams(req.URL.String()),
|
|
"Accept", req.Header.Get("Accept"),
|
|
)
|
|
|
|
return
|
|
}
|
|
|
|
bodyPreview := string(truncateBytesToValidUTF8([]byte(body), DebugBodyPreviewBytes))
|
|
|
|
s.getLogger().Debug(
|
|
"HTTP request",
|
|
"method", req.Method,
|
|
"url", req.URL.String(),
|
|
"Content-Type", req.Header.Get("Content-Type"),
|
|
"Accept", req.Header.Get("Accept"),
|
|
"body", bodyPreview,
|
|
)
|
|
}
|
|
|
|
func (s *SearXNGSearcher) logDebugResponse(resp *http.Response, err error) {
|
|
if !s.debug || err != nil || resp == nil {
|
|
return
|
|
}
|
|
|
|
s.getLogger().Debug(
|
|
"HTTP response",
|
|
"status", resp.StatusCode,
|
|
"content_type", resp.Header.Get("Content-Type"),
|
|
)
|
|
}
|
|
|
|
func (s *SearXNGSearcher) logDebugRetry(attempt, maxAttempts int, delay time.Duration, err error) {
|
|
if !s.debug {
|
|
return
|
|
}
|
|
|
|
s.getLogger().Debug(
|
|
"retrying search request",
|
|
"attempt", attempt+1,
|
|
"max_attempts", maxAttempts,
|
|
"delay", delay,
|
|
"error", err,
|
|
)
|
|
}
|
|
|
|
func (s *SearXNGSearcher) executeGETfallback(
|
|
ctx context.Context,
|
|
resp *http.Response,
|
|
postReq *http.Request,
|
|
postBodyStr string,
|
|
) (*http.Response, error) {
|
|
s.getLogger().Warn(
|
|
"GET fallback used after POST search was rejected. "+getFallbackLogRisk,
|
|
"status", resp.StatusCode,
|
|
)
|
|
|
|
if s.debug {
|
|
s.getLogger().Debug("Redirecting to GET fallback",
|
|
"status", resp.StatusCode,
|
|
"reason", "POST not supported by server")
|
|
}
|
|
|
|
// Capture the original 405/501 error before closing the response body so the
|
|
// hint (including the "set SEARXNG_ALLOW_GET_FALLBACK=1" message) is preserved
|
|
// when GET fallback also fails.
|
|
origErr := HTTPStatusError(resp.StatusCode, resp.Header.Get("Content-Type"), nil)
|
|
|
|
closeResponseBody(resp, s.getLogger())
|
|
|
|
getURL := *postReq.URL
|
|
getURL.RawQuery = postBodyStr
|
|
|
|
//nolint:gosec // GET fallback reuses the validated SearXNG base URL and only changes query parameters.
|
|
getReq, reqErr := http.NewRequestWithContext(ctx, http.MethodGet, getURL.String(), http.NoBody)
|
|
if reqErr != nil {
|
|
return nil, errors.Join(origErr, NewSearXNGError(0, "", "", fmt.Errorf("%w: %w", errRequestCreateFailed, reqErr)))
|
|
}
|
|
|
|
setBrowserHeaders(getReq)
|
|
s.logDebugRequest(getReq, "")
|
|
|
|
getResp, err := s.doHTTP(getReq)
|
|
if err != nil {
|
|
err = fmt.Errorf("%w: %w", errGETFallbackUsed, err)
|
|
err = errors.Join(origErr, err)
|
|
}
|
|
|
|
s.logDebugResponse(getResp, err)
|
|
|
|
return getResp, err
|
|
}
|
|
|
|
func redactSearchURLParams(rawURL string) string {
|
|
parsed, err := url.Parse(rawURL)
|
|
if err != nil {
|
|
return rawURL
|
|
}
|
|
|
|
if parsed.RawQuery != "" {
|
|
parsed.RawQuery = "[REDACTED]"
|
|
}
|
|
|
|
return parsed.String()
|
|
}
|
|
|
|
func (s *SearXNGSearcher) handleNonOKResponse(resp *http.Response) error {
|
|
body, truncated, readErr := readBodyWithLimit(resp.Body, MaxErrorBodySize)
|
|
if readErr != nil {
|
|
return NewSearXNGError(
|
|
resp.StatusCode, resp.Header.Get("Content-Type"), "",
|
|
fmt.Errorf("failed to read error response body: %w", readErr),
|
|
)
|
|
}
|
|
|
|
if s.debug {
|
|
errBodyPreview := string(truncateBytesToValidUTF8(body, DebugBodyPreviewBytes))
|
|
|
|
s.getLogger().Debug(
|
|
"HTTP error response body",
|
|
"status", resp.StatusCode,
|
|
"content_type", resp.Header.Get("Content-Type"),
|
|
"body_size", len(body),
|
|
"body_preview", errBodyPreview,
|
|
)
|
|
}
|
|
|
|
if truncated {
|
|
return NewSearXNGError(resp.StatusCode, resp.Header.Get("Content-Type"), buildErrorPreview(body),
|
|
fmt.Errorf("%w of %d bytes", errErrorBodyTooLarge, MaxErrorBodySize))
|
|
}
|
|
|
|
return HTTPStatusError(resp.StatusCode, resp.Header.Get("Content-Type"), body)
|
|
}
|
|
|
|
// getLogger returns the searcher's logger, falling back to slog.Default() if nil.
|
|
func (s *SearXNGSearcher) getLogger() *slog.Logger {
|
|
if s.logger != nil {
|
|
return s.logger
|
|
}
|
|
|
|
return slog.Default()
|
|
}
|