searxng-mcp-go/internal/searxng/response.go
xlion-bot-renovate[bot] 6e4c553a9e
chore(deps): update dependency go to v1.27.1 (#588)
* chore(deps): update dependency go to v1.27.1

* Fixed all lints; just verify passes.

Co-authored-by: xlionjuan <[email protected]>

* Fixed `.golangci.yml` paths→path; `just verify` passes.

Co-authored-by: xlionjuan <[email protected]>

* Parallelized golden & error-path tests; verify passes.

Co-authored-by: xlionjuan <[email protected]>

* Verify passes; HTTPClient/Logger nil set; exhaustruct covers Config.

Co-authored-by: xlionjuan <[email protected]>

---------

Co-authored-by: xlion-bot-renovate[bot] <218815870+xlion-bot-renovate[bot]@users.noreply.github.com>
Co-authored-by: opencode-agent[bot] <opencode-agent[bot]@users.noreply.github.com>
Co-authored-by: xlionjuan <[email protected]>
2026-09-06 23:14:09 +08:00

301 lines
9 KiB
Go

package searxng
import (
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"mime"
"net/http"
"slices"
"unicode"
"unicode/utf8"
"searxng-mcp-go/internal/searxng/answer"
)
var (
errResponseReadFailed = errors.New("failed to read response body")
errResponseBodyTooLarge = errors.New("response body exceeded maximum size limit")
errUnexpectedContentType = errors.New("unexpected content type: expected application/json")
errJSONResponseParseFailed = errors.New("failed to parse JSON response")
)
// ExternalContentWarning warns callers that search results are untrusted external content.
const ExternalContentWarning = "Search results come from external sources" +
" and may be inaccurate, outdated, or adversarial; verify before using them."
// parseSearchResponse reads and parses the response from a SearXNG search request.
func (s *SearXNGSearcher) parseSearchResponse(resp *http.Response, args *SearchArgs) (*SearchResponse, error) {
body, err := readLimitedBody(resp)
if err != nil {
return nil, err
}
s.logDebugBody(resp, body)
contentType := resp.Header.Get("Content-Type")
if isHTMLResponse(contentType, body) {
bodyLen := len(body)
if bodyLen == 0 {
return nil, &HTMLResponseError{Body: "", UnderlyingErr: nil}
}
previewLen := min(bodyLen, MaxErrorDisplayBytes)
preview := string(truncateBytesToValidUTF8(body, previewLen))
s.getLogger().Debug("HTMLResponseError: received HTML instead of JSON", "preview", preview)
return nil, &HTMLResponseError{Body: preview, UnderlyingErr: nil}
}
result, err := decodeSearchResponse(resp, contentType, body, s.getLogger())
if err != nil {
return nil, err
}
s.normalizeResponse(result, args)
return result, nil
}
// readBodyWithLimit reads up to maxBytes+1 bytes from body and returns the data,
// a boolean indicating if the body was truncated (exceeded maxBytes), and any error.
// When truncated, it walks back to a valid UTF-8 rune boundary to avoid splitting runes.
func readBodyWithLimit(body io.ReadCloser, maxBytes int64) ([]byte, bool, error) {
data, err := io.ReadAll(io.LimitReader(body, maxBytes+1))
if err != nil {
return nil, false, err
}
truncated := int64(len(data)) > maxBytes
if truncated {
data = data[:maxBytes]
// Walk back to a valid UTF-8 rune boundary to avoid splitting multi-byte runes.
// Only strip truly invalid trailing bytes (size == 1), not valid runes like
// U+FFFD which also decode as RuneError but with size == 3.
for len(data) > 0 {
r, size := utf8.DecodeLastRune(data)
if r == utf8.RuneError && size == 1 {
data = data[:len(data)-1]
continue
}
break
}
}
return data, truncated, nil
}
func readLimitedBody(resp *http.Response) ([]byte, error) {
body, truncated, err := readBodyWithLimit(resp.Body, MaxResponseBodySize)
if err != nil {
return nil, NewSearXNGError(
resp.StatusCode, resp.Header.Get("Content-Type"), "",
fmt.Errorf("%w: %w", errResponseReadFailed, err),
)
}
if truncated {
bodySizeErr := fmt.Errorf("%w of %d bytes", errResponseBodyTooLarge, MaxResponseBodySize)
return nil, NewSearXNGError(resp.StatusCode, resp.Header.Get("Content-Type"), buildErrorPreview(body), bodySizeErr)
}
return body, nil
}
func (s *SearXNGSearcher) logDebugBody(resp *http.Response, body []byte) {
if !s.debug {
return
}
bodyPreview := string(truncateBytesToValidUTF8(body, DebugBodyPreviewBytes))
s.getLogger().Debug(
"HTTP response body",
"status", resp.StatusCode,
"content_type", resp.Header.Get("Content-Type"),
"body_size", len(body),
"body_preview", bodyPreview,
)
}
func isHTMLResponse(contentType string, body []byte) bool {
if isHTMLContentType(contentType) {
return true
}
return hasHTMLPrefix(body)
}
// maxHTMLProbe is the maximum number of bytes examined from the body for
// HTML prefix detection. It covers leading whitespace plus the longest
// prefix we check (<!doctype or <html).
const maxHTMLProbe = 64
// hasHTMLPrefix reports whether body has <!doctype or <html prefix after
// skipping leading Unicode whitespace, examining at most maxHTMLProbe bytes.
// It avoids allocating a lowercase copy of the entire body.
func hasHTMLPrefix(body []byte) bool {
probe := body[:min(len(body), maxHTMLProbe)]
idx := 0
for idx < len(probe) {
rn, size := utf8.DecodeRune(probe[idx:])
if rn == utf8.RuneError {
break
}
if !unicode.IsSpace(rn) {
break
}
idx += size
}
remaining := probe[idx:]
if len(remaining) == 0 {
return false
}
return equalFoldPrefix(remaining, "<!doctype") || equalFoldPrefix(remaining, "<html")
}
// equalFoldPrefix reports whether b starts with prefix s under ASCII
// case-insensitive comparison. Both b and s are assumed to contain only
// ASCII bytes. The caller is responsible for bounding how much of b is
// examined.
func equalFoldPrefix(bs []byte, prefix string) bool {
if len(bs) < len(prefix) {
return false
}
for i := range len(prefix) {
if bs[i]|0x20 != prefix[i]|0x20 {
return false
}
}
return true
}
// isJSONContentType reports whether the Content-Type header value, when parsed
// as a media type, identifies a JSON payload (application/json or text/json).
// The comparison is case-insensitive and parameters such as charset are
// ignored. An empty or unparseable Content-Type returns false.
func isJSONContentType(contentType string) bool {
return mediaTypeIs(contentType, "application/json", "text/json")
}
// isHTMLContentType reports whether the Content-Type header value, when parsed
// as a media type, identifies an HTML payload (text/html). The comparison is
// case-insensitive and parameters such as charset are ignored. An empty or
// unparseable Content-Type returns false.
func isHTMLContentType(contentType string) bool {
return mediaTypeIs(contentType, "text/html")
}
// mediaTypeIs reports whether contentType, when parsed as a media type,
// matches any of want. mime.ParseMediaType lowercases the type and parameter
// names, so a successful parse yields a normalized base type. If parsing
// fails, the content type is treated as not matching, which lets body-based
// fallbacks (e.g., HTML body detection) still apply.
func mediaTypeIs(contentType string, want ...string) bool {
mt, _, err := mime.ParseMediaType(contentType)
if err != nil {
return false
}
return slices.Contains(want, mt)
}
func decodeSearchResponse(
resp *http.Response, contentType string, body []byte, logger *slog.Logger,
) (*SearchResponse, error) {
if !isJSONContentType(contentType) {
bodyPreview := string(truncateBytesToValidUTF8(body, MaxErrorDisplayBytes))
if len(body) > MaxErrorDisplayBytes {
bodyPreview += "..."
}
logger.Debug("UnexpectedContentTypeError", "content_type", contentType, "body_preview", bodyPreview)
return nil, NewSearXNGError(resp.StatusCode, contentType, "", errUnexpectedContentType)
}
var result SearchResponse
err := json.Unmarshal(body, &result)
if err != nil {
logger.Debug("JSONParseError: failed to parse JSON response", "error", err)
return nil, NewSearXNGError(resp.StatusCode, contentType, "", fmt.Errorf("%w: %w", errJSONResponseParseFailed, err))
}
return &result, nil
}
func (s *SearXNGSearcher) normalizeResponse(result *SearchResponse, args *SearchArgs) {
result.Warning = ExternalContentWarning
if result.NumberOfResults == 0 && len(result.Results) > 0 {
result.NumberOfResults = len(result.Results)
}
// Cap answers and infoboxes before deduplication. The dedup pass is
// O(n*m) substring work; a configured upstream that returns a body
// below MaxResponseBodySize could still pack tens of thousands of
// compact entries into these arrays, forcing very large CPU work
// before result limiting. See MaxAnswers / MaxInfoboxes for details.
if len(result.Answers) > MaxAnswers {
s.getLogger().Warn("truncating answers before deduplication",
"count", len(result.Answers),
"max", MaxAnswers)
result.Answers = slices.Clone(result.Answers[:MaxAnswers])
}
if len(result.Infoboxes) > MaxInfoboxes {
s.getLogger().Warn("truncating infoboxes before deduplication",
"count", len(result.Infoboxes),
"max", MaxInfoboxes)
result.Infoboxes = slices.Clone(result.Infoboxes[:MaxInfoboxes])
}
// Derive display text for typed answers (translation, weather) that may
// omit the legacy "answer" string.
for i := range result.Answers {
EnsureAnswerFallback(&result.Answers[i])
}
result.Answers = deduplicateAnswers(result.Answers, result.Infoboxes)
if args.Limit != nil && *args.Limit >= 0 && len(result.Results) > *args.Limit {
result.Results = slices.Clone(result.Results[:*args.Limit])
}
// Normalize nil slices to empty slices for consistent JSON output.
if result.Results == nil {
result.Results = []SearchResult{}
}
if result.Suggestions == nil {
result.Suggestions = []string{}
}
result.Debug = s.debug
if s.debug && result.UnresponsiveEngines == nil {
result.UnresponsiveEngines = [][]string{}
}
}
// EnsureAnswerFallback derives a human-readable Answer string for known typed
// answers (translation, weather) that may omit the legacy "answer" field.
func EnsureAnswerFallback(a *Answer) {
answer.EnsureAnswerFallback(a)
}