searxng-mcp-go/internal/searxng/response_internal_test.go
xlion-bot-renovate[bot] 6e4c553a9e
chore(deps): update dependency go to v1.27.1 (#588)
* chore(deps): update dependency go to v1.27.1

* Fixed all lints; just verify passes.

Co-authored-by: xlionjuan <[email protected]>

* Fixed `.golangci.yml` paths→path; `just verify` passes.

Co-authored-by: xlionjuan <[email protected]>

* Parallelized golden & error-path tests; verify passes.

Co-authored-by: xlionjuan <[email protected]>

* Verify passes; HTTPClient/Logger nil set; exhaustruct covers Config.

Co-authored-by: xlionjuan <[email protected]>

---------

Co-authored-by: xlion-bot-renovate[bot] <218815870+xlion-bot-renovate[bot]@users.noreply.github.com>
Co-authored-by: opencode-agent[bot] <opencode-agent[bot]@users.noreply.github.com>
Co-authored-by: xlionjuan <[email protected]>
2026-09-06 23:14:09 +08:00

1232 lines
36 KiB
Go

package searxng
import (
"bytes"
"errors"
"io"
"log/slog"
"net/http"
"path/filepath"
"strings"
"testing"
"time"
"searxng-mcp-go/internal/searxng/answer"
"searxng-mcp-go/internal/testhelper"
)
// --- readBodyWithLimit tests ---
//nolint:gocognit // table-driven test covering read limits and edge cases
func TestReadBodyWithLimit(t *testing.T) {
t.Parallel()
t.Run("body shorter than limit", func(t *testing.T) {
t.Parallel()
body := io.NopCloser(strings.NewReader("hello"))
data, truncated, err := readBodyWithLimit(body, 100)
if err != nil {
t.Fatalf("readBodyWithLimit() error = %v", err)
}
if truncated {
t.Fatal("readBodyWithLimit() truncated = true, want false")
}
if string(data) != "hello" {
t.Fatalf("data = %q, want %q", data, "hello")
}
})
t.Run("body exactly at limit", func(t *testing.T) {
t.Parallel()
body := io.NopCloser(strings.NewReader("hello"))
data, truncated, err := readBodyWithLimit(body, 5)
if err != nil {
t.Fatalf("readBodyWithLimit() error = %v", err)
}
if truncated {
t.Fatal("readBodyWithLimit() truncated = true, want false")
}
if string(data) != "hello" {
t.Fatalf("data = %q, want %q", data, "hello")
}
})
t.Run("body exceeds limit", func(t *testing.T) {
t.Parallel()
body := io.NopCloser(strings.NewReader("hello world"))
data, truncated, err := readBodyWithLimit(body, 5)
if err != nil {
t.Fatalf("readBodyWithLimit() error = %v", err)
}
if !truncated {
t.Fatal("readBodyWithLimit() truncated = false, want true")
}
if string(data) != "hello" {
t.Fatalf("data = %q, want %q", data, "hello")
}
})
t.Run("empty body", func(t *testing.T) {
t.Parallel()
body := io.NopCloser(strings.NewReader(""))
data, truncated, err := readBodyWithLimit(body, 100)
if err != nil {
t.Fatalf("readBodyWithLimit() error = %v", err)
}
if truncated {
t.Fatal("readBodyWithLimit() truncated = true, want false")
}
if len(data) != 0 {
t.Fatalf("data length = %d, want 0", len(data))
}
})
}
// --- readLimitedBody tests ---
func TestReadLimitedBody(t *testing.T) {
t.Parallel()
t.Run("successful read", func(t *testing.T) {
t.Parallel()
resp := &http.Response{
StatusCode: http.StatusOK,
Header: http.Header{"Content-Type": []string{"application/json"}},
Body: io.NopCloser(strings.NewReader(`{"key": "value"}`)),
}
data, err := readLimitedBody(resp)
if err != nil {
t.Fatalf("readLimitedBody() error = %v", err)
}
if string(data) != `{"key": "value"}` {
t.Fatalf("data = %q, want %q", data, `{"key": "value"}`)
}
})
t.Run("body exceeds MaxResponseBodySize", func(t *testing.T) {
t.Parallel()
largeBody := bytes.Repeat([]byte("x"), int(MaxResponseBodySize)+100)
resp := &http.Response{
StatusCode: http.StatusOK,
Header: http.Header{"Content-Type": []string{"application/json"}},
Body: io.NopCloser(bytes.NewReader(largeBody)),
}
data, err := readLimitedBody(resp)
if err == nil {
t.Fatal("readLimitedBody() error = nil, want error")
}
if !strings.Contains(err.Error(), "response body exceeded maximum size limit") {
t.Fatalf("error = %q, want body exceeded maximum size limit", err.Error())
}
if data != nil {
t.Fatalf("data = %v, want nil on error", data)
}
})
}
// --- isHTMLResponse tests ---
func TestIsHTMLResponse(t *testing.T) {
t.Parallel()
tests := []struct {
name string
contentType string
body string
want bool
}{
{name: "content-type text/html", contentType: "text/html", body: "<html></html>", want: true},
{name: "content-type text/html; charset=utf-8", contentType: "text/html; charset=utf-8", body: "hello", want: true},
{name: "content-type TEXT/HTML (uppercase)", contentType: "TEXT/HTML", body: "hello", want: true},
{name: "content-type Text/Html (mixed case)", contentType: "Text/Html", body: "hello", want: true},
{
name: "content-type text/html with extra params",
contentType: "text/html; charset=utf-8; boundary=xyz",
body: "hello",
want: true,
},
{name: "body starts with <!DOCTYPE", contentType: "application/json", body: "<!DOCTYPE html>", want: true},
{
name: "body starts with <!doctype (lowercase)",
contentType: "application/json",
body: "<!doctype html>",
want: true,
},
{name: "body starts with <html", contentType: "application/json", body: `<html lang="en">`, want: true},
{name: "body starts with <HTML (uppercase)", contentType: "application/json", body: "<HTML>", want: true},
{
name: "body starts with <Html (mixed case)",
contentType: "application/json",
body: `<Html lang="en">`,
want: true,
},
{name: "json response", contentType: "application/json", body: `{"key": "value"}`, want: false},
{name: "empty body", contentType: "application/json", body: "", want: false},
{name: "trimmed body with spaces then DOCTYPE", contentType: "text/plain", body: " <!DOCTYPE html>", want: true},
{
name: "trimmed body with spaces then lowercase doctype",
contentType: "text/plain",
body: " <!doctype html>",
want: true,
},
{name: "near-match text/htmlish with json body", contentType: "text/htmlish", body: `{"key": "value"}`, want: false},
{name: "malformed content-type with html body", contentType: "not a mime type", body: "<!DOCTYPE html>", want: true},
{name: "empty content-type with json body", contentType: "", body: `{"key": "value"}`, want: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
if got := isHTMLResponse(tt.contentType, []byte(tt.body)); got != tt.want {
t.Fatalf("isHTMLResponse(%q, %q) = %v, want %v", tt.contentType, tt.body, got, tt.want)
}
})
}
}
func TestHasHTMLPrefix_ProbeBoundarySplitUnicodeWhitespace(t *testing.T) {
t.Parallel()
leadingWhitespace := strings.Repeat("\u2000", maxHTMLProbe/len("\u2000"))
if len(leadingWhitespace) != maxHTMLProbe-1 {
t.Fatalf("leading whitespace length = %d, want %d", len(leadingWhitespace), maxHTMLProbe-1)
}
// U+2006 begins within the probe window but ends beyond it; <html begins
// after the window and must not be considered by HTML prefix detection.
body := []byte(leadingWhitespace + "\u2006<html>")
if hasHTMLPrefix(body) {
t.Fatal("hasHTMLPrefix() = true, want false when <html is outside the probe window")
}
}
// --- isJSONContentType tests ---
func TestIsJSONContentType(t *testing.T) {
t.Parallel()
tests := []struct {
name string
contentType string
want bool
}{
{name: "application/json", contentType: "application/json", want: true},
{name: "application/json; charset=utf-8", contentType: "application/json; charset=utf-8", want: true},
{name: "application/json with extra params", contentType: "application/json; charset=utf-8; foo=bar", want: true},
{name: "APPLICATION/JSON (uppercase)", contentType: "APPLICATION/JSON", want: true},
{name: "Application/Json (mixed case)", contentType: "Application/Json", want: true},
{name: "text/json", contentType: "text/json", want: true},
{name: "text/json; charset=utf-8", contentType: "text/json; charset=utf-8", want: true},
{name: "TEXT/JSON (uppercase)", contentType: "TEXT/JSON", want: true},
{name: "Text/Json (mixed case)", contentType: "Text/Json", want: true},
{name: "application/jsonish (near-match rejected)", contentType: "application/jsonish", want: false},
{name: "application/json+ld (structured extension rejected)", contentType: "application/json+ld", want: false},
{name: "text/jsonish (near-match rejected)", contentType: "text/jsonish", want: false},
{name: "text/html is not json", contentType: "text/html", want: false},
{name: "empty content-type", contentType: "", want: false},
{name: "malformed content-type", contentType: "not a mime type", want: false},
{name: "missing subtype", contentType: "application/", want: false},
{name: "missing type", contentType: "/json", want: false},
{name: "whitespace only", contentType: " ", want: false},
{name: "application/json with surrounding whitespace", contentType: " application/json ", want: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
if got := isJSONContentType(tt.contentType); got != tt.want {
t.Fatalf("isJSONContentType(%q) = %v, want %v", tt.contentType, got, tt.want)
}
})
}
}
// --- isHTMLContentType tests ---
func TestIsHTMLContentType(t *testing.T) {
t.Parallel()
tests := []struct {
name string
contentType string
want bool
}{
{name: "text/html", contentType: "text/html", want: true},
{name: "text/html; charset=utf-8", contentType: "text/html; charset=utf-8", want: true},
{name: "text/html with extra params", contentType: "text/html; charset=utf-8; boundary=xyz", want: true},
{name: "TEXT/HTML (uppercase)", contentType: "TEXT/HTML", want: true},
{name: "Text/Html (mixed case)", contentType: "Text/Html", want: true},
{name: "text/htmlish (near-match rejected)", contentType: "text/htmlish", want: false},
{name: "application/xhtml+xml is not text/html", contentType: "application/xhtml+xml", want: false},
{name: "application/json is not html", contentType: "application/json", want: false},
{name: "empty content-type", contentType: "", want: false},
{name: "malformed content-type", contentType: "not a mime type", want: false},
{name: "missing subtype", contentType: "text/", want: false},
{name: "whitespace only", contentType: " ", want: false},
{name: "text/html with surrounding whitespace", contentType: " text/html ", want: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
if got := isHTMLContentType(tt.contentType); got != tt.want {
t.Fatalf("isHTMLContentType(%q) = %v, want %v", tt.contentType, got, tt.want)
}
})
}
}
// --- decodeSearchResponse tests ---
//nolint:gocognit,gocyclo,cyclop,maintidx // table-driven test covering many response decode scenarios
func TestDecodeSearchResponse(t *testing.T) {
t.Parallel()
t.Run("valid JSON response", func(t *testing.T) {
t.Parallel()
body := `{"query": "test", "results": [` +
`{"title": "Result 1", "url": "https://example.com", ` +
`"content": "Content", "engine": "google"}], ` +
`"suggestions": []}`
resp := &http.Response{
StatusCode: http.StatusOK,
Header: http.Header{"Content-Type": []string{"application/json"}},
}
result, err := decodeSearchResponse(resp, "application/json", []byte(body), slog.Default())
if err != nil {
t.Fatalf("decodeSearchResponse() error = %v", err)
}
if result.Query != "test" {
t.Fatalf("Query = %q, want %q", result.Query, "test")
}
if len(result.Results) != 1 {
t.Fatalf("len(Results) = %d, want 1", len(result.Results))
}
if result.Results[0].Title != "Result 1" {
t.Fatalf("Results[0].Title = %q, want %q", result.Results[0].Title, "Result 1")
}
})
t.Run("text/json content type", func(t *testing.T) {
t.Parallel()
body := `{"query": "test"}`
result, err := decodeSearchResponse(&http.Response{}, "text/json", []byte(body), slog.Default())
if err != nil {
t.Fatalf("decodeSearchResponse() error = %v", err)
}
if result.Query != "test" {
t.Fatalf("Query = %q, want %q", result.Query, "test")
}
})
t.Run("application/json with charset parameter", func(t *testing.T) {
t.Parallel()
body := `{"query": "test"}`
result, err := decodeSearchResponse(&http.Response{}, "application/json; charset=utf-8", []byte(body), slog.Default())
if err != nil {
t.Fatalf("decodeSearchResponse() error = %v", err)
}
if result.Query != "test" {
t.Fatalf("Query = %q, want %q", result.Query, "test")
}
})
t.Run("text/json with charset parameter", func(t *testing.T) {
t.Parallel()
body := `{"query": "test"}`
result, err := decodeSearchResponse(&http.Response{}, "text/json; charset=utf-8", []byte(body), slog.Default())
if err != nil {
t.Fatalf("decodeSearchResponse() error = %v", err)
}
if result.Query != "test" {
t.Fatalf("Query = %q, want %q", result.Query, "test")
}
})
t.Run("APPLICATION/JSON (uppercase)", func(t *testing.T) {
t.Parallel()
body := `{"query": "test"}`
result, err := decodeSearchResponse(&http.Response{}, "APPLICATION/JSON", []byte(body), slog.Default())
if err != nil {
t.Fatalf("decodeSearchResponse() error = %v", err)
}
if result.Query != "test" {
t.Fatalf("Query = %q, want %q", result.Query, "test")
}
})
t.Run("Application/Json (mixed case)", func(t *testing.T) {
t.Parallel()
body := `{"query": "test"}`
result, err := decodeSearchResponse(&http.Response{}, "Application/Json", []byte(body), slog.Default())
if err != nil {
t.Fatalf("decodeSearchResponse() error = %v", err)
}
if result.Query != "test" {
t.Fatalf("Query = %q, want %q", result.Query, "test")
}
})
t.Run("TEXT/JSON (uppercase)", func(t *testing.T) {
t.Parallel()
body := `{"query": "test"}`
result, err := decodeSearchResponse(&http.Response{}, "TEXT/JSON", []byte(body), slog.Default())
if err != nil {
t.Fatalf("decodeSearchResponse() error = %v", err)
}
if result.Query != "test" {
t.Fatalf("Query = %q, want %q", result.Query, "test")
}
})
t.Run("application/jsonish is rejected as near-match", func(t *testing.T) {
t.Parallel()
resp := &http.Response{StatusCode: http.StatusOK}
_, err := decodeSearchResponse(resp, "application/jsonish", []byte(`{"query": "test"}`), slog.Default())
if err == nil {
t.Fatal("decodeSearchResponse() error = nil, want error for application/jsonish")
}
if !strings.Contains(err.Error(), "unexpected content type: expected application/json") {
t.Fatalf("error = %q, want unexpected content type", err.Error())
}
})
t.Run("text/jsonish is rejected as near-match", func(t *testing.T) {
t.Parallel()
resp := &http.Response{StatusCode: http.StatusOK}
_, err := decodeSearchResponse(resp, "text/jsonish", []byte(`{"query": "test"}`), slog.Default())
if err == nil {
t.Fatal("decodeSearchResponse() error = nil, want error for text/jsonish")
}
if !strings.Contains(err.Error(), "unexpected content type: expected application/json") {
t.Fatalf("error = %q, want unexpected content type", err.Error())
}
})
t.Run("malformed content-type falls back to unexpected error", func(t *testing.T) {
t.Parallel()
resp := &http.Response{StatusCode: http.StatusOK}
_, err := decodeSearchResponse(resp, "not a mime type", []byte(`{"query": "test"}`), slog.Default())
if err == nil {
t.Fatal("decodeSearchResponse() error = nil, want error for malformed content-type")
}
if !strings.Contains(err.Error(), "unexpected content type: expected application/json") {
t.Fatalf("error = %q, want unexpected content type", err.Error())
}
})
t.Run("empty content-type is rejected", func(t *testing.T) {
t.Parallel()
resp := &http.Response{StatusCode: http.StatusOK}
_, err := decodeSearchResponse(resp, "", []byte(`{"query": "test"}`), slog.Default())
if err == nil {
t.Fatal("decodeSearchResponse() error = nil, want error for empty content-type")
}
if !strings.Contains(err.Error(), "unexpected content type: expected application/json") {
t.Fatalf("error = %q, want unexpected content type", err.Error())
}
})
t.Run("unexpected content type", func(t *testing.T) {
t.Parallel()
resp := &http.Response{StatusCode: http.StatusOK}
_, err := decodeSearchResponse(resp, "text/plain", []byte("not json"), slog.Default())
if err == nil {
t.Fatal("decodeSearchResponse() error = nil, want error")
}
if !strings.Contains(err.Error(), "unexpected content type: expected application/json") {
t.Fatalf("error = %q, want unexpected content type", err.Error())
}
})
t.Run("invalid JSON", func(t *testing.T) {
t.Parallel()
_, err := decodeSearchResponse(
&http.Response{StatusCode: http.StatusOK},
"application/json", []byte("{invalid}"), slog.Default(),
)
if err == nil {
t.Fatal("decodeSearchResponse() error = nil, want error")
}
if _, ok := errors.AsType[*SearXNGError](err); !ok {
t.Fatalf("error type = %T, want *SearXNGError", err)
}
})
}
// --- normalizeResponse tests ---
//nolint:gocognit,gocyclo,cyclop,maintidx // table-driven test covering many normalize response scenarios
func TestNormalizeResponse(t *testing.T) {
t.Parallel()
t.Run("sets warning and normalizes nil slices", func(t *testing.T) {
t.Parallel()
s := &SearXNGSearcher{debug: false}
result := &SearchResponse{}
s.normalizeResponse(result, &SearchArgs{})
if result.Warning != ExternalContentWarning {
t.Fatalf("Warning = %q, want %q", result.Warning, ExternalContentWarning)
}
if result.Results == nil {
t.Fatal("Results is nil, want empty slice")
}
if result.Suggestions == nil {
t.Fatal("Suggestions is nil, want empty slice")
}
if result.NumberOfResults != 0 {
t.Fatalf("NumberOfResults = %d, want 0", result.NumberOfResults)
}
})
t.Run("sets NumberOfResults from results length", func(t *testing.T) {
t.Parallel()
s := &SearXNGSearcher{debug: false}
result := &SearchResponse{
Results: []SearchResult{
{Title: "A"},
{Title: "B"},
},
}
s.normalizeResponse(result, &SearchArgs{})
if result.NumberOfResults != 2 {
t.Fatalf("NumberOfResults = %d, want 2", result.NumberOfResults)
}
})
t.Run("keeps existing NumberOfResults", func(t *testing.T) {
t.Parallel()
s := &SearXNGSearcher{debug: false}
result := &SearchResponse{
NumberOfResults: 42,
Results: []SearchResult{{Title: "A"}},
}
s.normalizeResponse(result, &SearchArgs{})
if result.NumberOfResults != 42 {
t.Fatalf("NumberOfResults = %d, want 42", result.NumberOfResults)
}
})
t.Run("applies limit", func(t *testing.T) {
t.Parallel()
limit := 1
s := &SearXNGSearcher{debug: false}
result := &SearchResponse{
Results: []SearchResult{
{Title: "A"},
{Title: "B"},
{Title: "C"},
},
}
s.normalizeResponse(result, &SearchArgs{Limit: &limit})
if len(result.Results) != 1 {
t.Fatalf("len(Results) = %d, want 1", len(result.Results))
}
if result.Results[0].Title != "A" {
t.Fatalf("Results[0].Title = %q, want %q", result.Results[0].Title, "A")
}
})
t.Run("limit larger than results does not truncate", func(t *testing.T) {
t.Parallel()
limit := 10
s := &SearXNGSearcher{debug: false}
result := &SearchResponse{
Results: []SearchResult{
{Title: "A"},
{Title: "B"},
},
}
s.normalizeResponse(result, &SearchArgs{Limit: &limit})
if len(result.Results) != 2 {
t.Fatalf("len(Results) = %d, want 2", len(result.Results))
}
})
t.Run("debug mode includes unresponsive engines", func(t *testing.T) {
t.Parallel()
s := &SearXNGSearcher{debug: true}
result := &SearchResponse{}
s.normalizeResponse(result, &SearchArgs{})
if !result.Debug {
t.Fatal("Debug = false, want true")
}
if result.UnresponsiveEngines == nil {
t.Fatal("UnresponsiveEngines is nil, want empty slice")
}
})
t.Run("non-debug mode excludes unresponsive engines", func(t *testing.T) {
t.Parallel()
s := &SearXNGSearcher{debug: false}
result := &SearchResponse{}
s.normalizeResponse(result, &SearchArgs{})
if result.UnresponsiveEngines != nil {
t.Fatal("UnresponsiveEngines is not nil, want nil in non-debug mode")
}
})
t.Run("applies typed fallbacks before deduplication", func(t *testing.T) {
t.Parallel()
tests := []struct {
name string
fixture string
fallback func(*Answer) string
assertTypedPayload func(*testing.T, Answer)
}{
{
name: "translation",
fixture: "typed_translation_answer.json",
fallback: answer.TranslationAnswerFallback,
assertTypedPayload: func(t *testing.T, got Answer) {
t.Helper()
if len(got.Translations) == 0 || got.Translations[0].Text == "" {
t.Fatal("Translations is empty after normalization, want typed translation payload preserved")
}
},
},
{
name: "weather without summary falls back to components",
fixture: "typed_weather_no_summary_fallback.json",
fallback: answer.WeatherAnswerFallback,
assertTypedPayload: func(t *testing.T, got Answer) {
t.Helper()
if got.Current == nil || got.Current.Location.Name == "" {
t.Fatal("Current is empty after normalization, want typed weather payload preserved")
}
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
var result SearchResponse
testhelper.LoadJSONFixture(t, filepath.Join("..", "..", "testdata", tt.fixture), &result)
if len(result.Answers) != 1 {
t.Fatalf("fixture has %d answers, want 1", len(result.Answers))
}
wantAnswer := tt.fallback(&result.Answers[0])
// A non-empty infobox forces the deduplication pass. If
// deduplication runs before fallback derivation, it drops the
// typed answer while its legacy Answer field is still empty.
result.Infoboxes = []Infobox{{Infobox: "Other", Content: "unrelated content"}}
s := &SearXNGSearcher{debug: false}
s.normalizeResponse(&result, &SearchArgs{})
if len(result.Answers) != 1 {
t.Fatalf("len(Answers) = %d, want 1 typed answer to survive normalization", len(result.Answers))
}
got := result.Answers[0]
if got.Answer != wantAnswer {
t.Fatalf("Answer = %q, want canonical fallback %q", got.Answer, wantAnswer)
}
tt.assertTypedPayload(t, got)
})
}
})
t.Run("truncates answers to MaxAnswers", func(t *testing.T) {
t.Parallel()
s := &SearXNGSearcher{debug: false}
answers := make([]Answer, MaxAnswers+25)
for i := range answers {
answers[i] = Answer{Answer: "answer " + strings.Repeat("x", i%16), Engine: "e"}
}
result := &SearchResponse{Answers: answers}
s.normalizeResponse(result, &SearchArgs{})
if len(result.Answers) != MaxAnswers {
t.Fatalf("len(Answers) = %d, want %d", len(result.Answers), MaxAnswers)
}
})
t.Run("truncates infoboxes to MaxInfoboxes", func(t *testing.T) {
t.Parallel()
s := &SearXNGSearcher{debug: false}
infoboxes := make([]Infobox, MaxInfoboxes+25)
for i := range infoboxes {
infoboxes[i] = Infobox{Infobox: "topic", Content: "content"}
}
result := &SearchResponse{Infoboxes: infoboxes}
s.normalizeResponse(result, &SearchArgs{})
if len(result.Infoboxes) != MaxInfoboxes {
t.Fatalf("len(Infoboxes) = %d, want %d", len(result.Infoboxes), MaxInfoboxes)
}
})
t.Run("does not truncate when at or below cap", func(t *testing.T) {
t.Parallel()
s := &SearXNGSearcher{debug: false}
answers := make([]Answer, MaxAnswers)
for i := range answers {
answers[i] = Answer{Answer: "a"}
}
infoboxes := make([]Infobox, MaxInfoboxes)
for i := range infoboxes {
infoboxes[i] = Infobox{Infobox: "t", Content: "c"}
}
result := &SearchResponse{Answers: answers, Infoboxes: infoboxes}
s.normalizeResponse(result, &SearchArgs{})
if len(result.Answers) != MaxAnswers {
t.Fatalf("len(Answers) = %d, want %d", len(result.Answers), MaxAnswers)
}
if len(result.Infoboxes) != MaxInfoboxes {
t.Fatalf("len(Infoboxes) = %d, want %d", len(result.Infoboxes), MaxInfoboxes)
}
})
t.Run("caps answers before deduplication", func(t *testing.T) {
t.Parallel()
s := &SearXNGSearcher{debug: false}
// The retained prefix is all duplicates, while answers beyond the
// cap are distinct. If deduplication ran first, those later answers
// would survive and evade the cap after the duplicate prefix vanished.
wiki := "Apple Inc. is an American multinational technology company headquartered in Cupertino, California."
answers := make([]Answer, MaxAnswers+5)
for i := range MaxAnswers {
answers[i] = Answer{Answer: wiki, Engine: "duckduckgo"}
}
for i := MaxAnswers; i < len(answers); i++ {
answers[i] = Answer{Answer: "distinct answer", Engine: "calculator"}
}
infoboxes := []Infobox{{Infobox: "Apple Inc.", Content: wiki}}
result := &SearchResponse{Answers: answers, Infoboxes: infoboxes}
s.normalizeResponse(result, &SearchArgs{})
if len(result.Answers) != 0 {
t.Fatalf("len(Answers) = %d, want 0 after capping away distinct answers then deduplicating", len(result.Answers))
}
})
t.Run("caps infoboxes before deduplication", func(t *testing.T) {
t.Parallel()
infoboxes := make([]Infobox, MaxInfoboxes+1)
for i := range MaxInfoboxes {
infoboxes[i] = Infobox{Infobox: "Other", Content: "unrelated content"}
}
infoboxes[MaxInfoboxes] = Infobox{Infobox: "Match", Content: "answer text appears only beyond the cap"}
result := &SearchResponse{
Answers: []Answer{{Answer: "answer text", Engine: "duckduckgo"}},
Infoboxes: infoboxes,
}
s := &SearXNGSearcher{debug: false}
s.normalizeResponse(result, &SearchArgs{})
if len(result.Answers) != 1 {
t.Fatalf("len(Answers) = %d, want 1 because the only matching infobox is beyond the cap", len(result.Answers))
}
})
t.Run("pathological answers/infoboxes count terminates quickly", func(t *testing.T) {
t.Parallel()
// Build a response with many more answers and infoboxes than the
// cap. The model artifact in CAND-33fe0b85-RUNTIME-003 shows
// ~47k entries each fitting under MaxResponseBodySize. The cap
// must keep the dedup work bounded; we assert a generous time
// budget to catch unbounded regressions without flaking under
// load.
const oversized = 10 * MaxAnswers
const timeBudget = 2 * time.Second
answers := make([]Answer, oversized)
for i := range answers {
answers[i] = Answer{Answer: "answer text", Engine: "e"}
}
infoboxes := make([]Infobox, oversized)
for i := range infoboxes {
infoboxes[i] = Infobox{Infobox: "t", Content: "content"}
}
result := &SearchResponse{Answers: answers, Infoboxes: infoboxes}
s := &SearXNGSearcher{debug: false}
start := time.Now()
s.normalizeResponse(result, &SearchArgs{})
elapsed := time.Since(start)
if elapsed > timeBudget {
t.Fatalf("normalizeResponse took %v, want < %v (pathological input must stay bounded)", elapsed, timeBudget)
}
if len(result.Answers) > MaxAnswers {
t.Fatalf("len(Answers) = %d, want <= %d", len(result.Answers), MaxAnswers)
}
if len(result.Infoboxes) > MaxInfoboxes {
t.Fatalf("len(Infoboxes) = %d, want <= %d", len(result.Infoboxes), MaxInfoboxes)
}
})
}
// --- MarshalJSON tests ---
func TestMarshalSearchResponse_DebugUnresponsiveEngines(t *testing.T) {
t.Parallel()
resp := SearchResponse{
Query: "test",
Warning: ExternalContentWarning,
Results: []SearchResult{},
Suggestions: []string{},
UnresponsiveEngines: [][]string{{"brave", "msg"}},
Debug: true,
}
data, err := resp.MarshalJSON()
if err != nil {
t.Fatalf("MarshalJSON() error = %v", err)
}
jsonStr := string(data)
if !strings.Contains(jsonStr, `"brave","msg"`) {
t.Fatalf("MarshalJSON() output does not contain unresponsive engine entries\njson:\n%s", jsonStr)
}
if !strings.Contains(jsonStr, `"unresponsive_engines":[`) {
t.Fatalf("MarshalJSON() unresponsive_engines field is not an array\njson:\n%s", jsonStr)
}
}
// --- Logger routing regression tests ---
// These tests verify that response-processing diagnostics route through
// the configured Config.Logger rather than the package-global slog.
func TestNormalizeResponseLoggerRouting(t *testing.T) {
t.Parallel()
t.Run("answer truncation warning goes to configured logger", func(t *testing.T) {
t.Parallel()
var buf bytes.Buffer
logger := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelWarn}))
s := &SearXNGSearcher{logger: logger}
answers := make([]Answer, MaxAnswers+1)
for i := range answers {
answers[i] = Answer{Answer: "test", Engine: "e"}
}
result := &SearchResponse{Answers: answers}
s.normalizeResponse(result, &SearchArgs{})
if !strings.Contains(buf.String(), "truncating answers before deduplication") {
t.Fatal("expected truncation warning in configured logger, got none")
}
})
t.Run("infobox truncation warning goes to configured logger", func(t *testing.T) {
t.Parallel()
var buf bytes.Buffer
logger := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelWarn}))
s := &SearXNGSearcher{logger: logger}
infoboxes := make([]Infobox, MaxInfoboxes+1)
for i := range infoboxes {
infoboxes[i] = Infobox{Infobox: "topic", Content: "content"}
}
result := &SearchResponse{Infoboxes: infoboxes}
s.normalizeResponse(result, &SearchArgs{})
if !strings.Contains(buf.String(), "truncating infoboxes before deduplication") {
t.Fatal("expected infobox truncation warning in configured logger, got none")
}
})
t.Run("answer truncation does not go to configured logger when below cap", func(t *testing.T) {
t.Parallel()
var buf bytes.Buffer
logger := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelWarn}))
s := &SearXNGSearcher{logger: logger}
answers := make([]Answer, MaxAnswers)
for i := range answers {
answers[i] = Answer{Answer: "test", Engine: "e"}
}
result := &SearchResponse{Answers: answers}
s.normalizeResponse(result, &SearchArgs{})
if buf.Len() != 0 {
t.Fatal("expected no log output when answers are at or below cap")
}
})
t.Run("infobox truncation does not go to configured logger when below cap", func(t *testing.T) {
t.Parallel()
var buf bytes.Buffer
logger := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelWarn}))
s := &SearXNGSearcher{logger: logger}
infoboxes := make([]Infobox, MaxInfoboxes)
for i := range infoboxes {
infoboxes[i] = Infobox{Infobox: "topic", Content: "content"}
}
result := &SearchResponse{Infoboxes: infoboxes}
s.normalizeResponse(result, &SearchArgs{})
if buf.Len() != 0 {
t.Fatal("expected no log output when infoboxes are at or below cap")
}
})
}
func TestHTMLResponseErrorLoggerRouting(t *testing.T) {
t.Parallel()
var buf bytes.Buffer
logger := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))
s := &SearXNGSearcher{
logger: logger,
}
resp := &http.Response{
StatusCode: http.StatusOK,
Header: http.Header{"Content-Type": []string{"text/html"}},
Body: io.NopCloser(strings.NewReader("<html><body>JSON not enabled</body></html>")),
}
_, err := s.parseSearchResponse(resp, &SearchArgs{})
if err == nil {
t.Fatal("expected HTMLResponseError for HTML response")
}
if !strings.Contains(buf.String(), "HTMLResponseError: received HTML instead of JSON") {
t.Fatal("expected HTMLResponseError log in configured logger, got none")
}
}
func TestCloseResponseBodyLoggerRouting(t *testing.T) {
t.Parallel()
var buf bytes.Buffer
logger := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))
body := &errorCloseReader{}
resp := &http.Response{Body: body}
closeResponseBody(resp, logger)
if !strings.Contains(buf.String(), "failed to close response body") {
t.Fatal("expected close error log in configured logger, got none")
}
}
func TestLogDebugBodyLoggerRouting(t *testing.T) {
t.Parallel()
var buf bytes.Buffer
logger := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))
s := &SearXNGSearcher{debug: true, logger: logger}
resp := &http.Response{
StatusCode: http.StatusOK,
Header: http.Header{"Content-Type": []string{"application/json"}},
}
body := []byte(`{"key": "value"}`)
s.logDebugBody(resp, body)
if !strings.Contains(buf.String(), "HTTP response body") {
t.Fatal("expected HTTP response body log in configured logger, got none")
}
}
func TestDecodeSearchResponseLoggerRouting(t *testing.T) {
t.Parallel()
t.Run("unexpected content type logs to configured logger", func(t *testing.T) {
t.Parallel()
var buf bytes.Buffer
logger := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))
resp := &http.Response{StatusCode: http.StatusOK}
_, err := decodeSearchResponse(resp, "text/plain", []byte("not json"), logger)
if err == nil {
t.Fatal("expected error for text/plain content type")
}
if !strings.Contains(buf.String(), "UnexpectedContentTypeError") {
t.Fatal("expected UnexpectedContentTypeError log in configured logger, got none")
}
})
t.Run("JSON parse error logs to configured logger", func(t *testing.T) {
t.Parallel()
var buf bytes.Buffer
logger := slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))
resp := &http.Response{StatusCode: http.StatusOK}
_, err := decodeSearchResponse(resp, "application/json", []byte("{invalid}"), logger)
if err == nil {
t.Fatal("expected error for invalid JSON")
}
if !strings.Contains(buf.String(), "JSONParseError") {
t.Fatal("expected JSONParseError log in configured logger, got none")
}
})
}
// FuzzDecodeSearchResponse feeds arbitrary bytes through the full SearXNG
// response parsing path (body reading, HTML detection, content-type handling,
// JSON decoding, and normalization). The fuzz input encodes a content-type
// string and a body separated by a single null byte (0x00). If no null byte
// is present, the entire input is treated as the body with a default content
// type of "application/json". Inputs larger than MaxResponseBodySize are
// ignored to keep memory bounded.
func FuzzDecodeSearchResponse(f *testing.F) {
// Seed corpus entries are content-type + null byte + body.
addSeed := func(ct, body string) {
f.Add([]byte(ct + "\x00" + body))
}
addSeed("application/json", `{}`)
addSeed("application/json", `{"query":"fuzz"}`)
addSeed("application/json", `{"results":[]}`)
addSeed("application/json",
`{"results":[{"title":"t","url":"https://example.com","content":"c","engine":"e"}]}`)
addSeed("application/json", `{"suggestions":[]}`)
addSeed("application/json", `{"number_of_results":42}`)
addSeed("application/json", `{"answers":[{"answer":"a"}]}`)
addSeed("application/json", `{"infoboxes":[{"infobox":"i","content":"c"}]}`)
addSeed("application/json", `{`)
addSeed("application/json", `{invalid}`)
addSeed("application/json", `[]`)
addSeed("application/json", `""`)
addSeed("application/json", `null`)
addSeed("application/json", `{"results":null}`)
addSeed("application/json", `<html></html>`)
addSeed("application/json", `{"results":[{"title":"`+strings.Repeat("a", 1000)+`"}]}`)
// Content-type seeds.
addSeed("application/json; charset=utf-8", `{"query":"charset"}`)
addSeed("text/html", `<html></html>`)
addSeed("text/plain", `not json`)
addSeed("", `{"query":"empty-ct"}`)
// High-value seeds: duplicate JSON keys.
addSeed("application/json",
`{"results":[{"title":"a"}], "results":[{"title":"b"}]}`)
// High-value seeds: very large numeric values.
addSeed("application/json", `{"number_of_results": 999999999999999999999}`)
// High-value seeds: Unicode control characters / null bytes in strings.
addSeed("application/json", `{"query":"hello\u0000world"}`)
// High-value seeds: nested JSON payload.
addSeed("application/json",
`{"results":[{"title":"a"}],"infoboxes":[{"infobox":"i","content":"c","attributes":[{"key":"k","value":"v"}]}]}`)
// Seed with an actual null byte (not just JSON \u0000 escape) in the body.
f.Add([]byte("application/json\x00{\"query\":\"hello\x00world\"}"))
f.Fuzz(func(t *testing.T, data []byte) {
if len(data) > MaxResponseBodySize {
return
}
// Split on first null byte to separate content-type from body.
ctBytes, body, ok := bytes.Cut(data, []byte{0})
if !ok {
ctBytes = nil
body = data
}
contentType := string(ctBytes)
resp := &http.Response{
StatusCode: http.StatusOK,
Header: http.Header{"Content-Type": []string{contentType}},
Body: io.NopCloser(bytes.NewReader(body)),
}
logger := slog.New(slog.DiscardHandler)
searcher := &SearXNGSearcher{logger: logger}
result, err := searcher.parseSearchResponse(resp, &SearchArgs{})
if err != nil {
return
}
if result.Results == nil {
t.Errorf("Results is nil after normalization")
}
if result.Suggestions == nil {
t.Errorf("Suggestions is nil after normalization")
}
if result.Warning != ExternalContentWarning {
t.Errorf("Warning = %q, want %q", result.Warning, ExternalContentWarning)
}
})
}
var errMockClose = errors.New("mock close error")
// errorCloseReader implements io.ReadCloser and returns an error on Close.
type errorCloseReader struct{}
func (e *errorCloseReader) Read(_ []byte) (int, error) {
return 0, io.EOF
}
func (e *errorCloseReader) Close() error {
return errMockClose
}