mirror of
https://github.com/xlionjuan/searxng-mcp-go.git
synced 2026-11-17 04:17:51 +00:00
* docs: align documentation with code behavior (issues #547-#551) - INSTALL.md: drop SEARXNG_MAX_RETRIES=-1 from the unparsable-value example list; cross-reference the hard-error semantic path - PROMPT_INJECTION_SAFETY.md: mark Option A (warning header + warning field) as implemented, referencing #448 and #50 - MCP_TESTING.md: E2E binary is built once into a package-level cache, not per test; e2eMCPEnv lives in e2e_helper_test.go; goroutine-leak guard is suite-wide, not stress-file-specific - e2e-tests.md: fix workflow line citations (:87, :104) - verification.md: drop misleading '(except E2E)' qualifier; narrow the E2E-workflow coverage claim for exit-code tests - test-server.md: add missing test-server-pid-helper recipe row - .env.example: document SEARXNG_ALLOW_GET_FALLBACK opt-in - MCP_TOOLS.md: include 'failed to execute search request' segment in the network-failure error format - OUTPUT_FORMAT.md: sync Example 1 count line with the (showing N) format - adr/014: suppression was relocated to searchWithRetries, not removed - CONTEXT.md: note empty-text answers are dropped before the engine gate * docs: address review findings on doc accuracy (P2 items) - INSTALL.md: SEARXNG_MAX_RETRIES format is an integer in the range 0-20, not 'non-negative integer' — -1 parses and is rejected by SetMaxRetries, so the old wording conflated syntax and semantics - MCP_TESTING.md: example and E2E_MCP_BINARY section no longer claim a per-test build; list e2e_mockserver_test.go among the ungated CLI tests that build per-test via buildTestBinary - CONTEXT.md: dedup description notes the early return when no non-empty infobox text exists (original answers returned unchanged) * docs: fix remaining dedup and E2E_MCP_BINARY wording (P2) - CONTEXT.md: example dialogue no longer claims non-DuckDuckGo answers are never touched; empty answers are dropped before the engine check when dedup runs, matching the Search Behavior entry - test-server.md, verification.md: E2E_MCP_BINARY skips the MCP E2E package-level build, not the ungated CLI per-test builds * docs: scope dedup pass-through claim to non-empty answers SEARXNG_ANSWER_DEDUP.md and the DeduplicateAnswers doc comment both claimed non-DuckDuckGo answers always pass through. In fact, when dedup runs (non-empty infobox content exists), empty-text answers are dropped before the engine gate; only non-empty non-DuckDuckGo answers pass through. Aligns both with the implementation in deduplicate.go. * docs: fix dedup algorithm order in SEARXNG_ANSWER_DEDUP.md Document the empty-text filter running before the engine gate (matching deduplicate.go) and add the early-return precondition; sync the Edge Cases table so retain behavior is scoped to non-empty answers. * docs: qualify empty-answer drop with early-return precondition SEARXNG_ANSWER_DEDUP.md edge-case row and the deduplicateAnswers wrapper comment both stated empty-text answers are always filtered. When no non-empty infobox exists the implementation early-returns the original answers unchanged; scope the drop to dedup runs only.
93 lines
2.6 KiB
Go
93 lines
2.6 KiB
Go
package answer
|
|
|
|
import "strings"
|
|
|
|
const dedupPrefixRunes = 200
|
|
|
|
// DeduplicateAnswers filters out DuckDuckGo answers whose text is a prefix
|
|
// (substring) of any infobox content. For each DuckDuckGo answer, the
|
|
// exact-case match runs first; the lowercase index is built lazily on the
|
|
// first miss and reused for subsequent answers that also miss the exact-case
|
|
// check.
|
|
//
|
|
// Only answers whose Engine is "duckduckgo" are checked — when dedup runs,
|
|
// answers with empty text are dropped before the engine gate, and non-empty
|
|
// non-DuckDuckGo answers (including those with unknown Engine) pass through.
|
|
// This prevents discarding legitimate results like calculator answers or IP
|
|
// addresses that happen to appear in infobox text. The " More at Wikipedia"
|
|
// suffix stripping and 200-rune substring heuristic are specific to the
|
|
// DuckDuckGo Wikipedia duplicate pattern.
|
|
func DeduplicateAnswers(answers []Answer, infoboxContents []string) []Answer {
|
|
// Move collectInfoboxText (content filtering) inline.
|
|
filteredContents := make([]string, 0, len(infoboxContents))
|
|
for _, text := range infoboxContents {
|
|
if text != "" {
|
|
filteredContents = append(filteredContents, text)
|
|
}
|
|
}
|
|
|
|
if len(answers) == 0 || len(filteredContents) == 0 {
|
|
return answers
|
|
}
|
|
|
|
var lowerInfoboxTexts []string
|
|
|
|
filtered := make([]Answer, 0, len(answers))
|
|
for _, ans := range answers {
|
|
if ans.Answer == "" {
|
|
continue
|
|
}
|
|
|
|
if ans.Engine == "duckduckgo" {
|
|
if answerPrefixMatch(ans.Answer, filteredContents, false) {
|
|
continue
|
|
}
|
|
|
|
if lowerInfoboxTexts == nil {
|
|
lowerInfoboxTexts = buildLowerInfoboxTexts(filteredContents)
|
|
}
|
|
|
|
if answerPrefixMatch(ans.Answer, lowerInfoboxTexts, true) {
|
|
continue
|
|
}
|
|
}
|
|
|
|
filtered = append(filtered, ans)
|
|
}
|
|
|
|
return filtered
|
|
}
|
|
|
|
func buildLowerInfoboxTexts(infoboxTexts []string) []string {
|
|
lower := make([]string, 0, len(infoboxTexts))
|
|
for _, text := range infoboxTexts {
|
|
lower = append(lower, strings.ToLower(text))
|
|
}
|
|
|
|
return lower
|
|
}
|
|
|
|
// answerPrefixMatch reports whether the answer (or its lowercased form when
|
|
// lower is true), with the " More at Wikipedia" suffix stripped and
|
|
// truncated to dedupPrefixRunes characters, is a substring of any element
|
|
// of haystack.
|
|
func answerPrefixMatch(answer string, haystack []string, lower bool) bool {
|
|
needle := answer
|
|
suffix := " More at Wikipedia"
|
|
|
|
if lower {
|
|
needle = strings.ToLower(needle)
|
|
suffix = " more at wikipedia"
|
|
}
|
|
|
|
needle = strings.TrimSuffix(needle, suffix)
|
|
needle = TruncateRunes(needle, dedupPrefixRunes)
|
|
|
|
for _, text := range haystack {
|
|
if strings.Contains(text, needle) {
|
|
return true
|
|
}
|
|
}
|
|
|
|
return false
|
|
}
|