hermes-agent/tests/agent/test_repetition_guard.py
kshitijk4poor d38af0fcab fix(agent): keep window-count repetition scan alongside periodic runs
The periodic-run detector only matched exact repeats, so counter/noise loops
(#86581 shape) passed. OR it with main's window-count scan, and memoise
rejected runs so anchors inside them skip multiple-period re-expansion.
2026-09-24 22:32:05 +05:30

88 lines
3.9 KiB
Python

"""Unit tests for repetition-dominated model output detection."""
from __future__ import annotations
import random
import pytest
from agent.repetition_guard import MIN_FRAGMENT_LENGTH, is_repetition_dominated
# The exact sentence from the #86581 incident (echoed hundreds of times by
# the model before the provider cut it off at finish_reason=length).
_INCIDENT_ECHO = "好,你幫我更改成 Google Gemini 4 31B。"
class TestRepetitionGuard:
def test_incident_shape_flags_repetition(self):
# Narration + the echoed sentence on its own line, repeated (line path).
text = ("We need to verify the model setting.\n" + _INCIDENT_ECHO + "\n") * 800
assert is_repetition_dominated(text) is True
def test_repeated_sentence_without_line_breaks_flags(self):
# Repetition loop with no line breaks — exercises the window path.
text = _INCIDENT_ECHO * 2000
assert len(text) >= MIN_FRAGMENT_LENGTH
assert is_repetition_dominated(text) is True
def test_multiline_paragraph_run_uses_true_period_coverage(self):
rng = random.Random(11)
paragraph = "\n".join(
"".join(rng.choice("abcdefghijklmnopqrstuvwxyz ") for _ in range(151))
for _ in range(5)
) + "\n"
# The incident unit was approximately 764 characters. Detection must
# remain scale-free as the number of exact repeats grows.
for repeat_count in (100, 1_000, 10_000):
assert is_repetition_dominated(paragraph * repeat_count) is True
@pytest.mark.parametrize("shape", ["unique_prefix_suffix", "counter_loop"])
def test_dominant_run_with_unique_prefix_and_suffix_flags(self, shape):
if shape == "counter_loop":
# A changing counter breaks exact periodicity; main's window scan (#86581) must
# still flag it.
text = "".join(
f"Step {i}: I will now carefully re-check the configuration file for the error again.\n"
for i in range(200)
)
assert is_repetition_dominated(text) is True
return
paragraph = (
"A deliberately long repeated paragraph has enough distinct text "
"to make its period exceed the guard's minimum anchor length.\n"
"It also spans multiple lines, matching the real incident shape.\n"
)
repeated = paragraph * 20
text = ("unique introduction " * 20) + repeated + (" unique ending" * 20)
assert len(repeated) > len(text) * 0.5
assert is_repetition_dominated(text) is True
def test_long_legitimate_text_not_flagged(self):
# Long, unique prose — no 60-char window ever repeats.
text = " ".join(
f"Sentence number {i} describes a distinct topic with unique words "
f"such as quasar-{i} and nebula-{i} to keep every window distinct."
for i in range(1200)
)
assert len(text) >= MIN_FRAGMENT_LENGTH
assert is_repetition_dominated(text) is False
def test_short_fragment_never_flagged(self):
# Below MIN_FRAGMENT_LENGTH the guard fails open — short truncations
# are legitimately continued even if they look repetitive.
assert is_repetition_dominated("A. " * 50) is False
assert is_repetition_dominated("hello ") is False
def test_repeat_not_dominant_not_flagged(self):
# A repeated sentence scattered through a long unique text: repeated
# windows exist but cover far less than half of the fragment.
filler = " ".join(f"unique filler token {i}" for i in range(3000))
text = filler + ("\n" + _INCIDENT_ECHO + "\n") * 30
assert is_repetition_dominated(text) is False
def test_non_string_inputs(self):
assert is_repetition_dominated("") is False
assert is_repetition_dominated(None) is False
assert is_repetition_dominated(12345) is False