hermes-agent/tests/agent/test_degenerate_final_recovery.py
kshitijk4poor 112446afd1 fix(agent): judge "wrong script" against the user's own message; one continuation kind per stop
Review findings on the first cut:

* The letters-but-no-ASCII arm flagged every terse non-Latin answer
  ("是", "Готово", "はい", "تم") after tool work, so a Chinese/Russian
  conversation on the Responses transport would pay a re-prompt on most
  one-word answers. "Wrong script" now means wrong relative to the
  conversation: when the user's message carries non-ASCII letters, a
  non-Latin reply is an answer. "пар" after an English prompt still counts.
* The leading-punctuation arm caught ":8080", ":)", ";;", ":=", "}". It now
  requires a letter right after the punctuation ("?warming", "!warming").
* finalize_turn chose the continuation kind twice (log ladder and nudge
  pick); it is computed once, the tool-row count too.
* tool_results_this_turn documents the load-bearing invariant: any user
  row (the nudges included) closes the window, which is what bounds the
  guard to one re-prompt per collapse.

Tests: user-script negatives in Chinese/Russian/Japanese/Arabic incl. a
multi-part user message; ":8080"/":)"/";;"/"}" negatives; both red when
the script check or the letter-after rule is dropped.
2026-09-19 12:12:04 +05:30

132 lines
6.2 KiB
Python

"""Degenerate-final recovery (#103483).
Provider-side collapse: a turn executes its tool work correctly, then ends on ``finish_reason=stop``
with an ENTIRE visible answer that is a fragment — a stray wrong-script word (``пар``), a token
starting mid-punctuation (``?warming up``). Before the fix the loop accepted the fragment as the
answer, the turn reported ``completed``, and an unattended job silently abandoned the task.
The guard rides the existing ack-continuation path: same scope knob (``agent.intent_ack_continuation``,
default ``auto`` = Responses transports), same bounded per-turn counter, same durable interim +
nudge rows. Shape alone cannot prove a collapse, so the predicate is narrow and the nudge asks for
the same answer again when it WAS complete — a false positive costs one call, never the answer.
"""
from __future__ import annotations
from unittest.mock import MagicMock, patch
import pytest
from agent.agent_runtime_helpers import looks_like_degenerate_final
from agent.conversation_loop import _DEGENERATE_FINAL_NUDGE
@pytest.fixture()
def loop_agent():
"""AIAgent with a mocked OpenAI client (mirrors test_dropped_tool_call_recovery)."""
from run_agent import AIAgent
with (
patch("model_tools.get_tool_definitions", return_value=[]),
patch("model_tools.check_toolset_requirements", return_value={}),
patch("agent.process_bootstrap.OpenAI"),
):
agent = AIAgent(
api_key="test-key-1234567890",
base_url="https://openrouter.ai/api/v1",
quiet_mode=True,
skip_context_files=True,
skip_memory=True,
)
agent.client = MagicMock()
agent._cached_system_prompt = "You are helpful."
agent._use_prompt_caching = False
agent.tool_delay = 0
agent.compression_enabled = False
agent.save_trajectories = False
# Explicit opt-in: the loop tests exercise the mechanism, not the transport-scoped default.
agent._intent_ack_continuation = True
return agent
def _run(agent, stages):
agent.client.chat.completions.create.side_effect = stages
with (
patch.object(agent, "_persist_session"),
patch.object(agent, "_save_trajectory"),
patch.object(agent, "_cleanup_task_resources"),
patch("model_tools.handle_function_call", return_value="ok"),
):
return agent.run_conversation("build the workbook")
def _tool_round(call_id):
from tests.agent.test_run_agent import _mock_response, _mock_tool_call
return _mock_response(
content="", finish_reason="tool_calls",
tool_calls=[_mock_tool_call(name="web_search", arguments="{}", call_id=call_id)],
)
def _final(text):
from tests.agent.test_run_agent import _mock_response
return _mock_response(content=text, finish_reason="stop")
def _user_rows_sent(agent, call_index):
kwargs = agent.client.chat.completions.create.call_args_list[call_index].kwargs
return [m["content"] for m in kwargs["messages"] if m.get("role") == "user"]
class TestFragmentAfterToolWork:
def test_fragment_after_tool_work_reprompts_instead_of_exiting(self, loop_agent):
result = _run(loop_agent, [_tool_round("call_1"), _final("пар"), _final("Workbook built: 3 sheets.")])
assert result["final_response"] == "Workbook built: 3 sheets."
assert loop_agent.client.chat.completions.create.call_count == 3
# The re-prompt carried the degenerate-final nudge, not the generic ack nudge.
assert _user_rows_sent(loop_agent, 2)[-1] == _DEGENERATE_FINAL_NUDGE
# The fragment stays in the transcript as an interim row, the nudge right after it.
contents = [m.get("content") for m in result["messages"]]
assert contents.index("пар") + 1 == contents.index(_DEGENERATE_FINAL_NUDGE)
def test_persistent_collapse_is_bounded_and_keeps_the_fragment(self, loop_agent):
"""One re-prompt per collapse: the nudge row closes the tool-work window, so a second
fragment is the answer — today's behaviour, never a loop."""
result = _run(loop_agent, [_tool_round("call_1"), _final("пар"), _final("?warming up")])
assert loop_agent.client.chat.completions.create.call_count == 3
assert result["final_response"] == "?warming up"
def test_fragment_without_tool_work_is_a_plain_answer(self, loop_agent):
"""A chat-only turn that answers tersely is not a collapse; nothing is re-prompted."""
result = _run(loop_agent, [_final("пар")])
assert loop_agent.client.chat.completions.create.call_count == 1
assert result["final_response"] == "пар"
class TestLooksLikeDegenerateFinal:
EN = "Build the workbook and report back."
@pytest.mark.parametrize("text", ["пар", "?warming up", ",and", "ошибка", "!warming"])
def test_fragments_in_an_english_conversation(self, text):
assert looks_like_degenerate_final(text, user_message=self.EN)
@pytest.mark.parametrize("text", [
# Terse legitimate answers that reviewers showed a shape-only guard re-prompting (#111472).
"42", "SQLite", "report.csv", "3.14159", "€12.50", "你好。", "Done.", "True", "a51143fbbe",
"2026-09-19", "/tmp/out.log", "$5", "#123", "-1", "(a)", ".env", "~/x", "+1", "✅",
":8080", "::1", ":)", ";;", ":=", "}", "N/A", "**Done**", "`ok`", "{}", "null",
"Reading is a skill", "The answer is 43.", "x" * 25, "",
])
def test_terse_answers_are_not_fragments(self, text):
assert not looks_like_degenerate_final(text, user_message=self.EN)
@pytest.mark.parametrize("prompt, text", [
("把工作簿建好然后告诉我", "是"), ("把工作簿建好然后告诉我", "已完成"),
("Собери таблицу и отчитайся", "Готово"), ("Собери таблицу и отчитайся", "да"),
("ワークブックを作って報告して", "はい"), ("أنشئ المصنف ثم أخبرني", "تم"),
([{"type": "text", "text": "Собери таблицу"}], "нет"), # multi-part user content
])
def test_terse_answers_in_the_users_own_script_are_not_fragments(self, prompt, text):
assert not looks_like_degenerate_final(text, user_message=prompt)