mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-16 19:36:54 +00:00
Every hard interrupt reached `begin_iteration` through the same `_interrupt_requested` flag, so the turn loop booked all of them as `interrupted_by_user` — a cron run killed by the scheduler's inactivity watchdog, a turn aborted by the liveness watchdog, a lost session turn lease and a gateway inactivity timeout all read as a human pressing stop, and the investigation went to the wrong subsystem (#112647). `interrupt()` already records a trusted category per interrupt (`_tool_interrupt_reason`, fed by the `tool_reason` every producer can pass through `request_hard_interrupt`). The exit reason is now derived from it: the three categories `interrupt()` itself mints for human stops keep `interrupted_by_user` / `interrupted_during_api_call`; any other category names its producer — `interrupted_by_system(cron_inactivity_watchdog)`, `interrupted_during_api_call(turn_liveness_watchdog)`. The system producers that passed no `tool_reason` (cron inactivity watchdog, turn liveness watchdog, lease loss, gateway inactivity timeout) now name themselves, so the model-visible tool-cancellation text says the same thing. `_publish_interrupt_state` logs ONE line naming the source so the turn record and the log agree. No change to WHEN anything interrupts. `interrupted_during_api_call` moves to the prefix-matched explanation table so the parameterised form keeps its user copy. Slim redo of #112652 by @KoNit-K, which added a parallel `issuer` attribute and keyword; this reuses the existing `tool_reason` plumbing instead. Co-authored-by: KoNit-K <[email protected]>
92 lines
4.3 KiB
Python
92 lines
4.3 KiB
Python
"""``apply_retry_restarts`` bounds the refunding restarts (#106108).
|
|
|
|
The redirect and rebuilt-for-fallback paths refund the iteration budget and re-issue the
|
|
iteration; nothing else in the turn loop counts them, so a flag that keeps re-arming
|
|
(a request cancelled on every attempt) refunded forever and held the turn lease.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
from agent.turn_iteration_prep import apply_retry_restarts, begin_iteration
|
|
from agent.turn_retry_state import TurnRetryState
|
|
|
|
RESTART_FLAGS = ["restart_with_redirected_messages", "restart_with_rebuilt_messages"]
|
|
MAX_RETRIES = 3
|
|
|
|
|
|
def _apply(agent, flag: str, restart_count: int):
|
|
_retry = TurnRetryState()
|
|
setattr(_retry, flag, True)
|
|
return apply_retry_restarts(
|
|
agent, _retry=_retry, response=None, interrupted=False, messages=[],
|
|
conversation_history=[], user_message="hi", api_kwargs={}, current_turn_user_idx=0,
|
|
final_response=None, retry_count=0, max_retries=MAX_RETRIES, api_call_count=1,
|
|
restart_count=restart_count, length_continue_retries=0,
|
|
_preflight_compression_blocked=True, _turn_exit_reason="unknown",
|
|
)
|
|
|
|
|
|
def _agent():
|
|
budget = SimpleNamespace(refunds=0)
|
|
budget.refund = lambda: setattr(budget, "refunds", budget.refunds + 1)
|
|
agent = SimpleNamespace(iteration_budget=budget, steered=[])
|
|
agent._drain_pending_redirect = lambda: "last correction"
|
|
agent.steer = agent.steered.append
|
|
return agent
|
|
|
|
|
|
@pytest.mark.parametrize("flag", RESTART_FLAGS)
|
|
def test_single_restart_still_reissues_the_iteration(flag):
|
|
"""A lone correction / fallback activation keeps its refund-and-continue contract."""
|
|
agent = _agent()
|
|
verdict = _apply(agent, flag, restart_count=0)
|
|
assert verdict.action == "continue"
|
|
assert (agent.iteration_budget.refunds, verdict.api_call_count) == (1, 0)
|
|
if flag == "restart_with_rebuilt_messages":
|
|
assert verdict._preflight_compression_blocked is False # still the single consumer
|
|
|
|
|
|
@pytest.mark.parametrize("flag", RESTART_FLAGS)
|
|
def test_restart_refunds_are_bounded_per_turn(flag):
|
|
"""Re-arming the flag every iteration breaks after ``max_retries`` refunds instead of
|
|
refunding forever (the turn ends, so the session turn lease is released)."""
|
|
agent = _agent()
|
|
restart_count, verdicts = 0, []
|
|
while len(verdicts) < MAX_RETRIES + 5 and (not verdicts or verdicts[-1].action != "break"):
|
|
verdicts.append(_apply(agent, flag, restart_count))
|
|
restart_count = verdicts[-1].restart_count
|
|
assert [v.action for v in verdicts] == ["continue"] * MAX_RETRIES + ["break"]
|
|
assert agent.iteration_budget.refunds == MAX_RETRIES
|
|
assert verdicts[-1]._turn_exit_reason.endswith("restart_limit_exceeded")
|
|
# The correction that tripped the redirect cap is handed back as the next user turn.
|
|
assert agent.steered == (["last correction"] if flag == "restart_with_redirected_messages" else [])
|
|
|
|
|
|
def _interrupted_agent(tool_interrupt_reason):
|
|
agent = SimpleNamespace(
|
|
_interrupt_requested=True, _tool_interrupt_reason=tool_interrupt_reason, quiet_mode=True,
|
|
_drain_pending_redirect=lambda: None, _checkpoint_mgr=SimpleNamespace(new_turn=lambda: None),
|
|
)
|
|
return begin_iteration(
|
|
agent, messages=[], conversation_history=[], original_user_message="hi",
|
|
api_call_count=0, interrupted=False, _turn_exit_reason="unknown",
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("tool_interrupt_reason, expected", [
|
|
# Human stops keep the historical reason (every ``interrupt()`` category, and a bare flag write).
|
|
("explicit stop requested", "interrupted_by_user"),
|
|
("user sent a new message", "interrupted_by_user"),
|
|
(None, "interrupted_by_user"),
|
|
# A producer that named itself via ``tool_reason`` is booked as the issuer (#112647).
|
|
("cron inactivity watchdog", "interrupted_by_system(cron_inactivity_watchdog)"),
|
|
("turn liveness watchdog", "interrupted_by_system(turn_liveness_watchdog)"),
|
|
])
|
|
def test_interrupt_exit_reason_names_the_system_issuer(tool_interrupt_reason, expected):
|
|
"""A watchdog abort must not be recorded as a user stop: the exit reason carries the issuer."""
|
|
verdict = _interrupted_agent(tool_interrupt_reason)
|
|
assert (verdict.action, verdict.interrupted, verdict._turn_exit_reason) == ("break", True, expected)
|