hermes-agent/tests/agent/test_stream_stale_circuit_breaker.py
teknium1 9863980c61 fix: a stale stream attempt advances the cross-turn breaker once, not per timer re-fire
The streaming stale monitor re-fires every stale window while its worker
has not started/dispatched the attempt yet or is still unwinding a kill,
and each re-fire bumped agent._consecutive_stale_streams. One provider
attempt could therefore count twice (or more), so a turn with four hung
attempts reached HERMES_STREAM_STALE_GIVEUP=5 and the breaker refused the
NEXT turn although the provider only ever saw four stale requests. The
non-streaming and inline watchdogs already count once per call.

Count each started stream attempt once (attempt 0, i.e. nothing started,
never counts), and let the interrupted-wait bump honour the same ledger.

This is the root cause of the intermittent
test_agent_turn_liveness[provider_hang] red on CI (fault calls 4, five
"Stream stale for 3s" kills, probe requests 0, breaker text "5
consecutive stale attempts"): on a busy runner the first attempt's
worker takes >3 s to reach the wire, the monitor kills it before
dispatch (tcp_force_closed=0) and again 3 s after dispatch.

Repro: a 3.6 s sleep before the worker opens its first stream
reproduces the CI signature 4/4 on origin/main and 0/6 with the fix;
under taskset CPU starvation the unmodified module goes red 2/3 on base.
2026-09-26 06:26:48 +05:30

202 lines
8.8 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Cross-turn stream-stale circuit breaker (issue #58962).
A session wedged against an unresponsive provider can hit the stale-stream
detector on every turn and loop forever, burning the full 180s×retries each
turn with no response (observed: 494 consecutive failures over 3+ days).
These tests cover the guard added to ``interruptible_streaming_api_call``:
- a session that has already tripped the consecutive-stale threshold short
circuits immediately (no network attempt, no 180s wait) with a clear error;
- a successful stream resets the consecutive-stale streak;
- a stale-stream kill increments the consecutive-stale streak.
The harness mirrors tests/agent/test_anthropic_stream_pool_cleanup.py.
"""
import threading
import httpx
import pytest
from unittest.mock import MagicMock
from types import SimpleNamespace
def _make_anthropic_agent(**kwargs):
from run_agent import AIAgent
defaults = dict(
api_key="test-key",
base_url="https://example.com/v1",
model="claude-opus-4-7",
quiet_mode=True,
skip_context_files=True,
skip_memory=True,
)
defaults.update(kwargs)
agent = AIAgent(**defaults)
agent.api_mode = "anthropic_messages"
agent._anthropic_client = MagicMock()
agent._anthropic_api_key = "test-anthropic-key"
# #67142: anthropic streams now run on a request-local client; route it to
# the test mock so .messages.stream is exercised.
agent._create_request_anthropic_client = lambda *a, **k: agent._anthropic_client
return agent
def _good_stream_cm():
"""Context manager whose stream completes (message_stop) and returns a valid message."""
cm = MagicMock()
stream = MagicMock()
stream.__iter__ = MagicMock(return_value=iter([SimpleNamespace(type="message_stop")]))
msg = MagicMock()
msg.content = []
msg.stop_reason = "end_turn"
msg.usage = SimpleNamespace(input_tokens=10, output_tokens=5)
stream.get_final_message = MagicMock(return_value=msg)
cm.__enter__ = MagicMock(return_value=stream)
cm.__exit__ = MagicMock(return_value=False)
return cm
class TestStreamStaleCircuitBreaker:
def test_interrupted_pre_response_wait_advances_streak(self, monkeypatch):
"""Qualified pre-response interrupts advance the breaker, while early
and mid-stream user cancellations remain neutral."""
from agent.chat_completion_helpers import (
_check_stale_giveup,
_record_interrupted_provider_wait,
)
monkeypatch.setenv("HERMES_STREAM_STALE_GIVEUP", "2")
agent = _make_anthropic_agent()
agent._consecutive_stale_streams = 0
assert _record_interrupted_provider_wait(agent, 29.9, response_started=False) is False
assert _record_interrupted_provider_wait(agent, 45.0, response_started=True) is False
assert agent._consecutive_stale_streams == 0
assert _record_interrupted_provider_wait(agent, 45.0, response_started=False) is True
assert agent._consecutive_stale_streams == 1
assert _record_interrupted_provider_wait(agent, 60.0, response_started=False) is True
with pytest.raises(RuntimeError, match="2 consecutive stale attempts"):
_check_stale_giveup(agent)
@pytest.mark.filterwarnings("ignore::pytest.PytestUnhandledThreadExceptionWarning")
def test_short_circuits_when_streak_at_threshold(self, monkeypatch):
"""A session already past the consecutive-stale threshold must abort
immediately without opening a stream or waiting out the stale timeout."""
monkeypatch.setenv("HERMES_STREAM_STALE_GIVEUP", "3")
agent = _make_anthropic_agent()
agent._consecutive_stale_streams = 3 # simulate prior wedged turns
# The stream must never be opened on the short-circuit path.
with pytest.raises(RuntimeError, match="unresponsive"):
agent._interruptible_streaming_api_call({})
agent._anthropic_client.messages.stream.assert_not_called()
# The streak is NOT reset on the short-circuit so subsequent turns
# keep failing fast instead of re-attempting forever.
assert agent._consecutive_stale_streams == 3
@pytest.mark.filterwarnings("ignore::pytest.PytestUnhandledThreadExceptionWarning")
def test_success_resets_streak(self, monkeypatch):
"""A stream that completes successfully clears the consecutive-stale
streak so a recovered provider resumes normally."""
monkeypatch.setenv("HERMES_STREAM_STALE_GIVEUP", "3")
agent = _make_anthropic_agent()
agent._consecutive_stale_streams = 2 # below the giveup=3 threshold
agent._anthropic_client.messages.stream.return_value = _good_stream_cm()
resp = agent._interruptible_streaming_api_call({})
assert resp is not None
assert agent._consecutive_stale_streams == 0
@pytest.mark.filterwarnings("ignore::pytest.PytestUnhandledThreadExceptionWarning")
def test_stale_kill_increments_streak(self, monkeypatch):
"""Each stale-stream kill increments the consecutive-stale streak so a
wedged session eventually trips the breaker."""
monkeypatch.setenv("HERMES_STREAM_STALE_TIMEOUT", "0.1")
monkeypatch.setenv("HERMES_STREAM_STALE_GIVEUP", "50")
agent = _make_anthropic_agent()
agent._consecutive_stale_streams = 0
unblock = threading.Event()
def _blocking_gen():
unblock.wait(timeout=5.0)
raise httpx.ConnectError("connection dropped after close()")
yield # make this a generator so next() triggers the wait
def _stream_side_effect(*args, **kwargs):
cm = MagicMock()
stream = MagicMock()
stream.__iter__ = MagicMock(return_value=_blocking_gen())
cm.__enter__ = MagicMock(return_value=stream)
cm.__exit__ = MagicMock(return_value=False)
return cm
# Every attempt blocks, trips the stale detector, and fails.
agent._anthropic_client.messages.stream.side_effect = _stream_side_effect
# #67142: the stale detector now aborts the request-local client's
# sockets from the poll thread (not close() on the shared client), so
# unblock on the abort to simulate the socket shutdown waking the read.
agent._abort_request_anthropic_client = lambda *a, **k: unblock.set()
with pytest.raises(Exception):
agent._interruptible_streaming_api_call({})
# At least one stale kill happened; the streak must have advanced.
assert agent._consecutive_stale_streams >= 1
@pytest.mark.filterwarnings("ignore::pytest.PytestUnhandledThreadExceptionWarning")
@pytest.mark.parametrize("slow", ["worker_start", "worker_unwind"])
def test_one_attempt_counts_once_however_often_the_timer_fires(self, monkeypatch, caplog, slow):
"""The stale timer re-fires every window while the worker has not started its
attempt yet (CPU-starved host) or has not unwound a kill; one attempt must still
count once, or a single wedged turn trips the breaker for the next one."""
from agent import chat_completion_helpers as h
monkeypatch.setenv("HERMES_STREAM_STALE_TIMEOUT", "0.1")
monkeypatch.setenv("HERMES_STREAM_STALE_GIVEUP", "50")
monkeypatch.setenv("HERMES_STREAM_RETRIES", "0")
agent = _make_anthropic_agent()
agent._consecutive_stale_streams = 0
unblock = threading.Event()
if slow == "worker_start":
start = h._StreamingCall._start_stream_attempt
def _late_start(self):
threading.Event().wait(1.0)
return start(self)
monkeypatch.setattr(h._StreamingCall, "_start_stream_attempt", _late_start)
agent._abort_request_anthropic_client = lambda *a, **k: unblock.set()
else:
agent._abort_request_anthropic_client = lambda *a, **k: None # the kill never wakes the read
def _blocking_gen():
unblock.wait(timeout=1.2)
raise httpx.ConnectError("connection dropped")
yield
def _stream_side_effect(*args, **kwargs):
cm = MagicMock()
stream = MagicMock()
stream.__iter__ = MagicMock(return_value=_blocking_gen())
cm.__enter__ = MagicMock(return_value=stream)
cm.__exit__ = MagicMock(return_value=False)
return cm
agent._anthropic_client.messages.stream.side_effect = _stream_side_effect
with caplog.at_level("WARNING", logger="agent.chat_completion_helpers"), pytest.raises(Exception):
agent._interruptible_streaming_api_call({})
kills = sum("Stream stale for" in r.getMessage() for r in caplog.records)
assert kills >= 2, f"scenario needs the timer to re-fire; saw {kills} kill(s)"
assert agent._consecutive_stale_streams == 1