hermes-agent/tests/agent/test_overflow_overhead_aware_tokens.py
teknium1 76cfc7b257 test: restore state.db / SessionDB regression guards dropped by #120071
Restored (each proven red against a sabotaged production path, green on main):

- tests/agent/test_overflow_overhead_aware_tokens.py::
  test_overflow_recovery_compresses_with_tool_overhead_counted[413,
  context_overflow, prompt_too_long, long_context_tier_429]
  Rewritten from the four estimator-to-compressor tests: the real estimator
  runs with a ~10k-token tool schema and the compressor must receive an
  approx_tokens that counts it (LCM issue 441). No helper-name pinning.
- tests/hermes_cli/test_web_server_sessiondb_eventloop.py::
  test_bulk_delete_sessiondb_work_runs_off_event_loop,
  test_session_read_handlers_run_sessiondb_work_off_event_loop[search,stats,detail]
  Real route handlers; every SessionDB open/call lands off the event-loop
  thread (#60747). AST/source tests stay deleted.
- tests/hermes_state/test_hermes_state.py::TestCompressionChainProjection::
  test_list_batches_tip_row_fetch_into_one_query
  list_sessions_rich resolves a page of compression tips in one batched
  query, not one per root (N+1 guard).
- tests/hermes_state/test_tail_rewind_semantics.py::TestCompressTagsCarriedTail::
  test_compress_marks_carried_forward_tail_dicts
  Repaired from a skip-on-exception body that asserted nothing: compress()
  tags exactly the verbatim carried tail so archive_and_compact gets an
  accurate tail_count (#86366).
- tests/hermes_state/test_wal_checkpoint_strategy.py::
  test_failed_checkpoint_never_raises_and_is_not_silent,
  test_failed_checkpoint_at_close_still_releases_the_handle
  Periodic PASSIVE checkpoint failure neither raises into the write path nor
  passes silently; a failing checkpoint at close() still releases the
  handle (#45383 strategy). Log-wording asserts dropped.
- tests/hermes_state/test_hermes_state_ux_messages.py::
  test_db_unavailable_details_line_carries_raw_cause_only_below_the_lead,
  test_db_unavailable_network_drive_cause_does_not_offer_doctor_fix
  CLI details=True keeps the raw SQLite cause on its own Details line only;
  a locking-protocol (network FS) cause never offers `doctor --fix`.
- tests/tools/test_code_execution.py::TestStubSchemaDrift::
  test_generated_module_accepts_all_params
  Replaces the syntax-only test_generated_module_compiles: executes the
  generated hermes_tools module (uds and file transports) and asserts every
  stub accepts all its params as kwargs and forwards each by name and value.
2026-09-23 10:34:54 -07:00

215 lines
7.9 KiB
Python

"""Overflow recovery handlers (413, context overflow, Anthropic long-context
tier 429) must hand the compressor an overhead-aware token estimate that counts
tool schemas, not a messages-only estimate (LCM issue 441). Also: the retry
diagnostics buffered during long-context recovery must flush correctly (muted
vs. unmuted) on terminal failure."""
import pytest
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
from run_agent import AIAgent
# ---------------------------------------------------------------------------
# Shared fixtures / helpers (mirrored from test_413_compression.py)
# ---------------------------------------------------------------------------
@pytest.fixture(autouse=True)
def _no_sleep(monkeypatch):
"""Short-circuit all time.sleep and jittered_backoff calls."""
import time as _time
monkeypatch.setattr(_time, "sleep", lambda *_a, **_k: None)
from agent import retry_utils as _retry_utils
monkeypatch.setattr(_retry_utils, "jittered_backoff", lambda *a, **k: 0.0)
def _make_tool_defs(*names: str) -> list:
return [
{
"type": "function",
"function": {
"name": n,
"description": f"{n} tool",
"parameters": {"type": "object", "properties": {}},
},
}
for n in names
]
def _mock_response(content="Hello", finish_reason="stop", tool_calls=None, usage=None):
msg = SimpleNamespace(
content=content,
tool_calls=tool_calls,
reasoning_content=None,
reasoning=None,
)
choice = SimpleNamespace(message=msg, finish_reason=finish_reason)
resp = SimpleNamespace(choices=[choice], model="test/model")
resp.usage = SimpleNamespace(**usage) if usage else None
return resp
@pytest.fixture()
def agent():
with (
patch("model_tools.get_tool_definitions", return_value=_make_tool_defs("web_search")),
patch("model_tools.check_toolset_requirements", return_value={}),
patch("agent.process_bootstrap.OpenAI"),
):
a = AIAgent(
api_key="test-key-1234567890",
base_url="https://openrouter.ai/api/v1",
quiet_mode=True,
skip_context_files=True,
skip_memory=True,
)
a.client = MagicMock()
a._cached_system_prompt = "You are helpful."
a._use_prompt_caching = False
a.compression_enabled = True
a.save_trajectories = False
return a
def _prefill():
return [
{"role": "user", "content": "previous question"},
{"role": "assistant", "content": "previous answer"},
]
_SENTINEL_TOKENS = 987_654
def _make_413_error():
err = Exception("Request entity too large")
err.status_code = 413
return err
def _make_context_overflow_error():
"""A 400 the classifier routes to context_overflow."""
err = Exception(
"Error code: 400 - {'error': {'message': "
"\"This endpoint's maximum context length is 128000 tokens. "
"However, you requested about 200000 tokens. "
"Please reduce the length of the messages.\"}}"
)
err.status_code = 400
return err
def _make_prompt_too_long_error():
"""Anthropic's 'prompt is too long' variant of context overflow."""
err = Exception(
"Error code: 400 - {'type': 'error', 'error': {'type': 'invalid_request_error', "
"'message': 'prompt is too long: 233153 tokens > 200000 maximum'}}"
)
err.status_code = 400
return err
def _make_long_context_tier_error():
"""Build a 429 'extra usage required for long context requests' error."""
err = Exception(
"Error code: 429 - {'error': {'type': 'rate_limit_error', "
"'message': 'Extra usage is required for long context requests. "
"Please enable extra usage in your account settings.'}}"
)
err.status_code = 429
return err
@pytest.mark.parametrize("muted", [False, True])
def test_long_context_retry_carrier_survives_failure_flush(agent, muted, monkeypatch):
from gateway.warning_notifications import DiagnosticText
agent._notification_config = {"display": {"suppress_warning_notifications": muted}}
err = _make_long_context_tier_error()
agent.client.chat.completions.create.side_effect = [err, _mock_response(content="Recovered")]
captured = []
original = agent._buffer_retry_message
def capture(kind, text):
captured.append((kind, text))
original(kind, text)
monkeypatch.setattr(agent, "_buffer_retry_message", capture)
with (patch("agent.model_metadata.estimate_request_tokens_rough", return_value=_SENTINEL_TOKENS),
patch.object(agent, "_compress_context", return_value=([{"role": "user", "content": "compressed"}], "compressed prompt")),
patch.object(agent, "_persist_session"), patch.object(agent, "_save_trajectory"),
patch.object(agent, "_cleanup_task_resources")):
result = agent.run_conversation("hello", conversation_history=_prefill())
assert result["final_response"] == "Recovered"
assert captured
assert any(kind == "status" for kind, _ in captured)
assert all(kind == "vprint" or isinstance(text, DiagnosticText) for kind, text in captured), captured
# Recovery clears buffered diagnostics; the same carriers must also work on
# the terminal-failure flush instead of leaking their retry sibling copy.
assert not agent._retry_status_buffer
agent._retry_status_buffer = list(captured)
printed, observed = [], []
agent._print_fn = lambda *a, **k: printed.append(a)
agent.status_callback = lambda *a: observed.append(a)
agent._flush_status_buffer()
assert bool(printed) is not muted
assert observed
# A tool schema big enough (~10k tokens) to dwarf the tiny test conversation,
# so a messages-only estimate cannot reach the floor asserted below.
_HEAVY_TOOL_DESCRIPTION = "Heavy schema padding. " * 2_000
_TOOL_OVERHEAD_FLOOR = 5_000
@pytest.mark.parametrize(
"make_error",
[
pytest.param(_make_413_error, id="413"),
pytest.param(_make_context_overflow_error, id="context_overflow"),
pytest.param(_make_prompt_too_long_error, id="prompt_too_long"),
pytest.param(_make_long_context_tier_error, id="long_context_tier_429"),
],
)
def test_overflow_recovery_compresses_with_tool_overhead_counted(agent, make_error):
"""Each recovery path must pass approx_tokens that include tool-schema
overhead: a messages-only estimate under-reports the request, so the
compressor sizes its target wrong and the retry overflows again."""
from agent.model_metadata import estimate_messages_tokens_rough
agent.tools = [
{
"type": "function",
"function": {
"name": "heavy_tool",
"description": _HEAVY_TOOL_DESCRIPTION,
"parameters": {"type": "object", "properties": {}},
},
}
]
history = _prefill()
assert estimate_messages_tokens_rough(history + [{"role": "user", "content": "hello"}]) < 500
agent.client.chat.completions.create.side_effect = [
make_error(),
_mock_response(content="Recovered", finish_reason="stop"),
]
with (
patch.object(agent, "_compress_context") as mock_compress,
patch.object(agent, "_persist_session"),
patch.object(agent, "_save_trajectory"),
patch.object(agent, "_cleanup_task_resources"),
):
mock_compress.return_value = (
[{"role": "user", "content": "compressed"}],
"compressed prompt",
)
result = agent.run_conversation("hello", conversation_history=history)
assert result["final_response"] == "Recovered"
passed = [c.kwargs.get("approx_tokens") for c in mock_compress.call_args_list]
assert passed, "recovery never invoked the compressor"
assert any(t is not None and t >= _TOOL_OVERHEAD_FLOOR for t in passed), (
f"compressor got messages-only estimates {passed}; tool schemas were not counted"
)