hermes-trace/tests/test_hook_state.py

1335 lines
49 KiB
Python

"""Tests for the composite state key fix (hermes-trace race condition).
Verifies that ``_state_key`` produces correct composite keys and that
``on_pre_api_request`` / ``on_post_api_request`` / ``on_post_tool_call``
use those keys instead of bare ``session_id``, preventing cross-thread
clobbering when spawned subagents share the parent's session_id.
"""
from __future__ import annotations
import importlib.util
import json
import os
import sys
import unittest
from collections.abc import ItemsView, Iterator
from types import SimpleNamespace
from unittest.mock import MagicMock, call, create_autospec, patch
# The plugin lives in a directory with a hyphen, which is not a valid Python
# identifier for a regular import. Load it via importlib so the package
# machinery resolves ``from . import db`` relative imports correctly.
_plugin_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
_plugin_init = os.path.join(_plugin_root, "__init__.py")
_module_name = "hermes_trace_hook_plugin"
_original_module = sys.modules.get(_module_name)
_spec = importlib.util.spec_from_file_location(
_module_name,
_plugin_init,
submodule_search_locations=[_plugin_root],
)
_plugin = importlib.util.module_from_spec(_spec)
sys.modules[_module_name] = _plugin
_spec.loader.exec_module(_plugin)
def tearDownModule() -> None:
"""Restore the module registry entry changed during test import."""
if _original_module is None:
sys.modules.pop(_module_name, None)
else:
sys.modules[_module_name] = _original_module
_STATE_LOCK = _plugin._STATE_LOCK
_preview_text = _plugin._preview_text
_state_key = _plugin._state_key
TelemetryDB = _plugin.db.TelemetryDB
class TestPreviewText(unittest.TestCase):
"""Unit tests for preview content normalization."""
def test_none_returns_empty_string(self) -> None:
"""Tool-call-only responses do not render as the string ``None``."""
self.assertEqual(_preview_text(None), "")
def test_string_is_returned_unchanged(self) -> None:
"""Plain text remains plain text."""
self.assertEqual(_preview_text("hello"), "hello")
def test_list_joins_text_parts(self) -> None:
"""Multipart text content is joined with newlines."""
self.assertEqual(
_preview_text([{"text": "first"}, {"text": "second"}]),
"first\nsecond",
)
def test_list_without_text_falls_back_to_json(self) -> None:
"""Non-text multipart content retains a JSON representation."""
self.assertEqual(
_preview_text([{"type": "image", "url": "cat.png"}]),
'[{"type": "image", "url": "cat.png"}]',
)
def test_empty_list_falls_back_to_json(self) -> None:
"""An empty multipart value is represented as valid JSON."""
self.assertEqual(_preview_text([]), "[]")
def test_dict_and_non_dict_iterable_use_string_conversion(self) -> None:
"""Other containers use their ordinary string form."""
self.assertEqual(_preview_text({"answer": 42}), "{'answer': 42}")
self.assertEqual(_preview_text(("a", "b")), "('a', 'b')")
class _LockCheckingState(dict[str, object]):
"""State mapping that rejects field access without ``_STATE_LOCK``."""
def get(self, key: str, default: object = None) -> object:
if not _STATE_LOCK.locked():
raise AssertionError("state.get used without _STATE_LOCK")
return super().get(key, default)
def __setitem__(self, key: str, value: object) -> None:
if not _STATE_LOCK.locked():
raise AssertionError("state.__setitem__ used without _STATE_LOCK")
super().__setitem__(key, value)
def __getitem__(self, key: str) -> object:
if not _STATE_LOCK.locked():
raise AssertionError("state.__getitem__ used without _STATE_LOCK")
return super().__getitem__(key)
def __contains__(self, key: object) -> bool:
if not _STATE_LOCK.locked():
raise AssertionError("state.__contains__ used without _STATE_LOCK")
return super().__contains__(key)
def __iter__(self) -> Iterator[str]:
if not _STATE_LOCK.locked():
raise AssertionError("state.__iter__ used without _STATE_LOCK")
return super().__iter__()
class _LockCheckingSessionState(dict[str, _LockCheckingState]):
"""Session mapping that rejects compound access without ``_STATE_LOCK``."""
def items(self) -> ItemsView[str, _LockCheckingState]:
if not _STATE_LOCK.locked():
raise AssertionError("session_state.items used without _STATE_LOCK")
return super().items()
def get(self, key: str, default: object = None) -> object:
if not _STATE_LOCK.locked():
raise AssertionError("session_state.get used without _STATE_LOCK")
return super().get(key, default)
def setdefault(self, key: str, default: object = None) -> _LockCheckingState:
if not _STATE_LOCK.locked():
raise AssertionError("session_state.setdefault used without _STATE_LOCK")
if not dict.__contains__(self, key):
dict.__setitem__(self, key, _LockCheckingState())
return dict.__getitem__(self, key)
def pop(self, key: str, default: object = None) -> object:
if not _STATE_LOCK.locked():
raise AssertionError("session_state.pop used without _STATE_LOCK")
return super().pop(key, default)
def __getitem__(self, key: str) -> _LockCheckingState:
if not _STATE_LOCK.locked():
raise AssertionError("session_state.__getitem__ used without _STATE_LOCK")
return super().__getitem__(key)
def __contains__(self, key: object) -> bool:
if not _STATE_LOCK.locked():
raise AssertionError("session_state.__contains__ used without _STATE_LOCK")
return super().__contains__(key)
def __iter__(self) -> Iterator[str]:
if not _STATE_LOCK.locked():
raise AssertionError("session_state.__iter__ used without _STATE_LOCK")
return super().__iter__()
class TestStateKey(unittest.TestCase):
"""Unit tests for the ``_state_key`` helper."""
def test_session_with_turn_id(self) -> None:
"""session_id + turn_id → composite turn-scoped key."""
key = _state_key("sess-1", turn_id="turn-abc")
self.assertEqual(key, "sess-1:turn:turn-abc")
def test_session_with_api_request_id(self) -> None:
"""session_id + api_request_id → composite API-scoped key."""
key = _state_key("sess-1", api_request_id="req-xyz")
self.assertEqual(key, "sess-1:api:req-xyz")
def test_session_with_task_id(self) -> None:
"""session_id + task_id produces a task-scoped key."""
self.assertEqual(
_state_key("sess-1", task_id="task-xyz"),
"sess-1:task:task-xyz",
)
def test_bare_session_fallback(self) -> None:
"""No correlation IDs → bare session_id (backward compat)."""
key = _state_key("sess-1")
self.assertEqual(key, "sess-1")
def test_turn_id_preferred_over_api_request_id(self) -> None:
"""turn_id takes precedence when both are provided."""
key = _state_key("sess-1", turn_id="turn-abc", api_request_id="req-xyz")
self.assertEqual(key, "sess-1:turn:turn-abc")
def test_priority_is_turn_then_task_then_api_request(self) -> None:
"""All correlation identifiers obey the documented priority."""
self.assertEqual(
_state_key(
"sess-1",
turn_id="turn-abc",
task_id="task-abc",
api_request_id="req-abc",
),
"sess-1:turn:turn-abc",
)
self.assertEqual(
_state_key(
"sess-1",
task_id="task-abc",
api_request_id="req-abc",
),
"sess-1:task:task-abc",
)
def test_empty_turn_id_allows_api_request_id(self) -> None:
"""Empty string turn_id should not match; fall through to api_request_id."""
key = _state_key("sess-1", turn_id="", api_request_id="req-xyz")
self.assertEqual(key, "sess-1:api:req-xyz")
def test_empty_ids_returns_session(self) -> None:
"""Both empty strings → bare session fallback."""
key = _state_key("sess-1", turn_id="", api_request_id="")
self.assertEqual(key, "sess-1")
class TestRaceIsolation(unittest.TestCase):
"""Verify that concurrent pre→post cycles with same session_id but
different turn_id get their own state entries."""
def setUp(self) -> None:
self.state_patcher = patch(
f"{_module_name}._state._session_state",
new_callable=dict,
)
self.mock_state = self.state_patcher.start()
# We also need to mock db.get_db() so write_turn() doesn't
# actually hit SQLite during the state-scoping tests below.
self.db_patcher = patch(f"{_module_name}.db.get_db")
self.mock_db = self.db_patcher.start()
self.mock_db_instance = create_autospec(TelemetryDB, instance=True)
self.mock_db_instance.write_turn.return_value = 42 # fake db_turn_id
self.mock_db.return_value = self.mock_db_instance
def tearDown(self) -> None:
self.state_patcher.stop()
self.db_patcher.stop()
def _fire_pre_post(
self,
session_id: str,
turn_id: str,
*,
assistant_tool_call_count: int = 0,
) -> None:
"""Simulate a single pre→post hook cycle."""
# Import locally so patches are active
on_post_api_request = _plugin.on_post_api_request
on_pre_api_request = _plugin.on_pre_api_request
on_pre_api_request(
session_id=session_id,
turn_id=turn_id,
request_messages=[{"role": "user", "content": "hello"}],
)
on_post_api_request(
session_id=session_id,
turn_id=turn_id,
model="gpt-4",
usage={"input_tokens": 10, "output_tokens": 20},
api_duration=0.5,
finish_reason="stop",
assistant_message=MagicMock(content="hi", reasoning=None),
assistant_tool_call_count=assistant_tool_call_count,
api_call_count=1,
response={"id": "resp-1"},
)
def test_two_turns_isolation(self) -> None:
"""Two turns sharing session_id but with different turn_id each get
their own composite key and do NOT clobber each other."""
self._fire_pre_post("sess-1", turn_id="turn-a", assistant_tool_call_count=2)
self._fire_pre_post("sess-1", turn_id="turn-b", assistant_tool_call_count=2)
# Both entries should exist in state (no clobbering)
self.assertIn("sess-1:turn:turn-a", self.mock_state)
self.assertIn("sess-1:turn:turn-b", self.mock_state)
# Bare session_id should NOT exist (we used turn_id)
self.assertNotIn("sess-1", self.mock_state)
# Each key has its own latest_turn_id
self.assertEqual(
self.mock_state["sess-1:turn:turn-a"]["latest_turn_id"],
42,
)
self.assertEqual(
self.mock_state["sess-1:turn:turn-b"]["latest_turn_id"],
42,
)
def test_zero_tool_calls_cleans_up(self) -> None:
"""When assistant_tool_call_count == 0, the state is cleaned up
immediately after on_post_api_request."""
self._fire_pre_post("sess-1", turn_id="turn-a", assistant_tool_call_count=0)
# State should have been popped
self.assertNotIn("sess-1:turn:turn-a", self.mock_state)
def test_api_request_error_writes_and_cleans_up(self) -> None:
"""Failed API calls are persisted and their pending state is removed."""
_plugin.on_pre_api_request(
session_id="sess-1",
turn_id="turn-error",
request_messages=[{"role": "user", "content": "hello"}],
)
_plugin.on_api_request_error(
session_id="sess-1",
turn_id="turn-error",
model="gpt-4",
provider="provider-1",
api_call_count=3,
api_duration=1.25,
error={"type": "TimeoutError", "message": "provider timed out"},
request={"body": {"messages": [{"role": "user", "content": "hello"}]}},
)
call_kwargs = self.mock_db_instance.write_turn.call_args.kwargs
self.assertEqual(call_kwargs["finish_reason"], "error")
self.assertEqual(call_kwargs["model"], "gpt-4")
self.assertEqual(call_kwargs["provider"], "provider-1")
self.assertEqual(call_kwargs["api_call_count"], 3)
self.assertEqual(call_kwargs["elapsed_ms"], 1250)
self.assertEqual(
json.loads(call_kwargs["full_request"]),
{"body": {"messages": [{"role": "user", "content": "hello"}]}},
)
self.assertEqual(
json.loads(call_kwargs["full_response"]),
{"error_type": "TimeoutError", "error_message": "provider timed out"},
)
self.assertNotIn("sess-1:turn:turn-error", self.mock_state)
def test_api_request_error_without_pre_state_is_logged(self) -> None:
"""An error without a pre-hook still writes a minimal turn record."""
with self.assertLogs(f"{_module_name}.hooks", level="WARNING") as logs:
_plugin.on_api_request_error(
session_id="sess-1",
turn_id="turn-without-pre",
model="gpt-4o",
provider="provider-2",
api_call_count=4,
api_duration=None,
error={"type": "AuthError", "message": "invalid credentials"},
request={"body": {"messages": []}},
)
self.mock_db_instance.write_turn.assert_called_once()
call_kwargs = self.mock_db_instance.write_turn.call_args.kwargs
self.assertEqual(call_kwargs["model"], "gpt-4o")
self.assertEqual(call_kwargs["finish_reason"], "error")
self.assertEqual(call_kwargs["api_call_count"], 4)
self.assertIn("no session state", "\n".join(logs.output))
self.assertNotIn("sess-1:turn:turn-without-pre", self.mock_state)
def test_interleaved_turns_keep_their_own_requests(self) -> None:
"""pre(A), pre(B), post(A), post(B) never clobbers request data."""
self.mock_db_instance.write_turn.side_effect = [41, 42]
for turn_id, content in (("turn-a", "request A"), ("turn-b", "request B")):
_plugin.on_pre_api_request(
session_id="sess-1",
turn_id=turn_id,
request_messages=[{"role": "user", "content": content}],
)
for turn_id, content in (("turn-a", "response A"), ("turn-b", "response B")):
_plugin.on_post_api_request(
session_id="sess-1",
turn_id=turn_id,
model="gpt-4",
usage={},
api_duration=0.1,
finish_reason="tool_calls",
assistant_message=SimpleNamespace(content=content, reasoning=None),
assistant_tool_call_count=1,
api_call_count=1,
response={"content": content},
)
calls = self.mock_db_instance.write_turn.call_args_list
self.assertEqual(calls[0].kwargs["request_preview"], "request A")
self.assertEqual(
calls[0].kwargs["full_request"],
'[{"role": "user", "content": "request A"}]',
)
self.assertEqual(calls[1].kwargs["request_preview"], "request B")
self.assertEqual(
calls[1].kwargs["full_request"],
'[{"role": "user", "content": "request B"}]',
)
self.assertEqual(
self.mock_state["sess-1:turn:turn-a"]["latest_turn_id"],
41,
)
self.assertEqual(
self.mock_state["sess-1:turn:turn-b"]["latest_turn_id"],
42,
)
class TestToolCallIsolation(unittest.TestCase):
"""Verify that tool calls are associated with the correct turn when using
composite state keys."""
def setUp(self) -> None:
self.state_patcher = patch(
f"{_module_name}._state._session_state",
new_callable=dict,
)
self.mock_state = self.state_patcher.start()
self.db_patcher = patch(f"{_module_name}.db.get_db")
self.mock_db = self.db_patcher.start()
self.mock_db_instance = create_autospec(TelemetryDB, instance=True)
self.mock_db_instance.write_turn.return_value = 42
self.mock_db.return_value = self.mock_db_instance
def tearDown(self) -> None:
self.state_patcher.stop()
self.db_patcher.stop()
def _setup_turn_with_tool_calls(
self, turn_id: str, session_id: str = "sess-1"
) -> str:
"""Set up a turn that expects tool calls; return the composite key."""
on_post_api_request = _plugin.on_post_api_request
on_pre_api_request = _plugin.on_pre_api_request
on_pre_api_request(
session_id=session_id,
turn_id=turn_id,
request_messages=[{"role": "user", "content": "do a tool call"}],
)
on_post_api_request(
session_id=session_id,
turn_id=turn_id,
model="gpt-4",
usage={"input_tokens": 10, "output_tokens": 20},
api_duration=0.5,
finish_reason="tool_calls",
assistant_message=MagicMock(content=None, reasoning=None),
assistant_tool_call_count=2,
api_call_count=1,
response={"id": "resp-1"},
)
return _state_key(session_id, turn_id=turn_id)
def test_tool_call_uses_turn_key(self) -> None:
"""on_post_tool_call reads state from the composite key."""
on_post_tool_call = _plugin.on_post_tool_call
key = self._setup_turn_with_tool_calls(turn_id="turn-1")
on_post_tool_call(
session_id="sess-1",
turn_id="turn-1",
tool_name="get_weather",
args={"city": "London"},
result={"temp": 15},
duration_ms=200,
)
# Verify state was found (not None) — tool call was written
# tool_call_counter should be 1 after one call
self.assertEqual(self.mock_state[key]["tool_call_counter"], 1)
self.mock_db_instance.write_tool_call.assert_called_once_with(
turn_id=42,
call_order=0,
name="get_weather",
args='{"city": "London"}',
result='{"temp": 15}',
duration_ms=200,
error=None,
)
# Second tool call should reach expected_tool_calls (2) and clean up
on_post_tool_call(
session_id="sess-1",
turn_id="turn-1",
tool_name="get_time",
args={"tz": "UTC"},
result={"time": "12:00"},
duration_ms=100,
)
# After last expected tool call, state key should be removed
self.assertNotIn(key, self.mock_state)
self.mock_db_instance.write_tool_call.assert_has_calls(
[
call(
turn_id=42,
call_order=0,
name="get_weather",
args='{"city": "London"}',
result='{"temp": 15}',
duration_ms=200,
error=None,
),
call(
turn_id=42,
call_order=1,
name="get_time",
args='{"tz": "UTC"}',
result='{"time": "12:00"}',
duration_ms=100,
error=None,
),
]
)
def test_different_turn_different_tool_state(self) -> None:
"""Tool calls from different turns with same session_id use
independent state entries."""
on_post_tool_call = _plugin.on_post_tool_call
key_a = self._setup_turn_with_tool_calls(turn_id="turn-a")
key_b = self._setup_turn_with_tool_calls(turn_id="turn-b")
on_post_tool_call(
session_id="sess-1",
turn_id="turn-a",
tool_name="ping",
args={},
result={"pong": True},
duration_ms=50,
)
# turn-a counter incremented
self.assertEqual(self.mock_state[key_a]["tool_call_counter"], 1)
# turn-b counter still 0
self.assertEqual(self.mock_state[key_b]["tool_call_counter"], 0)
class TestStateLockCoverage(unittest.TestCase):
"""Verify compound state access is locked without locking DB writes."""
def test_hook_lifecycle_locks_state_only(self) -> None:
"""All state access is protected while database writes stay unlocked."""
on_post_api_request = _plugin.on_post_api_request
on_post_tool_call = _plugin.on_post_tool_call
on_pre_api_request = _plugin.on_pre_api_request
state = _LockCheckingSessionState()
mock_db = create_autospec(TelemetryDB, instance=True)
def write_turn(**_: object) -> int:
self.assertFalse(_STATE_LOCK.locked())
return 42
def write_tool_call(**_: object) -> None:
self.assertFalse(_STATE_LOCK.locked())
mock_db.write_turn.side_effect = write_turn
mock_db.write_tool_call.side_effect = write_tool_call
with (
patch(f"{_module_name}._state._session_state", state),
patch(f"{_module_name}.db.get_db", return_value=mock_db),
):
on_pre_api_request(
session_id="sess-locked",
turn_id="turn-locked",
request_messages=[{"role": "user", "content": "use a tool"}],
)
on_post_api_request(
session_id="sess-locked",
turn_id="turn-locked",
model="gpt-4",
usage={"input_tokens": 10, "output_tokens": 20},
api_duration=0.5,
finish_reason="tool_calls",
assistant_message=MagicMock(content=None, reasoning=None),
assistant_tool_call_count=1,
api_call_count=1,
response={"id": "resp-locked"},
)
on_post_tool_call(
session_id="sess-locked",
turn_id="turn-locked",
tool_name="locked_tool",
args={},
result={"ok": True},
duration_ms=100,
)
self.assertFalse(dict.__contains__(state, "sess-locked:turn:turn-locked"))
mock_db.write_turn.assert_called_once()
mock_db.write_tool_call.assert_called_once()
class TestBackwardCompatibility(unittest.TestCase):
"""Verify backward compatibility when hooks are called without
turn_id/api_request_id (legacy path)."""
def setUp(self) -> None:
self.state_patcher = patch(
f"{_module_name}._state._session_state",
new_callable=dict,
)
self.mock_state = self.state_patcher.start()
self.db_patcher = patch(f"{_module_name}.db.get_db")
self.mock_db = self.db_patcher.start()
self.mock_db_instance = create_autospec(TelemetryDB, instance=True)
self.mock_db_instance.write_turn.return_value = 99
self.mock_db.return_value = self.mock_db_instance
def tearDown(self) -> None:
self.state_patcher.stop()
self.db_patcher.stop()
def test_bare_session_key_fallback(self) -> None:
"""Calling on_pre_api_request / on_post_api_request without turn_id or
api_request_id uses bare session_id as the key."""
on_post_api_request = _plugin.on_post_api_request
on_pre_api_request = _plugin.on_pre_api_request
on_pre_api_request(
session_id="sess-legacy",
request_messages=[{"role": "user", "content": "legacy call"}],
)
self.assertIn("sess-legacy", self.mock_state)
self.assertEqual(
self.mock_state["sess-legacy"]["request_preview"],
"legacy call",
)
on_post_api_request(
session_id="sess-legacy",
model="gpt-3.5",
usage={"input_tokens": 5, "output_tokens": 10},
api_duration=0.3,
finish_reason="stop",
assistant_message=MagicMock(content="ok", reasoning=None),
assistant_tool_call_count=0,
api_call_count=1,
response={"id": "resp-legacy"},
)
# state cleaned up because tool_calls_count == 0
self.assertNotIn("sess-legacy", self.mock_state)
def test_tool_call_with_bare_session(self) -> None:
"""Tool call without turn_id uses and cleans up the bare session key."""
on_post_api_request = _plugin.on_post_api_request
on_post_tool_call = _plugin.on_post_tool_call
on_pre_api_request = _plugin.on_pre_api_request
on_pre_api_request(
session_id="sess-legacy",
request_messages=[{"role": "user", "content": "legacy tool call"}],
)
on_post_api_request(
session_id="sess-legacy",
model="gpt-4",
usage={"input_tokens": 10, "output_tokens": 20},
api_duration=0.5,
finish_reason="tool_calls",
assistant_message=MagicMock(content=None, reasoning=None),
assistant_tool_call_count=1,
api_call_count=1,
response={"id": "resp-legacy"},
)
self.assertIn("sess-legacy", self.mock_state)
self.assertEqual(self.mock_state["sess-legacy"]["expected_tool_calls"], 1)
on_post_tool_call(
session_id="sess-legacy",
tool_name="legacy_tool",
args={"key": "value"},
result={"ok": True},
duration_ms=100,
)
self.assertNotIn("sess-legacy", self.mock_state)
class TestCodexInputConversion(unittest.TestCase):
"""Verify Responses API input items become standard messages."""
def test_converts_text_content_types(self) -> None:
"""text, input_text, and output_text parts are preserved."""
converted = _plugin._convert_codex_input_to_standard_messages(
[
{
"type": "message",
"role": "user",
"content": [
{"type": "input_text", "text": "input"},
{"type": "output_text", "text": "output"},
{"type": "image", "url": "ignored"},
],
}
]
)
self.assertEqual(converted, [{"role": "user", "content": "input\noutput"}])
def test_groups_reasoning_message_and_function_call(self) -> None:
"""Contiguous assistant-side items become one chat-style message."""
reasoning = {
"type": "reasoning",
"encrypted_content": "secret",
"summary": [{"type": "summary_text", "text": "Checking weather"}],
}
converted = _plugin._convert_codex_input_to_standard_messages(
[
reasoning,
{"role": "assistant", "content": ""},
{
"type": "message",
"role": "assistant",
"content": [{"type": "output_text", "text": "One moment"}],
},
{
"type": "function_call",
"call_id": "call-1",
"name": "weather",
"arguments": '{"city":"Taipei"}',
},
{
"type": "function_call_output",
"call_id": "call-1",
"output": '{"temp":30}',
},
]
)
self.assertEqual(
converted,
[
{
"role": "assistant",
"content": "One moment",
"reasoning_details": [reasoning],
"reasoning_content": "Checking weather",
"tool_calls": [
{
"id": "call-1",
"type": "function",
"function": {
"name": "weather",
"arguments": '{"city":"Taipei"}',
},
}
],
},
{
"role": "tool",
"tool_call_id": "call-1",
"content": '{"temp":30}',
},
],
)
class TestAnthropicInputConversion(unittest.TestCase):
"""Verify Anthropic thinking blocks become chat-style reasoning."""
def test_converts_assistant_thinking_text_and_tool_use(self) -> None:
thinking = {
"type": "thinking",
"thinking": "Inspect the repository first.",
"signature": "sig-1",
}
converted = _plugin._state._convert_anthropic_messages_to_standard_messages(
[
{
"role": "assistant",
"content": [
thinking,
{"type": "text", "text": "I will check it."},
{
"type": "tool_use",
"id": "toolu-1",
"name": "terminal",
"input": {"command": "git status"},
},
],
}
]
)
self.assertEqual(
converted,
[
{
"role": "assistant",
"content": "I will check it.",
"reasoning_content": "Inspect the repository first.",
"reasoning_details": [thinking],
"tool_calls": [
{
"id": "toolu-1",
"type": "function",
"function": {
"name": "terminal",
"arguments": '{"command": "git status"}',
},
}
],
}
],
)
def test_preserves_redacted_thinking_without_inventing_text(self) -> None:
redacted = {"type": "redacted_thinking", "data": "opaque"}
converted = _plugin._state._convert_anthropic_messages_to_standard_messages(
[{"role": "assistant", "content": [redacted]}]
)
self.assertEqual(
converted,
[
{
"role": "assistant",
"content": None,
"reasoning_details": [redacted],
}
],
)
class TestPreAPIRequest(unittest.TestCase):
"""Verify request serialization and system-prompt fallback behavior."""
def setUp(self) -> None:
self.state: dict[str, dict[str, object]] = {}
self.state_patcher = patch(f"{_module_name}._state._session_state", self.state)
self.state_patcher.start()
def tearDown(self) -> None:
self.state_patcher.stop()
def _full_request(self, session_id: str = "session-1") -> list[dict[str, object]]:
return json.loads(str(self.state[session_id]["full_request"]))
def test_explicit_system_prompt_has_first_priority(self) -> None:
"""Hermes provider-bound system prompt wins over fallback sources."""
with patch(
f"{_module_name}._state._get_system_prompt_from_state_db",
return_value="database system",
) as state_db:
_plugin.on_pre_api_request(
session_id="session-1",
request_messages=[{"role": "user", "content": "hello"}],
system_prompt="provider system",
conversation_history=[{"role": "system", "content": "history system"}],
request={"body": {"instructions": "request system"}},
)
self.assertEqual(self._full_request()[0]["content"], "provider system")
state_db.assert_not_called()
def test_explicit_system_prompt_blocks_are_flattened(self) -> None:
"""Anthropic-style system text blocks produce one readable system entry."""
_plugin.on_pre_api_request(
session_id="session-1",
request_messages=[{"role": "user", "content": "hello"}],
system_prompt=[
{"type": "text", "text": "first"},
{"type": "text", "text": "second"},
],
)
self.assertEqual(self._full_request()[0]["content"], "first\nsecond")
def test_conversation_history_system_prompt_has_first_priority(self) -> None:
"""Conversation history wins over request instructions and state DB."""
with patch(
f"{_module_name}._state._get_system_prompt_from_state_db",
return_value="database system",
) as state_db:
_plugin.on_pre_api_request(
session_id="session-1",
request_messages=[{"role": "user", "content": "hello"}],
conversation_history=[{"role": "system", "content": "history system"}],
request={"body": {"instructions": "request system"}},
)
self.assertEqual(
self._full_request(),
[
{"role": "system", "content": "history system"},
{"role": "user", "content": "hello"},
],
)
state_db.assert_not_called()
def test_request_instructions_fall_back_before_state_database(self) -> None:
"""Request body instructions are used when history has no system role."""
with patch(
f"{_module_name}._state._get_system_prompt_from_state_db",
return_value="database system",
) as state_db:
_plugin.on_pre_api_request(
session_id="session-1",
request_messages=[{"role": "user", "content": "hello"}],
conversation_history=[{"role": "user", "content": "older"}],
request={"body": {"instructions": "request system"}},
)
self.assertEqual(self._full_request()[0]["content"], "request system")
state_db.assert_not_called()
def test_state_database_is_final_system_prompt_fallback(self) -> None:
"""The Hermes state DB supplies instructions when other sources do not."""
with patch(
f"{_module_name}._state._get_system_prompt_from_state_db",
return_value="database system",
) as state_db:
_plugin.on_pre_api_request(
session_id="session-1",
request_messages=[{"role": "user", "content": "hello"}],
)
self.assertEqual(self._full_request()[0]["content"], "database system")
state_db.assert_called_once_with("session-1")
def test_anthropic_full_request_projects_thinking_to_reasoning_content(
self,
) -> None:
thinking = {
"type": "thinking",
"thinking": "Use the tool next.",
"signature": "sig-1",
}
_plugin.on_pre_api_request(
session_id="session-1",
api_mode="anthropic_messages",
system_prompt="system",
request_messages=[
{
"role": "assistant",
"content": [
thinking,
{"type": "text", "text": "Checking."},
],
},
{"role": "user", "content": "continue"},
],
)
self.assertEqual(
self._full_request(),
[
{"role": "system", "content": "system"},
{
"role": "assistant",
"content": "Checking.",
"reasoning_content": "Use the tool next.",
"reasoning_details": [thinking],
},
{"role": "user", "content": "continue"},
],
)
def test_anthropic_full_request_uses_canonical_history_reasoning_when_wire_strips_it(
self,
) -> None:
thinking = {
"type": "thinking",
"thinking": "Reason through the tool result.",
"signature": "sig-1",
}
_plugin.on_pre_api_request(
session_id="session-1",
api_mode="anthropic_messages",
system_prompt="system",
request_messages=[
{
"role": "assistant",
"content": [{"type": "text", "text": "Checking."}],
},
{"role": "user", "content": "continue"},
],
conversation_history=[
{
"role": "assistant",
"content": "Checking.",
"reasoning": "Reason through the tool result.",
"reasoning_content": "Reason through the tool result.",
"reasoning_details": [thinking],
},
{"role": "user", "content": "continue"},
],
)
self.assertEqual(
self._full_request(),
[
{"role": "system", "content": "system"},
{
"role": "assistant",
"content": "Checking.",
"reasoning_content": "Reason through the tool result.",
"reasoning_details": [thinking],
},
{"role": "user", "content": "continue"},
],
)
def test_existing_system_message_is_not_duplicated(self) -> None:
"""A request system role suppresses every fallback source."""
messages = [
{"role": "system", "content": "existing"},
{"role": "user", "content": "hello"},
]
with patch(
f"{_module_name}._state._get_system_prompt_from_state_db"
) as state_db:
_plugin.on_pre_api_request(
session_id="session-1",
request_messages=messages,
conversation_history=[{"role": "system", "content": "history"}],
request={"body": {"instructions": "request"}},
)
self.assertEqual(self._full_request(), messages)
state_db.assert_not_called()
class TestPostAPIRequest(unittest.TestCase):
"""Verify completed API calls are normalized and persisted safely."""
def setUp(self) -> None:
self.state = {
"session-1:turn:turn-1": {
"request_preview": "request preview",
"full_request": '[{"role":"user","content":"hello"}]',
}
}
self.repository = create_autospec(TelemetryDB, instance=True)
self.repository.write_turn.return_value = 91
def test_normalizes_provider_usage_reasoning_and_none_duration(self) -> None:
"""Provider aliases and optional metadata reach write_turn unchanged."""
assistant = SimpleNamespace(
content=[{"text": "answer"}],
reasoning=None,
provider_data={"reasoning_content": "provider reasoning"},
reasoning_content="secondary reasoning",
)
with (
patch(f"{_module_name}._state._session_state", self.state),
patch(f"{_module_name}.db.get_db", return_value=self.repository),
):
_plugin.on_post_api_request(
session_id="session-1",
turn_id="turn-1",
model="model-1",
provider="provider-1",
usage={
"prompt_tokens": 11,
"completion_tokens": 7,
"total_tokens": 18,
"cache_read_tokens": 5,
"cache_write_tokens": 2,
"reasoning_tokens": 3,
},
api_duration=None,
finish_reason="stop",
assistant_message=assistant,
assistant_tool_call_count=0,
api_call_count=4,
response={"id": "response-1"},
)
self.repository.write_turn.assert_called_once_with(
session_id="session-1",
model="model-1",
provider="provider-1",
api_call_count=4,
request_preview="request preview",
response_preview="answer",
reasoning="provider reasoning",
finish_reason="stop",
input_tokens=11,
output_tokens=7,
total_tokens=18,
cache_read_tokens=5,
cache_write_tokens=2,
reasoning_tokens=3,
elapsed_ms=None,
tool_calls_count=0,
full_request='[{"role":"user","content":"hello"}]',
full_response='{"id": "response-1"}',
)
self.assertNotIn("session-1:turn:turn-1", self.state)
def test_responses_reclassifies_message_items_as_response_before_storage(
self,
) -> None:
assistant = SimpleNamespace(
content="",
reasoning="Planning second terminal call\n\n第二回合也很順。",
codex_reasoning_items=[
{
"type": "reasoning",
"encrypted_content": "opaque",
"summary": [
{
"type": "summary_text",
"text": "Planning second terminal call",
}
],
}
],
codex_message_items=[
{
"type": "message",
"role": "assistant",
"phase": "commentary",
"content": [{"type": "output_text", "text": "第二回合也很順。"}],
}
],
)
response = {
"assistant_message": {
"role": "assistant",
"content": "",
"tool_calls": [{"id": "call-1"}],
}
}
with (
patch(f"{_module_name}._state._session_state", self.state),
patch(f"{_module_name}.db.get_db", return_value=self.repository),
):
_plugin.on_post_api_request(
session_id="session-1",
turn_id="turn-1",
model="gpt-5.6-luna",
provider="chatgpt",
usage={},
api_duration=0.2,
finish_reason="tool_calls",
assistant_message=assistant,
assistant_tool_call_count=1,
api_call_count=1,
response=response,
)
kwargs = self.repository.write_turn.call_args.kwargs
self.assertEqual(kwargs["response_preview"], "第二回合也很順。")
self.assertEqual(kwargs["reasoning"], "Planning second terminal call")
full_response = json.loads(kwargs["full_response"])
self.assertEqual(
full_response["assistant_message"]["content"], "第二回合也很順。"
)
self.assertEqual(
full_response["assistant_message"]["reasoning_content"],
"Planning second terminal call",
)
def test_anthropic_reasoning_details_fallback_and_full_response_projection(
self,
) -> None:
thinking = {
"type": "thinking",
"thinking": "Reason through the tool result.",
"signature": "sig-1",
}
assistant = SimpleNamespace(
content="answer",
reasoning=None,
reasoning_content=None,
reasoning_details=[thinking],
provider_data={"reasoning_details": [thinking]},
)
response = {
"assistant_message": {
"role": "assistant",
"content": "answer",
"tool_calls": [],
}
}
with (
patch(f"{_module_name}._state._session_state", self.state),
patch(f"{_module_name}.db.get_db", return_value=self.repository),
):
_plugin.on_post_api_request(
session_id="session-1",
turn_id="turn-1",
model="MiniMax-M3",
provider="minimax",
api_mode="anthropic_messages",
usage={},
api_duration=0.2,
finish_reason="stop",
assistant_message=assistant,
assistant_tool_call_count=0,
api_call_count=1,
response=response,
)
kwargs = self.repository.write_turn.call_args.kwargs
self.assertEqual(kwargs["reasoning"], "Reason through the tool result.")
self.assertEqual(
json.loads(kwargs["full_response"])["assistant_message"][
"reasoning_content"
],
"Reason through the tool result.",
)
def test_database_failure_is_logged_and_does_not_escape(self) -> None:
"""Telemetry write failures never crash Hermes hook execution."""
self.repository.write_turn.side_effect = RuntimeError("database offline")
with (
patch(f"{_module_name}._state._session_state", self.state),
patch(f"{_module_name}.db.get_db", return_value=self.repository),
self.assertLogs(f"{_module_name}.hooks", level="ERROR") as logs,
):
_plugin.on_post_api_request(
session_id="session-1",
turn_id="turn-1",
model="model-1",
usage={},
api_duration=0.5,
finish_reason="stop",
assistant_message=SimpleNamespace(content="answer", reasoning=None),
assistant_tool_call_count=0,
api_call_count=1,
response={},
)
self.assertIn("failed to write turn", "\n".join(logs.output))
class TestPostToolCallFailures(unittest.TestCase):
"""Verify tool-call guard clauses and failure containment."""
def setUp(self) -> None:
self.repository = create_autospec(TelemetryDB, instance=True)
def test_missing_state_skips_database_write(self) -> None:
"""A tool result without correlated state is ignored with a warning."""
with (
patch(f"{_module_name}._state._session_state", {}),
patch(f"{_module_name}.db.get_db", return_value=self.repository),
self.assertLogs(f"{_module_name}.hooks", level="WARNING") as logs,
):
_plugin.on_post_tool_call(
session_id="missing",
tool_name="tool",
args={},
result={},
duration_ms=None,
)
self.repository.write_tool_call.assert_not_called()
self.assertIn("no session state", "\n".join(logs.output))
def test_wrong_latest_turn_id_type_skips_database_write(self) -> None:
"""Only an integer database turn id may be used as a foreign key."""
state = {"session-1": {"latest_turn_id": "91"}}
with (
patch(f"{_module_name}._state._session_state", state),
patch(f"{_module_name}.db.get_db", return_value=self.repository),
self.assertLogs(f"{_module_name}.hooks", level="WARNING") as logs,
):
_plugin.on_post_tool_call(
session_id="session-1",
tool_name="tool",
args={},
result={},
duration_ms=None,
)
self.repository.write_tool_call.assert_not_called()
self.assertIn("no latest_turn_id", "\n".join(logs.output))
def test_serializes_non_json_values_with_default_str(self) -> None:
"""Hook objects outside JSON's native types still persist safely."""
marker = SimpleNamespace(value="custom")
state = {
"session-1": {
"latest_turn_id": 91,
"tool_call_counter": 0,
"expected_tool_calls": 2,
}
}
with (
patch(f"{_module_name}._state._session_state", state),
patch(f"{_module_name}.db.get_db", return_value=self.repository),
):
_plugin.on_post_tool_call(
session_id="session-1",
tool_name="custom",
args={"value": marker},
result=[marker],
duration_ms=12,
)
self.repository.write_tool_call.assert_called_once_with(
turn_id=91,
call_order=0,
name="custom",
args=json.dumps({"value": marker}, ensure_ascii=False, default=str),
result=json.dumps([marker], ensure_ascii=False, default=str),
duration_ms=12,
error=None,
)
def test_database_failure_is_logged_and_does_not_escape(self) -> None:
"""A final tool write failure remains contained after state cleanup."""
self.repository.write_tool_call.side_effect = RuntimeError("database offline")
state = {
"session-1": {
"latest_turn_id": 91,
"tool_call_counter": 0,
"expected_tool_calls": 1,
}
}
with (
patch(f"{_module_name}._state._session_state", state),
patch(f"{_module_name}.db.get_db", return_value=self.repository),
self.assertLogs(f"{_module_name}.hooks", level="ERROR") as logs,
):
_plugin.on_post_tool_call(
session_id="session-1",
tool_name="tool",
args={},
result={},
duration_ms=1,
)
self.assertNotIn("session-1", state)
self.assertIn("failed to write tool call", "\n".join(logs.output))
if __name__ == "__main__":
unittest.main()