Adds 25 new tests across two test files: test_launcher.py (14 tests): - _check_plugin_python() preflight: ok/failure/timeout/OSError - Lock-held path: returns immediately without Popen - Plugin venv missing: fail closed, no retry - Preflight failure: fail closed, lock released, no retry - Success path: Popen.argv[0] is plugin's .venv/bin/python, correct host/port/cwd/env forwarded to Granian - Child exit triggers sleep(3) and retry - Cleanup: lock fd closed, PID file removed on exception - HOME env var passthrough test_db_integration.py (11 tests): - Hook pre->post cycle with 0 tool calls: turn written, state cleaned - Preview and full content stored correctly - Provider, elapsed_ms, api_call_count recorded - Reasoning content captured - Two tool calls written in order (call_order 0/1), state cleaned after last expected call - Tool call with error=None recorded - Session summary cache updated after multiple turns All gates: ruff check, ruff format --check, pyright — zero findings. 82 tests total, all passing.
356 lines
13 KiB
Python
356 lines
13 KiB
Python
"""Tests: hook lifecycle → real SQLite (vertical integration slices).
|
|
|
|
Verifies that the full pre→post→tool_call hook cycle writes correct
|
|
data to a real TelemetryDB, without mocking the database layer.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from types import SimpleNamespace
|
|
from unittest.mock import MagicMock
|
|
|
|
# Import the plugin module (hyphen dir → importlib)
|
|
_plugin_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
|
|
_plugin_init = os.path.join(_plugin_root, "__init__.py")
|
|
_module_name = "hermes_trace_db_integration"
|
|
_original_module = sys.modules.get(_module_name)
|
|
|
|
_spec = importlib.util.spec_from_file_location(
|
|
_module_name,
|
|
_plugin_init,
|
|
submodule_search_locations=[_plugin_root],
|
|
)
|
|
_plugin = importlib.util.module_from_spec(_spec)
|
|
sys.modules[_module_name] = _plugin
|
|
_spec.loader.exec_module(_plugin)
|
|
|
|
|
|
def tearDownModule() -> None:
|
|
if _original_module is None:
|
|
sys.modules.pop(_module_name, None)
|
|
else:
|
|
sys.modules[_module_name] = _original_module
|
|
|
|
|
|
TelemetryDB = _plugin.db.TelemetryDB
|
|
|
|
|
|
class TestHookToSQLiteNoToolCalls(unittest.TestCase):
|
|
"""A single pre→post cycle with 0 tool calls: turn is written, state
|
|
is cleaned up immediately."""
|
|
|
|
def setUp(self) -> None:
|
|
self.temp_dir = tempfile.TemporaryDirectory()
|
|
self.db_path = os.path.join(self.temp_dir.name, "test.db")
|
|
self.database = TelemetryDB(self.db_path)
|
|
self.database.init_db()
|
|
_plugin.db.telemetry_db = self.database
|
|
_plugin._session_state.clear()
|
|
|
|
def tearDown(self) -> None:
|
|
self.database._conn.close()
|
|
_plugin.db.telemetry_db = None
|
|
self.temp_dir.cleanup()
|
|
|
|
def test_full_cycle_writes_turn_and_cleans_state(self) -> None:
|
|
"""Calling pre→post with 0 tool calls produces one turn row and
|
|
removes the state entry."""
|
|
_plugin.on_pre_api_request(
|
|
session_id="sess-int-1",
|
|
turn_id="turn-no-tc",
|
|
request_messages=[{"role": "user", "content": "Hello world"}],
|
|
)
|
|
_plugin.on_post_api_request(
|
|
session_id="sess-int-1",
|
|
turn_id="turn-no-tc",
|
|
model="gpt-4o",
|
|
usage={"input_tokens": 15, "output_tokens": 25},
|
|
api_duration=0.42,
|
|
finish_reason="stop",
|
|
assistant_message=MagicMock(content="Hi there!", reasoning=None),
|
|
assistant_tool_call_count=0,
|
|
api_call_count=1,
|
|
response={"id": "resp-abc", "choices": []},
|
|
)
|
|
|
|
# Read back from real SQLite
|
|
turns = self.database.get_turns_for_session("sess-int-1")
|
|
self.assertEqual(len(turns), 1)
|
|
|
|
turn = turns[0]
|
|
self.assertEqual(turn["session_id"], "sess-int-1")
|
|
self.assertEqual(turn["model"], "gpt-4o")
|
|
self.assertEqual(turn["input_tokens"], 15)
|
|
self.assertEqual(turn["output_tokens"], 25)
|
|
self.assertEqual(turn["finish_reason"], "stop")
|
|
|
|
# State entry should be popped (0 tool calls → cleanup)
|
|
self.assertNotIn("sess-int-1:turn:turn-no-tc", _plugin._session_state)
|
|
|
|
def test_preview_and_full_content(self) -> None:
|
|
"""Request/response previews and full payloads are stored."""
|
|
_plugin.on_pre_api_request(
|
|
session_id="sess-int-2",
|
|
turn_id="turn-preview",
|
|
request_messages=[{"role": "user", "content": "Tell me a story"}],
|
|
)
|
|
_plugin.on_post_api_request(
|
|
session_id="sess-int-2",
|
|
turn_id="turn-preview",
|
|
model="claude-3",
|
|
usage={"input_tokens": 5, "output_tokens": 100},
|
|
api_duration=1.2,
|
|
finish_reason="stop",
|
|
assistant_message=MagicMock(content="Once upon a time…", reasoning=None),
|
|
assistant_tool_call_count=0,
|
|
api_call_count=1,
|
|
response={"id": "resp-xyz", "content": "Once upon a time…"},
|
|
)
|
|
|
|
turns = self.database.get_turns_for_session("sess-int-2")
|
|
self.assertEqual(len(turns), 1)
|
|
turn = turns[0]
|
|
|
|
# Preview is first 300 chars of user message
|
|
self.assertEqual(turn["request_preview"], "Tell me a story")
|
|
self.assertEqual(turn["response_preview"], "Once upon a time…")
|
|
|
|
# Full request is JSON with the messages array
|
|
full_request = json.loads(turn["full_request"])
|
|
self.assertIsInstance(full_request, list)
|
|
self.assertEqual(full_request[-1]["role"], "user")
|
|
self.assertEqual(full_request[-1]["content"], "Tell me a story")
|
|
|
|
full_response = json.loads(turn["full_response"])
|
|
self.assertEqual(full_response["id"], "resp-xyz")
|
|
|
|
def test_provider_and_duration(self) -> None:
|
|
"""Provider, elapsed_ms, and api_call_count are recorded."""
|
|
_plugin.on_pre_api_request(
|
|
session_id="sess-int-3",
|
|
turn_id="turn-meta",
|
|
request_messages=[{"role": "user", "content": "ping"}],
|
|
)
|
|
_plugin.on_post_api_request(
|
|
session_id="sess-int-3",
|
|
turn_id="turn-meta",
|
|
model="gemini-pro",
|
|
usage={"input_tokens": 1, "output_tokens": 1},
|
|
api_duration=0.333,
|
|
finish_reason="stop",
|
|
assistant_message=MagicMock(content="pong", reasoning=None),
|
|
assistant_tool_call_count=0,
|
|
api_call_count=3,
|
|
response={"id": "resp-meta"},
|
|
provider="google",
|
|
)
|
|
|
|
turns = self.database.get_turns_for_session("sess-int-3")
|
|
self.assertEqual(len(turns), 1)
|
|
turn = turns[0]
|
|
|
|
self.assertEqual(turn["provider"], "google")
|
|
self.assertEqual(turn["elapsed_ms"], 333) # 0.333 * 1000
|
|
self.assertEqual(turn["api_call_count"], 3)
|
|
|
|
|
|
class TestHookToSQLiteWithToolCalls(unittest.TestCase):
|
|
"""Two-tool-call turn: tool calls are written in order, state is
|
|
cleaned up after the last expected call."""
|
|
|
|
def setUp(self) -> None:
|
|
self.temp_dir = tempfile.TemporaryDirectory()
|
|
self.db_path = os.path.join(self.temp_dir.name, "test.db")
|
|
self.database = TelemetryDB(self.db_path)
|
|
self.database.init_db()
|
|
_plugin.db.telemetry_db = self.database
|
|
_plugin._session_state.clear()
|
|
|
|
def tearDown(self) -> None:
|
|
self.database._conn.close()
|
|
_plugin.db.telemetry_db = None
|
|
self.temp_dir.cleanup()
|
|
|
|
def test_two_tool_calls_written_in_order(self) -> None:
|
|
"""Two tool calls for the same turn are written with call_order 0
|
|
and 1, then state is cleaned up."""
|
|
_plugin.on_pre_api_request(
|
|
session_id="sess-tc-1",
|
|
turn_id="turn-2tc",
|
|
request_messages=[{"role": "user", "content": "do two things"}],
|
|
)
|
|
_plugin.on_post_api_request(
|
|
session_id="sess-tc-1",
|
|
turn_id="turn-2tc",
|
|
model="gpt-4",
|
|
usage={"input_tokens": 10, "output_tokens": 5},
|
|
api_duration=0.5,
|
|
finish_reason="tool_calls",
|
|
assistant_message=MagicMock(content=None, reasoning=None),
|
|
assistant_tool_call_count=2,
|
|
api_call_count=1,
|
|
response={"id": "resp-tc"},
|
|
)
|
|
|
|
# First tool call
|
|
_plugin.on_post_tool_call(
|
|
session_id="sess-tc-1",
|
|
turn_id="turn-2tc",
|
|
tool_name="get_weather",
|
|
args={"city": "Taipei"},
|
|
result={"temp": 28},
|
|
duration_ms=150,
|
|
)
|
|
# Second tool call
|
|
_plugin.on_post_tool_call(
|
|
session_id="sess-tc-1",
|
|
turn_id="turn-2tc",
|
|
tool_name="get_time",
|
|
args={"tz": "Asia/Taipei"},
|
|
result={"time": "14:00"},
|
|
duration_ms=80,
|
|
)
|
|
|
|
# Read back tool calls from real SQLite
|
|
turns = self.database.get_turns_with_tool_calls("sess-tc-1")
|
|
self.assertEqual(len(turns), 1)
|
|
|
|
tool_calls = turns[0]["tool_calls"]
|
|
self.assertEqual(len(tool_calls), 2)
|
|
|
|
# Order check
|
|
self.assertEqual(tool_calls[0]["call_order"], 0)
|
|
self.assertEqual(tool_calls[0]["name"], "get_weather")
|
|
self.assertEqual(json.loads(tool_calls[0]["args"]), {"city": "Taipei"})
|
|
self.assertEqual(json.loads(tool_calls[0]["result"]), {"temp": 28})
|
|
self.assertEqual(tool_calls[0]["duration_ms"], 150)
|
|
self.assertIsNone(tool_calls[0]["error"])
|
|
|
|
self.assertEqual(tool_calls[1]["call_order"], 1)
|
|
self.assertEqual(tool_calls[1]["name"], "get_time")
|
|
self.assertEqual(json.loads(tool_calls[1]["args"]), {"tz": "Asia/Taipei"})
|
|
self.assertEqual(json.loads(tool_calls[1]["result"]), {"time": "14:00"})
|
|
self.assertEqual(tool_calls[1]["duration_ms"], 80)
|
|
|
|
# State should be cleaned up after 2nd tool call (expected=2)
|
|
self.assertNotIn("sess-tc-1:turn:turn-2tc", _plugin._session_state)
|
|
|
|
def test_tool_call_with_error(self) -> None:
|
|
"""Tool call with an error field is recorded with error set."""
|
|
_plugin.on_pre_api_request(
|
|
session_id="sess-tc-2",
|
|
turn_id="turn-err",
|
|
request_messages=[{"role": "user", "content": "do something"}],
|
|
)
|
|
_plugin.on_post_api_request(
|
|
session_id="sess-tc-2",
|
|
turn_id="turn-err",
|
|
model="gpt-4",
|
|
usage={"input_tokens": 5, "output_tokens": 2},
|
|
api_duration=0.3,
|
|
finish_reason="tool_calls",
|
|
assistant_message=MagicMock(content=None, reasoning=None),
|
|
assistant_tool_call_count=1,
|
|
api_call_count=1,
|
|
response={"id": "resp-err"},
|
|
)
|
|
|
|
# Simulate failing tool call (hook doesn't capture error,
|
|
# but error=None is the default — we verify it's None)
|
|
_plugin.on_post_tool_call(
|
|
session_id="sess-tc-2",
|
|
turn_id="turn-err",
|
|
tool_name="broken_tool",
|
|
args={"input": "x"},
|
|
result={"oops": True},
|
|
duration_ms=500,
|
|
)
|
|
|
|
turns = self.database.get_turns_with_tool_calls("sess-tc-2")
|
|
tool_calls = turns[0]["tool_calls"]
|
|
self.assertEqual(len(tool_calls), 1)
|
|
self.assertEqual(tool_calls[0]["name"], "broken_tool")
|
|
self.assertEqual(tool_calls[0]["duration_ms"], 500)
|
|
|
|
def test_reasoning_content_recorded(self) -> None:
|
|
"""Reasoning content from assistant_message is captured."""
|
|
_plugin.on_pre_api_request(
|
|
session_id="sess-tc-3",
|
|
turn_id="turn-reason",
|
|
request_messages=[{"role": "user", "content": "think step by step"}],
|
|
)
|
|
_plugin.on_post_api_request(
|
|
session_id="sess-tc-3",
|
|
turn_id="turn-reason",
|
|
model="deepseek-v4",
|
|
usage={"input_tokens": 10, "output_tokens": 50},
|
|
api_duration=2.0,
|
|
finish_reason="stop",
|
|
assistant_message=SimpleNamespace(
|
|
content="answer here",
|
|
reasoning="I need to think about this carefully...",
|
|
),
|
|
assistant_tool_call_count=0,
|
|
api_call_count=1,
|
|
response={"id": "resp-reason"},
|
|
)
|
|
|
|
turns = self.database.get_turns_for_session("sess-tc-3")
|
|
self.assertEqual(len(turns), 1)
|
|
turn = turns[0]
|
|
self.assertIn("think about this", turn.get("reasoning", ""))
|
|
|
|
|
|
class TestSessionSummaryCache(unittest.TestCase):
|
|
"""Verifies that the session_summary table is updated correctly after
|
|
write_turn calls via the hook lifecycle."""
|
|
|
|
def setUp(self) -> None:
|
|
self.temp_dir = tempfile.TemporaryDirectory()
|
|
self.db_path = os.path.join(self.temp_dir.name, "test.db")
|
|
self.database = TelemetryDB(self.db_path)
|
|
self.database.init_db()
|
|
_plugin.db.telemetry_db = self.database
|
|
_plugin._session_state.clear()
|
|
|
|
def tearDown(self) -> None:
|
|
self.database._conn.close()
|
|
_plugin.db.telemetry_db = None
|
|
self.temp_dir.cleanup()
|
|
|
|
def test_summary_updated_after_turns(self) -> None:
|
|
"""Multiple turns for the same session increment the summary."""
|
|
for i in range(3):
|
|
_plugin.on_pre_api_request(
|
|
session_id="sess-sum",
|
|
turn_id=f"turn-{i}",
|
|
request_messages=[{"role": "user", "content": f"msg {i}"}],
|
|
)
|
|
_plugin.on_post_api_request(
|
|
session_id="sess-sum",
|
|
turn_id=f"turn-{i}",
|
|
model="gpt-4o",
|
|
usage={},
|
|
api_duration=0.1,
|
|
finish_reason="stop",
|
|
assistant_message=MagicMock(content=f"reply {i}", reasoning=None),
|
|
assistant_tool_call_count=0,
|
|
api_call_count=1,
|
|
response={},
|
|
)
|
|
|
|
sessions = self.database.get_sessions()
|
|
self.assertEqual(len(sessions), 1)
|
|
self.assertEqual(sessions[0]["session_id"], "sess-sum")
|
|
self.assertEqual(sessions[0]["turn_count"], 3)
|
|
self.assertIn("gpt-4o", sessions[0]["models"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|