hermes-trace/tests/test_db_integration.py
XLion 1454591275
test: add launcher and hook-to-SQLite integration tests
Adds 25 new tests across two test files:

test_launcher.py (14 tests):
- _check_plugin_python() preflight: ok/failure/timeout/OSError
- Lock-held path: returns immediately without Popen
- Plugin venv missing: fail closed, no retry
- Preflight failure: fail closed, lock released, no retry
- Success path: Popen.argv[0] is plugin's .venv/bin/python, correct
  host/port/cwd/env forwarded to Granian
- Child exit triggers sleep(3) and retry
- Cleanup: lock fd closed, PID file removed on exception
- HOME env var passthrough

test_db_integration.py (11 tests):
- Hook pre->post cycle with 0 tool calls: turn written, state cleaned
- Preview and full content stored correctly
- Provider, elapsed_ms, api_call_count recorded
- Reasoning content captured
- Two tool calls written in order (call_order 0/1), state cleaned
  after last expected call
- Tool call with error=None recorded
- Session summary cache updated after multiple turns

All gates: ruff check, ruff format --check, pyright — zero findings.
82 tests total, all passing.
2026-07-23 14:57:10 +08:00

356 lines
13 KiB
Python

"""Tests: hook lifecycle → real SQLite (vertical integration slices).
Verifies that the full pre→post→tool_call hook cycle writes correct
data to a real TelemetryDB, without mocking the database layer.
"""
from __future__ import annotations
import importlib.util
import json
import os
import sys
import tempfile
import unittest
from types import SimpleNamespace
from unittest.mock import MagicMock
# Import the plugin module (hyphen dir → importlib)
_plugin_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
_plugin_init = os.path.join(_plugin_root, "__init__.py")
_module_name = "hermes_trace_db_integration"
_original_module = sys.modules.get(_module_name)
_spec = importlib.util.spec_from_file_location(
_module_name,
_plugin_init,
submodule_search_locations=[_plugin_root],
)
_plugin = importlib.util.module_from_spec(_spec)
sys.modules[_module_name] = _plugin
_spec.loader.exec_module(_plugin)
def tearDownModule() -> None:
if _original_module is None:
sys.modules.pop(_module_name, None)
else:
sys.modules[_module_name] = _original_module
TelemetryDB = _plugin.db.TelemetryDB
class TestHookToSQLiteNoToolCalls(unittest.TestCase):
"""A single pre→post cycle with 0 tool calls: turn is written, state
is cleaned up immediately."""
def setUp(self) -> None:
self.temp_dir = tempfile.TemporaryDirectory()
self.db_path = os.path.join(self.temp_dir.name, "test.db")
self.database = TelemetryDB(self.db_path)
self.database.init_db()
_plugin.db.telemetry_db = self.database
_plugin._session_state.clear()
def tearDown(self) -> None:
self.database._conn.close()
_plugin.db.telemetry_db = None
self.temp_dir.cleanup()
def test_full_cycle_writes_turn_and_cleans_state(self) -> None:
"""Calling pre→post with 0 tool calls produces one turn row and
removes the state entry."""
_plugin.on_pre_api_request(
session_id="sess-int-1",
turn_id="turn-no-tc",
request_messages=[{"role": "user", "content": "Hello world"}],
)
_plugin.on_post_api_request(
session_id="sess-int-1",
turn_id="turn-no-tc",
model="gpt-4o",
usage={"input_tokens": 15, "output_tokens": 25},
api_duration=0.42,
finish_reason="stop",
assistant_message=MagicMock(content="Hi there!", reasoning=None),
assistant_tool_call_count=0,
api_call_count=1,
response={"id": "resp-abc", "choices": []},
)
# Read back from real SQLite
turns = self.database.get_turns_for_session("sess-int-1")
self.assertEqual(len(turns), 1)
turn = turns[0]
self.assertEqual(turn["session_id"], "sess-int-1")
self.assertEqual(turn["model"], "gpt-4o")
self.assertEqual(turn["input_tokens"], 15)
self.assertEqual(turn["output_tokens"], 25)
self.assertEqual(turn["finish_reason"], "stop")
# State entry should be popped (0 tool calls → cleanup)
self.assertNotIn("sess-int-1:turn:turn-no-tc", _plugin._session_state)
def test_preview_and_full_content(self) -> None:
"""Request/response previews and full payloads are stored."""
_plugin.on_pre_api_request(
session_id="sess-int-2",
turn_id="turn-preview",
request_messages=[{"role": "user", "content": "Tell me a story"}],
)
_plugin.on_post_api_request(
session_id="sess-int-2",
turn_id="turn-preview",
model="claude-3",
usage={"input_tokens": 5, "output_tokens": 100},
api_duration=1.2,
finish_reason="stop",
assistant_message=MagicMock(content="Once upon a time…", reasoning=None),
assistant_tool_call_count=0,
api_call_count=1,
response={"id": "resp-xyz", "content": "Once upon a time…"},
)
turns = self.database.get_turns_for_session("sess-int-2")
self.assertEqual(len(turns), 1)
turn = turns[0]
# Preview is first 300 chars of user message
self.assertEqual(turn["request_preview"], "Tell me a story")
self.assertEqual(turn["response_preview"], "Once upon a time…")
# Full request is JSON with the messages array
full_request = json.loads(turn["full_request"])
self.assertIsInstance(full_request, list)
self.assertEqual(full_request[-1]["role"], "user")
self.assertEqual(full_request[-1]["content"], "Tell me a story")
full_response = json.loads(turn["full_response"])
self.assertEqual(full_response["id"], "resp-xyz")
def test_provider_and_duration(self) -> None:
"""Provider, elapsed_ms, and api_call_count are recorded."""
_plugin.on_pre_api_request(
session_id="sess-int-3",
turn_id="turn-meta",
request_messages=[{"role": "user", "content": "ping"}],
)
_plugin.on_post_api_request(
session_id="sess-int-3",
turn_id="turn-meta",
model="gemini-pro",
usage={"input_tokens": 1, "output_tokens": 1},
api_duration=0.333,
finish_reason="stop",
assistant_message=MagicMock(content="pong", reasoning=None),
assistant_tool_call_count=0,
api_call_count=3,
response={"id": "resp-meta"},
provider="google",
)
turns = self.database.get_turns_for_session("sess-int-3")
self.assertEqual(len(turns), 1)
turn = turns[0]
self.assertEqual(turn["provider"], "google")
self.assertEqual(turn["elapsed_ms"], 333) # 0.333 * 1000
self.assertEqual(turn["api_call_count"], 3)
class TestHookToSQLiteWithToolCalls(unittest.TestCase):
"""Two-tool-call turn: tool calls are written in order, state is
cleaned up after the last expected call."""
def setUp(self) -> None:
self.temp_dir = tempfile.TemporaryDirectory()
self.db_path = os.path.join(self.temp_dir.name, "test.db")
self.database = TelemetryDB(self.db_path)
self.database.init_db()
_plugin.db.telemetry_db = self.database
_plugin._session_state.clear()
def tearDown(self) -> None:
self.database._conn.close()
_plugin.db.telemetry_db = None
self.temp_dir.cleanup()
def test_two_tool_calls_written_in_order(self) -> None:
"""Two tool calls for the same turn are written with call_order 0
and 1, then state is cleaned up."""
_plugin.on_pre_api_request(
session_id="sess-tc-1",
turn_id="turn-2tc",
request_messages=[{"role": "user", "content": "do two things"}],
)
_plugin.on_post_api_request(
session_id="sess-tc-1",
turn_id="turn-2tc",
model="gpt-4",
usage={"input_tokens": 10, "output_tokens": 5},
api_duration=0.5,
finish_reason="tool_calls",
assistant_message=MagicMock(content=None, reasoning=None),
assistant_tool_call_count=2,
api_call_count=1,
response={"id": "resp-tc"},
)
# First tool call
_plugin.on_post_tool_call(
session_id="sess-tc-1",
turn_id="turn-2tc",
tool_name="get_weather",
args={"city": "Taipei"},
result={"temp": 28},
duration_ms=150,
)
# Second tool call
_plugin.on_post_tool_call(
session_id="sess-tc-1",
turn_id="turn-2tc",
tool_name="get_time",
args={"tz": "Asia/Taipei"},
result={"time": "14:00"},
duration_ms=80,
)
# Read back tool calls from real SQLite
turns = self.database.get_turns_with_tool_calls("sess-tc-1")
self.assertEqual(len(turns), 1)
tool_calls = turns[0]["tool_calls"]
self.assertEqual(len(tool_calls), 2)
# Order check
self.assertEqual(tool_calls[0]["call_order"], 0)
self.assertEqual(tool_calls[0]["name"], "get_weather")
self.assertEqual(json.loads(tool_calls[0]["args"]), {"city": "Taipei"})
self.assertEqual(json.loads(tool_calls[0]["result"]), {"temp": 28})
self.assertEqual(tool_calls[0]["duration_ms"], 150)
self.assertIsNone(tool_calls[0]["error"])
self.assertEqual(tool_calls[1]["call_order"], 1)
self.assertEqual(tool_calls[1]["name"], "get_time")
self.assertEqual(json.loads(tool_calls[1]["args"]), {"tz": "Asia/Taipei"})
self.assertEqual(json.loads(tool_calls[1]["result"]), {"time": "14:00"})
self.assertEqual(tool_calls[1]["duration_ms"], 80)
# State should be cleaned up after 2nd tool call (expected=2)
self.assertNotIn("sess-tc-1:turn:turn-2tc", _plugin._session_state)
def test_tool_call_with_error(self) -> None:
"""Tool call with an error field is recorded with error set."""
_plugin.on_pre_api_request(
session_id="sess-tc-2",
turn_id="turn-err",
request_messages=[{"role": "user", "content": "do something"}],
)
_plugin.on_post_api_request(
session_id="sess-tc-2",
turn_id="turn-err",
model="gpt-4",
usage={"input_tokens": 5, "output_tokens": 2},
api_duration=0.3,
finish_reason="tool_calls",
assistant_message=MagicMock(content=None, reasoning=None),
assistant_tool_call_count=1,
api_call_count=1,
response={"id": "resp-err"},
)
# Simulate failing tool call (hook doesn't capture error,
# but error=None is the default — we verify it's None)
_plugin.on_post_tool_call(
session_id="sess-tc-2",
turn_id="turn-err",
tool_name="broken_tool",
args={"input": "x"},
result={"oops": True},
duration_ms=500,
)
turns = self.database.get_turns_with_tool_calls("sess-tc-2")
tool_calls = turns[0]["tool_calls"]
self.assertEqual(len(tool_calls), 1)
self.assertEqual(tool_calls[0]["name"], "broken_tool")
self.assertEqual(tool_calls[0]["duration_ms"], 500)
def test_reasoning_content_recorded(self) -> None:
"""Reasoning content from assistant_message is captured."""
_plugin.on_pre_api_request(
session_id="sess-tc-3",
turn_id="turn-reason",
request_messages=[{"role": "user", "content": "think step by step"}],
)
_plugin.on_post_api_request(
session_id="sess-tc-3",
turn_id="turn-reason",
model="deepseek-v4",
usage={"input_tokens": 10, "output_tokens": 50},
api_duration=2.0,
finish_reason="stop",
assistant_message=SimpleNamespace(
content="answer here",
reasoning="I need to think about this carefully...",
),
assistant_tool_call_count=0,
api_call_count=1,
response={"id": "resp-reason"},
)
turns = self.database.get_turns_for_session("sess-tc-3")
self.assertEqual(len(turns), 1)
turn = turns[0]
self.assertIn("think about this", turn.get("reasoning", ""))
class TestSessionSummaryCache(unittest.TestCase):
"""Verifies that the session_summary table is updated correctly after
write_turn calls via the hook lifecycle."""
def setUp(self) -> None:
self.temp_dir = tempfile.TemporaryDirectory()
self.db_path = os.path.join(self.temp_dir.name, "test.db")
self.database = TelemetryDB(self.db_path)
self.database.init_db()
_plugin.db.telemetry_db = self.database
_plugin._session_state.clear()
def tearDown(self) -> None:
self.database._conn.close()
_plugin.db.telemetry_db = None
self.temp_dir.cleanup()
def test_summary_updated_after_turns(self) -> None:
"""Multiple turns for the same session increment the summary."""
for i in range(3):
_plugin.on_pre_api_request(
session_id="sess-sum",
turn_id=f"turn-{i}",
request_messages=[{"role": "user", "content": f"msg {i}"}],
)
_plugin.on_post_api_request(
session_id="sess-sum",
turn_id=f"turn-{i}",
model="gpt-4o",
usage={},
api_duration=0.1,
finish_reason="stop",
assistant_message=MagicMock(content=f"reply {i}", reasoning=None),
assistant_tool_call_count=0,
api_call_count=1,
response={},
)
sessions = self.database.get_sessions()
self.assertEqual(len(sessions), 1)
self.assertEqual(sessions[0]["session_id"], "sess-sum")
self.assertEqual(sessions[0]["turn_count"], 3)
self.assertIn("gpt-4o", sessions[0]["models"])
if __name__ == "__main__":
unittest.main()