hermes-trace/tests/test_db_preview.py

562 lines
19 KiB
Python

"""Tests for lightweight session-listing database queries."""
from __future__ import annotations
import json
import os
import sys
import unittest
_plugin_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
sys.path.insert(0, _plugin_root)
from db import TelemetryDB
try:
sys.path.remove(_plugin_root)
except ValueError:
pass
class TestTurnsWithPreview(unittest.TestCase):
"""Verify projected turns and tool calls preserve listing behavior."""
def setUp(self) -> None:
"""Create an initialized in-memory telemetry database."""
self.db = TelemetryDB(":memory:")
self.db.init_db()
def tearDown(self) -> None:
"""Close the in-memory database connection."""
self.db._conn.close()
def _write_turn(self, *, created_at: str, marker: str) -> int:
"""Insert a turn containing payloads larger than listing previews."""
turn_id = self.db.write_turn(
session_id="session-1",
input_tokens=10,
output_tokens=20,
total_tokens=30,
cache_read_tokens=2,
cache_write_tokens=3,
reasoning_tokens=4,
elapsed_ms=500,
model="model-1",
provider="provider-1",
finish_reason="stop",
api_call_count=1,
tool_calls_count=1,
request_preview=marker * 100,
response_preview=marker.lower() * 100,
reasoning="reasoning" * 100,
full_request="request" * 100,
full_response="response" * 100,
)
with self.db._conn:
self.db._conn.execute(
"UPDATE turns SET created_at = ? WHERE id = ?",
(created_at, turn_id),
)
return turn_id
def test_projects_and_truncates_listing_columns(self) -> None:
"""Only listing columns are returned and previews stop at 80 chars."""
turn_id = self._write_turn(
created_at="2026-07-12 10:00:00",
marker="A",
)
self.db.write_tool_call(
turn_id=turn_id,
call_order=1,
name="terminal",
args="x" * 100,
result="y" * 100,
duration_ms=25,
error=None,
)
turns = self.db.get_turns_with_preview("session-1")
self.assertEqual(len(turns), 1)
turn = turns[0]
self.assertEqual(
set(turn),
{
"id",
"session_id",
"created_at",
"input_tokens",
"output_tokens",
"total_tokens",
"cache_read_tokens",
"cache_write_tokens",
"reasoning_tokens",
"elapsed_ms",
"model",
"provider",
"finish_reason",
"api_call_count",
"tool_calls_count",
"request_preview",
"response_preview",
"tool_calls",
"_order",
},
)
self.assertEqual(turn["request_preview"], "A" * 80)
self.assertEqual(turn["response_preview"], "a" * 80)
self.assertEqual(
set(turn["tool_calls"][0]),
{
"id",
"turn_id",
"call_order",
"name",
"duration_ms",
"error",
"args",
"result",
},
)
self.assertEqual(turn["tool_calls"][0]["args"], "x" * 80)
self.assertEqual(turn["tool_calls"][0]["result"], "y" * 80)
def test_full_query_still_returns_untruncated_payloads(self) -> None:
"""The existing full query remains available and unmodified."""
turn_id = self._write_turn(
created_at="2026-07-12 10:00:00",
marker="A",
)
self.db.write_tool_call(
turn_id=turn_id,
call_order=1,
name="terminal",
args="x" * 100,
result="y" * 100,
duration_ms=25,
error=None,
)
turns = self.db.get_turns_with_tool_calls("session-1")
self.assertEqual(turns[0]["full_request"], "request" * 100)
self.assertEqual(turns[0]["full_response"], "response" * 100)
self.assertEqual(turns[0]["reasoning"], "reasoning" * 100)
self.assertEqual(turns[0]["tool_calls"][0]["args"], "x" * 100)
self.assertEqual(turns[0]["tool_calls"][0]["result"], "y" * 100)
def test_detail_preview_excludes_full_payloads_and_truncates_tools(self) -> None:
"""Detail previews omit complete turn and tool-call payloads."""
turn_id = self._write_turn(
created_at="2026-07-12 10:00:00",
marker="A",
)
self.db.write_tool_call(
turn_id=turn_id,
call_order=1,
name="terminal",
args="x" * 100,
result="y" * 100,
duration_ms=25,
error=None,
)
turn = self.db.get_turn_preview(turn_id)
self.assertIsNotNone(turn)
assert turn is not None
self.assertNotIn("full_request", turn)
self.assertNotIn("full_response", turn)
self.assertEqual(turn["reasoning"], "reasoning" * 100)
self.assertEqual(turn["tool_calls"][0]["args"], "x" * 80)
self.assertEqual(turn["tool_calls"][0]["result"], "y" * 80)
self.assertNotIn("full_request", turn["tool_calls"][0])
self.assertNotIn("full_response", turn["tool_calls"][0])
def test_payload_query_returns_only_requested_full_payload(self) -> None:
"""Payload queries do not load metadata or the other full payload."""
turn_id = self._write_turn(
created_at="2026-07-12 10:00:00",
marker="A",
)
request_payload = self.db.get_turn_payload(turn_id, "request")
response_payload = self.db.get_turn_payload(turn_id, "response")
self.assertEqual(request_payload, {"full_request": "request" * 100})
self.assertEqual(response_payload, {"full_response": "response" * 100})
self.assertIsNone(self.db.get_turn_payload(999, "request"))
def test_payload_query_rejects_unknown_type(self) -> None:
"""Only the two full payload columns can be selected."""
with self.assertRaises(ValueError):
self.db.get_turn_payload(1, "other")
def test_user_input_query_returns_last_user_message(self) -> None:
"""User input comes from the last user message, untruncated."""
turn_id = self._write_turn(
created_at="2026-07-12 10:00:00",
marker="A",
)
messages = [
{"role": "user", "content": "first message"},
{"role": "assistant", "content": "tool call"},
{"role": "user", "content": "second message" + "x" * 600},
]
with self.db._conn:
self.db._conn.execute(
"UPDATE turns SET full_request = ? WHERE id = ?",
(json.dumps(messages), turn_id),
)
self.assertEqual(
self.db.get_turn_user_input(turn_id),
"second message" + "x" * 600,
)
self.assertEqual(self.db.get_turn_user_input(999), "")
def test_orders_turns_and_nested_tool_calls(self) -> None:
"""Turns remain newest-first and tool calls remain call-order-first."""
older_id = self._write_turn(
created_at="2026-07-12 09:00:00",
marker="A",
)
newer_id = self._write_turn(
created_at="2026-07-12 10:00:00",
marker="B",
)
self.db.write_tool_call(
turn_id=newer_id,
call_order=2,
name="second",
args="{}",
result="{}",
duration_ms=None,
error=None,
)
self.db.write_tool_call(
turn_id=newer_id,
call_order=1,
name="first",
args="{}",
result="{}",
duration_ms=None,
error=None,
)
turns = self.db.get_turns_with_preview("session-1")
self.assertEqual([turn["id"] for turn in turns], [newer_id, older_id])
self.assertEqual([turn["_order"] for turn in turns], [2, 1])
self.assertEqual(
[tool_call["name"] for tool_call in turns[0]["tool_calls"]],
["first", "second"],
)
self.assertEqual(turns[1]["tool_calls"], [])
def test_limit_applies_before_batch_loading_tool_calls(self) -> None:
"""Tool calls are loaded only for turns retained by the turn limit."""
older_id = self._write_turn(
created_at="2026-07-12 09:00:00",
marker="A",
)
newer_id = self._write_turn(
created_at="2026-07-12 10:00:00",
marker="B",
)
for turn_id in (older_id, newer_id):
self.db.write_tool_call(
turn_id=turn_id,
call_order=1,
name=f"tool-{turn_id}",
args="{}",
result="{}",
duration_ms=None,
error=None,
)
traced_sql: list[str] = []
self.db._conn.set_trace_callback(traced_sql.append)
try:
turns = self.db.get_turns_with_preview("session-1", limit=1)
finally:
self.db._conn.set_trace_callback(None)
self.assertEqual([turn["id"] for turn in turns], [newer_id])
self.assertEqual(turns[0]["tool_calls"][0]["turn_id"], newer_id)
tool_queries = [
statement
for statement in traced_sql
if "FROM tool_calls" in statement and "turn_id IN" in statement
]
self.assertEqual(len(tool_queries), 1)
self.assertIn(f"turn_id IN ({newer_id})", tool_queries[0])
self.assertNotIn(f"turn_id IN ({older_id})", tool_queries[0])
def test_empty_session_returns_empty_list(self) -> None:
"""An unknown session performs no tool-call batch query."""
self.assertEqual(self.db.get_turns_with_preview("missing"), [])
def test_null_and_unicode_previews_preserve_sqlite_semantics(self) -> None:
"""NULL remains NULL and Unicode truncation counts characters."""
null_id = self._write_turn(
created_at="2026-07-12 09:00:00",
marker="A",
)
unicode_id = self._write_turn(
created_at="2026-07-12 10:00:00",
marker="界",
)
with self.db._conn:
self.db._conn.execute(
"UPDATE turns SET request_preview = NULL, response_preview = NULL "
"WHERE id = ?",
(null_id,),
)
turns = self.db.get_turns_with_preview("session-1")
self.assertEqual([turn["id"] for turn in turns], [unicode_id, null_id])
self.assertEqual(turns[0]["request_preview"], "界" * 80)
self.assertIsNone(turns[1]["request_preview"])
self.assertIsNone(turns[1]["response_preview"])
def test_same_timestamp_orders_by_id_descending(self) -> None:
"""The higher turn id wins ties in created_at."""
first_id = self._write_turn(
created_at="2026-07-12 10:00:00",
marker="A",
)
second_id = self._write_turn(
created_at="2026-07-12 10:00:00",
marker="B",
)
turns = self.db.get_turns_with_preview("session-1")
self.assertEqual([turn["id"] for turn in turns], [second_id, first_id])
class TestSessionStats(unittest.TestCase):
"""Verify session statistics aggregate all matching turns in SQL."""
def setUp(self) -> None:
"""Create an initialized in-memory telemetry database."""
self.db = TelemetryDB(":memory:")
self.db.init_db()
def tearDown(self) -> None:
"""Close the in-memory database connection."""
self.db._conn.close()
def _write_turn(
self,
*,
session_id: str,
output_tokens: int | None,
elapsed_ms: int | None,
tool_calls_count: int,
) -> int:
"""Insert one turn with the values used by session aggregates."""
return self.db.write_turn(
session_id=session_id,
model="model-1",
api_call_count=1,
request_preview="request",
response_preview="response",
reasoning="",
finish_reason="stop",
input_tokens=10,
output_tokens=output_tokens,
total_tokens=None,
cache_read_tokens=None,
cache_write_tokens=None,
reasoning_tokens=None,
elapsed_ms=elapsed_ms,
tool_calls_count=tool_calls_count,
full_request="[]",
full_response="{}",
)
def test_aggregates_all_turns_independently_of_preview_limit(self) -> None:
"""Statistics include turns excluded from a limited preview query."""
first_turn_id = self._write_turn(
session_id="session-1",
output_tokens=100,
elapsed_ms=1000,
tool_calls_count=2,
)
second_turn_id = self._write_turn(
session_id="session-1",
output_tokens=200,
elapsed_ms=3000,
tool_calls_count=3,
)
other_turn_id = self._write_turn(
session_id="other-session",
output_tokens=999,
elapsed_ms=999,
tool_calls_count=9,
)
for turn_id, call_order in (
(first_turn_id, 1),
(second_turn_id, 1),
(second_turn_id, 2),
(other_turn_id, 1),
):
self.db.write_tool_call(
turn_id=turn_id,
call_order=call_order,
name="terminal",
args="{}",
result="{}",
duration_ms=10,
error=None,
)
self.assertEqual(len(self.db.get_turns_with_preview("session-1", limit=1)), 1)
stats = self.db.get_session_stats("session-1")
self.assertEqual(stats["total_output_tokens"], 300)
self.assertEqual(stats["total_elapsed_ms"], 4000)
self.assertEqual(stats["total_tool_calls"], 3)
self.assertEqual(stats["avg_tps"], 75.0)
self.assertEqual(stats["avg_latency"], 2.0)
self.assertEqual(stats["known_duration_turns"], 2)
def test_returns_zero_totals_and_none_averages_for_empty_session(self) -> None:
"""An unknown session has neutral totals and undefined averages."""
stats = self.db.get_session_stats("missing-session")
self.assertEqual(
stats,
{
"total_output_tokens": 0,
"total_elapsed_ms": 0,
"total_tool_calls": 0,
"avg_tps": None,
"avg_latency": None,
"known_duration_turns": 0,
},
)
def test_null_output_tokens_make_tps_undefined(self) -> None:
"""Elapsed time alone cannot define output-token throughput."""
self._write_turn(
session_id="session-null-output",
output_tokens=None,
elapsed_ms=1000,
tool_calls_count=0,
)
stats = self.db.get_session_stats("session-null-output")
self.assertEqual(stats["total_output_tokens"], 0)
self.assertEqual(stats["total_elapsed_ms"], 1000)
self.assertIsNone(stats["avg_tps"])
self.assertEqual(stats["known_duration_turns"], 1)
def test_null_elapsed_time_makes_tps_undefined(self) -> None:
"""Output tokens alone cannot define throughput."""
self._write_turn(
session_id="session-null-elapsed",
output_tokens=100,
elapsed_ms=None,
tool_calls_count=0,
)
stats = self.db.get_session_stats("session-null-elapsed")
self.assertEqual(stats["total_output_tokens"], 100)
self.assertEqual(stats["total_elapsed_ms"], 0)
self.assertIsNone(stats["avg_tps"])
self.assertEqual(stats["known_duration_turns"], 0)
def test_avg_latency_ignores_turns_without_duration(self) -> None:
"""Known durations determine latency even when other turns are NULL."""
self._write_turn(
session_id="session-mixed-duration",
output_tokens=100,
elapsed_ms=1000,
tool_calls_count=0,
)
self._write_turn(
session_id="session-mixed-duration",
output_tokens=100,
elapsed_ms=None,
tool_calls_count=0,
)
stats = self.db.get_session_stats("session-mixed-duration")
self.assertEqual(stats["total_elapsed_ms"], 1000)
self.assertEqual(stats["known_duration_turns"], 1)
self.assertEqual(stats["avg_latency"], 1.0)
class TestTurnIndexes(unittest.TestCase):
"""Verify the session-listing composite index and its migration."""
def setUp(self) -> None:
"""Create a database representing the schema before index migration."""
self.db = TelemetryDB(":memory:")
self.db.init_db()
with self.db._conn:
self.db._conn.execute("DROP INDEX idx_turns_session_created")
self.db._conn.execute("CREATE INDEX idx_turns_session ON turns(session_id)")
self.db._conn.execute(
"CREATE INDEX idx_turns_created ON turns(created_at DESC)"
)
self.db._conn.execute("PRAGMA user_version = 3")
def tearDown(self) -> None:
"""Close the in-memory database connection."""
self.db._conn.close()
def test_migrates_to_composite_index_with_expected_columns(self) -> None:
"""Migration replaces legacy indexes with the structural contract."""
self.db.init_db()
version = self.db._conn.execute("PRAGMA user_version").fetchone()[0]
indexes = {
row[1]
for row in self.db._conn.execute("PRAGMA index_list(turns)").fetchall()
}
index_columns = [
(row[2], row[3])
for row in self.db._conn.execute(
"PRAGMA index_xinfo(idx_turns_session_created)"
).fetchall()
if row[5]
]
self.assertEqual(version, 4)
self.assertIn("idx_turns_session_created", indexes)
self.assertNotIn("idx_turns_session", indexes)
self.assertNotIn("idx_turns_created", indexes)
self.assertEqual(
index_columns,
[("session_id", 0), ("created_at", 1), ("id", 1)],
)
def test_query_planner_uses_composite_index_without_temp_sort(self) -> None:
"""Check planner output; wording is SQLite-version-sensitive."""
self.db.init_db()
query_plan = [
row[3]
for row in self.db._conn.execute(
"EXPLAIN QUERY PLAN "
"SELECT * FROM turns WHERE session_id = ? "
"ORDER BY created_at DESC, id DESC LIMIT ?",
("session-1", 100),
).fetchall()
]
self.assertTrue(
any("USING INDEX idx_turns_session_created" in step for step in query_plan)
)
self.assertFalse(any("USE TEMP B-TREE" in step for step in query_plan))
if __name__ == "__main__":
unittest.main()