mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-17 03:54:08 +00:00
Restored (each proven red against a sabotaged production path, green on main): - tests/agent/test_overflow_overhead_aware_tokens.py:: test_overflow_recovery_compresses_with_tool_overhead_counted[413, context_overflow, prompt_too_long, long_context_tier_429] Rewritten from the four estimator-to-compressor tests: the real estimator runs with a ~10k-token tool schema and the compressor must receive an approx_tokens that counts it (LCM issue 441). No helper-name pinning. - tests/hermes_cli/test_web_server_sessiondb_eventloop.py:: test_bulk_delete_sessiondb_work_runs_off_event_loop, test_session_read_handlers_run_sessiondb_work_off_event_loop[search,stats,detail] Real route handlers; every SessionDB open/call lands off the event-loop thread (#60747). AST/source tests stay deleted. - tests/hermes_state/test_hermes_state.py::TestCompressionChainProjection:: test_list_batches_tip_row_fetch_into_one_query list_sessions_rich resolves a page of compression tips in one batched query, not one per root (N+1 guard). - tests/hermes_state/test_tail_rewind_semantics.py::TestCompressTagsCarriedTail:: test_compress_marks_carried_forward_tail_dicts Repaired from a skip-on-exception body that asserted nothing: compress() tags exactly the verbatim carried tail so archive_and_compact gets an accurate tail_count (#86366). - tests/hermes_state/test_wal_checkpoint_strategy.py:: test_failed_checkpoint_never_raises_and_is_not_silent, test_failed_checkpoint_at_close_still_releases_the_handle Periodic PASSIVE checkpoint failure neither raises into the write path nor passes silently; a failing checkpoint at close() still releases the handle (#45383 strategy). Log-wording asserts dropped. - tests/hermes_state/test_hermes_state_ux_messages.py:: test_db_unavailable_details_line_carries_raw_cause_only_below_the_lead, test_db_unavailable_network_drive_cause_does_not_offer_doctor_fix CLI details=True keeps the raw SQLite cause on its own Details line only; a locking-protocol (network FS) cause never offers `doctor --fix`. - tests/tools/test_code_execution.py::TestStubSchemaDrift:: test_generated_module_accepts_all_params Replaces the syntax-only test_generated_module_compiles: executes the generated hermes_tools module (uds and file transports) and asserts every stub accepts all its params as kwargs and forwards each by name and value.
225 lines
8.3 KiB
Python
225 lines
8.3 KiB
Python
"""Tests for SessionDB WAL checkpoint strategy (issue #45383).
|
|
|
|
Verifies that ALL checkpoints on the shared state.db use PASSIVE mode:
|
|
periodic, close(), and pre-VACUUM. TRUNCATE fires a full WAL reset, and
|
|
transient per-cron-run connections closing many times an hour would race
|
|
the live gateway writer and corrupt B-tree pages (#45383).
|
|
"""
|
|
|
|
import logging
|
|
import sqlite3
|
|
from unittest.mock import MagicMock
|
|
|
|
import pytest
|
|
|
|
from hermes_state import SessionDB
|
|
|
|
|
|
class TrackingConnection:
|
|
"""sqlite3.Connection proxy that records executed SQL strings."""
|
|
|
|
def __init__(self, conn):
|
|
self._conn = conn
|
|
self.execute_calls = []
|
|
|
|
def execute(self, sql, *args, **kwargs):
|
|
self.execute_calls.append(sql)
|
|
return self._conn.execute(sql, *args, **kwargs)
|
|
|
|
def __getattr__(self, name):
|
|
return getattr(self._conn, name)
|
|
|
|
|
|
@pytest.fixture()
|
|
def db(tmp_path):
|
|
"""Create a SessionDB with a temp database file."""
|
|
db_path = tmp_path / "test_state.db"
|
|
session_db = SessionDB(db_path=db_path)
|
|
yield session_db
|
|
try:
|
|
session_db.close()
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
class TestTryWalCheckpointPassive:
|
|
"""_try_wal_checkpoint() should use PASSIVE mode for periodic use."""
|
|
|
|
def test_checkpoint_uses_passive_mode(self, db):
|
|
"""PASSIVE checkpoint does not require exclusive lock — safe for large DBs."""
|
|
# Capture the real connection's execute before mocking
|
|
real_conn = db._conn
|
|
execute_calls = []
|
|
|
|
def tracking_execute(sql, *args, **kwargs):
|
|
execute_calls.append(sql)
|
|
return real_conn.execute(sql, *args, **kwargs)
|
|
|
|
# sqlite3.Connection.execute is read-only (C extension) — replace _conn
|
|
mock_conn = MagicMock()
|
|
mock_conn.execute.side_effect = tracking_execute
|
|
mock_conn.fetchone.return_value = None
|
|
db._conn = mock_conn
|
|
|
|
db._try_wal_checkpoint()
|
|
|
|
passive_calls = [c for c in execute_calls if "wal_checkpoint(PASSIVE)" in c]
|
|
truncate_calls = [c for c in execute_calls if "wal_checkpoint(TRUNCATE)" in c]
|
|
assert len(passive_calls) == 1, (
|
|
f"Expected 1 PASSIVE checkpoint call, got {len(passive_calls)}"
|
|
)
|
|
assert len(truncate_calls) == 0, (
|
|
"Periodic checkpoint should NOT use TRUNCATE"
|
|
)
|
|
|
|
def test_failed_checkpoint_never_raises_and_is_not_silent(self, db, caplog):
|
|
"""The periodic checkpoint runs from the write path: a failing PASSIVE
|
|
checkpoint must not raise into the caller, and must leave a WARNING."""
|
|
real_conn = db._conn
|
|
mock_conn = MagicMock()
|
|
mock_conn.execute.side_effect = sqlite3.OperationalError("disk I/O error")
|
|
db._conn = mock_conn
|
|
try:
|
|
with caplog.at_level(logging.WARNING, logger="hermes_state"):
|
|
db._try_wal_checkpoint()
|
|
finally:
|
|
db._conn = real_conn
|
|
assert any(r.levelno >= logging.WARNING for r in caplog.records)
|
|
|
|
|
|
|
|
|
|
class TestCloseUsesPassive:
|
|
"""close() must use PASSIVE. Transient per-cron-run SessionDB connections
|
|
close many times an hour; a TRUNCATE reset there races the live gateway
|
|
writer on the large WAL DB and corrupts B-tree pages (#45383)."""
|
|
|
|
def test_close_uses_passive_mode(self, db):
|
|
"""close() checkpoints PASSIVE, never TRUNCATE."""
|
|
real_conn = db._conn
|
|
execute_calls = []
|
|
|
|
def tracking_execute(sql, *args, **kwargs):
|
|
execute_calls.append(sql)
|
|
return real_conn.execute(sql, *args, **kwargs)
|
|
|
|
mock_conn = MagicMock()
|
|
mock_conn.execute.side_effect = tracking_execute
|
|
db._conn = mock_conn
|
|
|
|
db.close()
|
|
|
|
truncate_calls = [c for c in execute_calls if "wal_checkpoint(TRUNCATE)" in c]
|
|
passive_calls = [c for c in execute_calls if "wal_checkpoint(PASSIVE)" in c]
|
|
assert len(truncate_calls) == 0, (
|
|
"close() must NOT TRUNCATE (races the live gateway writer, #45383)"
|
|
)
|
|
assert len(passive_calls) == 1, (
|
|
f"Expected 1 PASSIVE checkpoint at close, got {len(passive_calls)}"
|
|
)
|
|
|
|
def test_failed_checkpoint_at_close_still_releases_the_handle(self, db):
|
|
"""close() is best-effort about the checkpoint: a failing PASSIVE at
|
|
close must neither raise nor leave the handle open."""
|
|
real_conn = db._conn
|
|
mock_conn = MagicMock()
|
|
mock_conn.execute.side_effect = sqlite3.OperationalError("database is locked")
|
|
db._conn = mock_conn
|
|
try:
|
|
db.close()
|
|
finally:
|
|
real_conn.close()
|
|
|
|
assert db._conn is None
|
|
mock_conn.close.assert_called()
|
|
|
|
|
|
|
|
class TestVacuumUsesPassive:
|
|
"""Manual vacuum paths must checkpoint PASSIVE, never TRUNCATE."""
|
|
|
|
def test_vacuum_uses_passive_before_vacuum(self, db):
|
|
"""SessionDB.vacuum() checkpoints PASSIVE before VACUUM.
|
|
|
|
A TRUNCATE checkpoint AFTER the VACUUM is expected: VACUUM rewrites
|
|
every page through the WAL, so without it a 3 GB database leaves a
|
|
3 GB state.db-wal behind and `sessions optimize` becomes a net disk
|
|
LOSS. The #45383 tearing concern is about truncating while another
|
|
writer is mid-transaction — the pre-VACUUM checkpoint stays PASSIVE
|
|
for that reason; the post-VACUUM truncate runs under the same
|
|
exclusive-lock window the VACUUM itself required.
|
|
"""
|
|
real_conn = db._conn
|
|
tracking_conn = TrackingConnection(real_conn)
|
|
db._conn = tracking_conn
|
|
|
|
db.vacuum()
|
|
|
|
checkpoint_calls = [
|
|
c for c in tracking_conn.execute_calls if "wal_checkpoint" in c.lower()
|
|
]
|
|
truncate_calls = [c for c in checkpoint_calls if "TRUNCATE" in c]
|
|
passive_calls = [c for c in checkpoint_calls if "PASSIVE" in c]
|
|
vacuum_calls = [
|
|
c for c in tracking_conn.execute_calls if c.strip().upper() == "VACUUM"
|
|
]
|
|
assert passive_calls == ["PRAGMA wal_checkpoint(PASSIVE)"]
|
|
assert vacuum_calls == ["VACUUM"]
|
|
assert truncate_calls == ["PRAGMA wal_checkpoint(TRUNCATE)"]
|
|
vacuum_index = tracking_conn.execute_calls.index(vacuum_calls[0])
|
|
assert tracking_conn.execute_calls.index(passive_calls[0]) < vacuum_index
|
|
# TRUNCATE must come only AFTER the VACUUM (never before it).
|
|
assert tracking_conn.execute_calls.index(truncate_calls[0]) > vacuum_index
|
|
|
|
def test_optimize_storage_uses_passive_after_vacuum(self, db):
|
|
"""optimize_fts_storage() checkpoints PASSIVE after its VACUUM."""
|
|
real_conn = db._conn
|
|
tracking_conn = TrackingConnection(real_conn)
|
|
db._conn = tracking_conn
|
|
|
|
result = db.optimize_fts_storage(vacuum=True)
|
|
|
|
checkpoint_calls = [
|
|
c for c in tracking_conn.execute_calls if "wal_checkpoint" in c.lower()
|
|
]
|
|
truncate_calls = [c for c in checkpoint_calls if "TRUNCATE" in c]
|
|
passive_calls = [c for c in checkpoint_calls if "PASSIVE" in c]
|
|
vacuum_calls = [
|
|
c for c in tracking_conn.execute_calls if c.strip().upper() == "VACUUM"
|
|
]
|
|
assert result["ok"] is True
|
|
assert result["vacuumed"] is True
|
|
assert truncate_calls == []
|
|
assert passive_calls == ["PRAGMA wal_checkpoint(PASSIVE)"]
|
|
assert vacuum_calls == ["VACUUM"]
|
|
assert tracking_conn.execute_calls.index(
|
|
vacuum_calls[0]
|
|
) < tracking_conn.execute_calls.index(passive_calls[0])
|
|
|
|
|
|
class TestCheckpointFrequency:
|
|
"""Checkpoint triggers every N writes."""
|
|
|
|
def test_checkpoint_triggers_at_interval(self, db):
|
|
"""_try_wal_checkpoint is called every _CHECKPOINT_EVERY_N_WRITES writes."""
|
|
call_count = [0]
|
|
original = db._try_wal_checkpoint
|
|
|
|
def counting_checkpoint():
|
|
call_count[0] += 1
|
|
original()
|
|
|
|
db._try_wal_checkpoint = counting_checkpoint
|
|
|
|
# Write exactly _CHECKPOINT_EVERY_N_WRITES sessions to trigger one checkpoint
|
|
n = db._CHECKPOINT_EVERY_N_WRITES
|
|
import time as _time
|
|
for i in range(n):
|
|
db._execute_write(lambda conn, _i=i: conn.execute(
|
|
"INSERT INTO sessions (id, source, started_at) VALUES (?, ?, ?)",
|
|
(f"sess_{_i}", "test", _time.time()),
|
|
))
|
|
|
|
assert call_count[0] == 1, (
|
|
f"Expected 1 checkpoint after {n} writes, got {call_count[0]}"
|
|
)
|