hermes-agent/tests/hermes_state/test_state_db_file_identity.py
teknium1 35272ce28b fix(state): /proc/locks readers no longer lose entries to other processes' lock churn
/proc/locks is host-wide and the kernel serves it over several read() calls,
resuming by record index. When other processes add or drop locks between
those reads the table shifts and entries are skipped, so a lock that is
still held can read as gone.

- tests: the live-DB preview and header-probe lock tests read their own
  process's locks from /proc/locks and flaked on busy CI hosts (1 in 12 on
  a quiet box; 9 in 12 with six processes churning locks). A shared
  tests/posix_lock_probe.own_posix_locks() reads /proc/self/fdinfo instead,
  which lists this process's locks per open file in one read.
- hermes_state_lockowners: state_db_write_lock_holders() names the writer
  in the "state.db write lock unavailable" warning. One pass could skip the
  holder and log "no write-class lock held at the deadline". It now unions
  three passes; a skip rarely repeats.
2026-09-28 02:26:21 -07:00

270 lines
9.6 KiB
Python

"""File-identity guard on SessionDB writes (#89332).
When state.db is replaced out-of-band under a live handle, in-place FTS
rebuild / fail-open cannot help: they operate on a generation mismatch.
The store must fail loudly instead of limping.
"""
import json
import os
import shutil
import sqlite3
from pathlib import Path
import pytest
from tests.posix_lock_probe import own_posix_locks
from hermes_state import (
SessionDB,
StateDbReplacedError,
classify_persistence_error,
divert_session_transcript_jsonl,
)
def _make_db(path: Path, session_id: str, content: str) -> SessionDB:
db = SessionDB(db_path=path)
db.create_session(session_id, "cli")
db.append_message(session_id, role="user", content=content)
return db
def _require_identity(db: SessionDB) -> None:
if db._db_file_identity is None:
pytest.skip("filesystem does not expose st_dev/st_ino for identity checks")
@pytest.mark.platforms("posix")
def test_replace_with_new_inode_fails_loudly_without_fts_repair(tmp_path):
live = tmp_path / "state.db"
other = tmp_path / "other.db"
db = _make_db(live, "live-sess", "original")
_require_identity(db)
alt = _make_db(other, "other-sess", "replacement")
alt.close()
recorded = db._db_file_identity
assert recorded is not None
os.replace(other, live)
assert _stat_changed(live, recorded)
with pytest.raises(StateDbReplacedError, match="replaced underneath"):
db.append_message("live-sess", role="user", content="after-replace")
assert db._db_replaced is True
# No FTS surgery ran: fail-open never detached the indexes.
assert db._fts_enabled is True
assert db._fts_stale is False
db.close()
@pytest.mark.platforms("posix")
def test_second_write_after_halt_does_not_attempt_repair(tmp_path):
live = tmp_path / "state.db"
other = tmp_path / "other.db"
db = _make_db(live, "s", "a")
_require_identity(db)
alt = _make_db(other, "t", "b")
alt.close()
os.replace(other, live)
with pytest.raises(StateDbReplacedError):
db.append_message("s", role="user", content="first")
with pytest.raises(StateDbReplacedError):
db.append_message("s", role="user", content="second")
assert db._fts_enabled is True
assert db._fts_stale is False
db.close()
def test_same_file_fts_corruption_still_fails_open(tmp_path):
"""Identity guard must not disable genuine in-file FTS recovery.
Since 18ac3c4fb6 the live write path never rebuilds FTS in place; the
recovery contract is the fail-open detach (stale marker + triggers
dropped) followed by a successful canonical retry.
"""
db = _make_db(tmp_path / "state.db", "s1", "hello world")
_require_identity(db)
identity = db._db_file_identity
raw = sqlite3.connect(str(tmp_path / "state.db"))
raw.execute(
"UPDATE messages_fts_data SET block = X'DEADBEEFDEADBEEFDEADBEEFDEADBEEF'"
)
raw.commit()
raw.close()
db.append_message("s1", role="user", content="healed append")
assert db._db_file_identity == identity
assert db._db_replaced is False
# Fail-open detach ran: canonical write landed, FTS marked stale.
assert db._fts_stale is True
assert db._fts_enabled is False
db.close()
def test_classify_replaced_is_not_disk_or_fts_repair():
err = StateDbReplacedError(
"FATAL: state.db was replaced underneath the gateway; refusing further writes"
)
assert classify_persistence_error(err) == "replaced"
assert classify_persistence_error(str(err)) == "replaced"
def test_new_sessiondb_on_replaced_path_records_new_identity(tmp_path):
live = tmp_path / "state.db"
other = tmp_path / "other.db"
db = _make_db(live, "s", "a")
old_id = db._db_file_identity
_require_identity(db)
db.close()
alt = _make_db(other, "t", "b")
alt.close()
os.replace(other, live)
reopened = SessionDB(db_path=live)
try:
assert reopened._db_file_identity != old_id
reopened.append_message("t", role="user", content="adopted after reopen")
assert reopened._db_replaced is False
finally:
reopened.close()
@pytest.mark.platforms("posix")
def test_fts_scoped_error_on_replaced_file_skips_fts_fail_open(tmp_path):
"""Even FTS-provenance corruption must not authorize surgery on a
replaced file. (A generic malformed error never reaches fail-open at
all since the provenance classifier of #99652 rejects it earlier.)"""
live = tmp_path / "state.db"
other = tmp_path / "other.db"
db = _make_db(live, "s", "a")
_require_identity(db)
alt = _make_db(other, "t", "b")
alt.close()
os.replace(other, live)
with pytest.raises(StateDbReplacedError):
db._enter_fts_fail_open(
sqlite3.DatabaseError(
'fts5: corrupt structure record for table "messages_fts"'
)
)
assert db._fts_enabled is True
assert db._fts_stale is False
db.close()
def test_copyfile_same_inode_fails_loudly_without_fts_repair(tmp_path):
"""``cp`` keeps st_ino; generation stamp must still halt (#89332)."""
live = tmp_path / "state.db"
other = tmp_path / "other.db"
db = _make_db(live, "live-sess", "original")
alt = _make_db(other, "other-sess", "replacement")
live_app = db._db_file_application_id
other_app = alt._db_file_application_id
if not live_app or not other_app:
alt.close()
db.close()
pytest.skip("generation stamp not recorded on this filesystem")
assert live_app != other_app
recorded = db._db_file_identity
alt.close()
shutil.copyfile(other, live)
if recorded is not None:
st = os.stat(live)
assert (st.st_dev, st.st_ino) == recorded
with pytest.raises(StateDbReplacedError, match="replaced underneath"):
db.append_message("live-sess", role="user", content="after-cp")
assert db._db_replaced is True
assert db._fts_enabled is True
assert db._fts_stale is False
db.close()
def test_divert_session_transcript_jsonl_appends(tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
path = divert_session_transcript_jsonl(
"sess-jsonl",
[{"role": "user", "content": "hello-jsonl"}],
)
assert path == tmp_path / "sessions" / "sess-jsonl.jsonl"
lines = path.read_text(encoding="utf-8-sig").strip().splitlines()
assert json.loads(lines[-1])["content"] == "hello-jsonl"
assert divert_session_transcript_jsonl("sess-jsonl", []) is None
def _stat_changed(path: Path, recorded) -> bool:
st = os.stat(path)
return (st.st_dev, st.st_ino) != recorded
# ---------------------------------------------------------------------------
# Lock safety of the identity probe itself (#100368 / howtocorrupt §2.2).
#
# _read_sqlite_application_id runs on EVERY write against the LIVE state.db.
# Before the _pread_db_header fix it did open("rb")/read/close, and that
# close() cancelled every POSIX advisory lock this process held on the file
# — including the WAL-mode DMS shared lock of the writer connection. These
# tests measure this process's kernel lock table (/proc/self/fdinfo), so they are
# Linux-only; the hazard itself is POSIX-only.
# ---------------------------------------------------------------------------
@pytest.mark.platforms("linux")
def test_identity_probe_does_not_cancel_live_posix_locks(tmp_path):
"""The on-write header probe must not drop the writer's DMS lock."""
from hermes_state import _read_sqlite_application_id
live = tmp_path / "state.db"
db = _make_db(live, "probe-sess", "seed")
try:
# Hold an open write transaction: that is when the connection holds
# POSIX range locks on the main db file, and exactly the state a
# concurrent _raise_if_db_replaced probe (another thread, same
# process) can destroy.
db._conn.execute("BEGIN IMMEDIATE")
db._conn.execute(
"UPDATE sessions SET source = source WHERE id = 'probe-sess'"
)
before = own_posix_locks(live)
assert before, "expected in-transaction WAL connection to lock state.db"
for _ in range(3):
_read_sqlite_application_id(live)
after = own_posix_locks(live)
db._conn.rollback()
# Only locks on the main file prove the header probe safe. WAL-index
# read-lock slots on -shm may legitimately move while SQLite runs, and
# sampling those unrelated locks made this test timing-sensitive. With
# the pre-fix open/read/close probe every main-file POSIX lock vanishes.
assert after, (
"live writer connection holds no POSIX lock on state.db itself — "
"the WAL DMS lock was cancelled by a raw open/close probe "
"(howtocorrupt §2.2)"
)
# The connection must still be able to commit.
db.append_message("probe-sess", role="user", content="post-probe")
finally:
db.close()
def test_identity_probe_still_detects_replacement_after_fd_cache(tmp_path):
"""The cached-fd probe rebinds when the path names a new inode."""
from hermes_state import _read_sqlite_application_id
live = tmp_path / "state.db"
other = tmp_path / "other.db"
db = _make_db(live, "live-sess", "original")
_require_identity(db)
first = _read_sqlite_application_id(live) # populates the fd cache
db.close()
alt = _make_db(other, "other-sess", "replacement")
alt.close()
os.replace(other, live)
second = _read_sqlite_application_id(live)
assert second is not None
assert second != first, (
"probe kept reading the retired inode instead of rebinding to the "
"replacement file"
)