mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-16 19:36:54 +00:00
/proc/locks is host-wide and the kernel serves it over several read() calls, resuming by record index. When other processes add or drop locks between those reads the table shifts and entries are skipped, so a lock that is still held can read as gone. - tests: the live-DB preview and header-probe lock tests read their own process's locks from /proc/locks and flaked on busy CI hosts (1 in 12 on a quiet box; 9 in 12 with six processes churning locks). A shared tests/posix_lock_probe.own_posix_locks() reads /proc/self/fdinfo instead, which lists this process's locks per open file in one read. - hermes_state_lockowners: state_db_write_lock_holders() names the writer in the "state.db write lock unavailable" warning. One pass could skip the holder and log "no write-class lock held at the deadline". It now unions three passes; a skip rarely repeats.
86 lines
3.2 KiB
Python
86 lines
3.2 KiB
Python
"""Regression: a state.db write-lock timeout names the process holding the lock.
|
|
|
|
Before, ``database is locked (another Hermes process held the state.db write lock for over 60s)``
|
|
identified the victim only; every Hermes process has the DB open, so the descriptor scan could not
|
|
single out the writer. ``/proc/locks`` can.
|
|
"""
|
|
|
|
import io
|
|
import os
|
|
import sqlite3
|
|
import subprocess
|
|
import sys
|
|
import textwrap
|
|
import time
|
|
|
|
import pytest
|
|
|
|
from hermes_state_lockowners import parse_proc_locks, state_db_write_lock_holders
|
|
|
|
|
|
def test_parse_proc_locks_keeps_only_write_locks_on_our_inodes_and_decodes_the_wal_write_byte():
|
|
inodes = {(os.makedev(0x103, 0x02), 4194737): "-shm", (os.makedev(0x103, 0x02), 4228330): ""}
|
|
text = textwrap.dedent("""\
|
|
1: POSIX ADVISORY WRITE 594094 103:02:4194737 120 120
|
|
2: POSIX ADVISORY READ 594094 103:02:4194737 125 125
|
|
3: POSIX ADVISORY READ 99493 103:02:4194737 128 128
|
|
4: OFDLCK ADVISORY WRITE -1 103:02:4228330 1073741825 1073741825
|
|
5: POSIX ADVISORY WRITE 4242 103:02:99999 120 120
|
|
6: -> POSIX ADVISORY WRITE 777 103:02:4194737 120 120
|
|
""")
|
|
assert parse_proc_locks(text, inodes) == [
|
|
(594094, "WAL write", "-shm"),
|
|
(-1, "RESERVED", ""),
|
|
]
|
|
|
|
|
|
@pytest.mark.platforms("linux")
|
|
def test_holder_skipped_by_one_proc_locks_pass_is_still_named(tmp_path, monkeypatch):
|
|
"""/proc/locks is served over several read()s, so churn elsewhere can skip an entry in one pass."""
|
|
import hermes_state_lockowners
|
|
|
|
db = tmp_path / "state.db"
|
|
db.write_bytes(b"")
|
|
st = os.stat(db)
|
|
held = (f"1: POSIX ADVISORY WRITE {os.getpid()} "
|
|
f"{os.major(st.st_dev):02x}:{os.minor(st.st_dev):02x}:{st.st_ino} 1073741825 1073741825\n")
|
|
passes = iter(["", held, ""])
|
|
monkeypatch.setattr(hermes_state_lockowners, "open", lambda *a, **k: io.StringIO(next(passes)), raising=False)
|
|
lines = state_db_write_lock_holders(db)
|
|
assert len(lines) == 1 and f"PID {os.getpid()} " in lines[0] and "RESERVED" in lines[0], lines
|
|
|
|
|
|
@pytest.mark.platforms("linux")
|
|
def test_live_writer_in_another_process_is_named_by_pid(tmp_path):
|
|
db = tmp_path / "state.db"
|
|
conn = sqlite3.connect(db)
|
|
conn.execute("PRAGMA journal_mode=wal")
|
|
conn.execute("CREATE TABLE t(x)")
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
holder = subprocess.Popen(
|
|
[sys.executable, "-c", textwrap.dedent(f"""
|
|
import sqlite3, sys, time
|
|
c = sqlite3.connect({str(db)!r}, isolation_level=None)
|
|
c.execute("BEGIN IMMEDIATE")
|
|
c.execute("INSERT INTO t VALUES (1)")
|
|
print("held", flush=True)
|
|
time.sleep(30)
|
|
""")],
|
|
stdout=subprocess.PIPE, text=True,
|
|
)
|
|
try:
|
|
assert holder.stdout.readline().strip() == "held"
|
|
deadline = time.monotonic() + 5
|
|
lines = []
|
|
while time.monotonic() < deadline:
|
|
lines = state_db_write_lock_holders(db)
|
|
if lines:
|
|
break
|
|
time.sleep(0.05)
|
|
assert any(f"PID {holder.pid} " in line and "WAL write" in line for line in lines), lines
|
|
finally:
|
|
holder.kill()
|
|
holder.wait()
|
|
assert state_db_write_lock_holders(db) == []
|