mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-17 04:11:35 +00:00
Rust live_marker_owner now REMOVES a stale marker (dead pid, past ceiling, unparseable) on read — previously stale bytes were only ignored, so a crashed updater whose Drop never ran refused every later acquire until the 20-minute ceiling (#77259). pid_is_alive gains pid-0 and zombie handling (Linux /proc state, macOS ps stat), closing the kill(pid,0) false-positive that kept a dead-but-unreaped updater 'live'. Python _pid_is_running gains the same zombie awareness via a new _process_state probe (Linux /proc, macOS ps), which read_live_update inherits through update_lock._pid_alive — so the CLI gate self-heals a zombie-owned marker in seconds instead of 20 minutes (#120635, #125932). Electron readLiveUpdateMarker consults a new posixProcessState probe (injectable, fail-open to the signal-0 verdict) so the desktop boot gate no longer parks on a zombie-owned marker. Rust fix cherry-picked from #77885 (authorship kept); Python and Electron are new parity work.
200 lines
7 KiB
Python
200 lines
7 KiB
Python
"""Startup triggers PM recovery without importing the damaged dependencies.
|
|
|
|
Real generation rebuilds and pre-activation startup live in tests/pm; these
|
|
checks keep marker ownership, retry limits and single-flight behavior intact.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from hermes_cli import _early_recovery as er
|
|
from pm import recovery
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
|
|
|
|
@pytest.mark.parametrize("prefix", [[], ["-p", "default"], ["--profile=default"]])
|
|
def test_bootstrap_and_pm_cli_work_without_site_packages(tmp_path, prefix):
|
|
env = {**os.environ, "HERMES_HOME": str(tmp_path / "home"), "PYTHONPATH": str(REPO_ROOT)}
|
|
result = subprocess.run(
|
|
[sys.executable, "-S", "-m", "hermes_cli.main", *prefix, "pm", "repair", "--help"],
|
|
cwd=tmp_path, env=env, capture_output=True, text=True, timeout=60,
|
|
)
|
|
assert result.returncode == 0, result.stderr
|
|
assert "hermes pm repair" in result.stdout
|
|
|
|
|
|
@pytest.mark.parametrize("marker_name", [".update-incomplete", ".lazy-refresh-incomplete"])
|
|
def test_marker_requests_pm_repair_then_clears(tmp_path, monkeypatch, capsys, marker_name):
|
|
root = _project(tmp_path)
|
|
marker = root / marker_name
|
|
marker.write_text("interrupted", encoding="utf-8")
|
|
calls = []
|
|
monkeypatch.setattr(recovery, "repair_dependencies", calls.append)
|
|
assert er.recover_if_needed(root, argv=[]) is True
|
|
assert calls == [root]
|
|
assert not marker.exists()
|
|
assert capsys.readouterr().out == ""
|
|
|
|
|
|
@pytest.mark.parametrize("body", ["", "not json", '{"attempts": 1}', "started=1\npid=0\n"])
|
|
def test_failed_repair_keeps_marker_and_stops_at_retry_limit(tmp_path, monkeypatch, capsys, body):
|
|
root = _project(tmp_path)
|
|
marker = root / ".update-incomplete"
|
|
marker.write_text(body, encoding="utf-8")
|
|
attempts = er._read_marker_attempts(marker)
|
|
calls = []
|
|
def fail(project):
|
|
calls.append(project)
|
|
raise RuntimeError("dependency build failed")
|
|
monkeypatch.setattr(recovery, "repair_dependencies", fail)
|
|
for expected in range(attempts + 1, er._EARLY_CORE_INSTALL_MAX_ATTEMPTS + 1):
|
|
assert er.recover_if_needed(root, argv=[]) is False
|
|
assert er._read_marker_attempts(marker) == expected
|
|
if "pid=" in body:
|
|
assert marker.read_text(encoding="utf-8").startswith(body)
|
|
before = len(calls)
|
|
assert er.recover_if_needed(root, argv=[]) is False
|
|
assert len(calls) == before
|
|
output = capsys.readouterr()
|
|
assert output.out == ""
|
|
assert "hermes pm repair" in output.err
|
|
|
|
|
|
def test_recovery_obeys_live_owner_and_single_flight(tmp_path, monkeypatch):
|
|
root = _project(tmp_path)
|
|
marker = root / ".update-incomplete"
|
|
marker.write_text(f"started=1\npid={os.getpid()}\n", encoding="utf-8")
|
|
calls = []
|
|
monkeypatch.setattr(recovery, "repair_dependencies", calls.append)
|
|
assert er.recover_if_needed(root, argv=[]) is False
|
|
assert calls == []
|
|
assert marker.exists()
|
|
marker.write_text("interrupted", encoding="utf-8")
|
|
fd = er._claim_recovery_lock(root)
|
|
assert fd is not None
|
|
try:
|
|
assert er.recover_if_needed(root, argv=[]) is False
|
|
assert calls == []
|
|
assert marker.exists()
|
|
finally:
|
|
os.close(fd)
|
|
assert er.recover_if_needed(root, argv=["update"]) is True
|
|
assert calls == [root]
|
|
|
|
|
|
def test_missing_environment_cannot_write_a_retry_marker_without_lock(tmp_path, monkeypatch):
|
|
from pm.environments import install_state_dir, runtime_facts_path
|
|
from pm.lock import Facts
|
|
|
|
root = _project(tmp_path)
|
|
state = install_state_dir(root)
|
|
Facts(runtime_facts_path(root)).record_state("venv", "old", [], environment=state / "environments" / "old" / "venv")
|
|
fd = er._claim_recovery_lock(root)
|
|
assert fd is not None
|
|
try:
|
|
assert er.recover_if_needed(root, argv=[]) is False
|
|
assert not (state / ".repair-incomplete").exists()
|
|
finally:
|
|
os.close(fd)
|
|
|
|
|
|
def test_pm_commands_and_healthy_startup_do_not_repair(tmp_path, monkeypatch):
|
|
root = _project(tmp_path)
|
|
monkeypatch.setattr(recovery, "repair_dependencies", lambda _: pytest.fail("unexpected install"))
|
|
assert er.recover_if_needed(root, argv=[]) is False
|
|
marker = root / ".update-incomplete"
|
|
marker.write_text("interrupted", encoding="utf-8")
|
|
assert er.recover_if_needed(root, argv=["pm", "repair"]) is False
|
|
assert marker.exists()
|
|
|
|
|
|
|
|
def test_pid_liveness_recognizes_current_process():
|
|
assert er._pid_is_running(os.getpid()) is True
|
|
assert er._pid_is_running(0) is False
|
|
|
|
|
|
@pytest.mark.platforms("posix")
|
|
def test_pid_liveness_counts_a_zombie_as_dead():
|
|
"""A crashed stage lingering unreaped must not read as a live owner.
|
|
|
|
``os.kill(pid, 0)`` succeeds for a zombie, so before the state probe a
|
|
crashed updater under an un-reaping parent pinned every lock keyed on its
|
|
pid (update marker, recovery marker) for the full age ceiling.
|
|
"""
|
|
pid = os.fork()
|
|
assert pid >= 0
|
|
if pid == 0:
|
|
os._exit(0) # noqa: P111 — child exits without running pytest teardown
|
|
|
|
# Do NOT wait() yet: the child must linger unreaped (a zombie). Poll until
|
|
# the state probe actually reports 'Z' so the assertion can't race the exit.
|
|
became_zombie = False
|
|
for _ in range(40):
|
|
state = er._process_state(pid)
|
|
if state is not None and state.upper().startswith("Z"):
|
|
became_zombie = True
|
|
break
|
|
time.sleep(0.05)
|
|
assert became_zombie, "child never reached zombie state on this platform"
|
|
|
|
assert er._pid_is_running(pid) is False, "a zombie is not a live owner"
|
|
|
|
os.waitpid(pid, 0) # reap so the test leaks no children
|
|
|
|
|
|
@pytest.mark.platforms("linux", "macos")
|
|
def test_process_state_reports_a_letter_for_a_live_pid():
|
|
state = er._process_state(os.getpid())
|
|
assert state is not None and len(state) == 1, "a live pid must expose a state"
|
|
assert not state.upper().startswith("Z"), "this process is not a zombie"
|
|
|
|
|
|
@pytest.mark.platforms("posix")
|
|
def test_process_state_is_none_for_a_dead_pid():
|
|
state = er._process_state(4294967294)
|
|
assert state is None, "an unprobeable pid must degrade to None (unknown)"
|
|
|
|
|
|
def test_marker_owner_liveness_uses_recorded_pid(tmp_path, monkeypatch):
|
|
marker = tmp_path / ".update-incomplete"
|
|
marker.write_text("started=1\npid=4321\n", encoding="utf-8")
|
|
seen = []
|
|
monkeypatch.setattr(
|
|
er, "_pid_is_running", lambda pid: seen.append(pid) or True
|
|
)
|
|
|
|
assert er._marker_owner_is_live(marker) is True
|
|
assert seen == [4321]
|
|
|
|
def _project(tmp_path: Path, *, pyproject: bool = True) -> Path:
|
|
root = tmp_path / "proj"
|
|
root.mkdir(exist_ok=True)
|
|
if pyproject:
|
|
(root / "pyproject.toml").write_text(
|
|
'[project]\nname = "x"\ndependencies = [\n'
|
|
' "ruamel.yaml==0.18.17",\n'
|
|
' "python-dotenv==1.2.2",\n'
|
|
' "PyJWT[crypto]==2.13.0",\n'
|
|
"]\n",
|
|
encoding="utf-8",
|
|
)
|
|
return root
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|