hermes-trace/launcher.py

219 lines
6.9 KiB
Python

from __future__ import annotations
import fcntl
import logging
import os
import signal
import subprocess
import time
from pathlib import Path
from . import _state, db
logger = logging.getLogger(__name__)
def _cleanup_loop() -> None:
"""Daemon thread: delete expired turns periodically.
Normal interval is 3600s (1 hour). On failure uses exponential backoff
starting at 60s, doubling up to a maximum of 3600s, and resets to 3600s
on success.
"""
backoff = 60 # first run after 60s, then stabilise to 3600
while True:
try:
time.sleep(backoff)
deleted = db.get_db().clean_expired_turns(_state._telemetry_retention_days)
if deleted > 0:
logger.info("Cleaned up %d expired turn(s)", deleted)
backoff = 3600 # reset on success (hourly)
except Exception:
logger.exception("Cleanup loop — error during expiry check")
backoff = min(backoff * 2, 3600)
def _check_plugin_python(python_path: str) -> bool:
"""Verify the candidate Python can import core dependencies.
Runs a minimal subprocess preflight — returns True only if
granian, fastapi, and jinja2 are all importable inside the
candidate Python runtime.
"""
try:
result = subprocess.run(
[
python_path,
"-c",
"import granian, fastapi, jinja2; print('ok')",
],
capture_output=True,
text=True,
timeout=15,
check=False,
)
return result.returncode == 0 and result.stdout.strip() == "ok"
except (subprocess.SubprocessError, OSError):
return False
def _granian_command(python_path: str) -> list[str]:
"""Build the exact command line used for the Granian subprocess."""
return [
python_path,
"-m",
"granian",
"app:app",
"--interface",
"asgi",
"--host",
_state._telemetry_host,
"--port",
str(_state._telemetry_port),
"--log-level",
"warning",
]
def _kill_orphan() -> None:
"""Kill any lingering Granian subprocess from a previous crashed gateway.
Reads the PID file and only sends SIGTERM if the process matches the
expected ``app:app`` module and port (verified via ``/proc/<pid>/cmdline``).
"""
pid_file = _state._LOCK_PATH + ".pid"
try:
with open(pid_file) as f:
pid = int(f.read().strip())
# Confirm it's really OUR Granian, not a reused PID.
try:
# Linux-only: /proc exposes the exact argv; no macOS fallback yet.
with open(f"/proc/{pid}/cmdline", "rb") as f:
cmdline = f.read().split(b"\0")
if cmdline and cmdline[-1] == b"":
cmdline.pop()
expected = [
os.fsencode(arg)
for arg in _granian_command(
str(Path(__file__).resolve().parent / ".venv" / "bin" / "python")
)
]
if cmdline != expected:
return # PID reused or wrong service — leave it alone
except (FileNotFoundError, OSError):
return # Process doesn't exist — nothing to kill
os.kill(pid, signal.SIGTERM)
time.sleep(0.5)
except (FileNotFoundError, ValueError, ProcessLookupError, OSError):
pass
finally:
try:
os.unlink(pid_file)
except FileNotFoundError:
pass
def _stop_child(proc: subprocess.Popen[bytes]) -> None:
"""Terminate a running frontend child and force-stop it if necessary."""
try:
if proc.poll() is not None:
return
except Exception:
logger.exception("Unable to poll the frontend child during cleanup")
try:
proc.terminate()
except ProcessLookupError:
return
except OSError:
logger.exception("Unable to terminate the frontend child during cleanup")
return
try:
proc.wait(timeout=5)
return
except subprocess.TimeoutExpired:
pass
except Exception:
logger.exception("Unexpected error waiting for the frontend child")
try:
if proc.poll() is None:
proc.kill()
except ProcessLookupError:
return
except OSError:
logger.exception("Unable to kill the frontend child during cleanup")
return
try:
proc.wait(timeout=5)
except subprocess.TimeoutExpired:
logger.error("Frontend child did not exit after SIGKILL")
except Exception:
logger.exception("Unexpected error waiting after killing the frontend child")
def _run_frontend() -> None:
"""Daemon thread: launch Granian with singleton lock.
Uses ``fcntl.flock(LOCK_NB)`` as a process-scoped lock — when the
process dies (exit / crash / SIGKILL), the kernel releases it
automatically. Only the process that holds the lock launches and
maintains the Granian frontend subprocess.
Before starting, kills any orphaned Granian from a previous crash
(checked via PID file + /proc/<pid>/cmdline).
"""
# Try to acquire the singleton lock
lock_fd = os.open(_state._LOCK_PATH, os.O_CREAT | os.O_RDWR, 0o644)
try:
fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
except OSError:
os.close(lock_fd)
return # Another Hermes process holds the lock — skip
plugin_dir = Path(__file__).resolve().parent
venv_python = plugin_dir / ".venv" / "bin" / "python"
env = {
"HERMES_TRACE_DB_PATH": _state._telemetry_db_path,
"HERMES_HOME": os.environ.get("HERMES_HOME", ""),
"PATH": os.environ.get("PATH", ""),
"HOME": os.environ.get("HOME", ""),
}
pid_file = _state._LOCK_PATH + ".pid"
proc: subprocess.Popen[bytes] | None = None
try:
# Use plugin's own venv Python — granian and frontend deps live here
if venv_python.is_file() and _check_plugin_python(str(venv_python)):
hermes_python = str(venv_python)
else:
logger.error(
"Plugin venv Python not found or missing dependencies at %s — "
"run `uv sync` in %s to set up the frontend environment",
venv_python,
plugin_dir,
)
return # fail closed, don't enter retry loop (finally cleans up lock)
_kill_orphan()
while True:
proc = subprocess.Popen(
_granian_command(hermes_python), cwd=plugin_dir, env=env
)
with open(pid_file, "w") as f:
f.write(str(proc.pid))
proc.wait()
# If Granian exited on its own, wait before respawning
time.sleep(3)
finally:
if proc is not None:
_stop_child(proc)
try:
os.unlink(pid_file)
except FileNotFoundError:
pass
fcntl.flock(lock_fd, fcntl.LOCK_UN)
os.close(lock_fd)