219 lines
6.9 KiB
Python
219 lines
6.9 KiB
Python
from __future__ import annotations
|
|
|
|
import fcntl
|
|
import logging
|
|
import os
|
|
import signal
|
|
import subprocess
|
|
import time
|
|
from pathlib import Path
|
|
|
|
from . import _state, db
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def _cleanup_loop() -> None:
|
|
"""Daemon thread: delete expired turns periodically.
|
|
|
|
Normal interval is 3600s (1 hour). On failure uses exponential backoff
|
|
starting at 60s, doubling up to a maximum of 3600s, and resets to 3600s
|
|
on success.
|
|
"""
|
|
backoff = 60 # first run after 60s, then stabilise to 3600
|
|
while True:
|
|
try:
|
|
time.sleep(backoff)
|
|
deleted = db.get_db().clean_expired_turns(_state._telemetry_retention_days)
|
|
if deleted > 0:
|
|
logger.info("Cleaned up %d expired turn(s)", deleted)
|
|
backoff = 3600 # reset on success (hourly)
|
|
except Exception:
|
|
logger.exception("Cleanup loop — error during expiry check")
|
|
backoff = min(backoff * 2, 3600)
|
|
|
|
|
|
def _check_plugin_python(python_path: str) -> bool:
|
|
"""Verify the candidate Python can import core dependencies.
|
|
|
|
Runs a minimal subprocess preflight — returns True only if
|
|
granian, fastapi, and jinja2 are all importable inside the
|
|
candidate Python runtime.
|
|
"""
|
|
try:
|
|
result = subprocess.run(
|
|
[
|
|
python_path,
|
|
"-c",
|
|
"import granian, fastapi, jinja2; print('ok')",
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=15,
|
|
check=False,
|
|
)
|
|
return result.returncode == 0 and result.stdout.strip() == "ok"
|
|
except (subprocess.SubprocessError, OSError):
|
|
return False
|
|
|
|
|
|
def _granian_command(python_path: str) -> list[str]:
|
|
"""Build the exact command line used for the Granian subprocess."""
|
|
return [
|
|
python_path,
|
|
"-m",
|
|
"granian",
|
|
"app:app",
|
|
"--interface",
|
|
"asgi",
|
|
"--host",
|
|
_state._telemetry_host,
|
|
"--port",
|
|
str(_state._telemetry_port),
|
|
"--log-level",
|
|
"warning",
|
|
]
|
|
|
|
|
|
def _kill_orphan() -> None:
|
|
"""Kill any lingering Granian subprocess from a previous crashed gateway.
|
|
|
|
Reads the PID file and only sends SIGTERM if the process matches the
|
|
expected ``app:app`` module and port (verified via ``/proc/<pid>/cmdline``).
|
|
"""
|
|
pid_file = _state._LOCK_PATH + ".pid"
|
|
try:
|
|
with open(pid_file) as f:
|
|
pid = int(f.read().strip())
|
|
# Confirm it's really OUR Granian, not a reused PID.
|
|
try:
|
|
# Linux-only: /proc exposes the exact argv; no macOS fallback yet.
|
|
with open(f"/proc/{pid}/cmdline", "rb") as f:
|
|
cmdline = f.read().split(b"\0")
|
|
if cmdline and cmdline[-1] == b"":
|
|
cmdline.pop()
|
|
expected = [
|
|
os.fsencode(arg)
|
|
for arg in _granian_command(
|
|
str(Path(__file__).resolve().parent / ".venv" / "bin" / "python")
|
|
)
|
|
]
|
|
if cmdline != expected:
|
|
return # PID reused or wrong service — leave it alone
|
|
except (FileNotFoundError, OSError):
|
|
return # Process doesn't exist — nothing to kill
|
|
os.kill(pid, signal.SIGTERM)
|
|
time.sleep(0.5)
|
|
except (FileNotFoundError, ValueError, ProcessLookupError, OSError):
|
|
pass
|
|
finally:
|
|
try:
|
|
os.unlink(pid_file)
|
|
except FileNotFoundError:
|
|
pass
|
|
|
|
|
|
def _stop_child(proc: subprocess.Popen[bytes]) -> None:
|
|
"""Terminate a running frontend child and force-stop it if necessary."""
|
|
try:
|
|
if proc.poll() is not None:
|
|
return
|
|
except Exception:
|
|
logger.exception("Unable to poll the frontend child during cleanup")
|
|
|
|
try:
|
|
proc.terminate()
|
|
except ProcessLookupError:
|
|
return
|
|
except OSError:
|
|
logger.exception("Unable to terminate the frontend child during cleanup")
|
|
return
|
|
|
|
try:
|
|
proc.wait(timeout=5)
|
|
return
|
|
except subprocess.TimeoutExpired:
|
|
pass
|
|
except Exception:
|
|
logger.exception("Unexpected error waiting for the frontend child")
|
|
|
|
try:
|
|
if proc.poll() is None:
|
|
proc.kill()
|
|
except ProcessLookupError:
|
|
return
|
|
except OSError:
|
|
logger.exception("Unable to kill the frontend child during cleanup")
|
|
return
|
|
|
|
try:
|
|
proc.wait(timeout=5)
|
|
except subprocess.TimeoutExpired:
|
|
logger.error("Frontend child did not exit after SIGKILL")
|
|
except Exception:
|
|
logger.exception("Unexpected error waiting after killing the frontend child")
|
|
|
|
|
|
def _run_frontend() -> None:
|
|
"""Daemon thread: launch Granian with singleton lock.
|
|
|
|
Uses ``fcntl.flock(LOCK_NB)`` as a process-scoped lock — when the
|
|
process dies (exit / crash / SIGKILL), the kernel releases it
|
|
automatically. Only the process that holds the lock launches and
|
|
maintains the Granian frontend subprocess.
|
|
|
|
Before starting, kills any orphaned Granian from a previous crash
|
|
(checked via PID file + /proc/<pid>/cmdline).
|
|
"""
|
|
# Try to acquire the singleton lock
|
|
lock_fd = os.open(_state._LOCK_PATH, os.O_CREAT | os.O_RDWR, 0o644)
|
|
try:
|
|
fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except OSError:
|
|
os.close(lock_fd)
|
|
return # Another Hermes process holds the lock — skip
|
|
|
|
plugin_dir = Path(__file__).resolve().parent
|
|
venv_python = plugin_dir / ".venv" / "bin" / "python"
|
|
|
|
env = {
|
|
"HERMES_TRACE_DB_PATH": _state._telemetry_db_path,
|
|
"HERMES_HOME": os.environ.get("HERMES_HOME", ""),
|
|
"PATH": os.environ.get("PATH", ""),
|
|
"HOME": os.environ.get("HOME", ""),
|
|
}
|
|
|
|
pid_file = _state._LOCK_PATH + ".pid"
|
|
proc: subprocess.Popen[bytes] | None = None
|
|
try:
|
|
# Use plugin's own venv Python — granian and frontend deps live here
|
|
if venv_python.is_file() and _check_plugin_python(str(venv_python)):
|
|
hermes_python = str(venv_python)
|
|
else:
|
|
logger.error(
|
|
"Plugin venv Python not found or missing dependencies at %s — "
|
|
"run `uv sync` in %s to set up the frontend environment",
|
|
venv_python,
|
|
plugin_dir,
|
|
)
|
|
return # fail closed, don't enter retry loop (finally cleans up lock)
|
|
|
|
_kill_orphan()
|
|
while True:
|
|
proc = subprocess.Popen(
|
|
_granian_command(hermes_python), cwd=plugin_dir, env=env
|
|
)
|
|
with open(pid_file, "w") as f:
|
|
f.write(str(proc.pid))
|
|
proc.wait()
|
|
# If Granian exited on its own, wait before respawning
|
|
time.sleep(3)
|
|
finally:
|
|
if proc is not None:
|
|
_stop_child(proc)
|
|
try:
|
|
os.unlink(pid_file)
|
|
except FileNotFoundError:
|
|
pass
|
|
fcntl.flock(lock_fd, fcntl.LOCK_UN)
|
|
os.close(lock_fd)
|