hermes-trace/tests/test_launcher.py

580 lines
22 KiB
Python

"""Tests for the frontend launcher: runtime selection, preflight, retry.
Covers ``_check_plugin_python()`` (dependency preflight) and
``_run_frontend()`` (Granian subprocess lifecycle).
"""
from __future__ import annotations
import importlib.util
import os
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
from unittest.mock import MagicMock, call, patch
# Import the plugin module (hyphen dir → importlib)
_plugin_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
_plugin_init = os.path.join(_plugin_root, "__init__.py")
_module_name = "hermes_trace_launcher_test"
_original_module = sys.modules.get(_module_name)
_spec = importlib.util.spec_from_file_location(
_module_name,
_plugin_init,
submodule_search_locations=[_plugin_root],
)
_plugin = importlib.util.module_from_spec(_spec)
sys.modules[_module_name] = _plugin
_spec.loader.exec_module(_plugin)
def tearDownModule() -> None:
if _original_module is None:
sys.modules.pop(_module_name, None)
else:
sys.modules[_module_name] = _original_module
_check_plugin_python = _plugin._check_plugin_python
class _SentinelExit(Exception):
"""Raised inside a mocked call to exit the infinite retry loop."""
# ---------------------------------------------------------------------------
# _check_plugin_python — dependency preflight
# ---------------------------------------------------------------------------
class TestCheckPluginPython(unittest.TestCase):
"""``_check_plugin_python`` correctly translates subprocess outcomes."""
@patch(f"{_module_name}.launcher.subprocess.run")
def test_imports_ok_returns_true(self, mock_run: MagicMock) -> None:
"""returncode=0 and stdout='ok' → True."""
mock_run.return_value = MagicMock(returncode=0, stdout="ok\n")
self.assertTrue(_check_plugin_python("/fake/python"))
@patch(f"{_module_name}.launcher.subprocess.run")
def test_import_failure_returns_false(self, mock_run: MagicMock) -> None:
"""non-zero returncode → False."""
mock_run.return_value = MagicMock(returncode=1, stdout="error\n")
self.assertFalse(_check_plugin_python("/fake/python"))
@patch(f"{_module_name}.launcher.subprocess.run")
def test_stdout_not_ok_returns_false(self, mock_run: MagicMock) -> None:
"""returncode=0 but stdout isn't 'ok' → False."""
mock_run.return_value = MagicMock(returncode=0, stdout="not ok\n")
self.assertFalse(_check_plugin_python("/fake/python"))
@patch(f"{_module_name}.launcher.subprocess.run")
def test_timeout_expired_returns_false(self, mock_run: MagicMock) -> None:
"""subprocess.TimeoutExpired → False."""
mock_run.side_effect = subprocess.TimeoutExpired(cmd="python", timeout=15)
self.assertFalse(_check_plugin_python("/fake/python"))
@patch(f"{_module_name}.launcher.subprocess.run")
def test_os_error_returns_false(self, mock_run: MagicMock) -> None:
"""OSError during subprocess.run → False."""
mock_run.side_effect = OSError("No such file or directory")
self.assertFalse(_check_plugin_python("/fake/python"))
@patch(f"{_module_name}.launcher.subprocess.run")
def test_python_path_forwarded(self, mock_run: MagicMock) -> None:
"""The candidate Python path is passed correctly to subprocess."""
mock_run.return_value = MagicMock(returncode=0, stdout="ok\n")
_check_plugin_python("/my/custom/python3")
args, kwargs = mock_run.call_args
executable_list = args[0]
self.assertEqual(executable_list[0], "/my/custom/python3")
self.assertIn("-c", executable_list)
self.assertIn("granian", executable_list[2])
self.assertIn("fastapi", executable_list[2])
self.assertIn("jinja2", executable_list[2])
self.assertEqual(kwargs.get("timeout"), 15)
self.assertTrue(kwargs.get("capture_output"))
self.assertTrue(kwargs.get("text"))
# ---------------------------------------------------------------------------
# _run_frontend — launcher lifecycle
# ---------------------------------------------------------------------------
class _RunFrontendBase(unittest.TestCase):
"""Shared setUp/tearDown for all ``_run_frontend`` tests."""
def setUp(self) -> None:
self.temp_dir = tempfile.TemporaryDirectory()
lock_path = os.path.join(self.temp_dir.name, "telemetry.db.lock")
_plugin._state._LOCK_PATH = lock_path
_plugin._state._telemetry_db_path = os.path.join(
self.temp_dir.name, "telemetry.db"
)
_plugin._state._telemetry_host = "127.0.0.1"
_plugin._state._telemetry_port = 7788
_plugin._session_state.clear()
def tearDown(self) -> None:
_plugin._state._LOCK_PATH = ""
_plugin._state._telemetry_db_path = ""
_plugin._state._telemetry_host = "127.0.0.1"
_plugin._state._telemetry_port = 7788
self.temp_dir.cleanup()
class TestRunFrontendLockHeld(_RunFrontendBase):
"""When another Hermes process already holds the singleton lock."""
@patch(f"{_module_name}.launcher.fcntl.flock")
@patch(f"{_module_name}.launcher.os.close")
def test_lock_held_returns_immediately(
self, mock_close: MagicMock, mock_flock: MagicMock
) -> None:
"""flock raises BlockingIOError → close fd, return, no Popen."""
mock_flock.side_effect = BlockingIOError("Resource temporarily unavailable")
# Also mock Path.is_file so we can detect if execution got past
with patch("pathlib.PosixPath.is_file") as mock_is_file:
_plugin._run_frontend()
mock_close.assert_called_once()
mock_is_file.assert_not_called()
class TestRunFrontendPreflightMissing(_RunFrontendBase):
"""Plugin .venv Python is missing entirely."""
@patch(f"{_module_name}.launcher.time.sleep")
@patch(f"{_module_name}.launcher.subprocess.Popen")
@patch(f"{_module_name}.launcher._check_plugin_python")
@patch(f"{_module_name}.launcher._kill_orphan")
def test_missing_venv_fails_closed(
self,
mock_kill: MagicMock,
mock_preflight: MagicMock,
mock_popen: MagicMock,
mock_sleep: MagicMock,
) -> None:
"""venv_python.is_file() returns False → no Popen, no retry."""
# The real .venv/bin/python exists on disk — mock is_file to
# simulate a missing environment so the function fails closed
# without entering the retry loop.
with patch("pathlib.PosixPath.is_file", return_value=False):
_plugin._run_frontend()
mock_popen.assert_not_called()
mock_preflight.assert_not_called()
mock_kill.assert_not_called() # orphan cleanup follows preflight
mock_sleep.assert_not_called()
class TestRunFrontendPreflightFails(_RunFrontendBase):
"""Plugin .venv Python exists but dependency check fails."""
@patch(f"{_module_name}.launcher.time.sleep")
@patch(f"{_module_name}.launcher.subprocess.Popen")
@patch(f"{_module_name}.launcher._check_plugin_python", return_value=False)
@patch(f"{_module_name}.launcher._kill_orphan")
def test_preflight_failure_fails_closed(
self,
mock_kill: MagicMock,
mock_preflight: MagicMock,
mock_popen: MagicMock,
mock_sleep: MagicMock,
) -> None:
"""_check_plugin_python returns False → no Popen, no retry."""
# Make Path.is_file return True so we reach the preflight step
with patch("pathlib.PosixPath.is_file", return_value=True):
_plugin._run_frontend()
mock_preflight.assert_called_once()
mock_popen.assert_not_called()
mock_kill.assert_not_called() # orphan cleanup follows preflight
mock_sleep.assert_not_called()
@patch(f"{_module_name}.launcher.time.sleep")
@patch(f"{_module_name}.launcher.subprocess.Popen")
@patch(f"{_module_name}.launcher._check_plugin_python", return_value=False)
@patch(f"{_module_name}.launcher._kill_orphan")
def test_preflight_failure_releases_lock(
self,
mock_kill: MagicMock,
mock_preflight: MagicMock,
mock_popen: MagicMock,
mock_sleep: MagicMock,
) -> None:
"""After fail-close, the lock should be released so another
process can acquire it."""
with patch("pathlib.PosixPath.is_file", return_value=True):
_plugin._run_frontend()
# After _run_frontend returns, a second call should be able
# to acquire the lock and proceed (indicates cleanup happened).
mock_popen.assert_not_called()
class TestRunFrontendSuccess(_RunFrontendBase):
"""Full success path: correct venv python passed to Granian."""
# The actual plugin .venv/bin/python exists on disk — let it resolve
# naturally, but mock _check_plugin_python to avoid a real subprocess.
@patch(f"{_module_name}.launcher.time.sleep")
@patch(f"{_module_name}.launcher.subprocess.Popen")
@patch(f"{_module_name}.launcher._check_plugin_python", return_value=True)
@patch(f"{_module_name}.launcher._kill_orphan")
def test_uses_plugin_venv_python(
self,
mock_kill: MagicMock,
mock_preflight: MagicMock,
mock_popen: MagicMock,
mock_sleep: MagicMock,
) -> None:
"""Popen.argv[0] is the plugin's .venv/bin/python path."""
mock_proc = MagicMock()
mock_proc.wait.side_effect = _SentinelExit("break loop")
mock_popen.return_value = mock_proc
with self.assertRaises(_SentinelExit):
_plugin._run_frontend()
mock_popen.assert_called_once()
popen_args = mock_popen.call_args[0][0]
expected_venv = str(
Path(_plugin.__file__).resolve().parent / ".venv" / "bin" / "python"
)
self.assertEqual(popen_args[0], expected_venv)
self.assertIn("-m", popen_args)
self.assertIn("granian", popen_args)
self.assertIn("app:app", popen_args)
@patch(f"{_module_name}.launcher.time.sleep")
@patch(f"{_module_name}.launcher.subprocess.Popen")
@patch(f"{_module_name}.launcher._check_plugin_python", return_value=True)
@patch(f"{_module_name}.launcher._kill_orphan")
def test_env_vars_passed_to_granian(
self,
mock_kill: MagicMock,
mock_preflight: MagicMock,
mock_popen: MagicMock,
mock_sleep: MagicMock,
) -> None:
"""Environment includes HERMES_TRACE_DB_PATH, PATH, HOME."""
mock_proc = MagicMock()
mock_proc.wait.side_effect = _SentinelExit("break loop")
mock_popen.return_value = mock_proc
with self.assertRaises(_SentinelExit):
_plugin._run_frontend()
_, kwargs = mock_popen.call_args
env = kwargs.get("env", {})
self.assertIn("HERMES_TRACE_DB_PATH", env)
self.assertIn("PATH", env)
self.assertIn("HOME", env)
self.assertEqual(
env["HERMES_TRACE_DB_PATH"],
_plugin._state._telemetry_db_path,
)
@patch(f"{_module_name}.launcher.time.sleep")
@patch(f"{_module_name}.launcher.subprocess.Popen")
@patch(f"{_module_name}.launcher._check_plugin_python", return_value=True)
@patch(f"{_module_name}.launcher._kill_orphan")
def test_cwd_is_plugin_dir(
self,
mock_kill: MagicMock,
mock_preflight: MagicMock,
mock_popen: MagicMock,
mock_sleep: MagicMock,
) -> None:
"""Popen is called with cwd=plugin directory."""
mock_proc = MagicMock()
mock_proc.wait.side_effect = _SentinelExit("break loop")
mock_popen.return_value = mock_proc
with self.assertRaises(_SentinelExit):
_plugin._run_frontend()
_, kwargs = mock_popen.call_args
expected_cwd = str(Path(_plugin.__file__).resolve().parent)
self.assertEqual(str(kwargs.get("cwd", "")), expected_cwd)
@patch(f"{_module_name}.launcher.time.sleep")
@patch(f"{_module_name}.launcher.subprocess.Popen")
@patch(f"{_module_name}.launcher._check_plugin_python", return_value=True)
@patch(f"{_module_name}.launcher._kill_orphan")
def test_granian_host_port_forwarded(
self,
mock_kill: MagicMock,
mock_preflight: MagicMock,
mock_popen: MagicMock,
mock_sleep: MagicMock,
) -> None:
"""Host and port from plugin config are passed to Granian."""
mock_proc = MagicMock()
mock_proc.wait.side_effect = _SentinelExit("break loop")
mock_popen.return_value = mock_proc
with self.assertRaises(_SentinelExit):
_plugin._run_frontend()
popen_args = mock_popen.call_args[0][0]
self.assertIn("--host", popen_args)
self.assertIn("127.0.0.1", popen_args)
self.assertIn("--port", popen_args)
self.assertIn("7788", popen_args)
class TestRunFrontendRetryAndCleanup(_RunFrontendBase):
"""Child exit triggers sleep-retry; cleanup runs on exit."""
@patch(f"{_module_name}.launcher.time.sleep")
@patch(f"{_module_name}.launcher.subprocess.Popen")
@patch(f"{_module_name}.launcher._check_plugin_python", return_value=True)
@patch(f"{_module_name}.launcher._kill_orphan")
def test_child_exit_triggers_sleep_then_retry(
self,
mock_kill: MagicMock,
mock_preflight: MagicMock,
mock_popen: MagicMock,
mock_sleep: MagicMock,
) -> None:
"""After child exits, sleep(3) is called, then Popen is called
again for a retry."""
call_count = 0
def popen_side(*args: object, **kwargs: object) -> MagicMock:
nonlocal call_count
call_count += 1
proc = MagicMock()
if call_count == 1:
proc.wait.return_value = 0 # normal exit
else:
proc.wait.side_effect = _SentinelExit("break loop after retry")
return proc
mock_popen.side_effect = popen_side
with self.assertRaises(_SentinelExit):
_plugin._run_frontend()
self.assertEqual(mock_popen.call_count, 2)
mock_sleep.assert_has_calls([call(3)])
@patch(f"{_module_name}.launcher.time.sleep")
@patch(f"{_module_name}.launcher.subprocess.Popen")
@patch(f"{_module_name}.launcher._check_plugin_python", return_value=True)
@patch(f"{_module_name}.launcher._kill_orphan")
def test_cleanup_runs_on_exception(
self,
mock_kill: MagicMock,
mock_preflight: MagicMock,
mock_popen: MagicMock,
mock_sleep: MagicMock,
) -> None:
"""When an exception escapes the retry loop, cleanup releases the lock
without removing its permanent pathname."""
mock_proc = MagicMock()
mock_proc.wait.side_effect = _SentinelExit("break loop")
mock_popen.return_value = mock_proc
with self.assertRaises(_SentinelExit):
_plugin._run_frontend()
# The PID file is transient, but the lock pathname must remain stable
# so every process flocks the same inode.
lock_path = _plugin._state._LOCK_PATH
self.assertTrue(os.path.exists(lock_path))
self.assertFalse(os.path.exists(lock_path + ".pid"))
@patch(f"{_module_name}.launcher.time.sleep")
@patch(f"{_module_name}.launcher.subprocess.Popen")
@patch(f"{_module_name}.launcher._check_plugin_python", return_value=True)
@patch(f"{_module_name}.launcher._kill_orphan")
def test_pid_file_cleaned_on_exit(
self,
mock_kill: MagicMock,
mock_preflight: MagicMock,
mock_popen: MagicMock,
mock_sleep: MagicMock,
) -> None:
"""PID file is removed after the function exits."""
mock_proc = MagicMock()
mock_proc.wait.side_effect = _SentinelExit("break loop")
mock_popen.return_value = mock_proc
with self.assertRaises(_SentinelExit):
_plugin._run_frontend()
pid_file = _plugin._state._LOCK_PATH + ".pid"
self.assertFalse(os.path.exists(pid_file))
@patch(f"{_module_name}.launcher.time.sleep")
@patch(f"{_module_name}.launcher.subprocess.Popen")
@patch(f"{_module_name}.launcher._check_plugin_python", return_value=True)
@patch(f"{_module_name}.launcher._kill_orphan")
def test_running_child_is_terminated_during_cleanup(
self,
mock_kill_orphan: MagicMock,
mock_preflight: MagicMock,
mock_popen: MagicMock,
mock_sleep: MagicMock,
) -> None:
"""A live child is terminated when the launcher exits unexpectedly."""
mock_proc = MagicMock()
mock_proc.poll.return_value = None
mock_proc.wait.side_effect = [_SentinelExit("break loop"), None]
mock_popen.return_value = mock_proc
with self.assertRaises(_SentinelExit):
_plugin._run_frontend()
mock_proc.terminate.assert_called_once_with()
mock_proc.kill.assert_not_called()
mock_proc.wait.assert_has_calls([call(), call(timeout=5)])
@patch(f"{_module_name}.launcher.time.sleep")
@patch(f"{_module_name}.launcher.subprocess.Popen")
@patch(f"{_module_name}.launcher._check_plugin_python", return_value=True)
@patch(f"{_module_name}.launcher._kill_orphan")
def test_child_is_killed_after_terminate_timeout(
self,
mock_kill_orphan: MagicMock,
mock_preflight: MagicMock,
mock_popen: MagicMock,
mock_sleep: MagicMock,
) -> None:
"""A child that ignores SIGTERM is force-killed after five seconds."""
mock_proc = MagicMock()
mock_proc.poll.side_effect = [None, None]
mock_proc.wait.side_effect = [
_SentinelExit("break loop"),
subprocess.TimeoutExpired(cmd="granian", timeout=5),
None,
]
mock_popen.return_value = mock_proc
with self.assertRaises(_SentinelExit):
_plugin._run_frontend()
mock_proc.terminate.assert_called_once_with()
mock_proc.kill.assert_called_once_with()
mock_proc.wait.assert_has_calls([call(), call(timeout=5), call(timeout=5)])
@patch(f"{_module_name}.launcher.time.sleep")
@patch(f"{_module_name}.launcher.subprocess.Popen")
@patch(f"{_module_name}.launcher._check_plugin_python", return_value=True)
@patch(f"{_module_name}.launcher._kill_orphan")
def test_pid_write_failure_stops_child(
self,
mock_kill_orphan: MagicMock,
mock_preflight: MagicMock,
mock_popen: MagicMock,
mock_sleep: MagicMock,
) -> None:
"""A PID metadata failure cannot leave the Granian child running."""
mock_proc = MagicMock()
mock_proc.poll.return_value = None
mock_proc.wait.return_value = None
mock_popen.return_value = mock_proc
with (
patch(f"{_module_name}.launcher.open", side_effect=OSError("read-only")),
self.assertRaises(OSError),
):
_plugin._run_frontend()
mock_proc.terminate.assert_called_once_with()
mock_proc.kill.assert_not_called()
class TestKillOrphan(unittest.TestCase):
"""Orphan cleanup only targets the exact Linux process argv."""
def setUp(self) -> None:
self.temp_dir = tempfile.TemporaryDirectory()
_plugin._state._LOCK_PATH = os.path.join(
self.temp_dir.name, "telemetry.db.lock"
)
_plugin._state._telemetry_host = "127.0.0.1"
_plugin._state._telemetry_port = 7788
def tearDown(self) -> None:
_plugin._state._LOCK_PATH = ""
_plugin._state._telemetry_host = "127.0.0.1"
_plugin._state._telemetry_port = 7788
self.temp_dir.cleanup()
def _run_with_cmdline(self, cmdline: bytes) -> MagicMock:
"""Run orphan cleanup with a mocked PID and /proc cmdline."""
mock_file = MagicMock()
mock_file.__enter__.return_value.read.side_effect = ["1234", cmdline]
with (
patch(f"{_module_name}.launcher.open", return_value=mock_file),
patch(f"{_module_name}.launcher.os.kill") as mock_os_kill,
patch(f"{_module_name}.launcher.os.unlink"),
patch(f"{_module_name}.launcher.time.sleep"),
):
_plugin._kill_orphan()
return mock_os_kill
def test_port_match_is_exact(self) -> None:
"""A port embedded in another argument does not identify our child."""
expected = _plugin.launcher._granian_command(
str(Path(_plugin.__file__).resolve().parent / ".venv" / "bin" / "python")
)
expected[expected.index("7788")] = "17788"
cmdline = b"\0".join(os.fsencode(arg) for arg in expected) + b"\0"
mock_os_kill = self._run_with_cmdline(cmdline)
mock_os_kill.assert_not_called()
def test_exact_command_match_is_killed(self) -> None:
"""The exact expected Granian argv is eligible for SIGTERM."""
expected = _plugin.launcher._granian_command(
str(Path(_plugin.__file__).resolve().parent / ".venv" / "bin" / "python")
)
cmdline = b"\0".join(os.fsencode(arg) for arg in expected) + b"\0"
mock_os_kill = self._run_with_cmdline(cmdline)
mock_os_kill.assert_called_once_with(1234, _plugin.launcher.signal.SIGTERM)
class TestRunFrontendEnvPassthrough(_RunFrontendBase):
"""Environment variable inheritance for the Granian subprocess."""
@patch(f"{_module_name}.launcher.time.sleep")
@patch(f"{_module_name}.launcher.subprocess.Popen")
@patch(f"{_module_name}.launcher._check_plugin_python", return_value=True)
@patch(f"{_module_name}.launcher._kill_orphan")
def test_home_from_environment(
self,
mock_kill: MagicMock,
mock_preflight: MagicMock,
mock_popen: MagicMock,
mock_sleep: MagicMock,
) -> None:
"""HOME is inherited from the parent environment."""
mock_proc = MagicMock()
mock_proc.wait.side_effect = _SentinelExit("break loop")
mock_popen.return_value = mock_proc
with (
patch.dict(os.environ, {"HOME": "/custom/home"}, clear=False),
self.assertRaises(_SentinelExit),
):
_plugin._run_frontend()
_, kwargs = mock_popen.call_args
self.assertEqual(kwargs["env"]["HOME"], "/custom/home")
if __name__ == "__main__":
unittest.main()