hermes-agent/tests/hermes_cli/test_gateway_job_teardown_live.py
ethernet 59624b7ace tests: replace every bare host skipif with platforms()
The OS lanes are marker-driven: list_os_marked_tests.py picks the files
a lane imports from their platforms() specs and the lane selects with
-m platforms. A test gated with skipif(sys.platform != "win32") is
therefore never imported on the Windows lane and skipped everywhere else
— it runs on no host. skipif(sys.platform == "win32") tests were merely
invisible to the lane bookkeeping, but the rule the tree now follows is
one host marker, never a bare skipif.

Mechanical mapping, semantics preserved: skip-on-Windows → "posix",
skip-off-Windows → "windows", skip-off-Linux → "linux", skip-on-macOS →
"not macos". The former skip reasons stay as trailing comments. A
non-host condition (os.geteuid() == 0) stays a separate skipif beside
the marker, spelled getattr(os, "geteuid", ...) so the decorator still
imports on Windows.

Where the conversion would stack two platforms() marks on one test (the
conftest rejects that at collection) the narrower mark wins:
- test_update_wedged_gateway: the class is already platforms("linux");
  its per-test "needs UNIX sockets" marks were redundant and are gone.
- test_process_registry.TestSystemdCgroupIsolation: the class-level
  skip-on-Windows moves onto the 11 methods that had no host mark; the
  11 platforms("linux") methods keep theirs.
- test_file_ops_single_roundtrip: the two fifo tests drop their
  platforms("linux") in favour of the module's "posix" (mkfifo exists on
  macOS; both tests already skip when it does not).
- test_linux_desktop_entry / test_gateway_job_teardown_live: duplicate
  or wider marks removed.
2026-09-21 19:18:15 -04:00

332 lines
13 KiB
Python

"""LIVE Windows E2E for #48820 (4th repro): Job-Object teardown vs the
gateway restart watcher, with real processes on a real windows-latest runner.
Three live proofs (no mocks of the code under test):
1. ``TestJobObjectMechanismLive`` — the mechanism everything rests on:
a child spawned with ``windows_detach_flags()`` (CREATE_BREAKAWAY_FROM_JOB)
from inside a kill-on-close Job Object SURVIVES the job teardown, while a
child spawned with ``windows_detach_flags_without_breakaway()`` is killed
by it. This is exactly the reporter's suspected kill path.
2. ``TestWatcherRespawnLive`` — drives the REAL
``hermes_cli.gateway._spawn_gateway_restart_watcher`` end to end with a
real stub gateway process, against a temp HERMES_HOME:
- the respawned process's stderr must land in ``logs/gateway-stdio.log``
(on unfixed main it went to DEVNULL: a job-teardown kill left ZERO trace);
- the respawn env must carry ``_HERMES_GATEWAY_BREAKAWAY=1`` (the stamp
that makes a later job-teardown death diagnosable in exit-diag).
3. ``TestResumeVerificationLive`` — the user-visible symptom: the updater's
``_resume_windows_gateways_after_update`` must NOT print
"✓ Restarting Windows gateway profile(s)" when the relaunched gateway is
dead. The relaunch chain runs for real; the "gateway" is a stub that exits
immediately (standing in for the job-teardown kill). On unfixed main the ✓
is printed anyway; after the fix the resume raises "not verified alive".
"""
from __future__ import annotations
import ctypes
import os
import subprocess
import sys
import time
from ctypes import wintypes
from pathlib import Path
import pytest
pytestmark = pytest.mark.platforms("windows") # native Windows only
_REPO_ROOT = Path(__file__).resolve().parents[2]
JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE = 0x00002000
JOB_OBJECT_LIMIT_BREAKAWAY_OK = 0x00000800
JobObjectExtendedLimitInformation = 9
PROCESS_ALL_ACCESS = 0x001FFFFF
class IO_COUNTERS(ctypes.Structure):
_fields_ = [
("ReadOperationCount", ctypes.c_ulonglong),
("WriteOperationCount", ctypes.c_ulonglong),
("OtherOperationCount", ctypes.c_ulonglong),
("ReadTransferCount", ctypes.c_ulonglong),
("WriteTransferCount", ctypes.c_ulonglong),
("OtherTransferCount", ctypes.c_ulonglong),
]
class JOBOBJECT_BASIC_LIMIT_INFORMATION(ctypes.Structure):
_fields_ = [
("PerProcessUserTimeLimit", ctypes.c_longlong),
("PerJobUserTimeLimit", ctypes.c_longlong),
("LimitFlags", wintypes.DWORD),
("MinimumWorkingSetSize", ctypes.c_size_t),
("MaximumWorkingSetSize", ctypes.c_size_t),
("ActiveProcessLimit", wintypes.DWORD),
("Affinity", ctypes.c_size_t),
("PriorityClass", wintypes.DWORD),
("SchedulingClass", wintypes.DWORD),
]
class JOBOBJECT_EXTENDED_LIMIT_INFORMATION(ctypes.Structure):
_fields_ = [
("BasicLimitInformation", JOBOBJECT_BASIC_LIMIT_INFORMATION),
("IoInfo", IO_COUNTERS),
("ProcessMemoryLimit", ctypes.c_size_t),
("JobMemoryLimit", ctypes.c_size_t),
("PeakProcessMemoryUsed", ctypes.c_size_t),
("PeakJobMemoryUsed", ctypes.c_size_t),
]
def _make_kill_on_close_job(allow_breakaway: bool) -> int:
kernel32 = ctypes.windll.kernel32
job = kernel32.CreateJobObjectW(None, None)
assert job, "CreateJobObjectW failed"
info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION()
flags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
if allow_breakaway:
flags |= JOB_OBJECT_LIMIT_BREAKAWAY_OK
info.BasicLimitInformation.LimitFlags = flags
ok = kernel32.SetInformationJobObject(
job,
JobObjectExtendedLimitInformation,
ctypes.byref(info),
ctypes.sizeof(info),
)
assert ok, "SetInformationJobObject failed"
return job
def _assign_to_job(job: int, proc: subprocess.Popen) -> None:
kernel32 = ctypes.windll.kernel32
ok = kernel32.AssignProcessToJobObject(job, int(proc._handle))
assert ok, f"AssignProcessToJobObject failed (winerror={ctypes.GetLastError()})"
def _pid_alive(pid: int) -> bool:
kernel32 = ctypes.windll.kernel32
PROCESS_QUERY_LIMITED_INFORMATION = 0x1000
h = kernel32.OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, False, pid)
if not h:
return False
try:
code = wintypes.DWORD()
kernel32.GetExitCodeProcess(h, ctypes.byref(code))
return code.value == 259 # STILL_ACTIVE
finally:
kernel32.CloseHandle(h)
_SLEEPER = "import time; time.sleep(120)"
def _wait_for(predicate, timeout_s: float = 30.0, interval_s: float = 0.25):
deadline = time.monotonic() + timeout_s
while time.monotonic() < deadline:
if predicate():
return True
time.sleep(interval_s)
return False
class TestJobObjectMechanismLive:
"""Real Job Objects, real children — the #48820 kill mechanism."""
def _driver_source(self, flags_helper: str, pid_file: str) -> str:
# The driver runs INSIDE the job and spawns a grandchild "gateway"
# with the flag bundle under test, then exits — mirroring the
# updater/watcher exiting while its job tears down.
return (
"import subprocess, sys, pathlib\n"
"sys.path.insert(0, r'%s')\n"
"from hermes_cli._subprocess_compat import (\n"
" windows_detach_flags, windows_detach_flags_without_breakaway)\n"
"flags = %s()\n"
"p = subprocess.Popen([sys.executable, '-c', %r],\n"
" creationflags=flags, stdin=subprocess.DEVNULL,\n"
" stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n"
"pathlib.Path(r'%s').write_text(str(p.pid), encoding='utf-8')\n"
) % (str(_REPO_ROOT), flags_helper, _SLEEPER, pid_file)
def _run_in_job(self, tmp_path: Path, flags_helper: str) -> int:
pid_file = tmp_path / f"{flags_helper}.pid"
job = _make_kill_on_close_job(allow_breakaway=True)
kernel32 = ctypes.windll.kernel32
try:
driver = subprocess.Popen(
[
sys.executable,
"-c",
# Handshake: wait until the test has assigned us to the
# job before spawning the grandchild.
"import pathlib, sys, time\n"
f"go = pathlib.Path(r'{tmp_path / 'go.marker'}')\n"
"deadline = time.monotonic() + 30\n"
"while not go.exists():\n"
" assert time.monotonic() < deadline, 'no go marker'\n"
" time.sleep(0.1)\n"
+ self._driver_source(flags_helper, str(pid_file)),
],
cwd=str(_REPO_ROOT),
)
_assign_to_job(job, driver)
(tmp_path / "go.marker").write_text("go", encoding="utf-8")
assert _wait_for(pid_file.exists), "driver never wrote the pid file"
gw_pid = int(pid_file.read_text(encoding="utf-8"))
assert _wait_for(lambda: driver.poll() is not None), (
"driver did not exit"
)
assert _pid_alive(gw_pid), "grandchild died before job teardown"
# THE teardown: closing the last job handle fires
# KILL_ON_JOB_CLOSE against every process still in the job.
kernel32.CloseHandle(job)
job = None
time.sleep(2.0)
return gw_pid
finally:
(tmp_path / "go.marker").unlink(missing_ok=True)
if job:
kernel32.CloseHandle(job)
def test_breakaway_child_survives_job_teardown(self, tmp_path):
pid = self._run_in_job(tmp_path, "windows_detach_flags")
try:
assert _pid_alive(pid), (
"CREATE_BREAKAWAY_FROM_JOB child must survive the parent "
"job's kill-on-close teardown"
)
finally:
subprocess.run(
["taskkill", "/PID", str(pid), "/T", "/F"], capture_output=True
)
def test_non_breakaway_child_killed_by_job_teardown(self, tmp_path):
"""The #48820 kill path, reproduced live: no breakaway → the job's
teardown reaps the freshly spawned gateway."""
pid = self._run_in_job(tmp_path, "windows_detach_flags_without_breakaway")
try:
assert not _pid_alive(pid), (
"child without breakaway must be killed by kill-on-close "
"job teardown — this is the silent gateway death of #48820"
)
finally:
subprocess.run(
["taskkill", "/PID", str(pid), "/T", "/F"], capture_output=True
)
class TestWatcherRespawnLive:
"""Drive the real ``_spawn_gateway_restart_watcher`` with real processes."""
def _run_watcher_cycle(self, tmp_path: Path, monkeypatch) -> tuple[Path, Path]:
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "home"))
(tmp_path / "home").mkdir(parents=True, exist_ok=True)
marker = tmp_path / "respawned.marker"
# The stub "gateway": records its breakaway stamp env, screams on
# stderr (so the stdio sidecar has something to capture), then exits.
stub = (
"import os, pathlib, sys\n"
f"pathlib.Path(r'{marker}').write_text(\n"
" os.environ.get('_HERMES_GATEWAY_BREAKAWAY', 'MISSING'),\n"
" encoding='utf-8')\n"
"print('stub-gateway-stderr-trace', file=sys.stderr)\n"
)
# A real old-pid that exits immediately — the watcher's poll loop
# sees it die and respawns.
old = subprocess.Popen([sys.executable, "-c", "pass"])
old.wait(timeout=30)
import hermes_cli.gateway as gateway
assert gateway._spawn_gateway_restart_watcher(
old.pid, [sys.executable, "-c", stub]
), "watcher spawn returned False"
assert _wait_for(marker.exists, timeout_s=60), (
"watcher never respawned the stub gateway"
)
stdio_log = tmp_path / "home" / "logs" / "gateway-stdio.log"
return marker, stdio_log
def test_respawn_stamps_breakaway_and_leaves_stdio_trace(
self, tmp_path, monkeypatch
):
marker, stdio_log = self._run_watcher_cycle(tmp_path, monkeypatch)
# (a) Breakaway stamp: on unfixed main the respawn env carried no
# stamp, so a job-teardown death was undiagnosable.
stamp = marker.read_text(encoding="utf-8").strip()
assert stamp in {"1", "0"}, (
f"respawned gateway must carry the breakaway stamp, got {stamp!r}"
)
# (b) Stdio trace: on unfixed main stderr went to DEVNULL — a dying
# gateway left zero trace (#48820 4th repro).
assert _wait_for(
lambda: stdio_log.exists()
and "stub-gateway-stderr-trace"
in stdio_log.read_text(encoding="utf-8", errors="replace"),
timeout_s=30,
), "respawned gateway stderr must land in logs/gateway-stdio.log"
class TestResumeVerificationLive:
"""The user-visible lie: '✓ Restarting' printed for a dead gateway."""
def test_dead_relaunch_is_not_reported_as_success(self, tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "home"))
(tmp_path / "home").mkdir(parents=True, exist_ok=True)
import hermes_cli.gateway as gateway
import hermes_cli.main as hm
from hermes_cli.update_cmd import _resume_windows_gateways_after_update
# Peripheral only: don't regenerate launcher scripts into the temp home.
monkeypatch.setattr(hm, "_refresh_windows_gateway_launchers", lambda: None)
# Real relaunch chain, real watcher, real spawn — but the respawned
# "gateway" exits immediately, standing in for the Job-Object
# teardown kill. It never registers in the process table as a
# gateway, exactly like the dead pid 48452 / 50456 of #48820.
def _relaunch(profile, old_pid):
dead = subprocess.Popen([sys.executable, "-c", "pass"])
dead.wait(timeout=30)
return gateway._spawn_gateway_restart_watcher(
dead.pid, [sys.executable, "-c", "pass"]
)
monkeypatch.setattr(
gateway, "launch_detached_profile_gateway_restart", _relaunch
)
token = {
"resume_needed": True,
"profiles": {"default": 999999},
"unmapped_pids": [],
"unmapped": [],
}
printed: list = []
real_print = print
monkeypatch.setattr(
"builtins.print", lambda *a, **k: printed.append(" ".join(map(str, a)))
)
try:
with pytest.raises(RuntimeError, match="not verified alive"):
_resume_windows_gateways_after_update(token)
finally:
monkeypatch.setattr("builtins.print", real_print)
text = "\n".join(printed)
assert "✓ Restarting" not in text, (
"the updater must not vouch for a gateway that is not alive "
f"(#48820). Printed:\n{text}"
)
assert "could not be verified" in text