mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-17 04:11:35 +00:00
The OS lanes are marker-driven: list_os_marked_tests.py picks the files
a lane imports from their platforms() specs and the lane selects with
-m platforms. A test gated with skipif(sys.platform != "win32") is
therefore never imported on the Windows lane and skipped everywhere else
— it runs on no host. skipif(sys.platform == "win32") tests were merely
invisible to the lane bookkeeping, but the rule the tree now follows is
one host marker, never a bare skipif.
Mechanical mapping, semantics preserved: skip-on-Windows → "posix",
skip-off-Windows → "windows", skip-off-Linux → "linux", skip-on-macOS →
"not macos". The former skip reasons stay as trailing comments. A
non-host condition (os.geteuid() == 0) stays a separate skipif beside
the marker, spelled getattr(os, "geteuid", ...) so the decorator still
imports on Windows.
Where the conversion would stack two platforms() marks on one test (the
conftest rejects that at collection) the narrower mark wins:
- test_update_wedged_gateway: the class is already platforms("linux");
its per-test "needs UNIX sockets" marks were redundant and are gone.
- test_process_registry.TestSystemdCgroupIsolation: the class-level
skip-on-Windows moves onto the 11 methods that had no host mark; the
11 platforms("linux") methods keep theirs.
- test_file_ops_single_roundtrip: the two fifo tests drop their
platforms("linux") in favour of the module's "posix" (mkfifo exists on
macOS; both tests already skip when it does not).
- test_linux_desktop_entry / test_gateway_job_teardown_live: duplicate
or wider marks removed.
332 lines
13 KiB
Python
332 lines
13 KiB
Python
"""LIVE Windows E2E for #48820 (4th repro): Job-Object teardown vs the
|
|
gateway restart watcher, with real processes on a real windows-latest runner.
|
|
|
|
Three live proofs (no mocks of the code under test):
|
|
|
|
1. ``TestJobObjectMechanismLive`` — the mechanism everything rests on:
|
|
a child spawned with ``windows_detach_flags()`` (CREATE_BREAKAWAY_FROM_JOB)
|
|
from inside a kill-on-close Job Object SURVIVES the job teardown, while a
|
|
child spawned with ``windows_detach_flags_without_breakaway()`` is killed
|
|
by it. This is exactly the reporter's suspected kill path.
|
|
|
|
2. ``TestWatcherRespawnLive`` — drives the REAL
|
|
``hermes_cli.gateway._spawn_gateway_restart_watcher`` end to end with a
|
|
real stub gateway process, against a temp HERMES_HOME:
|
|
- the respawned process's stderr must land in ``logs/gateway-stdio.log``
|
|
(on unfixed main it went to DEVNULL: a job-teardown kill left ZERO trace);
|
|
- the respawn env must carry ``_HERMES_GATEWAY_BREAKAWAY=1`` (the stamp
|
|
that makes a later job-teardown death diagnosable in exit-diag).
|
|
|
|
3. ``TestResumeVerificationLive`` — the user-visible symptom: the updater's
|
|
``_resume_windows_gateways_after_update`` must NOT print
|
|
"✓ Restarting Windows gateway profile(s)" when the relaunched gateway is
|
|
dead. The relaunch chain runs for real; the "gateway" is a stub that exits
|
|
immediately (standing in for the job-teardown kill). On unfixed main the ✓
|
|
is printed anyway; after the fix the resume raises "not verified alive".
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import ctypes
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from ctypes import wintypes
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
pytestmark = pytest.mark.platforms("windows") # native Windows only
|
|
|
|
_REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
|
|
JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE = 0x00002000
|
|
JOB_OBJECT_LIMIT_BREAKAWAY_OK = 0x00000800
|
|
JobObjectExtendedLimitInformation = 9
|
|
PROCESS_ALL_ACCESS = 0x001FFFFF
|
|
|
|
|
|
class IO_COUNTERS(ctypes.Structure):
|
|
_fields_ = [
|
|
("ReadOperationCount", ctypes.c_ulonglong),
|
|
("WriteOperationCount", ctypes.c_ulonglong),
|
|
("OtherOperationCount", ctypes.c_ulonglong),
|
|
("ReadTransferCount", ctypes.c_ulonglong),
|
|
("WriteTransferCount", ctypes.c_ulonglong),
|
|
("OtherTransferCount", ctypes.c_ulonglong),
|
|
]
|
|
|
|
|
|
class JOBOBJECT_BASIC_LIMIT_INFORMATION(ctypes.Structure):
|
|
_fields_ = [
|
|
("PerProcessUserTimeLimit", ctypes.c_longlong),
|
|
("PerJobUserTimeLimit", ctypes.c_longlong),
|
|
("LimitFlags", wintypes.DWORD),
|
|
("MinimumWorkingSetSize", ctypes.c_size_t),
|
|
("MaximumWorkingSetSize", ctypes.c_size_t),
|
|
("ActiveProcessLimit", wintypes.DWORD),
|
|
("Affinity", ctypes.c_size_t),
|
|
("PriorityClass", wintypes.DWORD),
|
|
("SchedulingClass", wintypes.DWORD),
|
|
]
|
|
|
|
|
|
class JOBOBJECT_EXTENDED_LIMIT_INFORMATION(ctypes.Structure):
|
|
_fields_ = [
|
|
("BasicLimitInformation", JOBOBJECT_BASIC_LIMIT_INFORMATION),
|
|
("IoInfo", IO_COUNTERS),
|
|
("ProcessMemoryLimit", ctypes.c_size_t),
|
|
("JobMemoryLimit", ctypes.c_size_t),
|
|
("PeakProcessMemoryUsed", ctypes.c_size_t),
|
|
("PeakJobMemoryUsed", ctypes.c_size_t),
|
|
]
|
|
|
|
|
|
def _make_kill_on_close_job(allow_breakaway: bool) -> int:
|
|
kernel32 = ctypes.windll.kernel32
|
|
job = kernel32.CreateJobObjectW(None, None)
|
|
assert job, "CreateJobObjectW failed"
|
|
info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION()
|
|
flags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
|
|
if allow_breakaway:
|
|
flags |= JOB_OBJECT_LIMIT_BREAKAWAY_OK
|
|
info.BasicLimitInformation.LimitFlags = flags
|
|
ok = kernel32.SetInformationJobObject(
|
|
job,
|
|
JobObjectExtendedLimitInformation,
|
|
ctypes.byref(info),
|
|
ctypes.sizeof(info),
|
|
)
|
|
assert ok, "SetInformationJobObject failed"
|
|
return job
|
|
|
|
|
|
def _assign_to_job(job: int, proc: subprocess.Popen) -> None:
|
|
kernel32 = ctypes.windll.kernel32
|
|
ok = kernel32.AssignProcessToJobObject(job, int(proc._handle))
|
|
assert ok, f"AssignProcessToJobObject failed (winerror={ctypes.GetLastError()})"
|
|
|
|
|
|
def _pid_alive(pid: int) -> bool:
|
|
kernel32 = ctypes.windll.kernel32
|
|
PROCESS_QUERY_LIMITED_INFORMATION = 0x1000
|
|
h = kernel32.OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, False, pid)
|
|
if not h:
|
|
return False
|
|
try:
|
|
code = wintypes.DWORD()
|
|
kernel32.GetExitCodeProcess(h, ctypes.byref(code))
|
|
return code.value == 259 # STILL_ACTIVE
|
|
finally:
|
|
kernel32.CloseHandle(h)
|
|
|
|
|
|
_SLEEPER = "import time; time.sleep(120)"
|
|
|
|
|
|
def _wait_for(predicate, timeout_s: float = 30.0, interval_s: float = 0.25):
|
|
deadline = time.monotonic() + timeout_s
|
|
while time.monotonic() < deadline:
|
|
if predicate():
|
|
return True
|
|
time.sleep(interval_s)
|
|
return False
|
|
|
|
|
|
class TestJobObjectMechanismLive:
|
|
"""Real Job Objects, real children — the #48820 kill mechanism."""
|
|
|
|
def _driver_source(self, flags_helper: str, pid_file: str) -> str:
|
|
# The driver runs INSIDE the job and spawns a grandchild "gateway"
|
|
# with the flag bundle under test, then exits — mirroring the
|
|
# updater/watcher exiting while its job tears down.
|
|
return (
|
|
"import subprocess, sys, pathlib\n"
|
|
"sys.path.insert(0, r'%s')\n"
|
|
"from hermes_cli._subprocess_compat import (\n"
|
|
" windows_detach_flags, windows_detach_flags_without_breakaway)\n"
|
|
"flags = %s()\n"
|
|
"p = subprocess.Popen([sys.executable, '-c', %r],\n"
|
|
" creationflags=flags, stdin=subprocess.DEVNULL,\n"
|
|
" stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n"
|
|
"pathlib.Path(r'%s').write_text(str(p.pid), encoding='utf-8')\n"
|
|
) % (str(_REPO_ROOT), flags_helper, _SLEEPER, pid_file)
|
|
|
|
def _run_in_job(self, tmp_path: Path, flags_helper: str) -> int:
|
|
pid_file = tmp_path / f"{flags_helper}.pid"
|
|
job = _make_kill_on_close_job(allow_breakaway=True)
|
|
kernel32 = ctypes.windll.kernel32
|
|
try:
|
|
driver = subprocess.Popen(
|
|
[
|
|
sys.executable,
|
|
"-c",
|
|
# Handshake: wait until the test has assigned us to the
|
|
# job before spawning the grandchild.
|
|
"import pathlib, sys, time\n"
|
|
f"go = pathlib.Path(r'{tmp_path / 'go.marker'}')\n"
|
|
"deadline = time.monotonic() + 30\n"
|
|
"while not go.exists():\n"
|
|
" assert time.monotonic() < deadline, 'no go marker'\n"
|
|
" time.sleep(0.1)\n"
|
|
+ self._driver_source(flags_helper, str(pid_file)),
|
|
],
|
|
cwd=str(_REPO_ROOT),
|
|
)
|
|
_assign_to_job(job, driver)
|
|
(tmp_path / "go.marker").write_text("go", encoding="utf-8")
|
|
assert _wait_for(pid_file.exists), "driver never wrote the pid file"
|
|
gw_pid = int(pid_file.read_text(encoding="utf-8"))
|
|
assert _wait_for(lambda: driver.poll() is not None), (
|
|
"driver did not exit"
|
|
)
|
|
assert _pid_alive(gw_pid), "grandchild died before job teardown"
|
|
# THE teardown: closing the last job handle fires
|
|
# KILL_ON_JOB_CLOSE against every process still in the job.
|
|
kernel32.CloseHandle(job)
|
|
job = None
|
|
time.sleep(2.0)
|
|
return gw_pid
|
|
finally:
|
|
(tmp_path / "go.marker").unlink(missing_ok=True)
|
|
if job:
|
|
kernel32.CloseHandle(job)
|
|
|
|
def test_breakaway_child_survives_job_teardown(self, tmp_path):
|
|
pid = self._run_in_job(tmp_path, "windows_detach_flags")
|
|
try:
|
|
assert _pid_alive(pid), (
|
|
"CREATE_BREAKAWAY_FROM_JOB child must survive the parent "
|
|
"job's kill-on-close teardown"
|
|
)
|
|
finally:
|
|
subprocess.run(
|
|
["taskkill", "/PID", str(pid), "/T", "/F"], capture_output=True
|
|
)
|
|
|
|
def test_non_breakaway_child_killed_by_job_teardown(self, tmp_path):
|
|
"""The #48820 kill path, reproduced live: no breakaway → the job's
|
|
teardown reaps the freshly spawned gateway."""
|
|
pid = self._run_in_job(tmp_path, "windows_detach_flags_without_breakaway")
|
|
try:
|
|
assert not _pid_alive(pid), (
|
|
"child without breakaway must be killed by kill-on-close "
|
|
"job teardown — this is the silent gateway death of #48820"
|
|
)
|
|
finally:
|
|
subprocess.run(
|
|
["taskkill", "/PID", str(pid), "/T", "/F"], capture_output=True
|
|
)
|
|
|
|
|
|
class TestWatcherRespawnLive:
|
|
"""Drive the real ``_spawn_gateway_restart_watcher`` with real processes."""
|
|
|
|
def _run_watcher_cycle(self, tmp_path: Path, monkeypatch) -> tuple[Path, Path]:
|
|
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "home"))
|
|
(tmp_path / "home").mkdir(parents=True, exist_ok=True)
|
|
|
|
marker = tmp_path / "respawned.marker"
|
|
# The stub "gateway": records its breakaway stamp env, screams on
|
|
# stderr (so the stdio sidecar has something to capture), then exits.
|
|
stub = (
|
|
"import os, pathlib, sys\n"
|
|
f"pathlib.Path(r'{marker}').write_text(\n"
|
|
" os.environ.get('_HERMES_GATEWAY_BREAKAWAY', 'MISSING'),\n"
|
|
" encoding='utf-8')\n"
|
|
"print('stub-gateway-stderr-trace', file=sys.stderr)\n"
|
|
)
|
|
|
|
# A real old-pid that exits immediately — the watcher's poll loop
|
|
# sees it die and respawns.
|
|
old = subprocess.Popen([sys.executable, "-c", "pass"])
|
|
old.wait(timeout=30)
|
|
|
|
import hermes_cli.gateway as gateway
|
|
|
|
assert gateway._spawn_gateway_restart_watcher(
|
|
old.pid, [sys.executable, "-c", stub]
|
|
), "watcher spawn returned False"
|
|
|
|
assert _wait_for(marker.exists, timeout_s=60), (
|
|
"watcher never respawned the stub gateway"
|
|
)
|
|
stdio_log = tmp_path / "home" / "logs" / "gateway-stdio.log"
|
|
return marker, stdio_log
|
|
|
|
def test_respawn_stamps_breakaway_and_leaves_stdio_trace(
|
|
self, tmp_path, monkeypatch
|
|
):
|
|
marker, stdio_log = self._run_watcher_cycle(tmp_path, monkeypatch)
|
|
|
|
# (a) Breakaway stamp: on unfixed main the respawn env carried no
|
|
# stamp, so a job-teardown death was undiagnosable.
|
|
stamp = marker.read_text(encoding="utf-8").strip()
|
|
assert stamp in {"1", "0"}, (
|
|
f"respawned gateway must carry the breakaway stamp, got {stamp!r}"
|
|
)
|
|
|
|
# (b) Stdio trace: on unfixed main stderr went to DEVNULL — a dying
|
|
# gateway left zero trace (#48820 4th repro).
|
|
assert _wait_for(
|
|
lambda: stdio_log.exists()
|
|
and "stub-gateway-stderr-trace"
|
|
in stdio_log.read_text(encoding="utf-8", errors="replace"),
|
|
timeout_s=30,
|
|
), "respawned gateway stderr must land in logs/gateway-stdio.log"
|
|
|
|
|
|
class TestResumeVerificationLive:
|
|
"""The user-visible lie: '✓ Restarting' printed for a dead gateway."""
|
|
|
|
def test_dead_relaunch_is_not_reported_as_success(self, tmp_path, monkeypatch):
|
|
monkeypatch.setenv("HERMES_HOME", str(tmp_path / "home"))
|
|
(tmp_path / "home").mkdir(parents=True, exist_ok=True)
|
|
|
|
import hermes_cli.gateway as gateway
|
|
import hermes_cli.main as hm
|
|
from hermes_cli.update_cmd import _resume_windows_gateways_after_update
|
|
|
|
# Peripheral only: don't regenerate launcher scripts into the temp home.
|
|
monkeypatch.setattr(hm, "_refresh_windows_gateway_launchers", lambda: None)
|
|
|
|
# Real relaunch chain, real watcher, real spawn — but the respawned
|
|
# "gateway" exits immediately, standing in for the Job-Object
|
|
# teardown kill. It never registers in the process table as a
|
|
# gateway, exactly like the dead pid 48452 / 50456 of #48820.
|
|
def _relaunch(profile, old_pid):
|
|
dead = subprocess.Popen([sys.executable, "-c", "pass"])
|
|
dead.wait(timeout=30)
|
|
return gateway._spawn_gateway_restart_watcher(
|
|
dead.pid, [sys.executable, "-c", "pass"]
|
|
)
|
|
|
|
monkeypatch.setattr(
|
|
gateway, "launch_detached_profile_gateway_restart", _relaunch
|
|
)
|
|
|
|
token = {
|
|
"resume_needed": True,
|
|
"profiles": {"default": 999999},
|
|
"unmapped_pids": [],
|
|
"unmapped": [],
|
|
}
|
|
|
|
printed: list = []
|
|
real_print = print
|
|
monkeypatch.setattr(
|
|
"builtins.print", lambda *a, **k: printed.append(" ".join(map(str, a)))
|
|
)
|
|
try:
|
|
with pytest.raises(RuntimeError, match="not verified alive"):
|
|
_resume_windows_gateways_after_update(token)
|
|
finally:
|
|
monkeypatch.setattr("builtins.print", real_print)
|
|
|
|
text = "\n".join(printed)
|
|
assert "✓ Restarting" not in text, (
|
|
"the updater must not vouch for a gateway that is not alive "
|
|
f"(#48820). Printed:\n{text}"
|
|
)
|
|
assert "could not be verified" in text
|