hermes-agent/tests/hermes_cli/test_update_cron_deadlock_guard.py
ethernet 612d542281 Merge remote-tracking branch 'origin/main' into ethie/pm-clean
# Conflicts:
#	.gitignore
#	Dockerfile
#	agent/onboarding.py
#	apps/desktop/electron/main.ts
#	apps/desktop/electron/pool-stop.ts
#	apps/desktop/src/components/model-picker.test.tsx
#	apps/desktop/src/store/updates.ts
#	apps/desktop/vite.config.ts
#	datagen-config-examples/run_browser_tasks.sh
#	docs/rca-ssl-cacert-post-git-pull.md
#	gateway/run.py
#	hermes_cli/backup.py
#	hermes_cli/credential_lifecycle.py
#	hermes_cli/dashboard_procs.py
#	hermes_cli/doctor_state.py
#	hermes_cli/env_loader.py
#	hermes_cli/gateway_windows.py
#	hermes_cli/local_runtime/endpoint.py
#	hermes_cli/psutil_android.py
#	hermes_cli/update_cmd.py
#	hermes_cli/update_cmd_windows.py
#	hermes_cli/web_routers/local_models.py
#	hermes_cli/web_server_config.py
#	hermes_cli/web_server_cron.py
#	plugins/memory/hindsight/__init__.py
#	plugins/memory/holographic/__init__.py
#	plugins/memory/honcho/cli.py
#	plugins/memory/mem0/__init__.py
#	plugins/platforms/google_chat/oauth.py
#	plugins/platforms/photon/adapter.py
#	scripts/ci/list_os_marked_tests.py
#	scripts/run_tests.sh
#	tests/agent/test_compression_stall_fallback.py
#	tests/agent/test_create_openai_client_ssl_verify.py
#	tests/gateway/test_google_chat_oauth_dependencies.py
#	tests/hermes_cli/conftest.py
#	tests/hermes_cli/test_cli_init.py
#	tests/hermes_cli/test_gateway_migrate_multiplex.py
#	tests/hermes_cli/test_psutil_android_extract.py
#	tests/hermes_cli/test_relaunch.py
#	tests/hermes_cli/test_update_check.py
#	tests/hermes_cli/test_update_handoff_desktop_rebuild.py
#	tests/hermes_cli/test_worktree_gc.py
#	tests/scripts/desktop_update/test_desktop_update_windows_python_handoff.py
#	tests/scripts/desktop_update/test_desktop_update_windows_retry_policy.py
#	tests/scripts/desktop_update/test_desktop_update_windows_timestamp.py
#	tests/scripts/install/test_install_autostash_conflict_recovery.py
#	tests/scripts/install/test_install_clone_throttle_fallback.py
#	tests/scripts/install/test_install_commit_pin_rollback.py
#	tests/scripts/install/test_install_diverged_update.py
#	tests/scripts/install/test_install_lockfile_churn.py
#	tests/scripts/install/test_install_macos_launcher.py
#	tests/scripts/install/test_install_no_initial_commit.py
#	tests/scripts/install/test_install_ps1_ascii_only.py
#	tests/scripts/install/test_install_ps1_browser_install.py
#	tests/scripts/install/test_install_ps1_managed_node_swap.py
#	tests/scripts/install/test_install_ps1_native_stderr_eap.py
#	tests/scripts/install/test_install_ps1_node_path_for_npm.py
#	tests/scripts/install/test_install_ps1_python_fallback_venv.py
#	tests/scripts/install/test_install_ps1_resolver_strictmode.py
#	tests/scripts/install/test_install_ps1_uv_install_fallback.py
#	tests/scripts/install/test_install_ps1_uv_powershell_host.py
#	tests/scripts/install/test_install_ps1_venv_process_tree.py
#	tests/scripts/install/test_install_ps1_venv_recreate_safety.py
#	tests/scripts/install/test_install_ps1_venv_rename_abort.py
#	tests/scripts/install/test_install_ps1_venv_transaction_boundary.py
#	tests/scripts/install/test_install_ps1_web_server_syntax_probe.py
#	tests/scripts/install/test_install_scripts_computer_use.py
#	tests/scripts/install/test_install_sh_acp_launcher.py
#	tests/scripts/install/test_install_sh_bootstrap_marker.py
#	tests/scripts/install/test_install_sh_browser_install.py
#	tests/scripts/install/test_install_sh_install_method_stamp.py
#	tests/scripts/install/test_install_sh_node_deps_failure.py
#	tests/scripts/install/test_install_sh_node_deps_workspaces.py
#	tests/scripts/install/test_install_sh_node_global_prefix.py
#	tests/scripts/install/test_install_sh_node_npm_check.py
#	tests/scripts/install/test_install_sh_node_prerelease.py
#	tests/scripts/install/test_install_sh_node_probe.py
#	tests/scripts/install/test_install_sh_node_tarball_without_xz.py
#	tests/scripts/install/test_install_sh_pythonpath_sanitization.py
#	tests/scripts/install/test_install_sh_reuse_supported_python.py
#	tests/scripts/install/test_install_sh_root_fhs_uv_python_path.py
#	tests/scripts/install/test_install_sh_setup_wizard_tty_probe.py
#	tests/scripts/install/test_install_sh_symlink_stomp.py
#	tests/scripts/install/test_install_sh_termux_network_prereqs.py
#	tests/scripts/install/test_install_sh_termux_python_bounds.py
#	tests/scripts/install/test_install_sh_uv_lock_config.py
#	tests/scripts/install/test_install_unmerged_index.py
#	tests/scripts/test_run_tests_parallel.py
#	tests/test_managed_runtime_resolution.py
#	tests/test_project_metadata.py
#	tests/tools/test_browser_use_cli.py
#	tests/tools/test_tts_pythonpath_fallback.py
#	tests/tui_gateway/test_hosted_room_driver_runtime.py
#	tests/tui_gateway/test_tui_gateway_server.py
#	tools/lazy_deps.py
#	tools/voice_mode.py
#	uv.lock
#	website/docs/developer-guide/macos-bundle-updates.md
#	website/docs/developer-guide/pm-audit-status.md
#	website/docs/developer-guide/shared-bundle-builds.md
#	website/docs/developer-guide/source-update-completion.md
#	website/docs/developer-guide/stable-releases.md
2026-09-14 15:38:34 -04:00

167 lines
6.2 KiB
Python

"""Regression tests for #100179: cron-update three-way restart deadlock.
When `hermes update` runs INSIDE the gateway's own process tree (the
hermes-auto-update cron job), waiting for that gateway to exit is a
circular wait:
gateway waits on all in-flight work units (#77184)
-> cron agent session waits on the `hermes update` process
-> `hermes update` waits on the gateway to exit [back to A]
The wedged-loop probe cannot break it: the cron session posts activity
every ~180s (process-tool poll return), so it is never marked wedged and
the gateway burns the full 1800s force-drain cap.
The fix: when the target gateway PID is an ancestor of this process,
fire-and-forget (SIGUSR1 + return) instead of drain-waiting.
"""
from unittest.mock import patch
import pytest
linux_only = pytest.mark.platforms("linux")
class TestAncestorDetectionGuard:
"""_is_pid_ancestor_of_current_process is the deadlock discriminator."""
def test_own_pid_is_ancestor(self):
import os
from hermes_cli.gateway import _is_pid_ancestor_of_current_process
assert _is_pid_ancestor_of_current_process(os.getpid()) is True
def test_parent_pid_is_ancestor(self):
import os
from hermes_cli.gateway import _is_pid_ancestor_of_current_process
ppid = os.getppid()
if ppid <= 1:
pytest.skip("no meaningful parent in this environment")
assert _is_pid_ancestor_of_current_process(ppid) is True
def test_unrelated_pid_is_not_ancestor(self):
from hermes_cli.gateway import _is_pid_ancestor_of_current_process
# PID 0 / negative are never ancestors; a very high unlikely PID isn't
# either. Use the documented zero/negative contract for determinism.
assert _is_pid_ancestor_of_current_process(0) is False
assert _is_pid_ancestor_of_current_process(-5) is False
@linux_only
class TestSelfRestartFireAndForget:
"""_request_gateway_self_restart signals without waiting for exit."""
def test_refuses_non_ancestor_pid(self):
from hermes_cli.gateway import _request_gateway_self_restart
# A non-ancestor must be refused — signalling an unrelated gateway
# and returning immediately would skip its drain entirely.
assert _request_gateway_self_restart(0) is False
def test_signals_ancestor_and_returns_immediately(self):
"""The ancestor path sends SIGUSR1 and does NOT poll for exit."""
import os
import signal as _signal
from hermes_cli import gateway as gw
sent = []
def _fake_kill(pid, sig):
sent.append((pid, sig))
with patch.object(gw.os, "kill", side_effect=_fake_kill), patch.object(
gw, "_wait_for_pid_exit",
side_effect=AssertionError(
"fire-and-forget must NOT wait for the gateway to exit — "
"that wait is the #100179 deadlock"
),
):
ok = gw._request_gateway_self_restart(os.getpid())
assert ok is True
assert sent == [(os.getpid(), _signal.SIGUSR1)]
def test_graceful_restart_does_wait(self):
"""Contrast: the non-ancestor path DOES drain-wait (unchanged)."""
import signal as _signal
from hermes_cli import gateway as gw
waited = []
with patch.object(gw.os, "kill"), patch.object(
gw, "_wait_for_pid_exit",
side_effect=lambda pid, t, **_: waited.append((pid, t)) or True,
):
ok = gw._graceful_restart_via_sigusr1(4242, drain_timeout=7.0)
assert ok is True
assert waited == [(4242, 7.0)], "drain path must still wait for exit"
class TestDrainOrSignalTriage:
"""_drain_or_signal_gateway_for_update routes the three cases correctly."""
def _patched(self, monkeypatch, *, ancestor, wedged):
from hermes_cli import gateway as gw
calls = {"self_restart": [], "escalate": [], "drain": []}
monkeypatch.setattr(
gw, "_is_pid_ancestor_of_current_process", lambda pid: ancestor
)
monkeypatch.setattr(
gw,
"probe_gateway_loop_liveness",
lambda pid: gw.GATEWAY_LOOP_WEDGED if wedged else "alive",
)
monkeypatch.setattr(
gw,
"_request_gateway_self_restart",
lambda pid: calls["self_restart"].append(pid) or True,
)
monkeypatch.setattr(
gw, "_escalate_wedged_gateway", lambda pid: calls["escalate"].append(pid)
)
monkeypatch.setattr(
gw,
"_graceful_restart_via_sigusr1",
lambda pid, drain_timeout, **_: calls["drain"].append((pid, drain_timeout))
or True,
)
return calls
def test_ancestor_gateway_is_signalled_fire_and_forget(self, monkeypatch):
"""In-tree gateway (#100179): SIGUSR1 request, NO drain wait."""
from hermes_cli.update_cmd import _drain_or_signal_gateway_for_update
calls = self._patched(monkeypatch, ancestor=True, wedged=False)
assert _drain_or_signal_gateway_for_update(1234, 900.0, "svc") is True
assert calls["self_restart"] == [1234]
assert calls["drain"] == [], "drain-waiting on an ancestor IS the deadlock"
assert calls["escalate"] == []
def test_wedged_gateway_is_escalated(self, monkeypatch):
"""Provably-dead loop (#81642): bounded escalation, no drain wait."""
from hermes_cli.update_cmd import _drain_or_signal_gateway_for_update
calls = self._patched(monkeypatch, ancestor=False, wedged=True)
assert _drain_or_signal_gateway_for_update(1234, 900.0, "svc") is True
assert calls["escalate"] == [1234]
assert calls["drain"] == []
assert calls["self_restart"] == []
def test_live_out_of_tree_gateway_still_drain_waits(self, monkeypatch):
"""Normal out-of-tree update keeps the full graceful drain semantics."""
from hermes_cli.update_cmd import _drain_or_signal_gateway_for_update
calls = self._patched(monkeypatch, ancestor=False, wedged=False)
assert _drain_or_signal_gateway_for_update(1234, 900.0, "svc") is True
assert calls["drain"] == [(1234, 900.0)]
assert calls["self_restart"] == []
assert calls["escalate"] == []