hermes-agent/tests/hermes_cli/test_update_wedged_gateway.py
ethernet 890bbbda1f Merge remote-tracking branch 'origin/main' into ethie/pm-clean
# Conflicts:
#	apps/desktop/e2e/archived-hidden-session-recoverable.spec.ts
#	apps/desktop/e2e/bot-chat-message-agent-friendly-name.spec.ts
#	apps/desktop/e2e/bot-mailbox-unreadable-ticket.spec.ts
#	apps/desktop/e2e/bot-mode-roster-localized.spec.ts
#	apps/desktop/e2e/bot-mode-row-click-mirrors-registry.spec.ts
#	apps/desktop/e2e/bot-mode-tab-shows-bot-name.spec.ts
#	apps/desktop/e2e/bot-roster-group-row-organisation.spec.ts
#	apps/desktop/e2e/bot-roster-ignores-infra-dirs.spec.ts
#	apps/desktop/e2e/bot-roster-timestamp-meta.spec.ts
#	apps/desktop/e2e/bot-roster-user-sections.spec.ts
#	apps/desktop/e2e/bot-routines-pane-narrow.spec.ts
#	apps/desktop/e2e/bot-row-open-recent-session.spec.ts
#	apps/desktop/e2e/bot-tile-ignores-ambient-composer-model.spec.ts
#	apps/desktop/e2e/group-composer-auto-grow.spec.ts
#	apps/desktop/e2e/group-create-gate-remote-roster.spec.ts
#	apps/desktop/e2e/group-prompt-renamed-primary-handle.spec.ts
#	apps/desktop/e2e/hosted-room-backend-continuity.spec.ts
#	apps/desktop/e2e/hosted-room-legacy-store-migration.spec.ts
#	apps/desktop/e2e/settings-scope-chips-bot-title.spec.ts
#	apps/desktop/e2e/worktree-branch-status.spec.ts
#	apps/desktop/electron/backend-probes.test.ts
#	apps/desktop/electron/connection-apply.test.ts
#	apps/desktop/electron/desktop-electron-pin.test.ts
#	apps/desktop/electron/desktop-uninstall.test.ts
#	apps/desktop/electron/gateway-file-download-transport.test.ts
#	apps/desktop/electron/gateway-stop-before-update.test.ts
#	apps/desktop/electron/github-api-auth.test.ts
#	apps/desktop/electron/registry-primary-profile-scope.test.ts
#	apps/desktop/electron/update-api-check.test.ts
#	apps/desktop/electron/update-handoff-marker.test.ts
#	apps/desktop/electron/venv-blocker-scan.test.ts
#	apps/desktop/scripts/after-extract.test.mjs
#	apps/desktop/scripts/local-pack-publish.test.mjs
#	apps/desktop/scripts/tasks-scroll.test.mjs
#	apps/desktop/src/app/settings/model-settings.test.tsx
#	apps/desktop/src/app/updates-overlay.blockers.test.tsx
#	apps/desktop/src/components/desktop-install-overlay.test.tsx
#	apps/desktop/src/lib/update-copy.test.ts
#	scripts/ci/check_os_marker_fakes.py
#	tests-js/desktop-mac-usage-descriptions.test.ts
#	tests-js/node-engine-alignment.test.ts
#	tests/agent/lsp/test_install_and_lint_fixes.py
#	tests/agent/test_command_token_source.py
#	tests/agent/test_compression_boundary_hook.py
#	tests/agent/test_create_openai_client_ssl_verify.py
#	tests/agent/test_custom_provider_ca_probes.py
#	tests/agent/test_endpoint_blackhole.py
#	tests/agent/test_estimator_parity.py
#	tests/agent/test_in_place_compaction.py
#	tests/agent/test_moa_loop_mode.py
#	tests/agent/test_model_metadata.py
#	tests/agent/test_skill_session_platform_gate.py
#	tests/agent/test_skill_utils.py
#	tests/agent/test_ssl_ca_guard.py
#	tests/computer_use/test_doctor.py
#	tests/cron/test_codex_execution_paths.py
#	tests/cron/test_cron_bot_chat_delivery.py
#	tests/cron/test_cron_script.py
#	tests/cron/test_media_delivery_parity.py
#	tests/cron/test_misfire_catchup.py
#	tests/cron/test_parallel_pool.py
#	tests/cron/test_recurring_eagain_redispatch.py
#	tests/gateway/test_choice_picker.py
#	tests/gateway/test_control_socket_windows_live.py
#	tests/gateway/test_dingtalk.py
#	tests/gateway/test_feishu.py
#	tests/gateway/test_feishu_onboard.py
#	tests/gateway/test_gateway_shutdown.py
#	tests/gateway/test_matrix.py
#	tests/gateway/test_model_command_custom_providers.py
#	tests/gateway/test_reasoning_command.py
#	tests/gateway/test_runtime_footer.py
#	tests/gateway/test_session.py
#	tests/gateway/test_session_hygiene.py
#	tests/gateway/test_status.py
#	tests/gateway/test_teams.py
#	tests/gateway/test_turn_lease.py
#	tests/gateway/test_whatsapp_connect.py
#	tests/hermes_cli/test_approvals_command.py
#	tests/hermes_cli/test_auth_store_lock_concurrent.py
#	tests/hermes_cli/test_backup.py
#	tests/hermes_cli/test_banner_git_state.py
#	tests/hermes_cli/test_certifi_repair.py
#	tests/hermes_cli/test_cmd_update.py
#	tests/hermes_cli/test_compat_manifest_targets.py
#	tests/hermes_cli/test_computer_use_cli.py
#	tests/hermes_cli/test_cpr_local_leak.py
#	tests/hermes_cli/test_dashboard_auth_gate.py
#	tests/hermes_cli/test_dashboard_procs_kill_grace.py
#	tests/hermes_cli/test_desktop_lifecycle_windows_live.py
#	tests/hermes_cli/test_doctor.py
#	tests/hermes_cli/test_doctor_command_install.py
#	tests/hermes_cli/test_fleet_config_migration_windows_live.py
#	tests/hermes_cli/test_gateway.py
#	tests/hermes_cli/test_gateway_platform_gating.py
#	tests/hermes_cli/test_gateway_restart_loop.py
#	tests/hermes_cli/test_gateway_task_probe.py
#	tests/hermes_cli/test_gateway_wsl.py
#	tests/hermes_cli/test_gui_command.py
#	tests/hermes_cli/test_install_cua_driver.py
#	tests/hermes_cli/test_kanban_db.py
#	tests/hermes_cli/test_lazy_command_exports.py
#	tests/hermes_cli/test_lazy_refresh_venv_repair.py
#	tests/hermes_cli/test_linux_desktop_entry.py
#	tests/hermes_cli/test_local_runtime.py
#	tests/hermes_cli/test_local_runtime_updates.py
#	tests/hermes_cli/test_managed_uv.py
#	tests/hermes_cli/test_mcp_reload_confirm_gate.py
#	tests/hermes_cli/test_nous_subscription.py
#	tests/hermes_cli/test_npm_engine.py
#	tests/hermes_cli/test_personality_none.py
#	tests/hermes_cli/test_pet_toggle.py
#	tests/hermes_cli/test_plan_reconciliation_windows_live.py
#	tests/hermes_cli/test_plugin_event_bus.py
#	tests/hermes_cli/test_plugin_manifest_v2.py
#	tests/hermes_cli/test_plugin_packs.py
#	tests/hermes_cli/test_plugins_cmd.py
#	tests/hermes_cli/test_plugins_cmd_enable_disable_nested.py
#	tests/hermes_cli/test_process_identity.py
#	tests/hermes_cli/test_profiles.py
#	tests/hermes_cli/test_profiles_sidebar_cache.py
#	tests/hermes_cli/test_pty_bridge.py
#	tests/hermes_cli/test_resolve_turn_limit.py
#	tests/hermes_cli/test_serve_runtime_inventory.py
#	tests/hermes_cli/test_session_vacuum_config.py
#	tests/hermes_cli/test_set_config_value.py
#	tests/hermes_cli/test_signal_handler_kanban_worker.py
#	tests/hermes_cli/test_slash_confirm_windows.py
#	tests/hermes_cli/test_stale_pid_guard.py
#	tests/hermes_cli/test_startup_fast_guards.py
#	tests/hermes_cli/test_status.py
#	tests/hermes_cli/test_telegram_managed_bot.py
#	tests/hermes_cli/test_tools_config.py
#	tests/hermes_cli/test_update_apply_shallow_count.py
#	tests/hermes_cli/test_update_autostash.py
#	tests/hermes_cli/test_update_concurrent_quarantine.py
#	tests/hermes_cli/test_update_fetch_failure_classifier.py
#	tests/hermes_cli/test_update_fleet_probe_resume_token.py
#	tests/hermes_cli/test_update_handoff_backend_reap.py
#	tests/hermes_cli/test_update_handoff_desktop_rebuild.py
#	tests/hermes_cli/test_update_head_moved_gate.py
#	tests/hermes_cli/test_update_host_obligation.py
#	tests/hermes_cli/test_update_import_guard.py
#	tests/hermes_cli/test_update_interrupted_recovery.py
#	tests/hermes_cli/test_update_inventory.py
#	tests/hermes_cli/test_update_launchd_unloaded_gateway.py
#	tests/hermes_cli/test_update_missing_configured_deps.py
#	tests/hermes_cli/test_update_modified_notice.py
#	tests/hermes_cli/test_update_multiplex_migration_hook.py
#	tests/hermes_cli/test_update_no_gateway_restart.py
#	tests/hermes_cli/test_update_orphan_backend_reap.py
#	tests/hermes_cli/test_update_parked_branch_guard.py
#	tests/hermes_cli/test_update_post_pull_syntax_guard.py
#	tests/hermes_cli/test_update_receipt.py
#	tests/hermes_cli/test_update_self_lock.py
#	tests/hermes_cli/test_update_shim_fail_closed.py
#	tests/hermes_cli/test_update_shim_self_lock.py
#	tests/hermes_cli/test_update_sqlite_remediation.py
#	tests/hermes_cli/test_update_stale_dashboard.py
#	tests/hermes_cli/test_update_stale_virtualenv.py
#	tests/hermes_cli/test_update_venv_health.py
#	tests/hermes_cli/test_update_venv_ownership_preflight.py
#	tests/hermes_cli/test_update_wedged_gateway.py
#	tests/hermes_cli/test_update_yes_flag.py
#	tests/hermes_cli/test_update_zip_two_phase.py
#	tests/hermes_cli/test_urllib_security.py
#	tests/hermes_cli/test_ux_messages_auth_config.py
#	tests/hermes_cli/test_ux_messages_startup.py
#	tests/hermes_cli/test_venv_holder_classifier.py
#	tests/hermes_cli/test_verify_console_scripts.py
#	tests/hermes_cli/test_verify_core_dependencies.py
#	tests/hermes_cli/test_web_server.py
#	tests/hermes_cli/test_web_server_console_ws.py
#	tests/hermes_cli/test_web_server_ws_ping.py
#	tests/hermes_cli/test_web_ui_build.py
#	tests/hermes_state/test_fts_rebuild_admission.py
#	tests/hermes_state/test_hermes_state.py
#	tests/plugins/memory/test_memory_lazy_install.py
#	tests/plugins/test_google_meet_plugin.py
#	tests/plugins/test_langfuse_plugin.py
#	tests/plugins/test_security_guidance_plugin.py
#	tests/plugins/test_transform_llm_output_hook.py
#	tests/scripts/desktop_update/test_desktop_update_windows_gateway_flag.py
#	tests/scripts/desktop_update/test_desktop_update_windows_python_handoff.py
#	tests/scripts/desktop_update/test_desktop_update_windows_timestamp.py
#	tests/scripts/install/test_install_clone_throttle_fallback.py
#	tests/scripts/install/test_install_lockfile_churn.py
#	tests/scripts/install/test_install_no_initial_commit.py
#	tests/scripts/install/test_install_sh_browser_install.py
#	tests/scripts/install/test_install_sh_node_prerelease.py
#	tests/scripts/install/test_install_sh_symlink_stomp.py
#	tests/scripts/install/test_install_sh_uv_lock_config.py
#	tests/scripts/install/test_install_unmerged_index.py
#	tests/scripts/test_contributor_map.py
#	tests/scripts/test_run_tests_parallel.py
#	tests/skills/test_competitor_news_monitor_skill.py
#	tests/skills/test_document_to_action_items_skill.py
#	tests/skills/test_google_workspace_setup.py
#	tests/skills/test_google_workspace_setup_deps.py
#	tests/skills/test_grounded_citations_skill.py
#	tests/skills/test_ip_as_logo_skill.py
#	tests/skills/test_live_dashboard_skill.py
#	tests/skills/test_mcp_oauth_remote_gateway_skill.py
#	tests/skills/test_office_document_skills.py
#	tests/skills/test_openclaw_migration.py
#	tests/skills/test_product_price_monitor_skill.py
#	tests/skills/test_scrollcraft_skill.py
#	tests/skills/test_setup_wizard_generator_skill.py
#	tests/skills/test_weekly_review_planning_skill.py
#	tests/test_engines_satisfiable.py
#	tests/test_fast_safe_load.py
#	tests/test_hermes_bootstrap.py
#	tests/test_hermes_constants.py
#	tests/test_hermes_logging.py
#	tests/test_managed_runtime_resolution.py
#	tests/test_model_tools_async_bridge.py
#	tests/test_packaging_build_guard.py
#	tests/test_packaging_metadata.py
#	tests/test_yaml_indent_consistency.py
#	tests/tools/test_approval_timeout_overflow.py
#	tests/tools/test_base_environment.py
#	tests/tools/test_bot_mode_dm.py
#	tests/tools/test_browser_chromium_check.py
#	tests/tools/test_browser_hardening.py
#	tests/tools/test_browser_homebrew_paths.py
#	tests/tools/test_browser_npx_warmup.py
#	tests/tools/test_browser_orphan_reaper.py
#	tests/tools/test_browser_real_profile.py
#	tests/tools/test_browser_use_cli.py
#	tests/tools/test_clipboard.py
#	tests/tools/test_code_execution.py
#	tests/tools/test_code_execution_modes.py
#	tests/tools/test_code_execution_windows_env.py
#	tests/tools/test_computer_use.py
#	tests/tools/test_delegate_liveness_timeout.py
#	tests/tools/test_execute_code_approval_cluster.py
#	tests/tools/test_execution_flag_detection.py
#	tests/tools/test_fal_common.py
#	tests/tools/test_file_operations.py
#	tests/tools/test_file_tools.py
#	tests/tools/test_file_tools_cwd_resolution.py
#	tests/tools/test_file_tools_live.py
#	tests/tools/test_lazy_deps.py
#	tests/tools/test_lazy_deps_durable_target.py
#	tests/tools/test_lazy_deps_managed.py
#	tests/tools/test_local_env_blocklist.py
#	tests/tools/test_local_tempdir.py
#	tests/tools/test_macos_protected_search.py
#	tests/tools/test_mcp_npx_cached_bin.py
#	tests/tools/test_oneshot_completion_linger.py
#	tests/tools/test_process_registry.py
#	tests/tools/test_read_file_schema_gating.py
#	tests/tools/test_skill_improvements.py
#	tests/tools/test_skills_sync.py
#	tests/tools/test_termux_api_detection.py
#	tests/tools/test_tirith_security.py
#	tests/tools/test_transcription_tools.py
#	tests/tools/test_tts_streaming.py
#	tests/tools/test_wake_word.py
#	tests/tui_gateway/test_compute_host_borrowed_lease.py
#	tests/tui_gateway/test_compute_host_turn_protocol.py
#	tests/tui_gateway/test_isolated_orphan_activity.py
#	tests/tui_gateway/test_protocol.py
#	tests/tui_gateway/test_slash_worker_profile_home.py
#	tests/tui_gateway/test_subprocess_encoding.py
#	tests/tui_gateway/test_tui_gateway_server.py
#	ui-tui/src/__tests__/terminalParity.test.ts
#	ui-tui/src/__tests__/termuxComposerLayout.test.ts
#	ui-tui/src/__tests__/textInputFastEcho.test.ts
2026-09-23 07:02:44 -04:00

895 lines
35 KiB
Python

"""Tests for the wedged-gateway health probe + bounded escalation (#81642).
A gateway whose event loop is stalled cannot process a graceful shutdown, so
the updater's drain wait used to burn the full 180s budget ("Gateway PID X
still running after 180.0s — restart may fail") and could deadlock `hermes
update`. The fix probes the loop-liveness heartbeat file BEFORE draining and,
only when the loop is provably dead, escalates SIGTERM → SIGKILL bounded to
seconds. A busy-but-alive gateway (fresh heartbeat) must keep the full drain
path — including the in-flight cron drain floor from #86684.
"""
import asyncio
import json
import os
import shutil
import socket
import sys
import tempfile
import threading
import time
from pathlib import Path
from unittest.mock import patch
import pytest
import hermes_cli.gateway as gateway_cli
from gateway.shutdown_watchdog import (
get_loop_heartbeat_path,
get_loop_tick_socket_path,
loop_heartbeat_forever,
write_loop_heartbeat,
)
# Native Windows exposes neither ``socket.AF_UNIX`` nor an asyncio UNIX
# server, so the witness cases that create real socket nodes
# (``_silent_socket_node``) or run the real producer
# (``loop_heartbeat_forever``) cannot execute there. Only those cases are
# skipped: the witness-absent contracts (mocked probes, file-only
# heartbeats) are platform-independent and keep running on Windows, per
# the Windows behavior pinned alongside the product-side guarantee.
_NEEDS_UNIX_SOCKETS = pytest.mark.skipif(
sys.platform == "win32",
reason="requires real UNIX-domain sockets "
"(socket.AF_UNIX / asyncio.start_unix_server), unavailable on native Windows",
)
@pytest.fixture()
def tmp_path():
"""Short-path override for this module (macOS AF_UNIX ~104-byte limit).
The loop-tick witness tests bind real UNIX sockets under HERMES_HOME.
pytest's default tmp_path nests deep enough on macOS that
``state/gateway.loop-tick.<pid>.sock`` exceeds the sockaddr_un limit and
``bind()`` raises ``OSError: AF_UNIX path too long``. A mkdtemp directly
under the system temp root keeps the socket path well under the limit
on every platform.
"""
path = Path(tempfile.mkdtemp(prefix="hwg-"))
try:
yield path
finally:
shutil.rmtree(path, ignore_errors=True)
def _write_heartbeat(home, pid, age_s=0.0):
"""Write a heartbeat file for ``pid`` whose mtime is ``age_s`` old."""
path = get_loop_heartbeat_path(home)
write_loop_heartbeat(pid=pid, home=home)
if age_s:
stamp = time.time() - age_s
os.utime(path, (stamp, stamp))
return path
def _mark_witness_flag(home, armed, age_s=0.0):
"""Set ``loop_tick_socket`` on the heartbeat payload; re-stamp mtime."""
path = get_loop_heartbeat_path(home)
payload = json.loads(path.read_text(encoding="utf-8"))
payload["loop_tick_socket"] = armed
path.write_text(json.dumps(payload), encoding="utf-8")
if age_s:
stamp = time.time() - age_s
os.utime(path, (stamp, stamp))
return path
def _silent_socket_node(path):
"""Create a socket node at ``path`` that never answers.
Bind + listen, then close the listener WITHOUT unlinking: the node stays,
so a probe's connect() gets ECONNREFUSED — a witness that exists but is
silent, exactly like a dead listener (or a loop that stopped scheduling).
"""
path.parent.mkdir(parents=True, exist_ok=True)
srv = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
try:
srv.bind(str(path))
srv.listen(1)
finally:
srv.close()
def _start_freezeable_producer(tmp_path, block_s, errors, write_stall_s=1.5):
"""Run the real heartbeat producer on a loop that can be frozen on demand.
Returns ``(state, ready)``:
- ``state["trigger"]`` — freezes the gateway loop synchronously for
``block_s`` seconds. While frozen, NO task runs on the loop — not the
heartbeat writer, not the tick-socket handler — which is exactly the
"loop not scheduling" condition the probe must detect (sustained).
- ``state["thread"]`` — the producer thread; join it in ``finally``.
- ``ready`` — set once the loop is running (socket may still be arming).
The heartbeat write is patched to stall ``write_stall_s`` after the first
write, so the file goes stale while the loop keeps dispatching.
"""
freeze_evt = asyncio.Event()
stop_evt = asyncio.Event()
state: dict = {"loop": None, "trigger": None, "thread": None, "stop": lambda: None}
ready = threading.Event()
def stalling_write(**_kwargs):
if not get_loop_heartbeat_path(tmp_path).exists():
return write_loop_heartbeat(**_kwargs)
time.sleep(write_stall_s)
return write_loop_heartbeat(**_kwargs)
async def freeze_gate() -> None:
while True:
await freeze_evt.wait()
freeze_evt.clear()
# Synchronous sleep: freezes the entire loop for block_s.
time.sleep(block_s)
async def producer() -> None:
loop = asyncio.get_running_loop()
state["loop"] = loop
state["trigger"] = lambda: loop.call_soon_threadsafe(freeze_evt.set)
state["stop"] = lambda: loop.call_soon_threadsafe(stop_evt.set)
with patch("gateway.shutdown_watchdog.write_loop_heartbeat", stalling_write):
task = asyncio.create_task(
loop_heartbeat_forever(interval_s=1.0, home=tmp_path)
)
gate = asyncio.create_task(freeze_gate())
try:
ready.set()
await stop_evt.wait()
finally:
task.cancel()
gate.cancel()
try:
await task
except asyncio.CancelledError:
pass
try:
await gate
except asyncio.CancelledError:
pass
def run_producer() -> None:
try:
asyncio.run(producer())
except Exception as exc: # surfaced via errors after join
errors.append(exc)
thread = threading.Thread(target=run_producer, daemon=True)
thread.start()
state["thread"] = thread
return state, ready
def _wait_heartbeat_stale(tmp_path, stale_after, timeout_s=5.0):
"""Block until the heartbeat file is older than ``stale_after``."""
hb_path = get_loop_heartbeat_path(tmp_path)
deadline = time.monotonic() + timeout_s
while True:
try:
age = time.time() - hb_path.stat().st_mtime
except FileNotFoundError:
# The socket is armed before the first write lands; the file
# appears a tick later.
age = 0.0
if age > stale_after:
return
assert time.monotonic() < deadline, "heartbeat never went stale"
time.sleep(0.02)
def _launchd_harness(monkeypatch, tmp_path, pid):
"""Patch the launchd_restart path so the REAL probe drives it.
Returns the ``events`` list: "probe" is recorded by the probe wrapper
below, "escalate" by ``_escalate_wedged_gateway``, ``("drain", t)`` by
the graceful drain wait, "kickstart" by the relaunch. The probe itself
is the real ``probe_gateway_loop_liveness`` (home resolved through the
patched ``_process_hermes_home``).
"""
events = []
monkeypatch.setattr(gateway_cli, "get_launchd_label", lambda: "ai.hermes.gateway")
monkeypatch.setattr(gateway_cli, "_launchd_domain", lambda: "gui/501")
monkeypatch.setattr(gateway_cli, "_get_restart_drain_timeout", lambda: 180.0)
monkeypatch.setattr("gateway.status.get_running_pid", lambda *a, **k: pid)
monkeypatch.setattr(
gateway_cli, "_request_gateway_self_restart", lambda pid: False
)
real_probe = gateway_cli.probe_gateway_loop_liveness
def recording_probe(pid, **kw):
events.append("probe")
return real_probe(pid, **kw)
monkeypatch.setattr(gateway_cli, "probe_gateway_loop_liveness", recording_probe)
monkeypatch.setattr(
gateway_cli,
"_escalate_wedged_gateway",
lambda pid, **kw: events.append("escalate") or True,
)
monkeypatch.setattr(
gateway_cli,
"terminate_pid",
lambda pid, force=False, **kwargs: events.append(("kill" if force else "term", pid)),
)
# Never let a real SIGUSR1 escape to the live test PID — the drain path
# goes through _graceful_restart_via_sigusr1 (in-place restart) before
# any exit-wait, and these tests feed launchd_restart os.getpid().
monkeypatch.setattr(
gateway_cli,
"_graceful_restart_via_sigusr1",
lambda pid, timeout, **_: events.append(("drain", pid, timeout)) or True,
)
monkeypatch.setattr(
gateway_cli,
"_wait_for_gateway_exit",
lambda timeout, force_after=None: events.append(("drain", timeout)) or True,
)
monkeypatch.setattr(
gateway_cli,
"_wait_for_launchd_service_pid",
lambda label, old_pid, timeout=10.0, *, domain: events.append("observe")
or True,
)
monkeypatch.setattr(
gateway_cli.subprocess,
"run",
lambda *a, **k: events.append("kickstart")
or __import__("types").SimpleNamespace(returncode=0, stdout="", stderr=""),
)
monkeypatch.setattr(gateway_cli, "_clear_launchd_unsupported_marker", lambda: None)
monkeypatch.setattr(
"gateway.shutdown_watchdog._process_hermes_home", lambda: tmp_path
)
return events
class TestProbeGatewayLoopLiveness:
def test_fresh_heartbeat_is_alive(self, tmp_path):
"""A gateway that refreshed its heartbeat recently is busy, not wedged."""
_write_heartbeat(tmp_path, pid=4242, age_s=5.0)
assert (
gateway_cli.probe_gateway_loop_liveness(4242, home=tmp_path)
== gateway_cli.GATEWAY_LOOP_ALIVE
)
def test_stale_heartbeat_is_wedged(self, tmp_path):
"""A heartbeat several missed beats old proves the loop is dead."""
_write_heartbeat(tmp_path, pid=4242, age_s=600.0)
assert (
gateway_cli.probe_gateway_loop_liveness(4242, home=tmp_path)
== gateway_cli.GATEWAY_LOOP_WEDGED
)
def test_heartbeat_just_inside_budget_is_alive(self, tmp_path):
"""Boundary: age below the stale budget must NOT classify as wedged."""
_write_heartbeat(tmp_path, pid=4242, age_s=60.0)
assert (
gateway_cli.probe_gateway_loop_liveness(
4242, stale_after=90.0, home=tmp_path
)
== gateway_cli.GATEWAY_LOOP_ALIVE
)
def test_missing_heartbeat_is_unknown(self, tmp_path):
"""No heartbeat file (older gateway, fresh start) is not evidence."""
assert (
gateway_cli.probe_gateway_loop_liveness(4242, home=tmp_path)
== gateway_cli.GATEWAY_LOOP_UNKNOWN
)
def test_pid_mismatch_is_unknown_even_when_stale(self, tmp_path):
"""A stale file from a PREVIOUS process must not condemn the new PID."""
_write_heartbeat(tmp_path, pid=1111, age_s=600.0)
assert (
gateway_cli.probe_gateway_loop_liveness(4242, home=tmp_path)
== gateway_cli.GATEWAY_LOOP_UNKNOWN
)
def test_corrupt_heartbeat_is_unknown(self, tmp_path):
path = get_loop_heartbeat_path(tmp_path)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("{not json", encoding="utf-8")
stamp = time.time() - 600.0
os.utime(path, (stamp, stamp))
assert (
gateway_cli.probe_gateway_loop_liveness(4242, home=tmp_path)
== gateway_cli.GATEWAY_LOOP_UNKNOWN
)
def test_nonpositive_pid_is_unknown(self, tmp_path):
_write_heartbeat(tmp_path, pid=4242, age_s=600.0)
assert (
gateway_cli.probe_gateway_loop_liveness(0, home=tmp_path)
== gateway_cli.GATEWAY_LOOP_UNKNOWN
)
def test_probe_never_raises_on_unreadable_path(self, monkeypatch):
monkeypatch.setattr(
"gateway.shutdown_watchdog.get_loop_heartbeat_path",
lambda home=None: (_ for _ in ()).throw(OSError("boom")),
)
assert (
gateway_cli.probe_gateway_loop_liveness(4242)
== gateway_cli.GATEWAY_LOOP_UNKNOWN
)
class TestEscalateWedgedGateway:
def test_sigterm_grace_suffices_without_sigkill(self, monkeypatch):
"""If SIGTERM lands (signal-handler thread alive), no SIGKILL is sent."""
signals = []
monkeypatch.setattr(
gateway_cli,
"terminate_pid",
lambda pid, force=False, **kwargs: signals.append(("kill" if force else "term", pid)),
)
monkeypatch.setattr(
gateway_cli, "_wait_for_pid_exit", lambda pid, timeout, **_: True
)
assert gateway_cli._escalate_wedged_gateway(4242) is True
assert signals == [("term", 4242)]
def test_escalates_to_sigkill_when_sigterm_ignored(self, monkeypatch):
signals = []
waits = []
def fake_wait(pid, timeout):
waits.append(timeout)
# First wait (SIGTERM grace) times out; second (post-SIGKILL) succeeds.
return len(waits) > 1
monkeypatch.setattr(
gateway_cli,
"terminate_pid",
lambda pid, force=False, **kwargs: signals.append(("kill" if force else "term", pid)),
)
monkeypatch.setattr(gateway_cli, "_wait_for_pid_exit", fake_wait)
assert gateway_cli._escalate_wedged_gateway(4242) is True
assert signals == [("term", 4242), ("kill", 4242)]
def test_total_wait_budget_is_bounded_well_under_drain(self, monkeypatch):
"""Worst case must be seconds, never the 180s drain budget."""
waits = []
monkeypatch.setattr(gateway_cli, "terminate_pid", lambda pid, force=False, **kwargs: None)
monkeypatch.setattr(
gateway_cli,
"_wait_for_pid_exit",
lambda pid, timeout, **_: waits.append(timeout) or False,
)
assert gateway_cli._escalate_wedged_gateway(4242) is False
assert sum(waits) < 30.0
def test_process_already_gone_is_success(self, monkeypatch):
def raise_gone(pid, force=False, **kwargs):
raise ProcessLookupError
monkeypatch.setattr(gateway_cli, "terminate_pid", raise_gone)
monkeypatch.setattr(
gateway_cli, "_wait_for_pid_exit", lambda pid, timeout, **_: True
)
assert gateway_cli._escalate_wedged_gateway(4242) is True
def test_sigkill_permission_error_does_not_raise(self, monkeypatch):
calls = []
def term(pid, force=False, **kwargs):
calls.append(force)
if force:
raise PermissionError
monkeypatch.setattr(gateway_cli, "terminate_pid", term)
monkeypatch.setattr(
gateway_cli, "_wait_for_pid_exit", lambda pid, timeout, **_: False
)
assert gateway_cli._escalate_wedged_gateway(4242) is False
assert calls == [False, True]
class TestLaunchdRestartWedgedIntegration:
"""launchd_restart must skip the 180s drain only for a wedged loop."""
def _setup(self, monkeypatch, liveness):
events = []
monkeypatch.setattr(gateway_cli, "get_launchd_label", lambda: "ai.hermes.gateway")
monkeypatch.setattr(gateway_cli, "_launchd_domain", lambda: "gui/501")
monkeypatch.setattr(gateway_cli, "_get_restart_drain_timeout", lambda: 180.0)
# Wait budget covers after-turn deferral + drain + headroom (#77184).
monkeypatch.setattr(gateway_cli, "_get_restart_exit_wait_budget", lambda: 195.0)
monkeypatch.setattr("gateway.status.get_running_pid", lambda *a, **k: 4242)
monkeypatch.setattr(
gateway_cli, "_request_gateway_self_restart", lambda pid: False
)
monkeypatch.setattr(
gateway_cli,
"probe_gateway_loop_liveness",
lambda pid, **kw: events.append("probe") or liveness,
)
monkeypatch.setattr(
gateway_cli,
"_escalate_wedged_gateway",
lambda pid, **kw: events.append("escalate") or True,
)
monkeypatch.setattr(
gateway_cli,
"terminate_pid",
lambda pid, force=False, **kwargs: events.append("sigterm"),
)
# Never let a real SIGUSR1 escape to PID 4242 during tests.
monkeypatch.setattr(
gateway_cli,
"_graceful_restart_via_sigusr1",
lambda pid, timeout, **_: events.append(("drain", pid, timeout)) or True,
)
# KeepAlive revival observed instantly — avoids the real 15s poll
# (mocked subprocess.run returns empty stdout, so the PID probe
# would otherwise burn the full observation timeout in time.sleep).
monkeypatch.setattr(
gateway_cli,
"_wait_for_launchd_service_pid",
lambda label, old_pid, timeout=10.0, *, domain: events.append("observe")
or True,
)
monkeypatch.setattr(
gateway_cli.subprocess,
"run",
lambda *a, **k: events.append("kickstart")
or __import__("types").SimpleNamespace(returncode=0, stdout="", stderr=""),
)
monkeypatch.setattr(
gateway_cli, "_clear_launchd_unsupported_marker", lambda: None
)
return events
def test_wedged_gateway_skips_drain_and_escalates(self, monkeypatch):
events = self._setup(monkeypatch, gateway_cli.GATEWAY_LOOP_WEDGED)
gateway_cli.launchd_restart()
assert "escalate" in events
# The 180s drain wait must never run for a wedged loop.
assert not any(isinstance(e, tuple) and e[0] == "drain" for e in events)
def test_busy_gateway_keeps_full_drain_budget(self, monkeypatch):
"""A busy-but-alive gateway (fresh heartbeat) must NOT be escalated —
that would bypass the in-flight cron drain floor (#86684)."""
events = self._setup(monkeypatch, gateway_cli.GATEWAY_LOOP_ALIVE)
gateway_cli.launchd_restart()
assert "escalate" not in events
assert ("drain", 4242, 195.0) in events
def test_unknown_liveness_keeps_full_drain_budget(self, monkeypatch):
"""Ambiguity (no heartbeat) must never trigger escalation."""
events = self._setup(monkeypatch, gateway_cli.GATEWAY_LOOP_UNKNOWN)
gateway_cli.launchd_restart()
assert "escalate" not in events
assert ("drain", 4242, 195.0) in events
class TestLoopTickWitness:
"""Two-witness liveness (#90502 review).
The heartbeat write moved off-loop, so a stale file no longer proves a
wedged loop and a fresh file no longer proves an alive one. The loop
answers a UNIX socket instead; the probe only escalates when BOTH
witnesses agree the loop stopped scheduling.
"""
@_NEEDS_UNIX_SOCKETS
def test_off_loop_completion_cannot_manufacture_fresh_liveness(self, tmp_path):
"""A write landing after the loop froze must not look alive.
File fresh (a late off-loop write completed) + loop silent: the probe
must NOT return ALIVE — the loop itself never answered, so freshness
is not a liveness proof. UNKNOWN keeps the safe drain path while
denying the false-fresh window the review described.
"""
pid = 4242
_silent_socket_node(get_loop_tick_socket_path(tmp_path, pid))
_write_heartbeat(tmp_path, pid, age_s=5.0)
_mark_witness_flag(tmp_path, armed=True)
assert (
gateway_cli.probe_gateway_loop_liveness(
pid, home=tmp_path, tick_timeout=0.2
)
== gateway_cli.GATEWAY_LOOP_UNKNOWN
)
@_NEEDS_UNIX_SOCKETS
def test_true_wedge_requires_sustained_witness_silence(self, tmp_path):
"""Stale file + armed socket silent across the whole window: WEDGED.
The destructive path must still exist for genuinely dead loops — a
frozen loop stops answering the socket for every attempt in the
bounded window, and the file goes stale. The reviewer's sustained-
proof contract (one sample is never destructive authority) is
satisfied here: all ``tick_strikes`` consecutive probes miss.
"""
pid = 4242
_silent_socket_node(get_loop_tick_socket_path(tmp_path, pid))
_write_heartbeat(tmp_path, pid, age_s=600.0)
_mark_witness_flag(tmp_path, armed=True, age_s=600.0)
assert (
gateway_cli.probe_gateway_loop_liveness(
pid,
home=tmp_path,
tick_timeout=0.2,
tick_strikes=3,
tick_gap_s=0.0,
)
== gateway_cli.GATEWAY_LOOP_WEDGED
)
def test_single_silent_probe_is_not_destructive(self, tmp_path, monkeypatch):
"""One miss is a transient stall, never a wedge (#90502 review).
Stale file + armed socket + a loop that misses the FIRST probe but
answers the second: the probe must recover to ALIVE — the loop is
demonstrably dispatching, and the stale file is just a stalled
write. A single silent sample must never grant the bounded kill
authority.
"""
pid = 4242
calls = []
def flaky_probe(_pid, _home, timeout=1.0):
calls.append(timeout)
# First sample misses (transient synchronous stall), the loop
# answers on the next attempt.
return len(calls) > 1
monkeypatch.setattr(gateway_cli, "_probe_loop_tick_socket", flaky_probe)
_write_heartbeat(tmp_path, pid, age_s=600.0)
_mark_witness_flag(tmp_path, armed=True, age_s=600.0)
assert (
gateway_cli.probe_gateway_loop_liveness(
pid,
home=tmp_path,
tick_timeout=0.2,
tick_strikes=2,
tick_gap_s=0.0,
)
== gateway_cli.GATEWAY_LOOP_ALIVE
)
def test_witness_vanishing_mid_window_is_unknown(self, tmp_path, monkeypatch):
"""A witness that disappears mid-window is ambiguity, not a wedge.
First sample silent (node existed), second sample finds no node:
the witness is gone and cannot condemn the loop. UNKNOWN keeps the
graceful drain path.
"""
pid = 4242
calls = []
monkeypatch.setattr(
gateway_cli,
"_probe_loop_tick_socket",
lambda _pid, _home, timeout=1.0: (calls.append(timeout), False, None)[
len(calls)
],
)
_write_heartbeat(tmp_path, pid, age_s=600.0)
_mark_witness_flag(tmp_path, armed=True, age_s=600.0)
assert (
gateway_cli.probe_gateway_loop_liveness(
pid,
home=tmp_path,
tick_timeout=0.2,
tick_strikes=2,
tick_gap_s=0.0,
)
== gateway_cli.GATEWAY_LOOP_UNKNOWN
)
def test_armed_witness_unreachable_is_unknown(self, tmp_path):
"""Producer claims the witness is armed but no node exists: ambiguity.
Never kill on it — the graceful drain remains the backstop.
"""
pid = 4242
_write_heartbeat(tmp_path, pid, age_s=600.0)
_mark_witness_flag(tmp_path, armed=True, age_s=600.0)
assert (
gateway_cli.probe_gateway_loop_liveness(
pid, home=tmp_path, tick_timeout=0.2
)
== gateway_cli.GATEWAY_LOOP_UNKNOWN
)
def test_unarmed_witness_disables_stale_escalation(self, tmp_path):
"""New producer whose bind failed: staleness is NOT proof.
The write is off-loop, so the file can age while the loop runs; with
no witness, a stale file must never escalate.
"""
pid = 4242
_write_heartbeat(tmp_path, pid, age_s=600.0)
_mark_witness_flag(tmp_path, armed=False, age_s=600.0)
assert (
gateway_cli.probe_gateway_loop_liveness(
pid, home=tmp_path, tick_timeout=0.2
)
== gateway_cli.GATEWAY_LOOP_UNKNOWN
)
def test_legacy_payload_keeps_single_witness_contract(self, tmp_path):
"""No witness flag = on-loop writer: staleness stays proof.
Old gateways never moved the write off-loop, so their stale file
still means a dead loop — the legacy WEDGED verdict is unchanged.
"""
pid = 4242
_write_heartbeat(tmp_path, pid, age_s=600.0)
assert (
gateway_cli.probe_gateway_loop_liveness(pid, home=tmp_path)
== gateway_cli.GATEWAY_LOOP_WEDGED
)
@_NEEDS_UNIX_SOCKETS
def test_legacy_fresh_file_with_dead_node_is_unknown(self, tmp_path):
"""A fresh legacy file stays safe under a dead-listener node.
A dead-listener node for the PID (leftover from a newer process):
the silent socket denies ALIVE, and UNKNOWN never escalates — the
drain path keeps the full budget either way.
"""
pid = 4242
_write_heartbeat(tmp_path, pid, age_s=5.0)
_silent_socket_node(get_loop_tick_socket_path(tmp_path, pid))
assert (
gateway_cli.probe_gateway_loop_liveness(
pid, home=tmp_path, tick_timeout=0.2
)
== gateway_cli.GATEWAY_LOOP_UNKNOWN
)
@_NEEDS_UNIX_SOCKETS
@pytest.mark.asyncio
async def test_producer_rebinds_over_stale_socket_node(self, tmp_path):
"""A leftover node from a dead process must not disarm the witness.
os._exit(75) / SIGKILL skip the finally-unlink, and PID reuse then
re-lands on the same PID-suffixed path. Without the pre-bind unlink
the bind fails EADDRINUSE, the except disarms the witness
(loop_tick_socket:false), and a stale heartbeat can never classify
WEDGED — precisely on crash-restart loops.
"""
sock_path = get_loop_tick_socket_path(tmp_path)
_silent_socket_node(sock_path) # dead process's leftover node
assert sock_path.exists()
task = asyncio.create_task(
loop_heartbeat_forever(interval_s=0.2, home=tmp_path)
)
try:
for _ in range(50):
hb = get_loop_heartbeat_path(tmp_path)
if hb.exists():
payload = json.loads(hb.read_text(encoding="utf-8"))
if payload.get("loop_tick_socket"):
break
await asyncio.sleep(0.05)
else:
raise AssertionError(
"witness never armed over the stale socket node"
)
# And it actually answers: the node is live, not the leftover.
reader, writer = await asyncio.open_unix_connection(str(sock_path))
data = await asyncio.wait_for(reader.read(64), timeout=2)
writer.close()
assert data, "re-bound tick socket gave no answer"
finally:
task.cancel()
try:
await task
except asyncio.CancelledError:
pass
@_NEEDS_UNIX_SOCKETS
def test_transient_stall_below_wedge_budget_never_escalates(
self, tmp_path, monkeypatch
):
"""Composed producer+consumer: transient loop stall below the budget.
Heartbeat write stalled (file stale) + loop frozen for longer than
one recv timeout but shorter than the wedge window: the probe must
NOT return WEDGED — the loop answers once it unfreezes, inside the
bounded window — and the restart path fed by the real probe must
take the graceful drain, never the bounded escalation. This is the
reviewer's required composed regression (#90502 review).
"""
pid = os.getpid()
block_s = 0.5
stale_after = 1.0
tick_timeout = 0.2
tick_strikes = 3
tick_gap_s = 0.05
wedge_window = tick_strikes * tick_timeout + (tick_strikes - 1) * tick_gap_s
assert tick_timeout < block_s < wedge_window, (tick_timeout, block_s, wedge_window)
errors = []
state, ready = _start_freezeable_producer(tmp_path, block_s, errors)
try:
assert ready.wait(timeout=5.0)
sock_path = get_loop_tick_socket_path(tmp_path, pid)
deadline = time.monotonic() + 5.0
while not sock_path.exists() and time.monotonic() < deadline:
time.sleep(0.02)
assert sock_path.exists(), "producer never armed the tick socket"
_wait_heartbeat_stale(tmp_path, stale_after)
state["trigger"]() # freeze the loop for block_s
verdict = gateway_cli.probe_gateway_loop_liveness(
pid,
home=tmp_path,
stale_after=stale_after,
tick_timeout=tick_timeout,
tick_strikes=tick_strikes,
tick_gap_s=tick_gap_s,
)
assert verdict == gateway_cli.GATEWAY_LOOP_ALIVE, verdict
events = _launchd_harness(monkeypatch, tmp_path, pid)
gateway_cli.launchd_restart()
assert "escalate" not in events
drains = [e for e in events if isinstance(e, tuple) and e[0] == "drain"]
assert drains, events
finally:
state["stop"]()
state["thread"].join(timeout=5.0)
assert not errors, errors
@_NEEDS_UNIX_SOCKETS
def test_sustained_stop_above_wedge_budget_still_escalates(
self, tmp_path
):
"""Composed producer+consumer: sustained stop above the budget.
The bounded-proof contract must not neuter the destructive path: a
loop frozen for longer than the wedge window stays silent for every
probe attempt, so the probe still returns WEDGED. (The WEDGED →
bounded-escalation wiring on the restart paths is covered by
``TestLaunchdRestartWedgedIntegration``.)
"""
pid = os.getpid()
block_s = 1.2
stale_after = 1.0
tick_timeout = 0.2
tick_strikes = 3
tick_gap_s = 0.05
wedge_window = tick_strikes * tick_timeout + (tick_strikes - 1) * tick_gap_s
assert block_s > wedge_window, (block_s, wedge_window)
errors = []
state, ready = _start_freezeable_producer(tmp_path, block_s, errors)
try:
assert ready.wait(timeout=5.0)
sock_path = get_loop_tick_socket_path(tmp_path, pid)
deadline = time.monotonic() + 5.0
while not sock_path.exists() and time.monotonic() < deadline:
time.sleep(0.02)
assert sock_path.exists(), "producer never armed the tick socket"
_wait_heartbeat_stale(tmp_path, stale_after)
state["trigger"]() # freeze the loop for block_s
verdict = gateway_cli.probe_gateway_loop_liveness(
pid,
home=tmp_path,
stale_after=stale_after,
tick_timeout=tick_timeout,
tick_strikes=tick_strikes,
tick_gap_s=tick_gap_s,
)
assert verdict == gateway_cli.GATEWAY_LOOP_WEDGED, verdict
finally:
state["stop"]()
state["thread"].join(timeout=5.0)
assert not errors, errors
class TestLoopTickTcpWitness:
"""Non-POSIX arm: the producer publishes ``loop_tick_tcp_port`` and the
consumer probes 127.0.0.1:<port> instead of the AF_UNIX node. The
two-witness contract must hold identically over TCP."""
@staticmethod
def _tcp_answerer():
"""A loopback listener that answers b"1" — the armed, dispatching loop."""
srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
srv.bind(("127.0.0.1", 0))
srv.listen(8)
stop = threading.Event()
def serve():
srv.settimeout(0.1)
while not stop.is_set():
try:
conn, _ = srv.accept()
except socket.timeout:
continue
try:
conn.sendall(b"1")
finally:
conn.close()
thread = threading.Thread(target=serve, daemon=True)
thread.start()
return srv.getsockname()[1], stop, srv
@staticmethod
def _write_tcp_heartbeat(home, pid, port, age_s=0.0):
write_loop_heartbeat(
pid=pid,
home=home,
extra={"loop_tick_socket": True, "loop_tick_tcp_port": port},
)
if age_s:
path = get_loop_heartbeat_path(home)
stamp = time.time() - age_s
os.utime(path, (stamp, stamp))
def test_stale_file_with_answering_tcp_witness_is_alive(self, tmp_path):
"""#90502 shape over TCP: a stalled write must not kill a live loop."""
port, stop, srv = self._tcp_answerer()
try:
self._write_tcp_heartbeat(tmp_path, 4343, port, age_s=600.0)
assert (
gateway_cli.probe_gateway_loop_liveness(4343, home=tmp_path)
== gateway_cli.GATEWAY_LOOP_ALIVE
)
finally:
stop.set()
srv.close()
def test_stale_file_with_silent_tcp_witness_is_wedged(self, tmp_path):
"""Armed TCP witness that never answers across the window: WEDGED."""
silent = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
silent.bind(("127.0.0.1", 0))
silent.listen(1) # accepts but never sends
try:
port = silent.getsockname()[1]
self._write_tcp_heartbeat(tmp_path, 4344, port, age_s=600.0)
assert (
gateway_cli.probe_gateway_loop_liveness(
4344, home=tmp_path, tick_timeout=0.2, tick_gap_s=0.05
)
== gateway_cli.GATEWAY_LOOP_WEDGED
)
finally:
silent.close()
def test_fresh_file_with_silent_tcp_witness_is_unknown(self, tmp_path):
"""Fresh file + silent TCP witness: an off-loop write landed after a
freeze — not proof of liveness, never destructive authority."""
silent = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
silent.bind(("127.0.0.1", 0))
silent.listen(1)
try:
port = silent.getsockname()[1]
self._write_tcp_heartbeat(tmp_path, 4345, port)
assert (
gateway_cli.probe_gateway_loop_liveness(
4345, home=tmp_path, tick_timeout=0.2
)
== gateway_cli.GATEWAY_LOOP_UNKNOWN
)
finally:
silent.close()
def test_garbage_tcp_port_falls_back_to_socket_contract(self, tmp_path):
"""A non-numeric port must not be treated as an armed witness."""
write_loop_heartbeat(
pid=4346,
home=tmp_path,
extra={"loop_tick_socket": False, "loop_tick_tcp_port": "nope"},
)
path = get_loop_heartbeat_path(tmp_path)
stamp = time.time() - 600.0
os.utime(path, (stamp, stamp))
# loop_tick_socket=False + no usable TCP port: witness could not be
# armed, staleness is not proof -> UNKNOWN, never WEDGED.
assert (
gateway_cli.probe_gateway_loop_liveness(4346, home=tmp_path)
== gateway_cli.GATEWAY_LOOP_UNKNOWN
)