mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-16 19:36:54 +00:00
# Conflicts: # apps/desktop/e2e/archived-hidden-session-recoverable.spec.ts # apps/desktop/e2e/bot-chat-message-agent-friendly-name.spec.ts # apps/desktop/e2e/bot-mailbox-unreadable-ticket.spec.ts # apps/desktop/e2e/bot-mode-roster-localized.spec.ts # apps/desktop/e2e/bot-mode-row-click-mirrors-registry.spec.ts # apps/desktop/e2e/bot-mode-tab-shows-bot-name.spec.ts # apps/desktop/e2e/bot-roster-group-row-organisation.spec.ts # apps/desktop/e2e/bot-roster-ignores-infra-dirs.spec.ts # apps/desktop/e2e/bot-roster-timestamp-meta.spec.ts # apps/desktop/e2e/bot-roster-user-sections.spec.ts # apps/desktop/e2e/bot-routines-pane-narrow.spec.ts # apps/desktop/e2e/bot-row-open-recent-session.spec.ts # apps/desktop/e2e/bot-tile-ignores-ambient-composer-model.spec.ts # apps/desktop/e2e/group-composer-auto-grow.spec.ts # apps/desktop/e2e/group-create-gate-remote-roster.spec.ts # apps/desktop/e2e/group-prompt-renamed-primary-handle.spec.ts # apps/desktop/e2e/hosted-room-backend-continuity.spec.ts # apps/desktop/e2e/hosted-room-legacy-store-migration.spec.ts # apps/desktop/e2e/settings-scope-chips-bot-title.spec.ts # apps/desktop/e2e/worktree-branch-status.spec.ts # apps/desktop/electron/backend-probes.test.ts # apps/desktop/electron/connection-apply.test.ts # apps/desktop/electron/desktop-electron-pin.test.ts # apps/desktop/electron/desktop-uninstall.test.ts # apps/desktop/electron/gateway-file-download-transport.test.ts # apps/desktop/electron/gateway-stop-before-update.test.ts # apps/desktop/electron/github-api-auth.test.ts # apps/desktop/electron/registry-primary-profile-scope.test.ts # apps/desktop/electron/update-api-check.test.ts # apps/desktop/electron/update-handoff-marker.test.ts # apps/desktop/electron/venv-blocker-scan.test.ts # apps/desktop/scripts/after-extract.test.mjs # apps/desktop/scripts/local-pack-publish.test.mjs # apps/desktop/scripts/tasks-scroll.test.mjs # apps/desktop/src/app/settings/model-settings.test.tsx # apps/desktop/src/app/updates-overlay.blockers.test.tsx # apps/desktop/src/components/desktop-install-overlay.test.tsx # apps/desktop/src/lib/update-copy.test.ts # scripts/ci/check_os_marker_fakes.py # tests-js/desktop-mac-usage-descriptions.test.ts # tests-js/node-engine-alignment.test.ts # tests/agent/lsp/test_install_and_lint_fixes.py # tests/agent/test_command_token_source.py # tests/agent/test_compression_boundary_hook.py # tests/agent/test_create_openai_client_ssl_verify.py # tests/agent/test_custom_provider_ca_probes.py # tests/agent/test_endpoint_blackhole.py # tests/agent/test_estimator_parity.py # tests/agent/test_in_place_compaction.py # tests/agent/test_moa_loop_mode.py # tests/agent/test_model_metadata.py # tests/agent/test_skill_session_platform_gate.py # tests/agent/test_skill_utils.py # tests/agent/test_ssl_ca_guard.py # tests/computer_use/test_doctor.py # tests/cron/test_codex_execution_paths.py # tests/cron/test_cron_bot_chat_delivery.py # tests/cron/test_cron_script.py # tests/cron/test_media_delivery_parity.py # tests/cron/test_misfire_catchup.py # tests/cron/test_parallel_pool.py # tests/cron/test_recurring_eagain_redispatch.py # tests/gateway/test_choice_picker.py # tests/gateway/test_control_socket_windows_live.py # tests/gateway/test_dingtalk.py # tests/gateway/test_feishu.py # tests/gateway/test_feishu_onboard.py # tests/gateway/test_gateway_shutdown.py # tests/gateway/test_matrix.py # tests/gateway/test_model_command_custom_providers.py # tests/gateway/test_reasoning_command.py # tests/gateway/test_runtime_footer.py # tests/gateway/test_session.py # tests/gateway/test_session_hygiene.py # tests/gateway/test_status.py # tests/gateway/test_teams.py # tests/gateway/test_turn_lease.py # tests/gateway/test_whatsapp_connect.py # tests/hermes_cli/test_approvals_command.py # tests/hermes_cli/test_auth_store_lock_concurrent.py # tests/hermes_cli/test_backup.py # tests/hermes_cli/test_banner_git_state.py # tests/hermes_cli/test_certifi_repair.py # tests/hermes_cli/test_cmd_update.py # tests/hermes_cli/test_compat_manifest_targets.py # tests/hermes_cli/test_computer_use_cli.py # tests/hermes_cli/test_cpr_local_leak.py # tests/hermes_cli/test_dashboard_auth_gate.py # tests/hermes_cli/test_dashboard_procs_kill_grace.py # tests/hermes_cli/test_desktop_lifecycle_windows_live.py # tests/hermes_cli/test_doctor.py # tests/hermes_cli/test_doctor_command_install.py # tests/hermes_cli/test_fleet_config_migration_windows_live.py # tests/hermes_cli/test_gateway.py # tests/hermes_cli/test_gateway_platform_gating.py # tests/hermes_cli/test_gateway_restart_loop.py # tests/hermes_cli/test_gateway_task_probe.py # tests/hermes_cli/test_gateway_wsl.py # tests/hermes_cli/test_gui_command.py # tests/hermes_cli/test_install_cua_driver.py # tests/hermes_cli/test_kanban_db.py # tests/hermes_cli/test_lazy_command_exports.py # tests/hermes_cli/test_lazy_refresh_venv_repair.py # tests/hermes_cli/test_linux_desktop_entry.py # tests/hermes_cli/test_local_runtime.py # tests/hermes_cli/test_local_runtime_updates.py # tests/hermes_cli/test_managed_uv.py # tests/hermes_cli/test_mcp_reload_confirm_gate.py # tests/hermes_cli/test_nous_subscription.py # tests/hermes_cli/test_npm_engine.py # tests/hermes_cli/test_personality_none.py # tests/hermes_cli/test_pet_toggle.py # tests/hermes_cli/test_plan_reconciliation_windows_live.py # tests/hermes_cli/test_plugin_event_bus.py # tests/hermes_cli/test_plugin_manifest_v2.py # tests/hermes_cli/test_plugin_packs.py # tests/hermes_cli/test_plugins_cmd.py # tests/hermes_cli/test_plugins_cmd_enable_disable_nested.py # tests/hermes_cli/test_process_identity.py # tests/hermes_cli/test_profiles.py # tests/hermes_cli/test_profiles_sidebar_cache.py # tests/hermes_cli/test_pty_bridge.py # tests/hermes_cli/test_resolve_turn_limit.py # tests/hermes_cli/test_serve_runtime_inventory.py # tests/hermes_cli/test_session_vacuum_config.py # tests/hermes_cli/test_set_config_value.py # tests/hermes_cli/test_signal_handler_kanban_worker.py # tests/hermes_cli/test_slash_confirm_windows.py # tests/hermes_cli/test_stale_pid_guard.py # tests/hermes_cli/test_startup_fast_guards.py # tests/hermes_cli/test_status.py # tests/hermes_cli/test_telegram_managed_bot.py # tests/hermes_cli/test_tools_config.py # tests/hermes_cli/test_update_apply_shallow_count.py # tests/hermes_cli/test_update_autostash.py # tests/hermes_cli/test_update_concurrent_quarantine.py # tests/hermes_cli/test_update_fetch_failure_classifier.py # tests/hermes_cli/test_update_fleet_probe_resume_token.py # tests/hermes_cli/test_update_handoff_backend_reap.py # tests/hermes_cli/test_update_handoff_desktop_rebuild.py # tests/hermes_cli/test_update_head_moved_gate.py # tests/hermes_cli/test_update_host_obligation.py # tests/hermes_cli/test_update_import_guard.py # tests/hermes_cli/test_update_interrupted_recovery.py # tests/hermes_cli/test_update_inventory.py # tests/hermes_cli/test_update_launchd_unloaded_gateway.py # tests/hermes_cli/test_update_missing_configured_deps.py # tests/hermes_cli/test_update_modified_notice.py # tests/hermes_cli/test_update_multiplex_migration_hook.py # tests/hermes_cli/test_update_no_gateway_restart.py # tests/hermes_cli/test_update_orphan_backend_reap.py # tests/hermes_cli/test_update_parked_branch_guard.py # tests/hermes_cli/test_update_post_pull_syntax_guard.py # tests/hermes_cli/test_update_receipt.py # tests/hermes_cli/test_update_self_lock.py # tests/hermes_cli/test_update_shim_fail_closed.py # tests/hermes_cli/test_update_shim_self_lock.py # tests/hermes_cli/test_update_sqlite_remediation.py # tests/hermes_cli/test_update_stale_dashboard.py # tests/hermes_cli/test_update_stale_virtualenv.py # tests/hermes_cli/test_update_venv_health.py # tests/hermes_cli/test_update_venv_ownership_preflight.py # tests/hermes_cli/test_update_wedged_gateway.py # tests/hermes_cli/test_update_yes_flag.py # tests/hermes_cli/test_update_zip_two_phase.py # tests/hermes_cli/test_urllib_security.py # tests/hermes_cli/test_ux_messages_auth_config.py # tests/hermes_cli/test_ux_messages_startup.py # tests/hermes_cli/test_venv_holder_classifier.py # tests/hermes_cli/test_verify_console_scripts.py # tests/hermes_cli/test_verify_core_dependencies.py # tests/hermes_cli/test_web_server.py # tests/hermes_cli/test_web_server_console_ws.py # tests/hermes_cli/test_web_server_ws_ping.py # tests/hermes_cli/test_web_ui_build.py # tests/hermes_state/test_fts_rebuild_admission.py # tests/hermes_state/test_hermes_state.py # tests/plugins/memory/test_memory_lazy_install.py # tests/plugins/test_google_meet_plugin.py # tests/plugins/test_langfuse_plugin.py # tests/plugins/test_security_guidance_plugin.py # tests/plugins/test_transform_llm_output_hook.py # tests/scripts/desktop_update/test_desktop_update_windows_gateway_flag.py # tests/scripts/desktop_update/test_desktop_update_windows_python_handoff.py # tests/scripts/desktop_update/test_desktop_update_windows_timestamp.py # tests/scripts/install/test_install_clone_throttle_fallback.py # tests/scripts/install/test_install_lockfile_churn.py # tests/scripts/install/test_install_no_initial_commit.py # tests/scripts/install/test_install_sh_browser_install.py # tests/scripts/install/test_install_sh_node_prerelease.py # tests/scripts/install/test_install_sh_symlink_stomp.py # tests/scripts/install/test_install_sh_uv_lock_config.py # tests/scripts/install/test_install_unmerged_index.py # tests/scripts/test_contributor_map.py # tests/scripts/test_run_tests_parallel.py # tests/skills/test_competitor_news_monitor_skill.py # tests/skills/test_document_to_action_items_skill.py # tests/skills/test_google_workspace_setup.py # tests/skills/test_google_workspace_setup_deps.py # tests/skills/test_grounded_citations_skill.py # tests/skills/test_ip_as_logo_skill.py # tests/skills/test_live_dashboard_skill.py # tests/skills/test_mcp_oauth_remote_gateway_skill.py # tests/skills/test_office_document_skills.py # tests/skills/test_openclaw_migration.py # tests/skills/test_product_price_monitor_skill.py # tests/skills/test_scrollcraft_skill.py # tests/skills/test_setup_wizard_generator_skill.py # tests/skills/test_weekly_review_planning_skill.py # tests/test_engines_satisfiable.py # tests/test_fast_safe_load.py # tests/test_hermes_bootstrap.py # tests/test_hermes_constants.py # tests/test_hermes_logging.py # tests/test_managed_runtime_resolution.py # tests/test_model_tools_async_bridge.py # tests/test_packaging_build_guard.py # tests/test_packaging_metadata.py # tests/test_yaml_indent_consistency.py # tests/tools/test_approval_timeout_overflow.py # tests/tools/test_base_environment.py # tests/tools/test_bot_mode_dm.py # tests/tools/test_browser_chromium_check.py # tests/tools/test_browser_hardening.py # tests/tools/test_browser_homebrew_paths.py # tests/tools/test_browser_npx_warmup.py # tests/tools/test_browser_orphan_reaper.py # tests/tools/test_browser_real_profile.py # tests/tools/test_browser_use_cli.py # tests/tools/test_clipboard.py # tests/tools/test_code_execution.py # tests/tools/test_code_execution_modes.py # tests/tools/test_code_execution_windows_env.py # tests/tools/test_computer_use.py # tests/tools/test_delegate_liveness_timeout.py # tests/tools/test_execute_code_approval_cluster.py # tests/tools/test_execution_flag_detection.py # tests/tools/test_fal_common.py # tests/tools/test_file_operations.py # tests/tools/test_file_tools.py # tests/tools/test_file_tools_cwd_resolution.py # tests/tools/test_file_tools_live.py # tests/tools/test_lazy_deps.py # tests/tools/test_lazy_deps_durable_target.py # tests/tools/test_lazy_deps_managed.py # tests/tools/test_local_env_blocklist.py # tests/tools/test_local_tempdir.py # tests/tools/test_macos_protected_search.py # tests/tools/test_mcp_npx_cached_bin.py # tests/tools/test_oneshot_completion_linger.py # tests/tools/test_process_registry.py # tests/tools/test_read_file_schema_gating.py # tests/tools/test_skill_improvements.py # tests/tools/test_skills_sync.py # tests/tools/test_termux_api_detection.py # tests/tools/test_tirith_security.py # tests/tools/test_transcription_tools.py # tests/tools/test_tts_streaming.py # tests/tools/test_wake_word.py # tests/tui_gateway/test_compute_host_borrowed_lease.py # tests/tui_gateway/test_compute_host_turn_protocol.py # tests/tui_gateway/test_isolated_orphan_activity.py # tests/tui_gateway/test_protocol.py # tests/tui_gateway/test_slash_worker_profile_home.py # tests/tui_gateway/test_subprocess_encoding.py # tests/tui_gateway/test_tui_gateway_server.py # ui-tui/src/__tests__/terminalParity.test.ts # ui-tui/src/__tests__/termuxComposerLayout.test.ts # ui-tui/src/__tests__/textInputFastEcho.test.ts
895 lines
35 KiB
Python
895 lines
35 KiB
Python
"""Tests for the wedged-gateway health probe + bounded escalation (#81642).
|
|
|
|
A gateway whose event loop is stalled cannot process a graceful shutdown, so
|
|
the updater's drain wait used to burn the full 180s budget ("Gateway PID X
|
|
still running after 180.0s — restart may fail") and could deadlock `hermes
|
|
update`. The fix probes the loop-liveness heartbeat file BEFORE draining and,
|
|
only when the loop is provably dead, escalates SIGTERM → SIGKILL bounded to
|
|
seconds. A busy-but-alive gateway (fresh heartbeat) must keep the full drain
|
|
path — including the in-flight cron drain floor from #86684.
|
|
"""
|
|
|
|
import asyncio
|
|
import json
|
|
import os
|
|
import shutil
|
|
import socket
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
|
|
import hermes_cli.gateway as gateway_cli
|
|
from gateway.shutdown_watchdog import (
|
|
get_loop_heartbeat_path,
|
|
get_loop_tick_socket_path,
|
|
loop_heartbeat_forever,
|
|
write_loop_heartbeat,
|
|
)
|
|
|
|
# Native Windows exposes neither ``socket.AF_UNIX`` nor an asyncio UNIX
|
|
# server, so the witness cases that create real socket nodes
|
|
# (``_silent_socket_node``) or run the real producer
|
|
# (``loop_heartbeat_forever``) cannot execute there. Only those cases are
|
|
# skipped: the witness-absent contracts (mocked probes, file-only
|
|
# heartbeats) are platform-independent and keep running on Windows, per
|
|
# the Windows behavior pinned alongside the product-side guarantee.
|
|
_NEEDS_UNIX_SOCKETS = pytest.mark.skipif(
|
|
sys.platform == "win32",
|
|
reason="requires real UNIX-domain sockets "
|
|
"(socket.AF_UNIX / asyncio.start_unix_server), unavailable on native Windows",
|
|
)
|
|
|
|
@pytest.fixture()
|
|
def tmp_path():
|
|
"""Short-path override for this module (macOS AF_UNIX ~104-byte limit).
|
|
|
|
The loop-tick witness tests bind real UNIX sockets under HERMES_HOME.
|
|
pytest's default tmp_path nests deep enough on macOS that
|
|
``state/gateway.loop-tick.<pid>.sock`` exceeds the sockaddr_un limit and
|
|
``bind()`` raises ``OSError: AF_UNIX path too long``. A mkdtemp directly
|
|
under the system temp root keeps the socket path well under the limit
|
|
on every platform.
|
|
"""
|
|
path = Path(tempfile.mkdtemp(prefix="hwg-"))
|
|
try:
|
|
yield path
|
|
finally:
|
|
shutil.rmtree(path, ignore_errors=True)
|
|
|
|
def _write_heartbeat(home, pid, age_s=0.0):
|
|
"""Write a heartbeat file for ``pid`` whose mtime is ``age_s`` old."""
|
|
path = get_loop_heartbeat_path(home)
|
|
write_loop_heartbeat(pid=pid, home=home)
|
|
if age_s:
|
|
stamp = time.time() - age_s
|
|
os.utime(path, (stamp, stamp))
|
|
return path
|
|
|
|
def _mark_witness_flag(home, armed, age_s=0.0):
|
|
"""Set ``loop_tick_socket`` on the heartbeat payload; re-stamp mtime."""
|
|
path = get_loop_heartbeat_path(home)
|
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
payload["loop_tick_socket"] = armed
|
|
path.write_text(json.dumps(payload), encoding="utf-8")
|
|
if age_s:
|
|
stamp = time.time() - age_s
|
|
os.utime(path, (stamp, stamp))
|
|
return path
|
|
|
|
def _silent_socket_node(path):
|
|
"""Create a socket node at ``path`` that never answers.
|
|
|
|
Bind + listen, then close the listener WITHOUT unlinking: the node stays,
|
|
so a probe's connect() gets ECONNREFUSED — a witness that exists but is
|
|
silent, exactly like a dead listener (or a loop that stopped scheduling).
|
|
"""
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
srv = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
|
try:
|
|
srv.bind(str(path))
|
|
srv.listen(1)
|
|
finally:
|
|
srv.close()
|
|
|
|
def _start_freezeable_producer(tmp_path, block_s, errors, write_stall_s=1.5):
|
|
"""Run the real heartbeat producer on a loop that can be frozen on demand.
|
|
|
|
Returns ``(state, ready)``:
|
|
|
|
- ``state["trigger"]`` — freezes the gateway loop synchronously for
|
|
``block_s`` seconds. While frozen, NO task runs on the loop — not the
|
|
heartbeat writer, not the tick-socket handler — which is exactly the
|
|
"loop not scheduling" condition the probe must detect (sustained).
|
|
- ``state["thread"]`` — the producer thread; join it in ``finally``.
|
|
- ``ready`` — set once the loop is running (socket may still be arming).
|
|
|
|
The heartbeat write is patched to stall ``write_stall_s`` after the first
|
|
write, so the file goes stale while the loop keeps dispatching.
|
|
"""
|
|
freeze_evt = asyncio.Event()
|
|
stop_evt = asyncio.Event()
|
|
state: dict = {"loop": None, "trigger": None, "thread": None, "stop": lambda: None}
|
|
ready = threading.Event()
|
|
|
|
def stalling_write(**_kwargs):
|
|
if not get_loop_heartbeat_path(tmp_path).exists():
|
|
return write_loop_heartbeat(**_kwargs)
|
|
time.sleep(write_stall_s)
|
|
return write_loop_heartbeat(**_kwargs)
|
|
|
|
async def freeze_gate() -> None:
|
|
while True:
|
|
await freeze_evt.wait()
|
|
freeze_evt.clear()
|
|
# Synchronous sleep: freezes the entire loop for block_s.
|
|
time.sleep(block_s)
|
|
|
|
async def producer() -> None:
|
|
loop = asyncio.get_running_loop()
|
|
state["loop"] = loop
|
|
state["trigger"] = lambda: loop.call_soon_threadsafe(freeze_evt.set)
|
|
state["stop"] = lambda: loop.call_soon_threadsafe(stop_evt.set)
|
|
with patch("gateway.shutdown_watchdog.write_loop_heartbeat", stalling_write):
|
|
task = asyncio.create_task(
|
|
loop_heartbeat_forever(interval_s=1.0, home=tmp_path)
|
|
)
|
|
gate = asyncio.create_task(freeze_gate())
|
|
try:
|
|
ready.set()
|
|
await stop_evt.wait()
|
|
finally:
|
|
task.cancel()
|
|
gate.cancel()
|
|
try:
|
|
await task
|
|
except asyncio.CancelledError:
|
|
pass
|
|
try:
|
|
await gate
|
|
except asyncio.CancelledError:
|
|
pass
|
|
|
|
def run_producer() -> None:
|
|
try:
|
|
asyncio.run(producer())
|
|
except Exception as exc: # surfaced via errors after join
|
|
errors.append(exc)
|
|
|
|
thread = threading.Thread(target=run_producer, daemon=True)
|
|
thread.start()
|
|
state["thread"] = thread
|
|
return state, ready
|
|
|
|
def _wait_heartbeat_stale(tmp_path, stale_after, timeout_s=5.0):
|
|
"""Block until the heartbeat file is older than ``stale_after``."""
|
|
hb_path = get_loop_heartbeat_path(tmp_path)
|
|
deadline = time.monotonic() + timeout_s
|
|
while True:
|
|
try:
|
|
age = time.time() - hb_path.stat().st_mtime
|
|
except FileNotFoundError:
|
|
# The socket is armed before the first write lands; the file
|
|
# appears a tick later.
|
|
age = 0.0
|
|
if age > stale_after:
|
|
return
|
|
assert time.monotonic() < deadline, "heartbeat never went stale"
|
|
time.sleep(0.02)
|
|
|
|
def _launchd_harness(monkeypatch, tmp_path, pid):
|
|
"""Patch the launchd_restart path so the REAL probe drives it.
|
|
|
|
Returns the ``events`` list: "probe" is recorded by the probe wrapper
|
|
below, "escalate" by ``_escalate_wedged_gateway``, ``("drain", t)`` by
|
|
the graceful drain wait, "kickstart" by the relaunch. The probe itself
|
|
is the real ``probe_gateway_loop_liveness`` (home resolved through the
|
|
patched ``_process_hermes_home``).
|
|
"""
|
|
events = []
|
|
monkeypatch.setattr(gateway_cli, "get_launchd_label", lambda: "ai.hermes.gateway")
|
|
monkeypatch.setattr(gateway_cli, "_launchd_domain", lambda: "gui/501")
|
|
monkeypatch.setattr(gateway_cli, "_get_restart_drain_timeout", lambda: 180.0)
|
|
monkeypatch.setattr("gateway.status.get_running_pid", lambda *a, **k: pid)
|
|
monkeypatch.setattr(
|
|
gateway_cli, "_request_gateway_self_restart", lambda pid: False
|
|
)
|
|
real_probe = gateway_cli.probe_gateway_loop_liveness
|
|
|
|
def recording_probe(pid, **kw):
|
|
events.append("probe")
|
|
return real_probe(pid, **kw)
|
|
|
|
monkeypatch.setattr(gateway_cli, "probe_gateway_loop_liveness", recording_probe)
|
|
monkeypatch.setattr(
|
|
gateway_cli,
|
|
"_escalate_wedged_gateway",
|
|
lambda pid, **kw: events.append("escalate") or True,
|
|
)
|
|
monkeypatch.setattr(
|
|
gateway_cli,
|
|
"terminate_pid",
|
|
lambda pid, force=False, **kwargs: events.append(("kill" if force else "term", pid)),
|
|
)
|
|
# Never let a real SIGUSR1 escape to the live test PID — the drain path
|
|
# goes through _graceful_restart_via_sigusr1 (in-place restart) before
|
|
# any exit-wait, and these tests feed launchd_restart os.getpid().
|
|
monkeypatch.setattr(
|
|
gateway_cli,
|
|
"_graceful_restart_via_sigusr1",
|
|
lambda pid, timeout, **_: events.append(("drain", pid, timeout)) or True,
|
|
)
|
|
monkeypatch.setattr(
|
|
gateway_cli,
|
|
"_wait_for_gateway_exit",
|
|
lambda timeout, force_after=None: events.append(("drain", timeout)) or True,
|
|
)
|
|
monkeypatch.setattr(
|
|
gateway_cli,
|
|
"_wait_for_launchd_service_pid",
|
|
lambda label, old_pid, timeout=10.0, *, domain: events.append("observe")
|
|
or True,
|
|
)
|
|
monkeypatch.setattr(
|
|
gateway_cli.subprocess,
|
|
"run",
|
|
lambda *a, **k: events.append("kickstart")
|
|
or __import__("types").SimpleNamespace(returncode=0, stdout="", stderr=""),
|
|
)
|
|
monkeypatch.setattr(gateway_cli, "_clear_launchd_unsupported_marker", lambda: None)
|
|
monkeypatch.setattr(
|
|
"gateway.shutdown_watchdog._process_hermes_home", lambda: tmp_path
|
|
)
|
|
return events
|
|
|
|
class TestProbeGatewayLoopLiveness:
|
|
def test_fresh_heartbeat_is_alive(self, tmp_path):
|
|
"""A gateway that refreshed its heartbeat recently is busy, not wedged."""
|
|
_write_heartbeat(tmp_path, pid=4242, age_s=5.0)
|
|
assert (
|
|
gateway_cli.probe_gateway_loop_liveness(4242, home=tmp_path)
|
|
== gateway_cli.GATEWAY_LOOP_ALIVE
|
|
)
|
|
|
|
def test_stale_heartbeat_is_wedged(self, tmp_path):
|
|
"""A heartbeat several missed beats old proves the loop is dead."""
|
|
_write_heartbeat(tmp_path, pid=4242, age_s=600.0)
|
|
assert (
|
|
gateway_cli.probe_gateway_loop_liveness(4242, home=tmp_path)
|
|
== gateway_cli.GATEWAY_LOOP_WEDGED
|
|
)
|
|
|
|
def test_heartbeat_just_inside_budget_is_alive(self, tmp_path):
|
|
"""Boundary: age below the stale budget must NOT classify as wedged."""
|
|
_write_heartbeat(tmp_path, pid=4242, age_s=60.0)
|
|
assert (
|
|
gateway_cli.probe_gateway_loop_liveness(
|
|
4242, stale_after=90.0, home=tmp_path
|
|
)
|
|
== gateway_cli.GATEWAY_LOOP_ALIVE
|
|
)
|
|
|
|
def test_missing_heartbeat_is_unknown(self, tmp_path):
|
|
"""No heartbeat file (older gateway, fresh start) is not evidence."""
|
|
assert (
|
|
gateway_cli.probe_gateway_loop_liveness(4242, home=tmp_path)
|
|
== gateway_cli.GATEWAY_LOOP_UNKNOWN
|
|
)
|
|
|
|
def test_pid_mismatch_is_unknown_even_when_stale(self, tmp_path):
|
|
"""A stale file from a PREVIOUS process must not condemn the new PID."""
|
|
_write_heartbeat(tmp_path, pid=1111, age_s=600.0)
|
|
assert (
|
|
gateway_cli.probe_gateway_loop_liveness(4242, home=tmp_path)
|
|
== gateway_cli.GATEWAY_LOOP_UNKNOWN
|
|
)
|
|
|
|
def test_corrupt_heartbeat_is_unknown(self, tmp_path):
|
|
path = get_loop_heartbeat_path(tmp_path)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text("{not json", encoding="utf-8")
|
|
stamp = time.time() - 600.0
|
|
os.utime(path, (stamp, stamp))
|
|
assert (
|
|
gateway_cli.probe_gateway_loop_liveness(4242, home=tmp_path)
|
|
== gateway_cli.GATEWAY_LOOP_UNKNOWN
|
|
)
|
|
|
|
def test_nonpositive_pid_is_unknown(self, tmp_path):
|
|
_write_heartbeat(tmp_path, pid=4242, age_s=600.0)
|
|
assert (
|
|
gateway_cli.probe_gateway_loop_liveness(0, home=tmp_path)
|
|
== gateway_cli.GATEWAY_LOOP_UNKNOWN
|
|
)
|
|
|
|
def test_probe_never_raises_on_unreadable_path(self, monkeypatch):
|
|
monkeypatch.setattr(
|
|
"gateway.shutdown_watchdog.get_loop_heartbeat_path",
|
|
lambda home=None: (_ for _ in ()).throw(OSError("boom")),
|
|
)
|
|
assert (
|
|
gateway_cli.probe_gateway_loop_liveness(4242)
|
|
== gateway_cli.GATEWAY_LOOP_UNKNOWN
|
|
)
|
|
|
|
class TestEscalateWedgedGateway:
|
|
def test_sigterm_grace_suffices_without_sigkill(self, monkeypatch):
|
|
"""If SIGTERM lands (signal-handler thread alive), no SIGKILL is sent."""
|
|
signals = []
|
|
monkeypatch.setattr(
|
|
gateway_cli,
|
|
"terminate_pid",
|
|
lambda pid, force=False, **kwargs: signals.append(("kill" if force else "term", pid)),
|
|
)
|
|
monkeypatch.setattr(
|
|
gateway_cli, "_wait_for_pid_exit", lambda pid, timeout, **_: True
|
|
)
|
|
|
|
assert gateway_cli._escalate_wedged_gateway(4242) is True
|
|
assert signals == [("term", 4242)]
|
|
|
|
def test_escalates_to_sigkill_when_sigterm_ignored(self, monkeypatch):
|
|
signals = []
|
|
waits = []
|
|
|
|
def fake_wait(pid, timeout):
|
|
waits.append(timeout)
|
|
# First wait (SIGTERM grace) times out; second (post-SIGKILL) succeeds.
|
|
return len(waits) > 1
|
|
|
|
monkeypatch.setattr(
|
|
gateway_cli,
|
|
"terminate_pid",
|
|
lambda pid, force=False, **kwargs: signals.append(("kill" if force else "term", pid)),
|
|
)
|
|
monkeypatch.setattr(gateway_cli, "_wait_for_pid_exit", fake_wait)
|
|
|
|
assert gateway_cli._escalate_wedged_gateway(4242) is True
|
|
assert signals == [("term", 4242), ("kill", 4242)]
|
|
|
|
def test_total_wait_budget_is_bounded_well_under_drain(self, monkeypatch):
|
|
"""Worst case must be seconds, never the 180s drain budget."""
|
|
waits = []
|
|
monkeypatch.setattr(gateway_cli, "terminate_pid", lambda pid, force=False, **kwargs: None)
|
|
monkeypatch.setattr(
|
|
gateway_cli,
|
|
"_wait_for_pid_exit",
|
|
lambda pid, timeout, **_: waits.append(timeout) or False,
|
|
)
|
|
|
|
assert gateway_cli._escalate_wedged_gateway(4242) is False
|
|
assert sum(waits) < 30.0
|
|
|
|
def test_process_already_gone_is_success(self, monkeypatch):
|
|
def raise_gone(pid, force=False, **kwargs):
|
|
raise ProcessLookupError
|
|
|
|
monkeypatch.setattr(gateway_cli, "terminate_pid", raise_gone)
|
|
monkeypatch.setattr(
|
|
gateway_cli, "_wait_for_pid_exit", lambda pid, timeout, **_: True
|
|
)
|
|
|
|
assert gateway_cli._escalate_wedged_gateway(4242) is True
|
|
|
|
def test_sigkill_permission_error_does_not_raise(self, monkeypatch):
|
|
calls = []
|
|
|
|
def term(pid, force=False, **kwargs):
|
|
calls.append(force)
|
|
if force:
|
|
raise PermissionError
|
|
|
|
monkeypatch.setattr(gateway_cli, "terminate_pid", term)
|
|
monkeypatch.setattr(
|
|
gateway_cli, "_wait_for_pid_exit", lambda pid, timeout, **_: False
|
|
)
|
|
|
|
assert gateway_cli._escalate_wedged_gateway(4242) is False
|
|
assert calls == [False, True]
|
|
|
|
class TestLaunchdRestartWedgedIntegration:
|
|
"""launchd_restart must skip the 180s drain only for a wedged loop."""
|
|
|
|
def _setup(self, monkeypatch, liveness):
|
|
events = []
|
|
monkeypatch.setattr(gateway_cli, "get_launchd_label", lambda: "ai.hermes.gateway")
|
|
monkeypatch.setattr(gateway_cli, "_launchd_domain", lambda: "gui/501")
|
|
monkeypatch.setattr(gateway_cli, "_get_restart_drain_timeout", lambda: 180.0)
|
|
# Wait budget covers after-turn deferral + drain + headroom (#77184).
|
|
monkeypatch.setattr(gateway_cli, "_get_restart_exit_wait_budget", lambda: 195.0)
|
|
monkeypatch.setattr("gateway.status.get_running_pid", lambda *a, **k: 4242)
|
|
monkeypatch.setattr(
|
|
gateway_cli, "_request_gateway_self_restart", lambda pid: False
|
|
)
|
|
monkeypatch.setattr(
|
|
gateway_cli,
|
|
"probe_gateway_loop_liveness",
|
|
lambda pid, **kw: events.append("probe") or liveness,
|
|
)
|
|
monkeypatch.setattr(
|
|
gateway_cli,
|
|
"_escalate_wedged_gateway",
|
|
lambda pid, **kw: events.append("escalate") or True,
|
|
)
|
|
monkeypatch.setattr(
|
|
gateway_cli,
|
|
"terminate_pid",
|
|
lambda pid, force=False, **kwargs: events.append("sigterm"),
|
|
)
|
|
# Never let a real SIGUSR1 escape to PID 4242 during tests.
|
|
monkeypatch.setattr(
|
|
gateway_cli,
|
|
"_graceful_restart_via_sigusr1",
|
|
lambda pid, timeout, **_: events.append(("drain", pid, timeout)) or True,
|
|
)
|
|
# KeepAlive revival observed instantly — avoids the real 15s poll
|
|
# (mocked subprocess.run returns empty stdout, so the PID probe
|
|
# would otherwise burn the full observation timeout in time.sleep).
|
|
monkeypatch.setattr(
|
|
gateway_cli,
|
|
"_wait_for_launchd_service_pid",
|
|
lambda label, old_pid, timeout=10.0, *, domain: events.append("observe")
|
|
or True,
|
|
)
|
|
monkeypatch.setattr(
|
|
gateway_cli.subprocess,
|
|
"run",
|
|
lambda *a, **k: events.append("kickstart")
|
|
or __import__("types").SimpleNamespace(returncode=0, stdout="", stderr=""),
|
|
)
|
|
monkeypatch.setattr(
|
|
gateway_cli, "_clear_launchd_unsupported_marker", lambda: None
|
|
)
|
|
return events
|
|
|
|
def test_wedged_gateway_skips_drain_and_escalates(self, monkeypatch):
|
|
events = self._setup(monkeypatch, gateway_cli.GATEWAY_LOOP_WEDGED)
|
|
gateway_cli.launchd_restart()
|
|
assert "escalate" in events
|
|
# The 180s drain wait must never run for a wedged loop.
|
|
assert not any(isinstance(e, tuple) and e[0] == "drain" for e in events)
|
|
|
|
def test_busy_gateway_keeps_full_drain_budget(self, monkeypatch):
|
|
"""A busy-but-alive gateway (fresh heartbeat) must NOT be escalated —
|
|
that would bypass the in-flight cron drain floor (#86684)."""
|
|
events = self._setup(monkeypatch, gateway_cli.GATEWAY_LOOP_ALIVE)
|
|
gateway_cli.launchd_restart()
|
|
assert "escalate" not in events
|
|
assert ("drain", 4242, 195.0) in events
|
|
|
|
def test_unknown_liveness_keeps_full_drain_budget(self, monkeypatch):
|
|
"""Ambiguity (no heartbeat) must never trigger escalation."""
|
|
events = self._setup(monkeypatch, gateway_cli.GATEWAY_LOOP_UNKNOWN)
|
|
gateway_cli.launchd_restart()
|
|
assert "escalate" not in events
|
|
assert ("drain", 4242, 195.0) in events
|
|
|
|
class TestLoopTickWitness:
|
|
"""Two-witness liveness (#90502 review).
|
|
|
|
The heartbeat write moved off-loop, so a stale file no longer proves a
|
|
wedged loop and a fresh file no longer proves an alive one. The loop
|
|
answers a UNIX socket instead; the probe only escalates when BOTH
|
|
witnesses agree the loop stopped scheduling.
|
|
"""
|
|
|
|
@_NEEDS_UNIX_SOCKETS
|
|
def test_off_loop_completion_cannot_manufacture_fresh_liveness(self, tmp_path):
|
|
"""A write landing after the loop froze must not look alive.
|
|
|
|
File fresh (a late off-loop write completed) + loop silent: the probe
|
|
must NOT return ALIVE — the loop itself never answered, so freshness
|
|
is not a liveness proof. UNKNOWN keeps the safe drain path while
|
|
denying the false-fresh window the review described.
|
|
"""
|
|
pid = 4242
|
|
_silent_socket_node(get_loop_tick_socket_path(tmp_path, pid))
|
|
_write_heartbeat(tmp_path, pid, age_s=5.0)
|
|
_mark_witness_flag(tmp_path, armed=True)
|
|
assert (
|
|
gateway_cli.probe_gateway_loop_liveness(
|
|
pid, home=tmp_path, tick_timeout=0.2
|
|
)
|
|
== gateway_cli.GATEWAY_LOOP_UNKNOWN
|
|
)
|
|
|
|
@_NEEDS_UNIX_SOCKETS
|
|
def test_true_wedge_requires_sustained_witness_silence(self, tmp_path):
|
|
"""Stale file + armed socket silent across the whole window: WEDGED.
|
|
|
|
The destructive path must still exist for genuinely dead loops — a
|
|
frozen loop stops answering the socket for every attempt in the
|
|
bounded window, and the file goes stale. The reviewer's sustained-
|
|
proof contract (one sample is never destructive authority) is
|
|
satisfied here: all ``tick_strikes`` consecutive probes miss.
|
|
"""
|
|
pid = 4242
|
|
_silent_socket_node(get_loop_tick_socket_path(tmp_path, pid))
|
|
_write_heartbeat(tmp_path, pid, age_s=600.0)
|
|
_mark_witness_flag(tmp_path, armed=True, age_s=600.0)
|
|
assert (
|
|
gateway_cli.probe_gateway_loop_liveness(
|
|
pid,
|
|
home=tmp_path,
|
|
tick_timeout=0.2,
|
|
tick_strikes=3,
|
|
tick_gap_s=0.0,
|
|
)
|
|
== gateway_cli.GATEWAY_LOOP_WEDGED
|
|
)
|
|
|
|
def test_single_silent_probe_is_not_destructive(self, tmp_path, monkeypatch):
|
|
"""One miss is a transient stall, never a wedge (#90502 review).
|
|
|
|
Stale file + armed socket + a loop that misses the FIRST probe but
|
|
answers the second: the probe must recover to ALIVE — the loop is
|
|
demonstrably dispatching, and the stale file is just a stalled
|
|
write. A single silent sample must never grant the bounded kill
|
|
authority.
|
|
"""
|
|
pid = 4242
|
|
calls = []
|
|
|
|
def flaky_probe(_pid, _home, timeout=1.0):
|
|
calls.append(timeout)
|
|
# First sample misses (transient synchronous stall), the loop
|
|
# answers on the next attempt.
|
|
return len(calls) > 1
|
|
|
|
monkeypatch.setattr(gateway_cli, "_probe_loop_tick_socket", flaky_probe)
|
|
_write_heartbeat(tmp_path, pid, age_s=600.0)
|
|
_mark_witness_flag(tmp_path, armed=True, age_s=600.0)
|
|
assert (
|
|
gateway_cli.probe_gateway_loop_liveness(
|
|
pid,
|
|
home=tmp_path,
|
|
tick_timeout=0.2,
|
|
tick_strikes=2,
|
|
tick_gap_s=0.0,
|
|
)
|
|
== gateway_cli.GATEWAY_LOOP_ALIVE
|
|
)
|
|
|
|
def test_witness_vanishing_mid_window_is_unknown(self, tmp_path, monkeypatch):
|
|
"""A witness that disappears mid-window is ambiguity, not a wedge.
|
|
|
|
First sample silent (node existed), second sample finds no node:
|
|
the witness is gone and cannot condemn the loop. UNKNOWN keeps the
|
|
graceful drain path.
|
|
"""
|
|
pid = 4242
|
|
calls = []
|
|
monkeypatch.setattr(
|
|
gateway_cli,
|
|
"_probe_loop_tick_socket",
|
|
lambda _pid, _home, timeout=1.0: (calls.append(timeout), False, None)[
|
|
len(calls)
|
|
],
|
|
)
|
|
_write_heartbeat(tmp_path, pid, age_s=600.0)
|
|
_mark_witness_flag(tmp_path, armed=True, age_s=600.0)
|
|
assert (
|
|
gateway_cli.probe_gateway_loop_liveness(
|
|
pid,
|
|
home=tmp_path,
|
|
tick_timeout=0.2,
|
|
tick_strikes=2,
|
|
tick_gap_s=0.0,
|
|
)
|
|
== gateway_cli.GATEWAY_LOOP_UNKNOWN
|
|
)
|
|
|
|
def test_armed_witness_unreachable_is_unknown(self, tmp_path):
|
|
"""Producer claims the witness is armed but no node exists: ambiguity.
|
|
|
|
Never kill on it — the graceful drain remains the backstop.
|
|
"""
|
|
pid = 4242
|
|
_write_heartbeat(tmp_path, pid, age_s=600.0)
|
|
_mark_witness_flag(tmp_path, armed=True, age_s=600.0)
|
|
assert (
|
|
gateway_cli.probe_gateway_loop_liveness(
|
|
pid, home=tmp_path, tick_timeout=0.2
|
|
)
|
|
== gateway_cli.GATEWAY_LOOP_UNKNOWN
|
|
)
|
|
|
|
def test_unarmed_witness_disables_stale_escalation(self, tmp_path):
|
|
"""New producer whose bind failed: staleness is NOT proof.
|
|
|
|
The write is off-loop, so the file can age while the loop runs; with
|
|
no witness, a stale file must never escalate.
|
|
"""
|
|
pid = 4242
|
|
_write_heartbeat(tmp_path, pid, age_s=600.0)
|
|
_mark_witness_flag(tmp_path, armed=False, age_s=600.0)
|
|
assert (
|
|
gateway_cli.probe_gateway_loop_liveness(
|
|
pid, home=tmp_path, tick_timeout=0.2
|
|
)
|
|
== gateway_cli.GATEWAY_LOOP_UNKNOWN
|
|
)
|
|
|
|
def test_legacy_payload_keeps_single_witness_contract(self, tmp_path):
|
|
"""No witness flag = on-loop writer: staleness stays proof.
|
|
|
|
Old gateways never moved the write off-loop, so their stale file
|
|
still means a dead loop — the legacy WEDGED verdict is unchanged.
|
|
"""
|
|
pid = 4242
|
|
_write_heartbeat(tmp_path, pid, age_s=600.0)
|
|
assert (
|
|
gateway_cli.probe_gateway_loop_liveness(pid, home=tmp_path)
|
|
== gateway_cli.GATEWAY_LOOP_WEDGED
|
|
)
|
|
|
|
@_NEEDS_UNIX_SOCKETS
|
|
def test_legacy_fresh_file_with_dead_node_is_unknown(self, tmp_path):
|
|
"""A fresh legacy file stays safe under a dead-listener node.
|
|
|
|
A dead-listener node for the PID (leftover from a newer process):
|
|
the silent socket denies ALIVE, and UNKNOWN never escalates — the
|
|
drain path keeps the full budget either way.
|
|
"""
|
|
pid = 4242
|
|
_write_heartbeat(tmp_path, pid, age_s=5.0)
|
|
_silent_socket_node(get_loop_tick_socket_path(tmp_path, pid))
|
|
assert (
|
|
gateway_cli.probe_gateway_loop_liveness(
|
|
pid, home=tmp_path, tick_timeout=0.2
|
|
)
|
|
== gateway_cli.GATEWAY_LOOP_UNKNOWN
|
|
)
|
|
|
|
@_NEEDS_UNIX_SOCKETS
|
|
@pytest.mark.asyncio
|
|
async def test_producer_rebinds_over_stale_socket_node(self, tmp_path):
|
|
"""A leftover node from a dead process must not disarm the witness.
|
|
|
|
os._exit(75) / SIGKILL skip the finally-unlink, and PID reuse then
|
|
re-lands on the same PID-suffixed path. Without the pre-bind unlink
|
|
the bind fails EADDRINUSE, the except disarms the witness
|
|
(loop_tick_socket:false), and a stale heartbeat can never classify
|
|
WEDGED — precisely on crash-restart loops.
|
|
"""
|
|
sock_path = get_loop_tick_socket_path(tmp_path)
|
|
_silent_socket_node(sock_path) # dead process's leftover node
|
|
assert sock_path.exists()
|
|
|
|
task = asyncio.create_task(
|
|
loop_heartbeat_forever(interval_s=0.2, home=tmp_path)
|
|
)
|
|
try:
|
|
for _ in range(50):
|
|
hb = get_loop_heartbeat_path(tmp_path)
|
|
if hb.exists():
|
|
payload = json.loads(hb.read_text(encoding="utf-8"))
|
|
if payload.get("loop_tick_socket"):
|
|
break
|
|
await asyncio.sleep(0.05)
|
|
else:
|
|
raise AssertionError(
|
|
"witness never armed over the stale socket node"
|
|
)
|
|
# And it actually answers: the node is live, not the leftover.
|
|
reader, writer = await asyncio.open_unix_connection(str(sock_path))
|
|
data = await asyncio.wait_for(reader.read(64), timeout=2)
|
|
writer.close()
|
|
assert data, "re-bound tick socket gave no answer"
|
|
finally:
|
|
task.cancel()
|
|
try:
|
|
await task
|
|
except asyncio.CancelledError:
|
|
pass
|
|
|
|
@_NEEDS_UNIX_SOCKETS
|
|
def test_transient_stall_below_wedge_budget_never_escalates(
|
|
self, tmp_path, monkeypatch
|
|
):
|
|
"""Composed producer+consumer: transient loop stall below the budget.
|
|
|
|
Heartbeat write stalled (file stale) + loop frozen for longer than
|
|
one recv timeout but shorter than the wedge window: the probe must
|
|
NOT return WEDGED — the loop answers once it unfreezes, inside the
|
|
bounded window — and the restart path fed by the real probe must
|
|
take the graceful drain, never the bounded escalation. This is the
|
|
reviewer's required composed regression (#90502 review).
|
|
"""
|
|
pid = os.getpid()
|
|
block_s = 0.5
|
|
stale_after = 1.0
|
|
tick_timeout = 0.2
|
|
tick_strikes = 3
|
|
tick_gap_s = 0.05
|
|
wedge_window = tick_strikes * tick_timeout + (tick_strikes - 1) * tick_gap_s
|
|
assert tick_timeout < block_s < wedge_window, (tick_timeout, block_s, wedge_window)
|
|
|
|
errors = []
|
|
state, ready = _start_freezeable_producer(tmp_path, block_s, errors)
|
|
try:
|
|
assert ready.wait(timeout=5.0)
|
|
sock_path = get_loop_tick_socket_path(tmp_path, pid)
|
|
deadline = time.monotonic() + 5.0
|
|
while not sock_path.exists() and time.monotonic() < deadline:
|
|
time.sleep(0.02)
|
|
assert sock_path.exists(), "producer never armed the tick socket"
|
|
_wait_heartbeat_stale(tmp_path, stale_after)
|
|
|
|
state["trigger"]() # freeze the loop for block_s
|
|
verdict = gateway_cli.probe_gateway_loop_liveness(
|
|
pid,
|
|
home=tmp_path,
|
|
stale_after=stale_after,
|
|
tick_timeout=tick_timeout,
|
|
tick_strikes=tick_strikes,
|
|
tick_gap_s=tick_gap_s,
|
|
)
|
|
assert verdict == gateway_cli.GATEWAY_LOOP_ALIVE, verdict
|
|
|
|
events = _launchd_harness(monkeypatch, tmp_path, pid)
|
|
gateway_cli.launchd_restart()
|
|
assert "escalate" not in events
|
|
drains = [e for e in events if isinstance(e, tuple) and e[0] == "drain"]
|
|
assert drains, events
|
|
finally:
|
|
state["stop"]()
|
|
state["thread"].join(timeout=5.0)
|
|
assert not errors, errors
|
|
|
|
@_NEEDS_UNIX_SOCKETS
|
|
def test_sustained_stop_above_wedge_budget_still_escalates(
|
|
self, tmp_path
|
|
):
|
|
"""Composed producer+consumer: sustained stop above the budget.
|
|
|
|
The bounded-proof contract must not neuter the destructive path: a
|
|
loop frozen for longer than the wedge window stays silent for every
|
|
probe attempt, so the probe still returns WEDGED. (The WEDGED →
|
|
bounded-escalation wiring on the restart paths is covered by
|
|
``TestLaunchdRestartWedgedIntegration``.)
|
|
"""
|
|
pid = os.getpid()
|
|
block_s = 1.2
|
|
stale_after = 1.0
|
|
tick_timeout = 0.2
|
|
tick_strikes = 3
|
|
tick_gap_s = 0.05
|
|
wedge_window = tick_strikes * tick_timeout + (tick_strikes - 1) * tick_gap_s
|
|
assert block_s > wedge_window, (block_s, wedge_window)
|
|
|
|
errors = []
|
|
state, ready = _start_freezeable_producer(tmp_path, block_s, errors)
|
|
try:
|
|
assert ready.wait(timeout=5.0)
|
|
sock_path = get_loop_tick_socket_path(tmp_path, pid)
|
|
deadline = time.monotonic() + 5.0
|
|
while not sock_path.exists() and time.monotonic() < deadline:
|
|
time.sleep(0.02)
|
|
assert sock_path.exists(), "producer never armed the tick socket"
|
|
_wait_heartbeat_stale(tmp_path, stale_after)
|
|
|
|
state["trigger"]() # freeze the loop for block_s
|
|
verdict = gateway_cli.probe_gateway_loop_liveness(
|
|
pid,
|
|
home=tmp_path,
|
|
stale_after=stale_after,
|
|
tick_timeout=tick_timeout,
|
|
tick_strikes=tick_strikes,
|
|
tick_gap_s=tick_gap_s,
|
|
)
|
|
assert verdict == gateway_cli.GATEWAY_LOOP_WEDGED, verdict
|
|
finally:
|
|
state["stop"]()
|
|
state["thread"].join(timeout=5.0)
|
|
assert not errors, errors
|
|
|
|
class TestLoopTickTcpWitness:
|
|
"""Non-POSIX arm: the producer publishes ``loop_tick_tcp_port`` and the
|
|
consumer probes 127.0.0.1:<port> instead of the AF_UNIX node. The
|
|
two-witness contract must hold identically over TCP."""
|
|
|
|
@staticmethod
|
|
def _tcp_answerer():
|
|
"""A loopback listener that answers b"1" — the armed, dispatching loop."""
|
|
srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
srv.bind(("127.0.0.1", 0))
|
|
srv.listen(8)
|
|
stop = threading.Event()
|
|
|
|
def serve():
|
|
srv.settimeout(0.1)
|
|
while not stop.is_set():
|
|
try:
|
|
conn, _ = srv.accept()
|
|
except socket.timeout:
|
|
continue
|
|
try:
|
|
conn.sendall(b"1")
|
|
finally:
|
|
conn.close()
|
|
|
|
thread = threading.Thread(target=serve, daemon=True)
|
|
thread.start()
|
|
return srv.getsockname()[1], stop, srv
|
|
|
|
@staticmethod
|
|
def _write_tcp_heartbeat(home, pid, port, age_s=0.0):
|
|
write_loop_heartbeat(
|
|
pid=pid,
|
|
home=home,
|
|
extra={"loop_tick_socket": True, "loop_tick_tcp_port": port},
|
|
)
|
|
if age_s:
|
|
path = get_loop_heartbeat_path(home)
|
|
stamp = time.time() - age_s
|
|
os.utime(path, (stamp, stamp))
|
|
|
|
def test_stale_file_with_answering_tcp_witness_is_alive(self, tmp_path):
|
|
"""#90502 shape over TCP: a stalled write must not kill a live loop."""
|
|
port, stop, srv = self._tcp_answerer()
|
|
try:
|
|
self._write_tcp_heartbeat(tmp_path, 4343, port, age_s=600.0)
|
|
assert (
|
|
gateway_cli.probe_gateway_loop_liveness(4343, home=tmp_path)
|
|
== gateway_cli.GATEWAY_LOOP_ALIVE
|
|
)
|
|
finally:
|
|
stop.set()
|
|
srv.close()
|
|
|
|
def test_stale_file_with_silent_tcp_witness_is_wedged(self, tmp_path):
|
|
"""Armed TCP witness that never answers across the window: WEDGED."""
|
|
silent = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
silent.bind(("127.0.0.1", 0))
|
|
silent.listen(1) # accepts but never sends
|
|
try:
|
|
port = silent.getsockname()[1]
|
|
self._write_tcp_heartbeat(tmp_path, 4344, port, age_s=600.0)
|
|
assert (
|
|
gateway_cli.probe_gateway_loop_liveness(
|
|
4344, home=tmp_path, tick_timeout=0.2, tick_gap_s=0.05
|
|
)
|
|
== gateway_cli.GATEWAY_LOOP_WEDGED
|
|
)
|
|
finally:
|
|
silent.close()
|
|
|
|
def test_fresh_file_with_silent_tcp_witness_is_unknown(self, tmp_path):
|
|
"""Fresh file + silent TCP witness: an off-loop write landed after a
|
|
freeze — not proof of liveness, never destructive authority."""
|
|
silent = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
silent.bind(("127.0.0.1", 0))
|
|
silent.listen(1)
|
|
try:
|
|
port = silent.getsockname()[1]
|
|
self._write_tcp_heartbeat(tmp_path, 4345, port)
|
|
assert (
|
|
gateway_cli.probe_gateway_loop_liveness(
|
|
4345, home=tmp_path, tick_timeout=0.2
|
|
)
|
|
== gateway_cli.GATEWAY_LOOP_UNKNOWN
|
|
)
|
|
finally:
|
|
silent.close()
|
|
|
|
def test_garbage_tcp_port_falls_back_to_socket_contract(self, tmp_path):
|
|
"""A non-numeric port must not be treated as an armed witness."""
|
|
write_loop_heartbeat(
|
|
pid=4346,
|
|
home=tmp_path,
|
|
extra={"loop_tick_socket": False, "loop_tick_tcp_port": "nope"},
|
|
)
|
|
path = get_loop_heartbeat_path(tmp_path)
|
|
stamp = time.time() - 600.0
|
|
os.utime(path, (stamp, stamp))
|
|
# loop_tick_socket=False + no usable TCP port: witness could not be
|
|
# armed, staleness is not proof -> UNKNOWN, never WEDGED.
|
|
assert (
|
|
gateway_cli.probe_gateway_loop_liveness(4346, home=tmp_path)
|
|
== gateway_cli.GATEWAY_LOOP_UNKNOWN
|
|
)
|