hermes-agent/tests/agent/test_turn_liveness_watchdog.py
ethernet 890bbbda1f Merge remote-tracking branch 'origin/main' into ethie/pm-clean
# Conflicts:
#	apps/desktop/e2e/archived-hidden-session-recoverable.spec.ts
#	apps/desktop/e2e/bot-chat-message-agent-friendly-name.spec.ts
#	apps/desktop/e2e/bot-mailbox-unreadable-ticket.spec.ts
#	apps/desktop/e2e/bot-mode-roster-localized.spec.ts
#	apps/desktop/e2e/bot-mode-row-click-mirrors-registry.spec.ts
#	apps/desktop/e2e/bot-mode-tab-shows-bot-name.spec.ts
#	apps/desktop/e2e/bot-roster-group-row-organisation.spec.ts
#	apps/desktop/e2e/bot-roster-ignores-infra-dirs.spec.ts
#	apps/desktop/e2e/bot-roster-timestamp-meta.spec.ts
#	apps/desktop/e2e/bot-roster-user-sections.spec.ts
#	apps/desktop/e2e/bot-routines-pane-narrow.spec.ts
#	apps/desktop/e2e/bot-row-open-recent-session.spec.ts
#	apps/desktop/e2e/bot-tile-ignores-ambient-composer-model.spec.ts
#	apps/desktop/e2e/group-composer-auto-grow.spec.ts
#	apps/desktop/e2e/group-create-gate-remote-roster.spec.ts
#	apps/desktop/e2e/group-prompt-renamed-primary-handle.spec.ts
#	apps/desktop/e2e/hosted-room-backend-continuity.spec.ts
#	apps/desktop/e2e/hosted-room-legacy-store-migration.spec.ts
#	apps/desktop/e2e/settings-scope-chips-bot-title.spec.ts
#	apps/desktop/e2e/worktree-branch-status.spec.ts
#	apps/desktop/electron/backend-probes.test.ts
#	apps/desktop/electron/connection-apply.test.ts
#	apps/desktop/electron/desktop-electron-pin.test.ts
#	apps/desktop/electron/desktop-uninstall.test.ts
#	apps/desktop/electron/gateway-file-download-transport.test.ts
#	apps/desktop/electron/gateway-stop-before-update.test.ts
#	apps/desktop/electron/github-api-auth.test.ts
#	apps/desktop/electron/registry-primary-profile-scope.test.ts
#	apps/desktop/electron/update-api-check.test.ts
#	apps/desktop/electron/update-handoff-marker.test.ts
#	apps/desktop/electron/venv-blocker-scan.test.ts
#	apps/desktop/scripts/after-extract.test.mjs
#	apps/desktop/scripts/local-pack-publish.test.mjs
#	apps/desktop/scripts/tasks-scroll.test.mjs
#	apps/desktop/src/app/settings/model-settings.test.tsx
#	apps/desktop/src/app/updates-overlay.blockers.test.tsx
#	apps/desktop/src/components/desktop-install-overlay.test.tsx
#	apps/desktop/src/lib/update-copy.test.ts
#	scripts/ci/check_os_marker_fakes.py
#	tests-js/desktop-mac-usage-descriptions.test.ts
#	tests-js/node-engine-alignment.test.ts
#	tests/agent/lsp/test_install_and_lint_fixes.py
#	tests/agent/test_command_token_source.py
#	tests/agent/test_compression_boundary_hook.py
#	tests/agent/test_create_openai_client_ssl_verify.py
#	tests/agent/test_custom_provider_ca_probes.py
#	tests/agent/test_endpoint_blackhole.py
#	tests/agent/test_estimator_parity.py
#	tests/agent/test_in_place_compaction.py
#	tests/agent/test_moa_loop_mode.py
#	tests/agent/test_model_metadata.py
#	tests/agent/test_skill_session_platform_gate.py
#	tests/agent/test_skill_utils.py
#	tests/agent/test_ssl_ca_guard.py
#	tests/computer_use/test_doctor.py
#	tests/cron/test_codex_execution_paths.py
#	tests/cron/test_cron_bot_chat_delivery.py
#	tests/cron/test_cron_script.py
#	tests/cron/test_media_delivery_parity.py
#	tests/cron/test_misfire_catchup.py
#	tests/cron/test_parallel_pool.py
#	tests/cron/test_recurring_eagain_redispatch.py
#	tests/gateway/test_choice_picker.py
#	tests/gateway/test_control_socket_windows_live.py
#	tests/gateway/test_dingtalk.py
#	tests/gateway/test_feishu.py
#	tests/gateway/test_feishu_onboard.py
#	tests/gateway/test_gateway_shutdown.py
#	tests/gateway/test_matrix.py
#	tests/gateway/test_model_command_custom_providers.py
#	tests/gateway/test_reasoning_command.py
#	tests/gateway/test_runtime_footer.py
#	tests/gateway/test_session.py
#	tests/gateway/test_session_hygiene.py
#	tests/gateway/test_status.py
#	tests/gateway/test_teams.py
#	tests/gateway/test_turn_lease.py
#	tests/gateway/test_whatsapp_connect.py
#	tests/hermes_cli/test_approvals_command.py
#	tests/hermes_cli/test_auth_store_lock_concurrent.py
#	tests/hermes_cli/test_backup.py
#	tests/hermes_cli/test_banner_git_state.py
#	tests/hermes_cli/test_certifi_repair.py
#	tests/hermes_cli/test_cmd_update.py
#	tests/hermes_cli/test_compat_manifest_targets.py
#	tests/hermes_cli/test_computer_use_cli.py
#	tests/hermes_cli/test_cpr_local_leak.py
#	tests/hermes_cli/test_dashboard_auth_gate.py
#	tests/hermes_cli/test_dashboard_procs_kill_grace.py
#	tests/hermes_cli/test_desktop_lifecycle_windows_live.py
#	tests/hermes_cli/test_doctor.py
#	tests/hermes_cli/test_doctor_command_install.py
#	tests/hermes_cli/test_fleet_config_migration_windows_live.py
#	tests/hermes_cli/test_gateway.py
#	tests/hermes_cli/test_gateway_platform_gating.py
#	tests/hermes_cli/test_gateway_restart_loop.py
#	tests/hermes_cli/test_gateway_task_probe.py
#	tests/hermes_cli/test_gateway_wsl.py
#	tests/hermes_cli/test_gui_command.py
#	tests/hermes_cli/test_install_cua_driver.py
#	tests/hermes_cli/test_kanban_db.py
#	tests/hermes_cli/test_lazy_command_exports.py
#	tests/hermes_cli/test_lazy_refresh_venv_repair.py
#	tests/hermes_cli/test_linux_desktop_entry.py
#	tests/hermes_cli/test_local_runtime.py
#	tests/hermes_cli/test_local_runtime_updates.py
#	tests/hermes_cli/test_managed_uv.py
#	tests/hermes_cli/test_mcp_reload_confirm_gate.py
#	tests/hermes_cli/test_nous_subscription.py
#	tests/hermes_cli/test_npm_engine.py
#	tests/hermes_cli/test_personality_none.py
#	tests/hermes_cli/test_pet_toggle.py
#	tests/hermes_cli/test_plan_reconciliation_windows_live.py
#	tests/hermes_cli/test_plugin_event_bus.py
#	tests/hermes_cli/test_plugin_manifest_v2.py
#	tests/hermes_cli/test_plugin_packs.py
#	tests/hermes_cli/test_plugins_cmd.py
#	tests/hermes_cli/test_plugins_cmd_enable_disable_nested.py
#	tests/hermes_cli/test_process_identity.py
#	tests/hermes_cli/test_profiles.py
#	tests/hermes_cli/test_profiles_sidebar_cache.py
#	tests/hermes_cli/test_pty_bridge.py
#	tests/hermes_cli/test_resolve_turn_limit.py
#	tests/hermes_cli/test_serve_runtime_inventory.py
#	tests/hermes_cli/test_session_vacuum_config.py
#	tests/hermes_cli/test_set_config_value.py
#	tests/hermes_cli/test_signal_handler_kanban_worker.py
#	tests/hermes_cli/test_slash_confirm_windows.py
#	tests/hermes_cli/test_stale_pid_guard.py
#	tests/hermes_cli/test_startup_fast_guards.py
#	tests/hermes_cli/test_status.py
#	tests/hermes_cli/test_telegram_managed_bot.py
#	tests/hermes_cli/test_tools_config.py
#	tests/hermes_cli/test_update_apply_shallow_count.py
#	tests/hermes_cli/test_update_autostash.py
#	tests/hermes_cli/test_update_concurrent_quarantine.py
#	tests/hermes_cli/test_update_fetch_failure_classifier.py
#	tests/hermes_cli/test_update_fleet_probe_resume_token.py
#	tests/hermes_cli/test_update_handoff_backend_reap.py
#	tests/hermes_cli/test_update_handoff_desktop_rebuild.py
#	tests/hermes_cli/test_update_head_moved_gate.py
#	tests/hermes_cli/test_update_host_obligation.py
#	tests/hermes_cli/test_update_import_guard.py
#	tests/hermes_cli/test_update_interrupted_recovery.py
#	tests/hermes_cli/test_update_inventory.py
#	tests/hermes_cli/test_update_launchd_unloaded_gateway.py
#	tests/hermes_cli/test_update_missing_configured_deps.py
#	tests/hermes_cli/test_update_modified_notice.py
#	tests/hermes_cli/test_update_multiplex_migration_hook.py
#	tests/hermes_cli/test_update_no_gateway_restart.py
#	tests/hermes_cli/test_update_orphan_backend_reap.py
#	tests/hermes_cli/test_update_parked_branch_guard.py
#	tests/hermes_cli/test_update_post_pull_syntax_guard.py
#	tests/hermes_cli/test_update_receipt.py
#	tests/hermes_cli/test_update_self_lock.py
#	tests/hermes_cli/test_update_shim_fail_closed.py
#	tests/hermes_cli/test_update_shim_self_lock.py
#	tests/hermes_cli/test_update_sqlite_remediation.py
#	tests/hermes_cli/test_update_stale_dashboard.py
#	tests/hermes_cli/test_update_stale_virtualenv.py
#	tests/hermes_cli/test_update_venv_health.py
#	tests/hermes_cli/test_update_venv_ownership_preflight.py
#	tests/hermes_cli/test_update_wedged_gateway.py
#	tests/hermes_cli/test_update_yes_flag.py
#	tests/hermes_cli/test_update_zip_two_phase.py
#	tests/hermes_cli/test_urllib_security.py
#	tests/hermes_cli/test_ux_messages_auth_config.py
#	tests/hermes_cli/test_ux_messages_startup.py
#	tests/hermes_cli/test_venv_holder_classifier.py
#	tests/hermes_cli/test_verify_console_scripts.py
#	tests/hermes_cli/test_verify_core_dependencies.py
#	tests/hermes_cli/test_web_server.py
#	tests/hermes_cli/test_web_server_console_ws.py
#	tests/hermes_cli/test_web_server_ws_ping.py
#	tests/hermes_cli/test_web_ui_build.py
#	tests/hermes_state/test_fts_rebuild_admission.py
#	tests/hermes_state/test_hermes_state.py
#	tests/plugins/memory/test_memory_lazy_install.py
#	tests/plugins/test_google_meet_plugin.py
#	tests/plugins/test_langfuse_plugin.py
#	tests/plugins/test_security_guidance_plugin.py
#	tests/plugins/test_transform_llm_output_hook.py
#	tests/scripts/desktop_update/test_desktop_update_windows_gateway_flag.py
#	tests/scripts/desktop_update/test_desktop_update_windows_python_handoff.py
#	tests/scripts/desktop_update/test_desktop_update_windows_timestamp.py
#	tests/scripts/install/test_install_clone_throttle_fallback.py
#	tests/scripts/install/test_install_lockfile_churn.py
#	tests/scripts/install/test_install_no_initial_commit.py
#	tests/scripts/install/test_install_sh_browser_install.py
#	tests/scripts/install/test_install_sh_node_prerelease.py
#	tests/scripts/install/test_install_sh_symlink_stomp.py
#	tests/scripts/install/test_install_sh_uv_lock_config.py
#	tests/scripts/install/test_install_unmerged_index.py
#	tests/scripts/test_contributor_map.py
#	tests/scripts/test_run_tests_parallel.py
#	tests/skills/test_competitor_news_monitor_skill.py
#	tests/skills/test_document_to_action_items_skill.py
#	tests/skills/test_google_workspace_setup.py
#	tests/skills/test_google_workspace_setup_deps.py
#	tests/skills/test_grounded_citations_skill.py
#	tests/skills/test_ip_as_logo_skill.py
#	tests/skills/test_live_dashboard_skill.py
#	tests/skills/test_mcp_oauth_remote_gateway_skill.py
#	tests/skills/test_office_document_skills.py
#	tests/skills/test_openclaw_migration.py
#	tests/skills/test_product_price_monitor_skill.py
#	tests/skills/test_scrollcraft_skill.py
#	tests/skills/test_setup_wizard_generator_skill.py
#	tests/skills/test_weekly_review_planning_skill.py
#	tests/test_engines_satisfiable.py
#	tests/test_fast_safe_load.py
#	tests/test_hermes_bootstrap.py
#	tests/test_hermes_constants.py
#	tests/test_hermes_logging.py
#	tests/test_managed_runtime_resolution.py
#	tests/test_model_tools_async_bridge.py
#	tests/test_packaging_build_guard.py
#	tests/test_packaging_metadata.py
#	tests/test_yaml_indent_consistency.py
#	tests/tools/test_approval_timeout_overflow.py
#	tests/tools/test_base_environment.py
#	tests/tools/test_bot_mode_dm.py
#	tests/tools/test_browser_chromium_check.py
#	tests/tools/test_browser_hardening.py
#	tests/tools/test_browser_homebrew_paths.py
#	tests/tools/test_browser_npx_warmup.py
#	tests/tools/test_browser_orphan_reaper.py
#	tests/tools/test_browser_real_profile.py
#	tests/tools/test_browser_use_cli.py
#	tests/tools/test_clipboard.py
#	tests/tools/test_code_execution.py
#	tests/tools/test_code_execution_modes.py
#	tests/tools/test_code_execution_windows_env.py
#	tests/tools/test_computer_use.py
#	tests/tools/test_delegate_liveness_timeout.py
#	tests/tools/test_execute_code_approval_cluster.py
#	tests/tools/test_execution_flag_detection.py
#	tests/tools/test_fal_common.py
#	tests/tools/test_file_operations.py
#	tests/tools/test_file_tools.py
#	tests/tools/test_file_tools_cwd_resolution.py
#	tests/tools/test_file_tools_live.py
#	tests/tools/test_lazy_deps.py
#	tests/tools/test_lazy_deps_durable_target.py
#	tests/tools/test_lazy_deps_managed.py
#	tests/tools/test_local_env_blocklist.py
#	tests/tools/test_local_tempdir.py
#	tests/tools/test_macos_protected_search.py
#	tests/tools/test_mcp_npx_cached_bin.py
#	tests/tools/test_oneshot_completion_linger.py
#	tests/tools/test_process_registry.py
#	tests/tools/test_read_file_schema_gating.py
#	tests/tools/test_skill_improvements.py
#	tests/tools/test_skills_sync.py
#	tests/tools/test_termux_api_detection.py
#	tests/tools/test_tirith_security.py
#	tests/tools/test_transcription_tools.py
#	tests/tools/test_tts_streaming.py
#	tests/tools/test_wake_word.py
#	tests/tui_gateway/test_compute_host_borrowed_lease.py
#	tests/tui_gateway/test_compute_host_turn_protocol.py
#	tests/tui_gateway/test_isolated_orphan_activity.py
#	tests/tui_gateway/test_protocol.py
#	tests/tui_gateway/test_slash_worker_profile_home.py
#	tests/tui_gateway/test_subprocess_encoding.py
#	tests/tui_gateway/test_tui_gateway_server.py
#	ui-tui/src/__tests__/terminalParity.test.ts
#	ui-tui/src/__tests__/termuxComposerLayout.test.ts
#	ui-tui/src/__tests__/textInputFastEcho.test.ts
2026-09-23 07:02:44 -04:00

755 lines
31 KiB
Python

"""Turn liveness watchdog (#95548): force-abort turns that stall silently.
Regression coverage for the gateway turn hang reported in #95548: a turn can
stall in the middle (observed between "model returned tool_calls" and tool
execution) with no error logged, no further progress, and the durable turn
lease kept renewing — so nothing ever force-aborts it and the session is
stuck until the gateway process is killed.
The fix adds a turn liveness watchdog next to the durable lease refresher in
``AIAgent.run_conversation`` (policy lives in ``agent/turn_liveness.py``).
It keys off the agent's activity clock (``_last_activity_ts`` — the #72039
single progress source, which lease renewal never touches). When a turn
shows no observable progress for the configured bound
(``agent.turn_liveness.timeout_s`` in config.yaml), it:
1. logs the stall loudly (surface instead of silent blocking),
2. force-interrupts the turn so it unwinds as an interrupted turn,
3. stops lease renewal so the durable lease lapses and stale-turn cleanup
can reclaim the session even if the hard interrupt cannot unwind a
truly wedged loop.
The abort is bound to the sampled ``(activity_generation, timestamp)`` pair
and revalidated at the commit point under the lock shared with
``_touch_activity`` (#95663 review): a turn that resumes while the stall is
being surfaced continues running and its lease keeps renewing.
"""
from __future__ import annotations
import logging
import threading
import time
import pytest
from run_agent import AIAgent
class _DB:
def __init__(self, session_exists=True, acquire_result=True):
self.events = []
self.refresh_times = []
self.session_exists = session_exists
self.acquire_result = acquire_result
def get_session(self, session_id):
return {"id": session_id} if self.session_exists else None
def acquire_session_turn_lease(self, session_id, holder, **kwargs):
self.events.append(("acquire", session_id, holder))
on_wait = kwargs.get("on_wait")
if on_wait is not None and self.acquire_result is False:
on_wait(0.0)
return self.acquire_result
def resolve_resume_session_id(self, session_id):
self.events.append(("resolve", session_id))
return session_id
def get_messages_as_conversation(self, session_id, **kwargs):
self.events.append(("reload", session_id, kwargs))
return [{"role": "user", "content": "durable latest"}]
def refresh_session_turn_lease(self, session_id, holder, **kwargs):
self.events.append(("refresh", session_id, holder))
self.refresh_times.append(time.time())
return True
def release_session_turn_lease(self, session_id, holder):
self.events.append(("release", session_id, holder))
class _BlockingCommitFence:
"""Controllable compression commit fence for the stall-abort witness.
``cancel_before_commit`` parks the interrupt thread AFTER it has passed
the internal ``require_generation`` comparison, simulating the unbounded
compression-commit wait. The test resumes the turn (``_touch_activity``)
while the hammer is parked and then releases the fence; a correct
interrupt must abandon itself without publishing anything.
"""
def __init__(self):
self.entered = threading.Event()
self.release = threading.Event()
self.calls = 0
@property
def commit_in_flight(self) -> bool:
# Mirrors the production fence's lock-free phase marker; this
# double models an IN-FLIGHT commit, so the interrupt's pre-claim
# wait parks inside cancel_before_commit exactly as against a real
# started commit (the production started-commit branch blocks until
# finish_commit WITHOUT cancelling).
return True
def cancel_before_commit(self, cancel_event=None):
# `cancel_event` is accepted (and ignored) to mirror the production
# fence signature; publication happens at the final claim edge, so
# nothing observable is set here.
self.calls += 1
self.entered.set()
assert self.release.wait(10.0), "fence was never released"
return True
class _ParkingReleaseLock:
"""Activity-lock wrapper that parks the releasing thread at one exact
release boundary, so a test can deterministically land real activity
on a competing thread in the precise window under test (#95663
round-6 review: the consume→publication boundary).
``park_on_release=N`` parks the thread that performs the Nth release
AFTER the inner lock has actually been released — so the competing
thread can immediately acquire and stamp the clock while the parked
thread has not yet executed its next statement.
"""
def __init__(self, inner):
self._inner = inner
self.park_on_release = None
self.release_count = 0
self.parked = threading.Event()
self.release_park = threading.Event()
def __enter__(self):
return self._inner.__enter__()
def __exit__(self, *exc_info):
result = self._inner.__exit__(*exc_info)
self.release_count += 1
if (
self.park_on_release is not None
and self.release_count == self.park_on_release
):
self.parked.set()
assert self.release_park.wait(10.0), (
"parked activity-lock release was never released"
)
return result
def _agent_with_db(db, *, session_id="stalled-session", platform="desktop"):
agent = AIAgent.__new__(AIAgent)
agent.session_id = session_id
agent.platform = platform
agent.model = "test-model"
agent._session_db = db
agent._session_db_created = True
agent._persist_disabled = False
agent._parent_session_id = None
agent._relay_pending_turn_id = None
agent._reset_activity_labels_after_turn = lambda: None
agent._conversation_root_id = lambda: session_id
agent.log_prefix = ""
agent._vprint = lambda *a, **k: None
agent.status_callback = None
agent._interrupt_requested = False
agent._interrupt_message = None
agent._pending_redirect = None
agent._execution_thread_id = None
agent._interrupt_thread_signal_pending = False
agent._hard_interrupt_requested = threading.Event()
agent._active_children_lock = threading.Lock()
agent._active_children = set()
agent.quiet_mode = True
# A real cached agent entering a new turn holds the activity clock
# from its PREVIOUS turn: `_reset_activity_labels_after_turn` keeps
# `_last_activity_ts` across turns by design, so an agent that sat
# idle longer than the watchdog bound (user walked away, came back,
# sent a message) enters with a STALE clock. `AIAgent.run_conversation`
# stamps the clock at turn entry (#95663 review), so the watchdog
# measures idle from THIS turn's start — mirror that reality: stale
# entry clock, fresh measurement after the wrapper's turn-entry stamp.
agent._last_activity_ts = time.time() - 1000.0
agent._last_activity_desc = "previous turn (idle)"
agent._session_turn_lease_refresh_interval = 60.0
return agent
@pytest.fixture
def watchdog_config(monkeypatch):
"""Arm the watchdog fast through config.yaml — the only supported surface.
`agent.turn_liveness` is the config authority the watchdog resolves
(AGENTS.md rejects new non-secret HERMES_* env knobs); the resolver in
agent/turn_liveness.py validates the values and the env is never read.
"""
import hermes_cli.config as config_module
monkeypatch.setattr(
config_module,
"load_config_readonly",
lambda: {
"agent": {
"turn_liveness": {"timeout_s": 0.3, "poll_s": 0.05},
}
},
)
return monkeypatch
def _run_turn(agent, inner_loop, monkeypatch):
"""Drive AIAgent.run_conversation with a fake inner conversation loop."""
from agent import conversation_loop as loop_module
monkeypatch.setattr(loop_module, "run_conversation", inner_loop)
return AIAgent.run_conversation(
agent,
"new message",
conversation_history=[{"role": "user", "content": "stale"}],
)
def test_watchdog_force_aborts_silently_stalled_turn(watchdog_config, monkeypatch, caplog):
"""A turn with zero observable progress past the bound is surfaced and
force-aborted as an interrupted turn instead of hanging forever."""
db = _DB()
agent = _agent_with_db(db)
interrupt_seen = {}
def stalled_loop(_agent, _message, _system, history, *_args, **_kwargs):
# Simulate the #95548 zombie: the loop makes no progress and never
# touches the activity clock. It only notices the watchdog's
# hard interrupt (real wedges may not even do that — see the lease
# test below).
assert _agent._hard_interrupt_requested.wait(30.0), (
"watchdog did not publish a hard interrupt"
)
interrupt_seen["message"] = _agent._interrupt_message
return {
"final_response": "aborted",
"messages": history,
"api_calls": 0,
"completed": False,
"interrupted": True,
}
with caplog.at_level(logging.ERROR, logger="agent.turn_liveness"):
result = _run_turn(agent, stalled_loop, monkeypatch)
# The turn was surfaced as interrupted, not hung.
assert result["interrupted"] is True
assert result["final_response"] == "aborted"
# The stall was logged loudly with the session named.
assert any(
"Turn liveness watchdog fired" in record.getMessage()
and "stalled-session" in record.getMessage()
for record in caplog.records
)
# The interrupt message tells the UI why the turn ended.
assert agent._interrupt_message is None # cleared by the wrapper's finally
assert "no progress" in (interrupt_seen.get("message") or "") # watchdog fired
# The durable lease was released on the interrupted exit path.
assert db.events[-1][0] == "release"
assert db.events[-1][1] == "stalled-session"
def test_watchdog_does_not_fire_while_turn_still_making_progress(
watchdog_config, monkeypatch, caplog
):
"""A turn that keeps touching the activity clock (API waits, stream
tokens, tool heartbeats, tool completions) is never force-aborted, and
the lease keeps renewing normally."""
db = _DB()
agent = _agent_with_db(db)
# Fast lease refresh so the test can prove renewal stayed alive.
agent._session_turn_lease_refresh_interval = 0.05
t_start = time.time()
def busy_loop(_agent, _message, _system, history, *_args, **_kwargs):
# Keep making progress well past the 0.3s idle bound.
while time.time() - t_start < 0.6:
_agent._touch_activity("test tick")
time.sleep(0.02)
return {
"final_response": "done",
"messages": history,
"api_calls": 1,
"completed": True,
}
with caplog.at_level(logging.ERROR, logger="agent.turn_liveness"):
result = _run_turn(agent, busy_loop, monkeypatch)
assert result["completed"] is True
assert result.get("interrupted") is not True
assert agent._interrupt_requested is False
assert not any(
"Turn liveness watchdog fired" in record.getMessage()
for record in caplog.records
)
# The lease refresher ran during the turn — renewal is orthogonal to the
# watchdog and continued while the turn was alive.
assert len(db.refresh_times) >= 1
def test_watchdog_stops_lease_renewal_when_interrupt_cannot_unwind_wedge(
watchdog_config, monkeypatch
):
"""The issue's 'lease keeps renewing' masking: even when the hard
interrupt cannot immediately unwind the loop (a truly wedged frame), the
watchdog stops renewing the durable lease so TTL expiry lets stale-turn
cleanup reclaim the session."""
db = _DB()
agent = _agent_with_db(db)
agent._session_turn_lease_refresh_interval = 0.05
t_start = time.time()
fire_ts = {}
def wedged_loop(_agent, _message, _system, history, *_args, **_kwargs):
# Notice the interrupt but keep "wedging" (no activity) for another
# half second — simulating a blocked frame the interrupt cannot free
# immediately.
while not _agent._interrupt_requested:
if time.time() - t_start > 10:
break
time.sleep(0.005)
fire_ts["at"] = time.time()
while time.time() - fire_ts["at"] < 0.5:
time.sleep(0.005)
return {
"final_response": "recovered",
"messages": history,
"api_calls": 0,
"completed": True,
}
result = _run_turn(agent, wedged_loop, monkeypatch)
assert result["completed"] is True
assert time.time() - t_start < 10.0
# The watchdog fired during the wedge.
assert "at" in fire_ts
# Once the watchdog fired, lease renewal stopped: refreshes cadence at
# 0.05s would have produced ~8 more events in the 0.5s post-fire wedge if
# renewal had continued. Tolerate only in-flight refreshes racing the
# stop (<= 0.15s window).
late_refreshes = [
t for t in db.refresh_times if t > fire_ts["at"] + 0.15
]
assert late_refreshes == [], (
f"lease renewal continued after watchdog fired: {len(late_refreshes)} "
"post-fire refreshes"
)
# The lease row was still released when the turn finally unwound.
assert [e[0] for e in db.events][-1] == "release"
def test_watchdog_declines_abort_when_activity_resumes_during_warning(
watchdog_config, monkeypatch, caplog
):
"""#95663 review race: the watchdog sampled a stale activity clock, then
the turn resumed while the stall was being logged/emitted — before the
abort commits. The commit point revalidates the observed
(generation, timestamp) pair under the lock shared with
`_touch_activity`, so the turn continues and its lease keeps renewing
instead of being hard-cancelled mid-recovery."""
db = _DB()
agent = _agent_with_db(db)
agent._session_turn_lease_refresh_interval = 0.05
resume_event = threading.Event()
touched_event = threading.Event()
t_start = time.time()
interrupt_seen = {}
def stalled_then_resumed_loop(_agent, _message, _system, history, *_a, **_k):
# Stall (zero activity) until the watchdog surfaces the stall. The
# warning delivery itself is what unblocks the wedge: the turn
# resumes DURING the warning window, before the commit point.
assert resume_event.wait(10.0)
_agent._touch_activity("turn resumed")
touched_event.set()
# Keep the turn alive a while so lease renewal is observable.
while time.time() - t_start < 0.8:
_agent._touch_activity("still alive")
time.sleep(0.02)
# Capture DURING the turn — the wrapper's finally clears any
# interrupt after the loop returns, so post-run assertions on the
# agent's interrupt state cannot prove the abort never happened.
interrupt_seen["requested"] = _agent._interrupt_requested
interrupt_seen["message"] = _agent._interrupt_message
return {
"final_response": "recovered",
"messages": history,
"api_calls": 1,
"completed": True,
}
def blocking_warning(_message):
# Mirrors _emit_warning: the stall warning is being delivered when
# the turn resumes. Hold the window open until the resumed activity
# has definitively landed on the clock, so the commit revalidation
# runs against the new generation.
resume_event.set()
assert touched_event.wait(10.0)
agent._emit_warning = blocking_warning
with caplog.at_level(logging.ERROR, logger="agent.turn_liveness"):
result = _run_turn(agent, stalled_then_resumed_loop, monkeypatch)
assert time.time() - t_start < 10.0
# The turn resumed and completed normally — it was NOT hard-cancelled
# mid-flight (the commit point declined the stale observation).
assert result["completed"] is True
assert result.get("interrupted") is not True
assert interrupt_seen.get("requested") is False, (
f"turn was hard-interrupted while resumed: {interrupt_seen!r}"
)
# The stall was still surfaced loudly (so the regression isn't a
# vacuous pass from the watchdog never sampling)…
assert any(
"Turn liveness watchdog fired" in record.getMessage()
and "stalled-session" in record.getMessage()
for record in caplog.records
)
# …but the surface was OBSERVATIONAL only: the declined abort must
# never publish a committed-abort or lease-stop settlement
# (#95663 review — false settlement before commit veto). On the
# pre-fix tree the pre-commit surface itself logged the definitive
# "Force-aborting … stopping lease renewal" outcome — this assertion
# is what made that witness red.
assert not any(
"watchdog aborted turn" in record.getMessage()
for record in caplog.records
), "declined abort published a committed-abort settlement"
assert not any(
"Force-aborting" in record.getMessage()
for record in caplog.records
), "pre-commit surface published the definitive abort outcome"
# …and the lease kept renewing through the resumed turn.
assert len(db.refresh_times) >= 1
assert db.events[-1][0] == "release"
def test_watchdog_declines_abort_when_activity_resumes_after_revalidation(
watchdog_config, monkeypatch, caplog
):
"""#95663 round-3 review race: the commit point revalidates the observed
(generation, timestamp) pair under the activity lock and then releases
it before the hard interrupt. Real progress landing in that
post-revalidation window published a new generation yet was still
hard-cancelled by the already-authorized abort. The abort now carries
its revalidated generation into the interrupt path, which re-compares
it against the live clock at the last instant before the hammer and
abandons the stale claim — the turn continues and its lease keeps
renewing."""
db = _DB()
agent = _agent_with_db(db)
agent._session_turn_lease_refresh_interval = 0.05
t_start = time.time()
interrupt_seen = {}
injected = {"count": 0}
real_interrupt = agent.interrupt
def intercepting_interrupt(
message=None,
*,
hard_cancel=False,
tool_reason=None,
require_generation=None,
):
# Deterministically land real progress in the exact
# post-revalidation / pre-interrupt window: every abort hammer
# attempt is preceded by a fresh activity stamp on the clock.
injected["count"] += 1
agent._touch_activity("resumed after revalidation")
if require_generation is not None:
return real_interrupt(
message,
hard_cancel=hard_cancel,
tool_reason=tool_reason,
require_generation=require_generation,
)
return real_interrupt(
message, hard_cancel=hard_cancel, tool_reason=tool_reason
)
agent.interrupt = intercepting_interrupt
def stalled_loop(_agent, _message, _system, history, *_args, **_kwargs):
# Stall without touching the clock: the watchdog fires and tries to
# abort; the interceptor above injects the resumed activity at the
# commit's hammer point. Keep the turn alive long enough for the
# abort decision to play out and record whether the interrupt was
# ever published.
while time.time() - t_start < 1.0:
if _agent._interrupt_requested:
interrupt_seen["requested"] = True
break
time.sleep(0.005)
# Capture DURING the turn — the wrapper's finally clears interrupt
# state after the loop returns.
if "requested" not in interrupt_seen:
interrupt_seen["requested"] = _agent._interrupt_requested
interrupt_seen["message"] = _agent._interrupt_message
return {
"final_response": "recovered",
"messages": history,
"api_calls": 1,
"completed": True,
}
with caplog.at_level(logging.ERROR, logger="agent.turn_liveness"):
result = _run_turn(agent, stalled_loop, monkeypatch)
assert time.time() - t_start < 10.0
# The window was really hit: the interceptor's injected activity ran.
assert injected["count"] >= 1
# The turn resumed and completed normally — the stale abort claim was
# abandoned instead of hard-cancelling the resumed turn.
assert result["completed"] is True
assert result.get("interrupted") is not True
assert interrupt_seen.get("requested") is False, (
f"turn was hard-interrupted although activity resumed after "
f"revalidation: {interrupt_seen!r}"
)
# The stall was still surfaced loudly…
assert any(
"Turn liveness watchdog fired" in record.getMessage()
and "stalled-session" in record.getMessage()
for record in caplog.records
)
# …and the lease kept renewing through the resumed turn.
assert len(db.refresh_times) >= 1
assert db.events[-1][0] == "release"
def test_watchdog_declines_abort_when_activity_resumes_inside_interrupt_publication(
watchdog_config, monkeypatch, caplog
):
"""#95663 round-4 review race: the generation claim must survive every
blocking boundary inside ``interrupt()`` — including the compression
commit fence — and be consumed at the final mutation edge, immediately
before the first observable publication.
The witness parks ``interrupt()`` inside a controllable
``cancel_before_commit`` AFTER its internal generation comparison
passed, resumes the turn with real progress (``_touch_activity``
publishes generation G+1) while the hammer is parked, then releases the
fence. The abort must abandon itself: no ``_interrupt_requested`` flag,
no hard-cancel event, no tool signal, and the turn's lease keeps
renewing.
"""
db = _DB()
agent = _agent_with_db(db)
agent._session_turn_lease_refresh_interval = 0.05
fence = _BlockingCommitFence()
agent._active_compression_commit_fence = fence
t_start = time.time()
published = {"requested": False, "hard_event": False, "tool_signal": False}
def stalled_then_resumed_loop(_agent, _message, _system, history, *_a, **_k):
# Stall (zero activity) until the watchdog's abort attempt is parked
# inside the compression fence — i.e. AFTER interrupt() passed its
# internal generation comparison. Then resume with real progress
# while the hammer is mid-flight.
assert fence.entered.wait(10.0), "interrupt never reached the fence"
_agent._touch_activity("resumed inside interrupt publication window")
fence.release.set()
# Keep the turn making REAL progress while observing whether the
# parked interrupt published anything. Continuous activity also
# guarantees no NEW legitimate watchdog decision can fire, so any
# publication observed here is attributable to the stale attempt.
deadline = time.time() + 1.0
while time.time() < deadline:
_agent._touch_activity("still alive")
if _agent._interrupt_requested or _agent._hard_interrupt_requested.is_set():
break
time.sleep(0.02)
published["requested"] = _agent._interrupt_requested
published["hard_event"] = _agent._hard_interrupt_requested.is_set()
published["tool_signal"] = _agent._interrupt_thread_signal_pending
published["message"] = _agent._interrupt_message
return {
"final_response": "recovered",
"messages": history,
"api_calls": 1,
"completed": True,
}
with caplog.at_level(logging.ERROR, logger="agent.turn_liveness"):
result = _run_turn(agent, stalled_then_resumed_loop, monkeypatch)
assert time.time() - t_start < 10.0
# The window was really hit: the interrupt entered the fence exactly
# once — the stale attempt declined and no retry ever fired.
assert fence.calls == 1, f"unexpected fence admissions: {fence.calls}"
# The resumed turn completed normally — the stale claim was abandoned
# at the final mutation edge without publishing any interrupt state.
assert result["completed"] is True
assert result.get("interrupted") is not True
assert published["requested"] is False, (
f"_interrupt_requested published against stale generation: {published!r}"
)
assert published["hard_event"] is False, (
f"hard-cancel event published against stale generation: {published!r}"
)
assert published["tool_signal"] is False, (
f"tool interrupt signal published against stale generation: {published!r}"
)
assert published["message"] is None
# The stall was still surfaced loudly…
assert any(
"Turn liveness watchdog fired" in record.getMessage()
and "stalled-session" in record.getMessage()
for record in caplog.records
)
# …and the lease kept renewing through the resumed turn.
assert len(db.refresh_times) >= 1
assert db.events[-1][0] == "release"
def test_watchdog_declines_abort_when_interrupt_publish_raises(
watchdog_config, monkeypatch, caplog
):
"""#95663 round-4 review: the exceptional interrupt path must not
convert an unvalidated generation claim into direct interrupt-flag
mutation. When ``interrupt()`` raises, the abort declines fail-closed —
no ``_interrupt_requested`` / ``_interrupt_message`` is set, the
watchdog keeps sampling, and the turn's lease keeps renewing.
"""
db = _DB()
agent = _agent_with_db(db)
agent._session_turn_lease_refresh_interval = 0.05
t_start = time.time()
raised = {"count": 0}
seen = {"requested": False, "message": None}
def raising_interrupt(
message=None,
*,
hard_cancel=False,
tool_reason=None,
require_generation=None,
):
raised["count"] += 1
raise RuntimeError("synthetic interrupt publication failure")
agent.interrupt = raising_interrupt
def stalled_loop(_agent, _message, _system, history, *_args, **_kwargs):
# Stall so the watchdog fires; every abort attempt raises inside
# interrupt(). Run a fixed window while recording whether the
# exception fallback published interrupt state anyway.
while time.time() - t_start < 1.0:
seen["requested"] = _agent._interrupt_requested
seen["message"] = _agent._interrupt_message
if _agent._interrupt_requested:
break
time.sleep(0.005)
seen["requested"] = _agent._interrupt_requested
seen["message"] = _agent._interrupt_message
return {
"final_response": "recovered",
"messages": history,
"api_calls": 1,
"completed": True,
}
with caplog.at_level(logging.ERROR, logger="agent.turn_liveness"):
result = _run_turn(agent, stalled_loop, monkeypatch)
assert time.time() - t_start < 10.0
# The exceptional abort attempt really happened inside the window.
assert raised["count"] >= 1
# The turn completed without being interrupted…
assert result["completed"] is True
assert result.get("interrupted") is not True
# …and the exception fallback published NOTHING: fail-closed, no flag,
# no message.
assert seen["requested"] is False, (
f"exception fallback mutated interrupt state: {seen!r}"
)
assert seen["message"] is None
# The stall was surfaced loudly (the watchdog really fired)…
assert any(
"Turn liveness watchdog fired" in record.getMessage()
and "stalled-session" in record.getMessage()
for record in caplog.records
)
# …and the lease kept renewing because the abort was declined.
assert len(db.refresh_times) >= 1
assert db.events[-1][0] == "release"
def test_declined_abort_does_not_cancel_pending_compression_commit():
"""#99758 P1 review: a stale liveness claim must not cancel a legitimate
pending compression commit when the abort ultimately declines.
Schedule under test: the watchdog reserves generation G and parks at
the claim-reservation release; real progress lands (G+1, claim
invalidated) while the interrupt is parked; the interrupt then runs
its (no-op for a pending commit) in-flight wait, declines at the final
mutation edge, and the REAL CompressionCommitFence must still admit
begin_commit() — the fence must NOT be left cancelled by the stale
abort authority. The pre-fix tree cancelled the pending fence BEFORE
validating the claim, so begin_commit() refused forever.
"""
from agent.conversation_compression import CompressionCommitFence
agent = AIAgent.__new__(AIAgent)
agent._turn_liveness_activity_generation = 5
agent._turn_liveness_abort_claim = None
agent._interrupt_requested = False
agent._interrupt_message = None
agent._tool_interrupt_reason = None
agent._hard_interrupt_requested = threading.Event()
agent._execution_thread_id = None
agent._active_children_lock = threading.Lock()
agent._active_children = set()
agent.quiet_mode = True
# A REAL production fence with a PENDING (not started) commit.
fence = CompressionCommitFence()
agent._active_compression_commit_fence = fence
# Park the interrupt right after the claim reservation (release #1 of
# the activity lock) so real progress can land in the exact window.
parking_lock = _ParkingReleaseLock(threading.Lock())
parking_lock.park_on_release = 1
agent._turn_liveness_activity_lock = parking_lock
result = {}
def interrupt_fn():
result["ret"] = AIAgent.interrupt(
agent,
"watchdog: no progress",
hard_cancel=True,
require_generation=5,
)
interrupt_thread = threading.Thread(target=interrupt_fn)
interrupt_thread.start()
assert parking_lock.parked.wait(10.0), (
"interrupt never reached the claim-reservation boundary"
)
# Real progress lands while the interrupt is parked between the claim
# reservation and the final mutation edge.
touched = threading.Event()
def turn_fn():
agent._touch_activity("turn resumed")
touched.set()
turn_thread = threading.Thread(target=turn_fn)
turn_thread.start()
assert touched.wait(10.0), "competing activity never landed"
assert agent._turn_liveness_activity_generation == 6
parking_lock.release_park.set()
interrupt_thread.join(10.0)
turn_thread.join(10.0)
# The abort declined: the claim went stale against generation 6.
assert result["ret"] is False, "interrupt should have declined"
assert not agent._interrupt_requested
assert not agent._hard_interrupt_requested.is_set()
# THE P1 INVARIANT: the pending compression commit was NOT cancelled
# by the declined abort — begin_commit() still admits.
assert fence.begin_commit() is True, (
"declined liveness abort left the pending compression fence "
"cancelled: begin_commit() refused"
)
fence.finish_commit()