hermes-agent/tests/cron/test_ticker_startup_survival.py
teknium1 e56ec8c9f7 fix(chronos): a 403 invalid_client from NAS hands cron fires to the built-in ticker (#97494)
NAS maps the agent-cron bearer to a provisioned instance via an `agent:*` client or the
`hermes-cli-vps` bootstrap session (hermes-portal `server/agent-cron/instance-auth.ts`). A
container whose auth.json holds a plain `hermes-cli` user login is refused with 403
invalid_client on every arm, re-arm and list, for the life of that credential - and the
re-login users try first (`hermes auth logout nous` + device code) replaces the bootstrap
session, making it permanent. Chronos previously logged one bare warning per job and left
the jobs with no trigger at all: they only ran through the misfire sweep, minutes late.

NasCronClientError now carries the HTTP status and the OAuth `error` code. On an identity
rejection the provider logs ONE warning that names the real remedy (restore the hosted
credential from the Nous Portal; re-login cannot fix it), stops calling NAS, and starts the
built-in ticker with the gateway's own adapters/loop so scheduled jobs keep firing on time.
Transient 5xx/transport failures keep retrying on the next reconcile.

Supersedes #97566 (@wesleysimplicio), whose runtime-credential swap resolves to the same
bearer on main (`agent_key` is the access token) and so could not change the 403.
2026-09-18 20:07:21 -07:00

121 lines
5 KiB
Python

"""The ticker contract — "an exception must not silently kill the cron thread" — holds for every
step that runs on the cron-scheduler thread, not only the tick body: startup recovery, the
multiplex per-cycle enumeration/gate, and the status-marker writes. And when a ticker HAS ended,
gateway housekeeping respawns it (#111010).
"""
import threading
import time
from unittest.mock import patch
def _wait_until(predicate, timeout=10.0, interval=0.005):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
value = predicate()
if value:
return value
time.sleep(interval)
return predicate()
def _run_ticker(provider, stop, **kwargs):
thread = threading.Thread(target=provider.start, args=(stop,), kwargs={"interval": 0, **kwargs},
daemon=True, name="cron-scheduler")
thread.start()
return thread
def test_ticker_survives_a_corrupt_ledger_at_startup(tmp_path, monkeypatch):
"""A real corrupt ``executions.db`` (sqlite ``file is not a database``) used to escape
``start()`` before the guarded loop and end the thread: gateway up, no job ever fires."""
from cron.scheduler_provider import InProcessCronScheduler
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
(tmp_path / "cron").mkdir()
(tmp_path / "cron" / "executions.db").write_bytes(b"not a sqlite database" * 64)
ticks = []
stop = threading.Event()
with patch("cron.scheduler.tick", side_effect=lambda **kw: ticks.append(1)):
thread = _run_ticker(InProcessCronScheduler(), stop)
_wait_until(lambda: len(ticks) >= 2)
alive = thread.is_alive()
stop.set()
thread.join(timeout=5)
assert alive and len(ticks) >= 2, "ticker died on the pre-loop recovery scan"
assert (tmp_path / "cron" / "ticker_heartbeat").exists()
def test_multiplex_ticker_survives_a_raising_gate_without_ticking_ungated(tmp_path, caplog):
"""A ``profile_gate`` that raises (Desktop stand-down probe) must fail the CYCLE: thread alive,
error logged, and — critically — zero ticks, since an unfiltered home list would tick the very
profiles the gate stands down for (#100489)."""
from cron.scheduler_provider import InProcessCronScheduler
home = tmp_path / "default"
(home / "cron").mkdir(parents=True)
ticks = []
stop = threading.Event()
def broken_gate(name, h):
raise RuntimeError("gate boom")
with patch("cron.scheduler.tick", side_effect=lambda **kw: ticks.append(1)), caplog.at_level("ERROR"):
thread = _run_ticker(InProcessCronScheduler(), stop,
profile_homes=[("default", home)], profile_gate=broken_gate)
_wait_until(lambda: sum("gate boom" in r.getMessage() for r in caplog.records) >= 2)
alive = thread.is_alive()
stop.set()
thread.join(timeout=5)
assert alive, "ticker thread died when profile_gate raised"
assert ticks == [], "a raising gate must not tick the ungated home list"
def test_housekeeping_restarts_a_dead_ticker(monkeypatch):
"""The supervisor is the outer layer: a ticker that ended without a stop request is respawned
on the next housekeeping tick; one that ended BECAUSE of the stop request is not."""
import gateway.run as gateway_run
from cron.scheduler_thread import SupervisedTickerThread
class _OneTick:
def __init__(self):
self.waited = False
def is_set(self):
return self.waited
def wait(self, timeout=None):
self.waited = True
return True
starts = []
stop = threading.Event()
def dying_ticker(stop_event):
starts.append(threading.current_thread().name)
raise RuntimeError("escaped")
ticker = SupervisedTickerThread(dying_ticker, args=(stop,), stop_event=stop)
with patch("threading.excepthook", lambda args: None):
ticker.start()
_wait_until(lambda: not ticker.is_alive())
gateway_run._start_gateway_housekeeping(_OneTick(), interval=0, cron_thread=ticker)
_wait_until(lambda: len(starts) == 2 and not ticker.is_alive())
assert len(starts) == 2 and ticker.restarts == 1
stop.set()
gateway_run._start_gateway_housekeeping(_OneTick(), interval=0, cron_thread=ticker)
assert len(starts) == 2, "a stopped ticker must not be respawned"
def test_supervisor_leaves_a_returning_external_provider_alone():
"""An external provider's start() (Chronos) arms remote one-shots and RETURNS by design; the
supervisor must not read that as a dead ticker and re-run start() every housekeeping tick
(each rerun re-reconciles against NAS and logs a spurious "died without a stop request")."""
from cron.scheduler_thread import SupervisedTickerThread
starts = []
stop = threading.Event()
ticker = SupervisedTickerThread(lambda stop_event: starts.append(1), args=(stop,), stop_event=stop)
ticker.start()
_wait_until(lambda: not ticker.is_alive())
assert ticker.restart_if_dead() is False and starts == [1] and ticker.restarts == 0