mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-17 21:32:52 +00:00
_recovery_worthwhile subtracted same-tz aware datetimes directly, which is wall-clock arithmetic and off by an hour when the hold-to-occurrence span crosses a DST transition (weekly jobs make that reachable); use the module's canonical _elapsed_seconds. Derive the schedule from the job and gate the "scheduled fire" boolean at the call site under one name (recover_consumed_fire) from scheduler -> mark_job_run -> plan_hold instead of three. Collapse plan_hold's interval/cron arms, which duplicated the not-blocked early return and the boundary park. Persist the expr fingerprint only for the off-lattice recovery fire: the coalesce branch parks on the lattice (STALE_CRON_MATCH, also under the croniter-missing fallback), so is_recovery_fire is never consulted there. Drop a comment restating the docstring.
173 lines
8 KiB
Python
173 lines
8 KiB
Python
"""Provider quota windows park a cron job instead of re-firing into them (#89376;
|
|
ai-velho-oy/chairman#3).
|
|
|
|
Contract (cron/quota_hold.py): a failed run whose cause is a rate-limited ``AuthError`` with a
|
|
``retry after <N>s`` hint parks a recurring job's ``next_run_at`` past the window and stamps
|
|
``quota_hold_until``; the stale-error re-arm leaves a held job alone; a run that reaches the
|
|
model clears the marker. The hint is read only from the AuthError in the cause chain, never from
|
|
arbitrary failure text.
|
|
"""
|
|
|
|
from datetime import datetime, timedelta, timezone
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
import cron.scheduler as sched
|
|
from cron import quota_hold as qh
|
|
from cron.jobs import (
|
|
_job_is_stale_error_recurring, create_job, get_due_jobs, get_job, mark_job_run, update_job,
|
|
)
|
|
from hermes_cli.auth import CODEX_RATE_LIMITED_CODE, AuthError
|
|
|
|
QUOTA_MSG = "Codex provider quota exhausted (429); retry after 123518s. Credentials are still valid."
|
|
|
|
|
|
@pytest.fixture
|
|
def tmp_cron_home(tmp_path, monkeypatch):
|
|
home = tmp_path / ".hermes"
|
|
home.mkdir()
|
|
monkeypatch.setenv("HERMES_HOME", str(home))
|
|
return home
|
|
|
|
|
|
def _quota_error() -> AuthError:
|
|
return AuthError(QUOTA_MSG, provider="openai-codex", code=CODEX_RATE_LIMITED_CODE)
|
|
|
|
|
|
def test_hold_seconds_only_from_rate_limited_auth_error_in_cause_chain():
|
|
"""The scheduler wraps the resolve failure in a RuntimeError ``from`` the AuthError; the
|
|
hint survives through the cause chain, and text alone (or a re-login AuthError) never
|
|
parks a job."""
|
|
try:
|
|
raise RuntimeError(QUOTA_MSG) from _quota_error()
|
|
except RuntimeError as wrapped:
|
|
assert qh.hold_seconds_from_failure(wrapped) == 123518.0
|
|
|
|
assert qh.hold_seconds_from_failure(RuntimeError(QUOTA_MSG)) is None
|
|
assert qh.hold_seconds_from_failure(RuntimeError("HTTP 429: retry after 60s")) is None
|
|
relogin = AuthError(QUOTA_MSG, provider="openai-codex", code="expired", relogin_required=True)
|
|
assert qh.hold_seconds_from_failure(relogin) is None
|
|
structured = AuthError("quota", code=CODEX_RATE_LIMITED_CODE, retry_after=900)
|
|
assert qh.hold_seconds_from_failure(structured) == 900.0
|
|
|
|
|
|
def test_weekly_cron_retries_when_quota_recovers_before_next_occurrence(
|
|
tmp_cron_home, monkeypatch,
|
|
):
|
|
"""A weekly fire blocked by a shorter quota window retries when the provider reopens;
|
|
it is not silently deferred until the following week's natural occurrence."""
|
|
now = datetime(2026, 9, 18, 12, 1, tzinfo=timezone.utc)
|
|
natural_next = datetime(2026, 9, 25, 12, 0, tzinfo=timezone.utc)
|
|
monkeypatch.setattr("cron.jobs._hermes_now", lambda: now)
|
|
monkeypatch.setattr(qh, "_hermes_now", lambda: now)
|
|
job = create_job("weekly digest", "0 12 * * 5")
|
|
assert datetime.fromisoformat(job["next_run_at"]) == natural_next
|
|
|
|
assert mark_job_run(
|
|
job["id"], False, QUOTA_MSG, quota_hold_seconds=20 * 60 * 60,
|
|
recover_consumed_fire=True,
|
|
)
|
|
|
|
held = get_job(job["id"])
|
|
assert held is not None
|
|
retry_at = datetime.fromisoformat(held["next_run_at"])
|
|
assert retry_at == now + timedelta(hours=20, seconds=qh.HOLD_SLACK_SECONDS) < natural_next
|
|
assert qh.is_recovery_fire(held, held["next_run_at"])
|
|
edited = {**held, "schedule": {"kind": "cron", "expr": "0 9 * * *"}}
|
|
assert not qh.is_recovery_fire(edited, edited["next_run_at"])
|
|
|
|
monkeypatch.setattr("cron.jobs._hermes_now", lambda: retry_at + timedelta(seconds=1))
|
|
assert job["id"] in {due["id"] for due in get_due_jobs()}
|
|
|
|
|
|
def test_recovery_fire_skips_dense_schedules_and_never_re_parks(monkeypatch):
|
|
"""Recovery is one attempt for sparse schedules only: an hourly job whose hold ends two
|
|
minutes before :00 keeps its natural :00 (no off-lattice near-duplicate), and a job that is
|
|
already the recovery fire (carries quota_hold_until) failing again is not re-parked."""
|
|
now = datetime(2026, 9, 18, 12, 1, tzinfo=timezone.utc)
|
|
monkeypatch.setattr(qh, "_hermes_now", lambda: now)
|
|
|
|
dense = {
|
|
"schedule": {"kind": "cron", "expr": "0 * * * *"},
|
|
"next_run_at": datetime(2026, 9, 18, 13, 0, tzinfo=timezone.utc).isoformat(),
|
|
}
|
|
natural = dense["next_run_at"]
|
|
assert not qh.plan_hold(dense, hold_seconds=57 * 60 - qh.HOLD_SLACK_SECONDS,
|
|
recover_consumed_fire=True)
|
|
assert dense["next_run_at"] == natural
|
|
assert qh.STATE_KEY not in dense
|
|
|
|
held_again = {
|
|
"schedule": {"kind": "cron", "expr": "0 12 * * 5"},
|
|
"next_run_at": (now + timedelta(days=7)).isoformat(),
|
|
qh.STATE_KEY: now.isoformat(),
|
|
qh.SCHEDULE_EXPR_KEY: "0 12 * * 5",
|
|
}
|
|
natural = held_again["next_run_at"]
|
|
assert not qh.plan_hold(held_again, hold_seconds=20 * 60 * 60, recover_consumed_fire=True)
|
|
assert held_again["next_run_at"] == natural
|
|
assert qh.STATE_KEY not in held_again
|
|
|
|
|
|
def _raise_quota(**_kw):
|
|
raise _quota_error()
|
|
|
|
|
|
def _tick(job, home, deliveries, resolve):
|
|
"""One real scheduler tick (preflight ON) with the provider resolver replaced by *resolve*."""
|
|
with patch("cron.scheduler._hermes_home", home), \
|
|
patch("cron.scheduler_delivery._resolve_origin", return_value=None), \
|
|
patch("hermes_cli.env_loader.load_hermes_dotenv"), \
|
|
patch("hermes_cli.env_loader.reset_secret_source_cache"), \
|
|
patch("hermes_state_registry.acquire", return_value=MagicMock()), \
|
|
patch("tools.mcp_tool_discovery.discover_mcp_tools", return_value=[]), \
|
|
patch("hermes_cli.runtime_provider.resolve_runtime_provider", side_effect=resolve), \
|
|
patch.object(sched, "_deliver_result",
|
|
side_effect=lambda jb, content, **kw: deliveries.append(content)), \
|
|
patch("run_agent.AIAgent") as agent_cls:
|
|
agent_cls.return_value.run_conversation.side_effect = RuntimeError("model said no")
|
|
sched.run_one_job(dict(job))
|
|
|
|
|
|
def test_quota_hold_parks_past_window_survives_stale_rearm_and_clears_on_model_reach(tmp_cron_home):
|
|
"""A 30-minute job whose provider resolve raises the Codex quota AuthError ('retry after
|
|
123518s') is parked by the real scheduler tick: preflight lets the rate-limited AuthError
|
|
through (it is not a missing credential), the one delivered alert carries the hold notice,
|
|
and the job does not fire again inside the window (not even after the stale-error re-arm's
|
|
cadence+grace). The marker clears once a run reaches the model."""
|
|
job = create_job("portfolio triage", "every 30m", deliver="local")
|
|
job_id = job["id"]
|
|
now = datetime.now(timezone.utc)
|
|
deliveries: list = []
|
|
|
|
_tick(get_job(job_id), tmp_cron_home, deliveries, _raise_quota)
|
|
j = get_job(job_id)
|
|
assert j["last_status"] == "error"
|
|
assert len(deliveries) == 1, deliveries
|
|
parked = datetime.fromisoformat(j["next_run_at"])
|
|
assert parked - now >= timedelta(seconds=123518), "next_run_at must land past the window"
|
|
assert j[qh.STATE_KEY] == j["next_run_at"]
|
|
assert "_quota_hold_seconds" not in j
|
|
|
|
# Two hours later the job looks like a wedged stale-error record (#62002) — the hold says
|
|
# it is parked on purpose, so it is neither re-armed nor due.
|
|
update_job(job_id, {"last_run_at": (now - timedelta(hours=2)).isoformat()})
|
|
j = get_job(job_id)
|
|
assert not _job_is_stale_error_recurring(j, j["schedule"], now)
|
|
assert all(d["id"] != job_id for d in get_due_jobs())
|
|
assert datetime.fromisoformat(get_job(job_id)["next_run_at"]) == parked
|
|
|
|
# A run that reached the model (either outcome) clears the marker.
|
|
assert mark_job_run(job_id, False, "RuntimeError: model said no")
|
|
j = get_job(job_id)
|
|
assert qh.STATE_KEY not in j
|
|
assert datetime.fromisoformat(j["next_run_at"]) - now < timedelta(hours=1)
|
|
|
|
# Editing the schedule recomputes next_run_at from the new cadence; the stale marker must
|
|
# not linger on a record that is no longer parked where it says.
|
|
assert mark_job_run(job_id, False, QUOTA_MSG, quota_hold_seconds=123518)
|
|
assert qh.STATE_KEY in get_job(job_id)
|
|
j = update_job(job_id, {"schedule": "every 15m"})
|
|
assert qh.STATE_KEY not in j
|
|
assert datetime.fromisoformat(j["next_run_at"]) - now < timedelta(hours=1)
|