mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-17 21:32:52 +00:00
The incident ledger only ever grew. A one-off failure (a drift skip after a global model bump, a provider outage) stayed `detected`/`alerted` forever after the job recovered, so `hermes cron incidents` listed 32 "open" incidents on an install where all 32 jobs had since run OK, and the list stopped saying anything about current health. A successful run now marks that job's `detected`/`alerted` incidents `resolved` (new state). `resolved` is distinct from the operator's `closed` ack on purpose: `upsert_incident` re-opens a resolved incident as `detected` when the same error signature recurs, so the operator is alerted again for a job that broke a second time, while `closed` keeps the signature silent as before. Wired from `_compose_run_delivery` next to the failure-side upsert; best-effort, store errors never affect delivery. CLI: `--state resolved` filter and a green `resolved` colour; `closed` is now dim. Docs updated in the same change.
51 lines
2.2 KiB
Python
51 lines
2.2 KiB
Python
"""A successful run resolves the job's open incidents; the same error afterwards re-opens them.
|
|
|
|
The ledger only ever grew: a one-off failure (drift skip, provider outage) stayed ``detected``/``alerted``
|
|
after the job had been green for weeks, so ``hermes cron incidents`` listed 32 "open" incidents on an
|
|
install whose every job was healthy. ``resolved`` is auto and re-openable; ``closed`` is the operator's
|
|
ack and stays silent.
|
|
"""
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
|
|
|
|
import cron.incidents as incidents
|
|
import cron.scheduler as sched
|
|
|
|
|
|
def _point_db(monkeypatch, tmp_path):
|
|
monkeypatch.setattr(incidents, "EXECUTIONS_FILE", tmp_path / "cron" / "executions.db")
|
|
return incidents
|
|
|
|
|
|
def test_successful_run_resolves_open_incidents_and_a_repeat_reopens_them(monkeypatch, tmp_path):
|
|
inc = _point_db(monkeypatch, tmp_path)
|
|
inc_id, _ = inc.upsert_incident("job-1", "provider 503")
|
|
sched._mark_incident_alerted(inc_id)
|
|
other_id, _ = inc.upsert_incident("job-2", "provider 503")
|
|
|
|
content, *_ = sched._compose_run_delivery(
|
|
{"id": "job-1", "name": "j1"}, success=True, error=None, final_response="all good", output_file=None)
|
|
|
|
assert content == "all good"
|
|
assert inc.get_incident(inc_id)["state"] == "resolved"
|
|
assert inc.get_incident(inc_id)["closed_at"]
|
|
assert inc.get_incident(other_id)["state"] == "detected" # another job's incident is untouched
|
|
assert inc.count_incidents("resolved") == 1
|
|
|
|
same_id, reopened = inc.upsert_incident("job-1", "provider 503")
|
|
assert same_id == inc_id and reopened is True
|
|
assert inc.get_incident(inc_id)["state"] == "detected"
|
|
assert inc.get_incident(inc_id)["closed_at"] is None
|
|
|
|
|
|
def test_operator_ack_survives_a_successful_run(monkeypatch, tmp_path):
|
|
inc = _point_db(monkeypatch, tmp_path)
|
|
inc_id, _ = inc.upsert_incident("job-1", "known flaky")
|
|
inc.ack_incident(inc_id)
|
|
|
|
assert inc.close_incidents_for_recovered_job("job-1") == 0
|
|
assert inc.get_incident(inc_id)["state"] == "closed"
|
|
_, reopened = inc.upsert_incident("job-1", "known flaky")
|
|
assert reopened is False and inc.get_incident(inc_id)["state"] == "closed"
|