hermes-agent/tests/cron/test_cron_run_metrics.py
teknium1 f4075a20d6 feat(telemetry): gateway platform health, delivery, first-reply latency and cron run metrics
The gateway and cron ticker were blind spots in shared metrics: we could not
tell which messaging platforms fail to connect or drop, how often replies fail
to reach users, how long users wait for a first reply, or whether scheduled
jobs run, fail, get skipped or get missed while Hermes was down.

New opt-in counters (all through the existing enabled() / record_process_mark
gate, recorded fire-and-forget on one background worker that runs in a copy of
the caller's context so the owning profile gets the row):

- hermes.platform.health: platform, event (connect_ok/connect_failed/
  reconnect/disconnect), error_class. One seam in the runner
  (_connect_adapter_with_timeout, used by cold start, multiplex secondaries and
  the reconnect watcher) plus the fatal-error handler; classified from
  exception types, HTTP statuses and Hermes's own fatal codes, never text.
- hermes.platform.delivery: platform, outcome, failure_class. One row per
  logical BasePlatformAdapter._send_with_retry call (retries and plain-text
  fallback included), from SendResult's typed fields or the exception type.
- hermes.gateway.reply_latency: platform, first_response_bucket. Clock starts
  when _handle_message accepts a non-internal turn; stops on the stream
  consumer's first delivered text or send_final_ledgered. Busy acks and
  command replies never stop it.
- hermes.cron.run: outcome (success/failed/missed/skipped), delivery_kind,
  duration_bucket. Recorded at the write-once terminal execution row
  (finish_execution) and where the due scan drops an occurrence (catch-up
  disabled, expired one-shot). Job names, prompts and targets never leave.

Platform names: platforms Hermes ships under plugins/platforms/ now report by
name, and a plugin-catalog platform reports its catalog entry name only when
the installer-owned .install-metadata.json record proves a catalog install of
the plugin dir that defines the registered adapter factory (a URL install
cannot claim a catalog name via its tree or manifest). Everything else stays
"plugin". GATEWAY_PLATFORMS becomes catalog-backed (_CatalogValues), and the
v3 schema's platform fields accept published catalog identifiers.
2026-09-28 12:43:03 -07:00

61 lines
2.8 KiB
Python

"""hermes.cron.run: one row per terminal execution and per dropped occurrence."""
from datetime import timedelta
import pytest
from cron import executions, jobs
from hermes_cli.observability import relay_shared_metrics as rsm
from hermes_cli.observability import shared_metrics_contract as contract
from hermes_cli.observability import shared_metrics_gateway as smg
@pytest.fixture
def cron_rows(tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
got = []
monkeypatch.setattr(rsm, "enabled", lambda: True)
monkeypatch.setattr(rsm, "record_process_mark", lambda mark, data: got.append((mark, dict(data))))
def read():
smg.drain()
found = [data for mark, data in got if mark == contract.CRON_RUN_MARK]
assert all(contract.counter_dimensions_are_valid(contract.CRON_RUN_METRIC, d) for d in found)
return found
return read
def _execution(job_id, deliver, *, start=True):
row = executions.create_execution(job_id, source="builtin")
smg.note_cron_execution({"execution_id": row["id"], "deliver": deliver})
if start:
executions.mark_execution_running(row["id"])
return row["id"]
def test_each_terminal_execution_is_one_row_classified_without_job_details(cron_rows):
ok = _execution("job1", "telegram:-100555")
executions.finish_execution(ok, success=True, delivery_outcome="delivered")
executions.finish_execution(ok, success=False, error="late duplicate") # write-once: no second row
executions.finish_execution(_execution("job2", "local"), success=False, error="boom")
refused = _execution("job3", "webhook", start=False) # claim lost before any side effect
executions.finish_execution(refused, success=False, error="Fire claim lost; execution was not started.")
gated = _execution("job4", "origin")
smg.note_cron_skipped({"execution_id": gated}) # pre-run script said wakeAgent=false
executions.finish_execution(gated, success=True, delivery_outcome="suppressed")
assert [(r["outcome"], r["delivery_kind"]) for r in cron_rows()] == [
("success", "platform"), ("failed", "local"), ("skipped", "webhook"), ("skipped", "platform"),
]
def test_an_occurrence_dropped_after_downtime_is_one_missed_row(tmp_path, cron_rows):
(tmp_path / "config.yaml").write_text("cron:\n catch_up_missed: false\n", encoding="utf-8")
with jobs.use_cron_store(tmp_path / "cron"):
jobs.create_job(prompt="secret prompt", schedule="every 1h", model="fixture", deliver="local")
stored = jobs.load_jobs()
stored[0]["next_run_at"] = (jobs._hermes_now() - timedelta(hours=4)).isoformat()
jobs.save_jobs(stored)
assert jobs.get_due_jobs() == []
assert jobs.get_due_jobs() == []
assert cron_rows() == [{"delivery_kind": "local", "duration_bucket": "lt_1s", "outcome": "missed"}]