mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-17 21:32:52 +00:00
267 lines
11 KiB
Python
267 lines
11 KiB
Python
"""#105861 / #113357: a fire-claim that only *reads* as lost must not overwrite a completed run's
|
|
terminal status — an ok, or a failure notice carrying its real error.
|
|
|
|
`_FireOwnership.lost()` samples the claim from the store once. When that sample misses after
|
|
the notice already reached the channel, the run must fall through to the owner-fenced terminal
|
|
write in `_finish_completed_run` (the authoritative claim check) instead of recording an error.
|
|
|
|
These drive the real store (``jobs.json`` under a temp HERMES_HOME) so the assertion is the
|
|
actual on-disk ``last_status`` the health watchdog reads, not a mock's call list.
|
|
"""
|
|
|
|
import threading
|
|
|
|
import pytest
|
|
|
|
|
|
@pytest.fixture
|
|
def temp_home(tmp_path, monkeypatch):
|
|
"""Isolated HERMES_HOME so jobs.json/executions don't touch the real store."""
|
|
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
|
return tmp_path
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _clean_running_state():
|
|
import cron.scheduler as sched
|
|
|
|
sched._running_job_ids.clear()
|
|
sched._running_fire_owners.clear()
|
|
sched._interrupted_job_ids.clear()
|
|
yield
|
|
sched._running_job_ids.clear()
|
|
sched._running_fire_owners.clear()
|
|
sched._interrupted_job_ids.clear()
|
|
|
|
|
|
class _SampledHeartbeat:
|
|
"""The real ``heartbeat_fire_claim``, with exactly ONE armed sample missing."""
|
|
|
|
def __init__(self, real, samples_before_miss: int):
|
|
self._real = real
|
|
self._samples_before_miss = samples_before_miss
|
|
self._armed = False
|
|
self._seen = 0
|
|
self.missed = 0
|
|
|
|
def __call__(self, job_id, *, expected_owner):
|
|
if self._armed:
|
|
self._seen += 1
|
|
if self._seen > self._samples_before_miss:
|
|
self._armed = False
|
|
self.missed += 1
|
|
return False
|
|
return self._real(job_id, expected_owner=expected_owner)
|
|
|
|
def arm(self):
|
|
"""Answer ``samples_before_miss`` samples truthfully from here on, then miss once."""
|
|
self._seen = 0
|
|
self._armed = True
|
|
|
|
|
|
def _claimed_job():
|
|
"""A real recurring job record holding a live fire claim (as a firing tick has)."""
|
|
from cron.jobs import claim_job_for_fire, create_job, get_job
|
|
|
|
job = create_job(prompt="x", schedule="every 5m", name="105861")
|
|
assert claim_job_for_fire(job["id"]) is True
|
|
job = get_job(job["id"])
|
|
assert isinstance(job.get("fire_claim"), dict) and job["fire_claim"].get("by")
|
|
return job
|
|
|
|
|
|
def _drive(monkeypatch, *, run_result, samples_before_miss):
|
|
"""Run one job through run_one_job with the sampled-heartbeat harness.
|
|
|
|
After ``run_job`` returns the claim is sampled once (the pre-delivery check), once again just
|
|
before the delivery side effect, and once after it (the post-delivery check that the incidents
|
|
tripped on). ``samples_before_miss=2`` therefore makes the third, post-delivery sample miss.
|
|
"""
|
|
import cron.scheduler as sched
|
|
|
|
job = _claimed_job()
|
|
hb = _SampledHeartbeat(sched.heartbeat_fire_claim, samples_before_miss=samples_before_miss)
|
|
delivered = []
|
|
|
|
def fake_run_job(job, **kwargs):
|
|
hb.arm()
|
|
return run_result
|
|
|
|
def fake_deliver(job, content, **kwargs):
|
|
delivered.append(content)
|
|
return None
|
|
|
|
monkeypatch.setattr(sched, "heartbeat_fire_claim", hb)
|
|
monkeypatch.setattr(sched, "run_job", fake_run_job)
|
|
monkeypatch.setattr(sched, "_deliver_result", fake_deliver)
|
|
return sched, job, hb, delivered
|
|
|
|
|
|
_REAL_ERROR = "the real error text the failure path recorded"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"success, run_result, expected_status, expected_error",
|
|
[
|
|
(True, (True, "output text", "the report", None), "ok", None),
|
|
(False, (False, "output text", "", _REAL_ERROR), "error", _REAL_ERROR),
|
|
],
|
|
ids=["delivered-ok", "delivered-failure-notice"],
|
|
)
|
|
def test_delivered_run_keeps_its_terminal_status_when_claim_sample_misses_after_delivery(
|
|
temp_home, monkeypatch, success, run_result, expected_status, expected_error,
|
|
):
|
|
"""Delivery completed, then the claim sample missed → the on-disk record is the delivered
|
|
run's real outcome (ok, or the real error — never ``_OWNERSHIP_LOST_INTERRUPTED``), and the
|
|
loss latch stays on the raw source: the caller's idle transport event is left unset."""
|
|
from cron.jobs import get_job
|
|
|
|
sched, job, hb, delivered = _drive(monkeypatch, run_result=run_result, samples_before_miss=2)
|
|
cancel = threading.Event()
|
|
|
|
assert sched.run_one_job(job, cancel_event=cancel) is True
|
|
|
|
assert len(delivered) == 1, "the notice must have left the process"
|
|
if success:
|
|
assert delivered == ["the report"]
|
|
else:
|
|
assert "failed" in delivered[0], "the failure notice, not the (empty) report, was delivered"
|
|
assert hb.missed == 1, "exactly the post-delivery sample missed"
|
|
assert cancel.is_set() is False, "the run's loss latch must not cancel the caller's transport"
|
|
record = get_job(job["id"])
|
|
assert record["last_status"] == expected_status
|
|
assert record["last_error"] == expected_error
|
|
assert record["last_error"] != sched._OWNERSHIP_LOST_INTERRUPTED
|
|
assert record["failure_streak"] == (0 if success else 1)
|
|
|
|
|
|
def test_transport_cancel_during_delivery_stays_fail_closed(temp_home, monkeypatch):
|
|
"""An explicit transport cancel during delivery is not a sampled miss → still interrupted."""
|
|
from cron.jobs import get_job
|
|
|
|
sched, job, hb, delivered = _drive(
|
|
monkeypatch, run_result=(True, "output text", "the report", None),
|
|
samples_before_miss=99)
|
|
cancel = threading.Event()
|
|
deliver_result = sched._deliver_result
|
|
|
|
def deliver_then_cancel(job, content, **kwargs):
|
|
outcome = deliver_result(job, content, **kwargs)
|
|
cancel.set()
|
|
return outcome
|
|
|
|
monkeypatch.setattr(sched, "_deliver_result", deliver_then_cancel)
|
|
|
|
assert sched.run_one_job(job, cancel_event=cancel) is True
|
|
|
|
assert delivered == ["the report"], "the notice had already left the process"
|
|
assert hb.missed == 0, "the sampled claim never missed — only the transport event fired"
|
|
record = get_job(job["id"])
|
|
assert record["last_status"] == "error"
|
|
assert record["last_error"] == sched._OWNERSHIP_LOST_INTERRUPTED
|
|
|
|
|
|
class _HeartbeatThreadMisses:
|
|
"""The real ``heartbeat_fire_claim``; the first ``misses`` samples taken on the heartbeat
|
|
thread return False (optionally re-owning the stored claim so the loss is genuine)."""
|
|
|
|
def __init__(self, real, misses: int, *, steal: bool = False):
|
|
self._real, self._left, self._steal = real, misses, steal
|
|
self.missed = 0
|
|
self._samples = 0
|
|
self.processed = threading.Event()
|
|
|
|
def __call__(self, job_id, *, expected_owner):
|
|
if threading.current_thread().name == "cron-fire-claim-heartbeat":
|
|
self._samples += 1
|
|
if self._samples > 2:
|
|
# Reaching the next poll proves the confirmation was accepted without latching.
|
|
self.processed.set()
|
|
if threading.current_thread().name == "cron-fire-claim-heartbeat" and self._left:
|
|
self._left -= 1
|
|
self.missed += 1
|
|
if self._steal:
|
|
from cron.jobs import _with_job, save_jobs
|
|
|
|
def re_own(jobs, _i, job):
|
|
job["fire_claim"] = {**job["fire_claim"], "by": "replacement:deadbeef"}
|
|
save_jobs(jobs)
|
|
|
|
_with_job(job_id, re_own)
|
|
return False
|
|
return self._real(job_id, expected_owner=expected_owner)
|
|
|
|
|
|
def _drive_heartbeat_thread(monkeypatch, *, misses, steal=False):
|
|
"""run_one_job with the REAL fire-claim heartbeat thread sampling every 10 ms; ``run_job``
|
|
finishes after the heartbeat has processed its confirmation, not merely sampled it."""
|
|
import cron.scheduler as sched
|
|
|
|
job = _claimed_job()
|
|
hb = _HeartbeatThreadMisses(sched.heartbeat_fire_claim, misses, steal=steal)
|
|
delivered, run_cancel = [], []
|
|
start_heartbeat = sched._start_heartbeat_thread
|
|
|
|
def observed_heartbeat(loop_fn, name, fail_log):
|
|
def run():
|
|
try:
|
|
loop_fn()
|
|
finally:
|
|
# A confirmed loss exits the loop only after setting the real cancel latch.
|
|
hb.processed.set()
|
|
|
|
return start_heartbeat(run, name, fail_log)
|
|
|
|
def fake_run_job(job, **kwargs):
|
|
# Deadlock guard only: store I/O and thread scheduling have no 100 ms upper bound.
|
|
assert hb.processed.wait(timeout=30), "heartbeat never processed its confirmation"
|
|
run_cancel.append(kwargs["cancel_event"].is_set())
|
|
return True, "output text", "the report", None
|
|
|
|
monkeypatch.setattr(sched, "_start_heartbeat_thread", observed_heartbeat)
|
|
monkeypatch.setattr(sched, "_RUN_CLAIM_HEARTBEAT_SECONDS", 0.01)
|
|
monkeypatch.setattr(sched, "_FIRE_CLAIM_MISS_CONFIRM_SECONDS", 0.01, raising=False)
|
|
monkeypatch.setattr(sched, "heartbeat_fire_claim", hb)
|
|
monkeypatch.setattr(sched, "run_job", fake_run_job)
|
|
monkeypatch.setattr(sched, "_deliver_result", lambda job, content, **kw: delivered.append(content))
|
|
return sched, job, hb, delivered, run_cancel
|
|
|
|
|
|
@pytest.mark.parametrize("misses, latched", [(1, False), (2, True)], ids=["one-sample", "latched"])
|
|
def test_heartbeat_miss_mid_run_keeps_completed_run(temp_home, monkeypatch, misses, latched):
|
|
"""#113357: the heartbeat thread's miss is a sample, not the verdict. One miss is re-sampled
|
|
and never cancels the run; two consecutive misses latch, but a claim the store still validates
|
|
at completion records the completed run — not ``_OWNERSHIP_LOST_INTERRUPTED``."""
|
|
from cron.executions import get_execution
|
|
from cron.jobs import get_job
|
|
|
|
sched, job, hb, delivered, run_cancel = _drive_heartbeat_thread(monkeypatch, misses=misses)
|
|
|
|
assert sched.run_one_job(job) is True
|
|
|
|
assert hb.missed == misses
|
|
assert run_cancel == [latched], "only a confirmed miss reaches the run's cancel event"
|
|
assert delivered == ["the report"]
|
|
record = get_job(job["id"])
|
|
assert record["last_status"] == "ok"
|
|
assert record["last_error"] is None
|
|
assert get_execution(job["execution_id"])["status"] == "completed"
|
|
|
|
|
|
def test_confirmed_claim_loss_mid_run_still_yields(temp_home, monkeypatch):
|
|
"""A genuinely re-owned claim (two misses, stored ``by`` rewritten) still fences the run out:
|
|
nothing delivered, no terminal write over the new owner, ledger records the discard."""
|
|
from cron.executions import get_execution
|
|
from cron.jobs import get_job
|
|
|
|
sched, job, hb, delivered, run_cancel = _drive_heartbeat_thread(
|
|
monkeypatch, misses=2, steal=True)
|
|
|
|
assert sched.run_one_job(job) is True
|
|
|
|
assert run_cancel == [True]
|
|
assert delivered == []
|
|
record = get_job(job["id"])
|
|
assert record["last_status"] is None
|
|
assert record["fire_claim"]["by"] == "replacement:deadbeef"
|
|
assert "discarded" in get_execution(job["execution_id"])["error"]
|