hermes-agent/tests/agent/test_credential_pool_sole_rotate_recovery.py
liuhao1024 8530d81b55 fix(credential-pool): treat rotate-to-self after mark as no-recovery
mark_exhausted_and_rotate benched the failed entry, then _select_unlocked
could hand the very same entry straight back (the auth-store sync adopted
fresher tokens and cleared the bench mid-selection, or a false-positive
quota probe lifted it). The caller took the non-None result as a successful
rotation and retried the same 429 with no attempt cap: a sole-credential
openai-codex pool on a weekly usage_limit_reached spun at ~2 req/s for
hours, ~11k requests, turn never failed (#97315).

If the post-mark selection returns the just-marked entry, return None so
the failure surfaces and provider fallback / error propagation proceed —
mirroring the single-entry guard on the unmatched-identity branch.
Multi-entry pools still rotate to a healthy sibling. Ported from PR #97328.

Fixes #97315
2026-09-19 09:40:56 -07:00

86 lines
2.9 KiB
Python

"""#97315: a rotation that can only hand back the entry that was just marked
exhausted is no recovery — it must return None so the turn fails.
A sole-credential ``openai-codex`` pool hitting a 429 ``usage_limit_reached``
writes a correct bench (``last_error_reset_at`` days out), yet the selection
path can re-admit the just-marked entry within milliseconds (auth-store sync
adopting fresher tokens, a false-positive quota probe). ``mark_exhausted_and_rotate``
then reports a successful rotation, and the caller retries the same throttled
credential forever (~2 req/s for hours, gateway wedged until killed by hand).
"""
from __future__ import annotations
import json
import time
from dataclasses import replace
def _entry(idx: int, *, token: str, refresh: str) -> dict:
return {
"id": f"codex-{idx}",
"label": f"codex-login-{idx}",
"auth_type": "oauth",
"priority": idx,
"source": "device_code",
"access_token": token,
"refresh_token": refresh,
}
def _load(tmp_path, monkeypatch, entries: list[dict]):
hermes_home = tmp_path / "hermes"
hermes_home.mkdir(parents=True, exist_ok=True)
(hermes_home / "auth.json").write_text(
json.dumps({"version": 1, "credential_pool": {"openai-codex": entries}})
)
monkeypatch.setenv("HERMES_HOME", str(hermes_home))
from agent.credential_pool import load_pool
return load_pool("openai-codex")
def _revive_entry(pool, entry):
"""Simulate the auth-store sync adopting fresher tokens: the bench is cleared
mid-selection, exactly as ``_sync_entry_from_auth_store`` does for a
changed token pair."""
updated = replace(
entry,
access_token=entry.access_token + "-adopted",
refresh_token=(entry.refresh_token or "") + "-adopted",
last_status=None,
last_status_at=None,
last_error_code=None,
last_error_reason=None,
last_error_message=None,
last_error_reset_at=None,
)
pool._replace_entry(entry, updated)
return updated
def _mark(pool, credential_id: str):
return pool.mark_exhausted_and_rotate(
status_code=429,
error_context={"reason": "usage_limit_reached", "reset_at": time.time() + 95.5 * 3600},
credential_id=credential_id,
)
def test_revived_sole_entry_is_no_recovery(tmp_path, monkeypatch):
pool = _load(tmp_path, monkeypatch, [_entry(1, token="tok-a", refresh="rf-a")])
pool._sync_entry_from_auth_store = lambda entry: _revive_entry(pool, entry)
assert _mark(pool, "codex-1") is None
def test_multi_entry_pool_still_rotates_to_healthy_sibling(tmp_path, monkeypatch):
pool = _load(
tmp_path, monkeypatch,
[_entry(1, token="tok-a", refresh="rf-a"), _entry(2, token="tok-b", refresh="rf-b")],
)
pool._sync_entry_from_auth_store = lambda entry: entry
nxt = _mark(pool, "codex-1")
assert nxt is not None and nxt.id == "codex-2"