mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-17 04:11:35 +00:00
# Conflicts: # apps/desktop/e2e/archived-hidden-session-recoverable.spec.ts # apps/desktop/e2e/bot-chat-message-agent-friendly-name.spec.ts # apps/desktop/e2e/bot-mailbox-unreadable-ticket.spec.ts # apps/desktop/e2e/bot-mode-roster-localized.spec.ts # apps/desktop/e2e/bot-mode-row-click-mirrors-registry.spec.ts # apps/desktop/e2e/bot-mode-tab-shows-bot-name.spec.ts # apps/desktop/e2e/bot-roster-group-row-organisation.spec.ts # apps/desktop/e2e/bot-roster-ignores-infra-dirs.spec.ts # apps/desktop/e2e/bot-roster-timestamp-meta.spec.ts # apps/desktop/e2e/bot-roster-user-sections.spec.ts # apps/desktop/e2e/bot-routines-pane-narrow.spec.ts # apps/desktop/e2e/bot-row-open-recent-session.spec.ts # apps/desktop/e2e/bot-tile-ignores-ambient-composer-model.spec.ts # apps/desktop/e2e/group-composer-auto-grow.spec.ts # apps/desktop/e2e/group-create-gate-remote-roster.spec.ts # apps/desktop/e2e/group-prompt-renamed-primary-handle.spec.ts # apps/desktop/e2e/hosted-room-backend-continuity.spec.ts # apps/desktop/e2e/hosted-room-legacy-store-migration.spec.ts # apps/desktop/e2e/settings-scope-chips-bot-title.spec.ts # apps/desktop/e2e/worktree-branch-status.spec.ts # apps/desktop/electron/backend-probes.test.ts # apps/desktop/electron/connection-apply.test.ts # apps/desktop/electron/desktop-electron-pin.test.ts # apps/desktop/electron/desktop-uninstall.test.ts # apps/desktop/electron/gateway-file-download-transport.test.ts # apps/desktop/electron/gateway-stop-before-update.test.ts # apps/desktop/electron/github-api-auth.test.ts # apps/desktop/electron/registry-primary-profile-scope.test.ts # apps/desktop/electron/update-api-check.test.ts # apps/desktop/electron/update-handoff-marker.test.ts # apps/desktop/electron/venv-blocker-scan.test.ts # apps/desktop/scripts/after-extract.test.mjs # apps/desktop/scripts/local-pack-publish.test.mjs # apps/desktop/scripts/tasks-scroll.test.mjs # apps/desktop/src/app/settings/model-settings.test.tsx # apps/desktop/src/app/updates-overlay.blockers.test.tsx # apps/desktop/src/components/desktop-install-overlay.test.tsx # apps/desktop/src/lib/update-copy.test.ts # scripts/ci/check_os_marker_fakes.py # tests-js/desktop-mac-usage-descriptions.test.ts # tests-js/node-engine-alignment.test.ts # tests/agent/lsp/test_install_and_lint_fixes.py # tests/agent/test_command_token_source.py # tests/agent/test_compression_boundary_hook.py # tests/agent/test_create_openai_client_ssl_verify.py # tests/agent/test_custom_provider_ca_probes.py # tests/agent/test_endpoint_blackhole.py # tests/agent/test_estimator_parity.py # tests/agent/test_in_place_compaction.py # tests/agent/test_moa_loop_mode.py # tests/agent/test_model_metadata.py # tests/agent/test_skill_session_platform_gate.py # tests/agent/test_skill_utils.py # tests/agent/test_ssl_ca_guard.py # tests/computer_use/test_doctor.py # tests/cron/test_codex_execution_paths.py # tests/cron/test_cron_bot_chat_delivery.py # tests/cron/test_cron_script.py # tests/cron/test_media_delivery_parity.py # tests/cron/test_misfire_catchup.py # tests/cron/test_parallel_pool.py # tests/cron/test_recurring_eagain_redispatch.py # tests/gateway/test_choice_picker.py # tests/gateway/test_control_socket_windows_live.py # tests/gateway/test_dingtalk.py # tests/gateway/test_feishu.py # tests/gateway/test_feishu_onboard.py # tests/gateway/test_gateway_shutdown.py # tests/gateway/test_matrix.py # tests/gateway/test_model_command_custom_providers.py # tests/gateway/test_reasoning_command.py # tests/gateway/test_runtime_footer.py # tests/gateway/test_session.py # tests/gateway/test_session_hygiene.py # tests/gateway/test_status.py # tests/gateway/test_teams.py # tests/gateway/test_turn_lease.py # tests/gateway/test_whatsapp_connect.py # tests/hermes_cli/test_approvals_command.py # tests/hermes_cli/test_auth_store_lock_concurrent.py # tests/hermes_cli/test_backup.py # tests/hermes_cli/test_banner_git_state.py # tests/hermes_cli/test_certifi_repair.py # tests/hermes_cli/test_cmd_update.py # tests/hermes_cli/test_compat_manifest_targets.py # tests/hermes_cli/test_computer_use_cli.py # tests/hermes_cli/test_cpr_local_leak.py # tests/hermes_cli/test_dashboard_auth_gate.py # tests/hermes_cli/test_dashboard_procs_kill_grace.py # tests/hermes_cli/test_desktop_lifecycle_windows_live.py # tests/hermes_cli/test_doctor.py # tests/hermes_cli/test_doctor_command_install.py # tests/hermes_cli/test_fleet_config_migration_windows_live.py # tests/hermes_cli/test_gateway.py # tests/hermes_cli/test_gateway_platform_gating.py # tests/hermes_cli/test_gateway_restart_loop.py # tests/hermes_cli/test_gateway_task_probe.py # tests/hermes_cli/test_gateway_wsl.py # tests/hermes_cli/test_gui_command.py # tests/hermes_cli/test_install_cua_driver.py # tests/hermes_cli/test_kanban_db.py # tests/hermes_cli/test_lazy_command_exports.py # tests/hermes_cli/test_lazy_refresh_venv_repair.py # tests/hermes_cli/test_linux_desktop_entry.py # tests/hermes_cli/test_local_runtime.py # tests/hermes_cli/test_local_runtime_updates.py # tests/hermes_cli/test_managed_uv.py # tests/hermes_cli/test_mcp_reload_confirm_gate.py # tests/hermes_cli/test_nous_subscription.py # tests/hermes_cli/test_npm_engine.py # tests/hermes_cli/test_personality_none.py # tests/hermes_cli/test_pet_toggle.py # tests/hermes_cli/test_plan_reconciliation_windows_live.py # tests/hermes_cli/test_plugin_event_bus.py # tests/hermes_cli/test_plugin_manifest_v2.py # tests/hermes_cli/test_plugin_packs.py # tests/hermes_cli/test_plugins_cmd.py # tests/hermes_cli/test_plugins_cmd_enable_disable_nested.py # tests/hermes_cli/test_process_identity.py # tests/hermes_cli/test_profiles.py # tests/hermes_cli/test_profiles_sidebar_cache.py # tests/hermes_cli/test_pty_bridge.py # tests/hermes_cli/test_resolve_turn_limit.py # tests/hermes_cli/test_serve_runtime_inventory.py # tests/hermes_cli/test_session_vacuum_config.py # tests/hermes_cli/test_set_config_value.py # tests/hermes_cli/test_signal_handler_kanban_worker.py # tests/hermes_cli/test_slash_confirm_windows.py # tests/hermes_cli/test_stale_pid_guard.py # tests/hermes_cli/test_startup_fast_guards.py # tests/hermes_cli/test_status.py # tests/hermes_cli/test_telegram_managed_bot.py # tests/hermes_cli/test_tools_config.py # tests/hermes_cli/test_update_apply_shallow_count.py # tests/hermes_cli/test_update_autostash.py # tests/hermes_cli/test_update_concurrent_quarantine.py # tests/hermes_cli/test_update_fetch_failure_classifier.py # tests/hermes_cli/test_update_fleet_probe_resume_token.py # tests/hermes_cli/test_update_handoff_backend_reap.py # tests/hermes_cli/test_update_handoff_desktop_rebuild.py # tests/hermes_cli/test_update_head_moved_gate.py # tests/hermes_cli/test_update_host_obligation.py # tests/hermes_cli/test_update_import_guard.py # tests/hermes_cli/test_update_interrupted_recovery.py # tests/hermes_cli/test_update_inventory.py # tests/hermes_cli/test_update_launchd_unloaded_gateway.py # tests/hermes_cli/test_update_missing_configured_deps.py # tests/hermes_cli/test_update_modified_notice.py # tests/hermes_cli/test_update_multiplex_migration_hook.py # tests/hermes_cli/test_update_no_gateway_restart.py # tests/hermes_cli/test_update_orphan_backend_reap.py # tests/hermes_cli/test_update_parked_branch_guard.py # tests/hermes_cli/test_update_post_pull_syntax_guard.py # tests/hermes_cli/test_update_receipt.py # tests/hermes_cli/test_update_self_lock.py # tests/hermes_cli/test_update_shim_fail_closed.py # tests/hermes_cli/test_update_shim_self_lock.py # tests/hermes_cli/test_update_sqlite_remediation.py # tests/hermes_cli/test_update_stale_dashboard.py # tests/hermes_cli/test_update_stale_virtualenv.py # tests/hermes_cli/test_update_venv_health.py # tests/hermes_cli/test_update_venv_ownership_preflight.py # tests/hermes_cli/test_update_wedged_gateway.py # tests/hermes_cli/test_update_yes_flag.py # tests/hermes_cli/test_update_zip_two_phase.py # tests/hermes_cli/test_urllib_security.py # tests/hermes_cli/test_ux_messages_auth_config.py # tests/hermes_cli/test_ux_messages_startup.py # tests/hermes_cli/test_venv_holder_classifier.py # tests/hermes_cli/test_verify_console_scripts.py # tests/hermes_cli/test_verify_core_dependencies.py # tests/hermes_cli/test_web_server.py # tests/hermes_cli/test_web_server_console_ws.py # tests/hermes_cli/test_web_server_ws_ping.py # tests/hermes_cli/test_web_ui_build.py # tests/hermes_state/test_fts_rebuild_admission.py # tests/hermes_state/test_hermes_state.py # tests/plugins/memory/test_memory_lazy_install.py # tests/plugins/test_google_meet_plugin.py # tests/plugins/test_langfuse_plugin.py # tests/plugins/test_security_guidance_plugin.py # tests/plugins/test_transform_llm_output_hook.py # tests/scripts/desktop_update/test_desktop_update_windows_gateway_flag.py # tests/scripts/desktop_update/test_desktop_update_windows_python_handoff.py # tests/scripts/desktop_update/test_desktop_update_windows_timestamp.py # tests/scripts/install/test_install_clone_throttle_fallback.py # tests/scripts/install/test_install_lockfile_churn.py # tests/scripts/install/test_install_no_initial_commit.py # tests/scripts/install/test_install_sh_browser_install.py # tests/scripts/install/test_install_sh_node_prerelease.py # tests/scripts/install/test_install_sh_symlink_stomp.py # tests/scripts/install/test_install_sh_uv_lock_config.py # tests/scripts/install/test_install_unmerged_index.py # tests/scripts/test_contributor_map.py # tests/scripts/test_run_tests_parallel.py # tests/skills/test_competitor_news_monitor_skill.py # tests/skills/test_document_to_action_items_skill.py # tests/skills/test_google_workspace_setup.py # tests/skills/test_google_workspace_setup_deps.py # tests/skills/test_grounded_citations_skill.py # tests/skills/test_ip_as_logo_skill.py # tests/skills/test_live_dashboard_skill.py # tests/skills/test_mcp_oauth_remote_gateway_skill.py # tests/skills/test_office_document_skills.py # tests/skills/test_openclaw_migration.py # tests/skills/test_product_price_monitor_skill.py # tests/skills/test_scrollcraft_skill.py # tests/skills/test_setup_wizard_generator_skill.py # tests/skills/test_weekly_review_planning_skill.py # tests/test_engines_satisfiable.py # tests/test_fast_safe_load.py # tests/test_hermes_bootstrap.py # tests/test_hermes_constants.py # tests/test_hermes_logging.py # tests/test_managed_runtime_resolution.py # tests/test_model_tools_async_bridge.py # tests/test_packaging_build_guard.py # tests/test_packaging_metadata.py # tests/test_yaml_indent_consistency.py # tests/tools/test_approval_timeout_overflow.py # tests/tools/test_base_environment.py # tests/tools/test_bot_mode_dm.py # tests/tools/test_browser_chromium_check.py # tests/tools/test_browser_hardening.py # tests/tools/test_browser_homebrew_paths.py # tests/tools/test_browser_npx_warmup.py # tests/tools/test_browser_orphan_reaper.py # tests/tools/test_browser_real_profile.py # tests/tools/test_browser_use_cli.py # tests/tools/test_clipboard.py # tests/tools/test_code_execution.py # tests/tools/test_code_execution_modes.py # tests/tools/test_code_execution_windows_env.py # tests/tools/test_computer_use.py # tests/tools/test_delegate_liveness_timeout.py # tests/tools/test_execute_code_approval_cluster.py # tests/tools/test_execution_flag_detection.py # tests/tools/test_fal_common.py # tests/tools/test_file_operations.py # tests/tools/test_file_tools.py # tests/tools/test_file_tools_cwd_resolution.py # tests/tools/test_file_tools_live.py # tests/tools/test_lazy_deps.py # tests/tools/test_lazy_deps_durable_target.py # tests/tools/test_lazy_deps_managed.py # tests/tools/test_local_env_blocklist.py # tests/tools/test_local_tempdir.py # tests/tools/test_macos_protected_search.py # tests/tools/test_mcp_npx_cached_bin.py # tests/tools/test_oneshot_completion_linger.py # tests/tools/test_process_registry.py # tests/tools/test_read_file_schema_gating.py # tests/tools/test_skill_improvements.py # tests/tools/test_skills_sync.py # tests/tools/test_termux_api_detection.py # tests/tools/test_tirith_security.py # tests/tools/test_transcription_tools.py # tests/tools/test_tts_streaming.py # tests/tools/test_wake_word.py # tests/tui_gateway/test_compute_host_borrowed_lease.py # tests/tui_gateway/test_compute_host_turn_protocol.py # tests/tui_gateway/test_isolated_orphan_activity.py # tests/tui_gateway/test_protocol.py # tests/tui_gateway/test_slash_worker_profile_home.py # tests/tui_gateway/test_subprocess_encoding.py # tests/tui_gateway/test_tui_gateway_server.py # ui-tui/src/__tests__/terminalParity.test.ts # ui-tui/src/__tests__/termuxComposerLayout.test.ts # ui-tui/src/__tests__/textInputFastEcho.test.ts
376 lines
15 KiB
Python
376 lines
15 KiB
Python
"""Load / stress test for the Anthropic OAuth cross-process refresh race fix.
|
|
|
|
Companion to ``tests/agent/test_credential_pool_anthropic_refresh_race.py``,
|
|
which proves the bug in isolation with two racers. This test scales the same scenario up to look for bottlenecks and degradation
|
|
under real concurrency. The thread stress case keeps the suite fast while a
|
|
separate spawn-based case uses independent interpreters, distinct profile
|
|
homes, and one shared Claude Code credentials file. Both exercise the REAL
|
|
cross-process file lock (``_auth_store_lock``) and REAL credential-pool
|
|
persistence under throwaway directories — only the network call to Anthropic
|
|
is faked. The process case also counts refresh POSTs and requires exactly one
|
|
use of the stale single-use token, so a broken lock cannot remain green merely
|
|
because two in-process mocks happened to finish quickly.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import multiprocessing as mp
|
|
import os
|
|
import queue
|
|
import threading
|
|
import time
|
|
from dataclasses import replace as dc_replace
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from agent.credential_pool import (
|
|
AUTH_TYPE_OAUTH,
|
|
STATUS_EXHAUSTED,
|
|
CredentialPool,
|
|
PooledCredential,
|
|
)
|
|
|
|
CONCURRENCY = 20
|
|
|
|
|
|
def _process_claude_code_refresh_worker(
|
|
profile_home: str,
|
|
shared_credentials_path: str,
|
|
server_state_path: str,
|
|
start_event,
|
|
result_queue,
|
|
) -> None:
|
|
"""Refresh one shared Claude Code credential from an independent process."""
|
|
os.environ["HERMES_HOME"] = profile_home
|
|
|
|
from agent import anthropic_credentials as anthropic_mod
|
|
from agent import credential_pool as credential_pool_mod
|
|
from hermes_cli import auth as auth_mod
|
|
|
|
shared_path = Path(shared_credentials_path)
|
|
server_path = Path(server_state_path)
|
|
|
|
def read_shared_credentials():
|
|
data = json.loads(shared_path.read_text(encoding="utf-8"))
|
|
oauth = data["claudeAiOauth"]
|
|
return {
|
|
"accessToken": oauth["accessToken"],
|
|
"refreshToken": oauth.get("refreshToken", ""),
|
|
"expiresAt": oauth.get("expiresAt", 0),
|
|
"source": "claude_code_credentials_file",
|
|
}
|
|
|
|
def write_shared_credentials(access_token, refresh_token, expires_at_ms, **_kwargs):
|
|
data = json.loads(shared_path.read_text(encoding="utf-8"))
|
|
data["claudeAiOauth"] = {
|
|
"accessToken": access_token,
|
|
"refreshToken": refresh_token,
|
|
"expiresAt": expires_at_ms,
|
|
}
|
|
shared_path.write_text(json.dumps(data), encoding="utf-8")
|
|
|
|
def fake_refresh(refresh_token, *, use_json=False):
|
|
# The state file models a single-use token endpoint. The lock here
|
|
# protects only the fake server's accounting; the production lock is
|
|
# what must ensure that the second Hermes process never calls this
|
|
# function after the first one has rotated the shared credential.
|
|
with auth_mod._auth_store_lock(timeout_seconds=10, target_path=server_path):
|
|
state = json.loads(server_path.read_text(encoding="utf-8"))
|
|
state["calls"].append(refresh_token)
|
|
if refresh_token in state["spent"]:
|
|
server_path.write_text(json.dumps(state), encoding="utf-8")
|
|
raise ValueError("invalid_grant: refresh token already used")
|
|
state["spent"].append(refresh_token)
|
|
state["rotation"] += 1
|
|
rotation = state["rotation"]
|
|
server_path.write_text(json.dumps(state), encoding="utf-8")
|
|
# Keep the simulated network operation inside the production shared
|
|
# lock long enough for the second profile to prove it waits, then
|
|
# re-reads the newly-written shared credentials file.
|
|
time.sleep(0.1)
|
|
return {
|
|
"access_token": f"process-access-{rotation}",
|
|
"refresh_token": f"process-refresh-{rotation}",
|
|
"expires_at_ms": int(time.time() * 1000) + 3_600_000,
|
|
}
|
|
|
|
# Keep this worker hermetic: each profile has its own auth store, while
|
|
# both workers deliberately point at the same Claude credential source.
|
|
auth_mod._global_auth_file_path = lambda: None
|
|
anthropic_mod.claude_code_credentials_path = lambda: shared_path
|
|
anthropic_mod.read_claude_code_credentials = read_shared_credentials
|
|
anthropic_mod._write_claude_code_credentials = write_shared_credentials
|
|
anthropic_mod.refresh_anthropic_oauth_pure = fake_refresh
|
|
|
|
result_queue.put({"kind": "ready", "pid": os.getpid()})
|
|
if not start_event.wait(timeout=10):
|
|
result_queue.put({"kind": "result", "ok": False, "error": "start barrier timeout"})
|
|
return
|
|
|
|
entry = _entry(id="pool-entry", refresh_token="stale-rt", source="claude_code")
|
|
pool = credential_pool_mod.CredentialPool("anthropic", [entry])
|
|
try:
|
|
refreshed = pool._refresh_entry(pool.entries()[0], force=True)
|
|
result_queue.put({
|
|
"kind": "result",
|
|
"ok": refreshed is not None,
|
|
"refresh_token": refreshed.refresh_token if refreshed else None,
|
|
"pool_refresh_token": pool.entries()[0].refresh_token,
|
|
})
|
|
except BaseException as exc: # pragma: no cover - failure diagnostics
|
|
result_queue.put({"kind": "result", "ok": False, "error": repr(exc)})
|
|
|
|
|
|
def _entry(*, id: str, refresh_token: str, source: str) -> PooledCredential:
|
|
return PooledCredential(
|
|
provider="anthropic",
|
|
id=id,
|
|
label="anthropic oauth",
|
|
auth_type=AUTH_TYPE_OAUTH,
|
|
priority=0,
|
|
source=source,
|
|
access_token="stale-at",
|
|
refresh_token=refresh_token,
|
|
expires_at_ms=0,
|
|
)
|
|
|
|
|
|
class _SingleUseTokenServer:
|
|
"""Same single-use-refresh-token contract as the race test, tuned for
|
|
a wider fan-out (more callers, less per-call delay so the suite stays
|
|
fast while still exercising real contention).
|
|
"""
|
|
|
|
def __init__(self, delay_seconds: float = 0.02) -> None:
|
|
self._lock = threading.Lock()
|
|
self._spent: set[str] = set()
|
|
self._rotation = 0
|
|
self.calls: list[str] = []
|
|
self.delay_seconds = delay_seconds
|
|
|
|
def refresh(self, refresh_token: str, *, use_json: bool = False):
|
|
with self._lock:
|
|
self.calls.append(refresh_token)
|
|
time.sleep(self.delay_seconds)
|
|
with self._lock:
|
|
if refresh_token in self._spent:
|
|
raise ValueError("invalid_grant: refresh token already used")
|
|
self._spent.add(refresh_token)
|
|
self._rotation += 1
|
|
rotation = self._rotation
|
|
return {
|
|
"access_token": f"sk-ant-oat-rotated-{rotation}",
|
|
"refresh_token": f"sk-ant-ort-rotated-{rotation}",
|
|
"expires_at_ms": int(time.time() * 1000) + 3_600_000,
|
|
}
|
|
|
|
|
|
@pytest.fixture
|
|
def hermes_home(tmp_path, monkeypatch):
|
|
"""Real, throwaway HERMES_HOME so _auth_store_lock and
|
|
write_credential_pool/read_credential_pool exercise the genuine
|
|
file-lock + on-disk persistence path, not a mock.
|
|
"""
|
|
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
|
return tmp_path
|
|
|
|
|
|
def test_high_concurrency_anthropic_refresh_no_lost_updates_no_deadlock(
|
|
hermes_home, monkeypatch
|
|
):
|
|
"""CONCURRENCY 'Hermes processes' race the same stale refresh token
|
|
against the real cross-process lock + real on-disk pool persistence.
|
|
|
|
Every participant must end up with a usable, non-exhausted credential and
|
|
the single-use stale token must be POSTed exactly once.
|
|
"""
|
|
server = _SingleUseTokenServer(delay_seconds=0.02)
|
|
monkeypatch.setattr(
|
|
"agent.anthropic_credentials.refresh_anthropic_oauth_pure",
|
|
lambda refresh_token, use_json=False: server.refresh(refresh_token, use_json=use_json),
|
|
)
|
|
monkeypatch.setattr(
|
|
"agent.anthropic_credentials.read_claude_code_credentials", lambda: None
|
|
)
|
|
|
|
shared_stale_entry = _entry(
|
|
id="pool-entry", refresh_token="stale-rt", source="manual:hermes_pkce"
|
|
)
|
|
pools = [
|
|
CredentialPool("anthropic", [dc_replace(shared_stale_entry)])
|
|
for _ in range(CONCURRENCY)
|
|
]
|
|
|
|
results: dict[int, object] = {}
|
|
errors: dict[int, BaseException] = {}
|
|
|
|
def _run(idx: int) -> None:
|
|
try:
|
|
entry = pools[idx].entries()[0]
|
|
results[idx] = pools[idx]._refresh_entry(entry, force=True)
|
|
except BaseException as exc: # pragma: no cover - failure diagnostics
|
|
errors[idx] = exc
|
|
|
|
threads = [threading.Thread(target=_run, args=(i,)) for i in range(CONCURRENCY)]
|
|
start = time.monotonic()
|
|
for t in threads:
|
|
t.start()
|
|
# Generous per-thread join budget: a correct implementation serializes
|
|
# through one file lock, so worst case is roughly
|
|
# CONCURRENCY * (delay + lock overhead), well under this ceiling. A
|
|
# deadlock or livelock would blow straight through it.
|
|
deadline = start + max(10.0, CONCURRENCY * server.delay_seconds * 5)
|
|
for t in threads:
|
|
remaining = max(0.1, deadline - time.monotonic())
|
|
t.join(timeout=remaining)
|
|
|
|
still_alive = [t for t in threads if t.is_alive()]
|
|
assert not still_alive, (
|
|
f"{len(still_alive)}/{CONCURRENCY} threads never finished -- "
|
|
"possible deadlock in the cross-process refresh lock."
|
|
)
|
|
assert not errors, f"unexpected exceptions during concurrent refresh: {errors!r}"
|
|
|
|
assert len(results) == CONCURRENCY
|
|
assert all(r is not None for r in results.values()), (
|
|
"at least one of the concurrent processes could not recover a "
|
|
"usable Anthropic credential after the refresh race"
|
|
)
|
|
for idx, pool in enumerate(pools):
|
|
entry_after = pool.entries()[0]
|
|
assert entry_after.last_status != STATUS_EXHAUSTED, (
|
|
f"process {idx} ended up with an exhausted Anthropic credential "
|
|
"despite valid tokens existing on disk"
|
|
)
|
|
# Deterministic replacement for the old wall-clock bound: the single-use
|
|
# stale token is POSTed exactly once, however many racers there are.
|
|
assert server.calls.count("stale-rt") == 1, server.calls
|
|
|
|
|
|
@pytest.mark.live_system_guard_bypass
|
|
@pytest.mark.platforms("windows")
|
|
def test_distinct_profiles_share_one_claude_refresh_without_duplicate_post(
|
|
hermes_home,
|
|
):
|
|
"""Independent profiles must serialize a shared Claude Code refresh.
|
|
|
|
The profile auth locks intentionally have different paths here; only the
|
|
dedicated lock keyed to the shared Claude credentials file can prevent the
|
|
second process from POSTing the already-spent refresh token.
|
|
"""
|
|
shared_credentials_path = hermes_home / "shared-claude-credentials.json"
|
|
shared_credentials_path.write_text(
|
|
json.dumps({
|
|
"claudeAiOauth": {
|
|
"accessToken": "stale-at",
|
|
"refreshToken": "stale-rt",
|
|
"expiresAt": 0,
|
|
}
|
|
}),
|
|
encoding="utf-8",
|
|
)
|
|
server_state_path = hermes_home / "fake-token-server.json"
|
|
server_state_path.write_text(
|
|
json.dumps({"calls": [], "spent": [], "rotation": 0}),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
profile_homes = [hermes_home / "profile-a", hermes_home / "profile-b"]
|
|
for profile_home in profile_homes:
|
|
profile_home.mkdir(parents=True)
|
|
(profile_home / "auth.json").write_text(
|
|
json.dumps({
|
|
"version": 1,
|
|
"providers": {},
|
|
# claude_code is a borrowed source; its raw tokens must not
|
|
# be persisted in a profile pool. Each worker constructs the
|
|
# runtime entry from the shared credential source below.
|
|
"credential_pool": {},
|
|
}),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
ctx = mp.get_context("spawn")
|
|
start_event = ctx.Event()
|
|
result_queue = ctx.Queue()
|
|
processes = [
|
|
ctx.Process(
|
|
target=_process_claude_code_refresh_worker,
|
|
args=(
|
|
str(profile_home),
|
|
str(shared_credentials_path),
|
|
str(server_state_path),
|
|
start_event,
|
|
result_queue,
|
|
),
|
|
)
|
|
for profile_home in profile_homes
|
|
]
|
|
|
|
messages = []
|
|
try:
|
|
for process in processes:
|
|
process.start()
|
|
|
|
ready_deadline = time.monotonic() + 20.0
|
|
while len([m for m in messages if m.get("kind") == "ready"]) < len(processes):
|
|
remaining = max(0.1, ready_deadline - time.monotonic())
|
|
if remaining <= 0.1:
|
|
break
|
|
try:
|
|
messages.append(result_queue.get(timeout=remaining))
|
|
except queue.Empty:
|
|
break
|
|
assert len([m for m in messages if m.get("kind") == "ready"]) == len(processes), (
|
|
f"not all refresh workers reached the start barrier: {messages!r}"
|
|
)
|
|
start_event.set()
|
|
|
|
for process in processes:
|
|
process.join(timeout=30)
|
|
assert not [process for process in processes if process.is_alive()], (
|
|
"a profile refresh worker did not finish; possible shared-lock deadlock"
|
|
)
|
|
|
|
result_deadline = time.monotonic() + 5.0
|
|
results = [m for m in messages if m.get("kind") == "result"]
|
|
while len(results) < len(processes) and time.monotonic() < result_deadline:
|
|
try:
|
|
message = result_queue.get(timeout=0.5)
|
|
except queue.Empty:
|
|
break
|
|
messages.append(message)
|
|
if message.get("kind") == "result":
|
|
results.append(message)
|
|
finally:
|
|
start_event.set()
|
|
for process in processes:
|
|
process.join(timeout=2)
|
|
for process in processes:
|
|
if process.is_alive():
|
|
process.kill()
|
|
process.join(timeout=5)
|
|
result_queue.close()
|
|
result_queue.join_thread()
|
|
|
|
assert len(results) == len(processes), f"missing process results: {messages!r}"
|
|
assert all(result.get("ok") for result in results), results
|
|
assert {result.get("refresh_token") for result in results} == {"process-refresh-1"}
|
|
assert all((profile_home / "auth.lock").exists() for profile_home in profile_homes)
|
|
assert shared_credentials_path.with_suffix(".lock").exists()
|
|
|
|
server_state = json.loads(server_state_path.read_text(encoding="utf-8"))
|
|
assert server_state["calls"] == ["stale-rt"], (
|
|
"the shared stale refresh token must be POSTed exactly once across "
|
|
f"distinct profiles, got {server_state['calls']!r}"
|
|
)
|
|
assert server_state["spent"] == ["stale-rt"]
|
|
|
|
shared_credentials = json.loads(shared_credentials_path.read_text(encoding="utf-8"))
|
|
assert shared_credentials["claudeAiOauth"]["refreshToken"] == "process-refresh-1"
|
|
for profile_home in profile_homes:
|
|
profile_text = (profile_home / "auth.json").read_text(encoding="utf-8")
|
|
assert "stale-rt" not in profile_text
|
|
assert "process-refresh-1" not in profile_text
|