mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-16 19:36:54 +00:00
# Conflicts: # apps/desktop/e2e/archived-hidden-session-recoverable.spec.ts # apps/desktop/e2e/bot-chat-message-agent-friendly-name.spec.ts # apps/desktop/e2e/bot-mailbox-unreadable-ticket.spec.ts # apps/desktop/e2e/bot-mode-roster-localized.spec.ts # apps/desktop/e2e/bot-mode-row-click-mirrors-registry.spec.ts # apps/desktop/e2e/bot-mode-tab-shows-bot-name.spec.ts # apps/desktop/e2e/bot-roster-group-row-organisation.spec.ts # apps/desktop/e2e/bot-roster-ignores-infra-dirs.spec.ts # apps/desktop/e2e/bot-roster-timestamp-meta.spec.ts # apps/desktop/e2e/bot-roster-user-sections.spec.ts # apps/desktop/e2e/bot-routines-pane-narrow.spec.ts # apps/desktop/e2e/bot-row-open-recent-session.spec.ts # apps/desktop/e2e/bot-tile-ignores-ambient-composer-model.spec.ts # apps/desktop/e2e/group-composer-auto-grow.spec.ts # apps/desktop/e2e/group-create-gate-remote-roster.spec.ts # apps/desktop/e2e/group-prompt-renamed-primary-handle.spec.ts # apps/desktop/e2e/hosted-room-backend-continuity.spec.ts # apps/desktop/e2e/hosted-room-legacy-store-migration.spec.ts # apps/desktop/e2e/settings-scope-chips-bot-title.spec.ts # apps/desktop/e2e/worktree-branch-status.spec.ts # apps/desktop/electron/backend-probes.test.ts # apps/desktop/electron/connection-apply.test.ts # apps/desktop/electron/desktop-electron-pin.test.ts # apps/desktop/electron/desktop-uninstall.test.ts # apps/desktop/electron/gateway-file-download-transport.test.ts # apps/desktop/electron/gateway-stop-before-update.test.ts # apps/desktop/electron/github-api-auth.test.ts # apps/desktop/electron/registry-primary-profile-scope.test.ts # apps/desktop/electron/update-api-check.test.ts # apps/desktop/electron/update-handoff-marker.test.ts # apps/desktop/electron/venv-blocker-scan.test.ts # apps/desktop/scripts/after-extract.test.mjs # apps/desktop/scripts/local-pack-publish.test.mjs # apps/desktop/scripts/tasks-scroll.test.mjs # apps/desktop/src/app/settings/model-settings.test.tsx # apps/desktop/src/app/updates-overlay.blockers.test.tsx # apps/desktop/src/components/desktop-install-overlay.test.tsx # apps/desktop/src/lib/update-copy.test.ts # scripts/ci/check_os_marker_fakes.py # tests-js/desktop-mac-usage-descriptions.test.ts # tests-js/node-engine-alignment.test.ts # tests/agent/lsp/test_install_and_lint_fixes.py # tests/agent/test_command_token_source.py # tests/agent/test_compression_boundary_hook.py # tests/agent/test_create_openai_client_ssl_verify.py # tests/agent/test_custom_provider_ca_probes.py # tests/agent/test_endpoint_blackhole.py # tests/agent/test_estimator_parity.py # tests/agent/test_in_place_compaction.py # tests/agent/test_moa_loop_mode.py # tests/agent/test_model_metadata.py # tests/agent/test_skill_session_platform_gate.py # tests/agent/test_skill_utils.py # tests/agent/test_ssl_ca_guard.py # tests/computer_use/test_doctor.py # tests/cron/test_codex_execution_paths.py # tests/cron/test_cron_bot_chat_delivery.py # tests/cron/test_cron_script.py # tests/cron/test_media_delivery_parity.py # tests/cron/test_misfire_catchup.py # tests/cron/test_parallel_pool.py # tests/cron/test_recurring_eagain_redispatch.py # tests/gateway/test_choice_picker.py # tests/gateway/test_control_socket_windows_live.py # tests/gateway/test_dingtalk.py # tests/gateway/test_feishu.py # tests/gateway/test_feishu_onboard.py # tests/gateway/test_gateway_shutdown.py # tests/gateway/test_matrix.py # tests/gateway/test_model_command_custom_providers.py # tests/gateway/test_reasoning_command.py # tests/gateway/test_runtime_footer.py # tests/gateway/test_session.py # tests/gateway/test_session_hygiene.py # tests/gateway/test_status.py # tests/gateway/test_teams.py # tests/gateway/test_turn_lease.py # tests/gateway/test_whatsapp_connect.py # tests/hermes_cli/test_approvals_command.py # tests/hermes_cli/test_auth_store_lock_concurrent.py # tests/hermes_cli/test_backup.py # tests/hermes_cli/test_banner_git_state.py # tests/hermes_cli/test_certifi_repair.py # tests/hermes_cli/test_cmd_update.py # tests/hermes_cli/test_compat_manifest_targets.py # tests/hermes_cli/test_computer_use_cli.py # tests/hermes_cli/test_cpr_local_leak.py # tests/hermes_cli/test_dashboard_auth_gate.py # tests/hermes_cli/test_dashboard_procs_kill_grace.py # tests/hermes_cli/test_desktop_lifecycle_windows_live.py # tests/hermes_cli/test_doctor.py # tests/hermes_cli/test_doctor_command_install.py # tests/hermes_cli/test_fleet_config_migration_windows_live.py # tests/hermes_cli/test_gateway.py # tests/hermes_cli/test_gateway_platform_gating.py # tests/hermes_cli/test_gateway_restart_loop.py # tests/hermes_cli/test_gateway_task_probe.py # tests/hermes_cli/test_gateway_wsl.py # tests/hermes_cli/test_gui_command.py # tests/hermes_cli/test_install_cua_driver.py # tests/hermes_cli/test_kanban_db.py # tests/hermes_cli/test_lazy_command_exports.py # tests/hermes_cli/test_lazy_refresh_venv_repair.py # tests/hermes_cli/test_linux_desktop_entry.py # tests/hermes_cli/test_local_runtime.py # tests/hermes_cli/test_local_runtime_updates.py # tests/hermes_cli/test_managed_uv.py # tests/hermes_cli/test_mcp_reload_confirm_gate.py # tests/hermes_cli/test_nous_subscription.py # tests/hermes_cli/test_npm_engine.py # tests/hermes_cli/test_personality_none.py # tests/hermes_cli/test_pet_toggle.py # tests/hermes_cli/test_plan_reconciliation_windows_live.py # tests/hermes_cli/test_plugin_event_bus.py # tests/hermes_cli/test_plugin_manifest_v2.py # tests/hermes_cli/test_plugin_packs.py # tests/hermes_cli/test_plugins_cmd.py # tests/hermes_cli/test_plugins_cmd_enable_disable_nested.py # tests/hermes_cli/test_process_identity.py # tests/hermes_cli/test_profiles.py # tests/hermes_cli/test_profiles_sidebar_cache.py # tests/hermes_cli/test_pty_bridge.py # tests/hermes_cli/test_resolve_turn_limit.py # tests/hermes_cli/test_serve_runtime_inventory.py # tests/hermes_cli/test_session_vacuum_config.py # tests/hermes_cli/test_set_config_value.py # tests/hermes_cli/test_signal_handler_kanban_worker.py # tests/hermes_cli/test_slash_confirm_windows.py # tests/hermes_cli/test_stale_pid_guard.py # tests/hermes_cli/test_startup_fast_guards.py # tests/hermes_cli/test_status.py # tests/hermes_cli/test_telegram_managed_bot.py # tests/hermes_cli/test_tools_config.py # tests/hermes_cli/test_update_apply_shallow_count.py # tests/hermes_cli/test_update_autostash.py # tests/hermes_cli/test_update_concurrent_quarantine.py # tests/hermes_cli/test_update_fetch_failure_classifier.py # tests/hermes_cli/test_update_fleet_probe_resume_token.py # tests/hermes_cli/test_update_handoff_backend_reap.py # tests/hermes_cli/test_update_handoff_desktop_rebuild.py # tests/hermes_cli/test_update_head_moved_gate.py # tests/hermes_cli/test_update_host_obligation.py # tests/hermes_cli/test_update_import_guard.py # tests/hermes_cli/test_update_interrupted_recovery.py # tests/hermes_cli/test_update_inventory.py # tests/hermes_cli/test_update_launchd_unloaded_gateway.py # tests/hermes_cli/test_update_missing_configured_deps.py # tests/hermes_cli/test_update_modified_notice.py # tests/hermes_cli/test_update_multiplex_migration_hook.py # tests/hermes_cli/test_update_no_gateway_restart.py # tests/hermes_cli/test_update_orphan_backend_reap.py # tests/hermes_cli/test_update_parked_branch_guard.py # tests/hermes_cli/test_update_post_pull_syntax_guard.py # tests/hermes_cli/test_update_receipt.py # tests/hermes_cli/test_update_self_lock.py # tests/hermes_cli/test_update_shim_fail_closed.py # tests/hermes_cli/test_update_shim_self_lock.py # tests/hermes_cli/test_update_sqlite_remediation.py # tests/hermes_cli/test_update_stale_dashboard.py # tests/hermes_cli/test_update_stale_virtualenv.py # tests/hermes_cli/test_update_venv_health.py # tests/hermes_cli/test_update_venv_ownership_preflight.py # tests/hermes_cli/test_update_wedged_gateway.py # tests/hermes_cli/test_update_yes_flag.py # tests/hermes_cli/test_update_zip_two_phase.py # tests/hermes_cli/test_urllib_security.py # tests/hermes_cli/test_ux_messages_auth_config.py # tests/hermes_cli/test_ux_messages_startup.py # tests/hermes_cli/test_venv_holder_classifier.py # tests/hermes_cli/test_verify_console_scripts.py # tests/hermes_cli/test_verify_core_dependencies.py # tests/hermes_cli/test_web_server.py # tests/hermes_cli/test_web_server_console_ws.py # tests/hermes_cli/test_web_server_ws_ping.py # tests/hermes_cli/test_web_ui_build.py # tests/hermes_state/test_fts_rebuild_admission.py # tests/hermes_state/test_hermes_state.py # tests/plugins/memory/test_memory_lazy_install.py # tests/plugins/test_google_meet_plugin.py # tests/plugins/test_langfuse_plugin.py # tests/plugins/test_security_guidance_plugin.py # tests/plugins/test_transform_llm_output_hook.py # tests/scripts/desktop_update/test_desktop_update_windows_gateway_flag.py # tests/scripts/desktop_update/test_desktop_update_windows_python_handoff.py # tests/scripts/desktop_update/test_desktop_update_windows_timestamp.py # tests/scripts/install/test_install_clone_throttle_fallback.py # tests/scripts/install/test_install_lockfile_churn.py # tests/scripts/install/test_install_no_initial_commit.py # tests/scripts/install/test_install_sh_browser_install.py # tests/scripts/install/test_install_sh_node_prerelease.py # tests/scripts/install/test_install_sh_symlink_stomp.py # tests/scripts/install/test_install_sh_uv_lock_config.py # tests/scripts/install/test_install_unmerged_index.py # tests/scripts/test_contributor_map.py # tests/scripts/test_run_tests_parallel.py # tests/skills/test_competitor_news_monitor_skill.py # tests/skills/test_document_to_action_items_skill.py # tests/skills/test_google_workspace_setup.py # tests/skills/test_google_workspace_setup_deps.py # tests/skills/test_grounded_citations_skill.py # tests/skills/test_ip_as_logo_skill.py # tests/skills/test_live_dashboard_skill.py # tests/skills/test_mcp_oauth_remote_gateway_skill.py # tests/skills/test_office_document_skills.py # tests/skills/test_openclaw_migration.py # tests/skills/test_product_price_monitor_skill.py # tests/skills/test_scrollcraft_skill.py # tests/skills/test_setup_wizard_generator_skill.py # tests/skills/test_weekly_review_planning_skill.py # tests/test_engines_satisfiable.py # tests/test_fast_safe_load.py # tests/test_hermes_bootstrap.py # tests/test_hermes_constants.py # tests/test_hermes_logging.py # tests/test_managed_runtime_resolution.py # tests/test_model_tools_async_bridge.py # tests/test_packaging_build_guard.py # tests/test_packaging_metadata.py # tests/test_yaml_indent_consistency.py # tests/tools/test_approval_timeout_overflow.py # tests/tools/test_base_environment.py # tests/tools/test_bot_mode_dm.py # tests/tools/test_browser_chromium_check.py # tests/tools/test_browser_hardening.py # tests/tools/test_browser_homebrew_paths.py # tests/tools/test_browser_npx_warmup.py # tests/tools/test_browser_orphan_reaper.py # tests/tools/test_browser_real_profile.py # tests/tools/test_browser_use_cli.py # tests/tools/test_clipboard.py # tests/tools/test_code_execution.py # tests/tools/test_code_execution_modes.py # tests/tools/test_code_execution_windows_env.py # tests/tools/test_computer_use.py # tests/tools/test_delegate_liveness_timeout.py # tests/tools/test_execute_code_approval_cluster.py # tests/tools/test_execution_flag_detection.py # tests/tools/test_fal_common.py # tests/tools/test_file_operations.py # tests/tools/test_file_tools.py # tests/tools/test_file_tools_cwd_resolution.py # tests/tools/test_file_tools_live.py # tests/tools/test_lazy_deps.py # tests/tools/test_lazy_deps_durable_target.py # tests/tools/test_lazy_deps_managed.py # tests/tools/test_local_env_blocklist.py # tests/tools/test_local_tempdir.py # tests/tools/test_macos_protected_search.py # tests/tools/test_mcp_npx_cached_bin.py # tests/tools/test_oneshot_completion_linger.py # tests/tools/test_process_registry.py # tests/tools/test_read_file_schema_gating.py # tests/tools/test_skill_improvements.py # tests/tools/test_skills_sync.py # tests/tools/test_termux_api_detection.py # tests/tools/test_tirith_security.py # tests/tools/test_transcription_tools.py # tests/tools/test_tts_streaming.py # tests/tools/test_wake_word.py # tests/tui_gateway/test_compute_host_borrowed_lease.py # tests/tui_gateway/test_compute_host_turn_protocol.py # tests/tui_gateway/test_isolated_orphan_activity.py # tests/tui_gateway/test_protocol.py # tests/tui_gateway/test_slash_worker_profile_home.py # tests/tui_gateway/test_subprocess_encoding.py # tests/tui_gateway/test_tui_gateway_server.py # ui-tui/src/__tests__/terminalParity.test.ts # ui-tui/src/__tests__/termuxComposerLayout.test.ts # ui-tui/src/__tests__/textInputFastEcho.test.ts
1112 lines
47 KiB
Python
1112 lines
47 KiB
Python
"""Tests for OAuth 401 recovery: prompt exchange retry, invalid_grant handling,
|
|
forced refresh + single retry on sync and dialectic, backoff exemption, and
|
|
the one-time user-facing notice."""
|
|
|
|
import json
|
|
import time
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from unittest.mock import MagicMock
|
|
|
|
import pytest
|
|
|
|
from plugins.memory.honcho import HonchoMemoryProvider
|
|
from plugins.memory.honcho import oauth
|
|
from plugins.memory.honcho.client import HonchoClientConfig
|
|
from plugins.memory.honcho.session import (
|
|
HonchoAuthError,
|
|
HonchoSession,
|
|
HonchoSessionManager,
|
|
)
|
|
from plugins.memory.honcho.session_auth import _is_auth_error
|
|
|
|
def _host_block(refresh="hch-rt-old", expires_at=100):
|
|
return {
|
|
"apiKey": "hch-at-old",
|
|
"oauth": {
|
|
"refreshToken": refresh,
|
|
"expiresAt": expires_at,
|
|
"clientId": "hermes-desktop",
|
|
"tokenEndpoint": "http://localhost:8000/oauth/token",
|
|
"scope": "write",
|
|
"tokenType": "Bearer",
|
|
},
|
|
}
|
|
|
|
def _write(path: Path, raw: dict) -> None:
|
|
path.write_text(json.dumps(raw), encoding="utf-8")
|
|
|
|
def _rotated_body(n=1):
|
|
return {
|
|
"access_token": f"hch-at-new{n}",
|
|
"refresh_token": f"hch-rt-new{n}",
|
|
"expires_in": 3600,
|
|
"scope": "write",
|
|
"token_type": "Bearer",
|
|
}
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _reset_oauth_module_state():
|
|
"""Module-level oauth dicts persist across tests in one process; reset so
|
|
dead grants / cooldowns / memoized verdicts can't leak between tests."""
|
|
yield
|
|
oauth._dead_grants.clear()
|
|
oauth._refresh_failure_at.clear()
|
|
oauth._reauth_check_cache.clear()
|
|
oauth._expiry_cache.clear()
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# oauth: transient vs permanent exchange failures
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestExchangeRetry:
|
|
def test_transient_failure_recovers_on_immediate_retry(self, tmp_path, monkeypatch):
|
|
"""A timed-out exchange retries right away — the server honors the
|
|
replayed refresh token only within its rotation grace window."""
|
|
path = tmp_path / "honcho.json"
|
|
_write(path, {"hosts": {"hermes": _host_block()}})
|
|
monkeypatch.setattr(oauth, "_REFRESH_RETRY_DELAY_SECONDS", 0)
|
|
|
|
calls = []
|
|
|
|
def flaky(url, data, timeout):
|
|
calls.append(data["refresh_token"])
|
|
if len(calls) == 1:
|
|
raise TimeoutError("token exchange timed out")
|
|
return 200, _rotated_body()
|
|
|
|
monkeypatch.setattr(oauth, "_http_post_form_status", flaky)
|
|
token, refreshed = oauth.ensure_fresh_token(path, "hermes", now=1000)
|
|
|
|
assert token == "hch-at-new1" and refreshed is True
|
|
assert calls == ["hch-rt-old", "hch-rt-old"]
|
|
saved = json.loads(path.read_text())["hosts"]["hermes"]
|
|
assert saved["oauth"]["refreshToken"] == "hch-rt-new1"
|
|
|
|
def test_invalid_grant_stops_retries_and_marks_reauth_required(self, tmp_path, monkeypatch):
|
|
path = tmp_path / "honcho.json"
|
|
_write(path, {"hosts": {"hermes": _host_block()}})
|
|
monkeypatch.setattr(oauth, "_REFRESH_RETRY_DELAY_SECONDS", 0)
|
|
|
|
calls = []
|
|
|
|
def revoked(url, data, timeout):
|
|
calls.append(1)
|
|
return 400, {"error": "invalid_grant", "error_description": "grant revoked"}
|
|
|
|
monkeypatch.setattr(oauth, "_http_post_form_status", revoked)
|
|
token, refreshed = oauth.ensure_fresh_token(path, "hermes", now=1000)
|
|
|
|
# Fail-open return, but no retry of a permanently rejected grant.
|
|
assert token == "hch-at-old" and refreshed is False
|
|
assert len(calls) == 1
|
|
assert oauth.reauth_required(path, "hermes") is True
|
|
|
|
# Later refresh attempts skip the endpoint entirely.
|
|
token2, refreshed2 = oauth.ensure_fresh_token(path, "hermes", now=2000)
|
|
assert token2 == "hch-at-old" and refreshed2 is False
|
|
assert len(calls) == 1
|
|
|
|
# The forced (post-401) path refuses a dead grant too.
|
|
assert oauth.force_refresh_token(path, "hermes") is None
|
|
assert len(calls) == 1
|
|
|
|
def test_relogin_clears_the_dead_grant(self, tmp_path, monkeypatch):
|
|
path = tmp_path / "honcho.json"
|
|
_write(path, {"hosts": {"hermes": _host_block()}})
|
|
monkeypatch.setattr(oauth, "_REFRESH_RETRY_DELAY_SECONDS", 0)
|
|
monkeypatch.setattr(
|
|
oauth, "_http_post_form_status",
|
|
lambda *a, **k: (400, {"error": "invalid_grant"}),
|
|
)
|
|
oauth.ensure_fresh_token(path, "hermes", now=1000)
|
|
assert oauth.reauth_required(path, "hermes") is True
|
|
|
|
oauth.install_grant(
|
|
path, "hermes",
|
|
{"access_token": "hch-at-fresh", "refresh_token": "hch-rt-fresh", "expires_in": 3600},
|
|
client_id="hermes-desktop",
|
|
token_endpoint="http://localhost:8000/oauth/token",
|
|
now=2000,
|
|
)
|
|
assert oauth.reauth_required(path, "hermes") is False
|
|
token, _ = oauth.ensure_fresh_token(path, "hermes", now=2000)
|
|
assert token == "hch-at-fresh"
|
|
|
|
def test_honcho_token_prefixes_are_registered_with_the_shared_redactor(self):
|
|
"""Importing the plugin registers hch-at-/hch-rt- with agent.redact, so every surface that
|
|
runs the shared redactor (logs, tool output, chat egress) masks Honcho tokens, not only
|
|
this module's own error strings."""
|
|
from agent.redact import redact_sensitive_text
|
|
redacted = redact_sensitive_text(
|
|
"exchange failed for hch-rt-supersecret123 got hch-at-alsosecret456", force=True
|
|
)
|
|
assert "supersecret123" not in redacted
|
|
assert "alsosecret456" not in redacted
|
|
|
|
class TestForceRefreshToken:
|
|
def test_rotates_despite_local_validity(self, tmp_path, monkeypatch):
|
|
"""A server-side 401 forces a rotation even when the local clock says
|
|
the token is still live."""
|
|
path = tmp_path / "honcho.json"
|
|
_write(path, {"hosts": {"hermes": _host_block(expires_at=time.time() + 3600)}})
|
|
monkeypatch.setattr(
|
|
oauth, "_http_post_form_status", lambda *a, **k: (200, _rotated_body())
|
|
)
|
|
token = oauth.force_refresh_token(path, "hermes")
|
|
assert token == "hch-at-new1"
|
|
saved = json.loads(path.read_text())["hosts"]["hermes"]
|
|
assert saved["apiKey"] == "hch-at-new1"
|
|
|
|
def test_adopts_concurrent_rotation_without_exchange(self, tmp_path, monkeypatch):
|
|
path = tmp_path / "honcho.json"
|
|
far = time.time() + 7200
|
|
_write(path, {"hosts": {"hermes": _host_block(expires_at=far)}})
|
|
# Seed the expiry cache with the old token.
|
|
oauth.ensure_fresh_token(path, "hermes")
|
|
# Another process rotated the credential on disk.
|
|
rotated = _host_block(refresh="hch-rt-2", expires_at=far)
|
|
rotated["apiKey"] = "hch-at-2"
|
|
_write(path, {"hosts": {"hermes": rotated}})
|
|
monkeypatch.setattr(
|
|
oauth, "_http_post_form_status",
|
|
lambda *a, **k: pytest.fail("must adopt the on-disk rotation, not exchange"),
|
|
)
|
|
assert oauth.force_refresh_token(path, "hermes") == "hch-at-2"
|
|
|
|
@pytest.mark.parametrize("rotated_by", ["first waiter", "sibling process", "sibling process during our cooldown"])
|
|
def test_401_on_a_bearer_disk_has_moved_off_adopts_without_exchange(self, tmp_path, monkeypatch, rotated_by):
|
|
"""The failing bearer disagreeing with disk is enough to adopt: a replayed refresh token can
|
|
revoke the grant, and the expiry cache is empty in a sibling process. Adopting is a disk read,
|
|
so our own recent failed exchange (the cooldown) does not block it."""
|
|
path = tmp_path / "honcho.json"
|
|
far = time.time() + 7200
|
|
_write(path, {"hosts": {"hermes": _host_block(expires_at=far)}})
|
|
if rotated_by == "first waiter":
|
|
monkeypatch.setattr(oauth, "_http_post_form_status", lambda *a, **k: (200, {**_rotated_body(2), "expires_in": 7200}))
|
|
assert oauth.force_refresh_token(path, "hermes", failed_access_token="hch-at-old") == "hch-at-new2"
|
|
else:
|
|
_write(path, {"hosts": {"hermes": {**_host_block(refresh="hch-rt-new2", expires_at=far), "apiKey": "hch-at-new2"}}})
|
|
oauth._expiry_cache.clear()
|
|
if rotated_by.endswith("cooldown"):
|
|
oauth._refresh_failure_at[(str(path), "hermes")] = time.monotonic()
|
|
monkeypatch.setattr(oauth, "_http_post_form_status", lambda *a, **k: pytest.fail("must adopt the on-disk grant, not exchange"))
|
|
assert oauth.force_refresh_token(path, "hermes", failed_access_token="hch-at-old") == "hch-at-new2"
|
|
assert oauth._expiry_cache[(str(path), "hermes")][1] == "hch-at-new2"
|
|
|
|
@pytest.mark.parametrize("sibling_persists", [True, False], ids=["disk rotated during exchange", "disk unchanged"])
|
|
def test_invalid_grant_adopts_a_rotation_that_landed_during_the_exchange(self, tmp_path, monkeypatch, sibling_persists):
|
|
"""The endpoint says invalid_grant because a sibling used the refresh token first. Its rotation
|
|
on disk is adopted; only an unchanged disk marks the grant dead."""
|
|
path = tmp_path / "honcho.json"
|
|
far = time.time() + 7200
|
|
_write(path, {"hosts": {"hermes": _host_block(expires_at=far)}})
|
|
monkeypatch.setattr(oauth, "_REFRESH_RETRY_DELAY_SECONDS", 0)
|
|
|
|
def exchange(url, data, timeout):
|
|
if sibling_persists:
|
|
_write(path, {"hosts": {"hermes": {**_host_block(refresh="hch-rt-sibling", expires_at=far), "apiKey": "hch-at-sibling"}}})
|
|
return 400, {"error": "invalid_grant", "error_description": "reuse detected"}
|
|
|
|
monkeypatch.setattr(oauth, "_http_post_form_status", exchange)
|
|
token = oauth.force_refresh_token(path, "hermes", failed_access_token="hch-at-old")
|
|
assert token == ("hch-at-sibling" if sibling_persists else None)
|
|
assert oauth.reauth_required(path, "hermes") is (not sibling_persists)
|
|
if sibling_persists:
|
|
assert oauth.ensure_fresh_token(path, "hermes", now=time.time())[0] == "hch-at-sibling"
|
|
|
|
def test_transient_failure_returns_none(self, tmp_path, monkeypatch):
|
|
path = tmp_path / "honcho.json"
|
|
_write(path, {"hosts": {"hermes": _host_block(expires_at=time.time() + 3600)}})
|
|
monkeypatch.setattr(oauth, "_REFRESH_RETRY_DELAY_SECONDS", 0)
|
|
|
|
def boom(*a, **k):
|
|
raise ConnectionError("network down")
|
|
|
|
monkeypatch.setattr(oauth, "_http_post_form_status", boom)
|
|
assert oauth.force_refresh_token(path, "hermes") is None
|
|
# Not permanent: a later attempt may exchange again.
|
|
assert oauth.reauth_required(path, "hermes") is False
|
|
|
|
def test_static_api_key_is_noop(self, tmp_path):
|
|
path = tmp_path / "honcho.json"
|
|
_write(path, {"hosts": {"hermes": {"apiKey": "hch-v3-static"}}})
|
|
assert oauth.force_refresh_token(path, "hermes") is None
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# session: auth error detection
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestAuthErrorDetection:
|
|
def test_matches_honcho_token_message(self):
|
|
assert _is_auth_error(Exception("Invalid or expired access token"))
|
|
|
|
def test_matches_status_code_attr(self):
|
|
exc = Exception("boom")
|
|
exc.status_code = 401
|
|
assert _is_auth_error(exc)
|
|
|
|
def test_ignores_other_errors(self):
|
|
assert not _is_auth_error(Exception("connection reset by peer"))
|
|
assert not _is_auth_error(Exception("HTTP 500 internal error"))
|
|
|
|
def test_bare_401_digits_are_not_auth_errors(self):
|
|
"""A false positive spends a token rotation and can revoke the grant;
|
|
digits appearing in latency figures, request ids, or identifiers must
|
|
never classify as auth failures."""
|
|
for msg in (
|
|
"Rate limited, retry after 4010 ms",
|
|
"500 Internal Server Error (request id req-4012ab)",
|
|
"connection timeout to workspace ws-401-prod",
|
|
"peer 401k-planning not found",
|
|
):
|
|
assert not _is_auth_error(Exception(msg)), msg
|
|
|
|
def test_401_with_http_context_matches(self):
|
|
assert _is_auth_error(Exception("HTTP 401"))
|
|
assert _is_auth_error(Exception("status 401"))
|
|
assert _is_auth_error(Exception("status_code: 401"))
|
|
assert _is_auth_error(Exception("401 Unauthorized"))
|
|
|
|
def test_concrete_non_auth_status_wins_over_text(self):
|
|
exc = Exception("authentication failed")
|
|
exc.status = 429
|
|
assert not _is_auth_error(exc)
|
|
|
|
def test_bare_authentication_word_is_not_enough(self):
|
|
assert not _is_auth_error(Exception("authentication service unreachable"))
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# session: dialectic 401 recovery
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class _FlakyPeer:
|
|
"""chat() raises an auth error N times, then succeeds."""
|
|
|
|
def __init__(self, failures: int, result: str = "synthesized answer"):
|
|
self.failures = failures
|
|
self.result = result
|
|
self.calls = 0
|
|
|
|
def chat(self, query, **kw):
|
|
self.calls += 1
|
|
if self.calls <= self.failures:
|
|
raise Exception("Invalid or expired access token")
|
|
return self.result
|
|
|
|
def _make_manager(peer, *, reauth_ok=True):
|
|
cfg = HonchoClientConfig(host="hermes", api_key="hch-at-x", enabled=True)
|
|
mgr = HonchoSessionManager(config=cfg)
|
|
session = HonchoSession(
|
|
key="k", user_peer_id="u", assistant_peer_id="a", honcho_session_id="s"
|
|
)
|
|
mgr._cache["k"] = session
|
|
mgr._get_or_create_peer = lambda peer_id: peer
|
|
mgr._force_reauth = lambda **kw: reauth_ok
|
|
return mgr
|
|
|
|
class TestDialecticAuthRetry:
|
|
def test_401_forces_refresh_and_retries_once(self):
|
|
peer = _FlakyPeer(failures=1)
|
|
mgr = _make_manager(peer)
|
|
assert mgr.dialectic_query("k", "who is this user?") == "synthesized answer"
|
|
assert peer.calls == 2 # original + one retry
|
|
|
|
def test_persistent_401_raises_auth_error(self):
|
|
peer = _FlakyPeer(failures=99)
|
|
mgr = _make_manager(peer)
|
|
with pytest.raises(HonchoAuthError):
|
|
mgr.dialectic_query("k", "q")
|
|
assert peer.calls == 2 # exactly one retry, no loop
|
|
|
|
def test_failed_reauth_raises_without_retry(self):
|
|
peer = _FlakyPeer(failures=99)
|
|
mgr = _make_manager(peer, reauth_ok=False)
|
|
with pytest.raises(HonchoAuthError):
|
|
mgr.dialectic_query("k", "q")
|
|
assert peer.calls == 1 # no retry without a fresh token
|
|
|
|
def test_non_auth_errors_stay_fail_open(self):
|
|
class _BrokenPeer:
|
|
def chat(self, *a, **kw):
|
|
raise Exception("connection reset by peer")
|
|
|
|
mgr = _make_manager(_BrokenPeer())
|
|
assert mgr.dialectic_query("k", "q") == ""
|
|
|
|
def test_success_after_failure_clears_auth_state(self):
|
|
peer = _FlakyPeer(failures=99)
|
|
mgr = _make_manager(peer, reauth_ok=False)
|
|
with pytest.raises(HonchoAuthError):
|
|
mgr.dialectic_query("k", "q")
|
|
assert mgr._auth_failure is not None
|
|
|
|
peer.failures = 0
|
|
mgr._force_reauth = lambda **kw: True
|
|
assert mgr.dialectic_query("k", "q") == "synthesized answer"
|
|
assert mgr._auth_failure is None
|
|
assert mgr.pop_auth_notice() is None
|
|
|
|
class TestForceReauth:
|
|
def test_rotates_and_applies_to_live_client(self, tmp_path, monkeypatch):
|
|
from plugins.memory.honcho import client as client_mod
|
|
from plugins.memory.honcho import session as session_mod
|
|
|
|
fake_client = object()
|
|
applied = {}
|
|
monkeypatch.setattr(session_mod, "get_honcho_client", lambda *a, **k: fake_client)
|
|
monkeypatch.setattr(client_mod, "resolve_config_path", lambda: tmp_path / "honcho.json")
|
|
monkeypatch.setattr(oauth, "force_refresh_token", lambda p, h, **kw: "hch-at-new")
|
|
|
|
def apply(client, token):
|
|
applied["client"] = client
|
|
applied["token"] = token
|
|
return True
|
|
|
|
monkeypatch.setattr(oauth, "apply_token_to_client", apply)
|
|
|
|
mgr = HonchoSessionManager(config=HonchoClientConfig(host="hermes"))
|
|
assert mgr._force_reauth() is True
|
|
assert applied == {"client": fake_client, "token": "hch-at-new"}
|
|
|
|
def test_returns_false_when_refresh_yields_nothing(self, tmp_path, monkeypatch):
|
|
from plugins.memory.honcho import client as client_mod
|
|
|
|
monkeypatch.setattr(client_mod, "resolve_config_path", lambda: tmp_path / "honcho.json")
|
|
monkeypatch.setattr(oauth, "force_refresh_token", lambda p, h, **kw: None)
|
|
mgr = HonchoSessionManager(config=HonchoClientConfig(host="hermes"))
|
|
assert mgr._force_reauth() is False
|
|
|
|
def test_passes_the_bearer_the_operation_sent_not_the_rotated_one(self, tmp_path, monkeypatch):
|
|
"""A sibling waiter rotates the shared client's api_key in place while the operation fails with
|
|
the old bearer; passing the rotated one would match disk and exchange again."""
|
|
from plugins.memory.honcho import client as client_mod
|
|
from plugins.memory.honcho import session as session_mod
|
|
|
|
http = SimpleNamespace(api_key="hch-at-old")
|
|
shared_client = SimpleNamespace(_http=http)
|
|
monkeypatch.setattr(session_mod, "get_honcho_client", lambda *a, **k: shared_client)
|
|
monkeypatch.setattr(client_mod, "resolve_config_path", lambda: tmp_path / "honcho.json")
|
|
seen = {}
|
|
monkeypatch.setattr(oauth, "force_refresh_token", lambda p, h, **kw: seen.update(kw) or "hch-at-new1")
|
|
monkeypatch.setattr(oauth, "apply_token_to_client", lambda c, t: True)
|
|
mgr = HonchoSessionManager(config=HonchoClientConfig(host="hermes", enabled=True))
|
|
|
|
def operation():
|
|
if http.api_key == "hch-at-old":
|
|
http.api_key = "hch-at-new1"
|
|
raise Exception("Invalid or expired access token")
|
|
return "ok"
|
|
|
|
assert mgr._authed_call("test op", operation) == "ok"
|
|
assert seen == {"failed_access_token": "hch-at-old"}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# session: message sync 401 recovery
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class _FlakyHonchoSession:
|
|
"""add_messages() raises an auth error N times, then succeeds."""
|
|
|
|
def __init__(self, failures: int):
|
|
self.failures = failures
|
|
self.calls = 0
|
|
|
|
def add_messages(self, messages):
|
|
self.calls += 1
|
|
if self.calls <= self.failures:
|
|
raise Exception("Invalid or expired access token")
|
|
|
|
def _make_sync_manager(flaky_session, *, reauth_ok=True):
|
|
cfg = HonchoClientConfig(host="hermes", api_key="hch-at-x", enabled=True)
|
|
mgr = HonchoSessionManager(config=cfg)
|
|
peer = MagicMock()
|
|
peer.message.side_effect = lambda content: content
|
|
mgr._get_or_create_peer = lambda peer_id: peer
|
|
mgr._sessions_cache["s"] = flaky_session
|
|
mgr._force_reauth = lambda **kw: reauth_ok
|
|
session = HonchoSession(
|
|
key="k", user_peer_id="u", assistant_peer_id="a", honcho_session_id="s"
|
|
)
|
|
session.add_message("user", "hello")
|
|
session.add_message("assistant", "hi")
|
|
return mgr, session
|
|
|
|
class TestSyncAuthRetry:
|
|
def test_401_forces_refresh_and_retries_once(self):
|
|
flaky = _FlakyHonchoSession(failures=1)
|
|
mgr, session = _make_sync_manager(flaky)
|
|
assert mgr._flush_session(session) is True
|
|
assert flaky.calls == 2
|
|
assert all(m["_synced"] for m in session.messages)
|
|
|
|
def test_persistent_401_fails_and_records_auth_failure(self):
|
|
flaky = _FlakyHonchoSession(failures=99)
|
|
mgr, session = _make_sync_manager(flaky)
|
|
assert mgr._flush_session(session) is False
|
|
assert flaky.calls == 2 # exactly one retry, no loop
|
|
assert not any(m.get("_synced") for m in session.messages)
|
|
assert mgr._auth_failure is not None
|
|
|
|
def test_failed_reauth_fails_without_retry(self):
|
|
flaky = _FlakyHonchoSession(failures=99)
|
|
mgr, session = _make_sync_manager(flaky, reauth_ok=False)
|
|
assert mgr._flush_session(session) is False
|
|
assert flaky.calls == 1
|
|
assert mgr._auth_failure is not None
|
|
|
|
def test_later_success_recovers_and_clears_auth_state(self):
|
|
flaky = _FlakyHonchoSession(failures=2)
|
|
mgr, session = _make_sync_manager(flaky, reauth_ok=False)
|
|
assert mgr._flush_session(session) is False
|
|
assert mgr._auth_failure is not None
|
|
|
|
mgr._force_reauth = lambda **kw: True
|
|
assert mgr._flush_session(session) is True
|
|
assert all(m["_synced"] for m in session.messages)
|
|
assert mgr._auth_failure is None
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# dead grant: skip calls entirely until re-login
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _kill_grant(tmp_path, monkeypatch) -> Path:
|
|
"""Revoke the grant on a tmp config and point the manager's path at it."""
|
|
from plugins.memory.honcho import client as client_mod
|
|
|
|
path = tmp_path / "honcho.json"
|
|
_write(path, {"hosts": {"hermes": _host_block()}})
|
|
monkeypatch.setattr(oauth, "_REFRESH_RETRY_DELAY_SECONDS", 0)
|
|
monkeypatch.setattr(
|
|
oauth, "_http_post_form_status",
|
|
lambda *a, **k: (400, {"error": "invalid_grant"}),
|
|
)
|
|
oauth.ensure_fresh_token(path, "hermes", now=1000)
|
|
assert oauth.reauth_required(path, "hermes") is True
|
|
monkeypatch.setattr(client_mod, "resolve_config_path", lambda: path)
|
|
return path
|
|
|
|
def _relogin(path: Path) -> None:
|
|
oauth.install_grant(
|
|
path, "hermes",
|
|
{"access_token": "hch-at-fresh", "refresh_token": "hch-rt-fresh", "expires_in": 3600},
|
|
client_id="hermes-desktop",
|
|
token_endpoint="http://localhost:8000/oauth/token",
|
|
)
|
|
|
|
class TestDeadGrantSkipsCalls:
|
|
|
|
def test_relogin_resumes_dialectic_without_waiting(self, tmp_path, monkeypatch):
|
|
path = _kill_grant(tmp_path, monkeypatch)
|
|
peer = _FlakyPeer(failures=0)
|
|
mgr = _make_manager(peer)
|
|
with pytest.raises(HonchoAuthError):
|
|
mgr.dialectic_query("k", "q")
|
|
assert peer.calls == 0
|
|
|
|
_relogin(path)
|
|
assert mgr.dialectic_query("k", "q") == "synthesized answer"
|
|
assert peer.calls == 1
|
|
assert mgr._auth_failure is None
|
|
|
|
def test_relogin_resumes_sync_without_waiting(self, tmp_path, monkeypatch):
|
|
path = _kill_grant(tmp_path, monkeypatch)
|
|
flaky = _FlakyHonchoSession(failures=0)
|
|
mgr, session = _make_sync_manager(flaky)
|
|
assert mgr._flush_session(session) is False
|
|
assert flaky.calls == 0
|
|
|
|
_relogin(path)
|
|
assert mgr._flush_session(session) is True
|
|
assert flaky.calls == 1
|
|
assert all(m["_synced"] for m in session.messages)
|
|
assert mgr._auth_failure is None
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# one-time user-facing notice
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestAuthNotice:
|
|
def test_manager_emits_notice_exactly_once(self):
|
|
peer = _FlakyPeer(failures=99)
|
|
mgr = _make_manager(peer, reauth_ok=False)
|
|
with pytest.raises(HonchoAuthError):
|
|
mgr.dialectic_query("k", "q")
|
|
|
|
first = mgr.pop_auth_notice()
|
|
assert first and "Invalid or expired access token" in first
|
|
assert mgr.pop_auth_notice() is None
|
|
|
|
# A second failure inside the same episode does not re-arm the notice.
|
|
with pytest.raises(HonchoAuthError):
|
|
mgr.dialectic_query("k", "q")
|
|
assert mgr.pop_auth_notice() is None
|
|
|
|
def test_recorded_failure_and_notice_redact_token_values(self):
|
|
mgr = _make_manager(_FlakyPeer(failures=0))
|
|
mgr._record_auth_failure(Exception("rejected token hch-at-secretvalue99"))
|
|
notice = mgr.pop_auth_notice()
|
|
assert "secretvalue99" not in notice
|
|
|
|
def test_provider_prefetch_injects_notice_once(self):
|
|
class _FakeManager:
|
|
def __init__(self):
|
|
self.notices = ["Invalid or expired access token"]
|
|
|
|
def pop_auth_notice(self):
|
|
return self.notices.pop() if self.notices else None
|
|
|
|
def pop_context_result(self, session_key):
|
|
return {}
|
|
|
|
provider = HonchoMemoryProvider()
|
|
provider._manager = _FakeManager()
|
|
provider._config = SimpleNamespace(timeout=0.01, context_tokens=0)
|
|
provider._session_key = "k"
|
|
provider._session_initialized = True
|
|
provider._recall_mode = "context"
|
|
provider._turn_count = 2
|
|
provider._last_dialectic_turn = 0
|
|
provider._base_context_cache = ""
|
|
|
|
first = provider.prefetch("what did we decide about the schema?")
|
|
assert "hermes honcho setup" in first
|
|
assert "paused" in first
|
|
|
|
second = provider.prefetch("and the follow-up question?")
|
|
assert second == ""
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# cadence backoff exemption
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestBackoffExemption:
|
|
def test_auth_error_does_not_widen_backoff(self):
|
|
provider = HonchoMemoryProvider()
|
|
provider._note_dialectic_failure(HonchoAuthError("still 401 after refresh"))
|
|
assert provider._dialectic_empty_streak == 0
|
|
|
|
def test_other_errors_still_widen_backoff(self):
|
|
provider = HonchoMemoryProvider()
|
|
provider._note_dialectic_failure(RuntimeError("timeout"))
|
|
assert provider._dialectic_empty_streak == 1
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# session: context/search 401 recovery through _authed_call
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class _FlakyContextPeer:
|
|
"""context() raises an auth error N times, then succeeds."""
|
|
|
|
def __init__(self, failures: int, representation: str = "knows Python"):
|
|
self.failures = failures
|
|
self.representation = representation
|
|
self.calls = 0
|
|
|
|
def context(self, **kw):
|
|
self.calls += 1
|
|
if self.calls <= self.failures:
|
|
raise Exception("Invalid or expired access token")
|
|
return SimpleNamespace(representation=self.representation, peer_card=["fact one"])
|
|
|
|
class TestContextAuthRetry:
|
|
def test_401_forces_refresh_and_retries_once(self):
|
|
peer = _FlakyContextPeer(failures=1)
|
|
mgr = _make_manager(peer)
|
|
ctx = mgr.get_session_context("k")
|
|
assert ctx["representation"] == "knows Python"
|
|
assert peer.calls == 2 # original + one retry
|
|
|
|
def test_persistent_401_records_failure_and_notices_once(self):
|
|
peer = _FlakyContextPeer(failures=99)
|
|
mgr = _make_manager(peer)
|
|
with pytest.raises(HonchoAuthError):
|
|
mgr.get_session_context("k")
|
|
assert peer.calls == 2 # exactly one retry, no loop
|
|
assert mgr._auth_failure is not None
|
|
assert mgr.pop_auth_notice() is not None
|
|
assert mgr.pop_auth_notice() is None
|
|
|
|
def test_peer_card_401_raises_instead_of_reading_empty(self):
|
|
class _FlakyCardPeer:
|
|
calls = 0
|
|
|
|
def get_card(self, **kw):
|
|
type(self).calls += 1
|
|
raise Exception("Invalid or expired access token")
|
|
|
|
mgr = _make_manager(_FlakyCardPeer(), reauth_ok=False)
|
|
with pytest.raises(HonchoAuthError):
|
|
mgr.get_peer_card("k")
|
|
assert _FlakyCardPeer.calls == 1
|
|
|
|
class TestDeadGrantSkipsContextAndSearch:
|
|
def test_dead_grant_issues_no_context_call(self, tmp_path, monkeypatch):
|
|
_kill_grant(tmp_path, monkeypatch)
|
|
peer = _FlakyContextPeer(failures=0)
|
|
mgr = _make_manager(peer)
|
|
with pytest.raises(HonchoAuthError):
|
|
mgr.get_session_context("k")
|
|
assert peer.calls == 0
|
|
|
|
def test_dead_grant_issues_no_search_call(self, tmp_path, monkeypatch):
|
|
from plugins.memory.honcho import session as session_mod
|
|
|
|
_kill_grant(tmp_path, monkeypatch)
|
|
client = MagicMock()
|
|
monkeypatch.setattr(session_mod, "get_honcho_client", lambda *a, **k: client)
|
|
mgr = _make_manager(_FlakyContextPeer(failures=0))
|
|
with pytest.raises(HonchoAuthError):
|
|
mgr.search_context("k", "query")
|
|
client.search.assert_not_called()
|
|
|
|
def test_dead_grant_prefetch_returns_empty_and_arms_notice(self, tmp_path, monkeypatch):
|
|
_kill_grant(tmp_path, monkeypatch)
|
|
peer = _FlakyContextPeer(failures=0)
|
|
mgr = _make_manager(peer)
|
|
assert mgr.get_prefetch_context("k") == {}
|
|
assert peer.calls == 0
|
|
assert mgr.pop_auth_notice() is not None
|
|
|
|
class TestNonAuthFailuresNotRetried:
|
|
def test_context_timeout_fails_open_without_refresh(self):
|
|
class _TimeoutPeer:
|
|
calls = 0
|
|
|
|
def _fail(self):
|
|
type(self).calls += 1
|
|
raise TimeoutError("request timed out")
|
|
|
|
def context(self, **kw):
|
|
self._fail()
|
|
|
|
def representation(self, **kw):
|
|
self._fail()
|
|
|
|
def get_card(self, **kw):
|
|
self._fail()
|
|
|
|
_TimeoutPeer.calls = 0
|
|
mgr = _make_manager(_TimeoutPeer())
|
|
reauths = []
|
|
mgr._force_reauth = lambda **kw: reauths.append(1) or True
|
|
|
|
ctx = mgr.get_session_context("k")
|
|
assert ctx == {"representation": "", "card": []}
|
|
assert _TimeoutPeer.calls == 3 # context, representation, card — no retries
|
|
assert reauths == []
|
|
|
|
def test_search_timeout_fails_open_without_refresh(self, monkeypatch):
|
|
from plugins.memory.honcho import session as session_mod
|
|
|
|
class _TimeoutSearchPeer:
|
|
calls = 0
|
|
|
|
def search(self, *a, **kw):
|
|
type(self).calls += 1
|
|
raise TimeoutError("request timed out")
|
|
|
|
_TimeoutSearchPeer.calls = 0
|
|
client = MagicMock()
|
|
client.search.side_effect = TimeoutError("request timed out")
|
|
monkeypatch.setattr(session_mod, "get_honcho_client", lambda *a, **k: client)
|
|
mgr = _make_manager(_TimeoutSearchPeer())
|
|
reauths = []
|
|
mgr._force_reauth = lambda **kw: reauths.append(1) or True
|
|
|
|
assert mgr.search_context("k", "q") == ""
|
|
assert client.search.call_count == 1
|
|
assert _TimeoutSearchPeer.calls == 1
|
|
assert reauths == []
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# client rebuild: the retry must use freshly resolved SDK objects
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _wire_rebuild(tmp_path, monkeypatch, fresh_client):
|
|
"""Route _force_reauth down its client-rebuild path, swapping in fresh_client."""
|
|
from plugins.memory.honcho import client as client_mod
|
|
from plugins.memory.honcho import session as session_mod
|
|
|
|
clients = {"current": MagicMock()}
|
|
monkeypatch.setattr(session_mod, "get_honcho_client", lambda *a, **k: clients["current"])
|
|
monkeypatch.setattr(client_mod, "resolve_config_path", lambda: tmp_path / "honcho.json")
|
|
monkeypatch.setattr(oauth, "force_refresh_token", lambda p, h, **kw: "hch-at-rotated")
|
|
monkeypatch.setattr(oauth, "apply_token_to_client", lambda c, t: False)
|
|
monkeypatch.setattr(
|
|
client_mod, "reset_honcho_client",
|
|
lambda: clients.__setitem__("current", fresh_client),
|
|
)
|
|
return clients
|
|
|
|
class TestClientRebuildRetry:
|
|
def test_flush_retry_uses_rebuilt_session_not_stale(self, tmp_path, monkeypatch):
|
|
stale_session = MagicMock()
|
|
stale_session.add_messages.side_effect = Exception("Invalid or expired access token")
|
|
stale_peer = MagicMock()
|
|
stale_peer.message.side_effect = lambda content: content
|
|
|
|
fresh_session = MagicMock()
|
|
fresh_session.context.return_value = SimpleNamespace(summary=None, messages=[])
|
|
fresh_peer = MagicMock()
|
|
fresh_peer.message.side_effect = lambda content: content
|
|
fresh_client = MagicMock()
|
|
fresh_client.session.return_value = fresh_session
|
|
fresh_client.peer.return_value = fresh_peer
|
|
|
|
_wire_rebuild(tmp_path, monkeypatch, fresh_client)
|
|
|
|
cfg = HonchoClientConfig(host="hermes", api_key="hch-at-x", enabled=True)
|
|
mgr = HonchoSessionManager(config=cfg)
|
|
mgr._peers_cache.update({"u": stale_peer, "a": stale_peer})
|
|
mgr._sessions_cache["s"] = stale_session
|
|
session = HonchoSession(
|
|
key="k", user_peer_id="u", assistant_peer_id="a", honcho_session_id="s"
|
|
)
|
|
session.add_message("user", "hello")
|
|
session.add_message("assistant", "hi")
|
|
|
|
assert mgr._flush_session(session) is True
|
|
# The stale pre-rebuild session must not be retried.
|
|
assert stale_session.add_messages.call_count == 1
|
|
assert fresh_session.add_messages.call_count == 1
|
|
assert all(m["_synced"] for m in session.messages)
|
|
assert mgr._auth_failure is None
|
|
|
|
def test_context_retry_uses_rebuilt_peer_not_stale(self, tmp_path, monkeypatch):
|
|
stale_peer = MagicMock()
|
|
stale_peer.context.side_effect = Exception("Invalid or expired access token")
|
|
|
|
fresh_peer = MagicMock()
|
|
fresh_peer.context.return_value = SimpleNamespace(
|
|
representation="rep after rebuild", peer_card=["fact"]
|
|
)
|
|
fresh_client = MagicMock()
|
|
fresh_client.peer.return_value = fresh_peer
|
|
|
|
_wire_rebuild(tmp_path, monkeypatch, fresh_client)
|
|
|
|
cfg = HonchoClientConfig(host="hermes", api_key="hch-at-x", enabled=True)
|
|
mgr = HonchoSessionManager(config=cfg)
|
|
mgr._peers_cache["u"] = stale_peer
|
|
mgr._cache["k"] = HonchoSession(
|
|
key="k", user_peer_id="u", assistant_peer_id="a", honcho_session_id="s"
|
|
)
|
|
|
|
ctx = mgr.get_session_context("k")
|
|
assert ctx["representation"] == "rep after rebuild"
|
|
assert stale_peer.context.call_count == 1
|
|
assert fresh_peer.context.call_count == 1
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# tools: auth failures must never read as "no context"
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestToolAuthVisibility:
|
|
def _provider(self, manager):
|
|
provider = HonchoMemoryProvider()
|
|
provider._manager = manager
|
|
provider._session_key = "k"
|
|
provider._session_initialized = True
|
|
return provider
|
|
|
|
def test_context_tool_reports_auth_failure(self):
|
|
class _Mgr:
|
|
def get_session_context(self, key, peer="user"):
|
|
raise HonchoAuthError("Honcho rejected our credentials")
|
|
|
|
out = self._provider(_Mgr()).handle_tool_call("honcho_context", {})
|
|
assert "No context available" not in out
|
|
assert "authentication failed" in out
|
|
|
|
def test_search_tool_reports_auth_failure(self):
|
|
class _Mgr:
|
|
def search_context(self, key, query, max_tokens=800, peer="user"):
|
|
raise HonchoAuthError("Honcho rejected our credentials")
|
|
|
|
out = self._provider(_Mgr()).handle_tool_call("honcho_search", {"query": "schema"})
|
|
assert "No relevant context found" not in out
|
|
assert "authentication failed" in out
|
|
|
|
def test_profile_tool_reports_auth_failure_not_empty_profile(self):
|
|
class _Mgr:
|
|
def get_peer_card(self, key, peer="user"):
|
|
raise HonchoAuthError("Honcho rejected our credentials")
|
|
|
|
out = self._provider(_Mgr()).handle_tool_call("honcho_profile", {})
|
|
assert "No profile facts" not in out
|
|
assert "authentication failed" in out
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# initialization-time auth failures: the notice must survive the manager discard
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _healthy_client():
|
|
"""A mock SDK client whose peers and sessions behave like an empty backend."""
|
|
client = MagicMock()
|
|
peer = MagicMock()
|
|
peer.chat.return_value = ""
|
|
peer.get_card.return_value = None
|
|
peer.context.return_value = SimpleNamespace(representation="", peer_card=[])
|
|
client.peer.return_value = peer
|
|
sdk_session = MagicMock()
|
|
sdk_session.context.return_value = SimpleNamespace(summary=None, messages=[])
|
|
client.session.return_value = sdk_session
|
|
return client
|
|
|
|
def _wire_init(tmp_path, monkeypatch, client, *, recall_mode="hybrid", dead_refresh=True):
|
|
"""Route provider initialization through a real manager backed by ``client``."""
|
|
from plugins.memory.honcho import client as client_mod
|
|
from plugins.memory.honcho import session as session_mod
|
|
|
|
path = tmp_path / "honcho.json"
|
|
_write(path, {"hosts": {"hermes": _host_block(expires_at=time.time() + 3600)}})
|
|
monkeypatch.setattr(client_mod, "resolve_config_path", lambda: path)
|
|
monkeypatch.setattr(client_mod, "get_honcho_client", lambda *a, **k: client)
|
|
monkeypatch.setattr(session_mod, "get_honcho_client", lambda *a, **k: client)
|
|
if dead_refresh:
|
|
monkeypatch.setattr(oauth, "force_refresh_token", lambda p, h, **kw: None)
|
|
cfg = HonchoClientConfig(
|
|
host="hermes", api_key="hch-at-old", enabled=True, recall_mode=recall_mode,
|
|
timeout=0.5, session_strategy="per-session", peer_name="operator",
|
|
)
|
|
monkeypatch.setattr(
|
|
client_mod.HonchoClientConfig, "from_global_config", lambda *a, **k: cfg
|
|
)
|
|
return path
|
|
|
|
def _initialized_provider():
|
|
provider = HonchoMemoryProvider()
|
|
provider.initialize(session_id="init-auth-session")
|
|
if provider._init_thread:
|
|
provider._init_thread.join(timeout=5)
|
|
return provider
|
|
|
|
class TestInitAuthFailureNotice:
|
|
|
|
def test_notice_is_emitted_exactly_once(self, tmp_path, monkeypatch):
|
|
client = MagicMock()
|
|
client.peer.side_effect = Exception("HTTP 401 Unauthorized")
|
|
_wire_init(tmp_path, monkeypatch, client)
|
|
provider = _initialized_provider()
|
|
|
|
assert "hermes honcho setup" in provider.prefetch("first question")
|
|
# Retries keep failing, but the same episode never re-arms the notice.
|
|
for query in ("second question", "third question"):
|
|
assert provider.prefetch(query) == ""
|
|
|
|
def test_dead_grant_during_session_setup_produces_notice(self, tmp_path, monkeypatch):
|
|
client = _healthy_client()
|
|
env = {}
|
|
|
|
def _session_dies(*a, **k):
|
|
path = env["path"]
|
|
block = json.loads(path.read_text())["hosts"]["hermes"]
|
|
cred = oauth.OAuthCredential.from_host_block(block)
|
|
oauth._mark_grant_dead((str(path), "hermes"), cred)
|
|
raise Exception("Invalid or expired access token")
|
|
|
|
client.session.side_effect = _session_dies
|
|
env["path"] = _wire_init(tmp_path, monkeypatch, client, dead_refresh=False)
|
|
provider = _initialized_provider()
|
|
|
|
assert provider._manager is None
|
|
assert client.peer.called # failure hit session setup, not peer setup
|
|
notice = provider.prefetch("what happened before the grant died?")
|
|
assert "hermes honcho setup" in notice
|
|
|
|
def test_relogin_resumes_init_and_clears_failure(self, tmp_path, monkeypatch):
|
|
client = _healthy_client()
|
|
client.peer.side_effect = Exception("HTTP 401 Unauthorized")
|
|
path = _wire_init(tmp_path, monkeypatch, client)
|
|
provider = _initialized_provider()
|
|
assert "hermes honcho setup" in provider.prefetch("first question")
|
|
|
|
_relogin(path)
|
|
client.peer.side_effect = None
|
|
provider.prefetch("after re-login")
|
|
if provider._init_thread:
|
|
provider._init_thread.join(timeout=5)
|
|
|
|
assert provider._session_initialized is True
|
|
assert provider._manager is not None
|
|
assert provider._init_auth_failure is None
|
|
|
|
def test_tools_relogin_resumes_without_restart(self, tmp_path, monkeypatch):
|
|
client = _healthy_client()
|
|
client.peer.side_effect = Exception("Invalid or expired access token")
|
|
path = _wire_init(tmp_path, monkeypatch, client, recall_mode="tools")
|
|
provider = _initialized_provider()
|
|
assert "authentication failed" in provider.handle_tool_call("honcho_profile", {})
|
|
|
|
_relogin(path)
|
|
client.peer.side_effect = None
|
|
out = provider.handle_tool_call("honcho_profile", {})
|
|
|
|
assert "authentication failed" not in out
|
|
assert provider._session_initialized is True
|
|
assert provider._init_auth_failure is None
|
|
|
|
def test_non_auth_init_timeout_fails_open_without_notice(self, tmp_path, monkeypatch):
|
|
client = MagicMock()
|
|
client.peer.side_effect = TimeoutError("request timed out")
|
|
_wire_init(tmp_path, monkeypatch, client, dead_refresh=False)
|
|
reauths = []
|
|
monkeypatch.setattr(oauth, "force_refresh_token", lambda p, h, **kw: reauths.append(1))
|
|
provider = _initialized_provider()
|
|
|
|
assert provider._manager is None
|
|
assert provider._init_auth_failure is None
|
|
assert provider.prefetch("a real question") == ""
|
|
assert reauths == []
|
|
|
|
def test_non_auth_tools_init_failure_keeps_generic_error(self, tmp_path, monkeypatch):
|
|
client = MagicMock()
|
|
client.peer.side_effect = TimeoutError("request timed out")
|
|
_wire_init(tmp_path, monkeypatch, client, recall_mode="tools", dead_refresh=False)
|
|
provider = _initialized_provider()
|
|
|
|
out = provider.handle_tool_call("honcho_profile", {})
|
|
assert "could not be initialized" in out
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# hardening: exchange budget, failure cooldown, client-generation cache guard
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestExchangeBudget:
|
|
def test_timed_out_first_attempt_skips_retry_when_budget_spent(self, tmp_path, monkeypatch):
|
|
"""A first attempt that consumed the whole budget must not start a
|
|
second full-timeout exchange while holding the global refresh locks."""
|
|
path = tmp_path / "honcho.json"
|
|
_write(path, {"hosts": {"hermes": _host_block()}})
|
|
monkeypatch.setattr(oauth, "_REFRESH_RETRY_DELAY_SECONDS", 0)
|
|
|
|
calls = []
|
|
clock = {"now": 1000.0}
|
|
monkeypatch.setattr(oauth.time, "monotonic", lambda: clock["now"])
|
|
|
|
def slow_timeout(url, data, timeout):
|
|
calls.append(timeout)
|
|
clock["now"] += oauth._REFRESH_TOTAL_BUDGET_SECONDS + 1
|
|
raise TimeoutError("token exchange timed out")
|
|
|
|
monkeypatch.setattr(oauth, "_http_post_form_status", slow_timeout)
|
|
token, refreshed = oauth.ensure_fresh_token(path, "hermes", now=1000)
|
|
|
|
assert token == "hch-at-old" and refreshed is False
|
|
assert len(calls) == 1 # no second exchange after the budget is gone
|
|
|
|
def test_fast_failure_retry_gets_remaining_budget(self, tmp_path, monkeypatch):
|
|
path = tmp_path / "honcho.json"
|
|
_write(path, {"hosts": {"hermes": _host_block()}})
|
|
monkeypatch.setattr(oauth, "_REFRESH_RETRY_DELAY_SECONDS", 0)
|
|
|
|
timeouts = []
|
|
|
|
def flaky(url, data, timeout):
|
|
timeouts.append(timeout)
|
|
if len(timeouts) == 1:
|
|
raise ConnectionError("reset")
|
|
return 200, _rotated_body()
|
|
|
|
monkeypatch.setattr(oauth, "_http_post_form_status", flaky)
|
|
token, refreshed = oauth.ensure_fresh_token(path, "hermes", now=1000)
|
|
|
|
assert refreshed is True and token == "hch-at-new1"
|
|
assert len(timeouts) == 2
|
|
# Retry timeout is bounded by both the per-attempt cap and the budget.
|
|
assert 0 < timeouts[1] <= oauth._REFRESH_TIMEOUT_SECONDS
|
|
|
|
class TestFailureCooldown:
|
|
def test_repeated_calls_within_cooldown_do_not_reexchange(self, tmp_path, monkeypatch):
|
|
"""After a transient failure, waiting callers fail open instead of
|
|
serializing their own full exchange cycles (dogpile guard)."""
|
|
path = tmp_path / "honcho.json"
|
|
_write(path, {"hosts": {"hermes": _host_block()}})
|
|
monkeypatch.setattr(oauth, "_REFRESH_RETRY_DELAY_SECONDS", 0)
|
|
|
|
calls = []
|
|
|
|
def boom(*a, **k):
|
|
calls.append(1)
|
|
raise ConnectionError("network down")
|
|
|
|
monkeypatch.setattr(oauth, "_http_post_form_status", boom)
|
|
oauth.ensure_fresh_token(path, "hermes", now=1000)
|
|
assert len(calls) == 2 # first attempt + its one retry
|
|
|
|
# Subsequent callers inside the cooldown window skip the endpoint.
|
|
for _ in range(3):
|
|
token, refreshed = oauth.ensure_fresh_token(path, "hermes", now=1000)
|
|
assert token == "hch-at-old" and refreshed is False
|
|
assert oauth.force_refresh_token(path, "hermes") is None
|
|
assert len(calls) == 2
|
|
|
|
# After the cooldown expires the exchange is attempted again.
|
|
key = (str(path), "hermes")
|
|
oauth._refresh_failure_at[key] -= oauth._REFRESH_FAILURE_COOLDOWN_SECONDS + 1
|
|
oauth.ensure_fresh_token(path, "hermes", now=1000)
|
|
assert len(calls) == 4
|
|
|
|
def test_relogin_clears_the_cooldown(self, tmp_path, monkeypatch):
|
|
path = tmp_path / "honcho.json"
|
|
_write(path, {"hosts": {"hermes": _host_block()}})
|
|
monkeypatch.setattr(oauth, "_REFRESH_RETRY_DELAY_SECONDS", 0)
|
|
|
|
def boom(*a, **k):
|
|
raise ConnectionError("network down")
|
|
|
|
monkeypatch.setattr(oauth, "_http_post_form_status", boom)
|
|
oauth.ensure_fresh_token(path, "hermes", now=1000)
|
|
assert oauth._in_failure_cooldown((str(path), "hermes")) is True
|
|
|
|
_relogin(path)
|
|
assert oauth._in_failure_cooldown((str(path), "hermes")) is False
|
|
|
|
def test_successful_rotation_clears_the_cooldown(self, tmp_path, monkeypatch):
|
|
path = tmp_path / "honcho.json"
|
|
_write(path, {"hosts": {"hermes": _host_block()}})
|
|
monkeypatch.setattr(oauth, "_REFRESH_RETRY_DELAY_SECONDS", 0)
|
|
key = (str(path), "hermes")
|
|
oauth._refresh_failure_at[key] = (
|
|
oauth.time.monotonic() - oauth._REFRESH_FAILURE_COOLDOWN_SECONDS - 1
|
|
)
|
|
monkeypatch.setattr(
|
|
oauth, "_http_post_form_status", lambda *a, **k: (200, _rotated_body())
|
|
)
|
|
token, refreshed = oauth.ensure_fresh_token(path, "hermes", now=1000)
|
|
assert refreshed is True
|
|
assert key not in oauth._refresh_failure_at
|
|
|
|
class TestClientGenerationGuard:
|
|
def test_stale_object_resolved_across_rebuild_is_not_cached(self, monkeypatch):
|
|
"""A resolver that fetched from the OLD client must not store its
|
|
object into the cache after _force_reauth rebuilt the client."""
|
|
from plugins.memory.honcho import session as session_mod
|
|
|
|
cfg = HonchoClientConfig(host="hermes", api_key="hch-at-x", enabled=True)
|
|
mgr = HonchoSessionManager(config=cfg)
|
|
|
|
stale_session = object()
|
|
fresh_session = object()
|
|
resolutions = []
|
|
|
|
class _Client:
|
|
def session(self, sid):
|
|
# First resolve returns the stale object and simulates a
|
|
# concurrent rebuild landing mid-flight; the retry gets fresh.
|
|
if not resolutions:
|
|
resolutions.append("stale")
|
|
with mgr._cache_lock:
|
|
mgr._client_generation += 1
|
|
mgr._sessions_cache.clear()
|
|
return stale_session
|
|
resolutions.append("fresh")
|
|
return fresh_session
|
|
|
|
client = _Client()
|
|
monkeypatch.setattr(session_mod, "get_honcho_client", lambda *a, **k: client)
|
|
got = mgr._sdk_session("s")
|
|
assert got is fresh_session
|
|
assert mgr._sessions_cache["s"] is fresh_session
|
|
assert resolutions == ["stale", "fresh"]
|