hermes-agent/tests/agent/test_actual_auxiliary_routing.py
ethernet 890bbbda1f Merge remote-tracking branch 'origin/main' into ethie/pm-clean
# Conflicts:
#	apps/desktop/e2e/archived-hidden-session-recoverable.spec.ts
#	apps/desktop/e2e/bot-chat-message-agent-friendly-name.spec.ts
#	apps/desktop/e2e/bot-mailbox-unreadable-ticket.spec.ts
#	apps/desktop/e2e/bot-mode-roster-localized.spec.ts
#	apps/desktop/e2e/bot-mode-row-click-mirrors-registry.spec.ts
#	apps/desktop/e2e/bot-mode-tab-shows-bot-name.spec.ts
#	apps/desktop/e2e/bot-roster-group-row-organisation.spec.ts
#	apps/desktop/e2e/bot-roster-ignores-infra-dirs.spec.ts
#	apps/desktop/e2e/bot-roster-timestamp-meta.spec.ts
#	apps/desktop/e2e/bot-roster-user-sections.spec.ts
#	apps/desktop/e2e/bot-routines-pane-narrow.spec.ts
#	apps/desktop/e2e/bot-row-open-recent-session.spec.ts
#	apps/desktop/e2e/bot-tile-ignores-ambient-composer-model.spec.ts
#	apps/desktop/e2e/group-composer-auto-grow.spec.ts
#	apps/desktop/e2e/group-create-gate-remote-roster.spec.ts
#	apps/desktop/e2e/group-prompt-renamed-primary-handle.spec.ts
#	apps/desktop/e2e/hosted-room-backend-continuity.spec.ts
#	apps/desktop/e2e/hosted-room-legacy-store-migration.spec.ts
#	apps/desktop/e2e/settings-scope-chips-bot-title.spec.ts
#	apps/desktop/e2e/worktree-branch-status.spec.ts
#	apps/desktop/electron/backend-probes.test.ts
#	apps/desktop/electron/connection-apply.test.ts
#	apps/desktop/electron/desktop-electron-pin.test.ts
#	apps/desktop/electron/desktop-uninstall.test.ts
#	apps/desktop/electron/gateway-file-download-transport.test.ts
#	apps/desktop/electron/gateway-stop-before-update.test.ts
#	apps/desktop/electron/github-api-auth.test.ts
#	apps/desktop/electron/registry-primary-profile-scope.test.ts
#	apps/desktop/electron/update-api-check.test.ts
#	apps/desktop/electron/update-handoff-marker.test.ts
#	apps/desktop/electron/venv-blocker-scan.test.ts
#	apps/desktop/scripts/after-extract.test.mjs
#	apps/desktop/scripts/local-pack-publish.test.mjs
#	apps/desktop/scripts/tasks-scroll.test.mjs
#	apps/desktop/src/app/settings/model-settings.test.tsx
#	apps/desktop/src/app/updates-overlay.blockers.test.tsx
#	apps/desktop/src/components/desktop-install-overlay.test.tsx
#	apps/desktop/src/lib/update-copy.test.ts
#	scripts/ci/check_os_marker_fakes.py
#	tests-js/desktop-mac-usage-descriptions.test.ts
#	tests-js/node-engine-alignment.test.ts
#	tests/agent/lsp/test_install_and_lint_fixes.py
#	tests/agent/test_command_token_source.py
#	tests/agent/test_compression_boundary_hook.py
#	tests/agent/test_create_openai_client_ssl_verify.py
#	tests/agent/test_custom_provider_ca_probes.py
#	tests/agent/test_endpoint_blackhole.py
#	tests/agent/test_estimator_parity.py
#	tests/agent/test_in_place_compaction.py
#	tests/agent/test_moa_loop_mode.py
#	tests/agent/test_model_metadata.py
#	tests/agent/test_skill_session_platform_gate.py
#	tests/agent/test_skill_utils.py
#	tests/agent/test_ssl_ca_guard.py
#	tests/computer_use/test_doctor.py
#	tests/cron/test_codex_execution_paths.py
#	tests/cron/test_cron_bot_chat_delivery.py
#	tests/cron/test_cron_script.py
#	tests/cron/test_media_delivery_parity.py
#	tests/cron/test_misfire_catchup.py
#	tests/cron/test_parallel_pool.py
#	tests/cron/test_recurring_eagain_redispatch.py
#	tests/gateway/test_choice_picker.py
#	tests/gateway/test_control_socket_windows_live.py
#	tests/gateway/test_dingtalk.py
#	tests/gateway/test_feishu.py
#	tests/gateway/test_feishu_onboard.py
#	tests/gateway/test_gateway_shutdown.py
#	tests/gateway/test_matrix.py
#	tests/gateway/test_model_command_custom_providers.py
#	tests/gateway/test_reasoning_command.py
#	tests/gateway/test_runtime_footer.py
#	tests/gateway/test_session.py
#	tests/gateway/test_session_hygiene.py
#	tests/gateway/test_status.py
#	tests/gateway/test_teams.py
#	tests/gateway/test_turn_lease.py
#	tests/gateway/test_whatsapp_connect.py
#	tests/hermes_cli/test_approvals_command.py
#	tests/hermes_cli/test_auth_store_lock_concurrent.py
#	tests/hermes_cli/test_backup.py
#	tests/hermes_cli/test_banner_git_state.py
#	tests/hermes_cli/test_certifi_repair.py
#	tests/hermes_cli/test_cmd_update.py
#	tests/hermes_cli/test_compat_manifest_targets.py
#	tests/hermes_cli/test_computer_use_cli.py
#	tests/hermes_cli/test_cpr_local_leak.py
#	tests/hermes_cli/test_dashboard_auth_gate.py
#	tests/hermes_cli/test_dashboard_procs_kill_grace.py
#	tests/hermes_cli/test_desktop_lifecycle_windows_live.py
#	tests/hermes_cli/test_doctor.py
#	tests/hermes_cli/test_doctor_command_install.py
#	tests/hermes_cli/test_fleet_config_migration_windows_live.py
#	tests/hermes_cli/test_gateway.py
#	tests/hermes_cli/test_gateway_platform_gating.py
#	tests/hermes_cli/test_gateway_restart_loop.py
#	tests/hermes_cli/test_gateway_task_probe.py
#	tests/hermes_cli/test_gateway_wsl.py
#	tests/hermes_cli/test_gui_command.py
#	tests/hermes_cli/test_install_cua_driver.py
#	tests/hermes_cli/test_kanban_db.py
#	tests/hermes_cli/test_lazy_command_exports.py
#	tests/hermes_cli/test_lazy_refresh_venv_repair.py
#	tests/hermes_cli/test_linux_desktop_entry.py
#	tests/hermes_cli/test_local_runtime.py
#	tests/hermes_cli/test_local_runtime_updates.py
#	tests/hermes_cli/test_managed_uv.py
#	tests/hermes_cli/test_mcp_reload_confirm_gate.py
#	tests/hermes_cli/test_nous_subscription.py
#	tests/hermes_cli/test_npm_engine.py
#	tests/hermes_cli/test_personality_none.py
#	tests/hermes_cli/test_pet_toggle.py
#	tests/hermes_cli/test_plan_reconciliation_windows_live.py
#	tests/hermes_cli/test_plugin_event_bus.py
#	tests/hermes_cli/test_plugin_manifest_v2.py
#	tests/hermes_cli/test_plugin_packs.py
#	tests/hermes_cli/test_plugins_cmd.py
#	tests/hermes_cli/test_plugins_cmd_enable_disable_nested.py
#	tests/hermes_cli/test_process_identity.py
#	tests/hermes_cli/test_profiles.py
#	tests/hermes_cli/test_profiles_sidebar_cache.py
#	tests/hermes_cli/test_pty_bridge.py
#	tests/hermes_cli/test_resolve_turn_limit.py
#	tests/hermes_cli/test_serve_runtime_inventory.py
#	tests/hermes_cli/test_session_vacuum_config.py
#	tests/hermes_cli/test_set_config_value.py
#	tests/hermes_cli/test_signal_handler_kanban_worker.py
#	tests/hermes_cli/test_slash_confirm_windows.py
#	tests/hermes_cli/test_stale_pid_guard.py
#	tests/hermes_cli/test_startup_fast_guards.py
#	tests/hermes_cli/test_status.py
#	tests/hermes_cli/test_telegram_managed_bot.py
#	tests/hermes_cli/test_tools_config.py
#	tests/hermes_cli/test_update_apply_shallow_count.py
#	tests/hermes_cli/test_update_autostash.py
#	tests/hermes_cli/test_update_concurrent_quarantine.py
#	tests/hermes_cli/test_update_fetch_failure_classifier.py
#	tests/hermes_cli/test_update_fleet_probe_resume_token.py
#	tests/hermes_cli/test_update_handoff_backend_reap.py
#	tests/hermes_cli/test_update_handoff_desktop_rebuild.py
#	tests/hermes_cli/test_update_head_moved_gate.py
#	tests/hermes_cli/test_update_host_obligation.py
#	tests/hermes_cli/test_update_import_guard.py
#	tests/hermes_cli/test_update_interrupted_recovery.py
#	tests/hermes_cli/test_update_inventory.py
#	tests/hermes_cli/test_update_launchd_unloaded_gateway.py
#	tests/hermes_cli/test_update_missing_configured_deps.py
#	tests/hermes_cli/test_update_modified_notice.py
#	tests/hermes_cli/test_update_multiplex_migration_hook.py
#	tests/hermes_cli/test_update_no_gateway_restart.py
#	tests/hermes_cli/test_update_orphan_backend_reap.py
#	tests/hermes_cli/test_update_parked_branch_guard.py
#	tests/hermes_cli/test_update_post_pull_syntax_guard.py
#	tests/hermes_cli/test_update_receipt.py
#	tests/hermes_cli/test_update_self_lock.py
#	tests/hermes_cli/test_update_shim_fail_closed.py
#	tests/hermes_cli/test_update_shim_self_lock.py
#	tests/hermes_cli/test_update_sqlite_remediation.py
#	tests/hermes_cli/test_update_stale_dashboard.py
#	tests/hermes_cli/test_update_stale_virtualenv.py
#	tests/hermes_cli/test_update_venv_health.py
#	tests/hermes_cli/test_update_venv_ownership_preflight.py
#	tests/hermes_cli/test_update_wedged_gateway.py
#	tests/hermes_cli/test_update_yes_flag.py
#	tests/hermes_cli/test_update_zip_two_phase.py
#	tests/hermes_cli/test_urllib_security.py
#	tests/hermes_cli/test_ux_messages_auth_config.py
#	tests/hermes_cli/test_ux_messages_startup.py
#	tests/hermes_cli/test_venv_holder_classifier.py
#	tests/hermes_cli/test_verify_console_scripts.py
#	tests/hermes_cli/test_verify_core_dependencies.py
#	tests/hermes_cli/test_web_server.py
#	tests/hermes_cli/test_web_server_console_ws.py
#	tests/hermes_cli/test_web_server_ws_ping.py
#	tests/hermes_cli/test_web_ui_build.py
#	tests/hermes_state/test_fts_rebuild_admission.py
#	tests/hermes_state/test_hermes_state.py
#	tests/plugins/memory/test_memory_lazy_install.py
#	tests/plugins/test_google_meet_plugin.py
#	tests/plugins/test_langfuse_plugin.py
#	tests/plugins/test_security_guidance_plugin.py
#	tests/plugins/test_transform_llm_output_hook.py
#	tests/scripts/desktop_update/test_desktop_update_windows_gateway_flag.py
#	tests/scripts/desktop_update/test_desktop_update_windows_python_handoff.py
#	tests/scripts/desktop_update/test_desktop_update_windows_timestamp.py
#	tests/scripts/install/test_install_clone_throttle_fallback.py
#	tests/scripts/install/test_install_lockfile_churn.py
#	tests/scripts/install/test_install_no_initial_commit.py
#	tests/scripts/install/test_install_sh_browser_install.py
#	tests/scripts/install/test_install_sh_node_prerelease.py
#	tests/scripts/install/test_install_sh_symlink_stomp.py
#	tests/scripts/install/test_install_sh_uv_lock_config.py
#	tests/scripts/install/test_install_unmerged_index.py
#	tests/scripts/test_contributor_map.py
#	tests/scripts/test_run_tests_parallel.py
#	tests/skills/test_competitor_news_monitor_skill.py
#	tests/skills/test_document_to_action_items_skill.py
#	tests/skills/test_google_workspace_setup.py
#	tests/skills/test_google_workspace_setup_deps.py
#	tests/skills/test_grounded_citations_skill.py
#	tests/skills/test_ip_as_logo_skill.py
#	tests/skills/test_live_dashboard_skill.py
#	tests/skills/test_mcp_oauth_remote_gateway_skill.py
#	tests/skills/test_office_document_skills.py
#	tests/skills/test_openclaw_migration.py
#	tests/skills/test_product_price_monitor_skill.py
#	tests/skills/test_scrollcraft_skill.py
#	tests/skills/test_setup_wizard_generator_skill.py
#	tests/skills/test_weekly_review_planning_skill.py
#	tests/test_engines_satisfiable.py
#	tests/test_fast_safe_load.py
#	tests/test_hermes_bootstrap.py
#	tests/test_hermes_constants.py
#	tests/test_hermes_logging.py
#	tests/test_managed_runtime_resolution.py
#	tests/test_model_tools_async_bridge.py
#	tests/test_packaging_build_guard.py
#	tests/test_packaging_metadata.py
#	tests/test_yaml_indent_consistency.py
#	tests/tools/test_approval_timeout_overflow.py
#	tests/tools/test_base_environment.py
#	tests/tools/test_bot_mode_dm.py
#	tests/tools/test_browser_chromium_check.py
#	tests/tools/test_browser_hardening.py
#	tests/tools/test_browser_homebrew_paths.py
#	tests/tools/test_browser_npx_warmup.py
#	tests/tools/test_browser_orphan_reaper.py
#	tests/tools/test_browser_real_profile.py
#	tests/tools/test_browser_use_cli.py
#	tests/tools/test_clipboard.py
#	tests/tools/test_code_execution.py
#	tests/tools/test_code_execution_modes.py
#	tests/tools/test_code_execution_windows_env.py
#	tests/tools/test_computer_use.py
#	tests/tools/test_delegate_liveness_timeout.py
#	tests/tools/test_execute_code_approval_cluster.py
#	tests/tools/test_execution_flag_detection.py
#	tests/tools/test_fal_common.py
#	tests/tools/test_file_operations.py
#	tests/tools/test_file_tools.py
#	tests/tools/test_file_tools_cwd_resolution.py
#	tests/tools/test_file_tools_live.py
#	tests/tools/test_lazy_deps.py
#	tests/tools/test_lazy_deps_durable_target.py
#	tests/tools/test_lazy_deps_managed.py
#	tests/tools/test_local_env_blocklist.py
#	tests/tools/test_local_tempdir.py
#	tests/tools/test_macos_protected_search.py
#	tests/tools/test_mcp_npx_cached_bin.py
#	tests/tools/test_oneshot_completion_linger.py
#	tests/tools/test_process_registry.py
#	tests/tools/test_read_file_schema_gating.py
#	tests/tools/test_skill_improvements.py
#	tests/tools/test_skills_sync.py
#	tests/tools/test_termux_api_detection.py
#	tests/tools/test_tirith_security.py
#	tests/tools/test_transcription_tools.py
#	tests/tools/test_tts_streaming.py
#	tests/tools/test_wake_word.py
#	tests/tui_gateway/test_compute_host_borrowed_lease.py
#	tests/tui_gateway/test_compute_host_turn_protocol.py
#	tests/tui_gateway/test_isolated_orphan_activity.py
#	tests/tui_gateway/test_protocol.py
#	tests/tui_gateway/test_slash_worker_profile_home.py
#	tests/tui_gateway/test_subprocess_encoding.py
#	tests/tui_gateway/test_tui_gateway_server.py
#	ui-tui/src/__tests__/terminalParity.test.ts
#	ui-tui/src/__tests__/termuxComposerLayout.test.ts
#	ui-tui/src/__tests__/textInputFastEcho.test.ts
2026-09-23 07:02:44 -04:00

577 lines
20 KiB
Python

"""Actual background tasks exercise the same configured route as foreground chat."""
import asyncio
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import json
import socket
import threading
import time
import pytest
import hermes_yaml as yaml
@pytest.fixture
def actual_endpoint(monkeypatch):
from agent.auxiliary_client import (
shutdown_cached_clients,
_reset_aux_unhealthy_cache,
)
shutdown_cached_clients()
_reset_aux_unhealthy_cache()
requests = []
resolve_address = socket.getaddrinfo
def local_actual_address(host, *args, **kwargs):
if host in ("api.actual.inc", b"api.actual.inc"):
host = "127.0.0.1"
return resolve_address(host, *args, **kwargs)
monkeypatch.setattr(socket, "getaddrinfo", local_actual_address)
monkeypatch.setenv("NO_PROXY", "127.0.0.1,localhost,api.actual.inc")
class Handler(BaseHTTPRequestHandler):
def do_POST(self):
payload = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
requests.append((self.path, payload))
if self.path != "/v1/chat/completions":
self.send_error(404)
return
if payload["model"] == "unavailable-model":
body = json.dumps({
"error": {
"message": "The model is not supported with this account",
"type": "invalid_request_error",
"code": "unsupported_model",
}
}).encode()
self.send_response(400)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
return
content = (
'{"title":"Actual background routing"}'
if "response_format" in payload
else "The task is complete."
)
response = {
"id": "chatcmpl-background",
"created": 1,
"model": payload["model"],
"object": "chat.completion",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": content},
"finish_reason": "stop",
}
],
"usage": {
"prompt_tokens": 10,
"completion_tokens": 5,
"total_tokens": 15,
},
}
if payload.get("stream"):
response["object"] = "chat.completion.chunk"
response["choices"][0]["delta"] = response["choices"][0].pop("message")
body = f"data: {json.dumps(response)}\n\ndata: [DONE]\n\n".encode()
content_type = "text/event-stream"
else:
body = json.dumps(response).encode()
content_type = "application/json"
self.send_response(200)
self.send_header("Content-Type", content_type)
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def log_message(self, *_args):
pass
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
thread = threading.Thread(
target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True
)
thread.start()
try:
yield f"http://127.0.0.1:{server.server_port}", requests
finally:
shutdown_cached_clients()
server.shutdown()
server.server_close()
thread.join(timeout=5)
@pytest.mark.parametrize(
"aux_provider",
[
"auto",
"actual",
"aci",
"custom",
"custom:actual-relay",
],
)
@pytest.mark.parametrize(
"hosted,use_api_key", [(False, False), (False, True), (True, True)]
)
@pytest.mark.parametrize("stale_mode", [None, "codex_responses"])
def test_actual_background_tasks_reach_chat_completions(
tmp_path,
monkeypatch,
actual_endpoint,
aux_provider,
hosted,
use_api_key,
stale_mode,
):
from agent.auxiliary_client import async_call_llm
from agent.context_compressor import ContextCompressor
from agent.title_generator import generate_title
from hermes_cli.runtime_provider import resolve_runtime_provider
base_url, requests = actual_endpoint
if hosted:
base_url = base_url.replace("127.0.0.1", "api.actual.inc")
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
if use_api_key:
monkeypatch.setenv("ACTUAL_API_KEY", "actual-test-key")
monkeypatch.setenv("ACTUAL_BASE_URL", "http://127.0.0.1:1")
aux_model = "override-model" if aux_provider.startswith("custom") else "test-model"
config = {
"model": {
"provider": "aci" if aux_provider == "aci" else "actual",
"default": "test-model",
"base_url": base_url,
},
"auxiliary": {
task: {
"provider": aux_provider,
"model": aux_model,
"api_mode": stale_mode,
**(
{"base_url": base_url if stale_mode else base_url + "/v1"}
if aux_provider == "custom"
else {}
),
}
for task in (
"compression",
"title_generation",
"session_search",
)
},
}
config["providers"] = {
"actual-relay": {
"base_url": base_url if stale_mode else base_url + "/v1",
"key_env": "ACTUAL_API_KEY",
"transport": stale_mode or "chat_completions",
}
}
(tmp_path / "config.yaml").write_text(yaml.safe_dump(config), encoding="utf-8")
runtime = resolve_runtime_provider(requested="actual")
runtime["model"] = "test-model"
assert runtime["api_mode"] == "chat_completions"
if stale_mode:
runtime["api_mode"] = stale_mode
assert (
generate_title(
"Stale conversation", main_runtime=runtime, runtime_validator=lambda: False
)
is None
)
assert requests == []
assert (
generate_title("Check the background routing", timeout=5, main_runtime=runtime)
== "Actual background routing"
)
compressor = ContextCompressor(
model=runtime["model"],
provider=runtime["provider"],
api_key=runtime["api_key"],
base_url=runtime["base_url"],
api_mode=runtime["api_mode"],
config_context_length=32768,
quiet_mode=True,
)
assert (
compressor._call_summary_llm("Summarize this conversation.", time.monotonic())
== "The task is complete."
)
response = asyncio.run(
async_call_llm(
task="session_search",
messages=[{"role": "user", "content": "Summarize the results."}],
main_runtime=runtime,
timeout=5,
)
)
assert response.choices[0].message.content == "The task is complete."
assert len(requests) == 3
assert [(path, payload["model"]) for path, payload in requests] == [
("/v1/chat/completions", aux_model)
] * 3
@pytest.mark.parametrize(
"provider,hosted",
[
("actual", False),
("aci", False),
("custom", True),
("custom:actual-relay", True),
],
)
@pytest.mark.parametrize(
"entrypoint",
[
"init",
"auto",
"switch",
"fallback",
"restore",
"rotation",
"init_fallback",
"init_auto",
],
)
def test_actual_runtime_transitions_reach_chat_completions(
tmp_path, monkeypatch, actual_endpoint, provider, hosted, entrypoint
):
from agent.error_classifier import FailoverReason
from hermes_cli.runtime_provider import resolve_runtime_provider
from run_agent import AIAgent
base_url, requests = actual_endpoint
if hosted:
base_url = base_url.replace("127.0.0.1", "api.actual.inc")
base_url += "/v1"
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
monkeypatch.setenv("ACTUAL_API_KEY", "actual-test-key")
monkeypatch.setenv("OPENAI_API_KEY", "actual-test-key")
config = {
"model": {
"provider": provider,
"default": "gpt-5.4",
"base_url": base_url,
"api_mode": "codex_responses",
},
"providers": {
"actual-relay": {
"base_url": base_url,
"key_env": "ACTUAL_API_KEY",
"transport": "codex_responses",
}
},
}
(tmp_path / "config.yaml").write_text(yaml.safe_dump(config), encoding="utf-8")
runtime = resolve_runtime_provider(requested=provider)
assert runtime["api_mode"] == "chat_completions"
initial_base = base_url.removesuffix("/v1")
initial_provider = provider
initial_key = "actual-test-key"
if entrypoint in {"init_fallback", "init_auto"}:
initial_base = initial_key = None
initial_provider = (
"missing-provider" if entrypoint == "init_fallback" else "auto"
)
elif entrypoint == "rotation":
initial_base = base_url.replace("api.actual.inc", "127.0.0.1")
agent = AIAgent(
provider=initial_provider,
base_url=initial_base,
api_key=initial_key,
api_mode=None if entrypoint == "auto" else "codex_responses",
model="primary-model" if entrypoint == "fallback" else "gpt-5.4",
enabled_toolsets=[],
quiet_mode=True,
skip_context_files=True,
skip_memory=True,
save_trajectories=False,
fallback_model={
"provider": provider,
"model": "gpt-5.4",
"base_url": base_url.removesuffix("/v1"),
"api_key": "actual-test-key",
"api_mode": "codex_responses",
},
)
try:
if entrypoint == "switch":
agent.switch_model(
"gpt-5.4",
provider,
api_key="actual-test-key",
base_url=base_url.removesuffix("/v1"),
api_mode="codex_responses",
)
elif entrypoint == "fallback":
assert agent._try_activate_fallback(FailoverReason.rate_limit)
elif entrypoint == "restore":
agent._primary_runtime["api_mode"] = "codex_responses"
agent._fallback_activated = True
assert agent._restore_primary_runtime()
elif entrypoint == "rotation":
from agent.credential_pool import PooledCredential
agent._swap_credential(
PooledCredential.from_dict(
provider,
{
"id": "rotated",
"access_token": "actual-rotated-key",
"base_url": base_url.removesuffix("/v1"),
},
)
)
assert agent.api_mode == "chat_completions"
response = agent._interruptible_api_call(
agent._build_api_kwargs([{"role": "user", "content": "Reply briefly."}], [])
)
assert response.choices[0].message.content == "The task is complete."
inference_requests = [
(path, body) for path, body in requests if path != "/api/show"
]
assert len(inference_requests) == 1
assert inference_requests[0][0] == "/v1/chat/completions"
assert inference_requests[0][1]["model"] == "gpt-5.4"
finally:
agent.client.close()
@pytest.mark.parametrize(
"provider,hosted",
[
("actual", False),
("aci", False),
("custom", True),
("custom:actual-relay", True),
],
)
@pytest.mark.parametrize("send_site", ["main", "auxiliary", "async_auxiliary"])
def test_actual_rejects_forced_responses_before_http(
tmp_path, monkeypatch, actual_endpoint, provider, hosted, send_site
):
from agent.auxiliary_client import (
AsyncCodexAuxiliaryClient,
CodexAuxiliaryClient,
resolve_provider_client,
)
from run_agent import AIAgent
base_url, requests = actual_endpoint
if hosted:
base_url = base_url.replace("127.0.0.1", "api.actual.inc")
base_url += "/v1"
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
config = {
"model": {"provider": provider, "base_url": base_url, "default": "test-model"},
"providers": {
"actual-relay": {"base_url": base_url, "transport": "codex_responses"}
},
}
(tmp_path / "config.yaml").write_text(yaml.safe_dump(config), encoding="utf-8")
if send_site == "main":
agent = AIAgent(
provider=provider,
base_url=base_url,
api_key="actual-test-key",
model="test-model",
enabled_toolsets=[],
quiet_mode=True,
skip_context_files=True,
skip_memory=True,
save_trajectories=False,
)
client = agent.client
agent.api_mode = "codex_responses"
def force_responses():
return agent._run_codex_stream(
{"model": "test-model", "input": "Reply briefly."}, client=client
)
else:
client, model = resolve_provider_client(
provider,
model="test-model",
explicit_base_url=base_url,
explicit_api_key="actual-test-key",
api_mode="codex_responses",
)
wrapper = CodexAuxiliaryClient(client, model)
if send_site == "async_auxiliary":
wrapper = AsyncCodexAuxiliaryClient(wrapper)
def force_responses():
result = wrapper.chat.completions.create(
model=model, messages=[{"role": "user", "content": "Reply briefly."}]
)
return asyncio.run(result) if send_site == "async_auxiliary" else result
try:
initial_requests = list(requests)
with pytest.raises(ValueError, match="Actual.*Chat Completions"):
force_responses()
assert requests == initial_requests
finally:
client.close()
@pytest.mark.parametrize("provider", ["actual", "aci", "custom:actual-relay"])
@pytest.mark.parametrize("async_mode", [False, True])
def test_actual_auxiliary_fallback_reaches_chat_completions(
tmp_path, monkeypatch, actual_endpoint, provider, async_mode
):
from agent.auxiliary_client import call_llm, async_call_llm
local_url, requests = actual_endpoint
actual_url = local_url.replace("127.0.0.1", "api.actual.inc") + "/v1"
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
monkeypatch.setenv("ACTUAL_API_KEY", "actual-test-key")
config = {
"model": {
"provider": "actual",
"default": "test-model",
"base_url": actual_url,
},
"providers": {
"actual-relay": {
"base_url": actual_url,
"key_env": "ACTUAL_API_KEY",
"transport": "codex_responses",
}
},
"auxiliary": {
"session_search": {
"provider": "custom",
"model": "unavailable-model",
"base_url": local_url + "/v1",
"fallback_chain": [
{
"provider": provider,
"model": "test-model",
"api_mode": "codex_responses",
"base_url": actual_url,
}
],
}
},
}
(tmp_path / "config.yaml").write_text(yaml.safe_dump(config), encoding="utf-8")
kwargs = {
"task": "session_search",
"messages": [{"role": "user", "content": "Summarize the results."}],
"timeout": 5,
}
response = (
asyncio.run(async_call_llm(**kwargs)) if async_mode else call_llm(**kwargs)
)
assert response.choices[0].message.content == "The task is complete."
assert requests[0][1]["model"] == "unavailable-model"
assert requests[-1][1]["model"] == "test-model"
assert all(path == "/v1/chat/completions" for path, _payload in requests)
@pytest.mark.parametrize("override", ["", "http://127.0.0.1:8081", "invalid-url"])
@pytest.mark.parametrize("configured_provider", ["actual", "aci"])
def test_actual_setup_keeps_provider_settings_in_yaml(
tmp_path, monkeypatch, override, configured_provider
):
from hermes_cli import config as config_module
from hermes_cli import model_setup_flows as setup
from hermes_cli.auth import resolve_api_key_provider_credentials
from hermes_cli.runtime_provider import resolve_runtime_provider
from providers import get_provider_profile
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
monkeypatch.setenv("ACTUAL_API_KEY", "actual-test-key")
monkeypatch.setenv("ACTUAL_BASE_URL", "http://127.0.0.1:8089")
configured_url = "http://127.0.0.1:8080"
raw = {
"model": {
"provider": configured_provider,
"default": "old-model",
"base_url": configured_url,
}
}
config_path = tmp_path / "config.yaml"
config_path.write_text(yaml.safe_dump(raw), encoding="utf-8")
env_path = tmp_path / ".env"
env_path.write_text("ACTUAL_API_KEY=actual-test-key\n", encoding="utf-8")
monkeypatch.setattr(
setup,
"_ensure_flow_api_key",
lambda *_a, **_kw: ("actual-test-key", "actual-test-key", False),
)
monkeypatch.setattr(setup, "_ask", lambda *_a, **_kw: override)
monkeypatch.setattr(setup, "_api_key_provider_model_list", lambda *_a: [])
monkeypatch.setattr(setup, "_pick_model_or_prompt", lambda *_a, **_kw: "test-model")
setup._model_flow_api_key_provider(config_module.load_config(), "actual")
expected_url = override if override.startswith("http://") else configured_url
saved = yaml.safe_load(config_path.read_text(encoding="utf-8"))
assert saved["model"]["provider"] == "actual"
assert saved["model"]["base_url"] == expected_url
assert saved["model"]["default"] == "test-model"
assert env_path.read_text(encoding="utf-8") == "ACTUAL_API_KEY=actual-test-key\n"
assert get_provider_profile("actual").env_vars == ("ACTUAL_API_KEY",)
assert "ACTUAL_BASE_URL" not in config_module.OPTIONAL_ENV_VARS
assert "ACTUAL_API_MODE" not in config_module.OPTIONAL_ENV_VARS
assert (
resolve_api_key_provider_credentials("actual")["base_url"]
== expected_url + "/v1"
)
for kwargs in ({}, {"explicit_api_key": "actual-test-key"}):
runtime = resolve_runtime_provider(requested="actual", **kwargs)
assert runtime["base_url"] == expected_url + "/v1"
assert runtime["api_mode"] == "chat_completions"
def test_actual_key_reload_keeps_yaml_endpoint(tmp_path, monkeypatch, actual_endpoint):
from run_agent import AIAgent
base_url, requests = actual_endpoint
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
monkeypatch.setenv("ACTUAL_API_KEY", "actual-test-key")
monkeypatch.setenv("ACTUAL_BASE_URL", "http://127.0.0.1:1")
(tmp_path / "config.yaml").write_text(
yaml.safe_dump({
"model": {"provider": "aci", "default": "test-model", "base_url": base_url},
}),
encoding="utf-8",
)
env_path = tmp_path / ".env"
env_path.write_text("ACTUAL_API_KEY=actual-test-key\n", encoding="utf-8")
agent = AIAgent(
provider="actual",
base_url=base_url,
api_key="actual-test-key",
model="test-model",
enabled_toolsets=[],
quiet_mode=True,
skip_context_files=True,
skip_memory=True,
save_trajectories=False,
)
try:
assert not agent._try_refresh_env_client_credentials()
env_path.write_text("ACTUAL_API_KEY=actual-rotated-key\n", encoding="utf-8")
assert agent._try_refresh_env_client_credentials()
assert agent.api_key == "actual-rotated-key"
assert agent.base_url == base_url + "/v1"
assert agent.api_mode == "chat_completions"
response = agent._interruptible_api_call(
agent._build_api_kwargs([{"role": "user", "content": "Reply briefly."}], [])
)
assert response.choices[0].message.content == "The task is complete."
assert [
path for path, body in requests if body.get("model") == "test-model"
] == ["/v1/chat/completions"]
finally:
agent.client.close()