hermes-agent/tests/gateway/test_multiplex_hot_serve.py
ethernet 16652eea18 Merge remote-tracking branch 'origin/main' into ethie/pm-clean
# Conflicts:
#	gateway/config.py
#	gateway/config_loader.py
#	gateway/readiness.py
#	hermes_cli/managed_scope.py
#	hermes_cli/plugin_python_deps.py
#	hermes_cli/plugins_cmd.py
#	hermes_cli/update_cmd_maint.py
#	plugin-catalog/hindsight.yaml
#	plugins/plugin_loader.py
#	providers/__init__.py
#	scripts/run_tests.sh
#	tests/gateway/test_control_socket_windows_live.py
#	tests/gateway/test_gateway_streaming_nested_config.py
#	tests/hermes_cli/test_doctor.py
#	tests/hermes_cli/test_plan_reconciliation_windows_live.py
#	tests/hermes_cli/test_update_apply_shallow_count.py
#	tests/hermes_cli/test_update_concurrent_quarantine.py
#	tests/hermes_cli/test_update_shim_self_lock.py
#	tests/hermes_cli/test_verify_console_scripts.py
#	tests/tools/test_lazy_deps.py
#	tests/tui_gateway/test_subprocess_encoding.py
#	tools/lazy_deps.py
2026-09-23 15:26:34 -04:00

381 lines
18 KiB
Python

"""Hot-serve invariants for ``gateway.multiplex_profiles`` (``gateway/run_profile_reconcile.py``).
The multiplexer used to enumerate ``profiles/`` once at boot; these pin the runtime reconcile: a
profile created afterwards is served, a deleted one is torn down and unrouted, a served profile whose
config/.env changed (bot token added after create) gets its adapters, and none of it touches the other
profiles' live adapters. The cron ticker's live enumerator is covered in ``tests/cron``.
"""
import asyncio
import json
from pathlib import Path
from unittest.mock import MagicMock, patch
import pytest
from gateway.config import GatewayConfig, Platform
from gateway.run import GatewayRunner
from gateway.run_profile_reconcile import profile_serve_signature
from gateway.status import flush_runtime_status
class _Adapter:
platform = Platform.DISCORD
def __init__(self, token):
self.token = token
self.disconnected = False
self.cancelled = False
async def disconnect(self):
self.disconnected = True
async def cancel_background_tasks(self):
self.cancelled = True
def _runner(tmp_path, monkeypatch):
home = tmp_path / ".hermes"
(home / "profiles").mkdir(parents=True)
monkeypatch.setenv("HERMES_HOME", str(home))
monkeypatch.setattr(Path, "home", lambda: tmp_path)
runner = object.__new__(GatewayRunner)
runner.config = GatewayConfig(multiplex_profiles=True)
runner._running = True
runner._primary_profile_name = "default"
runner.adapters = {}
runner._profile_adapters = {}
runner._profile_failed_platforms = {}
runner._failed_platforms = {}
runner._agent_cache = {}
runner._agent_cache_lock = None
runner.pairing_store = MagicMock()
runner.pairing_stores = {}
runner._adapter_disconnect_timeout_secs = lambda: 0.5
started = []
async def _start(profile_name, profile_home, claimed):
started.append(profile_name)
token = (profile_home / ".env").read_text(encoding="utf-8") if (profile_home / ".env").exists() else ""
if "DISCORD_BOT_TOKEN" not in token:
return 0
runner._profile_adapters.setdefault(profile_name, {})[Platform.DISCORD] = _Adapter(token)
return 1
runner._start_one_profile_adapters = _start
runner._adapter_credential_fingerprint = lambda adapter: getattr(adapter, "token", None)
runner._started = started
return runner, home
def _mkprofile(home, name, env=""):
d = home / "profiles" / name
d.mkdir(parents=True, exist_ok=True)
(d / "config.yaml").write_text("model: {default: m}\n", encoding="utf-8")
(d / ".env").write_text(env, encoding="utf-8")
return d
def _served_record(home):
flush_runtime_status()
return json.loads((home / "gateway_state.json").read_text(encoding="utf-8")).get("served_profiles")
@pytest.mark.asyncio
async def test_opt_out_rescans_and_opt_in_waits_for_own_gateway_to_stop(tmp_path, monkeypatch, caplog):
runner, home = _runner(tmp_path, monkeypatch)
solo = _mkprofile(home, "solo", "DISCORD_BOT_TOKEN=solo-token\n")
own_pids = {}
monkeypatch.setattr("gateway.status.live_gateway_pid_for_home", lambda h: own_pids.get(h))
with patch("hermes_cli.profiles.get_active_profile_name", return_value="default"):
await runner._start_secondary_profile_adapters()
adapter = runner._profile_adapters["solo"][Platform.DISCORD]
(solo / "config.yaml").write_text("gateway:\n standalone: true\n")
result = await runner.reconcile_served_profiles()
assert result["removed"] == ["solo"]
assert adapter.disconnected
assert _served_record(home) == ["default"]
own_pids[solo] = 12345
(solo / "config.yaml").write_text("gateway:\n standalone: false\n")
for _ in range(2):
result = await runner.reconcile_served_profiles()
assert result["added"] == []
assert result["served_profiles"] == ["default"]
assert runner._started.count("solo") == 1
assert len([r for r in caplog.records if "still runs its own gateway" in r.message]) == 1
own_pids.clear()
result = await runner.reconcile_served_profiles()
assert result["added"] == ["solo"]
assert _served_record(home) == ["default", "solo"]
assert runner._started.count("solo") == 2
@pytest.mark.asyncio
async def test_parked_profile_boot_and_reconcile(tmp_path, monkeypatch, caplog):
runner, home = _runner(tmp_path, monkeypatch)
secondary = _mkprofile(home, "worker")
marker = secondary / "gateway.parked"
marker.touch()
# Boot uses the real directory enumerator and config loaders, no bot/network.
del runner._start_one_profile_adapters
runner._register_config_hooks = lambda *a, **kw: None
caplog.set_level("INFO")
with patch("hermes_cli.profiles.get_active_profile_name", return_value="default"):
await runner._start_secondary_profile_adapters()
assert _served_record(home) == ["default"]
assert "profile 'worker' is parked (gateway.parked); not served by this gateway" in caplog.text
marker.unlink()
assert (await runner.reconcile_served_profiles())["added"] == ["worker"]
assert _served_record(home) == ["default", "worker"]
marker.touch()
assert (await runner.reconcile_served_profiles())["removed"] == ["worker"]
assert _served_record(home) == ["default"]
@pytest.mark.asyncio
async def test_profile_control_verbs_round_trip_and_refusals(tmp_path, monkeypatch):
from gateway import run_profile_reconcile as verbs
runner, home = _runner(tmp_path, monkeypatch)
secondary = _mkprofile(home, "worker", "DISCORD_BOT_TOKEN=worker-token\n")
with patch("hermes_cli.profiles.get_active_profile_name", return_value="default"):
await runner._start_secondary_profile_adapters()
stop = verbs.unserve_profile_verb(runner)
start = verbs.serve_profile_verb(runner)
for handler, name in [(stop, "default"), (stop, "missing"), (start, "missing"),
(start, "worker"), (start, "default")]:
assert (await asyncio.to_thread(handler, {"name": name}))["error"]
old = runner._profile_adapters["worker"][Platform.DISCORD]
answer = await asyncio.to_thread(stop, {"name": "worker"})
assert answer["unserved"] == "worker"
assert answer["served_profiles"] == _served_record(home) == ["default"]
assert old.disconnected
marker = secondary / "gateway.parked"
marker.touch()
assert (await asyncio.to_thread(start, {"name": "worker"}))["error"]
marker.unlink()
(secondary / ".env").write_text("DISCORD_BOT_TOKEN=new-worker-token\n")
answer = await asyncio.to_thread(start, {"name": "worker"})
assert answer["served"] == "worker"
assert answer["served_profiles"] == _served_record(home) == ["default", "worker"]
assert runner._profile_adapters["worker"][Platform.DISCORD].token.endswith("new-worker-token\n")
@pytest.mark.platforms("linux")
@pytest.mark.asyncio
async def test_profile_lifecycle_over_real_control_socket(tmp_path, monkeypatch):
from gateway.run import _start_gateway_start_control_socket
from gateway import control_socket
runner, home = _runner(tmp_path, monkeypatch)
secondary = _mkprofile(home, "worker")
with patch("hermes_cli.profiles.get_active_profile_name", return_value="default"):
await runner._start_secondary_profile_adapters()
server = await _start_gateway_start_control_socket(runner)
assert server is not None
try:
(secondary / "gateway.parked").touch()
stopped = await asyncio.to_thread(control_socket.request_unserve_profile, home, "worker")
assert stopped["unserved"] == "worker"
assert _served_record(home) == ["default"]
refused = await asyncio.to_thread(control_socket.request_serve_profile_hot, home, "worker")
assert "parked" in refused["error"]
(secondary / "gateway.parked").unlink()
started = await asyncio.to_thread(control_socket.request_serve_profile_hot, home, "worker")
assert started["served"] == "worker"
assert _served_record(home) == ["default", "worker"]
finally:
await server.stop()
@pytest.mark.asyncio
async def test_created_then_credentialed_profile_is_served_without_restart(tmp_path, monkeypatch):
runner, home = _runner(tmp_path, monkeypatch)
alpha_dir = _mkprofile(home, "alpha", "DISCORD_BOT_TOKEN=alpha-token\n")
with patch("hermes_cli.profiles.get_active_profile_name", return_value="default"):
await runner._start_secondary_profile_adapters()
alpha_adapter = runner._profile_adapters["alpha"][Platform.DISCORD]
assert _served_record(home) == ["default", "alpha"]
# 1. Created while running, no token yet: served (routes/prefixes/cron), zero adapters.
gamma_dir = _mkprofile(home, "gamma")
result = await runner.reconcile_served_profiles()
assert result["added"] == ["gamma"]
assert _served_record(home) == ["default", "alpha", "gamma"]
assert "gamma" in runner.pairing_stores
assert Platform.DISCORD not in runner._profile_adapters.get("gamma", {})
# 2. Token added afterwards: the rescan builds the adapter (never "adapter-less forever").
(gamma_dir / ".env").write_text("DISCORD_BOT_TOKEN=gamma-token\n", encoding="utf-8")
result = await runner.reconcile_served_profiles()
assert result["rescanned"] == ["gamma"]
assert runner._profile_adapters["gamma"][Platform.DISCORD].token.strip().endswith("gamma-token")
# 3. A no-op rescan and the whole sequence never touched alpha's live adapter.
assert await runner.reconcile_served_profiles() == {
"added": [], "removed": [], "rescanned": [], "reason": "request",
"served_profiles": ["default", "alpha", "gamma"],
}
assert runner._profile_adapters["alpha"][Platform.DISCORD] is alpha_adapter
assert alpha_adapter.disconnected is False
assert runner._started.count("alpha") == 1
assert profile_serve_signature(alpha_dir) == runner._served_profile_signatures["alpha"]
@pytest.mark.asyncio
async def test_deleted_profile_is_torn_down_and_unrouted_others_untouched(tmp_path, monkeypatch):
runner, home = _runner(tmp_path, monkeypatch)
_mkprofile(home, "alpha", "DISCORD_BOT_TOKEN=alpha-token\n")
gamma_dir = _mkprofile(home, "gamma", "DISCORD_BOT_TOKEN=gamma-token\n")
with patch("hermes_cli.profiles.get_active_profile_name", return_value="default"):
await runner._start_secondary_profile_adapters()
alpha_adapter = runner._profile_adapters["alpha"][Platform.DISCORD]
gamma_adapter = runner._profile_adapters["gamma"][Platform.DISCORD]
runner._agent_cache = {"agent:gamma:discord:dm:1": ("agent",), "agent:alpha:discord:dm:1": ("agent",)}
evicted = []
runner._evict_cached_agent = evicted.append
reconnect = asyncio.get_running_loop().create_task(asyncio.sleep(3600))
runner._profile_failed_platforms = {"gamma": {Platform.TELEGRAM: reconnect}}
from hermes_constants import mark_named_profile_deleted
mark_named_profile_deleted(gamma_dir) # what ``delete_profile`` does before rmtree
result = await runner.reconcile_served_profiles()
assert result["removed"] == ["gamma"]
assert gamma_adapter.disconnected is True and gamma_adapter.cancelled is True
assert "gamma" not in runner._profile_adapters
assert "gamma" not in runner.pairing_stores
assert reconnect.cancelled()
assert evicted == ["agent:gamma:discord:dm:1"]
assert _served_record(home) == ["default", "alpha"]
assert runner._profile_adapters["alpha"][Platform.DISCORD] is alpha_adapter
assert alpha_adapter.disconnected is False
@pytest.mark.asyncio
async def test_transient_start_failure_is_retried_on_next_reconcile(tmp_path, monkeypatch):
"""A hot-added profile whose adapter start raises a transient error (secret backend
unreachable, a half-written config.yaml) must not be recorded as scanned: the next
reconcile retries it. Only the deliberate MultiplexConfigError park is acknowledged."""
runner, home = _runner(tmp_path, monkeypatch)
_mkprofile(home, "alpha", "DISCORD_BOT_TOKEN=alpha-token\n")
with patch("hermes_cli.profiles.get_active_profile_name", return_value="default"):
await runner._start_secondary_profile_adapters()
_mkprofile(home, "gamma", "DISCORD_BOT_TOKEN=gamma-token\n")
attempts = []
real_start = runner._start_one_profile_adapters
async def flaky(name, profile_home, claimed):
attempts.append(name)
if len(attempts) == 1:
raise OSError("secret backend unreachable")
return await real_start(name, profile_home, claimed)
runner._start_one_profile_adapters = flaky
first = await runner.reconcile_served_profiles()
assert first["added"] == ["gamma"]
assert attempts == ["gamma"]
assert Platform.DISCORD not in runner._profile_adapters.get("gamma", {})
# Unacknowledged: the failed scan's signature must not survive ``_note_served_profiles``.
assert "gamma" not in runner._served_profile_signatures
second = await runner.reconcile_served_profiles()
assert attempts == ["gamma", "gamma"]
assert second["rescanned"] == ["gamma"]
assert runner._profile_adapters["gamma"][Platform.DISCORD].token.strip().endswith("gamma-token")
# The deliberate park stays distinct: a MultiplexConfigError is acknowledged, not retried.
from gateway.run import MultiplexConfigError
delta_dir = _mkprofile(home, "delta", "DISCORD_BOT_TOKEN=delta-token\n")
parked = []
async def park(name, profile_home, claimed):
parked.append(name)
raise MultiplexConfigError("open dm_policy")
runner._start_one_profile_adapters = park
await runner.reconcile_served_profiles()
await runner.reconcile_served_profiles()
assert parked == ["delta"]
assert runner._served_profile_signatures["delta"] == profile_serve_signature(delta_dir)
@pytest.mark.asyncio
async def test_transient_secret_hydrate_failure_retries_through_real_start_path(tmp_path, monkeypatch):
"""End to end through the real ``_start_one_profile_adapters``: the failure is raised inside
it at ``hydrate_profile_secret_sources`` (the network-dependent step of
``_load_secondary_profile_config``), not at a mocked seam. The boot failure leaves the
profile served-but-unsigned; the next reconcile runs the real config load, profile scope,
adapter creation and connect to success."""
from types import SimpleNamespace
runner, home = _runner(tmp_path, monkeypatch)
del runner._start_one_profile_adapters # instance stub off: exercise the real method
runner._register_config_hooks = lambda *a, **kw: None
runner._configure_profile_adapter = lambda *a: None
runner._sync_voice_mode_state_to_adapter = lambda *a: None
runner._restore_secondary_completion_ledgers = lambda *a: None
runner._adapter_credential_claim = lambda *a: None
runner._adapter_listener_claim = lambda *a: None
runner._busy_text_modes_by_profile = {}
runner._busy_input_modes_by_profile = {}
connected = []
async def _connect(adapter, platform):
connected.append(platform)
return True
async def _noop_added(profiles):
pass
runner._create_adapter = lambda platform, config: SimpleNamespace(platform=platform)
runner._connect_initial_adapter_with_timeout = _connect
runner._after_profiles_added = _noop_added
import hermes_cli.env_loader as env_loader
hydrate_calls = []
real_hydrate = env_loader.hydrate_profile_secret_sources
def _flaky(home_arg):
hydrate_calls.append(str(home_arg))
if len(hydrate_calls) == 1:
raise OSError("secret backend unreachable")
return real_hydrate(home_arg)
monkeypatch.setattr(env_loader, "hydrate_profile_secret_sources", _flaky)
gamma_dir = _mkprofile(home, "gamma", "DISCORD_BOT_TOKEN=gamma-token\n")
with patch("hermes_cli.profiles.get_active_profile_name", return_value="default"):
await runner._start_secondary_profile_adapters()
assert connected == []
assert "gamma" not in runner._served_profile_signatures
assert _served_record(home) == ["default", "gamma"]
result = await runner.reconcile_served_profiles()
assert result["rescanned"] == ["gamma"]
assert connected == [Platform.DISCORD]
assert runner._profile_adapters["gamma"][Platform.DISCORD].platform == Platform.DISCORD
assert runner._served_profile_signatures["gamma"] == profile_serve_signature(gamma_dir)
@pytest.mark.asyncio
async def test_hot_added_profile_cannot_double_claim_a_live_secondary_token(tmp_path, monkeypatch):
"""Boot's duplicate-credential guard sees every profile at once; a hot add must see the LIVE
secondaries' claims too, or the new profile starts a second poller on alpha's bot."""
runner, home = _runner(tmp_path, monkeypatch)
_mkprofile(home, "alpha", "DISCORD_BOT_TOKEN=shared\n")
seen_claims = {}
async def _start(profile_name, profile_home, claimed):
seen_claims[profile_name] = dict(claimed)
runner._profile_adapters.setdefault(profile_name, {})[Platform.DISCORD] = _Adapter("shared")
return 1
runner._start_one_profile_adapters = _start
with patch("hermes_cli.profiles.get_active_profile_name", return_value="default"):
await runner._start_secondary_profile_adapters()
_mkprofile(home, "dupe", "DISCORD_BOT_TOKEN=shared\n")
await runner.reconcile_served_profiles()
fp = GatewayRunner._adapter_credential_fingerprint(_Adapter("shared"))
assert seen_claims["dupe"].get((Platform.DISCORD, fp)) == "alpha"