hermes-agent/tests/hermes_cli/test_resume_model_restore.py
teknium1 485979ddf4 fix(review): validate the logsRoot owner name; scope the whole palette and fail 4064 on an unknown profile
Independent-review findings on #126159:

- MAJOR: `hermes:fs:logsRoot` joined a renderer-supplied profile name into
  `profiles/<name>` unvalidated (session-owner data; from the remote backend in
  remote mode), so `..`/absolute values escaped the profiles root, were created
  and revealed. `localPluginsRoot` now accepts only `default` or a
  `DESKTOP_PROFILE_NAME_RE` match — the gate `localSkinProfileKey` and main.ts
  already apply — and otherwise falls back to the active Desktop profile.
- MINOR (b): `commands.catalog` scoped skill discovery to the routed profile but
  read quick_commands and plugin commands unscoped, listing the LAUNCH profile's
  quick commands in a profile-routed palette. All loaders run inside the one
  `_session_home_scope` block.
- MINOR (c): `commands.catalog` swallowed ProfileUnavailableError into a warning
  (ok + empty skills) while complete.slash returned 4064; `_profile_scoped_rpc`
  re-raises it so dispatch maps it to 4064 (also fixes skills.reload's 5025).
- MINOR (a): the shared `session_gateway_runtime` reader's new merge (top-level
  base_url/api_mode + billing_provider) gets a CLI-side test; the CLI-visible
  precedence change is disclosed in the PR body.
2026-09-28 03:13:19 -07:00

441 lines
18 KiB
Python

"""Tests for CLI resume model restoration and /model session persistence.
Covers _restore_session_model, _persist_model_switch_to_session (cli.py) and
SessionDB.session_gateway_runtime (hermes_state.py) — the round trip that
makes `hermes --resume` reopen a session on the model/provider it actually
used instead of the ambient config default (#57588-class, #79536).
"""
import json
import cli as cli_mod
from hermes_state import SessionDB
def _make_stub(**overrides):
"""Bare HermesCLI the way resume paths see it (no __init__)."""
stub = object.__new__(cli_mod.HermesCLI)
stub.model = "ambient-model"
stub.provider = "openrouter"
stub.requested_provider = "openrouter"
stub.base_url = "https://openrouter.ai/api/v1"
stub.api_key = "ambient-key"
stub.api_mode = ""
stub.agent = None
stub._console_print = lambda s: None
for key, value in overrides.items():
setattr(stub, key, value)
return stub
def _row(model="glm-4.7", model_config=None):
return {
"model": model,
"model_config": json.dumps(model_config) if model_config else None,
}
# ── SessionDB.session_gateway_runtime ───────────────────────────────
def test_session_gateway_runtime_prefers_nested_key():
meta = _row(model_config={
"gateway_runtime": {"provider": "custom:feather", "base_url": "https://f/v1"},
"provider": "openrouter",
})
runtime = SessionDB.session_gateway_runtime(meta)
assert runtime["provider"] == "custom:feather"
assert runtime["base_url"] == "https://f/v1"
def test_session_gateway_runtime_falls_back_to_top_level_keys():
# The TUI gateway's _runtime_model_config writes top-level keys only.
meta = _row(model_config={"provider": "nous", "api_mode": "chat_completions"})
runtime = SessionDB.session_gateway_runtime(meta)
assert runtime == {"provider": "nous", "api_mode": "chat_completions"}
def test_session_gateway_runtime_tolerates_garbage():
assert SessionDB.session_gateway_runtime(None) == {}
assert SessionDB.session_gateway_runtime({}) == {}
assert SessionDB.session_gateway_runtime({"model_config": "{not json"}) == {}
assert SessionDB.session_gateway_runtime({"model_config": json.dumps([1, 2])}) == {}
# ── _restore_session_model ──────────────────────────────────────────
def test_restore_session_model_restores_model_and_provider():
stub = _make_stub()
stub._restore_session_model(_row(model_config={
"gateway_runtime": {"provider": "custom:feather", "base_url": "https://f/v1"},
}))
assert stub.model == "glm-4.7"
assert stub.provider == "custom:feather"
assert stub.requested_provider == "custom:feather"
assert stub.base_url == "https://f/v1"
# Stale launch-time explicit overrides must not leak into the restored
# provider's credential resolution.
assert stub._explicit_api_key is None
assert stub._explicit_base_url == "https://f/v1"
def test_restore_llamacpp_session_follows_live_managed_endpoint(monkeypatch):
"""Managed llama.cpp owns its live port. A snapshot of last boot's loopback
URL must not pin the resumed client to a dead ephemeral endpoint."""
live = {
"provider": "custom",
"base_url": "http://127.0.0.1:18434/v1",
"api_key": "sk-live-managed",
"api_mode": "chat_completions",
}
monkeypatch.setattr(
"hermes_cli.runtime_provider.resolve_runtime_provider",
lambda **_kw: live)
stub = _make_stub()
stub._restore_session_model(_row(
model="Qwen3.8-27B-UD-IQ3_XXS",
model_config={
"provider": "llamacpp",
"base_url": "http://127.0.0.1:51489/v1",
"api_mode": "chat_completions",
}))
assert stub.provider == "llamacpp"
assert stub.requested_provider == "llamacpp"
assert stub.base_url == live["base_url"]
assert stub._explicit_base_url in (None, "")
assert stub.api_key == live["api_key"]
def test_restore_llamacpp_session_keeps_launch_base_url_for_same_provider(monkeypatch):
"""`hermes --provider llamacpp --base-url X --resume` is user intent for the SAME provider the
session ran on; the live-endpoint re-resolution must not re-point it at the local supervisor."""
calls = []
monkeypatch.setattr(
"hermes_cli.runtime_provider.resolve_runtime_provider",
lambda **kw: calls.append(kw) or {"base_url": "http://127.0.0.1:18434/v1"})
user_url = "http://gpu-box:8080/v1"
stub = _make_stub(provider="llamacpp", requested_provider="llamacpp", base_url=user_url,
_explicit_base_url=user_url)
stub._restore_session_model(_row(
model="Qwen3.8-27B-UD-IQ3_XXS",
model_config={"provider": "llamacpp", "base_url": "http://127.0.0.1:51489/v1"}))
assert stub.model == "Qwen3.8-27B-UD-IQ3_XXS"
assert stub.base_url == user_url
assert stub._explicit_base_url == user_url
assert calls == []
def test_restore_session_model_explicit_cli_flag_wins():
stub = _make_stub(model="cli-flag-model", _explicit_model_override=True)
stub._restore_session_model(_row())
assert stub.model == "cli-flag-model"
assert stub.provider == "openrouter"
def test_restore_session_model_no_stored_model_is_noop():
stub = _make_stub()
stub._restore_session_model(_row(model=None))
assert stub.model == "ambient-model"
def test_restore_session_model_matching_state_is_silent_noop():
notes = []
stub = _make_stub(model="glm-4.7", provider="custom:feather",
requested_provider="custom:feather",
_console_print=lambda s: notes.append(s))
stub._restore_session_model(_row(model_config={
"gateway_runtime": {"provider": "custom:feather"},
}))
assert not notes
def test_restore_session_model_swaps_running_agent_in_place():
calls = {}
class _Agent:
def switch_model(self, **kwargs):
calls.update(kwargs)
stub = _make_stub(agent=_Agent())
stub._restore_session_model(_row())
assert calls["new_model"] == "glm-4.7"
# ── _persist_model_switch_to_session ────────────────────────────────
class _Result:
new_model = "deepseek-v4-flash-free"
target_provider = "custom:opencode-zen"
base_url = "https://oz/v1"
api_mode = ""
def test_persist_model_switch_writes_model_and_both_route_shapes():
written = {}
class _DB:
def update_session_model(self, sid, model):
written["model"] = (sid, model)
def patch_session_model_config(self, sid, patch):
written["patch"] = (sid, patch)
stub = _make_stub(_session_db=_DB(), session_id="s1")
stub._persist_model_switch_to_session(_Result())
assert written["model"] == ("s1", "deepseek-v4-flash-free")
sid, patch = written["patch"]
# Nested shape for the CLI reader...
assert patch["gateway_runtime"]["provider"] == "custom:opencode-zen"
# ...and top-level for the TUI gateway's _stored_session_runtime_overrides.
assert patch["provider"] == "custom:opencode-zen"
assert patch["base_url"] == "https://oz/v1"
# Both shapes use or-None so stale keys are deleted (not merely omitted)
# in BOTH gateway_runtime and top-level — the asymmetry that caused the
# original stale-key bug.
assert patch["gateway_runtime"]["api_mode"] is None
assert patch["api_mode"] is None
def test_persist_model_switch_clears_stale_route_keys(tmp_path, monkeypatch):
"""A later switch must not inherit the previous switch's api_mode/base_url.
patch_session_model_config merges key-level and only deletes on explicit
None — dropping falsy values from the patch left the FIRST switch's
api_mode (e.g. anthropic_messages) alive under the SECOND switch's
provider, corrupting the wire protocol on TUI/desktop resume.
"""
monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes"))
db = SessionDB(db_path=tmp_path / "state.db")
db.create_session(session_id="stale1", source="cli", model="m0")
stub = _make_stub(_session_db=db, session_id="stale1")
class _First:
new_model = "claude-x"
target_provider = "custom:feather"
base_url = "https://feather/v1"
api_mode = "anthropic_messages"
class _Second:
new_model = "gpt-5.4"
target_provider = "openrouter"
base_url = "https://openrouter.ai/api/v1"
api_mode = "" # openrouter default — must ERASE the anthropic mode
stub._persist_model_switch_to_session(_First())
stub._persist_model_switch_to_session(_Second())
meta = db.get_session("stale1")
config = json.loads(meta["model_config"])
# Top-level keys: stale values deleted.
assert config["provider"] == "openrouter"
assert "api_mode" not in config, config # stale anthropic_messages deleted
# Nested gateway_runtime: stale values replaced with None (the merge
# replaces the entire gateway_runtime dict, not deep-merging its keys).
# The reader's `or None` / `if v` filtering treats None the same as
# absent, so stale values are effectively erased.
gw = config.get("gateway_runtime", {})
assert gw.get("provider") == "openrouter"
assert gw.get("api_mode") is None # stale anthropic_messages erased
runtime = SessionDB.session_gateway_runtime(meta)
assert runtime["provider"] == "openrouter"
assert "api_mode" not in runtime
def test_persist_model_switch_swallows_db_errors():
class _DB:
def update_session_model(self, *a):
raise RuntimeError("disk full")
stub = _make_stub(_session_db=_DB(), session_id="s1")
stub._persist_model_switch_to_session(_Result()) # must not raise
def test_persist_model_switch_heals_bare_custom(monkeypatch):
"""Bare 'custom' is not routable — heal to custom:<name> or drop (C1)."""
written = {}
class _DB:
def update_session_model(self, sid, model):
written["model"] = model
def patch_session_model_config(self, sid, patch):
written["patch"] = patch
class _BareResult:
new_model = "qwen3.6-plus"
target_provider = "custom"
base_url = "https://my-endpoint/v1"
api_mode = ""
import hermes_cli.runtime_provider as rp
monkeypatch.setattr(rp, "canonical_custom_identity",
lambda base_url=None, model=None: "custom:myendpoint")
stub = _make_stub(_session_db=_DB(), session_id="s1")
stub._persist_model_switch_to_session(_BareResult())
assert written["patch"]["provider"] == "custom:myendpoint"
# Healing fails -> provider dropped (explicit None deletes any stale
# persisted provider), never persisted bare.
monkeypatch.setattr(rp, "canonical_custom_identity",
lambda base_url=None, model=None: None)
written.clear()
stub._persist_model_switch_to_session(_BareResult())
assert written["patch"]["provider"] is None
assert written["patch"]["gateway_runtime"]["provider"] is None
def test_restore_session_model_heals_bare_custom_stored_rows(monkeypatch):
"""Rows persisted by older builds may carry bare 'custom' — heal or drop."""
import hermes_cli.runtime_provider as rp
monkeypatch.setattr(rp, "canonical_custom_identity",
lambda base_url=None, model=None: None)
stub = _make_stub()
stub._restore_session_model(_row(model_config={
"gateway_runtime": {"provider": "custom"},
}))
# Provider dropped -> model restored but provider stays ambient.
assert stub.model == "glm-4.7"
assert stub.provider == "openrouter"
def test_restore_session_model_rederives_per_model_wire_for_opencode_rows(monkeypatch):
"""A row persisted while an opencode-go session ran an anthropic_messages model (MiniMax) must not
pin that wire onto a chat_completions model on resume — api_mode and the relay URL follow the
stored model, and a fixed-wire provider's row is still honored verbatim (#96066)."""
import hermes_cli.runtime_provider as rp
monkeypatch.setattr(rp, "resolve_runtime_provider", lambda **kw: {"api_key": "go-key"})
stub = _make_stub(provider="opencode-go", requested_provider="opencode-go",
base_url="https://opencode.ai/zen/go/v1", api_mode="chat_completions")
stub._restore_session_model(_row(model="deepseek-v4-flash-vision-exp", model_config={
"gateway_runtime": {"provider": "opencode-go", "base_url": "https://opencode.ai/zen/go",
"api_mode": "anthropic_messages"}}))
assert (stub.api_mode, stub.base_url) == ("chat_completions", "https://opencode.ai/zen/go/v1")
stub = _make_stub()
stub._restore_session_model(_row(model="MiniMax-M2.5", model_config={
"gateway_runtime": {"provider": "minimax", "base_url": "https://api.minimax.io/anthropic",
"api_mode": "anthropic_messages"}}))
assert (stub.api_mode, stub.base_url) == ("anthropic_messages", "https://api.minimax.io/anthropic")
# ── round trip: persist → get_session shape → restore ───────────────
def test_round_trip_persist_then_restore(tmp_path, monkeypatch):
monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes"))
db = SessionDB(db_path=tmp_path / "state.db")
db.create_session(session_id="rt1", source="cli", model="ambient-model")
stub = _make_stub(_session_db=db, session_id="rt1")
stub._persist_model_switch_to_session(_Result())
meta = db.get_session("rt1")
restored = _make_stub()
restored._restore_session_model(meta)
assert restored.model == "deepseek-v4-flash-free"
assert restored.provider == "custom:opencode-zen"
assert restored.base_url == "https://oz/v1"
# ── update_session_model provider persistence (#79536) ──────────────
def test_update_session_model_persists_provider(tmp_path, monkeypatch):
"""update_session_model writes $.model + $.provider into model_config."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes"))
db = SessionDB(db_path=tmp_path / "state.db")
db.create_session(session_id="s1", source="cli", model="m0")
db.update_session_model("s1", "claude-x", provider="custom:feather")
meta = db.get_session("s1")
assert meta["model"] == "claude-x"
config = json.loads(meta["model_config"])
assert config["model"] == "claude-x"
assert config["provider"] == "custom:feather"
def test_update_session_model_without_provider_preserves_existing(tmp_path, monkeypatch):
"""Without provider, existing $.provider is left untouched."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes"))
db = SessionDB(db_path=tmp_path / "state.db")
db.create_session(session_id="s2", source="cli", model="m0")
db.update_session_model("s2", "claude-x", provider="custom:feather")
db.update_session_model("s2", "gpt-5.4") # no provider
meta = db.get_session("s2")
config = json.loads(meta["model_config"])
assert config["model"] == "gpt-5.4"
assert config["provider"] == "custom:feather" # preserved
def test_update_session_model_null_model_config_with_provider(tmp_path, monkeypatch):
"""Provider persistence works when model_config starts as NULL."""
monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes"))
db = SessionDB(db_path=tmp_path / "state.db")
db.create_session(session_id="s3", source="cli", model="m0")
# model_config is NULL at creation — update_session_model must create it
db.update_session_model("s3", "claude-x", provider="minimax")
meta = db.get_session("s3")
config = json.loads(meta["model_config"])
assert config["model"] == "claude-x"
assert config["provider"] == "minimax"
# ── session_gateway_runtime billing_provider fallback (#85721) ─────
def test_session_gateway_runtime_falls_back_to_billing_provider():
"""Sessions that never ran /model have only billing_provider."""
meta = {
"model": "glm-4.7",
"model_config": None,
"billing_provider": "minimax",
}
runtime = SessionDB.session_gateway_runtime(meta)
assert runtime == {"provider": "minimax"}
def test_session_gateway_runtime_billing_provider_bare_bucket_ignored():
"""Bare billing buckets (auto/custom) are not routable — skip them."""
for bare in ("auto", "custom"):
meta = {
"model": "m",
"model_config": None,
"billing_provider": bare,
}
assert SessionDB.session_gateway_runtime(meta) == {}
def test_session_gateway_runtime_explicit_provider_wins_over_billing():
"""Explicit model_config provider takes precedence over billing_provider."""
meta = _row(model_config={"provider": "nous"})
meta["billing_provider"] = "minimax"
runtime = SessionDB.session_gateway_runtime(meta)
assert runtime == {"provider": "nous"}
def test_session_gateway_runtime_billing_provider_fills_top_level_route_without_provider():
"""A TUI row with top-level base_url/api_mode but no provider keeps that endpoint AND takes the
provider the session was billed to — the same merge the TUI gateway always did (#125942); a bare
bucket still contributes nothing."""
meta = _row(model_config={"base_url": "https://f/v1", "api_mode": "chat_completions"})
meta["billing_provider"] = "minimax"
assert SessionDB.session_gateway_runtime(meta) == {
"provider": "minimax", "base_url": "https://f/v1", "api_mode": "chat_completions"}
meta["billing_provider"] = "custom"
assert SessionDB.session_gateway_runtime(meta) == {"base_url": "https://f/v1", "api_mode": "chat_completions"}
def test_restore_session_model_restores_billing_provider_fallback():
"""End-to-end: _restore_session_model uses billing_provider fallback."""
stub = _make_stub()
stub._restore_session_model({
"model": "glm-4.7",
"model_config": None,
"billing_provider": "minimax",
})
assert stub.model == "glm-4.7"
assert stub.provider == "minimax"