mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-16 19:36:54 +00:00
327 lines
15 KiB
Python
327 lines
15 KiB
Python
"""Launch windows are sized beside what other programs hold on the card, not from capacity alone.
|
|
|
|
On a 32 GiB card, the capacity plan booted Qwen3.8 27B at 216K. That fit only while other
|
|
programs held about 3.4 GiB or less. With 4.5-6.3 GiB held, Windows paged part of the model to
|
|
host memory without an error, and decode fell from ~90 to ~24 tok/s. The window must shrink to
|
|
what fits now, and the weights must stay on the GPU.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import replace
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
from hermes_cli.local_runtime import hardware
|
|
from hermes_cli.local_runtime.estimator import HardwareBudget
|
|
|
|
GIB = 1 << 30
|
|
# The measured card: RTX 5090, 31.84 GiB total, capacity = total minus the 9% margin.
|
|
CARD_TOTAL = int(31.84 * GIB)
|
|
CARD_CAPACITY = HardwareBudget(
|
|
usable_vram_bytes=CARD_TOTAL - max(hardware._MARGIN_FLOOR, int(CARD_TOTAL * hardware._MARGIN_FRACTION)),
|
|
total_device_bytes=CARD_TOTAL, ram_available_bytes=256 * GIB, uma=False)
|
|
MODEL_ID = "Qwen3.8-27B-UD-Q4_K_M"
|
|
|
|
|
|
@pytest.fixture
|
|
def hermes_home(tmp_path, monkeypatch):
|
|
monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes"))
|
|
return tmp_path
|
|
|
|
|
|
def _card(monkeypatch, others_gib: float, *, total: int = CARD_TOTAL):
|
|
monkeypatch.setattr(hardware, "_nvidia_vram",
|
|
lambda: (total, total - int(others_gib * GIB), "NVIDIA GeForce RTX 5090", None))
|
|
|
|
|
|
@pytest.fixture
|
|
def capacity_probe(monkeypatch):
|
|
"""Launch decisions run through a server bounce, so they price the card's capacity
|
|
(``planning=True``), never raw free memory, which still counts the outgoing server. Free
|
|
memory enters only through ``launch_budget``. Every call is recorded, so a probe inside a
|
|
path that swallows exceptions still fails the test."""
|
|
calls = []
|
|
|
|
def probe(**kwargs):
|
|
calls.append(kwargs)
|
|
return CARD_CAPACITY
|
|
|
|
monkeypatch.setattr(hardware, "probe_budget", probe)
|
|
yield calls
|
|
assert all(call == {"planning": True} for call in calls), calls
|
|
|
|
|
|
# ── launch_budget ────────────────────────────────────────────
|
|
|
|
|
|
def test_launch_budget_subtracts_other_programs_plus_headroom(monkeypatch):
|
|
_card(monkeypatch, 5.4)
|
|
budget = hardware.launch_budget(CARD_CAPACITY)
|
|
assert budget.usable_vram_bytes == CARD_TOTAL - int(5.4 * GIB) - hardware._LAUNCH_HEADROOM
|
|
assert budget.total_device_bytes == CARD_TOTAL and budget.uma is False
|
|
|
|
|
|
def test_launch_budget_never_exceeds_capacity(monkeypatch):
|
|
_card(monkeypatch, 0.0)
|
|
assert hardware.launch_budget(CARD_CAPACITY).usable_vram_bytes == CARD_CAPACITY.usable_vram_bytes
|
|
|
|
|
|
def test_launch_budget_counts_own_server_as_free(monkeypatch):
|
|
"""The managed server exits before the next instance loads; counting it as another program
|
|
once pinned a fitting model's weights to the CPU."""
|
|
_card(monkeypatch, 5.0 + 24.6) # 5 GiB of other programs plus our own 24.6 GiB model
|
|
budget = hardware.launch_budget(CARD_CAPACITY, own_bytes=int(24.6 * GIB))
|
|
assert budget.usable_vram_bytes == pytest.approx(
|
|
CARD_TOTAL - 5 * GIB - hardware._LAUNCH_HEADROOM, abs=1 << 20)
|
|
|
|
|
|
def test_launch_budget_skips_unified_memory_and_failed_probes(monkeypatch):
|
|
uma = replace(CARD_CAPACITY, uma=True)
|
|
monkeypatch.setattr(hardware, "_nvidia_vram", lambda: pytest.fail("UMA must not probe"))
|
|
assert hardware.launch_budget(uma) is None
|
|
monkeypatch.setattr(hardware, "_nvidia_vram", lambda: None)
|
|
assert hardware.launch_budget(CARD_CAPACITY) is None
|
|
|
|
|
|
# ── launch windows for the real catalog 27B ─────────────────
|
|
|
|
|
|
def _stage_27b(hermes_home, monkeypatch):
|
|
"""The real catalog entry (mmproj staged), priced from its catalog profile."""
|
|
from hermes_cli.local_runtime import bootstrap, catalog, presets
|
|
|
|
entry = catalog.catalog_by_id()["qwen3.8-27b"]
|
|
variant = next(v for v in entry.variants if v.model_id == MODEL_ID)
|
|
mdir = hermes_home / "models"
|
|
mdir.mkdir(parents=True, exist_ok=True)
|
|
# First: assets_dir() resolves under models_dir(), so the projector lands in this test's tree.
|
|
monkeypatch.setattr(bootstrap, "models_dir", lambda: mdir)
|
|
gguf = mdir / f"{MODEL_ID}.gguf"
|
|
gguf.write_bytes(b"GGUF" + b"\x00" * 64)
|
|
asset = bootstrap.assets_dir() / entry.mmproj.local_name
|
|
asset.parent.mkdir(parents=True, exist_ok=True)
|
|
asset.touch()
|
|
profile = entry.profile(variant)
|
|
monkeypatch.setattr(presets, "read_gguf_header", lambda p: SimpleNamespace(sampling_defaults={}))
|
|
monkeypatch.setattr(presets, "profile_from_gguf", lambda h: profile)
|
|
monkeypatch.setattr(bootstrap, "staged_models", lambda: [gguf])
|
|
monkeypatch.setattr(bootstrap, "staged_in", lambda d: [gguf])
|
|
return mdir, gguf
|
|
|
|
|
|
@pytest.mark.parametrize("others_gib,window_k", [
|
|
(0.0, 216), # quiet card: capacity plan unchanged
|
|
(3.0, 216),
|
|
(3.6, 144), # measured: 216K ran with 0.7 GiB spare here — too close to keep
|
|
(4.5, 144), # measured: 216K demoted to ~24 tok/s here
|
|
(5.4, 144),
|
|
(6.5, 96), # measured smoke run held 6.3 GiB at peak
|
|
(8.5, 64),
|
|
])
|
|
def test_27b_window_follows_other_programs(hermes_home, monkeypatch, others_gib, window_k):
|
|
from hermes_cli.local_runtime import presets
|
|
|
|
_mdir, gguf = _stage_27b(hermes_home, monkeypatch)
|
|
_card(monkeypatch, others_gib)
|
|
entry = presets.preset_for_model(gguf, CARD_CAPACITY, set(), requested_window=0,
|
|
live=hardware.launch_budget(CARD_CAPACITY))
|
|
assert entry.window == window_k * 1024
|
|
assert entry.spilled is False and "override-tensor" not in entry.keys
|
|
assert entry.keys["ctx-size"] == str(window_k * 1024)
|
|
assert entry.keys["mmproj"].endswith("mmproj-Qwen3.8-27B-BF16.gguf")
|
|
|
|
|
|
def test_busy_card_holds_the_floor_and_keeps_weights_on_the_gpu(hermes_home, monkeypatch):
|
|
"""A live reading may still include memory about to be freed; it never moves weights."""
|
|
from hermes_cli.local_runtime import presets
|
|
|
|
_mdir, gguf = _stage_27b(hermes_home, monkeypatch)
|
|
_card(monkeypatch, 14.0)
|
|
entry = presets.preset_for_model(gguf, CARD_CAPACITY, set(), requested_window=0,
|
|
live=hardware.launch_budget(CARD_CAPACITY))
|
|
assert entry.window == 64 * 1024
|
|
assert entry.spilled is False and "override-tensor" not in entry.keys
|
|
|
|
|
|
def test_a_capacity_plan_that_spills_is_left_alone(hermes_home, monkeypatch):
|
|
"""Small cards spill by design; a live reading must not rearrange that placement."""
|
|
from hermes_cli.local_runtime import presets
|
|
|
|
_mdir, gguf = _stage_27b(hermes_home, monkeypatch)
|
|
small = HardwareBudget(usable_vram_bytes=10 * GIB, total_device_bytes=12 * GIB,
|
|
ram_available_bytes=64 * GIB)
|
|
_card(monkeypatch, 3.0, total=12 * GIB)
|
|
capacity_only = presets.preset_for_model(gguf, small, set(), requested_window=0)
|
|
with_live = presets.preset_for_model(gguf, small, set(), requested_window=0,
|
|
live=hardware.launch_budget(small))
|
|
assert capacity_only.spilled and with_live == capacity_only
|
|
|
|
|
|
def test_recommendations_still_price_against_capacity(hermes_home, monkeypatch):
|
|
"""The catalog and picker describe the card, not this minute's desktop."""
|
|
from hermes_cli.local_runtime import catalog
|
|
|
|
_card(monkeypatch, 8.5)
|
|
entry = catalog.catalog_by_id()["qwen3.8-27b"]
|
|
variant = next(v for v in entry.variants if v.model_id == MODEL_ID)
|
|
assert entry.launch_plan(variant, CARD_CAPACITY).decision.window == 216 * 1024
|
|
|
|
|
|
# ── boot wiring ──────────────────────────────────────────────
|
|
|
|
|
|
def test_boot_writes_the_launch_window(hermes_home, monkeypatch, capacity_probe):
|
|
from hermes_cli.local_runtime import bootstrap, presets
|
|
|
|
mdir, _gguf = _stage_27b(hermes_home, monkeypatch)
|
|
_card(monkeypatch, 5.4)
|
|
path = hermes_home / "presets.ini"
|
|
assert bootstrap._generate_presets(mdir, path) == path
|
|
assert presets.read_preset_decisions(path)[MODEL_ID].window == 144 * 1024
|
|
|
|
|
|
def test_boot_keeps_capacity_when_the_probe_fails(hermes_home, monkeypatch, capacity_probe):
|
|
"""The probe is policy, not a prerequisite: a failure must never block a boot."""
|
|
from hermes_cli.local_runtime import bootstrap, presets
|
|
|
|
mdir, _gguf = _stage_27b(hermes_home, monkeypatch)
|
|
|
|
def boom():
|
|
raise OSError("nvidia-smi vanished")
|
|
|
|
monkeypatch.setattr(hardware, "_nvidia_vram", boom)
|
|
path = hermes_home / "presets.ini"
|
|
assert bootstrap._generate_presets(mdir, path) == path
|
|
assert presets.read_preset_decisions(path)[MODEL_ID].window == 216 * 1024
|
|
|
|
|
|
# ── re-planning while nothing is loaded ──────────────────────
|
|
|
|
|
|
class _Router:
|
|
def __init__(self, preset_path, statuses):
|
|
self.preset_path = preset_path
|
|
self.statuses = statuses
|
|
self.reloads = 0
|
|
self.proc = SimpleNamespace(poll=lambda: None)
|
|
self._refit_usable = None
|
|
import threading
|
|
|
|
self._lifecycle_lock = threading.RLock()
|
|
|
|
def models(self, timeout_s=30):
|
|
return dict(self.statuses)
|
|
|
|
def reload_presets(self):
|
|
self.reloads += 1
|
|
|
|
|
|
def _booted(hermes_home, monkeypatch, others_gib):
|
|
from hermes_cli.local_runtime import bootstrap
|
|
|
|
mdir, _gguf = _stage_27b(hermes_home, monkeypatch)
|
|
_card(monkeypatch, others_gib)
|
|
path = hermes_home / "presets.ini"
|
|
bootstrap._generate_presets(mdir, path)
|
|
return path
|
|
|
|
|
|
def test_idle_router_gets_windows_for_the_current_desktop(hermes_home, monkeypatch, capacity_probe):
|
|
from hermes_cli.local_runtime import bootstrap, presets
|
|
|
|
path = _booted(hermes_home, monkeypatch, 6.5)
|
|
assert presets.read_preset_decisions(path)[MODEL_ID].window == 96 * 1024
|
|
router = _Router(path, {MODEL_ID: "unloaded"})
|
|
_card(monkeypatch, 2.0) # the user closed the heavy apps
|
|
assert bootstrap.refit_idle_presets(router) is True
|
|
assert router.reloads == 1
|
|
assert presets.read_preset_decisions(path)[MODEL_ID].window == 216 * 1024
|
|
# Nothing changed since: no rewrite, no reload.
|
|
assert bootstrap.refit_idle_presets(router) is False and router.reloads == 1
|
|
|
|
|
|
@pytest.mark.parametrize("status", ["loaded", "loading", "sleeping"])
|
|
def test_never_rewrites_while_a_model_holds_memory(hermes_home, monkeypatch, capacity_probe, status):
|
|
"""The router unloads a loaded model whose launch flags change on reload."""
|
|
from hermes_cli.local_runtime import bootstrap
|
|
|
|
path = _booted(hermes_home, monkeypatch, 6.5)
|
|
before = path.read_text(encoding="utf-8-sig")
|
|
router = _Router(path, {MODEL_ID: status})
|
|
_card(monkeypatch, 2.0)
|
|
assert bootstrap.refit_idle_presets(router) is False
|
|
assert router.reloads == 0 and path.read_text(encoding="utf-8-sig") == before
|
|
|
|
|
|
def test_small_changes_in_free_memory_do_not_replan(hermes_home, monkeypatch, capacity_probe):
|
|
from hermes_cli.local_runtime import bootstrap, presets
|
|
|
|
path = _booted(hermes_home, monkeypatch, 5.4)
|
|
router = _Router(path, {MODEL_ID: "unloaded"})
|
|
assert bootstrap.refit_idle_presets(router) is False # same reading as boot: same file
|
|
monkeypatch.setattr(presets, "plan_presets", lambda *a, **k: pytest.fail("re-planned"))
|
|
_card(monkeypatch, 5.5)
|
|
assert bootstrap.refit_idle_presets(router) is False
|
|
|
|
|
|
# ── growth ───────────────────────────────────────────────────
|
|
|
|
|
|
def test_growth_counts_the_growing_model_as_free(hermes_home, monkeypatch, capacity_probe):
|
|
"""The grown instance loads after the current one exits. Counting the model's own memory as
|
|
held would veto every rung."""
|
|
from hermes_cli.local_runtime import bootstrap, growth, presets
|
|
|
|
mdir, gguf = _stage_27b(hermes_home, monkeypatch)
|
|
monkeypatch.setattr(bootstrap, "get_supervisor", lambda: SimpleNamespace(is_idle=lambda m: True))
|
|
monkeypatch.setattr(growth, "is_managed_endpoint", lambda url: True)
|
|
from hermes_cli.local_runtime import estimator, gguf as gguf_mod
|
|
|
|
monkeypatch.setattr(gguf_mod, "read_gguf_header", presets.read_gguf_header)
|
|
monkeypatch.setattr(estimator, "profile_from_gguf", presets.profile_from_gguf)
|
|
own = presets.resident_footprint(gguf, CARD_CAPACITY, 144 * 1024)
|
|
_card(monkeypatch, 2.0 + own / GIB) # 2 GiB of other programs plus our loaded 144K model
|
|
path = hermes_home / "presets.ini"
|
|
|
|
def refresh():
|
|
presets.generate_presets(mdir, CARD_CAPACITY, path)
|
|
return True
|
|
|
|
monkeypatch.setattr(bootstrap, "refresh_local_runtime", refresh)
|
|
real_read = presets.read_preset_decisions
|
|
monkeypatch.setattr(presets, "read_preset_decisions", lambda p=None: real_read(path))
|
|
grown = growth.maybe_grow_window(MODEL_ID, base_url="http://127.0.0.1:1/v1",
|
|
session_tokens=140 * 1024, current_window=144 * 1024)
|
|
assert grown == 216 * 1024
|
|
|
|
|
|
def test_growth_refuses_a_rung_other_programs_leave_no_room_for(hermes_home, monkeypatch, capacity_probe):
|
|
from hermes_cli.local_runtime import bootstrap, growth, presets
|
|
|
|
_mdir, gguf = _stage_27b(hermes_home, monkeypatch)
|
|
monkeypatch.setattr(bootstrap, "get_supervisor", lambda: SimpleNamespace(is_idle=lambda m: True))
|
|
monkeypatch.setattr(growth, "is_managed_endpoint", lambda url: True)
|
|
from hermes_cli.local_runtime import estimator, gguf as gguf_mod
|
|
|
|
monkeypatch.setattr(gguf_mod, "read_gguf_header", presets.read_gguf_header)
|
|
monkeypatch.setattr(estimator, "profile_from_gguf", presets.profile_from_gguf)
|
|
own = presets.resident_footprint(gguf, CARD_CAPACITY, 144 * 1024)
|
|
_card(monkeypatch, 5.4 + own / GIB)
|
|
monkeypatch.setattr(bootstrap, "refresh_local_runtime", lambda: pytest.fail("bounced"))
|
|
assert growth.maybe_grow_window(MODEL_ID, base_url="http://127.0.0.1:1/v1",
|
|
session_tokens=140 * 1024, current_window=144 * 1024) is None
|
|
assert not growth.load_window_overrides()
|
|
|
|
|
|
# ── budget source ────────────────────────────────────────────
|
|
|
|
|
|
def test_residency_cap_prices_capacity(hermes_home, monkeypatch, capacity_probe):
|
|
"""The residency cap is also decided before a server bounce. It swallows probe errors to keep
|
|
boots unblocked, so the recorded call is the proof it priced capacity."""
|
|
from hermes_cli.local_runtime import bootstrap
|
|
|
|
mdir, _gguf = _stage_27b(hermes_home, monkeypatch)
|
|
bootstrap._admitted_models_max(mdir, 4)
|
|
assert capacity_probe == [{"planning": True}]
|