hermes-agent/tests/hermes_cli/test_shared_metrics_gateway.py
teknium1 49aeaa1526 fix(telemetry): name every gateway adapter platform from the Platform enum
hermes.platform.health / delivery / reply_latency built their core platform
vocabulary from hermes_cli.platforms.PLATFORMS (the setup wizard list), which
has no relay, msgraph_webhook or local entry, so those first-party adapters
were reported as the anonymous "plugin" bucket. The adapter vocabulary now
comes from gateway.config.Platform's declared members (loaded on first use:
gateway.config is too heavy for the contract's import); plugin pseudo-members
are not declared members, so unknown plugin platforms still collapse to
"plugin" and catalog/bundled plugin names keep working. The v3 task platform
vocabulary (GATEWAY_PLATFORMS) is unchanged.

Invariant test: every declared Platform member is named as itself, is
contract-valid for the three adapter metrics and validates against the v3
JSON schema counter definitions (red at d2f70bfaa0: relay/local -> plugin).
2026-09-28 12:43:03 -07:00

117 lines
5.7 KiB
Python

"""Invariants for gateway / cron shared metrics: bounded platform names with proven catalog provenance,
opt-in gating, and contract-valid rows."""
import json
import textwrap
import pytest
from hermes_cli.observability import relay_shared_metrics as rsm
from hermes_cli.observability import shared_metrics_catalog as catalog
from hermes_cli.observability import shared_metrics_contract as contract
from hermes_cli.observability import shared_metrics_gateway as smg
@pytest.fixture
def marks(monkeypatch):
got = []
monkeypatch.setattr(rsm, "enabled", lambda: True)
monkeypatch.setattr(rsm, "record_process_mark", lambda mark, data: got.append((mark, dict(data))))
def read():
smg.drain()
return [(contract._DECISION_MARK_METRICS[mark], data) for mark, data in got]
return read
def _install_platform_plugin(home, monkeypatch, *, dir_name, platform, catalog_name=None):
"""A user plugin under $HERMES_HOME/plugins registering ``platform``; ``catalog_name`` writes the
installer-owned record a plugin-catalog install leaves (a URL install writes none)."""
from gateway.platform_registry import PlatformEntry, platform_registry
plugin_dir = home / "plugins" / dir_name
plugin_dir.mkdir(parents=True)
(plugin_dir / "adapter.py").write_text(textwrap.dedent("""
def make_adapter(config):
return None
"""), encoding="utf-8")
# Anything inside the tree is the repo's to write: it must never prove a catalog install.
(plugin_dir / ".hermes-catalog.json").write_text(json.dumps({"catalog_name": "sendblue"}), encoding="utf-8")
namespace = {}
exec(compile((plugin_dir / "adapter.py").read_text(), str(plugin_dir / "adapter.py"), "exec"), namespace)
if catalog_name:
(home / "plugins" / ".install-metadata.json").write_text(json.dumps({
dir_name: {"source": "https://example.invalid/repo", "catalog": {"name": catalog_name, "tier": "official"}},
}), encoding="utf-8")
platform_registry.register(PlatformEntry(
name=platform, label=platform, adapter_factory=namespace["make_adapter"], check_fn=lambda: True,
source="plugin", plugin_name="sendblue"))
monkeypatch.setattr(catalog, "catalog_platform_names", lambda: frozenset({"sendblue", "vk-platform"}))
catalog._catalog_platform_owner.cache_clear()
return lambda: platform_registry.unregister(platform)
def test_platform_names_are_published_or_proven_by_the_installer_record(tmp_path, monkeypatch):
home = tmp_path / ".hermes"
monkeypatch.setenv("HERMES_HOME", str(home))
cleanup = [
_install_platform_plugin(home, monkeypatch, dir_name="vk", platform="vk", catalog_name="vk-platform"),
# A URL install claiming a catalog name everywhere it can (tree sidecar, manifest name) stays anonymous.
_install_platform_plugin(tmp_path / "other", monkeypatch, dir_name="x", platform="sneaky"),
]
try:
assert contract.adapter_platform("telegram") == "telegram"
assert contract.adapter_platform("irc") == "irc" # bundled plugins/platforms/irc
assert contract.adapter_platform("vk") == "vk-platform"
assert contract.adapter_platform("sneaky") == "plugin"
assert contract.adapter_platform("my-private-bridge") == "plugin"
assert contract.gateway_platform({"platform": "vk", "surface": "gateway"}, "gateway") == "vk-platform"
assert "vk-platform" in contract.GATEWAY_PLATFORMS
finally:
for undo in cleanup:
undo()
catalog._catalog_platform_owner.cache_clear()
def test_every_gateway_row_is_contract_valid_and_disabled_config_records_nothing(marks, monkeypatch):
class Adapter:
platform = "telegram"
fatal_error_code = "telegram_auth_error"
smg.record_platform_connect(Adapter(), "telegram", is_reconnect=False, ok=False, exc=TimeoutError("x"))
smg.record_platform_disconnect(Adapter())
smg.record_cron_missed({"deliver": "telegram:-100123"})
rows = marks()
assert rows == [
("hermes.platform.health", {"error_class": "network", "event": "connect_failed", "platform": "telegram"}),
("hermes.platform.health", {"error_class": "auth", "event": "disconnect", "platform": "telegram"}),
("hermes.cron.run", {"delivery_kind": "platform", "duration_bucket": "lt_1s", "outcome": "missed"}),
]
assert all(contract.counter_dimensions_are_valid(name, dims) for name, dims in rows)
monkeypatch.setattr(rsm, "enabled", lambda: False)
smg.record_platform_disconnect(Adapter())
assert len(marks()) == 3
def test_every_gateway_adapter_platform_is_named_and_accepted_by_contract_and_schema():
jsonschema = pytest.importorskip("jsonschema")
from pathlib import Path
from gateway.config import Platform
schema = json.loads((Path(contract.__file__).parent / "schemas/hermes.shared_metrics.v3.schema.json").read_text())
rows = {
"platform_health_counter": ("hermes.platform.health", {"error_class": "network", "event": "disconnect"}),
"platform_delivery_counter": ("hermes.platform.delivery", {"failure_class": "none", "outcome": "sent"}),
"reply_latency_counter": ("hermes.gateway.reply_latency", {"first_response_bucket": "lt_2s"}),
}
for member in Platform: # relay and msgraph_webhook are gateway adapters with no setup-wizard entry
name = contract.adapter_platform(member)
assert name == member.value
for definition, (metric, dims) in rows.items():
dims = {**dims, "platform": name}
assert contract.counter_dimensions_are_valid(metric, dims)
jsonschema.validate({"name": metric, "type": "counter", "dimensions": dims, "value": 1},
{"$defs": schema["$defs"], "$ref": f"#/$defs/{definition}"})