hermes-agent/agent/monitoring/gateway_health.py
Teknium a5bd246865
Old pre-decomposition import paths are gone: plugin compat layer removed on schedule (#126164)
* refactor(plugins): remove the Sep 2026 decomposition compat layer on schedule

The PLUGIN-COMPAT layer (2776813df3 + d63e380324 + 0a5164cebe) kept pre-#102117 import paths
alive for external plugins until 2026-09-14. That window closed two weeks ago; since then the loader
has already been skipping plugins that use the old paths. This removes the layer itself:

- 328 appended `PLUGIN-COMPAT` blocks (lazy `__getattr__` pointer tables, re-exported third-party
  names, restored dead definitions) and the three re-export stub modules
  (gateway/startup_watchdog, hermes_cli/observability/relay_runtime, tools/environments/modal_utils)
- COMPAT_MANIFEST.md, compat_manifest.json, scripts/check_compat_pointers.py and its lint step
- the reporting surfaces: CLI banner notice, `hermes plugins compat`, the `hermes doctor` section,
  the post-update notice, the Desktop one-time dialog, the loader's pre-import skip and the
  `plugins.allow_deprecated_imports` escape hatch

An external plugin that still imports an old path now fails to load with its ImportError as the
reason in `hermes plugins list`, the same path as any broken plugin.

hermes_cli/plugin_compat.py stays as three inert stubs (compat_report, removal_in_effect,
summary_lines): an already-running pre-removal `hermes update` lazy-imports them after the checkout
swap (tests/compat/old_updater_surface.json).

In-tree fallout, both already dead: hermes_cli/setup.py::_check_espeak_ng (no callers; its
`shutil` came from a compat block) and gateway/config.py::SessionResetPolicy ("retained solely for
the scheduled plugin-compat window"). Two test_run_agent patches targeted the removed
`run_agent.handle_function_call` pointer; they now patch `model_tools.handle_function_call`, the
seam production reads, like every sibling test in that file.

* chore: retrigger CI (zero-job startup_failure phantom)

* test: drop resolution allowlist rows for the two deleted which() sites

hermes_cli/setup.py::_check_espeak_ng (dead) and tools/skillevaluator_scan.py::scanner_available
(a restored definition inside a PLUGIN-COMPAT block) no longer exist; the stale-row gate requires
their allowlist entries go with them.
2026-09-28 10:21:41 -07:00

331 lines
15 KiB
Python

"""Gateway health and diagnostics signal producer.
Keeps the plane narrow: service health plus redacted operational diagnostics,
derived from the existing gateway runtime-status contract. No prompts,
messages, tool args, session history, audit records, or product analytics.
"""
from __future__ import annotations
import hashlib
import logging
import re
from dataclasses import dataclass
from typing import Any, Callable, Dict, List, Optional
from agent.monitoring import emitter
from agent.monitoring.events import GatewayDiagnosticEvent, GatewayHealthEvent
from agent.monitoring.redaction import redact_bounded
logger = logging.getLogger(__name__)
@dataclass(frozen=True, slots=True)
class GatewayMetric:
name: str
value: int | float
attributes: Dict[str, str]
@dataclass(frozen=True, slots=True)
class GatewayHealthSnapshot:
metrics: List[GatewayMetric]
events: List[GatewayHealthEvent | GatewayDiagnosticEvent]
_RUNNING_PLATFORM_STATES = {"running", "connected", "ok", "ready"}
_FATAL_PLATFORM_STATES = {"fatal", "degraded", "error", "failed"}
_KNOWN_GATEWAY_STATES = _RUNNING_PLATFORM_STATES | _FATAL_PLATFORM_STATES | {"starting", "draining", "stopping", "stopped", "startup_failed", "unknown"}
_KNOWN_PLATFORM_STATES = _RUNNING_PLATFORM_STATES | _FATAL_PLATFORM_STATES | {"connecting", "disconnected", "disabled", "paused", "retrying", "unknown"}
_SUPERVISION_MODES = {"systemd", "s6", "container", "launchd", "manual", "unknown"}
_SOURCE_LOGGER_RE = re.compile(r"^gateway(?:\.[A-Za-z_][A-Za-z0-9_]*)*$")
def source_logger_for_export(name: Any) -> Optional[str]:
"""Return a bounded source-controlled gateway logger name for OTLP scope."""
value = str(name or "")
return value if len(value) <= 128 and _SOURCE_LOGGER_RE.fullmatch(value) else None
def _contains_any(*needles: str) -> Callable[[str], bool]:
return lambda text: any(needle in text for needle in needles)
# Ordered (predicate, class) rules; first match wins, so auth outranks rate-limit etc.
_GATEWAY_ERROR_RULES: tuple[tuple[Callable[[str], bool], str], ...] = (
(_contains_any("auth", "token", "unauthorized", "forbidden", "401", "403"), "auth_failed"),
(lambda s: "rate" in s and "limit" in s, "rate_limited"),
(_contains_any("timeout", "timed out"), "timeout"),
(_contains_any("network", "connection", "dns", "socket", "connect call failed", "failed to connect", "cannot connect",
"unreachable", "name resolution"), "network_error"),
(_contains_any("config", "missing", "invalid"), "invalid_config"),
(_contains_any("startup"), "startup_failed"),
(_contains_any("fatal"), "platform_fatal"),
)
def classify_gateway_error(raw: Any) -> str:
s = str(raw or "").lower()
return next((label for match, label in _GATEWAY_ERROR_RULES if match(s)), "unknown")
def classify_exit_reason(raw: Any, *, state: Any, restart_requested: bool) -> Optional[str]:
"""Reduce free-form shutdown text to a bounded operational class."""
if restart_requested:
return "restart_requested"
state_name = str(state or "").lower()
if raw is None and state_name != "startup_failed":
return None
classified = classify_gateway_error(raw)
if state_name == "startup_failed":
return classified if classified != "unknown" else "startup_failed"
text = str(raw or "").lower()
if "signal" in text or "sigterm" in text or "sigint" in text:
return "signal"
if state_name == "stopped" and any(word in text for word in ("shutdown", "stop")):
return "planned_stop"
return classified
def _bounded_state(raw: Any, *, allowed: set[str]) -> str:
state = str(raw or "unknown").lower()
return state if state in allowed else "unknown"
def _optional_state(raw: Any, *, allowed: set[str]) -> Optional[str]:
"""``_bounded_state`` that preserves "absent" (None) instead of coercing to unknown."""
return None if raw is None else _bounded_state(raw, allowed=allowed)
def _safe_metric_value(raw: Any, *, limit: int = 128) -> str:
return redact_bounded(raw, limit=limit, empty="unknown", unavailable="unknown")
def _safe_instance_id(raw: Any) -> str:
"""Return a stable opaque instance key without exporting the source ID."""
value = str(raw or "unknown").encode("utf-8", errors="replace")
return f"sha256:{hashlib.sha256(value).hexdigest()[:24]}"
def subsystem_for_logger(logger_name: str) -> str:
parts = logger_name.split(".")
if parts[:2] == ["gateway", "relay"]:
return "platform.relay"
if parts[:2] == ["gateway", "platforms"] and len(parts) >= 3 and parts[2]:
return f"platform.{parts[2]}"
return "platform" if logger_name.startswith("gateway.platforms") else "gateway"
def platform_for_subsystem(subsystem: str) -> Optional[str]:
return (subsystem.split(".", 1)[1] or None) if subsystem.startswith("platform.") else None
def _coerce_pid(raw: Any) -> Optional[int]:
try:
pid = int(raw)
except (TypeError, ValueError):
return None
return pid if pid > 0 else None
def _gateway_status(name: str, fallback: Callable[[], Any], /, **kwargs: Any) -> Any:
"""Prefer ``gateway.status.<name>`` (the runtime-status contract); fall back to the local approximation."""
try:
import gateway.status as status
return getattr(status, name)(**kwargs)
except Exception:
return fallback()
def _parse_active_agents(raw: Any) -> int:
def fallback() -> int:
try:
return max(0, int(raw))
except (TypeError, ValueError):
return 0
return _gateway_status("parse_active_agents", fallback, raw=raw)
def _dict_or_empty(raw: Any) -> dict[str, Any]:
return raw if isinstance(raw, dict) else {}
def _platforms_of(runtime: Optional[dict[str, Any]]) -> dict[str, Any]:
return _dict_or_empty((runtime or {}).get("platforms"))
def build_gateway_health_snapshot(
runtime: Optional[dict[str, Any]], *, gateway_running: bool, profile: str, install_id: str, version: str,
supervision_mode: str = "unknown",
) -> GatewayHealthSnapshot:
"""Convert gateway_state.json-compatible runtime state into P0 signals."""
runtime = runtime or {}
gateway_state = _bounded_state(runtime.get("gateway_state"), allowed=_KNOWN_GATEWAY_STATES)
active_agents = _parse_active_agents(runtime.get("active_agents", 0))
running = gateway_running and gateway_state == "running"
busy = _gateway_status(
"derive_gateway_busy", lambda: bool(running and _parse_active_agents(active_agents) > 0),
gateway_running=gateway_running, gateway_state=gateway_state, active_agents=active_agents,
)
drainable = _gateway_status(
"derive_gateway_drainable", lambda: bool(running), gateway_running=gateway_running, gateway_state=gateway_state
)
platforms = _platforms_of(runtime)
mode = str(supervision_mode or "unknown").lower()
base = {
"service.instance.id": _safe_instance_id(install_id),
"service.version": _safe_metric_value(version, limit=64),
"hermes.supervision_mode": mode if mode in _SUPERVISION_MODES else "unknown",
}
def metric(name: str, value: int | float, **extra: str) -> GatewayMetric:
attrs = dict(base)
for key, val in extra.items():
if val is not None:
attrs[key] = _safe_metric_value(val)
return GatewayMetric(name=name, value=value, attributes=attrs)
metrics: list[GatewayMetric] = [
metric("hermes.gateway.up", int(bool(gateway_running))),
metric("hermes.gateway.active_agents", active_agents),
metric("hermes.gateway.busy", int(bool(busy))),
metric("hermes.gateway.drainable", int(bool(drainable))),
metric("hermes.gateway.restart_requested", int(bool(runtime.get("restart_requested")))),
metric("hermes.gateway.state", 1, **{"hermes.gateway.state": gateway_state}),
]
fatal_count = 0
events: list[GatewayHealthEvent | GatewayDiagnosticEvent] = []
# classify_* is idempotent on its own labels, so error_class == error_code downstream.
for platform, pdata in platforms.items():
pdata = _dict_or_empty(pdata)
state = _bounded_state(pdata.get("state"), allowed=_KNOWN_PLATFORM_STATES)
error_code = classify_gateway_error(pdata.get("error_code") or pdata.get("error_message"))
is_degraded = state in _FATAL_PLATFORM_STATES
fatal_count += is_degraded
pattrs = {"hermes.platform": str(platform), "hermes.platform.state": state}
metrics.append(metric("hermes.platform.up", int(state in _RUNNING_PLATFORM_STATES), **pattrs))
metrics.append(metric("hermes.platform.degraded", int(is_degraded), **pattrs, **{"hermes.error_code": error_code}))
if is_degraded:
events.append(GatewayDiagnosticEvent(
name="platform.fatal", subsystem=f"platform.{platform}", platform=str(platform),
error_code=error_code, error_class=error_code, profile=profile, version=version,
severity="error" if state == "fatal" else "warning",
))
events.insert(0, GatewayHealthEvent(
name="gateway.health_snapshot", gateway_state=gateway_state, active_agents=active_agents, gateway_busy=busy,
gateway_drainable=drainable, platform_count=len(platforms), fatal_platform_count=fatal_count, profile=profile,
install_id=install_id, version=version, supervision_mode=supervision_mode, pid=_coerce_pid(runtime.get("pid")),
))
return GatewayHealthSnapshot(metrics=metrics, events=events)
def _safe_profile() -> str:
try:
from hermes_cli.profiles import get_active_profile_name
return str(get_active_profile_name() or "default")
except Exception:
return "default"
def _safe_version() -> str:
try:
from hermes_cli.version_info import get_version_info
return get_version_info().base_version
except Exception:
return "unknown"
def _lifecycle_events(
previous: Optional[dict[str, Any]], current: dict[str, Any], *, profile: str, version: str
) -> list[GatewayHealthEvent | GatewayDiagnosticEvent]:
"""Gateway-level transition events: lifecycle, startup_failed diagnostic, exit."""
old_state = _optional_state((previous or {}).get("gateway_state"), allowed=_KNOWN_GATEWAY_STATES)
new_state = _optional_state(current.get("gateway_state"), allowed=_KNOWN_GATEWAY_STATES)
if old_state == new_state or not new_state:
return []
restart_requested = bool(current.get("restart_requested"))
def health(name: str) -> GatewayHealthEvent:
return GatewayHealthEvent(
name=name, gateway_state=new_state, old_state=old_state, new_state=new_state,
exit_reason=classify_exit_reason(current.get("exit_reason"), state=new_state, restart_requested=restart_requested),
restart_requested=restart_requested, active_agents=_parse_active_agents(current.get("active_agents", 0)),
profile=profile, version=version, pid=_coerce_pid(current.get("pid")),
)
out: list[GatewayHealthEvent | GatewayDiagnosticEvent] = [health("gateway.lifecycle")]
if new_state == "startup_failed":
error_class = classify_gateway_error(current.get("exit_reason") or "startup_failed")
out.append(GatewayDiagnosticEvent(
name="gateway.startup_failed", subsystem="gateway", error_class=error_class, error_code=error_class,
profile=profile, version=version, severity="error",
))
if new_state == "stopped":
out.append(health("gateway.exit"))
return out
def _platform_events(
previous: Optional[dict[str, Any]], current: dict[str, Any], *, profile: str, version: str
) -> list[GatewayDiagnosticEvent]:
"""Per-platform state_change diagnostics, plus platform.fatal when the new state is fatal."""
old_platforms = _platforms_of(previous)
out: list[GatewayDiagnosticEvent] = []
for platform, pdata in _platforms_of(current).items():
pdata = _dict_or_empty(pdata)
prev = _dict_or_empty(old_platforms.get(platform, {}))
old_state = _optional_state(prev.get("state"), allowed=_KNOWN_PLATFORM_STATES)
new_state = _optional_state(pdata.get("state"), allowed=_KNOWN_PLATFORM_STATES)
if old_state == new_state or not new_state:
continue
error_code = classify_gateway_error(pdata.get("error_code") or pdata.get("error_message"))
common: dict[str, Any] = dict(
subsystem=f"platform.{platform}", platform=str(platform), error_code=error_code, error_class=error_code,
profile=profile, version=version, severity="error" if new_state in {"fatal", "failed", "error"} else "warning",
)
out.append(GatewayDiagnosticEvent(name="platform.state_change", old_state=old_state, new_state=new_state, **common))
if new_state in _FATAL_PLATFORM_STATES:
out.append(GatewayDiagnosticEvent(name="platform.fatal", **common))
return out
def emit_runtime_status_transition(previous: Optional[dict[str, Any]], current: dict[str, Any]) -> None:
"""Emit immediate content-free gateway events for runtime status changes. Called by
gateway.status.write_runtime_status after persisting; fully fail-open."""
try:
ctx = dict(profile=_safe_profile(), version=_safe_version())
for ev in _lifecycle_events(previous, current, **ctx) + _platform_events(previous, current, **ctx):
emitter.emit(ev)
except Exception:
logger.debug("gateway runtime status transition emit failed", exc_info=True)
class GatewayDiagnosticLogHandler(logging.Handler):
"""Allowlisted warning/error bridge for gateway-owned diagnostics."""
def __init__(self, *, profile: str = "default", version: str = "unknown") -> None:
super().__init__(level=logging.WARNING)
self.profile = profile
self.version = version
def emit(self, record: logging.LogRecord) -> None:
try:
if record.levelno < logging.WARNING or not (record.name == "gateway" or record.name.startswith("gateway.")):
return
subsystem = subsystem_for_logger(record.name)
error_class = classify_gateway_error(record.getMessage())
emitter.get_emitter().emit(GatewayDiagnosticEvent(
name=f"gateway.log.{record.levelname.lower()}", subsystem=subsystem,
source_logger=source_logger_for_export(record.name), platform=platform_for_subsystem(subsystem),
error_class=error_class, error_code=error_class, profile=self.profile, version=self.version,
severity=record.levelname.lower(),
))
except Exception:
logger.debug("gateway diagnostic emit failed", exc_info=True)
__all__ = [
"GatewayMetric", "GatewayHealthSnapshot", "GatewayDiagnosticLogHandler",
"build_gateway_health_snapshot", "classify_gateway_error", "source_logger_for_export",
]