mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-16 19:36:54 +00:00
- `curator` latched on the default-on scheduled pass (outcome=success, all
buckets 0), so it meant "idled through one curator interval". It now needs
trigger=manual (the user ran it).
- `skills_created` and the v4 milestone `first_skill_created` latched when
the background-review fork created a skill (provenance=agent_created).
Both now require provenance != agent_created (foreground creates are
stamped "learn" -> not agent_created, see tools/skill_usage.record_created).
- days_since_install_bucket caught only sqlite3.Error; a non-numeric
sessions.started_at raised ValueError into the subscriber on every event,
re-opened state.db each time and never latched. It now also catches
OSError/TypeError/ValueError -> `unknown`.
Probes:
probe_internal_adoption.py
before: feature_adoption curator + skills_created, milestone first_skill_created
after: no feature_adoption rows, no milestone
probe_statedb_errors.py textstamp
before: ValueError warnings, 0 rows, settled 0 of 1
after: 1 state.db read, feature_adoption {unknown, projects}, settled 1 of 1
Tests test_hermes_internal_work_never_latches_adoption and
test_unreadable_first_session_reads_unknown are RED on base.
148 lines
6.7 KiB
Python
148 lines
6.7 KiB
Python
"""v5 signals: which disabled built-in tools models reach for, and when an install first adopts a feature.
|
|
|
|
``hermes.tool_unavailable.count`` is counted where the agent validates the calls a model emitted: a
|
|
name Hermes ships (``toolsets.BUILTIN_TOOL_NAMES``) that this session did not enable. Any other name
|
|
stays the v4 ``unknown_tool`` quality issue only, so a plugin, MCP or hallucinated name never leaves.
|
|
|
|
``hermes.feature_adoption.count`` rides on counters the subscriber already records: the first counter
|
|
row that proves a feature was really used latches it once per install, with the owning profile's age.
|
|
Features no counter can see report through :func:`record_feature_used`.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import sqlite3
|
|
import time
|
|
from pathlib import Path
|
|
from typing import Any, Callable
|
|
|
|
from . import shared_metrics_contract as contract
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_DAY_S = 86_400
|
|
_DAYS_SINCE_INSTALL_THRESHOLDS = (
|
|
(_DAY_S, "same_day"), (7 * _DAY_S, "1d_to_7d"), (30 * _DAY_S, "7d_to_30d"), (90 * _DAY_S, "30d_to_90d"),
|
|
)
|
|
|
|
|
|
# ---- tool unavailable ------------------------------------------------------------------------
|
|
|
|
def tool_unavailable_fields(agent: Any, name: Any, issue: str, route: dict[str, str]) -> dict[str, str] | None:
|
|
"""Fields for a call to a shipped built-in the session has disabled; None for anything else.
|
|
|
|
Background reviews, delegated children and cron jobs run with toolsets Hermes (or the parent
|
|
model) narrowed on purpose, so their misses say nothing about what users should get by default.
|
|
A built-in deferred behind ``tool_search`` is enabled (reachable through ``tool_call``), not missing.
|
|
"""
|
|
if issue != "unknown_tool" or not isinstance(name, str) or name not in contract.BUILTIN_TOOL_NAMES:
|
|
return None
|
|
if getattr(agent, "_delegate_depth", 0) or getattr(agent, "platform", None) == "cron":
|
|
return None
|
|
from agent.tool_executor import _tool_search_scoped_names
|
|
from tools.skill_provenance import is_background_review
|
|
|
|
if is_background_review() or name in _tool_search_scoped_names(agent):
|
|
return None
|
|
return {**route, "tool_name": name}
|
|
|
|
|
|
# ---- feature adoption ------------------------------------------------------------------------
|
|
|
|
def _success(d: dict[str, str]) -> bool:
|
|
return d.get("outcome") == "success"
|
|
|
|
|
|
def _tool(*names: str, prefix: str = "") -> Callable[[dict[str, str]], bool]:
|
|
def used(d: dict[str, str]) -> bool:
|
|
tool = d.get("tool_name") or ""
|
|
return _success(d) and (tool in names or bool(prefix) and tool.startswith(prefix))
|
|
|
|
return used
|
|
|
|
|
|
def user_created_skill(d: dict[str, str]) -> bool:
|
|
"""A skill created at the user's request (``agent_created`` is Hermes' own background review)."""
|
|
return d.get("action") == "created" and d.get("provenance") != "agent_created"
|
|
|
|
|
|
_TASK_SURFACE = {"desktop": "desktop", "tui": "tui"}
|
|
# metric -> [(feature, predicate over the recorded dimensions)]. Unattended Hermes-owned work only
|
|
# counts where the user set it up (a cron job, a curator run the user started).
|
|
_FEATURE_RULES: dict[str, tuple[tuple[str, Callable[[dict[str, str]], bool]], ...]] = {
|
|
contract.MEMORY_OP_METRIC: (("memory", lambda d: _success(d) and d.get("origin") == "foreground"),),
|
|
contract.SKILL_LIFECYCLE_METRIC: (("skills_created", user_created_skill),),
|
|
contract.DELEGATION_RUN_METRIC: (("delegation", lambda d: True),),
|
|
contract.CRON_RUN_METRIC: (("cron", lambda d: d.get("outcome") in {"success", "failed"}),),
|
|
contract.CURATOR_RUN_METRIC: (("curator", lambda d: _success(d) and d.get("trigger") == "manual"),),
|
|
contract.TASK_STARTED_METRIC: (
|
|
("gateway_platform", lambda d: d.get("platform") not in {None, "none"}),
|
|
("desktop", lambda d: d.get("execution_surface") == "desktop"),
|
|
("tui", lambda d: d.get("execution_surface") == "tui"),
|
|
),
|
|
contract.TOOL_USAGE_METRIC: (
|
|
("mcp", _tool("mcp")),
|
|
("plugins", _tool("plugin")),
|
|
("browser", _tool(prefix="browser_")),
|
|
("voice", _tool("text_to_speech")),
|
|
("kanban", _tool(prefix="kanban_")),
|
|
("projects", _tool("desktop_project")),
|
|
),
|
|
contract.SLASH_COMMAND_METRIC: (
|
|
("voice", lambda d: d.get("command") == "voice"),
|
|
("kanban", lambda d: d.get("command") == "kanban"),
|
|
),
|
|
contract.FEATURE_USED_MARK: tuple((feature, lambda d, f=feature: d.get("feature") == f) for feature in sorted(contract.FEATURES)),
|
|
}
|
|
|
|
|
|
def features_for(metric_name: str, dimensions: dict[str, str]) -> tuple[str, ...]:
|
|
"""Features a recorded counter proves were used (each is latched once per install by the store)."""
|
|
return tuple(name for name, used in _FEATURE_RULES.get(metric_name, ()) if used(dimensions))
|
|
|
|
|
|
def days_since_install_bucket(home: Path) -> str:
|
|
"""The owning profile's age (its first session), bucketed; ``unknown`` when state.db is unreadable."""
|
|
from .shared_metrics_snapshot import _first_session_started_at
|
|
|
|
try:
|
|
first = _first_session_started_at(home)
|
|
except (sqlite3.Error, OSError, TypeError, ValueError): # unreadable, or a non-numeric started_at
|
|
return "unknown"
|
|
age = contract._non_negative_number(time.time() - first) if first is not None else None
|
|
return "unknown" if age is None else contract._bucket(age, _DAYS_SINCE_INSTALL_THRESHOLDS, "gte_90d")
|
|
|
|
|
|
def feature_used_counter(event: Any) -> tuple[str, dict[str, str]] | None:
|
|
"""A first-use fact for a feature no counter sees (never stored as a counter row itself)."""
|
|
if not contract._valid_shape(event, **contract._MARK_SHAPE):
|
|
return None
|
|
if contract._event_text(event, "name") != contract.FEATURE_USED_MARK:
|
|
return None
|
|
data = getattr(event, "data", None)
|
|
if not isinstance(data, dict) or set(data) != {"feature"} or data["feature"] not in contract.FEATURES:
|
|
return None
|
|
return contract.FEATURE_USED_MARK, {"feature": data["feature"]}
|
|
|
|
|
|
def record_feature_used(feature: str, *, hermes_home: Any = None) -> None:
|
|
"""Report one real use of ``feature`` (only the first per install becomes a row). Never raises.
|
|
``hermes_home`` binds the owning profile for callers outside it."""
|
|
token = None
|
|
try:
|
|
if hermes_home:
|
|
from hermes_constants import set_hermes_home_override
|
|
|
|
token = set_hermes_home_override(str(hermes_home))
|
|
from .relay_shared_metrics import enabled, record_process_mark
|
|
|
|
if feature in contract.FEATURES and enabled():
|
|
record_process_mark(contract.FEATURE_USED_MARK, {"feature": feature})
|
|
except Exception:
|
|
logger.debug("Shared-metrics feature use not recorded", exc_info=True)
|
|
finally:
|
|
if token is not None:
|
|
from hermes_constants import reset_hermes_home_override
|
|
|
|
reset_hermes_home_override(token)
|