hermes-agent/evals/tool_search/analyze_livetest.py
ethernet 612d542281 Merge remote-tracking branch 'origin/main' into ethie/pm-clean
# Conflicts:
#	.gitignore
#	Dockerfile
#	agent/onboarding.py
#	apps/desktop/electron/main.ts
#	apps/desktop/electron/pool-stop.ts
#	apps/desktop/src/components/model-picker.test.tsx
#	apps/desktop/src/store/updates.ts
#	apps/desktop/vite.config.ts
#	datagen-config-examples/run_browser_tasks.sh
#	docs/rca-ssl-cacert-post-git-pull.md
#	gateway/run.py
#	hermes_cli/backup.py
#	hermes_cli/credential_lifecycle.py
#	hermes_cli/dashboard_procs.py
#	hermes_cli/doctor_state.py
#	hermes_cli/env_loader.py
#	hermes_cli/gateway_windows.py
#	hermes_cli/local_runtime/endpoint.py
#	hermes_cli/psutil_android.py
#	hermes_cli/update_cmd.py
#	hermes_cli/update_cmd_windows.py
#	hermes_cli/web_routers/local_models.py
#	hermes_cli/web_server_config.py
#	hermes_cli/web_server_cron.py
#	plugins/memory/hindsight/__init__.py
#	plugins/memory/holographic/__init__.py
#	plugins/memory/honcho/cli.py
#	plugins/memory/mem0/__init__.py
#	plugins/platforms/google_chat/oauth.py
#	plugins/platforms/photon/adapter.py
#	scripts/ci/list_os_marked_tests.py
#	scripts/run_tests.sh
#	tests/agent/test_compression_stall_fallback.py
#	tests/agent/test_create_openai_client_ssl_verify.py
#	tests/gateway/test_google_chat_oauth_dependencies.py
#	tests/hermes_cli/conftest.py
#	tests/hermes_cli/test_cli_init.py
#	tests/hermes_cli/test_gateway_migrate_multiplex.py
#	tests/hermes_cli/test_psutil_android_extract.py
#	tests/hermes_cli/test_relaunch.py
#	tests/hermes_cli/test_update_check.py
#	tests/hermes_cli/test_update_handoff_desktop_rebuild.py
#	tests/hermes_cli/test_worktree_gc.py
#	tests/scripts/desktop_update/test_desktop_update_windows_python_handoff.py
#	tests/scripts/desktop_update/test_desktop_update_windows_retry_policy.py
#	tests/scripts/desktop_update/test_desktop_update_windows_timestamp.py
#	tests/scripts/install/test_install_autostash_conflict_recovery.py
#	tests/scripts/install/test_install_clone_throttle_fallback.py
#	tests/scripts/install/test_install_commit_pin_rollback.py
#	tests/scripts/install/test_install_diverged_update.py
#	tests/scripts/install/test_install_lockfile_churn.py
#	tests/scripts/install/test_install_macos_launcher.py
#	tests/scripts/install/test_install_no_initial_commit.py
#	tests/scripts/install/test_install_ps1_ascii_only.py
#	tests/scripts/install/test_install_ps1_browser_install.py
#	tests/scripts/install/test_install_ps1_managed_node_swap.py
#	tests/scripts/install/test_install_ps1_native_stderr_eap.py
#	tests/scripts/install/test_install_ps1_node_path_for_npm.py
#	tests/scripts/install/test_install_ps1_python_fallback_venv.py
#	tests/scripts/install/test_install_ps1_resolver_strictmode.py
#	tests/scripts/install/test_install_ps1_uv_install_fallback.py
#	tests/scripts/install/test_install_ps1_uv_powershell_host.py
#	tests/scripts/install/test_install_ps1_venv_process_tree.py
#	tests/scripts/install/test_install_ps1_venv_recreate_safety.py
#	tests/scripts/install/test_install_ps1_venv_rename_abort.py
#	tests/scripts/install/test_install_ps1_venv_transaction_boundary.py
#	tests/scripts/install/test_install_ps1_web_server_syntax_probe.py
#	tests/scripts/install/test_install_scripts_computer_use.py
#	tests/scripts/install/test_install_sh_acp_launcher.py
#	tests/scripts/install/test_install_sh_bootstrap_marker.py
#	tests/scripts/install/test_install_sh_browser_install.py
#	tests/scripts/install/test_install_sh_install_method_stamp.py
#	tests/scripts/install/test_install_sh_node_deps_failure.py
#	tests/scripts/install/test_install_sh_node_deps_workspaces.py
#	tests/scripts/install/test_install_sh_node_global_prefix.py
#	tests/scripts/install/test_install_sh_node_npm_check.py
#	tests/scripts/install/test_install_sh_node_prerelease.py
#	tests/scripts/install/test_install_sh_node_probe.py
#	tests/scripts/install/test_install_sh_node_tarball_without_xz.py
#	tests/scripts/install/test_install_sh_pythonpath_sanitization.py
#	tests/scripts/install/test_install_sh_reuse_supported_python.py
#	tests/scripts/install/test_install_sh_root_fhs_uv_python_path.py
#	tests/scripts/install/test_install_sh_setup_wizard_tty_probe.py
#	tests/scripts/install/test_install_sh_symlink_stomp.py
#	tests/scripts/install/test_install_sh_termux_network_prereqs.py
#	tests/scripts/install/test_install_sh_termux_python_bounds.py
#	tests/scripts/install/test_install_sh_uv_lock_config.py
#	tests/scripts/install/test_install_unmerged_index.py
#	tests/scripts/test_run_tests_parallel.py
#	tests/test_managed_runtime_resolution.py
#	tests/test_project_metadata.py
#	tests/tools/test_browser_use_cli.py
#	tests/tools/test_tts_pythonpath_fallback.py
#	tests/tui_gateway/test_hosted_room_driver_runtime.py
#	tests/tui_gateway/test_tui_gateway_server.py
#	tools/lazy_deps.py
#	tools/voice_mode.py
#	uv.lock
#	website/docs/developer-guide/macos-bundle-updates.md
#	website/docs/developer-guide/pm-audit-status.md
#	website/docs/developer-guide/shared-bundle-builds.md
#	website/docs/developer-guide/source-update-completion.md
#	website/docs/developer-guide/stable-releases.md
2026-09-14 15:38:34 -04:00

124 lines
4.5 KiB
Python

#!/usr/bin/env python3
"""Compare enabled vs disabled runs and produce a readable report.
Reads scripts/out/_summary.json and the per-scenario JSONs, prints a side-by-
side comparison of what happened, and flags anomalies.
"""
from __future__ import annotations
import json
import sys
from pathlib import Path
HERE = Path(__file__).resolve().parent
OUT = HERE / "out"
def load_record(scenario_id: str, mode: str):
path = OUT / f"{scenario_id}__{mode}.json"
if not path.exists():
return None
return json.loads(path.read_text(encoding="utf-8-sig"))
def fmt_tool_seq(calls):
if not calls:
return "(none)"
return " → ".join(c["name"] for c in calls)
def fmt_bridge_seq(calls):
if not calls:
return "(none)"
parts = []
for c in calls:
if c["name"] == "tool_call":
inner = (c.get("args") or {}).get("name", "?")
parts.append(f"tool_call→{inner}")
elif c["name"] == "tool_search":
args = c.get("args") or {}
qs = args.get("queries")
if isinstance(qs, list):
q = "; ".join(str(x) for x in qs)
else: # legacy single-query transcripts
q = str(args["query"] if "query" in args else "?")
parts.append(f"search('{q[:30]}')")
elif c["name"] == "tool_describe":
args = c.get("args") or {}
ns = args.get("names")
if isinstance(ns, list):
n = ", ".join(str(x) for x in ns)
else: # legacy single-name transcripts
n = str(args.get("name", "?"))
parts.append(f"describe({n})")
return " → ".join(parts)
def main():
if not OUT.exists():
print("No output directory at", OUT)
sys.exit(1)
summary_path = OUT / "_summary.json"
if not summary_path.exists():
print("No _summary.json yet")
sys.exit(1)
summary = json.loads(summary_path.read_text(encoding="utf-8-sig"))
scenarios = sorted({row["scenario"] for row in summary})
print(f"{'='*78}")
print(" Live test results: tool_search ENABLED vs DISABLED")
print(f"{'='*78}\n")
fails = 0
for sid in scenarios:
en = load_record(sid, "enabled")
di = load_record(sid, "disabled")
if not en or not di:
continue
expected = set(en["expected_underlying_tools"])
print(f"┌─ {sid} ({en['scenario_description']})")
print(f"│ Prompt: {en['prompt'][:120]}")
print(f"│ Expected underlying tools: {sorted(expected) or '(none)'}")
print("│")
for label, rec in [("ENABLED ", en), ("DISABLED", di)]:
called_under = [c["name"] for c in rec["underlying_tool_calls"]]
called_set = set(called_under)
missing = expected - called_set
extra = called_set - expected - {"read_file", "search_files", "terminal", "todo", "memory"}
mark = "✓" if (expected.issubset(called_set) and not rec["error"]) else "✗"
if mark == "✗":
fails += 1
print(f"│ {label} {mark} bridges={len(rec['bridge_calls']):2} underlying={len(rec['underlying_tool_calls']):2} "
f"iters={rec['n_iterations']:2} elapsed={rec['elapsed_seconds']:5.1f}s err={bool(rec['error'])}")
print(f"│ underlying: {fmt_tool_seq(rec['underlying_tool_calls'])}")
if rec["bridge_calls"]:
print(f"│ bridges: {fmt_bridge_seq(rec['bridge_calls'])}")
if missing:
print(f"│ ⚠ MISSING expected tools: {sorted(missing)}")
if extra:
print(f"│ ⓘ extra tools called: {sorted(extra)}")
if rec["error"]:
print(f"│ 💥 error: {rec['error'][:200]}")
# Bridge-trip count vs direct (interesting comparator)
en_bridges = len(en["bridge_calls"])
di_underlying = len(di["underlying_tool_calls"])
en_underlying = len(en["underlying_tool_calls"])
overhead = en_bridges + en_underlying - di_underlying
print(f"│ Δ round-trip cost: enabled used {en_bridges + en_underlying} calls vs disabled {di_underlying} → +{overhead}")
print(f"│ Final (enabled): {(en.get('final_response') or '')[:140]}")
print(f"│ Final (disabled): {(di.get('final_response') or '')[:140]}")
print("└──")
print()
print(f"\nFails: {fails}/{2*len(scenarios)}")
if __name__ == "__main__":
main()