hermes-agent/evals/postmortem/live_ab/cache_prefix_live.py
teknium1 956732d4a8 chore(evals,desktop): drop literal /tmp from eval harnesses and desktop scripts
Eval probes wrote fixtures, receipts and evidence to hard-coded /tmp paths and
two live A/B tasks literally instructed the model to work in /tmp. They now
derive locations from tempfile.gettempdir() / os.tmpdir() (env overrides kept),
usage examples use relative output names, and the desktop e2e screenshot dirs,
perf scripts and the short-session repro fixture stop naming /tmp. Also adds
the explicit encoding= the windows-footgun check wants in the touched files.
2026-09-19 10:44:26 -07:00

50 lines
3.3 KiB
Python

"""Live A/B for the thinking-strip cache miss (F0). Runs a short real tool loop through AIAgent on
Fable 5.1 via Nous and prints per-call cache hit ratios from agent.log. ~10 calls, well under $1.
Arm A = current code. Arm B = HERMES_KEEP_ALL_THINKING=1 monkeypatch of _manage_thinking_signatures
that passes thinking blocks back unchanged for the Nous/Anthropic route."""
import os, sys, re, tempfile, time, subprocess, json
# LIVE: real provider calls (cents). Usage: python cache_prefix_live.py <repo_root> <A|B>
os.environ.setdefault("HERMES_HOME", os.path.expanduser("~/.hermes"))
sys.path.insert(0, sys.argv[1])
arm = sys.argv[2] if len(sys.argv) > 2 else "A"
import agent.anthropic_message_convert as amc
if arm == "B":
_orig = amc._manage_thinking_signatures
def _keep_all(result, base_url, model):
# keep-all model on a signature-validating route: pass every thinking block back unchanged,
# only drop cache_control on thinking blocks and the internal flag.
for idx, m in amc._assistant_block_lists(result):
for b in m["content"]:
if amc._block_type(b) in amc._THINKING_TYPES:
b.pop("cache_control", None)
m.pop("_thinking_signature_invalidated", None)
amc._manage_thinking_signatures = _keep_all
# the adapter imports the name at call time via module attribute? verify binding
import agent.anthropic_adapter as ad
if hasattr(ad, "_manage_thinking_signatures"):
ad._manage_thinking_signatures = _keep_all
from run_agent import AIAgent
from hermes_cli.runtime_provider import resolve_runtime_provider
rt = resolve_runtime_provider(requested="nous", target_model="anthropic/claude-fable-5.1")
sid = f"f0ab_{arm}_{int(time.time())}"
ag = AIAgent(model="anthropic/claude-fable-5.1", provider="nous", base_url=rt.get("base_url"), api_key=rt.get("api_key"),
api_mode=rt.get("api_mode"), session_id=sid, quiet_mode=True,
enabled_toolsets=["file", "terminal"], platform="cli", max_iterations=12,
skip_context_files=True, skip_memory=True, reasoning_config={"enabled": True, "effort": "medium"})
work = os.path.join(tempfile.gettempdir(), "f0ab_work")
task = (f"In {work} (create it), do these steps ONE tool call at a time, no parallel calls: "
"1) write a.txt with 'alpha', 2) write b.txt with 'beta', 3) read a.txt, 4) read b.txt, "
f"5) run `ls -la {work}`, 6) run `wc -c {work}/*`, 7) run `cat {work}/a.txt`, "
"then reply with one line: DONE.")
t0 = time.time()
r = ag.run_conversation(task)
print("final:", (r.get("final_response") or "")[:80], "| wall", round(time.time() - t0, 1), "s")
time.sleep(1)
log = subprocess.run(f"grep -h '\\[{sid}\\]' ~/.hermes/logs/agent.log | grep 'API call #'", shell=True, capture_output=True).stdout.decode("utf-8", "replace")
rows = re.findall(r"API call #(\d+): .*in=(\d+) out=(\d+) .*cache=(\d+)/(\d+) \((\d+)%\)", log)
tot_in = tot_c = 0
for n, i, o, c, ct, p in rows:
i, c = int(i), int(c); tot_in += i; tot_c += c
print(f" call {n:>2} in={i:>7} out={o:>5} cached={c:>7} ({p}%) uncached={i-c}")
print(f"ARM {arm}: calls={len(rows)} input={tot_in} cached={tot_c} uncached={tot_in-tot_c} hit={100*tot_c/max(tot_in,1):.1f}%")
print(json.dumps({"arm": arm, "calls": len(rows), "input": tot_in, "cached": tot_c}))