hermes-agent/evals/compaction/results/jev-cycles-2026-09-19/cycles-sigsegv-160k.json
teknium1 d659843dbf evals(compaction): Jev repeated-compaction simulator, cycle data and verdict
jev_cycles.py compacts a lineage with Jev every time the estimate crosses a
threshold and records freed tokens, text floor and fitting stage per cycle.
Six runs show the floor (never-removed user/assistant text) is the binding
limit: freed-per-cycle decays to 0-8% and a 200K-window host is stuck after
0.42M tokens; >~500 tool calls between compactions cannot fit the 32K state.
Scorecard gains a decisive verdict section.
2026-09-19 11:58:37 -07:00

165 lines
3 KiB
JSON

{
"transcript": "sigsegv",
"threshold": 160000,
"raw_tokens_consumed": 417257,
"raw_total": 8553913,
"msgs_consumed": 1115,
"cycles": [
{
"cycle": 1,
"at_msg": 449,
"before": 181861,
"after": 88112,
"freed_pct": 51.5,
"floor": 84762,
"candidates": 140,
"dropped": 140,
"stage": "old messages collapsed",
"state_tok": 24961,
"requests": 4,
"usd": 0.0048
},
{
"cycle": 2,
"at_msg": 581,
"before": 160149,
"after": 107770,
"freed_pct": 32.7,
"floor": 95258,
"candidates": 46,
"dropped": 45,
"stage": "old messages collapsed",
"state_tok": 24962,
"requests": 2,
"usd": 0.0023
},
{
"cycle": 3,
"at_msg": 820,
"before": 160418,
"after": 125852,
"freed_pct": 21.5,
"floor": 121184,
"candidates": 71,
"dropped": 71,
"stage": "old messages collapsed",
"state_tok": 24809,
"requests": 2,
"usd": 0.0025
},
{
"cycle": 4,
"at_msg": 939,
"before": 162465,
"after": 133530,
"freed_pct": 17.8,
"floor": 126311,
"candidates": 52,
"dropped": 52,
"stage": "old messages collapsed",
"state_tok": 24815,
"requests": 2,
"usd": 0.0024
},
{
"cycle": 5,
"at_msg": 990,
"before": 160308,
"after": 146620,
"freed_pct": 8.5,
"floor": 133799,
"candidates": 16,
"dropped": 15,
"stage": "old messages collapsed",
"state_tok": 24915,
"requests": 1,
"usd": 0.0012
},
{
"cycle": 6,
"at_msg": 1023,
"before": 160097,
"after": 146959,
"freed_pct": 8.2,
"floor": 141253,
"candidates": 12,
"dropped": 12,
"stage": "old messages collapsed",
"state_tok": 24869,
"requests": 1,
"usd": 0.0012
},
{
"cycle": 7,
"at_msg": 1049,
"before": 162325,
"after": 154537,
"freed_pct": 4.8,
"floor": 143551,
"candidates": 9,
"dropped": 8,
"stage": "old messages collapsed",
"state_tok": 24787,
"requests": 1,
"usd": 0.0011
},
{
"cycle": 8,
"at_msg": 1079,
"before": 160027,
"after": 150727,
"freed_pct": 5.8,
"floor": 146251,
"candidates": 6,
"dropped": 6,
"stage": "old messages collapsed",
"state_tok": 24914,
"requests": 1,
"usd": 0.0011
},
{
"cycle": 9,
"at_msg": 1104,
"before": 160066,
"after": 158598,
"freed_pct": 0.9,
"floor": 149902,
"candidates": 6,
"dropped": 6,
"stage": "old messages collapsed",
"state_tok": 24834,
"requests": 1,
"usd": 0.0011
},
{
"cycle": 10,
"at_msg": 1113,
"before": 160055,
"after": 158044,
"freed_pct": 1.3,
"floor": 151359,
"candidates": 3,
"dropped": 2,
"stage": "old messages collapsed",
"state_tok": 24921,
"requests": 1,
"usd": 0.0011
},
{
"cycle": 11,
"at_msg": 1115,
"before": 160235,
"after": 160235,
"freed_pct": 0.0,
"floor": 151359,
"candidates": 1,
"dropped": 0,
"stage": "old messages collapsed",
"state_tok": 24826,
"requests": 1,
"usd": 0.0011,
"stuck": true
}
],
"jev_usd_total": 0.02
}