hermes-agent/evals/compaction/results/jev-cycles-2026-09-19/cycles-prreview-500k.json
teknium1 d659843dbf evals(compaction): Jev repeated-compaction simulator, cycle data and verdict
jev_cycles.py compacts a lineage with Jev every time the estimate crosses a
threshold and records freed tokens, text floor and fitting stage per cycle.
Six runs show the floor (never-removed user/assistant text) is the binding
limit: freed-per-cycle decays to 0-8% and a 200K-window host is stuck after
0.42M tokens; >~500 tool calls between compactions cannot fit the 32K state.
Scorecard gains a decisive verdict section.
2026-09-19 11:58:37 -07:00

458 lines
8.4 KiB
JSON

{
"transcript": "prreview",
"threshold": 500000,
"raw_tokens_consumed": 11361152,
"raw_total": 11361152,
"msgs_consumed": 10074,
"cycles": [
{
"cycle": 1,
"at_msg": 531,
"before": 500120,
"after": 52660,
"freed_pct": 89.5,
"floor": 43449,
"candidates": 252,
"dropped": 252,
"stage": "old calls compacted",
"state_tok": 24975,
"requests": 7,
"usd": 0.0083
},
{
"cycle": 2,
"at_msg": 961,
"before": 511654,
"after": 111265,
"freed_pct": 78.3,
"floor": 66783,
"candidates": 199,
"dropped": 199,
"stage": "old messages collapsed",
"state_tok": 24916,
"requests": 6,
"usd": 0.0071
},
{
"cycle": 3,
"at_msg": 1313,
"before": 502998,
"after": 88652,
"freed_pct": 82.4,
"floor": 70824,
"candidates": 216,
"dropped": 216,
"stage": "old messages collapsed",
"state_tok": 24882,
"requests": 6,
"usd": 0.0071
},
{
"cycle": 4,
"at_msg": 1627,
"before": 501264,
"after": 92569,
"freed_pct": 81.5,
"floor": 72852,
"candidates": 177,
"dropped": 177,
"stage": "old messages collapsed",
"state_tok": 24921,
"requests": 5,
"usd": 0.0059
},
{
"cycle": 5,
"at_msg": 1947,
"before": 500375,
"after": 94749,
"freed_pct": 81.1,
"floor": 75759,
"candidates": 193,
"dropped": 191,
"stage": "old messages collapsed",
"state_tok": 24852,
"requests": 5,
"usd": 0.0059
},
{
"cycle": 6,
"at_msg": 2411,
"before": 519129,
"after": 114932,
"freed_pct": 77.9,
"floor": 78211,
"candidates": 219,
"dropped": 218,
"stage": "old calls compacted",
"state_tok": 24999,
"requests": 6,
"usd": 0.0072
},
{
"cycle": 7,
"at_msg": 2741,
"before": 502448,
"after": 104732,
"freed_pct": 79.2,
"floor": 80148,
"candidates": 168,
"dropped": 167,
"stage": "old messages collapsed",
"state_tok": 24954,
"requests": 5,
"usd": 0.0059
},
{
"cycle": 8,
"at_msg": 3236,
"before": 504872,
"after": 117119,
"freed_pct": 76.8,
"floor": 82732,
"candidates": 245,
"dropped": 245,
"stage": "old calls compacted",
"state_tok": 24979,
"requests": 7,
"usd": 0.0084
},
{
"cycle": 9,
"at_msg": 3636,
"before": 500075,
"after": 119512,
"freed_pct": 76.1,
"floor": 86298,
"candidates": 210,
"dropped": 209,
"stage": "old calls compacted",
"state_tok": 24973,
"requests": 6,
"usd": 0.0072
},
{
"cycle": 10,
"at_msg": 4000,
"before": 502270,
"after": 128473,
"freed_pct": 74.4,
"floor": 88838,
"candidates": 181,
"dropped": 181,
"stage": "old messages collapsed",
"state_tok": 24956,
"requests": 5,
"usd": 0.006
},
{
"cycle": 11,
"at_msg": 4354,
"before": 501400,
"after": 151425,
"freed_pct": 69.8,
"floor": 91026,
"candidates": 192,
"dropped": 192,
"stage": "old messages collapsed",
"state_tok": 24842,
"requests": 5,
"usd": 0.006
},
{
"cycle": 12,
"at_msg": 4592,
"before": 511090,
"after": 158483,
"freed_pct": 69.0,
"floor": 93179,
"candidates": 128,
"dropped": 127,
"stage": "old messages collapsed",
"state_tok": 24897,
"requests": 4,
"usd": 0.0046
},
{
"cycle": 13,
"at_msg": 4806,
"before": 501974,
"after": 142234,
"freed_pct": 71.7,
"floor": 95299,
"candidates": 110,
"dropped": 110,
"stage": "old messages collapsed",
"state_tok": 24984,
"requests": 3,
"usd": 0.0035
},
{
"cycle": 14,
"at_msg": 5124,
"before": 501941,
"after": 156309,
"freed_pct": 68.9,
"floor": 98017,
"candidates": 161,
"dropped": 160,
"stage": "old messages collapsed",
"state_tok": 24877,
"requests": 5,
"usd": 0.0059
},
{
"cycle": 15,
"at_msg": 5463,
"before": 500937,
"after": 154241,
"freed_pct": 69.2,
"floor": 101588,
"candidates": 176,
"dropped": 176,
"stage": "old messages collapsed",
"state_tok": 24813,
"requests": 5,
"usd": 0.006
},
{
"cycle": 16,
"at_msg": 5819,
"before": 503417,
"after": 162498,
"freed_pct": 67.7,
"floor": 102640,
"candidates": 233,
"dropped": 233,
"stage": "old calls compacted",
"state_tok": 24996,
"requests": 7,
"usd": 0.0083
},
{
"cycle": 17,
"at_msg": 6033,
"before": 507005,
"after": 188509,
"freed_pct": 62.8,
"floor": 104655,
"candidates": 110,
"dropped": 110,
"stage": "old messages collapsed",
"state_tok": 24876,
"requests": 3,
"usd": 0.0035
},
{
"cycle": 18,
"at_msg": 6314,
"before": 501223,
"after": 163564,
"freed_pct": 67.4,
"floor": 105926,
"candidates": 143,
"dropped": 143,
"stage": "old messages collapsed",
"state_tok": 24865,
"requests": 4,
"usd": 0.0047
},
{
"cycle": 19,
"at_msg": 6646,
"before": 501654,
"after": 178869,
"freed_pct": 64.3,
"floor": 109116,
"candidates": 170,
"dropped": 170,
"stage": "old calls compacted",
"state_tok": 24992,
"requests": 5,
"usd": 0.006
},
{
"cycle": 20,
"at_msg": 6883,
"before": 501207,
"after": 172627,
"freed_pct": 65.6,
"floor": 111444,
"candidates": 126,
"dropped": 126,
"stage": "old messages collapsed",
"state_tok": 24957,
"requests": 4,
"usd": 0.0047
},
{
"cycle": 21,
"at_msg": 7016,
"before": 506054,
"after": 218132,
"freed_pct": 56.9,
"floor": 114654,
"candidates": 73,
"dropped": 73,
"stage": "old messages collapsed",
"state_tok": 24816,
"requests": 2,
"usd": 0.0023
},
{
"cycle": 22,
"at_msg": 7283,
"before": 503596,
"after": 204002,
"freed_pct": 59.5,
"floor": 116369,
"candidates": 129,
"dropped": 125,
"stage": "old messages collapsed",
"state_tok": 24970,
"requests": 4,
"usd": 0.0048
},
{
"cycle": 23,
"at_msg": 7503,
"before": 504496,
"after": 191152,
"freed_pct": 62.1,
"floor": 117845,
"candidates": 109,
"dropped": 109,
"stage": "old messages collapsed",
"state_tok": 24917,
"requests": 3,
"usd": 0.0036
},
{
"cycle": 24,
"at_msg": 7749,
"before": 501038,
"after": 205540,
"freed_pct": 59.0,
"floor": 119715,
"candidates": 120,
"dropped": 119,
"stage": "old messages collapsed",
"state_tok": 24988,
"requests": 4,
"usd": 0.0048
},
{
"cycle": 25,
"at_msg": 8050,
"before": 515776,
"after": 211866,
"freed_pct": 58.9,
"floor": 122519,
"candidates": 155,
"dropped": 155,
"stage": "old calls compacted",
"state_tok": 24996,
"requests": 5,
"usd": 0.006
},
{
"cycle": 26,
"at_msg": 8314,
"before": 504042,
"after": 216111,
"freed_pct": 57.1,
"floor": 125001,
"candidates": 132,
"dropped": 132,
"stage": "old messages collapsed",
"state_tok": 24980,
"requests": 4,
"usd": 0.0048
},
{
"cycle": 27,
"at_msg": 8533,
"before": 500651,
"after": 205303,
"freed_pct": 59.0,
"floor": 126706,
"candidates": 110,
"dropped": 105,
"stage": "old messages collapsed",
"state_tok": 25000,
"requests": 3,
"usd": 0.0036
},
{
"cycle": 28,
"at_msg": 8819,
"before": 502475,
"after": 251171,
"freed_pct": 50.0,
"floor": 129085,
"candidates": 143,
"dropped": 138,
"stage": "old messages collapsed",
"state_tok": 24993,
"requests": 4,
"usd": 0.0049
},
{
"cycle": 29,
"at_msg": 9083,
"before": 507185,
"after": 215329,
"freed_pct": 57.5,
"floor": 131182,
"candidates": 137,
"dropped": 137,
"stage": "old messages collapsed",
"state_tok": 24856,
"requests": 4,
"usd": 0.0048
},
{
"cycle": 30,
"at_msg": 9432,
"before": 500044,
"after": 213241,
"freed_pct": 57.4,
"floor": 133547,
"candidates": 178,
"dropped": 173,
"stage": "old calls compacted",
"state_tok": 24983,
"requests": 5,
"usd": 0.0063
},
{
"cycle": 31,
"at_msg": 9654,
"before": 501024,
"after": 222477,
"freed_pct": 55.6,
"floor": 136282,
"candidates": 112,
"dropped": 112,
"stage": "old messages collapsed",
"state_tok": 24876,
"requests": 3,
"usd": 0.0036
},
{
"cycle": 32,
"at_msg": 9856,
"before": 505769,
"after": 228548,
"freed_pct": 54.8,
"floor": 138773,
"candidates": 98,
"dropped": 96,
"stage": "old messages collapsed",
"state_tok": 24861,
"requests": 3,
"usd": 0.0036
}
],
"jev_usd_total": 0.1755
}