hermes-agent/evals/compaction/results/jev-cycles-2026-09-19/cycles-sysprompt-500k.json
teknium1 d659843dbf evals(compaction): Jev repeated-compaction simulator, cycle data and verdict
jev_cycles.py compacts a lineage with Jev every time the estimate crosses a
threshold and records freed tokens, text floor and fitting stage per cycle.
Six runs show the floor (never-removed user/assistant text) is the binding
limit: freed-per-cycle decays to 0-8% and a 200K-window host is stuck after
0.42M tokens; >~500 tool calls between compactions cannot fit the 32K state.
Scorecard gains a decisive verdict section.
2026-09-19 11:58:37 -07:00

570 lines
10 KiB
JSON

{
"transcript": "sysprompt",
"threshold": 500000,
"raw_tokens_consumed": 8601919,
"raw_total": 23311571,
"msgs_consumed": 6663,
"cycles": [
{
"cycle": 1,
"at_msg": 508,
"before": 500281,
"after": 122136,
"freed_pct": 75.6,
"floor": 91399,
"candidates": 200,
"dropped": 200,
"stage": "old messages collapsed",
"state_tok": 24958,
"requests": 6,
"usd": 0.007
},
{
"cycle": 2,
"at_msg": 731,
"before": 503628,
"after": 130821,
"freed_pct": 74.0,
"floor": 110321,
"candidates": 109,
"dropped": 109,
"stage": "old messages collapsed",
"state_tok": 24975,
"requests": 3,
"usd": 0.0035
},
{
"cycle": 3,
"at_msg": 1006,
"before": 500183,
"after": 134695,
"freed_pct": 73.1,
"floor": 128713,
"candidates": 139,
"dropped": 139,
"stage": "old messages collapsed",
"state_tok": 24952,
"requests": 4,
"usd": 0.0047
},
{
"cycle": 4,
"at_msg": 1229,
"before": 500076,
"after": 158048,
"freed_pct": 68.4,
"floor": 151031,
"candidates": 111,
"dropped": 111,
"stage": "old messages collapsed",
"state_tok": 24870,
"requests": 3,
"usd": 0.0035
},
{
"cycle": 5,
"at_msg": 1459,
"before": 501400,
"after": 179777,
"freed_pct": 64.1,
"floor": 171380,
"candidates": 115,
"dropped": 115,
"stage": "old messages collapsed",
"state_tok": 24894,
"requests": 3,
"usd": 0.0035
},
{
"cycle": 6,
"at_msg": 1759,
"before": 501647,
"after": 199215,
"freed_pct": 60.3,
"floor": 182724,
"candidates": 143,
"dropped": 143,
"stage": "old messages collapsed",
"state_tok": 24976,
"requests": 4,
"usd": 0.0048
},
{
"cycle": 7,
"at_msg": 2164,
"before": 500531,
"after": 206717,
"freed_pct": 58.7,
"floor": 189181,
"candidates": 235,
"dropped": 235,
"stage": "old calls compacted",
"state_tok": 24878,
"requests": 6,
"usd": 0.0076
},
{
"cycle": 8,
"at_msg": 2515,
"before": 508820,
"after": 223053,
"freed_pct": 56.2,
"floor": 194669,
"candidates": 199,
"dropped": 199,
"stage": "old calls compacted",
"state_tok": 24953,
"requests": 6,
"usd": 0.0075
},
{
"cycle": 9,
"at_msg": 2861,
"before": 500524,
"after": 229101,
"freed_pct": 54.2,
"floor": 199926,
"candidates": 182,
"dropped": 182,
"stage": "old messages left out",
"state_tok": 24983,
"requests": 5,
"usd": 0.0064
},
{
"cycle": 10,
"at_msg": 3062,
"before": 500764,
"after": 237294,
"freed_pct": 52.6,
"floor": 202656,
"candidates": 107,
"dropped": 107,
"stage": "old calls compacted",
"state_tok": 24998,
"requests": 3,
"usd": 0.0038
},
{
"cycle": 11,
"at_msg": 3228,
"before": 500037,
"after": 255078,
"freed_pct": 49.0,
"floor": 203299,
"candidates": 119,
"dropped": 119,
"stage": "old calls compacted",
"state_tok": 24948,
"requests": 4,
"usd": 0.005
},
{
"cycle": 12,
"at_msg": 3471,
"before": 501341,
"after": 244372,
"freed_pct": 51.3,
"floor": 205213,
"candidates": 118,
"dropped": 118,
"stage": "old messages left out",
"state_tok": 24971,
"requests": 4,
"usd": 0.005
},
{
"cycle": 13,
"at_msg": 3621,
"before": 502278,
"after": 266125,
"freed_pct": 47.0,
"floor": 207160,
"candidates": 70,
"dropped": 70,
"stage": "old calls compacted",
"state_tok": 24969,
"requests": 2,
"usd": 0.0025
},
{
"cycle": 14,
"at_msg": 3769,
"before": 500791,
"after": 264091,
"freed_pct": 47.3,
"floor": 209372,
"candidates": 79,
"dropped": 79,
"stage": "old calls compacted",
"state_tok": 24980,
"requests": 3,
"usd": 0.0037
},
{
"cycle": 15,
"at_msg": 3946,
"before": 500330,
"after": 279988,
"freed_pct": 44.0,
"floor": 211711,
"candidates": 115,
"dropped": 115,
"stage": "old messages left out",
"state_tok": 24977,
"requests": 3,
"usd": 0.0039
},
{
"cycle": 16,
"at_msg": 4055,
"before": 505381,
"after": 268923,
"freed_pct": 46.8,
"floor": 213362,
"candidates": 65,
"dropped": 65,
"stage": "old calls compacted",
"state_tok": 24969,
"requests": 2,
"usd": 0.0025
},
{
"cycle": 17,
"at_msg": 4223,
"before": 501906,
"after": 310108,
"freed_pct": 38.2,
"floor": 214263,
"candidates": 82,
"dropped": 82,
"stage": "old messages left out",
"state_tok": 24983,
"requests": 3,
"usd": 0.0037
},
{
"cycle": 18,
"at_msg": 4359,
"before": 500152,
"after": 282762,
"freed_pct": 43.5,
"floor": 215250,
"candidates": 66,
"dropped": 66,
"stage": "old messages left out",
"state_tok": 24997,
"requests": 2,
"usd": 0.0025
},
{
"cycle": 19,
"at_msg": 4506,
"before": 504446,
"after": 305300,
"freed_pct": 39.5,
"floor": 217046,
"candidates": 69,
"dropped": 69,
"stage": "old messages left out",
"state_tok": 24978,
"requests": 2,
"usd": 0.0025
},
{
"cycle": 20,
"at_msg": 4660,
"before": 501723,
"after": 293927,
"freed_pct": 41.4,
"floor": 218023,
"candidates": 79,
"dropped": 79,
"stage": "old messages left out",
"state_tok": 24990,
"requests": 3,
"usd": 0.0037
},
{
"cycle": 21,
"at_msg": 4844,
"before": 505792,
"after": 306179,
"freed_pct": 39.5,
"floor": 219729,
"candidates": 86,
"dropped": 86,
"stage": "old messages left out",
"state_tok": 24992,
"requests": 3,
"usd": 0.0037
},
{
"cycle": 22,
"at_msg": 5081,
"before": 500256,
"after": 308778,
"freed_pct": 38.3,
"floor": 220875,
"candidates": 116,
"dropped": 116,
"stage": "old messages left out",
"state_tok": 24982,
"requests": 3,
"usd": 0.0038
},
{
"cycle": 23,
"at_msg": 5202,
"before": 500337,
"after": 330910,
"freed_pct": 33.9,
"floor": 222726,
"candidates": 55,
"dropped": 55,
"stage": "old messages left out",
"state_tok": 24970,
"requests": 2,
"usd": 0.0025
},
{
"cycle": 24,
"at_msg": 5281,
"before": 505966,
"after": 344354,
"freed_pct": 31.9,
"floor": 223692,
"candidates": 37,
"dropped": 37,
"stage": "old messages left out",
"state_tok": 24984,
"requests": 1,
"usd": 0.0013
},
{
"cycle": 25,
"at_msg": 5358,
"before": 500802,
"after": 360397,
"freed_pct": 28.0,
"floor": 225423,
"candidates": 35,
"dropped": 35,
"stage": "old messages left out",
"state_tok": 24994,
"requests": 1,
"usd": 0.0013
},
{
"cycle": 26,
"at_msg": 5459,
"before": 502896,
"after": 352048,
"freed_pct": 30.0,
"floor": 227627,
"candidates": 51,
"dropped": 51,
"stage": "old messages left out",
"state_tok": 24994,
"requests": 2,
"usd": 0.0025
},
{
"cycle": 27,
"at_msg": 5551,
"before": 502348,
"after": 361513,
"freed_pct": 28.0,
"floor": 228455,
"candidates": 60,
"dropped": 60,
"stage": "old messages left out",
"state_tok": 24969,
"requests": 2,
"usd": 0.0025
},
{
"cycle": 28,
"at_msg": 5665,
"before": 501853,
"after": 347094,
"freed_pct": 30.8,
"floor": 229579,
"candidates": 54,
"dropped": 54,
"stage": "old messages left out",
"state_tok": 24979,
"requests": 2,
"usd": 0.0025
},
{
"cycle": 29,
"at_msg": 5772,
"before": 502020,
"after": 349120,
"freed_pct": 30.5,
"floor": 230704,
"candidates": 52,
"dropped": 52,
"stage": "old messages left out",
"state_tok": 24987,
"requests": 2,
"usd": 0.0025
},
{
"cycle": 30,
"at_msg": 5839,
"before": 500681,
"after": 359578,
"freed_pct": 28.2,
"floor": 230907,
"candidates": 31,
"dropped": 31,
"stage": "old messages left out",
"state_tok": 24992,
"requests": 1,
"usd": 0.0013
},
{
"cycle": 31,
"at_msg": 5919,
"before": 501642,
"after": 372640,
"freed_pct": 25.7,
"floor": 232045,
"candidates": 39,
"dropped": 39,
"stage": "old messages left out",
"state_tok": 24994,
"requests": 2,
"usd": 0.0024
},
{
"cycle": 32,
"at_msg": 6017,
"before": 502268,
"after": 380565,
"freed_pct": 24.2,
"floor": 232644,
"candidates": 58,
"dropped": 58,
"stage": "old messages left out",
"state_tok": 24991,
"requests": 2,
"usd": 0.0025
},
{
"cycle": 33,
"at_msg": 6108,
"before": 501239,
"after": 379645,
"freed_pct": 24.3,
"floor": 233193,
"candidates": 43,
"dropped": 43,
"stage": "old messages left out",
"state_tok": 24980,
"requests": 2,
"usd": 0.0024
},
{
"cycle": 34,
"at_msg": 6178,
"before": 502893,
"after": 380806,
"freed_pct": 24.3,
"floor": 234168,
"candidates": 33,
"dropped": 33,
"stage": "old messages left out",
"state_tok": 24994,
"requests": 1,
"usd": 0.0013
},
{
"cycle": 35,
"at_msg": 6310,
"before": 502521,
"after": 376822,
"freed_pct": 25.0,
"floor": 236234,
"candidates": 68,
"dropped": 68,
"stage": "old messages left out",
"state_tok": 24972,
"requests": 2,
"usd": 0.0025
},
{
"cycle": 36,
"at_msg": 6374,
"before": 500684,
"after": 380201,
"freed_pct": 24.1,
"floor": 236266,
"candidates": 45,
"dropped": 45,
"stage": "old messages left out",
"state_tok": 24989,
"requests": 2,
"usd": 0.0025
},
{
"cycle": 37,
"at_msg": 6456,
"before": 506443,
"after": 402465,
"freed_pct": 20.5,
"floor": 237354,
"candidates": 40,
"dropped": 40,
"stage": "old messages left out",
"state_tok": 24992,
"requests": 2,
"usd": 0.0024
},
{
"cycle": 38,
"at_msg": 6516,
"before": 502959,
"after": 390405,
"freed_pct": 22.4,
"floor": 237724,
"candidates": 28,
"dropped": 28,
"stage": "old messages left out",
"state_tok": 24996,
"requests": 1,
"usd": 0.0012
},
{
"cycle": 39,
"at_msg": 6588,
"before": 500087,
"after": 403242,
"freed_pct": 19.4,
"floor": 238371,
"candidates": 35,
"dropped": 35,
"stage": "old messages left out",
"state_tok": 24987,
"requests": 1,
"usd": 0.0013
},
{
"cycle": 40,
"at_msg": 6663,
"before": 500286,
"after": 401251,
"freed_pct": 19.8,
"floor": 239653,
"candidates": 44,
"dropped": 44,
"stage": "old messages left out",
"state_tok": 24989,
"requests": 2,
"usd": 0.0025
}
],
"jev_usd_total": 0.132
}