hermes-agent/apps/desktop/electron/pool-retire.ts
Brooklyn Nicholson 654300240a fix(desktop): reap keepalive-pinned pool backends idle past a streamed-activity TTL
The renderer's 60s keepalive (touchPoolBackend) refreshes lastActiveAt
for every OPEN chat — the only clock both the idle reaper and LRU
eviction read — so the pinned tier (backends whose profile chat was ever
opened) was exempt from both the maxBackends cap and the idle reaper
forever: N profile chats held N ~120 MB serve children until app quit
(126 processes / 7.5 GB, all parented to the live Hermes.exe) (#105239).

A keepalive proves the chat is open, not that anything streamed.

- touchPoolBackend stamps entry.lastStreamedAt on activeTurn=true
  touches (a prompt turn leasing the backend IS streamed activity).
- pool-retire: PoolRetireEntry gains lastStreamedAt; retireIdle accepts
  an idle-eligibility override so the reaper can key on the streamed
  clock. Every other safeguard is unchanged: activeTurn veto, identity
  checks, admission permit, backend proof probe.
- startPoolIdleReaper retires a local child whose last streamed turn is
  older than POOL_PINNED_IDLE_MS (default 60 min,
  HERMES_DESKTOP_POOL_PINNED_IDLE_MS override) even while keepalive-fresh;
  entries without the stamp keep the legacy lastActiveAt clock. Killing
  is never user-observable: re-focusing the chat re-ensures idempotently
  via ensureBackend/ensureRegistryBackend entry reuse (#105239 req 3).

Note: #118246 already routes local profiles through one host backend;
this TTL bounds what still pools per-profile (isolated mode, unscopable
mutating REST requests, remote profiles' local descriptors aside).
2026-09-29 21:43:12 -05:00

195 lines
6.1 KiB
TypeScript

import type { LocalBackendSpawnCoordinator } from './pool-spawn-coordinator'
export interface PoolRetireEntry {
/** An early veto only; backend admission owns the proof. */
activeTurn?: boolean
lastActiveAt?: null | number
/**
* Last prompt turn that leased this backend (#105239): a keepalive touch
* only proves the chat is open, while this stamp proves streamed activity.
* The idle reaper reads it for the pinned-tier TTL; absent means the entry
* predates the stamp and keeps the legacy lastActiveAt clock.
*/
lastStreamedAt?: null | number
process?: unknown
}
export interface PoolRetirerDeps<E extends PoolRetireEntry> {
pool: Map<string, E>
coordinator: LocalBackendSpawnCoordinator
prepare: (key: string, entry: E) => Promise<string | null>
commit: (key: string, entry: E, token: string) => Promise<boolean>
cancel: (key: string, entry: E, token: string) => Promise<void>
stopBackend: (key: string) => Promise<void>
onRetiring?: (key: string) => void
log?: (message: string) => void
}
export function selectRetirementCandidates<K, E extends PoolRetireEntry>(
entries: Iterable<[K, E]>,
exclude: ReadonlySet<K>
): [K, E][] {
return [...entries]
.filter(([key, entry]) => Boolean(entry.process) && entry.activeTurn !== true && !exclude.has(key))
.sort((a, b) => (a[1].lastActiveAt || 0) - (b[1].lastActiveAt || 0))
}
/** One arbiter owns foreground recovery, idle reaping and stale LRU eviction.
* A permit freezes backend admission; a GET snapshot never authorizes a kill.
* Credit: @bounce12340's occupied-not-busy diagnosis, @austinpickett's parking,
* @chelsealong's backend-work proof and @sharkenstein3d's shared stop boundary.
*/
export function createPoolRetirer<E extends PoolRetireEntry>(deps: PoolRetirerDeps<E>) {
let serial: Promise<unknown> = Promise.resolve()
let scheduled = false
let disposed = false
const retiredScopes = new Set<string>()
let retry: ReturnType<typeof setTimeout> | undefined
const report = (error: unknown) => deps.log?.(`Pool retirement failed: ${String(error)}`)
function enqueue<T>(work: () => Promise<T>): Promise<T> {
const result = serial.then(work)
serial = result.catch(report)
return result
}
const needsCapacity = () =>
!disposed && deps.coordinator.foregroundWaiters.size > 0 && deps.coordinator.activeCount >= deps.coordinator.limit
async function retire(key: string, entry: E, needed: () => boolean): Promise<boolean> {
const eligible = () => !disposed && deps.pool.get(key) === entry && entry.activeTurn !== true && needed()
if (!entry.process || !eligible()) {
return false
}
const token = await deps.prepare(key, entry)
if (!token) {
return false
}
let committed = false
try {
// A different resident may have exited, the waiter withdrawn, or this
// key acquired a new generation while the HTTP prepare was outstanding.
if (!eligible()) {
return false
}
committed = await deps.commit(key, entry, token)
if (!committed || deps.pool.get(key) !== entry) {
return false
}
// Commit is irreversible: admission remains closed even if capacity or
// renderer intent changes now. Park before signalling the exact child.
retiredScopes.add(key)
deps.onRetiring?.(key)
await deps.stopBackend(key)
return true
} finally {
if (!committed) {
await deps.cancel(key, entry, token).catch(report)
}
}
}
async function reclaim(): Promise<void> {
while (needsCapacity()) {
let retired = false
const candidates = selectRetirementCandidates(deps.pool, deps.coordinator.foregroundWaiters)
for (const [key, entry] of candidates) {
if (!needsCapacity()) {
return
}
if (await retire(key, entry, needsCapacity)) {
retired = true
break
}
}
if (!retired) {
return
}
}
}
function wake(): void {
if (scheduled || disposed) {
return
}
clearTimeout(retry)
scheduled = true
void enqueue(reclaim)
.catch(report)
.finally(() => {
scheduled = false
// Busy work may finish without a renderer touch (cron / side agents).
// The coordinator's existing ticket deadline bounds these retries.
if (needsCapacity()) {
retry = setTimeout(wake, 1000)
retry.unref?.()
}
})
}
const unsubscribe = deps.coordinator.onChange(wake)
return {
wake,
assertCanOpen: (key: string, priority: 'foreground' | 'background') => {
if (priority === 'foreground') {
retiredScopes.delete(key)
} else if (retiredScopes.has(key)) {
throw new Error(`Backend for "${key}" was retired; open it explicitly to reconnect.`)
}
},
retireIdle: (key: string, idleMs: number, idleEligibility?: (entry: E) => boolean) =>
enqueue(async () => {
const entry = deps.pool.get(key)
if (!entry) {
return false
}
// The default clock is the keepalive-touched lastActiveAt; the reaper
// passes an override that also honours the pinned-tier TTL
// (lastStreamedAt) so a keepalive-fresh but stream-idle entry can
// still be retired (#105239). The retirer keeps every other
// safeguard: identity, admission permit, activeTurn veto, proof probe.
return retire(
key,
entry,
() => idleEligibility ? idleEligibility(entry) : Date.now() - (entry.lastActiveAt || 0) > idleMs
)
}),
evictTo: (keep: number, freshMs: number) =>
enqueue(async () => {
const retired: string[] = []
const overCap = () => [...deps.pool.values()].filter(entry => entry.process).length > Math.max(0, keep)
for (const [key, entry] of selectRetirementCandidates(deps.pool, deps.coordinator.foregroundWaiters)) {
if (await retire(key, entry, () => overCap() && Date.now() - (entry.lastActiveAt || 0) > freshMs)) {
retired.push(key)
}
}
return retired
}),
dispose: () => {
disposed = true
clearTimeout(retry)
unsubscribe()
}
}
}