mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-17 04:11:35 +00:00
Follow-up to the progress-aware probe (#96177): - Wait while the spawned child is alive instead of "alive and wrote output in the last 30s". The stall this fixes is silent: the backend answers HTTP, then holds the GIL importing gateway platform modules and prints nothing until the loop recovers (the web_server heartbeat only logs "event loop stalled" afterwards). Output recency counted down during exactly that window, so a long stall still failed. A live loopback child whose listener accepted the TCP connect is busy, not refusing; refusals and auth rejections are immediate error/close events, not timeouts. Drops the now-unused BackendOutputTail.lastActivityAt/backendMakingProgress. - One policy home: spawnedBackendProbeOptions() in gateway-ws-probe.ts (base 10s, cap 90s = the port-announcement cold-start budget); both the primary and pool spawn paths use it. Remote gateways, "Test remote" and host-backend attach keep the fixed budget. - Without keepWaitingWhile the probe is again a single one-shot timer (no 1s polling during the base budget). Timeout reasons name what ended the wait: base budget, backend stopped after the budget, or the cap. - Tests drive every deadline with fake timers (no wall-clock races) and cover the production policy: a 30s stall succeeds while alive, a child that exits mid-wait fails at the next check, a live backend that never answers fails at the cap.
134 lines
5.1 KiB
TypeScript
134 lines
5.1 KiB
TypeScript
import assert from 'node:assert/strict'
|
|
import { EventEmitter } from 'node:events'
|
|
|
|
import { test } from 'vitest'
|
|
|
|
import {
|
|
claimDecision,
|
|
createBackendOutputTail,
|
|
formatBackendExitLine,
|
|
isPidOnlyStartMarker,
|
|
pidOnlyStartMarker,
|
|
probeStartMarker,
|
|
processStartMarker
|
|
} from './backend-claim'
|
|
|
|
// --- claimDecision: the #93608 policy ---------------------------------------
|
|
|
|
test('probe success claims with the full start marker (unchanged behavior)', () => {
|
|
const decision = claimDecision(true, { ok: true, startMarker: 'linux:12345' })
|
|
|
|
assert.deepEqual(decision, { action: 'claim', startMarker: 'linux:12345' })
|
|
})
|
|
|
|
test('probe success claims even when the child already exited (ownership records the incarnation)', () => {
|
|
// The claim itself must not invent a failure: the exit handler owns cleanup.
|
|
const decision = claimDecision(false, { ok: true, startMarker: 'win:99' })
|
|
|
|
assert.deepEqual(decision, { action: 'claim', startMarker: 'win:99' })
|
|
})
|
|
|
|
test('probe failure on a LIVE child degrades to PID-only identity — never kills a healthy backend (#93608)', () => {
|
|
const decision = claimDecision(true, { ok: false, reason: 'powershell.exe timed out after 30000ms' })
|
|
|
|
assert.equal(decision.action, 'degrade')
|
|
assert.match((decision as { reason: string }).reason, /timed out/)
|
|
})
|
|
|
|
test('probe failure on a DEAD child fails closed so the caller can attach the stderr tail', () => {
|
|
const decision = claimDecision(false, { ok: false, reason: 'Get-Process: no process with ID 4242' })
|
|
|
|
assert.equal(decision.action, 'fail')
|
|
assert.match((decision as { reason: string }).reason, /4242/)
|
|
})
|
|
|
|
// --- probeStartMarker: throw → value ----------------------------------------
|
|
|
|
test('probeStartMarker converts a probe throw into { ok: false, reason }', async () => {
|
|
const probe = await probeStartMarker(4242, async () => {
|
|
throw new Error('PowerShell 5.1 cold start exceeded budget')
|
|
})
|
|
|
|
assert.deepEqual(probe, { ok: false, reason: 'PowerShell 5.1 cold start exceeded budget' })
|
|
})
|
|
|
|
// --- real probe: drives the actual OS helper (PowerShell on the Windows lane) ---
|
|
|
|
test('processStartMarker resolves a real marker for the current process', async () => {
|
|
const marker = await processStartMarker(process.pid)
|
|
|
|
assert.match(marker, /^(linux|win|winms|ps):.+/)
|
|
})
|
|
|
|
test('a missing PID is classified as ESRCH so reapOrphans can drop the record', async () => {
|
|
// Largest PIDs are bounded well below this on every supported platform.
|
|
// Windows Get-Process / macOS `ps -p` used to surface exit code 1, which
|
|
// the identity matchers treated as "unknown" and kept forever. The native
|
|
// gate throws ESRCH — the errno those catch blocks already map to gone.
|
|
await assert.rejects(processStartMarker(2 ** 30 + 12345), (error: NodeJS.ErrnoException) => error?.code === 'ESRCH')
|
|
})
|
|
|
|
// --- PID-only marker helpers --------------------------------------------------
|
|
|
|
test('pidOnlyStartMarker round-trips through isPidOnlyStartMarker', () => {
|
|
const marker = pidOnlyStartMarker(4242)
|
|
|
|
assert.equal(isPidOnlyStartMarker(marker), true)
|
|
assert.equal(isPidOnlyStartMarker('linux:12345'), false)
|
|
assert.equal(isPidOnlyStartMarker(undefined), false)
|
|
})
|
|
|
|
// --- output tail ring buffer ----------------------------------------------------
|
|
|
|
test('output tail keeps only the most recent bytes once past the limit', () => {
|
|
const tail = createBackendOutputTail(16)
|
|
|
|
tail.append('0123456789')
|
|
tail.append('abcdefghij')
|
|
|
|
assert.equal(tail.text(), '456789abcdefghij')
|
|
assert.equal(tail.text().length, 16)
|
|
})
|
|
|
|
test('output tail interleaves stdout and stderr attached from spawn time', () => {
|
|
const child = { stderr: new EventEmitter(), stdout: new EventEmitter() }
|
|
const tail = createBackendOutputTail(64)
|
|
|
|
tail.attach(child)
|
|
child.stdout.emit('data', Buffer.from('booting\n'))
|
|
child.stderr.emit('data', Buffer.from("ModuleNotFoundError: No module named 'hermes_cli'\n"))
|
|
|
|
assert.match(tail.text(), /booting/)
|
|
assert.match(tail.text(), /ModuleNotFoundError/)
|
|
})
|
|
|
|
test('attach tolerates a child with missing stdio streams', () => {
|
|
const tail = createBackendOutputTail(64)
|
|
|
|
tail.attach({ stderr: null, stdout: null })
|
|
assert.equal(tail.text(), '')
|
|
})
|
|
|
|
// --- formatBackendExitLine ---------------------------------------------------
|
|
|
|
test('exit line carries the buffered tail next to the exit code, preferring the signal', () => {
|
|
const tail = createBackendOutputTail(64)
|
|
tail.append('Traceback (most recent call last):\n')
|
|
|
|
assert.equal(
|
|
formatBackendExitLine('Ignoring stale Hermes backend exit', 1, null, tail),
|
|
'Ignoring stale Hermes backend exit (1)\nRecent backend output:\nTraceback (most recent call last):'
|
|
)
|
|
assert.equal(
|
|
formatBackendExitLine('Hermes backend exited', null, 'SIGTERM', tail),
|
|
'Hermes backend exited (SIGTERM)\nRecent backend output:\nTraceback (most recent call last):'
|
|
)
|
|
})
|
|
|
|
test('exit line stays byte-identical to the legacy shape when the tail is empty or missing', () => {
|
|
assert.equal(
|
|
formatBackendExitLine('Hermes backend exited', 0, null, createBackendOutputTail(64)),
|
|
'Hermes backend exited (0)'
|
|
)
|
|
assert.equal(formatBackendExitLine('Hermes backend exited', 1, null, null), 'Hermes backend exited (1)')
|
|
})
|