mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-17 04:11:35 +00:00
A hard-killed backend leaves its spawn-ledger record and its published
session token behind. Since the attach path started adopting published
tokens (f1247d2e01), such a record passed the token rung and then had its
refused port polled for the whole 45s readiness budget. The renderer's boot
timeout is the same 45s and starts first, so it gave up just before main
fell through to spawning, and every Retry/Repair killed the fresh backend
and produced another dead record: the app never started after an update.
A ledger record is written only after its backend binds, so a refused
connection on one means the process is gone. Readiness gains an opt-in
`alreadyBound` that fails at once on ECONNREFUSED; the attach probe and the
attached-backend liveness monitor set it. Remote/SSH readiness keeps
polling, since a tunnel still coming up refuses legitimately.
569 lines
19 KiB
TypeScript
569 lines
19 KiB
TypeScript
import assert from 'node:assert/strict'
|
|
|
|
import { test } from 'vitest'
|
|
|
|
import {
|
|
DEFAULT_HEALTH_PROBE_TIMEOUT_MS,
|
|
isAuthRejectionError,
|
|
isGatedMissingHealthError,
|
|
isMissingHealthEndpointError,
|
|
isNousCloudAgentUrl,
|
|
isReauthRequiredError,
|
|
isServerSideHttpError,
|
|
makeNousCloudBackendDownError,
|
|
makeUnsignedOauthError,
|
|
waitForHermesReady
|
|
} from './backend-health'
|
|
|
|
const GATE_401 = '401: {"error":"unauthenticated","detail":"Unauthorized","reason":"no_cookie","login_url":"/login"}'
|
|
|
|
test('uses lightweight /api/health for current backends', async () => {
|
|
const calls: string[][] = []
|
|
|
|
await waitForHermesReady('http://127.0.0.1:9000/', {
|
|
token: 'secret-token',
|
|
fetchPublicJson: async url => {
|
|
calls.push(['public', url])
|
|
|
|
return { ok: true }
|
|
},
|
|
fetchJson: async url => {
|
|
calls.push(['token', url])
|
|
throw new Error('status should not be called')
|
|
},
|
|
sleep: async () => {},
|
|
timeoutMs: 100,
|
|
pollMs: 1
|
|
})
|
|
|
|
assert.deepEqual(calls, [['public', 'http://127.0.0.1:9000/api/health']])
|
|
})
|
|
|
|
test('falls back to /api/status only for old backends without /api/health', async () => {
|
|
const calls: string[][] = []
|
|
|
|
await waitForHermesReady('http://127.0.0.1:9000', {
|
|
token: 'secret-token',
|
|
fetchPublicJson: async url => {
|
|
calls.push(['public', url])
|
|
|
|
throw new Error('404: {"detail":"Not Found"}')
|
|
},
|
|
fetchJson: async (url, token) => {
|
|
calls.push(['token', url, token ?? ''])
|
|
|
|
return { version: 'old' }
|
|
},
|
|
sleep: async () => {},
|
|
timeoutMs: 100,
|
|
pollMs: 1
|
|
})
|
|
|
|
assert.deepEqual(calls, [
|
|
['public', 'http://127.0.0.1:9000/api/health'],
|
|
['token', 'http://127.0.0.1:9000/api/status', 'secret-token']
|
|
])
|
|
})
|
|
|
|
test('does not fall back to heavyweight /api/status for transient health failures', async () => {
|
|
const calls: string[][] = []
|
|
let currentTime = 0
|
|
|
|
await assert.rejects(
|
|
waitForHermesReady('http://127.0.0.1:9000', {
|
|
fetchPublicJson: async url => {
|
|
calls.push(['public', url])
|
|
throw new Error('Timed out connecting to Hermes backend after 15000ms')
|
|
},
|
|
fetchJson: async url => {
|
|
calls.push(['token', url])
|
|
},
|
|
sleep: async () => {},
|
|
now: () => {
|
|
currentTime += 20
|
|
|
|
return currentTime
|
|
},
|
|
timeoutMs: 50,
|
|
pollMs: 1
|
|
}),
|
|
/Timed out connecting/
|
|
)
|
|
|
|
assert.ok(calls.length > 0)
|
|
assert.ok(calls.every(call => call[0] === 'public' && call[1].endsWith('/api/health')))
|
|
})
|
|
|
|
test('probes health on a short timeout but leaves the legacy fallback its own', async () => {
|
|
const timeouts: (number | undefined)[] = []
|
|
|
|
await waitForHermesReady('http://127.0.0.1:9000', {
|
|
fetchPublicJson: async (_url, options) => {
|
|
timeouts.push(options?.timeoutMs)
|
|
|
|
throw new Error('404: {"detail":"Not Found"}')
|
|
},
|
|
fetchJson: async (_url, _token, options) => {
|
|
timeouts.push(options?.timeoutMs)
|
|
|
|
return { version: 'old' }
|
|
},
|
|
sleep: async () => {},
|
|
timeoutMs: 100,
|
|
pollMs: 1
|
|
})
|
|
|
|
assert.deepEqual(timeouts, [DEFAULT_HEALTH_PROBE_TIMEOUT_MS, undefined])
|
|
})
|
|
|
|
function connectionRefused(port: number): Error {
|
|
return Object.assign(new Error(`connect ECONNREFUSED 127.0.0.1:${port}`), { code: 'ECONNREFUSED' })
|
|
}
|
|
|
|
async function probesUntilSettled(alreadyBound: boolean): Promise<number> {
|
|
let probes = 0
|
|
let currentTime = 0
|
|
|
|
await waitForHermesReady('http://127.0.0.1:2802', {
|
|
fetchPublicJson: async () => {
|
|
probes += 1
|
|
throw connectionRefused(2802)
|
|
},
|
|
fetchJson: async () => ({}),
|
|
sleep: async () => {},
|
|
now: () => (currentTime += 1),
|
|
timeoutMs: 1_000,
|
|
pollMs: 1,
|
|
alreadyBound
|
|
}).catch(() => undefined)
|
|
|
|
return probes
|
|
}
|
|
|
|
test('a refused port on a backend known to have bound fails at once; an unbound one keeps polling', async () => {
|
|
// A hard-killed backend leaves its ledger record and published token behind.
|
|
// Polling that dead port for the whole budget outlived the renderer's boot
|
|
// timeout and wedged every post-update launch.
|
|
assert.equal(await probesUntilSettled(true), 1)
|
|
assert.ok((await probesUntilSettled(false)) > 1)
|
|
})
|
|
|
|
test('aborts as superseded when the bootstrap signal fires', async () => {
|
|
const controller = new AbortController()
|
|
controller.abort()
|
|
|
|
await assert.rejects(
|
|
waitForHermesReady('http://127.0.0.1:9000', {
|
|
signal: controller.signal,
|
|
fetchPublicJson: async () => {
|
|
throw new Error('should not probe after abort')
|
|
},
|
|
fetchJson: async () => {
|
|
throw new Error('should not probe after abort')
|
|
},
|
|
timeoutMs: 100,
|
|
pollMs: 1
|
|
}),
|
|
(error: any) => error.kind === 'superseded'
|
|
)
|
|
})
|
|
|
|
test('recognizes missing-route shapes only', () => {
|
|
assert.equal(isMissingHealthEndpointError(new Error('404: {"detail":"Not Found"}')), true)
|
|
assert.equal(
|
|
isMissingHealthEndpointError(
|
|
new Error('Expected JSON from /api/health but got HTML. The endpoint is likely missing on the Hermes backend.')
|
|
),
|
|
true
|
|
)
|
|
assert.equal(isMissingHealthEndpointError(new Error('Timed out connecting to Hermes backend after 15000ms')), false)
|
|
assert.equal(isMissingHealthEndpointError(new Error('500: boom')), false)
|
|
})
|
|
|
|
// --- Gated backends that predate /api/health (release 0.19.0 and earlier) ---
|
|
//
|
|
// The dashboard auth gate runs ahead of the SPA catch-all, so on a backend
|
|
// without the route an ANONYMOUS probe is rejected as unauthenticated rather
|
|
// than 404 — verified against a simulated 0.19.0 backend:
|
|
// credential-free: /api/health -> 401 no_cookie, /api/status -> 200
|
|
// credentialed: /api/health -> 404, /api/sessions -> 200
|
|
|
|
test('anonymous gate-shaped 401 falls back to /api/status (backend predates /api/health)', async () => {
|
|
const calls: string[][] = []
|
|
|
|
await waitForHermesReady('http://192.168.1.132:9119', {
|
|
token: null,
|
|
fetchPublicJson: async url => {
|
|
calls.push(['public', url])
|
|
throw new Error(GATE_401)
|
|
},
|
|
fetchJson: async (url, token) => {
|
|
calls.push(['token', url, token == null ? 'null' : token])
|
|
|
|
return { version: '0.19.0', auth_required: true }
|
|
},
|
|
sleep: async () => {},
|
|
timeoutMs: 100,
|
|
pollMs: 1
|
|
})
|
|
|
|
assert.deepEqual(calls, [
|
|
['public', 'http://192.168.1.132:9119/api/health'],
|
|
['token', 'http://192.168.1.132:9119/api/status', 'null']
|
|
])
|
|
})
|
|
|
|
test('a credentialed 401 fails fast for reauth instead of reporting a dead session ready', async () => {
|
|
// The regression a blanket 401->fallback introduces: /api/status is public,
|
|
// so an expired session would answer 200 and boot would report "ready",
|
|
// deferring the no_cookie to the first real API call.
|
|
const calls: string[][] = []
|
|
|
|
await assert.rejects(
|
|
waitForHermesReady('https://gateway.example', {
|
|
token: 'session-token',
|
|
fetchPublicJson: async () => {
|
|
throw new Error('public probe must not be used when credentialed')
|
|
},
|
|
fetchJson: async url => {
|
|
calls.push(['status', url])
|
|
|
|
return { version: '0.19.0' }
|
|
},
|
|
probeHealth: async url => {
|
|
calls.push(['probe', url])
|
|
throw new Error(GATE_401)
|
|
},
|
|
probeIsCredentialed: true,
|
|
sleep: async () => {},
|
|
timeoutMs: 100,
|
|
pollMs: 1
|
|
}),
|
|
(error: any) => {
|
|
assert.equal(isReauthRequiredError(error), true)
|
|
assert.equal(error.needsOauthLogin, true)
|
|
assert.match(error.message, /remote gateway session has expired/i)
|
|
|
|
return true
|
|
}
|
|
)
|
|
|
|
// Fail fast: never reached the public /api/status leg.
|
|
assert.deepEqual(calls, [['probe', 'https://gateway.example/api/health']])
|
|
})
|
|
|
|
test('unsigned OAuth is a terminal reauth failure; a bare needsOauthLogin hint is not', () => {
|
|
// The unsigned-in throw must set isReauthRequired so startHermes latches.
|
|
// A bare needsOauthLogin (the IPC-shaped hint) stays Sign-in copy, not a
|
|
// latch. A CONFIRMED ticket 401/403 is different: the gateway has already
|
|
// tried the AT/RT rotation (cookie) or the desktop has forced one (native)
|
|
// before that rejection reaches gatewayTicketFailure, which tags it
|
|
// isReauthRequired itself (#95701).
|
|
const unsigned = makeUnsignedOauthError() as any
|
|
|
|
assert.equal(unsigned.needsOauthLogin, true)
|
|
assert.equal(unsigned.isReauthRequired, true)
|
|
assert.equal(isReauthRequiredError(unsigned), true)
|
|
assert.match(unsigned.message, /not signed in/i)
|
|
assert.equal(isReauthRequiredError({ needsOauthLogin: true }), false)
|
|
assert.equal(isReauthRequiredError(new Error('Could not reach the remote Hermes gateway')), false)
|
|
})
|
|
|
|
test('a credentialed 403 is also a terminal reauth failure', async () => {
|
|
await assert.rejects(
|
|
waitForHermesReady('https://gateway.example', {
|
|
fetchPublicJson: async () => ({}),
|
|
fetchJson: async () => ({}),
|
|
probeHealth: async () => {
|
|
throw new Error('403: {"detail":"Forbidden"}')
|
|
},
|
|
probeIsCredentialed: true,
|
|
sleep: async () => {},
|
|
timeoutMs: 100,
|
|
pollMs: 1
|
|
}),
|
|
(error: any) => isReauthRequiredError(error)
|
|
)
|
|
})
|
|
|
|
test('a credentialed probe still uses the 404 fallback for a genuinely missing route', async () => {
|
|
// With credentials the gate lets the request through to the SPA catch-all,
|
|
// so an old backend answers a real 404 — that must still fall back, not be
|
|
// mistaken for a rejected session.
|
|
const calls: string[][] = []
|
|
|
|
await waitForHermesReady('https://gateway.example', {
|
|
token: 'session-token',
|
|
fetchPublicJson: async () => {
|
|
throw new Error('public probe must not be used when credentialed')
|
|
},
|
|
fetchJson: async url => {
|
|
calls.push(['status', url])
|
|
|
|
return { version: '0.19.0' }
|
|
},
|
|
probeHealth: async url => {
|
|
calls.push(['probe', url])
|
|
throw new Error('404: {"detail":"Not Found"}')
|
|
},
|
|
probeIsCredentialed: true,
|
|
sleep: async () => {},
|
|
timeoutMs: 100,
|
|
pollMs: 1
|
|
})
|
|
|
|
assert.deepEqual(calls, [
|
|
['probe', 'https://gateway.example/api/health'],
|
|
['status', 'https://gateway.example/api/status']
|
|
])
|
|
})
|
|
|
|
test('a non-gate 401 keeps polling rather than skipping a misconfigured health route', async () => {
|
|
const calls: string[][] = []
|
|
let currentTime = 0
|
|
|
|
await assert.rejects(
|
|
waitForHermesReady('http://127.0.0.1:9000', {
|
|
fetchPublicJson: async url => {
|
|
calls.push(['public', url])
|
|
throw new Error('401: {"detail":"Unauthorized"}')
|
|
},
|
|
fetchJson: async url => {
|
|
calls.push(['token', url])
|
|
},
|
|
sleep: async () => {},
|
|
now: () => {
|
|
currentTime += 20
|
|
|
|
return currentTime
|
|
},
|
|
timeoutMs: 50,
|
|
pollMs: 1
|
|
}),
|
|
/401: \{"detail":"Unauthorized"\}/
|
|
)
|
|
|
|
assert.ok(calls.length > 0)
|
|
assert.ok(calls.every(call => call[0] === 'public' && call[1].endsWith('/api/health')))
|
|
})
|
|
|
|
test('credentialed 5xx and 429 keep polling — only 401/403 are terminal', async () => {
|
|
for (const transient of ['500: boom', '429: {"detail":"Too Many Requests"}']) {
|
|
let attempts = 0
|
|
let currentTime = 0
|
|
|
|
await assert.rejects(
|
|
waitForHermesReady('https://gateway.example', {
|
|
fetchPublicJson: async () => ({}),
|
|
fetchJson: async () => ({}),
|
|
probeHealth: async () => {
|
|
attempts += 1
|
|
throw new Error(transient)
|
|
},
|
|
probeIsCredentialed: true,
|
|
sleep: async () => {},
|
|
now: () => {
|
|
currentTime += 20
|
|
|
|
return currentTime
|
|
},
|
|
timeoutMs: 100,
|
|
pollMs: 1
|
|
}),
|
|
(error: any) => isReauthRequiredError(error) === false
|
|
)
|
|
|
|
assert.ok(attempts > 1, `${transient} should have retried, got ${attempts} attempt(s)`)
|
|
}
|
|
})
|
|
|
|
test('error-shape predicates', () => {
|
|
assert.equal(isGatedMissingHealthError(new Error(GATE_401)), true)
|
|
assert.equal(isGatedMissingHealthError(new Error('401: {"detail":"Unauthorized"}')), false)
|
|
assert.equal(isGatedMissingHealthError(new Error('404: {"detail":"Not Found"}')), false)
|
|
|
|
assert.equal(isAuthRejectionError(new Error(GATE_401)), true)
|
|
assert.equal(isAuthRejectionError(new Error('403: {"detail":"Forbidden"}')), true)
|
|
assert.equal(isAuthRejectionError(new Error('404: {"detail":"Not Found"}')), false)
|
|
assert.equal(isAuthRejectionError(new Error('429: slow down')), false)
|
|
assert.equal(isAuthRejectionError(new Error('500: boom')), false)
|
|
|
|
// A gated 401 must NOT be conflated with a missing route by the 404 predicate.
|
|
assert.equal(isMissingHealthEndpointError(new Error(GATE_401)), false)
|
|
})
|
|
|
|
test('isServerSideHttpError detects 502/503/504', () => {
|
|
// 503 — server-side fault
|
|
const result503 = isServerSideHttpError(new Error('503: Service Unavailable'))
|
|
assert.ok(result503, 'should detect 503')
|
|
assert.equal(result503?.statusCode, 503)
|
|
assert.equal(result503?.detail, '503: Service Unavailable')
|
|
|
|
// 502
|
|
const result502 = isServerSideHttpError(new Error('502: Bad Gateway'))
|
|
assert.ok(result502, 'should detect 502')
|
|
assert.equal(result502?.statusCode, 502)
|
|
|
|
// 504
|
|
const result504 = isServerSideHttpError(new Error('504: Gateway Timeout'))
|
|
assert.ok(result504, 'should detect 504')
|
|
assert.equal(result504?.statusCode, 504)
|
|
|
|
// 500 is NOT a server-side HTTP error per our definition (keeps polling)
|
|
const result500 = isServerSideHttpError(new Error('500: Internal Server Error'))
|
|
assert.equal(result500, null)
|
|
|
|
// 401/403/404/429 are not server-side faults
|
|
assert.equal(isServerSideHttpError(new Error('401: Unauthorized')), null)
|
|
assert.equal(isServerSideHttpError(new Error('403: Forbidden')), null)
|
|
assert.equal(isServerSideHttpError(new Error('404: Not Found')), null)
|
|
assert.equal(isServerSideHttpError(new Error('429: Too Many Requests')), null)
|
|
|
|
// Non-HTTP errors (timeouts, network failures) don't match the pattern
|
|
assert.equal(isServerSideHttpError(new Error('connect ECONNREFUSED')), null)
|
|
assert.equal(isServerSideHttpError(null), null)
|
|
assert.equal(isServerSideHttpError('503: something'), null) // not an Error
|
|
})
|
|
|
|
test('isNousCloudAgentUrl detects cloud agent hosts', () => {
|
|
// Positive cases
|
|
assert.equal(isNousCloudAgentUrl('https://ares-3009.agents.nousresearch.com'), true)
|
|
assert.equal(isNousCloudAgentUrl('https://ares-3009.agents.nousresearch.com/api/health'), true)
|
|
assert.equal(isNousCloudAgentUrl('http://test.agents.nousresearch.com'), true)
|
|
|
|
// Negative cases
|
|
assert.equal(isNousCloudAgentUrl('http://127.0.0.1:9000'), false)
|
|
assert.equal(isNousCloudAgentUrl('https://gateway.example.com'), false)
|
|
assert.equal(isNousCloudAgentUrl('https://nousresearch.com'), false)
|
|
assert.equal(isNousCloudAgentUrl('not-a-url'), false)
|
|
})
|
|
|
|
test('waitForHermesReady classifies a persistent cloud agent 503 as cloud-backend-down', async () => {
|
|
let attempts = 0
|
|
const currentTime = { value: 0 }
|
|
|
|
try {
|
|
await waitForHermesReady('https://ares-3009.agents.nousresearch.com', {
|
|
fetchPublicJson: async () => {
|
|
attempts++
|
|
// Always return 503
|
|
throw new Error('503: Service Unavailable')
|
|
},
|
|
fetchJson: async () => {
|
|
throw new Error('503: Service Unavailable')
|
|
},
|
|
sleep: async () => {},
|
|
// Advance the mock clock per poll — a frozen now() never crosses the
|
|
// deadline and the readiness loop spins forever (hung the whole vitest
|
|
// electron project for 20m in CI).
|
|
now: () => {
|
|
currentTime.value += 20
|
|
|
|
return currentTime.value
|
|
},
|
|
timeoutMs: 100,
|
|
pollMs: 1
|
|
})
|
|
assert.fail('should have thrown')
|
|
} catch (error: any) {
|
|
assert.equal(error.isCloudBackendDown, true)
|
|
assert.equal(error.statusCode, 503)
|
|
assert.ok(attempts > 1, 'should have retried before failing')
|
|
}
|
|
})
|
|
|
|
test('waitForHermesReady does not cloud-wrap non-cloud 503 errors', async () => {
|
|
const currentTime = { value: 0 }
|
|
|
|
try {
|
|
await waitForHermesReady('http://127.0.0.1:9000', {
|
|
fetchPublicJson: async () => {
|
|
throw new Error('503: Service Unavailable')
|
|
},
|
|
fetchJson: async () => {
|
|
throw new Error('503: Service Unavailable')
|
|
},
|
|
sleep: async () => {},
|
|
// Same advancing clock as above — frozen now() = infinite loop.
|
|
now: () => {
|
|
currentTime.value += 20
|
|
|
|
return currentTime.value
|
|
},
|
|
timeoutMs: 100,
|
|
pollMs: 1
|
|
})
|
|
assert.fail('should have thrown')
|
|
} catch (error: any) {
|
|
assert.equal(error.isCloudBackendDown, undefined)
|
|
}
|
|
})
|
|
|
|
test('isServerSideHttpError detects structured statusCode even when the message is opaque', () => {
|
|
const err = new Error('upstream unavailable') as any
|
|
err.statusCode = 503
|
|
const result = isServerSideHttpError(err)
|
|
assert.ok(result)
|
|
assert.equal(result?.statusCode, 503)
|
|
assert.equal(result?.detail, 'upstream unavailable')
|
|
|
|
const err502 = new Error('bad gateway') as any
|
|
err502.statusCode = 502
|
|
assert.equal(isServerSideHttpError(err502)?.statusCode, 502)
|
|
|
|
const err504 = new Error('gateway timeout') as any
|
|
err504.statusCode = 504
|
|
assert.equal(isServerSideHttpError(err504)?.statusCode, 504)
|
|
})
|
|
|
|
test('isServerSideHttpError rejects non-Error inputs even with a 503-shaped value', () => {
|
|
// The structured path requires an actual Error (the fetch layer attaches
|
|
// statusCode to an Error instance); a bare string/null/number must not be
|
|
// misclassified by the legacy prefix fallback.
|
|
assert.equal(isServerSideHttpError('503: something'), null)
|
|
assert.equal(isServerSideHttpError({ statusCode: 503 }), null)
|
|
assert.equal(isServerSideHttpError(null), null)
|
|
assert.equal(isServerSideHttpError(503), null)
|
|
})
|
|
|
|
test('isServerSideHttpError structured path excludes 500/401/403/404/429 even when statusCode is attached', () => {
|
|
for (const code of [500, 401, 403, 404, 429]) {
|
|
const err = new Error(`HTTP ${code}`) as any
|
|
err.statusCode = code
|
|
assert.equal(isServerSideHttpError(err), null, `should reject statusCode ${code}`)
|
|
}
|
|
})
|
|
|
|
test('makeNousCloudBackendDownError produces the Cloud shape and preserves cause', () => {
|
|
const err = new Error('upstream unavailable') as any
|
|
err.statusCode = 503
|
|
const result = makeNousCloudBackendDownError('https://ares-3009.agents.nousresearch.com', err)
|
|
assert.ok(result)
|
|
assert.equal((result as any).isCloudBackendDown, true)
|
|
assert.equal((result as any).statusCode, 503)
|
|
assert.equal((result as any).cause, err)
|
|
})
|
|
|
|
test('makeNousCloudBackendDownError returns null for a Cloud 401 (routes to reauth)', () => {
|
|
const err = new Error('Unauthorized') as any
|
|
err.statusCode = 401
|
|
assert.equal(makeNousCloudBackendDownError('https://ares-3009.agents.nousresearch.com', err), null)
|
|
})
|
|
|
|
test('makeNousCloudBackendDownError returns null for a non-Cloud 503 (generic remote failure)', () => {
|
|
const err = new Error('Service Unavailable') as any
|
|
err.statusCode = 503
|
|
assert.equal(makeNousCloudBackendDownError('https://gateway.example.com', err), null)
|
|
assert.equal(makeNousCloudBackendDownError('http://127.0.0.1:9000', err), null)
|
|
})
|
|
|
|
test('makeNousCloudBackendDownError preserves legacy string-prefix compatibility', () => {
|
|
const result = makeNousCloudBackendDownError(
|
|
'https://ares-3009.agents.nousresearch.com',
|
|
new Error('503: Service Unavailable')
|
|
)
|
|
|
|
assert.ok(result)
|
|
assert.equal((result as any).isCloudBackendDown, true)
|
|
assert.equal((result as any).statusCode, 503)
|
|
})
|