hermes-agent/apps/desktop/electron/backend-start-failure.test.ts
Austin Pickett a35f600b8a fix(desktop): resolve the Windows ssh client instead of hard-coding System32 OpenSSH
All ssh spawns (the -G config probe, the ssh-config:resolve IPC, SSH
terminals, and SshConnection) now go through one pure resolver:
desktop.ssh_path in config.yaml, then System32 OpenSSH, then Git for
Windows' usr\bin\ssh.exe, then bare ssh. Non-Windows keeps bare ssh.

A failed local ssh -G probe is now tagged terminal: boot latches on the
failure overlay (naming the client and the ssh_path setting) instead of
re-driving the same probe every ~2s.

Fixes #103288
2026-09-29 22:56:15 -04:00

255 lines
12 KiB
TypeScript

import assert from 'node:assert/strict'
import { test } from 'vitest'
import { isReauthRequiredError, makeUnsignedOauthError } from './backend-health'
import {
isHostKeyChangedBootFailure,
isRetryableRemoteBootFailure,
isSshAuthFailedBootFailure,
isSshClientFailedBootFailure,
shouldHoldBootProgressForReauth,
shouldLatchBackendStartFailure,
shouldLatchHostKeyChangedFailure,
shouldLatchRemoteReauthFailure,
shouldLatchSshAuthFailure,
shouldLatchSshClientFailure,
sshClientFailedError
} from './backend-start-failure'
import { SshConnection } from './ssh-connection'
test('latches a LOCAL backend failure so the install-retry loop is broken', () => {
assert.equal(shouldLatchBackendStartFailure({ attemptedRemote: false }), true)
})
test('never latches a REMOTE failure so recovery stays retryable without a restart', () => {
// A lapsed OAuth session / mint timeout / host briefly unreachable across a
// laptop sleep must not wedge the app: the next connect has to re-attempt and
// re-mint against the refreshed session.
assert.equal(shouldLatchBackendStartFailure({ attemptedRemote: true }), false)
})
test('never latches a supervisor-owned respawn failure (it has its own bounded crash-loop budget)', () => {
// A pre-ready child exit during a supervisor respawn must be able to spend
// the remaining crash-loop slots instead of becoming a permanent local latch.
assert.equal(shouldLatchBackendStartFailure({ attemptedRemote: false, supervisorRecovery: true }), false)
assert.equal(shouldLatchBackendStartFailure({ attemptedRemote: false, supervisorRecovery: false }), true)
})
test('latches a CONFIRMED remote reauth failure so the overlay stays clickable', () => {
// Without this the non-latching remote path re-runs startHermes on every
// getConnection/api call, re-emits running:true, and the overlay hides
// itself — the "Sign in" button flickers away before it can be clicked.
assert.equal(shouldLatchRemoteReauthFailure({ attemptedRemote: true, isReauth: true }), true)
})
test('does not latch a transient remote failure as reauth', () => {
// A mint timeout or a host unreachable across sleep must still self-heal.
assert.equal(shouldLatchRemoteReauthFailure({ attemptedRemote: true, isReauth: false }), false)
})
test('never latches a LOCAL failure as reauth (that is backendStartFailure job)', () => {
assert.equal(shouldLatchRemoteReauthFailure({ attemptedRemote: false, isReauth: true }), false)
assert.equal(shouldLatchRemoteReauthFailure({ attemptedRemote: false, isReauth: false }), false)
})
test('the two latches never fire for the same failure', () => {
// They are complementary, not overlapping: local failures latch via
// backendStartFailure, confirmed remote reauth latches via its own flag.
for (const attemptedRemote of [true, false]) {
for (const isReauth of [true, false]) {
const start = shouldLatchBackendStartFailure({ attemptedRemote })
const reauth = shouldLatchRemoteReauthFailure({ attemptedRemote, isReauth })
assert.ok(!(start && reauth), `both latched for remote=${attemptedRemote} reauth=${isReauth}`)
}
}
})
test('FIX #82679: a transient remote failure is retryable so a dropped SSH/HTTP connection self-heals', () => {
// The dropped-registered-connection class: "Could not verify the existing
// SSH backend", ERR_CONNECTION_RESET on an HTTP remote, mint timeouts. All
// surface as non-reauth remote boot failures and must enter the bounded
// renderer retry loop instead of parking on "Desktop boot failed".
assert.equal(isRetryableRemoteBootFailure({ attemptedRemote: true, isReauth: false }), true)
})
test('a CONFIRMED reauth rejection is never auto-retried (missing capability, not transient failure)', () => {
assert.equal(isRetryableRemoteBootFailure({ attemptedRemote: true, isReauth: true }), false)
})
test('unsigned OAuth latches and is never auto-retried; a bare needsOauthLogin hint still retries', () => {
// Production composition in startHermes: isReauth = isReauthRequiredError(error).
// A bare `{ needsOauthLogin: true }` is the IPC-shaped hint, not a confirmed
// rejection; gatewayTicketFailure tags a confirmed 401/403 with
// isReauthRequired itself (#95701, see remote-reauth-latch.test.ts).
const unsigned = isReauthRequiredError(makeUnsignedOauthError())
const ticketHint = isReauthRequiredError({ needsOauthLogin: true })
assert.equal(unsigned, true)
assert.equal(shouldLatchRemoteReauthFailure({ attemptedRemote: true, isReauth: unsigned }), true)
assert.equal(isRetryableRemoteBootFailure({ attemptedRemote: true, isReauth: unsigned }), false)
assert.equal(ticketHint, false)
assert.equal(shouldLatchRemoteReauthFailure({ attemptedRemote: true, isReauth: ticketHint }), false)
assert.equal(isRetryableRemoteBootFailure({ attemptedRemote: true, isReauth: ticketHint }), true)
})
test('local failures are never auto-retried by the remote self-heal loop', () => {
assert.equal(isRetryableRemoteBootFailure({ attemptedRemote: false, isReauth: false }), false)
assert.equal(isRetryableRemoteBootFailure({ attemptedRemote: false, isReauth: true }), false)
})
test('retryable and reauth-latch are mutually exclusive for remote failures', () => {
// Every remote failure either self-heals (transient) or latches for sign-in
// (confirmed reauth) — never both, never neither.
for (const isReauth of [true, false]) {
const retry = isRetryableRemoteBootFailure({ attemptedRemote: true, isReauth })
const latch = shouldLatchRemoteReauthFailure({ attemptedRemote: true, isReauth })
assert.equal(retry !== latch, true, `remote failure with reauth=${isReauth} must pick exactly one path`)
}
})
test('FIX host-key change: classified from the kind tag and from stringified ssh banners', () => {
// classifySshError tags the Error it built; errors that crossed an IPC or
// string boundary only keep the message. Both shapes must classify.
const tagged = Object.assign(new Error('SSH refused to connect.'), { kind: 'host-key-changed' })
assert.equal(isHostKeyChangedBootFailure(tagged), true)
assert.equal(
isHostKeyChangedBootFailure(new Error('@@@@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @@@@')),
true
)
assert.equal(isHostKeyChangedBootFailure(new Error('Host key verification failed.')), true)
assert.equal(
isHostKeyChangedBootFailure(new Error('The host key for [email protected] has CHANGED since you last connected.')),
true
)
assert.equal(isHostKeyChangedBootFailure(new Error('Connection refused')), false)
assert.equal(isHostKeyChangedBootFailure(null), false)
})
test('FIX host-key change: latches and is never auto-retried (157-failure loop, Aug 2026 bundle)', () => {
// SSH fails closed on a changed host key: every retry re-drives the same
// doomed boot until the user clears known_hosts. Terminal, like reauth.
const context = { attemptedRemote: true, isReauth: false, isHostKeyChanged: true }
assert.equal(shouldLatchHostKeyChangedFailure(context), true)
assert.equal(isRetryableRemoteBootFailure(context), false)
})
test('host-key latch never fires for local failures or ordinary remote faults', () => {
assert.equal(
shouldLatchHostKeyChangedFailure({ attemptedRemote: false, isReauth: false, isHostKeyChanged: true }),
false
)
assert.equal(
shouldLatchHostKeyChangedFailure({ attemptedRemote: true, isReauth: false, isHostKeyChanged: false }),
false
)
assert.equal(shouldLatchHostKeyChangedFailure({ attemptedRemote: true, isReauth: false }), false)
})
test('every remote failure picks exactly one path: retry, reauth latch, or host-key latch', () => {
for (const isReauth of [true, false]) {
for (const isHostKeyChanged of [true, false]) {
const retry = isRetryableRemoteBootFailure({ attemptedRemote: true, isReauth, isHostKeyChanged })
const reauth = shouldLatchRemoteReauthFailure({ attemptedRemote: true, isReauth })
const hostKey = shouldLatchHostKeyChangedFailure({ attemptedRemote: true, isReauth, isHostKeyChanged })
const picked = [retry, reauth, hostKey].filter(Boolean).length
assert.ok(picked >= 1, `remote failure reauth=${isReauth} hostKey=${isHostKeyChanged} fell through every path`)
}
}
})
test('FIX #72698: a rejected SSH key latches the boot failure and is never auto-retried', () => {
// The error exactly as SshConnection.open() rejects it, and as the SSH
// bootstrap re-wraps a lifecycle failure (message + sshError tag).
const fromOpen = new SshConnection({ host: '127.0.0.1', user: 'me' }, {})._fail(
'[email protected]: Permission denied (publickey,password,keyboard-interactive).'
)
const rewrapped = Object.assign(new Error(fromOpen.message), { sshError: fromOpen.kind, isSshBootstrap: true })
for (const error of [fromOpen, rewrapped, new Error(fromOpen.message)]) {
const isSshAuthFailed = isSshAuthFailedBootFailure(error)
const context = { attemptedRemote: true, isReauth: false, isHostKeyChanged: false, isSshAuthFailed }
assert.equal(shouldLatchSshAuthFailure(context), true)
assert.equal(isRetryableRemoteBootFailure(context), false)
}
// Connectivity faults keep self-healing; local boots use the local latch.
const unreachable = new SshConnection({ host: '127.0.0.1', user: 'me' }, {})._fail(
'ssh: connect to host 127.0.0.1 port 22: Connection refused'
)
const transient = {
attemptedRemote: true,
isReauth: false,
isSshAuthFailed: isSshAuthFailedBootFailure(unreachable)
}
assert.equal(shouldLatchSshAuthFailure(transient), false)
assert.equal(isRetryableRemoteBootFailure(transient), true)
assert.equal(shouldLatchSshAuthFailure({ attemptedRemote: false, isReauth: false, isSshAuthFailed: true }), false)
// A remote lifecycle's filesystem "Permission denied" is not a credential rejection.
assert.equal(isSshAuthFailedBootFailure(new Error('mkdir: /opt/hermes: Permission denied')), false)
})
test('FIX #95701: while a reauth rejection is latched, only re-emits of that failure reach the renderer', () => {
const latched = 'Your remote gateway session has expired. Sign in again.'
// The latched failure's own (re-)emit passes — it carries retryable:false.
assert.equal(shouldHoldBootProgressForReauth(latched, { error: latched }), false)
// Anything that would lift the overlay is held: a running phase from an
// attempt already in flight when the latch closed, a cleared error, or a
// sibling failure that would flip retryable back on.
assert.equal(shouldHoldBootProgressForReauth(latched, { error: null }), true)
assert.equal(shouldHoldBootProgressForReauth(latched, {}), true)
assert.equal(shouldHoldBootProgressForReauth(latched, { error: 'Could not reach the remote Hermes gateway' }), true)
})
test('FIX #95701: with no reauth latch every boot-progress update flows as before', () => {
for (const latch of [null, undefined, '']) {
assert.equal(shouldHoldBootProgressForReauth(latch, { error: null }), false)
assert.equal(shouldHoldBootProgressForReauth(latch, {}), false)
assert.equal(shouldHoldBootProgressForReauth(latch, { error: 'Desktop boot failed: spawn ENOENT' }), false)
}
})
test('FIX #103288: a failed local `ssh -G` probe latches and is never auto-retried', () => {
const error = sshClientFailedError(
'C:\\Windows\\System32\\OpenSSH\\ssh.exe',
new Error('Command failed: ssh.exe -G -- box'),
'win32'
)
assert.equal(isSshClientFailedBootFailure(error), true)
assert.match(error.message, /System32\\OpenSSH\\ssh\.exe/)
assert.match(error.message, /Command failed/)
assert.match(error.message, /desktop\.ssh_path/)
const context = { attemptedRemote: true, isReauth: false, isSshClientFailed: true }
assert.equal(shouldLatchSshClientFailure(context), true)
assert.equal(isRetryableRemoteBootFailure(context), false)
// Ordinary remote faults keep retrying; local boots never take this latch.
const unreachable = new Error('ssh: connect to host box port 22: Connection timed out')
assert.equal(isSshClientFailedBootFailure(unreachable), false)
assert.equal(
isRetryableRemoteBootFailure({
attemptedRemote: true,
isReauth: false,
isSshClientFailed: isSshClientFailedBootFailure(unreachable)
}),
true
)
assert.equal(shouldLatchSshClientFailure({ attemptedRemote: false, isReauth: false, isSshClientFailed: true }), false)
})
test('FIX #103288: the desktop.ssh_path hint is Windows-only', () => {
const error = sshClientFailedError('ssh', new Error('boom'), 'darwin')
assert.equal(isSshClientFailedBootFailure(error), true)
assert.doesNotMatch(error.message, /ssh_path/)
})