hermes-agent/apps/desktop/electron/backend-start-failure.ts
Austin Pickett a35f600b8a fix(desktop): resolve the Windows ssh client instead of hard-coding System32 OpenSSH
All ssh spawns (the -G config probe, the ssh-config:resolve IPC, SSH
terminals, and SshConnection) now go through one pure resolver:
desktop.ssh_path in config.yaml, then System32 OpenSSH, then Git for
Windows' usr\bin\ssh.exe, then bare ssh. Non-Windows keeps bare ssh.

A failed local ssh -G probe is now tagged terminal: boot latches on the
failure overlay (naming the client and the ssh_path setting) instead of
re-driving the same probe every ~2s.

Fixes #103288
2026-09-29 22:56:15 -04:00

273 lines
12 KiB
TypeScript

/**
* backend-start-failure.ts
*
* Decides whether a failed primary-backend boot should *latch* into
* `backendStartFailure`. A latched failure makes every subsequent
* startHermes() re-throw the cached error without re-attempting the connect —
* the right behavior for a LOCAL backend so the renderer's retry loop can't
* restart a broken install over and over.
*
* It is the WRONG behavior for a REMOTE backend. A remote connect can fail for
* transient reasons — a lapsed OAuth access-token cookie (the gateway rotates a
* fresh one from the live refresh-token cookie on the next request), a
* ws-ticket mint that timed out mid sleep/wake, or a host that was briefly
* unreachable across a laptop sleep. There is no child process whose 'exit'
* handler would clear the cache, so a latched remote failure sticks until the
* whole app is quit and relaunched: reconnect, "Sign out & sign in" (which only
* reloads the renderer), and the wake-recovery revalidate path all keep hitting
* the same stale error. Not latching lets the very next connect re-mint a
* ticket against the (now refreshed) session and self-heal.
*
* Extracted as a dependency-free pure predicate so the invariant is testable
* without booting Electron or reading main.ts source text.
*/
export interface BackendStartFailureContext {
/**
* True when the boot that just failed was resolving/dialing a REMOTE (or
* cloud) primary backend rather than spawning a local child.
*/
attemptedRemote: boolean
/**
* True when the boot that just failed was a supervisor-owned respawn after
* an unexpected primary exit, not an initial or user-driven start.
*/
supervisorRecovery?: boolean
}
/**
* Whether a startHermes() failure should latch into `backendStartFailure`.
* Latch local failures (prevent install-restart loops); never latch remote
* failures (they are transient and must stay retryable so recovery paths work
* without an app restart).
*
* A supervisor-owned respawn never latches either: it already has its own
* bounded crash-loop budget, so a pre-ready child exit must not become a
* permanent local boot latch before that budget can run. Initial and
* user-driven starts keep the fail-closed latch.
*/
export function shouldLatchBackendStartFailure(context: BackendStartFailureContext): boolean {
return !context.attemptedRemote && !context.supervisorRecovery
}
export interface RemoteReauthFailureContext {
/** True when the boot that just failed was dialing a REMOTE (or cloud) backend. */
attemptedRemote: boolean
/**
* True when the failure was a CONFIRMED auth rejection (a credentialed
* probe got 401/403), not a transient connectivity fault.
*/
isReauth: boolean
}
/**
* Whether a failed remote boot should latch as a reauth failure.
*
* This is the deliberate counterpart to `shouldLatchBackendStartFailure`,
* which never latches a remote failure because remote faults are usually
* transient and must stay retryable. A *confirmed* reauth rejection is the
* exception: it cannot self-heal, because nothing will change until the user
* signs in again.
*
* Without a latch, the non-latching remote path actively prevents recovery.
* Every subsequent `getConnection`/`api` call re-runs `startHermes`, re-emits
* `running: true`, and the boot-failure overlay (`visible = Boolean(boot.error)
* && !boot.running`) hides itself — so the "Sign in" button flickers out from
* under the user before they can click it. Latching holds the overlay still
* and clickable. Cleared on every recovery path (reset, repair, apply-config,
* and a confirmed sign-in) so a fresh session boots normally.
*/
export function shouldLatchRemoteReauthFailure(context: RemoteReauthFailureContext): boolean {
return context.attemptedRemote && context.isReauth
}
export interface RemoteBootRetryContext {
/** True when the boot that just failed was dialing a REMOTE (or cloud/SSH) backend. */
attemptedRemote: boolean
/**
* True when the failure was a CONFIRMED auth rejection (401/403), which can
* never self-heal without the user signing in again.
*/
isReauth: boolean
/**
* True when SSH refused to connect because the host's key CHANGED
* (StrictHostKeyChecking fails closed). Retrying cannot succeed until the
* user verifies the change and removes the stale known_hosts entry, so this
* is terminal like a reauth rejection — not connectivity.
*/
isHostKeyChanged?: boolean
/**
* True when SSH rejected the credentials (`auth-failed`). Desktop runs ssh
* in BatchMode, so nothing changes until the user loads the key into
* ssh-agent or fixes the connection settings: terminal, not connectivity.
*/
isSshAuthFailed?: boolean
/**
* True when the LOCAL ssh client failed (`ssh -G` could not run or exited
* non-zero). `-G` never touches the network, so a retry re-runs the same
* doomed probe: terminal until the user fixes the client or sets
* `desktop.ssh_path` (#103288).
*/
isSshClientFailed?: boolean
}
/**
* A host-key-change refusal is identifiable both by the `kind` tag
* classifySshError puts on the error and — for errors that crossed a
* stringifying boundary — by the stable phrases ssh/our own message carry.
* One user hit 157 consecutive boot-retry failures over 2.5h against a
* reinstalled VPS (Aug 2026 bundle) because this was classified as transient.
*/
export function isHostKeyChangedBootFailure(error: unknown): boolean {
if ((error as { kind?: string } | null | undefined)?.kind === 'host-key-changed') {
return true
}
const message = error instanceof Error ? error.message : String(error ?? '')
return /REMOTE HOST IDENTIFICATION HAS CHANGED|Host key verification failed|host key for .+ has CHANGED/i.test(
message
)
}
/**
* An SSH credential rejection is identifiable by the `auth-failed` kind that
* classifySshError puts on the error (`kind` from `SshConnection.open`,
* `sshError` once the bootstrap re-wraps a lifecycle failure) and, for errors
* that crossed a stringifying boundary, by our own message or ssh's banner.
*/
export function isSshAuthFailedBootFailure(error: unknown): boolean {
const tagged = error as { kind?: string; sshError?: string } | null | undefined
if (tagged?.kind === 'auth-failed' || tagged?.sshError === 'auth-failed') {
return true
}
const message = error instanceof Error ? error.message : String(error ?? '')
return /SSH authentication to .+ failed|Permission denied \((?:publickey|password|keyboard-interactive)/i.test(
message
)
}
export const SSH_CLIENT_FAILED = 'ssh-client-failed'
/**
* Wrap a failed local `ssh -G` probe as a terminal, tagged boot failure that
* names the client binary. On Windows the message points at
* `desktop.ssh_path`, the only way past a broken in-box OpenSSH (#103288).
*/
export function sshClientFailedError(sshBinary: string, cause: unknown, platform: string = process.platform): Error {
const detail = cause instanceof Error ? cause.message : String(cause ?? '')
const hint =
platform === 'win32'
? " Set desktop.ssh_path in config.yaml to a working ssh.exe (for example Git for Windows' usr\\bin\\ssh.exe) and retry."
: ''
const error = new Error(
`The local SSH client (${sshBinary}) failed to resolve the connection config: ${detail}.${hint}`
) as Error & {
kind: string
}
error.kind = SSH_CLIENT_FAILED
return error
}
/** A failed local ssh client probe, tagged by sshClientFailedError. */
export function isSshClientFailedBootFailure(error: unknown): boolean {
return (error as { kind?: string } | null | undefined)?.kind === SSH_CLIENT_FAILED
}
/**
* Whether a failed remote boot should latch because the local ssh client
* itself failed. Same rationale as the credential latch: unlatched, every
* api call re-drives boot and the overlay never holds still (#103288).
*/
export function shouldLatchSshClientFailure(context: RemoteBootRetryContext): boolean {
return context.attemptedRemote && context.isSshClientFailed === true
}
/**
* Whether a failed remote boot should latch (into `backendStartFailure`)
* because SSH rejected the credentials (#72698). Unlatched, every
* `getConnection`/api call re-runs startHermes, re-emits `running: true` and
* hides the boot-failure overlay, so its Gateway settings button — the only
* way to fix the key — ignores clicks. Released by reset/repair/apply-config
* like the host-key latch.
*/
export function shouldLatchSshAuthFailure(context: RemoteBootRetryContext): boolean {
return context.attemptedRemote && context.isSshAuthFailed === true
}
/**
* Whether a failed remote boot should latch (into `backendStartFailure`)
* because the host key changed. Same rationale as the reauth latch: the
* failure cannot self-heal, and an unlatched terminal failure makes every
* recovery surface re-drive the identical doomed boot. The latch is released
* by the existing reset/repair/apply-config paths once the user has run
* `ssh-keygen -R <host>`.
*/
export function shouldLatchHostKeyChangedFailure(context: RemoteBootRetryContext): boolean {
return context.attemptedRemote && context.isHostKeyChanged === true
}
/**
* Whether a failed primary-backend boot is a TRANSIENT remote failure the
* renderer may retry automatically (bounded, with backoff).
*
* This closes the self-heal gap of issue #82679: a dropped SSH/HTTP remote
* connection surfaces at the next boot as a transient transport failure
* ("Could not verify the existing SSH backend", ERR_CONNECTION_RESET, mint
* timeouts). Those never latch (see shouldLatchBackendStartFailure), but
* nothing ever RE-ATTEMPTED the boot either — the renderer's reconnect loop
* only arms after a completed boot, so the app sat on "Desktop boot failed"
* until the user manually re-entered the same connection details (which just
* forced a fresh bootstrap). A missing capability differs from a transient
* failure: confirmed reauth rejections, host-key changes, SSH credential
* rejections, and local failures stay out of the retry path; everything else remote is connectivity and
* should retry.
*/
export function isRetryableRemoteBootFailure(context: RemoteBootRetryContext): boolean {
return (
context.attemptedRemote &&
!context.isReauth &&
context.isHostKeyChanged !== true &&
context.isSshAuthFailed !== true &&
context.isSshClientFailed !== true
)
}
export interface BootProgressUpdateLike {
/** The failure text an error update carries; absent/null on progress updates. */
error?: unknown
}
/**
* Whether a boot-progress update must be dropped because a CONFIRMED remote
* reauth rejection is latched.
*
* `latchedMessage` is the message of the latched reauth failure (null when
* nothing is latched). While it is set, the recovery overlay with its Sign in
* button is the only correct surface until the user signs in, and any update
* that is not a re-emit of that same failure would lift it: a `running: true`
* phase or a cleared error from an attempt that was already in flight when the
* latch closed, or an unrelated failure from a sibling caller (which would
* also swap the retryable verdict back to true and re-arm the renderer's
* boot-retry loop — the flicker of #95701). Re-emits of the latched failure
* pass so the non-retryable verdict is never lost. Every recovery path
* (reset, repair, apply-config, confirmed sign-in) clears the latch BEFORE it
* re-drives boot, so this never holds a legitimate restart.
*/
export function shouldHoldBootProgressForReauth(
latchedMessage: string | null | undefined,
update: BootProgressUpdateLike
): boolean {
if (!latchedMessage) {
return false
}
return update.error !== latchedMessage
}