mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-17 04:11:35 +00:00
All ssh spawns (the -G config probe, the ssh-config:resolve IPC, SSH terminals, and SshConnection) now go through one pure resolver: desktop.ssh_path in config.yaml, then System32 OpenSSH, then Git for Windows' usr\bin\ssh.exe, then bare ssh. Non-Windows keeps bare ssh. A failed local ssh -G probe is now tagged terminal: boot latches on the failure overlay (naming the client and the ssh_path setting) instead of re-driving the same probe every ~2s. Fixes #103288
273 lines
12 KiB
TypeScript
273 lines
12 KiB
TypeScript
/**
|
|
* backend-start-failure.ts
|
|
*
|
|
* Decides whether a failed primary-backend boot should *latch* into
|
|
* `backendStartFailure`. A latched failure makes every subsequent
|
|
* startHermes() re-throw the cached error without re-attempting the connect —
|
|
* the right behavior for a LOCAL backend so the renderer's retry loop can't
|
|
* restart a broken install over and over.
|
|
*
|
|
* It is the WRONG behavior for a REMOTE backend. A remote connect can fail for
|
|
* transient reasons — a lapsed OAuth access-token cookie (the gateway rotates a
|
|
* fresh one from the live refresh-token cookie on the next request), a
|
|
* ws-ticket mint that timed out mid sleep/wake, or a host that was briefly
|
|
* unreachable across a laptop sleep. There is no child process whose 'exit'
|
|
* handler would clear the cache, so a latched remote failure sticks until the
|
|
* whole app is quit and relaunched: reconnect, "Sign out & sign in" (which only
|
|
* reloads the renderer), and the wake-recovery revalidate path all keep hitting
|
|
* the same stale error. Not latching lets the very next connect re-mint a
|
|
* ticket against the (now refreshed) session and self-heal.
|
|
*
|
|
* Extracted as a dependency-free pure predicate so the invariant is testable
|
|
* without booting Electron or reading main.ts source text.
|
|
*/
|
|
|
|
export interface BackendStartFailureContext {
|
|
/**
|
|
* True when the boot that just failed was resolving/dialing a REMOTE (or
|
|
* cloud) primary backend rather than spawning a local child.
|
|
*/
|
|
attemptedRemote: boolean
|
|
/**
|
|
* True when the boot that just failed was a supervisor-owned respawn after
|
|
* an unexpected primary exit, not an initial or user-driven start.
|
|
*/
|
|
supervisorRecovery?: boolean
|
|
}
|
|
|
|
/**
|
|
* Whether a startHermes() failure should latch into `backendStartFailure`.
|
|
* Latch local failures (prevent install-restart loops); never latch remote
|
|
* failures (they are transient and must stay retryable so recovery paths work
|
|
* without an app restart).
|
|
*
|
|
* A supervisor-owned respawn never latches either: it already has its own
|
|
* bounded crash-loop budget, so a pre-ready child exit must not become a
|
|
* permanent local boot latch before that budget can run. Initial and
|
|
* user-driven starts keep the fail-closed latch.
|
|
*/
|
|
export function shouldLatchBackendStartFailure(context: BackendStartFailureContext): boolean {
|
|
return !context.attemptedRemote && !context.supervisorRecovery
|
|
}
|
|
|
|
export interface RemoteReauthFailureContext {
|
|
/** True when the boot that just failed was dialing a REMOTE (or cloud) backend. */
|
|
attemptedRemote: boolean
|
|
/**
|
|
* True when the failure was a CONFIRMED auth rejection (a credentialed
|
|
* probe got 401/403), not a transient connectivity fault.
|
|
*/
|
|
isReauth: boolean
|
|
}
|
|
|
|
/**
|
|
* Whether a failed remote boot should latch as a reauth failure.
|
|
*
|
|
* This is the deliberate counterpart to `shouldLatchBackendStartFailure`,
|
|
* which never latches a remote failure because remote faults are usually
|
|
* transient and must stay retryable. A *confirmed* reauth rejection is the
|
|
* exception: it cannot self-heal, because nothing will change until the user
|
|
* signs in again.
|
|
*
|
|
* Without a latch, the non-latching remote path actively prevents recovery.
|
|
* Every subsequent `getConnection`/`api` call re-runs `startHermes`, re-emits
|
|
* `running: true`, and the boot-failure overlay (`visible = Boolean(boot.error)
|
|
* && !boot.running`) hides itself — so the "Sign in" button flickers out from
|
|
* under the user before they can click it. Latching holds the overlay still
|
|
* and clickable. Cleared on every recovery path (reset, repair, apply-config,
|
|
* and a confirmed sign-in) so a fresh session boots normally.
|
|
*/
|
|
export function shouldLatchRemoteReauthFailure(context: RemoteReauthFailureContext): boolean {
|
|
return context.attemptedRemote && context.isReauth
|
|
}
|
|
|
|
export interface RemoteBootRetryContext {
|
|
/** True when the boot that just failed was dialing a REMOTE (or cloud/SSH) backend. */
|
|
attemptedRemote: boolean
|
|
/**
|
|
* True when the failure was a CONFIRMED auth rejection (401/403), which can
|
|
* never self-heal without the user signing in again.
|
|
*/
|
|
isReauth: boolean
|
|
/**
|
|
* True when SSH refused to connect because the host's key CHANGED
|
|
* (StrictHostKeyChecking fails closed). Retrying cannot succeed until the
|
|
* user verifies the change and removes the stale known_hosts entry, so this
|
|
* is terminal like a reauth rejection — not connectivity.
|
|
*/
|
|
isHostKeyChanged?: boolean
|
|
/**
|
|
* True when SSH rejected the credentials (`auth-failed`). Desktop runs ssh
|
|
* in BatchMode, so nothing changes until the user loads the key into
|
|
* ssh-agent or fixes the connection settings: terminal, not connectivity.
|
|
*/
|
|
isSshAuthFailed?: boolean
|
|
/**
|
|
* True when the LOCAL ssh client failed (`ssh -G` could not run or exited
|
|
* non-zero). `-G` never touches the network, so a retry re-runs the same
|
|
* doomed probe: terminal until the user fixes the client or sets
|
|
* `desktop.ssh_path` (#103288).
|
|
*/
|
|
isSshClientFailed?: boolean
|
|
}
|
|
|
|
/**
|
|
* A host-key-change refusal is identifiable both by the `kind` tag
|
|
* classifySshError puts on the error and — for errors that crossed a
|
|
* stringifying boundary — by the stable phrases ssh/our own message carry.
|
|
* One user hit 157 consecutive boot-retry failures over 2.5h against a
|
|
* reinstalled VPS (Aug 2026 bundle) because this was classified as transient.
|
|
*/
|
|
export function isHostKeyChangedBootFailure(error: unknown): boolean {
|
|
if ((error as { kind?: string } | null | undefined)?.kind === 'host-key-changed') {
|
|
return true
|
|
}
|
|
|
|
const message = error instanceof Error ? error.message : String(error ?? '')
|
|
|
|
return /REMOTE HOST IDENTIFICATION HAS CHANGED|Host key verification failed|host key for .+ has CHANGED/i.test(
|
|
message
|
|
)
|
|
}
|
|
|
|
/**
|
|
* An SSH credential rejection is identifiable by the `auth-failed` kind that
|
|
* classifySshError puts on the error (`kind` from `SshConnection.open`,
|
|
* `sshError` once the bootstrap re-wraps a lifecycle failure) and, for errors
|
|
* that crossed a stringifying boundary, by our own message or ssh's banner.
|
|
*/
|
|
export function isSshAuthFailedBootFailure(error: unknown): boolean {
|
|
const tagged = error as { kind?: string; sshError?: string } | null | undefined
|
|
|
|
if (tagged?.kind === 'auth-failed' || tagged?.sshError === 'auth-failed') {
|
|
return true
|
|
}
|
|
|
|
const message = error instanceof Error ? error.message : String(error ?? '')
|
|
|
|
return /SSH authentication to .+ failed|Permission denied \((?:publickey|password|keyboard-interactive)/i.test(
|
|
message
|
|
)
|
|
}
|
|
|
|
export const SSH_CLIENT_FAILED = 'ssh-client-failed'
|
|
|
|
/**
|
|
* Wrap a failed local `ssh -G` probe as a terminal, tagged boot failure that
|
|
* names the client binary. On Windows the message points at
|
|
* `desktop.ssh_path`, the only way past a broken in-box OpenSSH (#103288).
|
|
*/
|
|
export function sshClientFailedError(sshBinary: string, cause: unknown, platform: string = process.platform): Error {
|
|
const detail = cause instanceof Error ? cause.message : String(cause ?? '')
|
|
|
|
const hint =
|
|
platform === 'win32'
|
|
? " Set desktop.ssh_path in config.yaml to a working ssh.exe (for example Git for Windows' usr\\bin\\ssh.exe) and retry."
|
|
: ''
|
|
|
|
const error = new Error(
|
|
`The local SSH client (${sshBinary}) failed to resolve the connection config: ${detail}.${hint}`
|
|
) as Error & {
|
|
kind: string
|
|
}
|
|
|
|
error.kind = SSH_CLIENT_FAILED
|
|
|
|
return error
|
|
}
|
|
|
|
/** A failed local ssh client probe, tagged by sshClientFailedError. */
|
|
export function isSshClientFailedBootFailure(error: unknown): boolean {
|
|
return (error as { kind?: string } | null | undefined)?.kind === SSH_CLIENT_FAILED
|
|
}
|
|
|
|
/**
|
|
* Whether a failed remote boot should latch because the local ssh client
|
|
* itself failed. Same rationale as the credential latch: unlatched, every
|
|
* api call re-drives boot and the overlay never holds still (#103288).
|
|
*/
|
|
export function shouldLatchSshClientFailure(context: RemoteBootRetryContext): boolean {
|
|
return context.attemptedRemote && context.isSshClientFailed === true
|
|
}
|
|
|
|
/**
|
|
* Whether a failed remote boot should latch (into `backendStartFailure`)
|
|
* because SSH rejected the credentials (#72698). Unlatched, every
|
|
* `getConnection`/api call re-runs startHermes, re-emits `running: true` and
|
|
* hides the boot-failure overlay, so its Gateway settings button — the only
|
|
* way to fix the key — ignores clicks. Released by reset/repair/apply-config
|
|
* like the host-key latch.
|
|
*/
|
|
export function shouldLatchSshAuthFailure(context: RemoteBootRetryContext): boolean {
|
|
return context.attemptedRemote && context.isSshAuthFailed === true
|
|
}
|
|
|
|
/**
|
|
* Whether a failed remote boot should latch (into `backendStartFailure`)
|
|
* because the host key changed. Same rationale as the reauth latch: the
|
|
* failure cannot self-heal, and an unlatched terminal failure makes every
|
|
* recovery surface re-drive the identical doomed boot. The latch is released
|
|
* by the existing reset/repair/apply-config paths once the user has run
|
|
* `ssh-keygen -R <host>`.
|
|
*/
|
|
export function shouldLatchHostKeyChangedFailure(context: RemoteBootRetryContext): boolean {
|
|
return context.attemptedRemote && context.isHostKeyChanged === true
|
|
}
|
|
|
|
/**
|
|
* Whether a failed primary-backend boot is a TRANSIENT remote failure the
|
|
* renderer may retry automatically (bounded, with backoff).
|
|
*
|
|
* This closes the self-heal gap of issue #82679: a dropped SSH/HTTP remote
|
|
* connection surfaces at the next boot as a transient transport failure
|
|
* ("Could not verify the existing SSH backend", ERR_CONNECTION_RESET, mint
|
|
* timeouts). Those never latch (see shouldLatchBackendStartFailure), but
|
|
* nothing ever RE-ATTEMPTED the boot either — the renderer's reconnect loop
|
|
* only arms after a completed boot, so the app sat on "Desktop boot failed"
|
|
* until the user manually re-entered the same connection details (which just
|
|
* forced a fresh bootstrap). A missing capability differs from a transient
|
|
* failure: confirmed reauth rejections, host-key changes, SSH credential
|
|
* rejections, and local failures stay out of the retry path; everything else remote is connectivity and
|
|
* should retry.
|
|
*/
|
|
export function isRetryableRemoteBootFailure(context: RemoteBootRetryContext): boolean {
|
|
return (
|
|
context.attemptedRemote &&
|
|
!context.isReauth &&
|
|
context.isHostKeyChanged !== true &&
|
|
context.isSshAuthFailed !== true &&
|
|
context.isSshClientFailed !== true
|
|
)
|
|
}
|
|
|
|
export interface BootProgressUpdateLike {
|
|
/** The failure text an error update carries; absent/null on progress updates. */
|
|
error?: unknown
|
|
}
|
|
|
|
/**
|
|
* Whether a boot-progress update must be dropped because a CONFIRMED remote
|
|
* reauth rejection is latched.
|
|
*
|
|
* `latchedMessage` is the message of the latched reauth failure (null when
|
|
* nothing is latched). While it is set, the recovery overlay with its Sign in
|
|
* button is the only correct surface until the user signs in, and any update
|
|
* that is not a re-emit of that same failure would lift it: a `running: true`
|
|
* phase or a cleared error from an attempt that was already in flight when the
|
|
* latch closed, or an unrelated failure from a sibling caller (which would
|
|
* also swap the retryable verdict back to true and re-arm the renderer's
|
|
* boot-retry loop — the flicker of #95701). Re-emits of the latched failure
|
|
* pass so the non-retryable verdict is never lost. Every recovery path
|
|
* (reset, repair, apply-config, confirmed sign-in) clears the latch BEFORE it
|
|
* re-drives boot, so this never holds a legitimate restart.
|
|
*/
|
|
export function shouldHoldBootProgressForReauth(
|
|
latchedMessage: string | null | undefined,
|
|
update: BootProgressUpdateLike
|
|
): boolean {
|
|
if (!latchedMessage) {
|
|
return false
|
|
}
|
|
|
|
return update.error !== latchedMessage
|
|
}
|