hermes-agent/apps/desktop/electron/update-marker.ts
Brooklyn Nicholson 0590ab25fe fix(marker): zombie-aware stale-marker self-heal parity across all readers
Rust live_marker_owner now REMOVES a stale marker (dead pid, past ceiling,
unparseable) on read — previously stale bytes were only ignored, so a crashed
updater whose Drop never ran refused every later acquire until the 20-minute
ceiling (#77259). pid_is_alive gains pid-0 and zombie handling (Linux /proc
state, macOS ps stat), closing the kill(pid,0) false-positive that kept a
dead-but-unreaped updater 'live'.

Python _pid_is_running gains the same zombie awareness via a new
_process_state probe (Linux /proc, macOS ps), which read_live_update inherits
through update_lock._pid_alive — so the CLI gate self-heals a zombie-owned
marker in seconds instead of 20 minutes (#120635, #125932).

Electron readLiveUpdateMarker consults a new posixProcessState probe
(injectable, fail-open to the signal-0 verdict) so the desktop boot gate no
longer parks on a zombie-owned marker.

Rust fix cherry-picked from #77885 (authorship kept); Python and Electron
are new parity work.
2026-09-29 20:57:17 -05:00

261 lines
9.2 KiB
TypeScript

/**
* In-app update mutual-exclusion marker (#50238).
*
* The Tauri updater writes HERMES_HOME/.hermes-update-in-progress for the whole
* duration of an `--update` run (see apps/bootstrap-installer/src-tauri/src/
* update.rs `UpdateMarkerGuard`). The marker body is two lines: the updater's
* pid and the unix-seconds it started.
*
* Why: if the user relaunches the desktop mid-update — the window vanished with
* no progress and looks crashed — a fresh instance must NOT spawn its own local
* backend. That backend re-locks the venv shim, the updater's straggler cleanup
* (`force_kill_other_hermes`, taskkill /IM hermes.exe) kills it, the launch
* fails with the 45s "backend didn't come up" timeout, and the user relaunches
* into the same trap — an infinite respawn/kill loop. The desktop gates local
* backend startup on this marker and parks until the update finishes.
*
* This module holds the PURE, side-effect-light logic (path, pid liveness,
* parse + staleness) so it is unit-testable without booting Electron. The
* polling/boot-progress wrapper lives in main.ts where the boot-progress and
* log sinks are.
*/
import fs from 'fs'
import { execFileSync } from 'node:child_process'
import path from 'path'
// Even with a live-looking PID, never treat a marker older than this as a live
// update. A full update (git pull + pip + desktop rebuild) is minutes, not tens
// of minutes; past this the marker is almost certainly stale (e.g. the OS
// recycled the pid onto an unrelated process), so the gate self-heals.
export const UPDATE_MARKER_MAX_AGE_MS = 20 * 60 * 1000
export function markerPath(hermesHome) {
return path.join(hermesHome, '.hermes-update-in-progress')
}
// True only if a host process with this pid is currently alive. Signal 0 does
// not deliver a signal — it just probes existence/permission. ESRCH => dead;
// EPERM => alive but owned by another user (still "alive" for our purposes).
// Injectable `kill` keeps it unit-testable.
//
// NOT zombie-aware on its own: signal 0 also succeeds for a process that
// exited but whose parent has not reaped it. Callers deciding whether an
// update marker's owner is still running must layer `posixProcessState` on
// top (see `readLiveUpdateMarker`).
export function isPidAlive(pid, kill: typeof process.kill = process.kill.bind(process)) {
if (!Number.isInteger(pid) || pid <= 0) {
return false
}
try {
kill(pid, 0)
return true
} catch (err) {
return Boolean(err && err.code === 'EPERM')
}
}
/**
* Single-letter process state (`ps` style) for a kill(0)-alive pid, or null
* when it cannot be determined.
*
* A ZOMBIE — exited, still in the table because its parent has not reaped
* it — answers signal 0 like a live process. A crashed updater lingering
* that way would keep its update marker "live" and park the desktop boot
* gate for the whole 20-minute ceiling (#77259, #120635, #125932). Linux
* exposes the state via /proc; on macOS `ps -o stat=` does. Failures return
* null so callers keep their signal-0 verdict (fail-open to alive, matching
* the EPERM behavior above).
*/
export function posixProcessState(pid: number): string | null {
if (process.platform === 'linux') {
try {
const stat = fs.readFileSync(`/proc/${pid}/stat`, 'utf8')
const commEnd = stat.lastIndexOf(')')
const state = commEnd >= 0 ? stat.slice(commEnd + 2, commEnd + 3) : ''
return state || null
} catch {
return null
}
}
if (process.platform === 'darwin') {
try {
const out = execFileSync('ps', ['-o', 'stat=', '-p', String(pid)], {
encoding: 'utf8',
timeout: 5000
})
return out.trim().charAt(0) || null
} catch {
return null
}
}
return null
}
// A state of 'Z'/'Z+' (and friends) means the process exited and only its
// unreaped table entry remains — dead for every liveness decision here.
function isZombieState(state: string | null | undefined): boolean {
return Boolean(state && state.toUpperCase().startsWith('Z'))
}
/**
* Read + interpret the marker.
*
* Returns `{ pid, ageMs }` only when an update is GENUINELY still running
* (parseable pid that is alive, within the age ceiling). Returns `null` for
* every "no live update" case — absent, unreadable, malformed, dead pid, or
* past the ceiling — and, when a stale marker file exists, deletes it so it
* cannot strand future launches.
*
* Pure-ish: file I/O against the given path, plus an injectable pid probe and
* clock for tests.
*/
export function readLiveUpdateMarker(
hermesHome,
{
kill,
now = Date.now,
maxAgeMs = UPDATE_MARKER_MAX_AGE_MS,
processState = posixProcessState
}: {
now?: () => number
maxAgeMs?: number
kill?: typeof process.kill
/** Injectable override of the zombie/state probe (see posixProcessState). */
processState?: (pid: number) => string | null
} = {}
) {
const file = markerPath(hermesHome)
let raw
try {
raw = fs.readFileSync(file, 'utf8')
} catch {
return null // absent or unreadable => no live update
}
const [pidLine, startedLine] = String(raw).split('\n')
const pid = Number.parseInt((pidLine || '').trim(), 10)
const startedAt = Number.parseInt((startedLine || '').trim(), 10)
const ageMs = Number.isFinite(startedAt) ? now() - startedAt * 1000 : Infinity
const alive = Number.isInteger(pid) && isPidAlive(pid, kill)
if (!alive || isZombieState(processState(pid)) || ageMs > maxAgeMs) {
try {
fs.unlinkSync(file)
} catch {
void 0
}
return null
}
return { pid, ageMs }
}
/**
* Write the update-in-progress marker *from the desktop* before handing off
* to the detached updater.
*
* The Tauri-based hermes-setup.exe takes several seconds to initialise its
* window and reach the Rust `run_update` entry point where it writes the
* marker itself. During that gap the desktop's `app.quit()` teardown kills
* the backend child, the renderer's WebSocket drops, and the renderer
* immediately calls `ensureBackend()` → `waitForUpdateToFinish()`. Because
* the updater hasn't written the marker yet, the gate sees no live update
* and spawns a *new* backend — which re-locks `.pyd` files in the venv.
* When the updater finally reaches the venv-rebuild stage it finds those
* files locked and the update bricks.
*
* Fix: the desktop writes the marker itself, using the spawned updater's
* PID, immediately after `spawn()`. The updater's `UpdateMarkerGuard` will
* later adopt it or another hand-off stage may replace the PID. A live
* holder's original timestamp is preserved across those transfers so retries
* cannot keep resetting the 20-minute stale ceiling. When the updater finishes
* it deletes the marker as before.
* If the updater never starts (spawn failure) the marker still contains a
* real PID, so `readLiveUpdateMarker` will self-heal once that PID exits.
*/
export function writeUpdateMarker(
hermesHome,
pid,
{
kill,
now = Date.now,
maxAgeMs = UPDATE_MARKER_MAX_AGE_MS,
startedAt
}: {
now?: () => number
maxAgeMs?: number
kill?: typeof process.kill
startedAt?: number
} = {}
) {
const file = markerPath(hermesHome)
const nowMs = now()
const owner = readLiveUpdateMarker(hermesHome, { kill, maxAgeMs, now: () => nowMs })
const acquiredAt =
typeof startedAt === 'number' && Number.isInteger(startedAt)
? startedAt
: owner
? Math.floor((nowMs - owner.ageMs) / 1000)
: Math.floor(nowMs / 1000)
try {
fs.writeFileSync(file, `${pid}\n${acquiredAt}\n`, 'utf8')
} catch {
// Best-effort: if we can't write the marker, proceed anyway. The
// updater will write its own when it reaches run_update.
}
}
/**
* Whether a NEW updater hand-off must be refused because a different,
* already-alive updater currently owns the marker (#75778).
*
* `writeUpdateMarker` unconditionally overwrites the marker file. Called
* before every hand-off with no conflict check, a user who clicks "Update"
* again while a prior updater is still parked mid-run (e.g. "waiting for
* Hermes to exit…") clobbers that still-running updater's claim: the
* retry's pre-write now names the NEW child, so the OLD process — alive
* and mutating the checkout — is no longer recorded as the owner. A second
* live updater can then run over the same tree unrecorded, the exact
* two-updaters-at-once hazard `UpdateMarkerGuard` in the Rust updater
* exists to prevent (apps/bootstrap-installer/src-tauri/src/update.rs).
*
* Returns the live foreign owner (with a ready-to-show message) when the
* hand-off must be refused, or `null` when it's safe to spawn — no marker,
* or the existing one is stale/dead and self-heals via
* `readLiveUpdateMarker`.
*/
export function updateHandoffConflict(
hermesHome,
opts: {
now?: () => number
maxAgeMs?: number
kill?: typeof process.kill
} = {}
) {
const owner = readLiveUpdateMarker(hermesHome, opts)
if (!owner) {
return null
}
const mins = Math.floor(owner.ageMs / 60_000)
const secs = Math.floor((owner.ageMs % 60_000) / 1000)
const elapsed = mins > 0 ? `${mins}m ${secs}s` : `${secs}s`
return {
pid: owner.pid,
ageMs: owner.ageMs,
message: `An update is already running (PID ${owner.pid}, started ${elapsed} ago). Wait for it to finish, then try again.`
}
}