mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-11-16 19:36:54 +00:00
The NVIDIA EGL fallback (added for the #40077 580-series crash, bounded to {580} by #123213) forced every 580 host onto all-CPU SwiftShader compositing. But the same driver point release breaks one host and not another: 580.178.04 kills the GPU process where NVIDIA drives the display, and renders fine on hybrid hosts whose session EGL lands on the iGPU (#124255 reporter, psukez on #123203). The all-CPU fallback then turns a known Chromium stuck-Button-observer leak (~0 cost with GPU compositing) into a silent 4-9 core burn. Replace the driver-major set with a behavioral probe (option A from the #124255 decision prep): a Linux+NVIDIA host boots with hardware GL and a booting marker; a GPU-process death before first window reveal (reasons crashed / launch-failure / killed — #40077 died to Chromium's health-check SIGTERM, exit_code=15) flips the marker sticky per app version + full driver version and relaunches once with --use-angle=swiftshader. A healthy boot marks ok; an app or driver update re-probes once. A leftover booting marker (the 'GPU process isn't usable' FATAL abort winning the race) engages on the next launch. HERMES_DESKTOP_NVIDIA_SWIFTSHADER still forces on/off; HERMES_DESKTOP_DISABLE_GPU=0 still wins. Never disableHardwareAcceleration() — that SIGKILLs the renderer on 580 + Electron 40 (#40119). Healthy 580 hosts get hardware rendering back; broken ones still reach the desktop in at most one extra restart per app/driver update. Follow-up to #123213. The in-app notice (#123598) and the warn-once stuck-observer log (#124275) remain complementary and are not included. Fixes #124255
226 lines
9.4 KiB
TypeScript
226 lines
9.4 KiB
TypeScript
import { mkdtempSync, rmSync } from 'node:fs'
|
|
import { tmpdir } from 'node:os'
|
|
import { join } from 'node:path'
|
|
|
|
import { afterEach, describe, expect, it } from 'vitest'
|
|
|
|
import {
|
|
decideNvidiaEglFallback,
|
|
NVIDIA_GPU_DEATH_REASONS,
|
|
nvidiaEglFallbackMarker,
|
|
nvidiaEglMarkerAfterSuccessfulBoot,
|
|
parseNvidiaDriverMajor,
|
|
parseNvidiaDriverVersion,
|
|
readNvidiaEglMarker,
|
|
shouldRelaunchForNvidiaGpuDeath,
|
|
writeNvidiaEglMarker
|
|
} from './linux-nvidia-egl-fallback'
|
|
|
|
const LINUX = { env: {}, platform: 'linux' as const, isWsl: false, remoteDisplayReason: null }
|
|
const PROBE = { ...LINUX, driverMajor: 580, driverVersion: '580.178.04', appVersion: '0.21.5', marker: null }
|
|
|
|
const tmpDirs: string[] = []
|
|
|
|
afterEach(() => {
|
|
for (const dir of tmpDirs.splice(0)) {
|
|
rmSync(dir, { recursive: true, force: true })
|
|
}
|
|
})
|
|
|
|
function tmpUserData(): string {
|
|
const dir = mkdtempSync(join(tmpdir(), 'nvidia-egl-'))
|
|
tmpDirs.push(dir)
|
|
|
|
return dir
|
|
}
|
|
|
|
describe('parseNvidiaDriverMajor / parseNvidiaDriverVersion', () => {
|
|
it('parses the major and full version from /proc/driver/nvidia/version content', () => {
|
|
const text = 'NVRM version: NVIDIA UNIX x86_64 Kernel Module 580.82.09 Mon Jul 21 19:44:16 UTC 2025\n'
|
|
|
|
expect(parseNvidiaDriverMajor(text)).toBe(580)
|
|
expect(parseNvidiaDriverVersion(text)).toBe('580.82.09')
|
|
})
|
|
|
|
it('returns null for garbage or empty input', () => {
|
|
expect(parseNvidiaDriverMajor('')).toBeNull()
|
|
expect(parseNvidiaDriverMajor('no driver version here')).toBeNull()
|
|
expect(parseNvidiaDriverVersion('')).toBeNull()
|
|
expect(parseNvidiaDriverVersion('no driver version here')).toBeNull()
|
|
})
|
|
})
|
|
|
|
// ─── the behavioral probe (#124255) ───────────────────────────────────────────
|
|
//
|
|
// The driver-major set gate burned 4-9 CPU cores on healthy 580 hosts whose
|
|
// session EGL lands on the iGPU; the same point release kills the GPU process
|
|
// on hosts where NVIDIA drives the display. Only a witnessed death may route
|
|
// ANGLE through SwiftShader.
|
|
|
|
describe('decideNvidiaEglFallback — behavioral probe', () => {
|
|
it('boots with hardware GL on a first launch (no marker): the 580 series is probed, not assumed broken', () => {
|
|
const decision = decideNvidiaEglFallback(PROBE)
|
|
|
|
expect(decision.enable).toBe(false)
|
|
expect(decision.nextMarker.state).toBe('booting')
|
|
})
|
|
|
|
it('a healthy prior boot (ok marker) keeps hardware GL', () => {
|
|
const decision = decideNvidiaEglFallback({ ...PROBE, marker: { state: 'ok' } })
|
|
|
|
expect(decision.enable).toBe(false)
|
|
expect(decision.nextMarker.state).toBe('booting')
|
|
})
|
|
|
|
it('a witnessed GPU death (fallback marker, same app + driver) engages SwiftShader and stays sticky', () => {
|
|
const marker = nvidiaEglFallbackMarker('0.21.5', '580.178.04')
|
|
const decision = decideNvidiaEglFallback({ ...PROBE, marker })
|
|
|
|
expect(decision.enable).toBe(true)
|
|
expect(decision.reason).toContain('witnessed')
|
|
expect(decision.nextMarker).toEqual(marker)
|
|
})
|
|
|
|
it('an app update re-probes hardware GL once instead of degrading forever', () => {
|
|
const marker = nvidiaEglFallbackMarker('0.21.4', '580.178.04')
|
|
const decision = decideNvidiaEglFallback({ ...PROBE, marker })
|
|
|
|
expect(decision.enable).toBe(false)
|
|
expect(decision.nextMarker.state).toBe('booting')
|
|
})
|
|
|
|
it('a driver update re-probes hardware GL once', () => {
|
|
const marker = nvidiaEglFallbackMarker('0.21.5', '580.178.04')
|
|
const decision = decideNvidiaEglFallback({ ...PROBE, marker, driverVersion: '580.182.10' })
|
|
|
|
expect(decision.enable).toBe(false)
|
|
expect(decision.nextMarker.state).toBe('booting')
|
|
})
|
|
|
|
it('a leftover booting marker engages the fallback: the previous boot died mid-probe', () => {
|
|
// Chromium's "GPU process isn't usable" FATAL abort can kill the process
|
|
// before the child-process-gone handler runs; the unresolved `booting`
|
|
// marker is the evidence it leaves behind.
|
|
const decision = decideNvidiaEglFallback({ ...PROBE, marker: { state: 'booting' } })
|
|
|
|
expect(decision.enable).toBe(true)
|
|
expect(decision.reason).toContain('aborted')
|
|
expect(decision.nextMarker.state).toBe('fallback')
|
|
})
|
|
|
|
it('the healthy 580 hosts from the issue thread get hardware rendering back', () => {
|
|
// #124255's reporter (Quadro P1000 + Intel UHD 630) and psukez on #123203
|
|
// (940MX + Intel HD 520) both render on hardware with this driver; under
|
|
// the old {580} series gate they were forced onto CPU SwiftShader.
|
|
const healthy = decideNvidiaEglFallback({ ...PROBE, marker: { state: 'ok' } })
|
|
|
|
expect(healthy.enable).toBe(false)
|
|
})
|
|
})
|
|
|
|
// ─── the same standing exclusions as before ────────────────────────────────
|
|
|
|
describe('decideNvidiaEglFallback — exclusions', () => {
|
|
it('stays off on non-linux platforms and without an NVIDIA driver', () => {
|
|
expect(decideNvidiaEglFallback({ ...PROBE, platform: 'darwin' }).enable).toBe(false)
|
|
expect(decideNvidiaEglFallback({ ...PROBE, platform: 'win32' }).enable).toBe(false)
|
|
expect(decideNvidiaEglFallback({ ...PROBE, driverMajor: null, marker: { state: 'booting' } }).enable).toBe(false)
|
|
})
|
|
|
|
it('stays off under WSLg and when a remote display already forced software rendering', () => {
|
|
expect(decideNvidiaEglFallback({ ...PROBE, isWsl: true, marker: { state: 'booting' } }).enable).toBe(false)
|
|
expect(
|
|
decideNvidiaEglFallback({ ...PROBE, remoteDisplayReason: 'ssh-session', marker: { state: 'booting' } }).enable
|
|
).toBe(false)
|
|
})
|
|
|
|
it('HERMES_DESKTOP_DISABLE_GPU=0 keeps the GPU on even over a witnessed marker', () => {
|
|
const marker = nvidiaEglFallbackMarker('0.21.5', '580.178.04')
|
|
|
|
expect(
|
|
decideNvidiaEglFallback({ ...PROBE, marker, env: { HERMES_DESKTOP_DISABLE_GPU: '0' } }).enable
|
|
).toBe(false)
|
|
})
|
|
|
|
it('HERMES_DESKTOP_NVIDIA_SWIFTSHADER forces the fallback on without any marker', () => {
|
|
const decision = decideNvidiaEglFallback({ ...PROBE, env: { HERMES_DESKTOP_NVIDIA_SWIFTSHADER: '1' }, marker: null })
|
|
|
|
expect(decision.enable).toBe(true)
|
|
expect(decision.reason).toContain('override')
|
|
})
|
|
|
|
it('HERMES_DESKTOP_NVIDIA_SWIFTSHADER=0 opts out even over a witnessed marker', () => {
|
|
const marker = nvidiaEglFallbackMarker('0.21.5', '580.178.04')
|
|
|
|
expect(
|
|
decideNvidiaEglFallback({ ...PROBE, marker, env: { HERMES_DESKTOP_NVIDIA_SWIFTSHADER: 'off' } }).enable
|
|
).toBe(false)
|
|
})
|
|
})
|
|
|
|
// ─── the runtime witness ────────────────────────────────────────────────────
|
|
|
|
describe('shouldRelaunchForNvidiaGpuDeath', () => {
|
|
const BASE = { platform: 'linux' as const, fallbackActive: false, relaunchAttempted: false }
|
|
|
|
it('relaunches on a GPU crash', () => {
|
|
expect(shouldRelaunchForNvidiaGpuDeath({ ...BASE, details: { type: 'GPU', reason: 'crashed' } })).toBe(true)
|
|
})
|
|
|
|
it('relaunches on a GPU launch failure', () => {
|
|
expect(shouldRelaunchForNvidiaGpuDeath({ ...BASE, details: { type: 'GPU', reason: 'launch-failure' } })).toBe(true)
|
|
})
|
|
|
|
it("counts `killed`: the #40077 GPU process died to Chromium's health-check SIGTERM (exit_code=15)", () => {
|
|
expect(NVIDIA_GPU_DEATH_REASONS.has('killed')).toBe(true)
|
|
expect(shouldRelaunchForNvidiaGpuDeath({ ...BASE, details: { type: 'GPU', reason: 'killed' } })).toBe(true)
|
|
})
|
|
|
|
it('never relaunches twice in one process, or when the fallback is already active', () => {
|
|
const details = { type: 'GPU', reason: 'crashed' }
|
|
|
|
expect(shouldRelaunchForNvidiaGpuDeath({ ...BASE, details, relaunchAttempted: true })).toBe(false)
|
|
expect(shouldRelaunchForNvidiaGpuDeath({ ...BASE, details, fallbackActive: true })).toBe(false)
|
|
})
|
|
|
|
it('ignores non-GPU deaths, clean exits, and other platforms', () => {
|
|
expect(shouldRelaunchForNvidiaGpuDeath({ ...BASE, details: { type: 'renderer', reason: 'crashed' } })).toBe(false)
|
|
expect(shouldRelaunchForNvidiaGpuDeath({ ...BASE, details: { type: 'GPU', reason: 'clean-exit' } })).toBe(false)
|
|
expect(shouldRelaunchForNvidiaGpuDeath({ ...BASE, details: null })).toBe(false)
|
|
expect(
|
|
shouldRelaunchForNvidiaGpuDeath({ ...BASE, platform: 'darwin', details: { type: 'GPU', reason: 'crashed' } })
|
|
).toBe(false)
|
|
})
|
|
})
|
|
|
|
// ─── marker lifecycle ───────────────────────────────────────────────────────
|
|
|
|
describe('marker persistence', () => {
|
|
it('round-trips through the userData dir', () => {
|
|
const dir = tmpUserData()
|
|
const marker = nvidiaEglFallbackMarker('0.21.5', '580.178.04')
|
|
|
|
writeNvidiaEglMarker(dir, marker)
|
|
|
|
expect(readNvidiaEglMarker(dir)).toEqual(marker)
|
|
})
|
|
|
|
it('returns null for a missing or garbage marker', () => {
|
|
const dir = tmpUserData()
|
|
|
|
expect(readNvidiaEglMarker(dir)).toBeNull()
|
|
expect(readNvidiaEglMarker(dir, { readFileSync: () => 'not json' as never })).toBeNull()
|
|
})
|
|
|
|
it('a healthy boot marks ok; a fallback boot keeps the sticky marker', () => {
|
|
expect(nvidiaEglMarkerAfterSuccessfulBoot({ fallbackActive: false })).toEqual({ state: 'ok' })
|
|
|
|
expect(
|
|
nvidiaEglMarkerAfterSuccessfulBoot({
|
|
fallbackActive: true,
|
|
appVersion: '0.21.5',
|
|
driverVersion: '580.178.04'
|
|
})
|
|
).toEqual(nvidiaEglFallbackMarker('0.21.5', '580.178.04'))
|
|
})
|
|
})
|