Handle NVIDIA GSP firmware init hang with timeout fallback
- bee-nvidia-load: run insmod in background, poll /proc/devices for nvidiactl; if GSP init doesn't complete in 90s, kill insmod and retry with NVreg_EnableGpuFirmware=0. Handles EBUSY case with clear error. - Write /run/bee-nvidia-mode (gsp-on/gsp-off/gsp-stuck) for audit layer - Show GSP mode badge in sidebar: yellow for gsp-off, red for gsp-stuck - Report NvidiaGSPMode in RuntimeHealth with issue entries - Simplify GRUB menu: default (KMS+GSP), advanced submenu (GSP=off, nomodeset, fail-safe), remove load-to-RAM entry - Add pcmanfm, ristretto, mupdf, mousepad to desktop packages Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -173,6 +173,22 @@ func (s *System) collectGPURuntimeHealth(vendor string, health *schema.RuntimeHe
|
||||
|
||||
switch vendor {
|
||||
case "nvidia":
|
||||
if raw, err := os.ReadFile("/run/bee-nvidia-mode"); err == nil {
|
||||
health.NvidiaGSPMode = strings.TrimSpace(string(raw))
|
||||
if health.NvidiaGSPMode == "gsp-stuck" {
|
||||
health.Issues = append(health.Issues, schema.RuntimeIssue{
|
||||
Code: "nvidia_gsp_stuck",
|
||||
Severity: "critical",
|
||||
Description: "NVIDIA GSP firmware init timed out and the kernel module is stuck. Reboot and select 'GSP=off' in the boot menu.",
|
||||
})
|
||||
} else if health.NvidiaGSPMode == "gsp-off" {
|
||||
health.Issues = append(health.Issues, schema.RuntimeIssue{
|
||||
Code: "nvidia_gsp_disabled",
|
||||
Severity: "warning",
|
||||
Description: "NVIDIA GSP firmware disabled (fallback). Power management runs via CPU path — power draw readings may differ from reference hardware.",
|
||||
})
|
||||
}
|
||||
}
|
||||
health.DriverReady = strings.Contains(lsmodText, "nvidia ")
|
||||
if !health.DriverReady {
|
||||
health.Issues = append(health.Issues, schema.RuntimeIssue{
|
||||
|
||||
@@ -20,6 +20,7 @@ type RuntimeHealth struct {
|
||||
ExportDir string `json:"export_dir,omitempty"`
|
||||
DriverReady bool `json:"driver_ready,omitempty"`
|
||||
CUDAReady bool `json:"cuda_ready,omitempty"`
|
||||
NvidiaGSPMode string `json:"nvidia_gsp_mode,omitempty"` // "gsp-on", "gsp-off", "gsp-stuck"
|
||||
NetworkStatus string `json:"network_status,omitempty"`
|
||||
Issues []RuntimeIssue `json:"issues,omitempty"`
|
||||
Tools []RuntimeToolStatus `json:"tools,omitempty"`
|
||||
|
||||
@@ -33,6 +33,9 @@ a{color:var(--accent);text-decoration:none}
|
||||
.sidebar-logo{padding:18px 16px 12px;font-size:18px;font-weight:700;color:#fff;letter-spacing:-.5px}
|
||||
.sidebar-logo span{color:rgba(255,255,255,.5);font-weight:400;font-size:12px;display:block;margin-top:2px}
|
||||
.sidebar-version{padding:0 16px 14px;font-size:11px;color:rgba(255,255,255,.45)}
|
||||
.sidebar-badge{margin:0 12px 12px;padding:5px 8px;border-radius:4px;font-size:11px;font-weight:600;text-align:center}
|
||||
.sidebar-badge-warn{background:#7a4f00;color:#f6c90e}
|
||||
.sidebar-badge-crit{background:#5c1a1a;color:#ff6b6b}
|
||||
.nav{flex:1}
|
||||
.nav-item{display:block;padding:10px 16px;color:rgba(255,255,255,.7);font-size:13px;border-left:3px solid transparent;transition:all .15s}
|
||||
.nav-item:hover{color:#fff;background:rgba(255,255,255,.08)}
|
||||
@@ -107,6 +110,15 @@ func layoutNav(active string, buildLabel string) string {
|
||||
buildLabel = "dev"
|
||||
}
|
||||
b.WriteString(`<div class="sidebar-version">Version ` + html.EscapeString(buildLabel) + `</div>`)
|
||||
if raw, err := os.ReadFile("/run/bee-nvidia-mode"); err == nil {
|
||||
gspMode := strings.TrimSpace(string(raw))
|
||||
switch gspMode {
|
||||
case "gsp-off":
|
||||
b.WriteString(`<div class="sidebar-badge sidebar-badge-warn">NVIDIA GSP=off</div>`)
|
||||
case "gsp-stuck":
|
||||
b.WriteString(`<div class="sidebar-badge sidebar-badge-crit">NVIDIA GSP stuck — reboot</div>`)
|
||||
}
|
||||
}
|
||||
b.WriteString(`<nav class="nav">`)
|
||||
for _, item := range items {
|
||||
cls := "nav-item"
|
||||
|
||||
@@ -11,33 +11,25 @@ echo " Hardware Audit LiveCD"
|
||||
echo ""
|
||||
|
||||
menuentry "EASY-BEE" {
|
||||
linux @KERNEL_LIVE@ @APPEND_LIVE@ nomodeset bee.nvidia.mode=normal net.ifnames=0 biosdevname=0 mitigations=off transparent_hugepage=always numa_balancing=disable nowatchdog nosoftlockup
|
||||
linux @KERNEL_LIVE@ @APPEND_LIVE@ bee.nvidia.mode=normal net.ifnames=0 biosdevname=0 mitigations=off transparent_hugepage=always numa_balancing=disable nowatchdog nosoftlockup
|
||||
initrd @INITRD_LIVE@
|
||||
}
|
||||
|
||||
menuentry "EASY-BEE (graphics/KMS)" {
|
||||
linux @KERNEL_LIVE@ @APPEND_LIVE@ bee.display=kms bee.nvidia.mode=normal net.ifnames=0 biosdevname=0 mitigations=off transparent_hugepage=always numa_balancing=disable nowatchdog nosoftlockup
|
||||
initrd @INITRD_LIVE@
|
||||
}
|
||||
submenu "EASY-BEE (advanced options) -->" {
|
||||
menuentry "EASY-BEE — GSP=off" {
|
||||
linux @KERNEL_LIVE@ @APPEND_LIVE@ bee.nvidia.mode=gsp-off net.ifnames=0 biosdevname=0 mitigations=off transparent_hugepage=always numa_balancing=disable nowatchdog nosoftlockup
|
||||
initrd @INITRD_LIVE@
|
||||
}
|
||||
|
||||
menuentry "EASY-BEE (load to RAM)" {
|
||||
linux @KERNEL_LIVE@ @APPEND_LIVE@ toram nomodeset bee.nvidia.mode=normal net.ifnames=0 biosdevname=0 mitigations=off transparent_hugepage=always numa_balancing=disable nowatchdog nosoftlockup
|
||||
initrd @INITRD_LIVE@
|
||||
}
|
||||
menuentry "EASY-BEE — nomodeset" {
|
||||
linux @KERNEL_LIVE@ @APPEND_LIVE@ nomodeset bee.nvidia.mode=normal net.ifnames=0 biosdevname=0 mitigations=off transparent_hugepage=always numa_balancing=disable nowatchdog nosoftlockup
|
||||
initrd @INITRD_LIVE@
|
||||
}
|
||||
|
||||
menuentry "EASY-BEE (NVIDIA GSP=off)" {
|
||||
linux @KERNEL_LIVE@ @APPEND_LIVE@ nomodeset bee.nvidia.mode=gsp-off net.ifnames=0 biosdevname=0 mitigations=off transparent_hugepage=always numa_balancing=disable nowatchdog nosoftlockup
|
||||
initrd @INITRD_LIVE@
|
||||
}
|
||||
|
||||
menuentry "EASY-BEE (graphics/KMS, GSP=off)" {
|
||||
linux @KERNEL_LIVE@ @APPEND_LIVE@ bee.display=kms bee.nvidia.mode=gsp-off net.ifnames=0 biosdevname=0 mitigations=off transparent_hugepage=always numa_balancing=disable nowatchdog nosoftlockup
|
||||
initrd @INITRD_LIVE@
|
||||
}
|
||||
|
||||
menuentry "EASY-BEE (fail-safe)" {
|
||||
linux @KERNEL_LIVE@ @APPEND_LIVE@ bee.nvidia.mode=gsp-off memtest noapic noapm nodma nomce nolapic nosmp vga=normal net.ifnames=0 biosdevname=0
|
||||
initrd @INITRD_LIVE@
|
||||
menuentry "EASY-BEE — fail-safe" {
|
||||
linux @KERNEL_LIVE@ @APPEND_LIVE@ nomodeset bee.nvidia.mode=gsp-off noapic noapm nodma nomce nolapic nosmp vga=normal net.ifnames=0 biosdevname=0
|
||||
initrd @INITRD_LIVE@
|
||||
}
|
||||
}
|
||||
|
||||
if [ "${grub_platform}" = "efi" ]; then
|
||||
|
||||
@@ -65,6 +65,10 @@ python3-pil
|
||||
xorg
|
||||
xterm
|
||||
chromium
|
||||
mousepad
|
||||
pcmanfm
|
||||
ristretto
|
||||
mupdf
|
||||
xserver-xorg-video-fbdev
|
||||
xserver-xorg-video-vesa
|
||||
lightdm
|
||||
|
||||
@@ -50,11 +50,93 @@ load_module() {
|
||||
log "WARN: not found: $ko"
|
||||
return 1
|
||||
fi
|
||||
if insmod "$ko" "$@"; then
|
||||
if timeout 90 insmod "$ko" "$@"; then
|
||||
log "loaded: $mod $*"
|
||||
return 0
|
||||
fi
|
||||
log "WARN: failed to load: $mod"
|
||||
log "WARN: failed to load: $mod (exit $?)"
|
||||
dmesg | tail -n 10 | sed 's/^/ dmesg: /' || true
|
||||
return 1
|
||||
}
|
||||
|
||||
nvidia_is_functional() {
|
||||
grep -q ' nvidiactl$' /proc/devices 2>/dev/null
|
||||
}
|
||||
|
||||
load_module_with_gsp_fallback() {
|
||||
ko="$NVIDIA_KO_DIR/nvidia.ko"
|
||||
if [ ! -f "$ko" ]; then
|
||||
log "ERROR: not found: $ko"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Run insmod in background — on some converted SXM→PCIe cards GSP enters an
|
||||
# infinite crash/reload loop and insmod never returns. We check for successful
|
||||
# initialization by polling /proc/devices for nvidiactl instead of waiting for
|
||||
# insmod to exit.
|
||||
log "loading nvidia (GSP enabled, timeout 90s)"
|
||||
insmod "$ko" &
|
||||
_insmod_pid=$!
|
||||
|
||||
_waited=0
|
||||
while [ $_waited -lt 90 ]; do
|
||||
if nvidia_is_functional; then
|
||||
log "loaded: nvidia (GSP enabled, ${_waited}s)"
|
||||
echo "gsp-on" > /run/bee-nvidia-mode
|
||||
return 0
|
||||
fi
|
||||
# Check if insmod exited with an error before timeout
|
||||
if ! kill -0 "$_insmod_pid" 2>/dev/null; then
|
||||
wait "$_insmod_pid"
|
||||
_rc=$?
|
||||
if [ $_rc -ne 0 ]; then
|
||||
log "nvidia load failed (exit $_rc)"
|
||||
dmesg | tail -n 10 | sed 's/^/ dmesg: /' || true
|
||||
return 1
|
||||
fi
|
||||
# insmod exited 0 but nvidiactl not yet in /proc/devices — give it a moment
|
||||
sleep 2
|
||||
if nvidia_is_functional; then
|
||||
log "loaded: nvidia (GSP enabled, ${_waited}s)"
|
||||
return 0
|
||||
fi
|
||||
log "insmod exited 0 but nvidiactl missing — treating as failure"
|
||||
return 1
|
||||
fi
|
||||
sleep 1
|
||||
_waited=$((_waited + 1))
|
||||
done
|
||||
|
||||
# GSP init timed out — kill the hanging insmod and attempt gsp-off fallback
|
||||
log "nvidia GSP init timed out after 90s"
|
||||
kill "$_insmod_pid" 2>/dev/null || true
|
||||
wait "$_insmod_pid" 2>/dev/null || true
|
||||
|
||||
# Attempt to unload the partially-initialized module
|
||||
if ! rmmod nvidia 2>/dev/null; then
|
||||
# Module is stuck in the kernel — cannot reload with different params.
|
||||
# User must reboot and select bee.nvidia.mode=gsp-off at boot menu.
|
||||
log "ERROR: rmmod nvidia failed (EBUSY) — module stuck in kernel"
|
||||
log "ERROR: reboot and select 'EASY-BEE (advanced) -> GSP=off' in boot menu"
|
||||
echo "gsp-stuck" > /run/bee-nvidia-mode
|
||||
return 1
|
||||
fi
|
||||
|
||||
sleep 2
|
||||
log "retrying with NVreg_EnableGpuFirmware=0"
|
||||
log "WARNING: GSP disabled — power management will run via CPU path, not GPU firmware"
|
||||
|
||||
if insmod "$ko" NVreg_EnableGpuFirmware=0; then
|
||||
if nvidia_is_functional; then
|
||||
log "loaded: nvidia (GSP disabled)"
|
||||
echo "gsp-off" > /run/bee-nvidia-mode
|
||||
return 0
|
||||
fi
|
||||
log "insmod gsp-off exited 0 but nvidiactl missing"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log "nvidia load failed (GSP=off)"
|
||||
dmesg | tail -n 10 | sed 's/^/ dmesg: /' || true
|
||||
return 1
|
||||
}
|
||||
@@ -70,7 +152,7 @@ load_host_module() {
|
||||
|
||||
case "$nvidia_mode" in
|
||||
normal|full)
|
||||
if ! load_module nvidia; then
|
||||
if ! load_module_with_gsp_fallback; then
|
||||
exit 1
|
||||
fi
|
||||
# nvidia-modeset on some server kernels needs ACPI video helper symbols
|
||||
|
||||
Reference in New Issue
Block a user