refactor: modularize audit and harden build validation
This commit is contained in:
Mikhail Chusavitin
@@ -1,5 +1,5 @@
|
||||
#!/bin/sh
|
||||
# bee-nvidia-load — load NVIDIA kernel modules and create device nodes
|
||||
# bee-nvidia-load - load NVIDIA kernel modules and create device nodes
|
||||
# Called by bee-nvidia.service at boot.
|
||||
|
||||
NVIDIA_KO_DIR="/usr/local/lib/nvidia"
|
||||
@@ -28,7 +28,7 @@ have_nvidia_gpu() {
|
||||
}
|
||||
|
||||
if ! have_nvidia_gpu; then
|
||||
log "no NVIDIA GPU detected — skipping module load"
|
||||
log "no NVIDIA GPU detected - skipping module load"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
@@ -65,7 +65,8 @@ load_module() {
|
||||
mod="$1"
|
||||
shift
|
||||
ko="$NVIDIA_KO_DIR/${mod}.ko"
|
||||
[ -f "$ko" ] || ko="$NVIDIA_KO_DIR/${mod//-/_}.ko"
|
||||
mod_file="$(printf '%s' "$mod" | tr '-' '_')"
|
||||
[ -f "$ko" ] || ko="$NVIDIA_KO_DIR/${mod_file}.ko"
|
||||
if [ ! -f "$ko" ]; then
|
||||
log "WARN: not found: $ko"
|
||||
return 1
|
||||
@@ -90,7 +91,7 @@ load_module_with_gsp_fallback() {
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Run insmod in background — on some converted SXM→PCIe cards GSP enters an
|
||||
# Run insmod in background. On some converted SXM-to-PCIe cards GSP enters an
|
||||
# infinite crash/reload loop and insmod never returns. We check for successful
|
||||
# initialization by polling /proc/devices for nvidiactl instead of waiting for
|
||||
# insmod to exit.
|
||||
@@ -114,29 +115,29 @@ load_module_with_gsp_fallback() {
|
||||
dmesg | tail -n 10 | sed 's/^/ dmesg: /' || true
|
||||
return 1
|
||||
fi
|
||||
# insmod exited 0 but nvidiactl not yet in /proc/devices — give it a moment
|
||||
# insmod exited 0 but nvidiactl is not yet in /proc/devices; give it a moment
|
||||
sleep 2
|
||||
if nvidia_is_functional; then
|
||||
log "loaded: nvidia (GSP enabled, ${_waited}s)"
|
||||
return 0
|
||||
fi
|
||||
log "insmod exited 0 but nvidiactl missing — treating as failure"
|
||||
log "insmod exited 0 but nvidiactl missing - treating as failure"
|
||||
return 1
|
||||
fi
|
||||
sleep 1
|
||||
_waited=$((_waited + 1))
|
||||
done
|
||||
|
||||
# GSP init timed out — kill the hanging insmod and attempt gsp-off fallback
|
||||
# GSP init timed out; kill the hanging insmod and attempt gsp-off fallback.
|
||||
log "nvidia GSP init timed out after 90s"
|
||||
kill "$_insmod_pid" 2>/dev/null || true
|
||||
wait "$_insmod_pid" 2>/dev/null || true
|
||||
|
||||
# Attempt to unload the partially-initialized module
|
||||
if ! rmmod nvidia 2>/dev/null; then
|
||||
# Module is stuck in the kernel — cannot reload with different params.
|
||||
# Module is stuck in the kernel; cannot reload with different params.
|
||||
# User must reboot and select bee.nvidia.mode=gsp-off at boot menu.
|
||||
log "ERROR: rmmod nvidia failed (EBUSY) — module stuck in kernel"
|
||||
log "ERROR: rmmod nvidia failed (EBUSY) - module stuck in kernel"
|
||||
log "ERROR: reboot and select 'EASY-BEE (advanced) -> GSP=off' in boot menu"
|
||||
echo "gsp-stuck" > /run/bee-nvidia-mode
|
||||
return 1
|
||||
@@ -144,7 +145,7 @@ load_module_with_gsp_fallback() {
|
||||
|
||||
sleep 2
|
||||
log "retrying with NVreg_EnableGpuFirmware=0"
|
||||
log "WARNING: GSP disabled — power management will run via CPU path, not GPU firmware"
|
||||
log "WARNING: GSP disabled - power management will run via CPU path, not GPU firmware"
|
||||
|
||||
if insmod "$ko" NVreg_EnableGpuFirmware=0; then
|
||||
if nvidia_is_functional; then
|
||||
@@ -208,7 +209,7 @@ else
|
||||
log "GSP-off mode: skipping nvidia-modeset and nvidia-uvm during boot"
|
||||
;;
|
||||
nomsi|*)
|
||||
# nomsi: disable MSI-X/MSI interrupts — use when RmInitAdapter fails with
|
||||
# nomsi: disable MSI-X/MSI interrupts; use when RmInitAdapter fails with
|
||||
# "Failed to enable MSI-X" on one or more GPUs (IOMMU group interrupt limits).
|
||||
# NVreg_EnableMSI=0 forces legacy INTx interrupts for all GPUs.
|
||||
if ! load_module nvidia NVreg_EnableGpuFirmware=0 NVreg_EnableMSI=0; then
|
||||
@@ -230,7 +231,7 @@ if [ -n "$nvidia_major" ]; then
|
||||
done
|
||||
log "created /dev/nvidia{0-7}"
|
||||
else
|
||||
log "WARN: nvidiactl not in /proc/devices — no GPU hardware present?"
|
||||
log "WARN: nvidiactl not in /proc/devices - no GPU hardware present?"
|
||||
fi
|
||||
|
||||
uvm_major=$(grep -m1 ' nvidia-uvm$' /proc/devices | awk '{print $1}')
|
||||
@@ -255,60 +256,40 @@ if command -v nvidia-smi >/dev/null 2>&1; then
|
||||
log "WARN: failed to enable NVIDIA persistence mode"
|
||||
fi
|
||||
else
|
||||
log "WARN: nvidia-smi not found — cannot enable persistence mode"
|
||||
log "WARN: nvidia-smi not found - cannot enable persistence mode"
|
||||
fi
|
||||
|
||||
# Bound every systemctl call below: a unit whose ExecStart/ExecCondition hangs
|
||||
# (e.g. fabricmanager stuck training a bad NVSwitch fabric) must not be able to
|
||||
# wedge bee-nvidia.service forever — that would keep nvidia-dcgm.service from
|
||||
# ever starting, since it's ordered After= this one. 60s comfortably covers a
|
||||
# normal fabricmanager/dcgm startup without blocking boot indefinitely.
|
||||
SYSTEMCTL_TIMEOUT=60
|
||||
timeout_systemctl() {
|
||||
timeout "${SYSTEMCTL_TIMEOUT}" systemctl "$@"
|
||||
# Refresh nvidia-fabricmanager and nvidia-dcgm so they (re)enumerate against
|
||||
# the device nodes we just created.
|
||||
#
|
||||
# These MUST NOT block. bee-nvidia.service is Type=oneshot and ordered
|
||||
# Before=nvidia-fabricmanager.service nvidia-dcgm.service, so systemd will not
|
||||
# run either unit until this script returns. A synchronous "systemctl restart"
|
||||
# here therefore deadlocks against our own ordering and only unwedges when its
|
||||
# timeout fires. "systemctl --no-block try-restart" queues a restart only for
|
||||
# an active unit and returns without waiting. An inactive enabled unit remains
|
||||
# in the normal boot transaction and can start after bee-nvidia.service exits.
|
||||
nvidia_refresh_unit() {
|
||||
unit="$1"
|
||||
if ! command -v systemctl >/dev/null 2>&1 ||
|
||||
! systemctl list-unit-files --no-legend 2>/dev/null | awk -v wanted="$unit" '$1 == wanted { found=1 } END { exit(found ? 0 : 1) }'; then
|
||||
log "WARN: ${unit} not installed"
|
||||
return
|
||||
fi
|
||||
if systemctl --no-block try-restart "$unit" >/dev/null 2>&1; then
|
||||
log "queued refresh of ${unit} (non-blocking)"
|
||||
else
|
||||
log "WARN: could not queue refresh of ${unit}"
|
||||
fi
|
||||
}
|
||||
|
||||
# Start or refresh Fabric Manager after the NVIDIA stack is ready. On NVSwitch
|
||||
# systems CUDA/DCGM can report "system not yet initialized" until fabric
|
||||
# training completes under nvidia-fabricmanager.
|
||||
if command -v systemctl >/dev/null 2>&1 && systemctl list-unit-files --no-legend 2>/dev/null | grep -q '^nvidia-fabricmanager\.service'; then
|
||||
log "restarting nvidia-fabricmanager.service (timeout ${SYSTEMCTL_TIMEOUT}s)"
|
||||
if timeout_systemctl restart nvidia-fabricmanager.service >/dev/null 2>&1; then
|
||||
log "nvidia-fabricmanager restarted"
|
||||
elif [ $? -eq 124 ]; then
|
||||
log "WARN: systemctl restart nvidia-fabricmanager.service timed out after ${SYSTEMCTL_TIMEOUT}s"
|
||||
elif timeout_systemctl start nvidia-fabricmanager.service >/dev/null 2>&1; then
|
||||
log "nvidia-fabricmanager started"
|
||||
else
|
||||
log "WARN: failed to start nvidia-fabricmanager.service"
|
||||
systemctl status nvidia-fabricmanager.service --no-pager 2>&1 | sed 's/^/ fabricmanager: /' || true
|
||||
fi
|
||||
else
|
||||
log "WARN: nvidia-fabricmanager.service not installed"
|
||||
fi
|
||||
# On NVSwitch systems CUDA/DCGM can report "system not yet initialized" until
|
||||
# fabric training completes under nvidia-fabricmanager; on non-NVSwitch boxes
|
||||
# the unit's ExecCondition skips it and this is a no-op.
|
||||
nvidia_refresh_unit nvidia-fabricmanager.service
|
||||
|
||||
# Restart the DCGM host engine so dcgmi can discover GPUs. nv-hostengine
|
||||
# enumerates GPUs once at startup and never rescans; bee-nvidia.service now
|
||||
# orders itself Before=nvidia-dcgm.service so systemd shouldn't start it until
|
||||
# modules/device nodes exist, but restart here too in case the unit was
|
||||
# already active from a previous boot/reload with a stale empty inventory.
|
||||
# Use systemctl (not a raw nv-hostengine invocation) so systemd's own
|
||||
# supervision of nvidia-dcgm.service stays authoritative and we don't end up
|
||||
# with two host engines racing for the same port.
|
||||
if command -v systemctl >/dev/null 2>&1 && systemctl list-unit-files --no-legend 2>/dev/null | grep -q '^nvidia-dcgm\.service'; then
|
||||
log "restarting nvidia-dcgm.service (timeout ${SYSTEMCTL_TIMEOUT}s)"
|
||||
if timeout_systemctl restart nvidia-dcgm.service >/dev/null 2>&1; then
|
||||
log "nvidia-dcgm restarted"
|
||||
elif [ $? -eq 124 ]; then
|
||||
log "WARN: systemctl restart nvidia-dcgm.service timed out after ${SYSTEMCTL_TIMEOUT}s"
|
||||
elif timeout_systemctl start nvidia-dcgm.service >/dev/null 2>&1; then
|
||||
log "nvidia-dcgm started"
|
||||
else
|
||||
log "WARN: failed to start nvidia-dcgm.service"
|
||||
systemctl status nvidia-dcgm.service --no-pager 2>&1 | sed 's/^/ nvidia-dcgm: /' || true
|
||||
fi
|
||||
else
|
||||
log "WARN: nvidia-dcgm.service not installed"
|
||||
fi
|
||||
# If nvidia-dcgm is already active, restart it after the device nodes exist so
|
||||
# its hostengine refreshes its device view. Otherwise normal boot starts it.
|
||||
nvidia_refresh_unit nvidia-dcgm.service
|
||||
|
||||
log "done"
|
||||
|
||||
Reference in New Issue
Block a user