reanimator/bee
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

sat/nvidia-load: make dcgmi discovery non-fatal and bound systemctl restarts

Browse Source 
dcgmi discovery -l is a preflight/metadata step ahead of the real DCGM
diag jobs; a transient failure racing nv-hostengine startup shouldn't
flip the whole pack's status, so it's now marked informational with a
couple of retries. Separately, bound the fabricmanager/nvidia-dcgm
systemctl restart/start calls in bee-nvidia-load with a timeout so a
wedged unit (e.g. fabric training stuck on a bad NVSwitch fabric)
can't hang bee-nvidia.service forever and block dcgm from ever
starting.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-07-07 15:25:14 +03:00
co-authored by Claude Sonnet 5
parent b7f015c713
commit 2599d9c5e3
2 changed files with 55 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
iso/overlay/usr/local/bin/bee-nvidia-load
+20 -4
View File
@@ -258,13 +258,26 @@ else
log "WARN: nvidia-smi not found — cannot enable persistence mode"
fi
# Bound every systemctl call below: a unit whose ExecStart/ExecCondition hangs
# (e.g. fabricmanager stuck training a bad NVSwitch fabric) must not be able to
# wedge bee-nvidia.service forever — that would keep nvidia-dcgm.service from
# ever starting, since it's ordered After= this one. 60s comfortably covers a
# normal fabricmanager/dcgm startup without blocking boot indefinitely.
SYSTEMCTL_TIMEOUT=60
timeout_systemctl() {
timeout "${SYSTEMCTL_TIMEOUT}" systemctl "$@"
}
# Start or refresh Fabric Manager after the NVIDIA stack is ready. On NVSwitch
# systems CUDA/DCGM can report "system not yet initialized" until fabric
# training completes under nvidia-fabricmanager.
if command -v systemctl >/dev/null 2>&1 && systemctl list-unit-files --no-legend 2>/dev/null | grep -q '^nvidia-fabricmanager\.service'; then
if systemctl restart nvidia-fabricmanager.service >/dev/null 2>&1; then
log "restarting nvidia-fabricmanager.service (timeout ${SYSTEMCTL_TIMEOUT}s)"
if timeout_systemctl restart nvidia-fabricmanager.service >/dev/null 2>&1; then
log "nvidia-fabricmanager restarted"
elif systemctl start nvidia-fabricmanager.service >/dev/null 2>&1; then
elif [ $? -eq 124 ]; then
log "WARN: systemctl restart nvidia-fabricmanager.service timed out after ${SYSTEMCTL_TIMEOUT}s"
elif timeout_systemctl start nvidia-fabricmanager.service >/dev/null 2>&1; then
log "nvidia-fabricmanager started"
else
log "WARN: failed to start nvidia-fabricmanager.service"
@@ -283,9 +296,12 @@ fi
# supervision of nvidia-dcgm.service stays authoritative and we don't end up
# with two host engines racing for the same port.
if command -v systemctl >/dev/null 2>&1 && systemctl list-unit-files --no-legend 2>/dev/null | grep -q '^nvidia-dcgm\.service'; then
if systemctl restart nvidia-dcgm.service >/dev/null 2>&1; then
log "restarting nvidia-dcgm.service (timeout ${SYSTEMCTL_TIMEOUT}s)"
if timeout_systemctl restart nvidia-dcgm.service >/dev/null 2>&1; then
log "nvidia-dcgm restarted"
elif systemctl start nvidia-dcgm.service >/dev/null 2>&1; then
elif [ $? -eq 124 ]; then
log "WARN: systemctl restart nvidia-dcgm.service timed out after ${SYSTEMCTL_TIMEOUT}s"
elif timeout_systemctl start nvidia-dcgm.service >/dev/null 2>&1; then
log "nvidia-dcgm started"
else
log "WARN: failed to start nvidia-dcgm.service"