sat: collect NVLink status/errors, fix nv-hostengine restart via systemd

Add NVLink port status (nvidia-smi nvlink -s), error counters
(nvidia-smi nvlink -e), and dcgmi nvlink status to the support bundle,
and enrich HardwarePCIeDevice entries with per-link telemetry.

Replace the raw nv-hostengine pkill/restart dance in bee-nvidia-load
with systemctl restart/start of nvidia-dcgm.service, and order
bee-nvidia.service Before= nvidia-dcgm.service and
nvidia-fabricmanager.service so modules/device nodes exist before
those units start.
This commit is contained in:
Mikhail Chusavitin
2026-07-07 10:21:56 +03:00
parent a98721b038
commit 33d6eee9cf
6 changed files with 339 additions and 81 deletions
+17 -30
View File
@@ -274,38 +274,25 @@ else
log "WARN: nvidia-fabricmanager.service not installed"
fi
# Start DCGM host engine so dcgmi can discover GPUs.
# nv-hostengine must run after the NVIDIA modules and device nodes are ready.
# If it started too early (for example via systemd before bee-nvidia-load), it can
# keep a stale empty inventory and dcgmi diag later reports no testable entities.
if command -v nv-hostengine >/dev/null 2>&1; then
if pgrep -x nv-hostengine >/dev/null 2>&1; then
if command -v pkill >/dev/null 2>&1; then
pkill -x nv-hostengine >/dev/null 2>&1 || true
tries=0
while pgrep -x nv-hostengine >/dev/null 2>&1; do
tries=$((tries + 1))
if [ "${tries}" -ge 10 ]; then
log "WARN: nv-hostengine is still running after restart request"
break
fi
sleep 1
done
if pgrep -x nv-hostengine >/dev/null 2>&1; then
log "WARN: keeping existing nv-hostengine process"
else
log "nv-hostengine restarted"
fi
else
log "WARN: pkill not found — cannot refresh nv-hostengine inventory"
fi
fi
if ! pgrep -x nv-hostengine >/dev/null 2>&1; then
nv-hostengine
log "nv-hostengine started"
# Restart the DCGM host engine so dcgmi can discover GPUs. nv-hostengine
# enumerates GPUs once at startup and never rescans; bee-nvidia.service now
# orders itself Before=nvidia-dcgm.service so systemd shouldn't start it until
# modules/device nodes exist, but restart here too in case the unit was
# already active from a previous boot/reload with a stale empty inventory.
# Use systemctl (not a raw nv-hostengine invocation) so systemd's own
# supervision of nvidia-dcgm.service stays authoritative and we don't end up
# with two host engines racing for the same port.
if command -v systemctl >/dev/null 2>&1 && systemctl list-unit-files --no-legend 2>/dev/null | grep -q '^nvidia-dcgm\.service'; then
if systemctl restart nvidia-dcgm.service >/dev/null 2>&1; then
log "nvidia-dcgm restarted"
elif systemctl start nvidia-dcgm.service >/dev/null 2>&1; then
log "nvidia-dcgm started"
else
log "WARN: failed to start nvidia-dcgm.service"
systemctl status nvidia-dcgm.service --no-pager 2>&1 | sed 's/^/ nvidia-dcgm: /' || true
fi
else
log "WARN: nv-hostengine not found — dcgmi diagnostics will not work"
log "WARN: nvidia-dcgm.service not installed"
fi
log "done"