reanimator/bee
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

sat/nvidia-load: make dcgmi discovery non-fatal and bound systemctl restarts

Browse Source 
dcgmi discovery -l is a preflight/metadata step ahead of the real DCGM
diag jobs; a transient failure racing nv-hostengine startup shouldn't
flip the whole pack's status, so it's now marked informational with a
couple of retries. Separately, bound the fabricmanager/nvidia-dcgm
systemctl restart/start calls in bee-nvidia-load with a timeout so a
wedged unit (e.g. fabric training stuck on a bad NVSwitch fabric)
can't hang bee-nvidia.service forever and block dcgm from ever
starting.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-07-07 15:25:14 +03:00
co-authored by Claude Sonnet 5
parent b7f015c713
commit 2599d9c5e3
2 changed files with 55 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
audit/internal/platform/sat.go
+35 -13
View File
@@ -493,7 +493,7 @@ func (s *System) RunNvidiaTargetedPowerPack(ctx context.Context, baseDir string,
} }
return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-targeted-power", withNvidiaPersistenceMode( return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-targeted-power", withNvidiaPersistenceMode(
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}}, satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}}, satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
satJob{ satJob{
name: "03-dcgmi-targeted-power.log", name: "03-dcgmi-targeted-power.log",
cmd: nvidiaDCGMNamedDiagCommand("targeted_power", normalizeNvidiaBurnDuration(durationSec), selected), cmd: nvidiaDCGMNamedDiagCommand("targeted_power", normalizeNvidiaBurnDuration(durationSec), selected),
@@ -518,7 +518,7 @@ func (s *System) RunNvidiaPulseTestPack(ctx context.Context, baseDir string, dur
} }
return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-pulse", withNvidiaPersistenceMode( return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-pulse", withNvidiaPersistenceMode(
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}}, satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}}, satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
satJob{ satJob{
name: "03-dcgmi-pulse-test.log", name: "03-dcgmi-pulse-test.log",
cmd: nvidiaDCGMNamedDiagCommand("pulse_test", normalizeNvidiaBurnDuration(durationSec), selected), cmd: nvidiaDCGMNamedDiagCommand("pulse_test", normalizeNvidiaBurnDuration(durationSec), selected),
@@ -543,7 +543,7 @@ func (s *System) RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpu
} }
return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-bandwidth", withNvidiaPersistenceMode( return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-bandwidth", withNvidiaPersistenceMode(
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}}, satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}}, satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
satJob{ satJob{
name: "03-dcgmi-nvbandwidth.log", name: "03-dcgmi-nvbandwidth.log",
cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, selected), cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, selected),
@@ -584,7 +584,7 @@ func (s *System) RunNvidiaTargetedStressValidatePack(ctx context.Context, baseDi
} }
return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-targeted-stress", withNvidiaPersistenceMode( return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-targeted-stress", withNvidiaPersistenceMode(
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}}, satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}}, satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
satJob{ satJob{
name: "03-dcgmi-targeted-stress.log", name: "03-dcgmi-targeted-stress.log",
cmd: nvidiaDCGMNamedDiagCommand("targeted_stress", normalizeNvidiaBurnDuration(durationSec), selected), cmd: nvidiaDCGMNamedDiagCommand("targeted_stress", normalizeNvidiaBurnDuration(durationSec), selected),
@@ -787,12 +787,20 @@ type satJob struct {
env []string // extra env vars (appended to os.Environ) env []string // extra env vars (appended to os.Environ)
collectGPU bool // collect GPU metrics via nvidia-smi while this job runs collectGPU bool // collect GPU metrics via nvidia-smi while this job runs
gpuIndices []int // GPU indices to collect metrics for (empty = all) gpuIndices []int // GPU indices to collect metrics for (empty = all)
// informational marks a preflight/metadata job (e.g. dcgmi discovery) whose
// failure shouldn't flip the pack's overall status — the diagnostic jobs
// that follow it are the actual test of GPU health.
informational bool
// retries is the number of extra attempts (with a short backoff) if the
// job's first run fails. Used for jobs racing nv-hostengine startup.
retries int
} }
type satStats struct { type satStats struct {
OK int OK int
Failed int Failed int
Unsupported int Unsupported int
Informational int
} }
func withNvidiaPersistenceMode(jobs ...satJob) []satJob { func withNvidiaPersistenceMode(jobs ...satJob) []satJob {
@@ -831,7 +839,7 @@ func nvidiaDCGMJobs(diagLevel int, gpuIndices []int) []satJob {
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}}, satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
satJob{name: "02-dmidecode-baseboard.log", cmd: []string{"dmidecode", "-t", "baseboard"}}, satJob{name: "02-dmidecode-baseboard.log", cmd: []string{"dmidecode", "-t", "baseboard"}},
satJob{name: "03-dmidecode-system.log", cmd: []string{"dmidecode", "-t", "system"}}, satJob{name: "03-dmidecode-system.log", cmd: []string{"dmidecode", "-t", "system"}},
satJob{name: "04-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}}, satJob{name: "04-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
satJob{name: "05-dcgmi-diag.log", cmd: diagArgs, gpuIndices: gpuIndices}, satJob{name: "05-dcgmi-diag.log", cmd: diagArgs, gpuIndices: gpuIndices},
) )
} }
@@ -924,10 +932,19 @@ func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []sa
} }
if err == nil { if err == nil {
if job.collectGPU { for attempt := 0; ; attempt++ {
out, err = runSATCommandWithMetrics(ctx, verboseLog, job.name, cmd, job.env, job.gpuIndices, runDir, logFunc) if job.collectGPU {
} else { out, err = runSATCommandWithMetrics(ctx, verboseLog, job.name, cmd, job.env, job.gpuIndices, runDir, logFunc)
out, err = runSATCommandCtx(ctx, verboseLog, job.name, cmd, job.env, logFunc) } else {
out, err = runSATCommandCtx(ctx, verboseLog, job.name, cmd, job.env, logFunc)
}
if err == nil || attempt >= job.retries || ctx.Err() != nil {
break
}
if logFunc != nil {
logFunc(fmt.Sprintf("%s: retrying after failure (attempt %d/%d)", job.name, attempt+1, job.retries))
}
time.Sleep(2 * time.Second)
} }
} }
@@ -953,7 +970,11 @@ func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []sa
return "", ctx.Err() return "", ctx.Err()
} }
status, rc := classifySATResult(job.name, out, err) status, rc := classifySATResult(job.name, out, err)
stats.Add(status) if job.informational && status != "OK" {
stats.Informational++
} else {
stats.Add(status)
}
if nvidiaPack && len(job.gpuIndices) > 0 && nvidiaJobNeedsHealthCheck(job) { if nvidiaPack && len(job.gpuIndices) > 0 && nvidiaJobNeedsHealthCheck(job) {
for _, idx := range job.gpuIndices { for _, idx := range job.gpuIndices {
updateNvidiaGPUStatus(perGPU, idx, status, job.name, string(out)) updateNvidiaGPUStatus(perGPU, idx, status, job.name, string(out))
@@ -1306,6 +1327,7 @@ func writeSATStats(summary *strings.Builder, stats satStats) {
fmt.Fprintf(summary, "job_ok=%d\n", stats.OK) fmt.Fprintf(summary, "job_ok=%d\n", stats.OK)
fmt.Fprintf(summary, "job_failed=%d\n", stats.Failed) fmt.Fprintf(summary, "job_failed=%d\n", stats.Failed)
fmt.Fprintf(summary, "job_unsupported=%d\n", stats.Unsupported) fmt.Fprintf(summary, "job_unsupported=%d\n", stats.Unsupported)
fmt.Fprintf(summary, "job_informational_failed=%d\n", stats.Informational)
} }
func classifySATResult(name string, out []byte, err error) (string, int) { func classifySATResult(name string, out []byte, err error) (string, int) {
iso/overlay/usr/local/bin/bee-nvidia-load
+20 -4
View File
@@ -258,13 +258,26 @@ else
log "WARN: nvidia-smi not found — cannot enable persistence mode" log "WARN: nvidia-smi not found — cannot enable persistence mode"
fi fi
# Bound every systemctl call below: a unit whose ExecStart/ExecCondition hangs
# (e.g. fabricmanager stuck training a bad NVSwitch fabric) must not be able to
# wedge bee-nvidia.service forever — that would keep nvidia-dcgm.service from
# ever starting, since it's ordered After= this one. 60s comfortably covers a
# normal fabricmanager/dcgm startup without blocking boot indefinitely.
SYSTEMCTL_TIMEOUT=60
timeout_systemctl() {
timeout "${SYSTEMCTL_TIMEOUT}" systemctl "$@"
}
# Start or refresh Fabric Manager after the NVIDIA stack is ready. On NVSwitch # Start or refresh Fabric Manager after the NVIDIA stack is ready. On NVSwitch
# systems CUDA/DCGM can report "system not yet initialized" until fabric # systems CUDA/DCGM can report "system not yet initialized" until fabric
# training completes under nvidia-fabricmanager. # training completes under nvidia-fabricmanager.
if command -v systemctl >/dev/null 2>&1 && systemctl list-unit-files --no-legend 2>/dev/null | grep -q '^nvidia-fabricmanager\.service'; then if command -v systemctl >/dev/null 2>&1 && systemctl list-unit-files --no-legend 2>/dev/null | grep -q '^nvidia-fabricmanager\.service'; then
if systemctl restart nvidia-fabricmanager.service >/dev/null 2>&1; then log "restarting nvidia-fabricmanager.service (timeout ${SYSTEMCTL_TIMEOUT}s)"
if timeout_systemctl restart nvidia-fabricmanager.service >/dev/null 2>&1; then
log "nvidia-fabricmanager restarted" log "nvidia-fabricmanager restarted"
elif systemctl start nvidia-fabricmanager.service >/dev/null 2>&1; then elif [ $? -eq 124 ]; then
log "WARN: systemctl restart nvidia-fabricmanager.service timed out after ${SYSTEMCTL_TIMEOUT}s"
elif timeout_systemctl start nvidia-fabricmanager.service >/dev/null 2>&1; then
log "nvidia-fabricmanager started" log "nvidia-fabricmanager started"
else else
log "WARN: failed to start nvidia-fabricmanager.service" log "WARN: failed to start nvidia-fabricmanager.service"
@@ -283,9 +296,12 @@ fi
# supervision of nvidia-dcgm.service stays authoritative and we don't end up # supervision of nvidia-dcgm.service stays authoritative and we don't end up
# with two host engines racing for the same port. # with two host engines racing for the same port.
if command -v systemctl >/dev/null 2>&1 && systemctl list-unit-files --no-legend 2>/dev/null | grep -q '^nvidia-dcgm\.service'; then if command -v systemctl >/dev/null 2>&1 && systemctl list-unit-files --no-legend 2>/dev/null | grep -q '^nvidia-dcgm\.service'; then
if systemctl restart nvidia-dcgm.service >/dev/null 2>&1; then log "restarting nvidia-dcgm.service (timeout ${SYSTEMCTL_TIMEOUT}s)"
if timeout_systemctl restart nvidia-dcgm.service >/dev/null 2>&1; then
log "nvidia-dcgm restarted" log "nvidia-dcgm restarted"
elif systemctl start nvidia-dcgm.service >/dev/null 2>&1; then elif [ $? -eq 124 ]; then
log "WARN: systemctl restart nvidia-dcgm.service timed out after ${SYSTEMCTL_TIMEOUT}s"
elif timeout_systemctl start nvidia-dcgm.service >/dev/null 2>&1; then
log "nvidia-dcgm started" log "nvidia-dcgm started"
else else
log "WARN: failed to start nvidia-dcgm.service" log "WARN: failed to start nvidia-dcgm.service"