diff --git a/audit/internal/platform/sat.go b/audit/internal/platform/sat.go index 94c02a6..8028e9b 100644 --- a/audit/internal/platform/sat.go +++ b/audit/internal/platform/sat.go @@ -493,7 +493,7 @@ func (s *System) RunNvidiaTargetedPowerPack(ctx context.Context, baseDir string, } return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-targeted-power", withNvidiaPersistenceMode( satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}}, - satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}}, + satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2}, satJob{ name: "03-dcgmi-targeted-power.log", cmd: nvidiaDCGMNamedDiagCommand("targeted_power", normalizeNvidiaBurnDuration(durationSec), selected), @@ -518,7 +518,7 @@ func (s *System) RunNvidiaPulseTestPack(ctx context.Context, baseDir string, dur } return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-pulse", withNvidiaPersistenceMode( satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}}, - satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}}, + satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2}, satJob{ name: "03-dcgmi-pulse-test.log", cmd: nvidiaDCGMNamedDiagCommand("pulse_test", normalizeNvidiaBurnDuration(durationSec), selected), @@ -543,7 +543,7 @@ func (s *System) RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpu } return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-bandwidth", withNvidiaPersistenceMode( satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}}, - satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}}, + satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2}, satJob{ name: "03-dcgmi-nvbandwidth.log", cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, selected), @@ -584,7 +584,7 @@ func (s *System) RunNvidiaTargetedStressValidatePack(ctx context.Context, baseDi } return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-targeted-stress", withNvidiaPersistenceMode( satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}}, - satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}}, + satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2}, satJob{ name: "03-dcgmi-targeted-stress.log", cmd: nvidiaDCGMNamedDiagCommand("targeted_stress", normalizeNvidiaBurnDuration(durationSec), selected), @@ -787,12 +787,20 @@ type satJob struct { env []string // extra env vars (appended to os.Environ) collectGPU bool // collect GPU metrics via nvidia-smi while this job runs gpuIndices []int // GPU indices to collect metrics for (empty = all) + // informational marks a preflight/metadata job (e.g. dcgmi discovery) whose + // failure shouldn't flip the pack's overall status — the diagnostic jobs + // that follow it are the actual test of GPU health. + informational bool + // retries is the number of extra attempts (with a short backoff) if the + // job's first run fails. Used for jobs racing nv-hostengine startup. + retries int } type satStats struct { - OK int - Failed int - Unsupported int + OK int + Failed int + Unsupported int + Informational int } func withNvidiaPersistenceMode(jobs ...satJob) []satJob { @@ -831,7 +839,7 @@ func nvidiaDCGMJobs(diagLevel int, gpuIndices []int) []satJob { satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}}, satJob{name: "02-dmidecode-baseboard.log", cmd: []string{"dmidecode", "-t", "baseboard"}}, satJob{name: "03-dmidecode-system.log", cmd: []string{"dmidecode", "-t", "system"}}, - satJob{name: "04-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}}, + satJob{name: "04-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2}, satJob{name: "05-dcgmi-diag.log", cmd: diagArgs, gpuIndices: gpuIndices}, ) } @@ -924,10 +932,19 @@ func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []sa } if err == nil { - if job.collectGPU { - out, err = runSATCommandWithMetrics(ctx, verboseLog, job.name, cmd, job.env, job.gpuIndices, runDir, logFunc) - } else { - out, err = runSATCommandCtx(ctx, verboseLog, job.name, cmd, job.env, logFunc) + for attempt := 0; ; attempt++ { + if job.collectGPU { + out, err = runSATCommandWithMetrics(ctx, verboseLog, job.name, cmd, job.env, job.gpuIndices, runDir, logFunc) + } else { + out, err = runSATCommandCtx(ctx, verboseLog, job.name, cmd, job.env, logFunc) + } + if err == nil || attempt >= job.retries || ctx.Err() != nil { + break + } + if logFunc != nil { + logFunc(fmt.Sprintf("%s: retrying after failure (attempt %d/%d)", job.name, attempt+1, job.retries)) + } + time.Sleep(2 * time.Second) } } @@ -953,7 +970,11 @@ func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []sa return "", ctx.Err() } status, rc := classifySATResult(job.name, out, err) - stats.Add(status) + if job.informational && status != "OK" { + stats.Informational++ + } else { + stats.Add(status) + } if nvidiaPack && len(job.gpuIndices) > 0 && nvidiaJobNeedsHealthCheck(job) { for _, idx := range job.gpuIndices { updateNvidiaGPUStatus(perGPU, idx, status, job.name, string(out)) @@ -1306,6 +1327,7 @@ func writeSATStats(summary *strings.Builder, stats satStats) { fmt.Fprintf(summary, "job_ok=%d\n", stats.OK) fmt.Fprintf(summary, "job_failed=%d\n", stats.Failed) fmt.Fprintf(summary, "job_unsupported=%d\n", stats.Unsupported) + fmt.Fprintf(summary, "job_informational_failed=%d\n", stats.Informational) } func classifySATResult(name string, out []byte, err error) (string, int) { diff --git a/iso/overlay/usr/local/bin/bee-nvidia-load b/iso/overlay/usr/local/bin/bee-nvidia-load index 106b2b5..0c06a51 100755 --- a/iso/overlay/usr/local/bin/bee-nvidia-load +++ b/iso/overlay/usr/local/bin/bee-nvidia-load @@ -258,13 +258,26 @@ else log "WARN: nvidia-smi not found — cannot enable persistence mode" fi +# Bound every systemctl call below: a unit whose ExecStart/ExecCondition hangs +# (e.g. fabricmanager stuck training a bad NVSwitch fabric) must not be able to +# wedge bee-nvidia.service forever — that would keep nvidia-dcgm.service from +# ever starting, since it's ordered After= this one. 60s comfortably covers a +# normal fabricmanager/dcgm startup without blocking boot indefinitely. +SYSTEMCTL_TIMEOUT=60 +timeout_systemctl() { + timeout "${SYSTEMCTL_TIMEOUT}" systemctl "$@" +} + # Start or refresh Fabric Manager after the NVIDIA stack is ready. On NVSwitch # systems CUDA/DCGM can report "system not yet initialized" until fabric # training completes under nvidia-fabricmanager. if command -v systemctl >/dev/null 2>&1 && systemctl list-unit-files --no-legend 2>/dev/null | grep -q '^nvidia-fabricmanager\.service'; then - if systemctl restart nvidia-fabricmanager.service >/dev/null 2>&1; then + log "restarting nvidia-fabricmanager.service (timeout ${SYSTEMCTL_TIMEOUT}s)" + if timeout_systemctl restart nvidia-fabricmanager.service >/dev/null 2>&1; then log "nvidia-fabricmanager restarted" - elif systemctl start nvidia-fabricmanager.service >/dev/null 2>&1; then + elif [ $? -eq 124 ]; then + log "WARN: systemctl restart nvidia-fabricmanager.service timed out after ${SYSTEMCTL_TIMEOUT}s" + elif timeout_systemctl start nvidia-fabricmanager.service >/dev/null 2>&1; then log "nvidia-fabricmanager started" else log "WARN: failed to start nvidia-fabricmanager.service" @@ -283,9 +296,12 @@ fi # supervision of nvidia-dcgm.service stays authoritative and we don't end up # with two host engines racing for the same port. if command -v systemctl >/dev/null 2>&1 && systemctl list-unit-files --no-legend 2>/dev/null | grep -q '^nvidia-dcgm\.service'; then - if systemctl restart nvidia-dcgm.service >/dev/null 2>&1; then + log "restarting nvidia-dcgm.service (timeout ${SYSTEMCTL_TIMEOUT}s)" + if timeout_systemctl restart nvidia-dcgm.service >/dev/null 2>&1; then log "nvidia-dcgm restarted" - elif systemctl start nvidia-dcgm.service >/dev/null 2>&1; then + elif [ $? -eq 124 ]; then + log "WARN: systemctl restart nvidia-dcgm.service timed out after ${SYSTEMCTL_TIMEOUT}s" + elif timeout_systemctl start nvidia-dcgm.service >/dev/null 2>&1; then log "nvidia-dcgm started" else log "WARN: failed to start nvidia-dcgm.service"