sat/nvidia-load: make dcgmi discovery non-fatal and bound systemctl restarts

dcgmi discovery -l is a preflight/metadata step ahead of the real DCGM
diag jobs; a transient failure racing nv-hostengine startup shouldn't
flip the whole pack's status, so it's now marked informational with a
couple of retries. Separately, bound the fabricmanager/nvidia-dcgm
systemctl restart/start calls in bee-nvidia-load with a timeout so a
wedged unit (e.g. fabric training stuck on a bad NVSwitch fabric)
can't hang bee-nvidia.service forever and block dcgm from ever
starting.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

audit/internal/platform/sat.go

@@ -493,7 +493,7 @@ func (s *System) RunNvidiaTargetedPowerPack(ctx context.Context, baseDir string,
 	}
 	return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-targeted-power", withNvidiaPersistenceMode(
 		satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
-		satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}},
+		satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
 		satJob{
 			name:       "03-dcgmi-targeted-power.log",
 			cmd:        nvidiaDCGMNamedDiagCommand("targeted_power", normalizeNvidiaBurnDuration(durationSec), selected),
@@ -518,7 +518,7 @@ func (s *System) RunNvidiaPulseTestPack(ctx context.Context, baseDir string, dur
 	}
 	return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-pulse", withNvidiaPersistenceMode(
 		satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
-		satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}},
+		satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
 		satJob{
 			name:       "03-dcgmi-pulse-test.log",
 			cmd:        nvidiaDCGMNamedDiagCommand("pulse_test", normalizeNvidiaBurnDuration(durationSec), selected),
@@ -543,7 +543,7 @@ func (s *System) RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpu
 	}
 	return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-bandwidth", withNvidiaPersistenceMode(
 		satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
-		satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}},
+		satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
 		satJob{
 			name:       "03-dcgmi-nvbandwidth.log",
 			cmd:        nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, selected),
@@ -584,7 +584,7 @@ func (s *System) RunNvidiaTargetedStressValidatePack(ctx context.Context, baseDi
 	}
 	return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-targeted-stress", withNvidiaPersistenceMode(
 		satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
-		satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}},
+		satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
 		satJob{
 			name:       "03-dcgmi-targeted-stress.log",
 			cmd:        nvidiaDCGMNamedDiagCommand("targeted_stress", normalizeNvidiaBurnDuration(durationSec), selected),
@@ -787,12 +787,20 @@ type satJob struct {
 	env        []string // extra env vars (appended to os.Environ)
 	collectGPU bool     // collect GPU metrics via nvidia-smi while this job runs
 	gpuIndices []int    // GPU indices to collect metrics for (empty = all)
+	// informational marks a preflight/metadata job (e.g. dcgmi discovery) whose
+	// failure shouldn't flip the pack's overall status — the diagnostic jobs
+	// that follow it are the actual test of GPU health.
+	informational bool
+	// retries is the number of extra attempts (with a short backoff) if the
+	// job's first run fails. Used for jobs racing nv-hostengine startup.
+	retries int
 }
 
 type satStats struct {
 	OK            int
 	Failed        int
 	Unsupported   int
+	Informational int
 }
 
 func withNvidiaPersistenceMode(jobs ...satJob) []satJob {
@@ -831,7 +839,7 @@ func nvidiaDCGMJobs(diagLevel int, gpuIndices []int) []satJob {
 		satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
 		satJob{name: "02-dmidecode-baseboard.log", cmd: []string{"dmidecode", "-t", "baseboard"}},
 		satJob{name: "03-dmidecode-system.log", cmd: []string{"dmidecode", "-t", "system"}},
-		satJob{name: "04-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}},
+		satJob{name: "04-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
 		satJob{name: "05-dcgmi-diag.log", cmd: diagArgs, gpuIndices: gpuIndices},
 	)
 }
@@ -924,11 +932,20 @@ func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []sa
 		}
 
 		if err == nil {
+			for attempt := 0; ; attempt++ {
 				if job.collectGPU {
 					out, err = runSATCommandWithMetrics(ctx, verboseLog, job.name, cmd, job.env, job.gpuIndices, runDir, logFunc)
 				} else {
 					out, err = runSATCommandCtx(ctx, verboseLog, job.name, cmd, job.env, logFunc)
 				}
+				if err == nil || attempt >= job.retries || ctx.Err() != nil {
+					break
+				}
+				if logFunc != nil {
+					logFunc(fmt.Sprintf("%s: retrying after failure (attempt %d/%d)", job.name, attempt+1, job.retries))
+				}
+				time.Sleep(2 * time.Second)
+			}
 		}
 
 		if nvidiaPack && nvidiaJobNeedsHealthCheck(job) {
@@ -953,7 +970,11 @@ func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []sa
 			return "", ctx.Err()
 		}
 		status, rc := classifySATResult(job.name, out, err)
+		if job.informational && status != "OK" {
+			stats.Informational++
+		} else {
 			stats.Add(status)
+		}
 		if nvidiaPack && len(job.gpuIndices) > 0 && nvidiaJobNeedsHealthCheck(job) {
 			for _, idx := range job.gpuIndices {
 				updateNvidiaGPUStatus(perGPU, idx, status, job.name, string(out))
@@ -1306,6 +1327,7 @@ func writeSATStats(summary *strings.Builder, stats satStats) {
 	fmt.Fprintf(summary, "job_ok=%d\n", stats.OK)
 	fmt.Fprintf(summary, "job_failed=%d\n", stats.Failed)
 	fmt.Fprintf(summary, "job_unsupported=%d\n", stats.Unsupported)
+	fmt.Fprintf(summary, "job_informational_failed=%d\n", stats.Informational)
 }
 
 func classifySATResult(name string, out []byte, err error) (string, int) {

iso/overlay/usr/local/bin/bee-nvidia-load

@@ -258,13 +258,26 @@ else
     log "WARN: nvidia-smi not found — cannot enable persistence mode"
 fi
 
+# Bound every systemctl call below: a unit whose ExecStart/ExecCondition hangs
+# (e.g. fabricmanager stuck training a bad NVSwitch fabric) must not be able to
+# wedge bee-nvidia.service forever — that would keep nvidia-dcgm.service from
+# ever starting, since it's ordered After= this one. 60s comfortably covers a
+# normal fabricmanager/dcgm startup without blocking boot indefinitely.
+SYSTEMCTL_TIMEOUT=60
+timeout_systemctl() {
+    timeout "${SYSTEMCTL_TIMEOUT}" systemctl "$@"
+}
+
 # Start or refresh Fabric Manager after the NVIDIA stack is ready. On NVSwitch
 # systems CUDA/DCGM can report "system not yet initialized" until fabric
 # training completes under nvidia-fabricmanager.
 if command -v systemctl >/dev/null 2>&1 && systemctl list-unit-files --no-legend 2>/dev/null | grep -q '^nvidia-fabricmanager\.service'; then
-    if systemctl restart nvidia-fabricmanager.service >/dev/null 2>&1; then
+    log "restarting nvidia-fabricmanager.service (timeout ${SYSTEMCTL_TIMEOUT}s)"
+    if timeout_systemctl restart nvidia-fabricmanager.service >/dev/null 2>&1; then
         log "nvidia-fabricmanager restarted"
-    elif systemctl start nvidia-fabricmanager.service >/dev/null 2>&1; then
+    elif [ $? -eq 124 ]; then
+        log "WARN: systemctl restart nvidia-fabricmanager.service timed out after ${SYSTEMCTL_TIMEOUT}s"
+    elif timeout_systemctl start nvidia-fabricmanager.service >/dev/null 2>&1; then
         log "nvidia-fabricmanager started"
     else
         log "WARN: failed to start nvidia-fabricmanager.service"
@@ -283,9 +296,12 @@ fi
 # supervision of nvidia-dcgm.service stays authoritative and we don't end up
 # with two host engines racing for the same port.
 if command -v systemctl >/dev/null 2>&1 && systemctl list-unit-files --no-legend 2>/dev/null | grep -q '^nvidia-dcgm\.service'; then
-    if systemctl restart nvidia-dcgm.service >/dev/null 2>&1; then
+    log "restarting nvidia-dcgm.service (timeout ${SYSTEMCTL_TIMEOUT}s)"
+    if timeout_systemctl restart nvidia-dcgm.service >/dev/null 2>&1; then
         log "nvidia-dcgm restarted"
-    elif systemctl start nvidia-dcgm.service >/dev/null 2>&1; then
+    elif [ $? -eq 124 ]; then
+        log "WARN: systemctl restart nvidia-dcgm.service timed out after ${SYSTEMCTL_TIMEOUT}s"
+    elif timeout_systemctl start nvidia-dcgm.service >/dev/null 2>&1; then
         log "nvidia-dcgm started"
     else
         log "WARN: failed to start nvidia-dcgm.service"