sat/nvidia-load: make dcgmi discovery non-fatal and bound systemctl restarts
dcgmi discovery -l is a preflight/metadata step ahead of the real DCGM diag jobs; a transient failure racing nv-hostengine startup shouldn't flip the whole pack's status, so it's now marked informational with a couple of retries. Separately, bound the fabricmanager/nvidia-dcgm systemctl restart/start calls in bee-nvidia-load with a timeout so a wedged unit (e.g. fabric training stuck on a bad NVSwitch fabric) can't hang bee-nvidia.service forever and block dcgm from ever starting. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
b7f015c713
commit
2599d9c5e3
@@ -493,7 +493,7 @@ func (s *System) RunNvidiaTargetedPowerPack(ctx context.Context, baseDir string,
|
||||
}
|
||||
return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-targeted-power", withNvidiaPersistenceMode(
|
||||
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
|
||||
satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}},
|
||||
satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
|
||||
satJob{
|
||||
name: "03-dcgmi-targeted-power.log",
|
||||
cmd: nvidiaDCGMNamedDiagCommand("targeted_power", normalizeNvidiaBurnDuration(durationSec), selected),
|
||||
@@ -518,7 +518,7 @@ func (s *System) RunNvidiaPulseTestPack(ctx context.Context, baseDir string, dur
|
||||
}
|
||||
return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-pulse", withNvidiaPersistenceMode(
|
||||
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
|
||||
satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}},
|
||||
satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
|
||||
satJob{
|
||||
name: "03-dcgmi-pulse-test.log",
|
||||
cmd: nvidiaDCGMNamedDiagCommand("pulse_test", normalizeNvidiaBurnDuration(durationSec), selected),
|
||||
@@ -543,7 +543,7 @@ func (s *System) RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpu
|
||||
}
|
||||
return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-bandwidth", withNvidiaPersistenceMode(
|
||||
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
|
||||
satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}},
|
||||
satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
|
||||
satJob{
|
||||
name: "03-dcgmi-nvbandwidth.log",
|
||||
cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, selected),
|
||||
@@ -584,7 +584,7 @@ func (s *System) RunNvidiaTargetedStressValidatePack(ctx context.Context, baseDi
|
||||
}
|
||||
return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-targeted-stress", withNvidiaPersistenceMode(
|
||||
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
|
||||
satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}},
|
||||
satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
|
||||
satJob{
|
||||
name: "03-dcgmi-targeted-stress.log",
|
||||
cmd: nvidiaDCGMNamedDiagCommand("targeted_stress", normalizeNvidiaBurnDuration(durationSec), selected),
|
||||
@@ -787,12 +787,20 @@ type satJob struct {
|
||||
env []string // extra env vars (appended to os.Environ)
|
||||
collectGPU bool // collect GPU metrics via nvidia-smi while this job runs
|
||||
gpuIndices []int // GPU indices to collect metrics for (empty = all)
|
||||
// informational marks a preflight/metadata job (e.g. dcgmi discovery) whose
|
||||
// failure shouldn't flip the pack's overall status — the diagnostic jobs
|
||||
// that follow it are the actual test of GPU health.
|
||||
informational bool
|
||||
// retries is the number of extra attempts (with a short backoff) if the
|
||||
// job's first run fails. Used for jobs racing nv-hostengine startup.
|
||||
retries int
|
||||
}
|
||||
|
||||
type satStats struct {
|
||||
OK int
|
||||
Failed int
|
||||
Unsupported int
|
||||
OK int
|
||||
Failed int
|
||||
Unsupported int
|
||||
Informational int
|
||||
}
|
||||
|
||||
func withNvidiaPersistenceMode(jobs ...satJob) []satJob {
|
||||
@@ -831,7 +839,7 @@ func nvidiaDCGMJobs(diagLevel int, gpuIndices []int) []satJob {
|
||||
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
|
||||
satJob{name: "02-dmidecode-baseboard.log", cmd: []string{"dmidecode", "-t", "baseboard"}},
|
||||
satJob{name: "03-dmidecode-system.log", cmd: []string{"dmidecode", "-t", "system"}},
|
||||
satJob{name: "04-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}},
|
||||
satJob{name: "04-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
|
||||
satJob{name: "05-dcgmi-diag.log", cmd: diagArgs, gpuIndices: gpuIndices},
|
||||
)
|
||||
}
|
||||
@@ -924,10 +932,19 @@ func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []sa
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
if job.collectGPU {
|
||||
out, err = runSATCommandWithMetrics(ctx, verboseLog, job.name, cmd, job.env, job.gpuIndices, runDir, logFunc)
|
||||
} else {
|
||||
out, err = runSATCommandCtx(ctx, verboseLog, job.name, cmd, job.env, logFunc)
|
||||
for attempt := 0; ; attempt++ {
|
||||
if job.collectGPU {
|
||||
out, err = runSATCommandWithMetrics(ctx, verboseLog, job.name, cmd, job.env, job.gpuIndices, runDir, logFunc)
|
||||
} else {
|
||||
out, err = runSATCommandCtx(ctx, verboseLog, job.name, cmd, job.env, logFunc)
|
||||
}
|
||||
if err == nil || attempt >= job.retries || ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
if logFunc != nil {
|
||||
logFunc(fmt.Sprintf("%s: retrying after failure (attempt %d/%d)", job.name, attempt+1, job.retries))
|
||||
}
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -953,7 +970,11 @@ func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []sa
|
||||
return "", ctx.Err()
|
||||
}
|
||||
status, rc := classifySATResult(job.name, out, err)
|
||||
stats.Add(status)
|
||||
if job.informational && status != "OK" {
|
||||
stats.Informational++
|
||||
} else {
|
||||
stats.Add(status)
|
||||
}
|
||||
if nvidiaPack && len(job.gpuIndices) > 0 && nvidiaJobNeedsHealthCheck(job) {
|
||||
for _, idx := range job.gpuIndices {
|
||||
updateNvidiaGPUStatus(perGPU, idx, status, job.name, string(out))
|
||||
@@ -1306,6 +1327,7 @@ func writeSATStats(summary *strings.Builder, stats satStats) {
|
||||
fmt.Fprintf(summary, "job_ok=%d\n", stats.OK)
|
||||
fmt.Fprintf(summary, "job_failed=%d\n", stats.Failed)
|
||||
fmt.Fprintf(summary, "job_unsupported=%d\n", stats.Unsupported)
|
||||
fmt.Fprintf(summary, "job_informational_failed=%d\n", stats.Informational)
|
||||
}
|
||||
|
||||
func classifySATResult(name string, out []byte, err error) (string, int) {
|
||||
|
||||
Reference in New Issue
Block a user