platform/app: fix nvbandwidth split fallback, blackbox discovery churn, and add blocking sync brackets around load steps
- gpuBandwidthSocketGroups: a single GPU whose NUMA node fails to resolve no longer collapses the whole per-socket nvbandwidth split into one fallback pass — it now folds into the last resolved group instead, preserving isolation for the sockets that did resolve. - blackbox discoverMarkedTargets: skip mounting/unmounting devices that already have a running worker on every 2s discovery tick. This was observed hammering the same USB target continuously (mount+unmount every ~2s for the whole session) and contending with the worker's own sync cycle, plausibly explaining multi-minute sync cycles seen on a real crash bundle. - syncFilesystem now calls syscall.Sync() directly instead of spawning /bin/sync per copied file; blackbox mounts removable targets with -o sync so writes are durable without relying on the app-level sync as the primary mechanism. - New platform.SetSyncBracketHook / satJob.syncBracket: blocks (with a bounded timeout) on blackbox actually reaching removable media right before and right after a diagnostic's real load step (nvbandwidth, memtester, stress-ng, dcgmi diag, nccl, smartctl/nvme self-test...), instead of only firing a fire-and-forget kick after the job's own log file is written. A crash mid-load now has durable evidence the load started, not just whatever streamed to the RAM-backed export dir before blackbox's next scheduled cycle. Found investigating a real support bundle where blackbox's last successful sync (19:55:25) predated both the previous job finishing and the crashing nvbandwidth job starting (19:56:57) — none of the crash window ever reached durable media. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
6d13c17d36
commit
e03267a72f
@@ -83,6 +83,34 @@ func SetJobBoundaryHook(hook func(jobName string)) {
|
||||
satJobBoundaryHook = hook
|
||||
}
|
||||
|
||||
// satSyncBracketHook, if set, is called synchronously immediately before and
|
||||
// after a satJob marked syncBracket runs — i.e. around the actual load
|
||||
// command of a diagnostic (nvbandwidth, memtester, stress-ng, dcgmi diag...),
|
||||
// not the cheap discovery/inventory steps around it. phase is "before" or
|
||||
// "after". Unlike satJobBoundaryHook (fire-and-forget, fires after every
|
||||
// job), this is meant to block until an external blackbox sync has actually
|
||||
// reached removable media, so that a crash during the load itself still
|
||||
// leaves durable evidence that the load started (and, on the far side, that
|
||||
// it finished). Nil by default. An error is logged, never fails the job —
|
||||
// a stuck blackbox target must not block the diagnostic the operator asked
|
||||
// for.
|
||||
var satSyncBracketHook func(jobName, phase string) error
|
||||
|
||||
// SetSyncBracketHook installs the callback invoked synchronously before and
|
||||
// after a syncBracket-marked SAT job. Pass nil to clear it.
|
||||
func SetSyncBracketHook(hook func(jobName, phase string) error) {
|
||||
satSyncBracketHook = hook
|
||||
}
|
||||
|
||||
func runSyncBracketHook(job satJob, phase string, logFunc func(string)) {
|
||||
if !job.syncBracket || satSyncBracketHook == nil {
|
||||
return
|
||||
}
|
||||
if err := satSyncBracketHook(job.name, phase); err != nil && logFunc != nil {
|
||||
logFunc(fmt.Sprintf("%s: blackbox sync wait (%s) did not complete cleanly: %v", job.name, phase, err))
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
satExecCommand = exec.Command
|
||||
satLookPath = exec.LookPath
|
||||
@@ -463,7 +491,7 @@ func (s *System) RunNCCLTests(ctx context.Context, baseDir string, gpuIndices []
|
||||
satJob{name: "02-all-reduce-perf.log", cmd: []string{
|
||||
"all_reduce_perf", "-b", "512M", "-e", "4G", "-f", "2",
|
||||
"-g", strconv.Itoa(gpuCount), "--iters", "20",
|
||||
}, env: nvidiaVisibleDevicesEnv(selected)},
|
||||
}, env: nvidiaVisibleDevicesEnv(selected), syncBracket: true},
|
||||
), logFunc)
|
||||
}
|
||||
|
||||
@@ -502,11 +530,12 @@ func (s *System) RunNvidiaOfficialComputePack(ctx context.Context, baseDir strin
|
||||
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
|
||||
satJob{name: "02-dcgmi-version.log", cmd: []string{"dcgmi", "-v"}},
|
||||
satJob{
|
||||
name: "03-dcgmproftester.log",
|
||||
cmd: profCmd,
|
||||
env: profEnv,
|
||||
collectGPU: true,
|
||||
gpuIndices: selected,
|
||||
name: "03-dcgmproftester.log",
|
||||
cmd: profCmd,
|
||||
env: profEnv,
|
||||
collectGPU: true,
|
||||
gpuIndices: selected,
|
||||
syncBracket: true,
|
||||
},
|
||||
satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
|
||||
), logFunc)
|
||||
@@ -528,10 +557,11 @@ func (s *System) RunNvidiaTargetedPowerPack(ctx context.Context, baseDir string,
|
||||
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
|
||||
satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
|
||||
satJob{
|
||||
name: "03-dcgmi-targeted-power.log",
|
||||
cmd: nvidiaDCGMNamedDiagCommand("targeted_power", normalizeNvidiaBurnDuration(durationSec), selected),
|
||||
collectGPU: true,
|
||||
gpuIndices: selected,
|
||||
name: "03-dcgmi-targeted-power.log",
|
||||
cmd: nvidiaDCGMNamedDiagCommand("targeted_power", normalizeNvidiaBurnDuration(durationSec), selected),
|
||||
collectGPU: true,
|
||||
gpuIndices: selected,
|
||||
syncBracket: true,
|
||||
},
|
||||
satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
|
||||
), logFunc)
|
||||
@@ -553,10 +583,11 @@ func (s *System) RunNvidiaPulseTestPack(ctx context.Context, baseDir string, dur
|
||||
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
|
||||
satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
|
||||
satJob{
|
||||
name: "03-dcgmi-pulse-test.log",
|
||||
cmd: nvidiaDCGMNamedDiagCommand("pulse_test", normalizeNvidiaBurnDuration(durationSec), selected),
|
||||
collectGPU: true,
|
||||
gpuIndices: selected,
|
||||
name: "03-dcgmi-pulse-test.log",
|
||||
cmd: nvidiaDCGMNamedDiagCommand("pulse_test", normalizeNvidiaBurnDuration(durationSec), selected),
|
||||
collectGPU: true,
|
||||
gpuIndices: selected,
|
||||
syncBracket: true,
|
||||
},
|
||||
satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
|
||||
), logFunc)
|
||||
@@ -591,27 +622,30 @@ func (s *System) RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpu
|
||||
socketGroups := gpuBandwidthSocketGroups(selected, logFunc)
|
||||
if len(socketGroups) <= 1 {
|
||||
jobs = append(jobs, satJob{
|
||||
name: fmt.Sprintf("%02d-dcgmi-nvbandwidth.log", step),
|
||||
cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, selected),
|
||||
collectGPU: true,
|
||||
gpuIndices: selected,
|
||||
name: fmt.Sprintf("%02d-dcgmi-nvbandwidth.log", step),
|
||||
cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, selected),
|
||||
collectGPU: true,
|
||||
gpuIndices: selected,
|
||||
syncBracket: true,
|
||||
})
|
||||
step++
|
||||
} else {
|
||||
for i, group := range socketGroups {
|
||||
jobs = append(jobs, satJob{
|
||||
name: fmt.Sprintf("%02d-dcgmi-nvbandwidth-socket%d.log", step, i),
|
||||
cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, group),
|
||||
collectGPU: true,
|
||||
gpuIndices: group,
|
||||
name: fmt.Sprintf("%02d-dcgmi-nvbandwidth-socket%d.log", step, i),
|
||||
cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, group),
|
||||
collectGPU: true,
|
||||
gpuIndices: group,
|
||||
syncBracket: true,
|
||||
})
|
||||
step++
|
||||
}
|
||||
jobs = append(jobs, satJob{
|
||||
name: fmt.Sprintf("%02d-dcgmi-nvbandwidth-all.log", step),
|
||||
cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, selected),
|
||||
collectGPU: true,
|
||||
gpuIndices: selected,
|
||||
name: fmt.Sprintf("%02d-dcgmi-nvbandwidth-all.log", step),
|
||||
cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, selected),
|
||||
collectGPU: true,
|
||||
gpuIndices: selected,
|
||||
syncBracket: true,
|
||||
})
|
||||
step++
|
||||
}
|
||||
@@ -655,10 +689,11 @@ func (s *System) RunNvidiaTargetedStressValidatePack(ctx context.Context, baseDi
|
||||
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
|
||||
satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
|
||||
satJob{
|
||||
name: "03-dcgmi-targeted-stress.log",
|
||||
cmd: nvidiaDCGMNamedDiagCommand("targeted_stress", normalizeNvidiaBurnDuration(durationSec), selected),
|
||||
collectGPU: true,
|
||||
gpuIndices: selected,
|
||||
name: "03-dcgmi-targeted-stress.log",
|
||||
cmd: nvidiaDCGMNamedDiagCommand("targeted_stress", normalizeNvidiaBurnDuration(durationSec), selected),
|
||||
collectGPU: true,
|
||||
gpuIndices: selected,
|
||||
syncBracket: true,
|
||||
},
|
||||
satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
|
||||
), logFunc)
|
||||
@@ -716,7 +751,7 @@ func (s *System) RunMemoryAcceptancePack(ctx context.Context, baseDir string, si
|
||||
}
|
||||
return runAcceptancePackCtx(ctx, baseDir, "memory", []satJob{
|
||||
{name: "01-free-before.log", cmd: []string{"free", "-h"}},
|
||||
{name: "02-memtester.log", cmd: []string{"timeout", fmt.Sprintf("%d", timeoutSec), "memtester", fmt.Sprintf("%dM", sizeMB), fmt.Sprintf("%d", passes)}},
|
||||
{name: "02-memtester.log", cmd: []string{"timeout", fmt.Sprintf("%d", timeoutSec), "memtester", fmt.Sprintf("%dM", sizeMB), fmt.Sprintf("%d", passes)}, syncBracket: true},
|
||||
{name: "03-free-after.log", cmd: []string{"free", "-h"}},
|
||||
}, logFunc)
|
||||
}
|
||||
@@ -737,7 +772,7 @@ func (s *System) RunMemoryStressPack(ctx context.Context, baseDir string, durati
|
||||
"--vm-method", "all",
|
||||
"--timeout", fmt.Sprintf("%d", seconds),
|
||||
"--metrics-brief",
|
||||
}},
|
||||
}, syncBracket: true},
|
||||
{name: "03-free-after.log", cmd: []string{"free", "-h"}},
|
||||
}, logFunc)
|
||||
}
|
||||
@@ -765,7 +800,7 @@ func (s *System) RunCPUAcceptancePack(ctx context.Context, baseDir string, durat
|
||||
return runAcceptancePackCtx(ctx, baseDir, "cpu", []satJob{
|
||||
{name: "01-lscpu.log", cmd: []string{"lscpu"}},
|
||||
{name: "02-sensors-before.log", cmd: []string{"sensors"}},
|
||||
{name: "03-stress-ng.log", cmd: []string{"stress-ng", "--cpu", "0", "--cpu-method", "all", "--timeout", fmt.Sprintf("%d", durationSec)}},
|
||||
{name: "03-stress-ng.log", cmd: []string{"stress-ng", "--cpu", "0", "--cpu-method", "all", "--timeout", fmt.Sprintf("%d", durationSec)}, syncBracket: true},
|
||||
{name: "04-sensors-after.log", cmd: []string{"sensors"}},
|
||||
}, logFunc)
|
||||
}
|
||||
@@ -819,6 +854,7 @@ func (s *System) RunStorageAcceptancePack(ctx context.Context, baseDir string, e
|
||||
}
|
||||
name := fmt.Sprintf("%s-%02d-%s.log", prefix, cmdIndex+1, job.name)
|
||||
livePath := filepath.Join(runDir, name)
|
||||
runSyncBracketHook(job, "before", logFunc)
|
||||
out, err := runSATCommandCtx(ctx, verboseLog, job.name, job.cmd, nil, logFunc, livePath)
|
||||
deviceOutputs[job.name] = out
|
||||
if writeErr := os.WriteFile(livePath, out, 0644); writeErr != nil {
|
||||
@@ -827,17 +863,23 @@ func (s *System) RunStorageAcceptancePack(ctx context.Context, baseDir string, e
|
||||
if satJobBoundaryHook != nil {
|
||||
satJobBoundaryHook(name)
|
||||
}
|
||||
// smartctl -t short only launches the self-test on the drive firmware and
|
||||
// returns immediately ("Testing has begun"); unlike `nvme device-self-test
|
||||
// --wait`, smartctl has no blocking mode, so we must poll the drive
|
||||
// ourselves until the self-test actually finishes. Hold the "after" sync
|
||||
// until that poll completes — the self-test itself, not just its launch,
|
||||
// is the load worth having durable evidence of.
|
||||
deferSyncBracketAfter := job.name == "smartctl-self-test-short" && err == nil
|
||||
if !deferSyncBracketAfter {
|
||||
runSyncBracketHook(job, "after", logFunc)
|
||||
}
|
||||
status, rc := classifySATResult(job.name, out, err)
|
||||
stats.Add(status)
|
||||
key := filepath.Base(devPath) + "_" + strings.ReplaceAll(job.name, "-", "_")
|
||||
fmt.Fprintf(&summary, "%s_rc=%d\n", key, rc)
|
||||
fmt.Fprintf(&summary, "%s_status=%s\n", key, status)
|
||||
|
||||
// smartctl -t short only launches the self-test on the drive firmware and
|
||||
// returns immediately ("Testing has begun"); unlike `nvme device-self-test
|
||||
// --wait`, smartctl has no blocking mode, so we must poll the drive
|
||||
// ourselves until the self-test actually finishes.
|
||||
if job.name == "smartctl-self-test-short" && err == nil {
|
||||
if deferSyncBracketAfter {
|
||||
statusName := "smartctl-self-test-status"
|
||||
statusOut := waitForSmartctlSelfTest(ctx, verboseLog, devPath, logFunc)
|
||||
deviceOutputs[statusName] = statusOut
|
||||
@@ -845,6 +887,7 @@ func (s *System) RunStorageAcceptancePack(ctx context.Context, baseDir string, e
|
||||
if writeErr := os.WriteFile(filepath.Join(runDir, statusFile), statusOut, 0644); writeErr != nil {
|
||||
return "", writeErr
|
||||
}
|
||||
runSyncBracketHook(job, "after", logFunc)
|
||||
sStatus, sRC := classifySATResult(statusName, statusOut, nil)
|
||||
stats.Add(sStatus)
|
||||
sKey := filepath.Base(devPath) + "_" + strings.ReplaceAll(statusName, "-", "_")
|
||||
@@ -878,6 +921,11 @@ type satJob struct {
|
||||
// retries is the number of extra attempts (with a short backoff) if the
|
||||
// job's first run fails. Used for jobs racing nv-hostengine startup.
|
||||
retries int
|
||||
// syncBracket marks a job as the actual load step of a pack (as opposed
|
||||
// to the cheap inventory/discovery steps around it) — see
|
||||
// satSyncBracketHook. Set this on the command that can hang or crash the
|
||||
// host, not on nvidia-smi/dcgmi discovery calls.
|
||||
syncBracket bool
|
||||
}
|
||||
|
||||
type satStats struct {
|
||||
@@ -903,7 +951,7 @@ func nvidiaSATJobs() []satJob {
|
||||
satJob{name: "02-dmidecode-baseboard.log", cmd: []string{"dmidecode", "-t", "baseboard"}},
|
||||
satJob{name: "03-dmidecode-system.log", cmd: []string{"dmidecode", "-t", "system"}},
|
||||
satJob{name: "04-nvidia-bug-report.log", cmd: []string{"nvidia-bug-report.sh", "--output-file", "{{run_dir}}/nvidia-bug-report.log"}},
|
||||
satJob{name: "05-bee-gpu-burn.log", cmd: []string{"bee-gpu-burn", "--seconds", "5", "--size-mb", "64"}},
|
||||
satJob{name: "05-bee-gpu-burn.log", cmd: []string{"bee-gpu-burn", "--seconds", "5", "--size-mb", "64"}, syncBracket: true},
|
||||
)
|
||||
}
|
||||
|
||||
@@ -924,7 +972,7 @@ func nvidiaDCGMJobs(diagLevel int, gpuIndices []int) []satJob {
|
||||
satJob{name: "02-dmidecode-baseboard.log", cmd: []string{"dmidecode", "-t", "baseboard"}},
|
||||
satJob{name: "03-dmidecode-system.log", cmd: []string{"dmidecode", "-t", "system"}},
|
||||
satJob{name: "04-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
|
||||
satJob{name: "05-dcgmi-diag.log", cmd: diagArgs, gpuIndices: gpuIndices},
|
||||
satJob{name: "05-dcgmi-diag.log", cmd: diagArgs, gpuIndices: gpuIndices, syncBracket: true},
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1016,6 +1064,7 @@ func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []sa
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
runSyncBracketHook(job, "before", logFunc)
|
||||
for attempt := 0; ; attempt++ {
|
||||
if job.collectGPU {
|
||||
out, err = runSATCommandWithMetrics(ctx, verboseLog, job.name, cmd, job.env, job.gpuIndices, runDir, logFunc)
|
||||
@@ -1053,6 +1102,7 @@ func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []sa
|
||||
if satJobBoundaryHook != nil {
|
||||
satJobBoundaryHook(job.name)
|
||||
}
|
||||
runSyncBracketHook(job, "after", logFunc)
|
||||
if ctx.Err() != nil {
|
||||
return "", ctx.Err()
|
||||
}
|
||||
@@ -1383,7 +1433,7 @@ func storageSATCommands(devPath string, extended bool) []satJob {
|
||||
{name: "nvme-smart-log", cmd: []string{"nvme", "smart-log", devPath, "-o", "json"}},
|
||||
}
|
||||
if extended {
|
||||
jobs = append(jobs, satJob{name: "nvme-device-self-test", cmd: []string{"nvme", "device-self-test", devPath, "-s", "1", "--wait"}})
|
||||
jobs = append(jobs, satJob{name: "nvme-device-self-test", cmd: []string{"nvme", "device-self-test", devPath, "-s", "1", "--wait"}, syncBracket: true})
|
||||
}
|
||||
return jobs
|
||||
}
|
||||
@@ -1391,7 +1441,7 @@ func storageSATCommands(devPath string, extended bool) []satJob {
|
||||
{name: "smartctl-health", cmd: []string{"smartctl", "-H", "-A", "-i", devPath}},
|
||||
}
|
||||
if extended {
|
||||
jobs = append(jobs, satJob{name: "smartctl-self-test-short", cmd: []string{"smartctl", "-t", "short", devPath}})
|
||||
jobs = append(jobs, satJob{name: "smartctl-self-test-short", cmd: []string{"smartctl", "-t", "short", devPath}, syncBracket: true})
|
||||
}
|
||||
return jobs
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user