platform/app: fix nvbandwidth split fallback, blackbox discovery churn, and add blocking sync brackets around load steps

- gpuBandwidthSocketGroups: a single GPU whose NUMA node fails to resolve
  no longer collapses the whole per-socket nvbandwidth split into one
  fallback pass — it now folds into the last resolved group instead,
  preserving isolation for the sockets that did resolve.
- blackbox discoverMarkedTargets: skip mounting/unmounting devices that
  already have a running worker on every 2s discovery tick. This was
  observed hammering the same USB target continuously (mount+unmount
  every ~2s for the whole session) and contending with the worker's own
  sync cycle, plausibly explaining multi-minute sync cycles seen on a
  real crash bundle.
- syncFilesystem now calls syscall.Sync() directly instead of spawning
  /bin/sync per copied file; blackbox mounts removable targets with
  -o sync so writes are durable without relying on the app-level sync as
  the primary mechanism.
- New platform.SetSyncBracketHook / satJob.syncBracket: blocks (with a
  bounded timeout) on blackbox actually reaching removable media right
  before and right after a diagnostic's real load step (nvbandwidth,
  memtester, stress-ng, dcgmi diag, nccl, smartctl/nvme self-test...),
  instead of only firing a fire-and-forget kick after the job's own log
  file is written. A crash mid-load now has durable evidence the load
  started, not just whatever streamed to the RAM-backed export dir before
  blackbox's next scheduled cycle.

Found investigating a real support bundle where blackbox's last
successful sync (19:55:25) predated both the previous job finishing and
the crashing nvbandwidth job starting (19:56:57) — none of the crash
window ever reached durable media.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-07-28 14:22:05 +03:00
co-authored by Claude Sonnet 5
parent 6d13c17d36
commit e03267a72f
8 changed files with 438 additions and 51 deletions
@@ -31,16 +31,22 @@ func gpuBandwidthSocketGroups(gpuIndices []int, logFunc func(string)) [][]int {
}
byNode := map[int][]int{}
var unresolved []int
for _, idx := range gpuIndices {
node, ok := nodes[idx]
if !ok {
if logFunc != nil {
logFunc(fmt.Sprintf("nvbandwidth: no NUMA node resolved for GPU %d; running all GPUs as one group", idx))
logFunc(fmt.Sprintf("nvbandwidth: no NUMA node resolved for GPU %d; will fold it into a resolved socket group instead of dropping the split", idx))
}
return [][]int{gpuIndices}
unresolved = append(unresolved, idx)
continue
}
byNode[node] = append(byNode[node], idx)
}
// Fewer than two resolved sockets means there's nothing to split either
// way: every GPU's node is unknown, or every resolved GPU shares one
// socket. A single unresolved GPU among an otherwise clean multi-socket
// system shouldn't cost us the split, so only bail out here.
if len(byNode) < 2 {
return [][]int{gpuIndices}
}
@@ -55,6 +61,14 @@ func gpuBandwidthSocketGroups(gpuIndices []int, logFunc func(string)) [][]int {
for _, node := range sortedNodes {
groups = append(groups, dedupeSortedIndices(byNode[node]))
}
if len(unresolved) > 0 {
// Fold into the last group rather than running unresolved GPUs in a
// group of their own — a lone GPU can't run a GPU-to-GPU bandwidth
// test by itself, and the point of the split is to isolate the
// sockets we *do* know about, not to also isolate the unknown one.
last := len(groups) - 1
groups[last] = dedupeSortedIndices(append(groups[last], unresolved...))
}
return groups
}
@@ -91,6 +91,33 @@ func TestGPUBandwidthSocketGroupsSplitsBySocket(t *testing.T) {
}
}
func TestGPUBandwidthSocketGroupsFoldsUnresolvedIntoLastGroup(t *testing.T) {
// GPU 4's NUMA node fails to resolve (e.g. a flaky sysfs read), but the
// other 5 GPUs still clearly span two sockets — the split should survive
// and GPU 4 should ride along with the last group rather than being
// tested alone or collapsing the whole thing to one pass.
fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:06:00.0\n2, 00000000:76:00.0\n3, 00000000:77:00.0\n4, 00000000:F4:00.0\n5, 00000000:F5:00.0\n")
fakeNUMANodes(t, map[string]string{
"0000:05:00.0": "0\n",
"0000:06:00.0": "0\n",
"0000:76:00.0": "0\n",
"0000:77:00.0": "0\n",
// GPU 4 (F4:00.0) deliberately missing.
"0000:F5:00.0": "1\n",
})
groups := gpuBandwidthSocketGroups([]int{0, 1, 2, 3, 4, 5}, nil)
if len(groups) != 2 {
t.Fatalf("groups=%v want 2 groups", groups)
}
if joinIndexList(groups[0]) != "0,1,2,3" {
t.Fatalf("groups[0]=%v want 0,1,2,3", groups[0])
}
if joinIndexList(groups[1]) != "4,5" {
t.Fatalf("groups[1]=%v want 4,5 (unresolved GPU 4 folded into last group)", groups[1])
}
}
func TestGPUBandwidthSocketGroupsFallsBackToSingleGroup(t *testing.T) {
t.Run("single NUMA node", func(t *testing.T) {
fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:06:00.0\n")
+92 -42
View File
@@ -83,6 +83,34 @@ func SetJobBoundaryHook(hook func(jobName string)) {
satJobBoundaryHook = hook
}
// satSyncBracketHook, if set, is called synchronously immediately before and
// after a satJob marked syncBracket runs — i.e. around the actual load
// command of a diagnostic (nvbandwidth, memtester, stress-ng, dcgmi diag...),
// not the cheap discovery/inventory steps around it. phase is "before" or
// "after". Unlike satJobBoundaryHook (fire-and-forget, fires after every
// job), this is meant to block until an external blackbox sync has actually
// reached removable media, so that a crash during the load itself still
// leaves durable evidence that the load started (and, on the far side, that
// it finished). Nil by default. An error is logged, never fails the job —
// a stuck blackbox target must not block the diagnostic the operator asked
// for.
var satSyncBracketHook func(jobName, phase string) error
// SetSyncBracketHook installs the callback invoked synchronously before and
// after a syncBracket-marked SAT job. Pass nil to clear it.
func SetSyncBracketHook(hook func(jobName, phase string) error) {
satSyncBracketHook = hook
}
func runSyncBracketHook(job satJob, phase string, logFunc func(string)) {
if !job.syncBracket || satSyncBracketHook == nil {
return
}
if err := satSyncBracketHook(job.name, phase); err != nil && logFunc != nil {
logFunc(fmt.Sprintf("%s: blackbox sync wait (%s) did not complete cleanly: %v", job.name, phase, err))
}
}
var (
satExecCommand = exec.Command
satLookPath = exec.LookPath
@@ -463,7 +491,7 @@ func (s *System) RunNCCLTests(ctx context.Context, baseDir string, gpuIndices []
satJob{name: "02-all-reduce-perf.log", cmd: []string{
"all_reduce_perf", "-b", "512M", "-e", "4G", "-f", "2",
"-g", strconv.Itoa(gpuCount), "--iters", "20",
}, env: nvidiaVisibleDevicesEnv(selected)},
}, env: nvidiaVisibleDevicesEnv(selected), syncBracket: true},
), logFunc)
}
@@ -502,11 +530,12 @@ func (s *System) RunNvidiaOfficialComputePack(ctx context.Context, baseDir strin
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
satJob{name: "02-dcgmi-version.log", cmd: []string{"dcgmi", "-v"}},
satJob{
name: "03-dcgmproftester.log",
cmd: profCmd,
env: profEnv,
collectGPU: true,
gpuIndices: selected,
name: "03-dcgmproftester.log",
cmd: profCmd,
env: profEnv,
collectGPU: true,
gpuIndices: selected,
syncBracket: true,
},
satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
), logFunc)
@@ -528,10 +557,11 @@ func (s *System) RunNvidiaTargetedPowerPack(ctx context.Context, baseDir string,
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
satJob{
name: "03-dcgmi-targeted-power.log",
cmd: nvidiaDCGMNamedDiagCommand("targeted_power", normalizeNvidiaBurnDuration(durationSec), selected),
collectGPU: true,
gpuIndices: selected,
name: "03-dcgmi-targeted-power.log",
cmd: nvidiaDCGMNamedDiagCommand("targeted_power", normalizeNvidiaBurnDuration(durationSec), selected),
collectGPU: true,
gpuIndices: selected,
syncBracket: true,
},
satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
), logFunc)
@@ -553,10 +583,11 @@ func (s *System) RunNvidiaPulseTestPack(ctx context.Context, baseDir string, dur
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
satJob{
name: "03-dcgmi-pulse-test.log",
cmd: nvidiaDCGMNamedDiagCommand("pulse_test", normalizeNvidiaBurnDuration(durationSec), selected),
collectGPU: true,
gpuIndices: selected,
name: "03-dcgmi-pulse-test.log",
cmd: nvidiaDCGMNamedDiagCommand("pulse_test", normalizeNvidiaBurnDuration(durationSec), selected),
collectGPU: true,
gpuIndices: selected,
syncBracket: true,
},
satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
), logFunc)
@@ -591,27 +622,30 @@ func (s *System) RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpu
socketGroups := gpuBandwidthSocketGroups(selected, logFunc)
if len(socketGroups) <= 1 {
jobs = append(jobs, satJob{
name: fmt.Sprintf("%02d-dcgmi-nvbandwidth.log", step),
cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, selected),
collectGPU: true,
gpuIndices: selected,
name: fmt.Sprintf("%02d-dcgmi-nvbandwidth.log", step),
cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, selected),
collectGPU: true,
gpuIndices: selected,
syncBracket: true,
})
step++
} else {
for i, group := range socketGroups {
jobs = append(jobs, satJob{
name: fmt.Sprintf("%02d-dcgmi-nvbandwidth-socket%d.log", step, i),
cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, group),
collectGPU: true,
gpuIndices: group,
name: fmt.Sprintf("%02d-dcgmi-nvbandwidth-socket%d.log", step, i),
cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, group),
collectGPU: true,
gpuIndices: group,
syncBracket: true,
})
step++
}
jobs = append(jobs, satJob{
name: fmt.Sprintf("%02d-dcgmi-nvbandwidth-all.log", step),
cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, selected),
collectGPU: true,
gpuIndices: selected,
name: fmt.Sprintf("%02d-dcgmi-nvbandwidth-all.log", step),
cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, selected),
collectGPU: true,
gpuIndices: selected,
syncBracket: true,
})
step++
}
@@ -655,10 +689,11 @@ func (s *System) RunNvidiaTargetedStressValidatePack(ctx context.Context, baseDi
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
satJob{
name: "03-dcgmi-targeted-stress.log",
cmd: nvidiaDCGMNamedDiagCommand("targeted_stress", normalizeNvidiaBurnDuration(durationSec), selected),
collectGPU: true,
gpuIndices: selected,
name: "03-dcgmi-targeted-stress.log",
cmd: nvidiaDCGMNamedDiagCommand("targeted_stress", normalizeNvidiaBurnDuration(durationSec), selected),
collectGPU: true,
gpuIndices: selected,
syncBracket: true,
},
satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
), logFunc)
@@ -716,7 +751,7 @@ func (s *System) RunMemoryAcceptancePack(ctx context.Context, baseDir string, si
}
return runAcceptancePackCtx(ctx, baseDir, "memory", []satJob{
{name: "01-free-before.log", cmd: []string{"free", "-h"}},
{name: "02-memtester.log", cmd: []string{"timeout", fmt.Sprintf("%d", timeoutSec), "memtester", fmt.Sprintf("%dM", sizeMB), fmt.Sprintf("%d", passes)}},
{name: "02-memtester.log", cmd: []string{"timeout", fmt.Sprintf("%d", timeoutSec), "memtester", fmt.Sprintf("%dM", sizeMB), fmt.Sprintf("%d", passes)}, syncBracket: true},
{name: "03-free-after.log", cmd: []string{"free", "-h"}},
}, logFunc)
}
@@ -737,7 +772,7 @@ func (s *System) RunMemoryStressPack(ctx context.Context, baseDir string, durati
"--vm-method", "all",
"--timeout", fmt.Sprintf("%d", seconds),
"--metrics-brief",
}},
}, syncBracket: true},
{name: "03-free-after.log", cmd: []string{"free", "-h"}},
}, logFunc)
}
@@ -765,7 +800,7 @@ func (s *System) RunCPUAcceptancePack(ctx context.Context, baseDir string, durat
return runAcceptancePackCtx(ctx, baseDir, "cpu", []satJob{
{name: "01-lscpu.log", cmd: []string{"lscpu"}},
{name: "02-sensors-before.log", cmd: []string{"sensors"}},
{name: "03-stress-ng.log", cmd: []string{"stress-ng", "--cpu", "0", "--cpu-method", "all", "--timeout", fmt.Sprintf("%d", durationSec)}},
{name: "03-stress-ng.log", cmd: []string{"stress-ng", "--cpu", "0", "--cpu-method", "all", "--timeout", fmt.Sprintf("%d", durationSec)}, syncBracket: true},
{name: "04-sensors-after.log", cmd: []string{"sensors"}},
}, logFunc)
}
@@ -819,6 +854,7 @@ func (s *System) RunStorageAcceptancePack(ctx context.Context, baseDir string, e
}
name := fmt.Sprintf("%s-%02d-%s.log", prefix, cmdIndex+1, job.name)
livePath := filepath.Join(runDir, name)
runSyncBracketHook(job, "before", logFunc)
out, err := runSATCommandCtx(ctx, verboseLog, job.name, job.cmd, nil, logFunc, livePath)
deviceOutputs[job.name] = out
if writeErr := os.WriteFile(livePath, out, 0644); writeErr != nil {
@@ -827,17 +863,23 @@ func (s *System) RunStorageAcceptancePack(ctx context.Context, baseDir string, e
if satJobBoundaryHook != nil {
satJobBoundaryHook(name)
}
// smartctl -t short only launches the self-test on the drive firmware and
// returns immediately ("Testing has begun"); unlike `nvme device-self-test
// --wait`, smartctl has no blocking mode, so we must poll the drive
// ourselves until the self-test actually finishes. Hold the "after" sync
// until that poll completes — the self-test itself, not just its launch,
// is the load worth having durable evidence of.
deferSyncBracketAfter := job.name == "smartctl-self-test-short" && err == nil
if !deferSyncBracketAfter {
runSyncBracketHook(job, "after", logFunc)
}
status, rc := classifySATResult(job.name, out, err)
stats.Add(status)
key := filepath.Base(devPath) + "_" + strings.ReplaceAll(job.name, "-", "_")
fmt.Fprintf(&summary, "%s_rc=%d\n", key, rc)
fmt.Fprintf(&summary, "%s_status=%s\n", key, status)
// smartctl -t short only launches the self-test on the drive firmware and
// returns immediately ("Testing has begun"); unlike `nvme device-self-test
// --wait`, smartctl has no blocking mode, so we must poll the drive
// ourselves until the self-test actually finishes.
if job.name == "smartctl-self-test-short" && err == nil {
if deferSyncBracketAfter {
statusName := "smartctl-self-test-status"
statusOut := waitForSmartctlSelfTest(ctx, verboseLog, devPath, logFunc)
deviceOutputs[statusName] = statusOut
@@ -845,6 +887,7 @@ func (s *System) RunStorageAcceptancePack(ctx context.Context, baseDir string, e
if writeErr := os.WriteFile(filepath.Join(runDir, statusFile), statusOut, 0644); writeErr != nil {
return "", writeErr
}
runSyncBracketHook(job, "after", logFunc)
sStatus, sRC := classifySATResult(statusName, statusOut, nil)
stats.Add(sStatus)
sKey := filepath.Base(devPath) + "_" + strings.ReplaceAll(statusName, "-", "_")
@@ -878,6 +921,11 @@ type satJob struct {
// retries is the number of extra attempts (with a short backoff) if the
// job's first run fails. Used for jobs racing nv-hostengine startup.
retries int
// syncBracket marks a job as the actual load step of a pack (as opposed
// to the cheap inventory/discovery steps around it) — see
// satSyncBracketHook. Set this on the command that can hang or crash the
// host, not on nvidia-smi/dcgmi discovery calls.
syncBracket bool
}
type satStats struct {
@@ -903,7 +951,7 @@ func nvidiaSATJobs() []satJob {
satJob{name: "02-dmidecode-baseboard.log", cmd: []string{"dmidecode", "-t", "baseboard"}},
satJob{name: "03-dmidecode-system.log", cmd: []string{"dmidecode", "-t", "system"}},
satJob{name: "04-nvidia-bug-report.log", cmd: []string{"nvidia-bug-report.sh", "--output-file", "{{run_dir}}/nvidia-bug-report.log"}},
satJob{name: "05-bee-gpu-burn.log", cmd: []string{"bee-gpu-burn", "--seconds", "5", "--size-mb", "64"}},
satJob{name: "05-bee-gpu-burn.log", cmd: []string{"bee-gpu-burn", "--seconds", "5", "--size-mb", "64"}, syncBracket: true},
)
}
@@ -924,7 +972,7 @@ func nvidiaDCGMJobs(diagLevel int, gpuIndices []int) []satJob {
satJob{name: "02-dmidecode-baseboard.log", cmd: []string{"dmidecode", "-t", "baseboard"}},
satJob{name: "03-dmidecode-system.log", cmd: []string{"dmidecode", "-t", "system"}},
satJob{name: "04-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
satJob{name: "05-dcgmi-diag.log", cmd: diagArgs, gpuIndices: gpuIndices},
satJob{name: "05-dcgmi-diag.log", cmd: diagArgs, gpuIndices: gpuIndices, syncBracket: true},
)
}
@@ -1016,6 +1064,7 @@ func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []sa
}
if err == nil {
runSyncBracketHook(job, "before", logFunc)
for attempt := 0; ; attempt++ {
if job.collectGPU {
out, err = runSATCommandWithMetrics(ctx, verboseLog, job.name, cmd, job.env, job.gpuIndices, runDir, logFunc)
@@ -1053,6 +1102,7 @@ func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []sa
if satJobBoundaryHook != nil {
satJobBoundaryHook(job.name)
}
runSyncBracketHook(job, "after", logFunc)
if ctx.Err() != nil {
return "", ctx.Err()
}
@@ -1383,7 +1433,7 @@ func storageSATCommands(devPath string, extended bool) []satJob {
{name: "nvme-smart-log", cmd: []string{"nvme", "smart-log", devPath, "-o", "json"}},
}
if extended {
jobs = append(jobs, satJob{name: "nvme-device-self-test", cmd: []string{"nvme", "device-self-test", devPath, "-s", "1", "--wait"}})
jobs = append(jobs, satJob{name: "nvme-device-self-test", cmd: []string{"nvme", "device-self-test", devPath, "-s", "1", "--wait"}, syncBracket: true})
}
return jobs
}
@@ -1391,7 +1441,7 @@ func storageSATCommands(devPath string, extended bool) []satJob {
{name: "smartctl-health", cmd: []string{"smartctl", "-H", "-A", "-i", devPath}},
}
if extended {
jobs = append(jobs, satJob{name: "smartctl-self-test-short", cmd: []string{"smartctl", "-t", "short", devPath}})
jobs = append(jobs, satJob{name: "smartctl-self-test-short", cmd: []string{"smartctl", "-t", "short", devPath}, syncBracket: true})
}
return jobs
}
@@ -2,9 +2,11 @@ package platform
import (
"context"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
)
@@ -95,3 +97,56 @@ func TestRunAcceptancePackCtxInvokesJobBoundaryHook(t *testing.T) {
t.Fatalf("seen=%v want [01-a.log 02-b.log]", seen)
}
}
func TestRunAcceptancePackCtxInvokesSyncBracketHookOnlyForMarkedJobs(t *testing.T) {
old := satSyncBracketHook
t.Cleanup(func() { satSyncBracketHook = old })
var calls []string
SetSyncBracketHook(func(jobName, phase string) error {
calls = append(calls, jobName+":"+phase)
return nil
})
oldExecCommand := satExecCommand
satExecCommand = func(name string, args ...string) *exec.Cmd {
return exec.Command(name, args...)
}
t.Cleanup(func() { satExecCommand = oldExecCommand })
dir := t.TempDir()
_, err := runAcceptancePackCtx(context.Background(), dir, "test-pack", []satJob{
{name: "01-inventory.log", cmd: []string{"printf", "a\n"}},
{name: "02-load.log", cmd: []string{"printf", "b\n"}, syncBracket: true},
}, nil)
if err != nil {
t.Fatalf("runAcceptancePackCtx error: %v", err)
}
want := []string{"02-load.log:before", "02-load.log:after"}
if len(calls) != len(want) {
t.Fatalf("calls=%v want %v", calls, want)
}
for i := range want {
if calls[i] != want[i] {
t.Fatalf("calls=%v want %v", calls, want)
}
}
}
func TestRunSyncBracketHookLogsErrorWithoutFailingJob(t *testing.T) {
old := satSyncBracketHook
t.Cleanup(func() { satSyncBracketHook = old })
SetSyncBracketHook(func(jobName, phase string) error {
return fmt.Errorf("blackbox target unreachable")
})
var logged []string
runSyncBracketHook(satJob{name: "02-load.log", syncBracket: true}, "before", func(line string) {
logged = append(logged, line)
})
if len(logged) != 1 || !strings.Contains(logged[0], "02-load.log") || !strings.Contains(logged[0], "blackbox target unreachable") {
t.Fatalf("logged=%v want a single line naming the job and the error", logged)
}
}