platform/app: fix nvbandwidth split fallback, blackbox discovery churn, and add blocking sync brackets around load steps
- gpuBandwidthSocketGroups: a single GPU whose NUMA node fails to resolve no longer collapses the whole per-socket nvbandwidth split into one fallback pass — it now folds into the last resolved group instead, preserving isolation for the sockets that did resolve. - blackbox discoverMarkedTargets: skip mounting/unmounting devices that already have a running worker on every 2s discovery tick. This was observed hammering the same USB target continuously (mount+unmount every ~2s for the whole session) and contending with the worker's own sync cycle, plausibly explaining multi-minute sync cycles seen on a real crash bundle. - syncFilesystem now calls syscall.Sync() directly instead of spawning /bin/sync per copied file; blackbox mounts removable targets with -o sync so writes are durable without relying on the app-level sync as the primary mechanism. - New platform.SetSyncBracketHook / satJob.syncBracket: blocks (with a bounded timeout) on blackbox actually reaching removable media right before and right after a diagnostic's real load step (nvbandwidth, memtester, stress-ng, dcgmi diag, nccl, smartctl/nvme self-test...), instead of only firing a fire-and-forget kick after the job's own log file is written. A crash mid-load now has durable evidence the load started, not just whatever streamed to the RAM-backed export dir before blackbox's next scheduled cycle. Found investigating a real support bundle where blackbox's last successful sync (19:55:25) predated both the previous job finishing and the crashing nvbandwidth job starting (19:56:57) — none of the crash window ever reached durable media. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
6d13c17d36
commit
e03267a72f
@@ -31,16 +31,22 @@ func gpuBandwidthSocketGroups(gpuIndices []int, logFunc func(string)) [][]int {
|
||||
}
|
||||
|
||||
byNode := map[int][]int{}
|
||||
var unresolved []int
|
||||
for _, idx := range gpuIndices {
|
||||
node, ok := nodes[idx]
|
||||
if !ok {
|
||||
if logFunc != nil {
|
||||
logFunc(fmt.Sprintf("nvbandwidth: no NUMA node resolved for GPU %d; running all GPUs as one group", idx))
|
||||
logFunc(fmt.Sprintf("nvbandwidth: no NUMA node resolved for GPU %d; will fold it into a resolved socket group instead of dropping the split", idx))
|
||||
}
|
||||
return [][]int{gpuIndices}
|
||||
unresolved = append(unresolved, idx)
|
||||
continue
|
||||
}
|
||||
byNode[node] = append(byNode[node], idx)
|
||||
}
|
||||
// Fewer than two resolved sockets means there's nothing to split either
|
||||
// way: every GPU's node is unknown, or every resolved GPU shares one
|
||||
// socket. A single unresolved GPU among an otherwise clean multi-socket
|
||||
// system shouldn't cost us the split, so only bail out here.
|
||||
if len(byNode) < 2 {
|
||||
return [][]int{gpuIndices}
|
||||
}
|
||||
@@ -55,6 +61,14 @@ func gpuBandwidthSocketGroups(gpuIndices []int, logFunc func(string)) [][]int {
|
||||
for _, node := range sortedNodes {
|
||||
groups = append(groups, dedupeSortedIndices(byNode[node]))
|
||||
}
|
||||
if len(unresolved) > 0 {
|
||||
// Fold into the last group rather than running unresolved GPUs in a
|
||||
// group of their own — a lone GPU can't run a GPU-to-GPU bandwidth
|
||||
// test by itself, and the point of the split is to isolate the
|
||||
// sockets we *do* know about, not to also isolate the unknown one.
|
||||
last := len(groups) - 1
|
||||
groups[last] = dedupeSortedIndices(append(groups[last], unresolved...))
|
||||
}
|
||||
return groups
|
||||
}
|
||||
|
||||
|
||||
@@ -91,6 +91,33 @@ func TestGPUBandwidthSocketGroupsSplitsBySocket(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGPUBandwidthSocketGroupsFoldsUnresolvedIntoLastGroup(t *testing.T) {
|
||||
// GPU 4's NUMA node fails to resolve (e.g. a flaky sysfs read), but the
|
||||
// other 5 GPUs still clearly span two sockets — the split should survive
|
||||
// and GPU 4 should ride along with the last group rather than being
|
||||
// tested alone or collapsing the whole thing to one pass.
|
||||
fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:06:00.0\n2, 00000000:76:00.0\n3, 00000000:77:00.0\n4, 00000000:F4:00.0\n5, 00000000:F5:00.0\n")
|
||||
fakeNUMANodes(t, map[string]string{
|
||||
"0000:05:00.0": "0\n",
|
||||
"0000:06:00.0": "0\n",
|
||||
"0000:76:00.0": "0\n",
|
||||
"0000:77:00.0": "0\n",
|
||||
// GPU 4 (F4:00.0) deliberately missing.
|
||||
"0000:F5:00.0": "1\n",
|
||||
})
|
||||
|
||||
groups := gpuBandwidthSocketGroups([]int{0, 1, 2, 3, 4, 5}, nil)
|
||||
if len(groups) != 2 {
|
||||
t.Fatalf("groups=%v want 2 groups", groups)
|
||||
}
|
||||
if joinIndexList(groups[0]) != "0,1,2,3" {
|
||||
t.Fatalf("groups[0]=%v want 0,1,2,3", groups[0])
|
||||
}
|
||||
if joinIndexList(groups[1]) != "4,5" {
|
||||
t.Fatalf("groups[1]=%v want 4,5 (unresolved GPU 4 folded into last group)", groups[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestGPUBandwidthSocketGroupsFallsBackToSingleGroup(t *testing.T) {
|
||||
t.Run("single NUMA node", func(t *testing.T) {
|
||||
fakeNvidiaSmiBusIDs(t, "0, 00000000:05:00.0\n1, 00000000:06:00.0\n")
|
||||
|
||||
@@ -83,6 +83,34 @@ func SetJobBoundaryHook(hook func(jobName string)) {
|
||||
satJobBoundaryHook = hook
|
||||
}
|
||||
|
||||
// satSyncBracketHook, if set, is called synchronously immediately before and
|
||||
// after a satJob marked syncBracket runs — i.e. around the actual load
|
||||
// command of a diagnostic (nvbandwidth, memtester, stress-ng, dcgmi diag...),
|
||||
// not the cheap discovery/inventory steps around it. phase is "before" or
|
||||
// "after". Unlike satJobBoundaryHook (fire-and-forget, fires after every
|
||||
// job), this is meant to block until an external blackbox sync has actually
|
||||
// reached removable media, so that a crash during the load itself still
|
||||
// leaves durable evidence that the load started (and, on the far side, that
|
||||
// it finished). Nil by default. An error is logged, never fails the job —
|
||||
// a stuck blackbox target must not block the diagnostic the operator asked
|
||||
// for.
|
||||
var satSyncBracketHook func(jobName, phase string) error
|
||||
|
||||
// SetSyncBracketHook installs the callback invoked synchronously before and
|
||||
// after a syncBracket-marked SAT job. Pass nil to clear it.
|
||||
func SetSyncBracketHook(hook func(jobName, phase string) error) {
|
||||
satSyncBracketHook = hook
|
||||
}
|
||||
|
||||
func runSyncBracketHook(job satJob, phase string, logFunc func(string)) {
|
||||
if !job.syncBracket || satSyncBracketHook == nil {
|
||||
return
|
||||
}
|
||||
if err := satSyncBracketHook(job.name, phase); err != nil && logFunc != nil {
|
||||
logFunc(fmt.Sprintf("%s: blackbox sync wait (%s) did not complete cleanly: %v", job.name, phase, err))
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
satExecCommand = exec.Command
|
||||
satLookPath = exec.LookPath
|
||||
@@ -463,7 +491,7 @@ func (s *System) RunNCCLTests(ctx context.Context, baseDir string, gpuIndices []
|
||||
satJob{name: "02-all-reduce-perf.log", cmd: []string{
|
||||
"all_reduce_perf", "-b", "512M", "-e", "4G", "-f", "2",
|
||||
"-g", strconv.Itoa(gpuCount), "--iters", "20",
|
||||
}, env: nvidiaVisibleDevicesEnv(selected)},
|
||||
}, env: nvidiaVisibleDevicesEnv(selected), syncBracket: true},
|
||||
), logFunc)
|
||||
}
|
||||
|
||||
@@ -502,11 +530,12 @@ func (s *System) RunNvidiaOfficialComputePack(ctx context.Context, baseDir strin
|
||||
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
|
||||
satJob{name: "02-dcgmi-version.log", cmd: []string{"dcgmi", "-v"}},
|
||||
satJob{
|
||||
name: "03-dcgmproftester.log",
|
||||
cmd: profCmd,
|
||||
env: profEnv,
|
||||
collectGPU: true,
|
||||
gpuIndices: selected,
|
||||
name: "03-dcgmproftester.log",
|
||||
cmd: profCmd,
|
||||
env: profEnv,
|
||||
collectGPU: true,
|
||||
gpuIndices: selected,
|
||||
syncBracket: true,
|
||||
},
|
||||
satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
|
||||
), logFunc)
|
||||
@@ -528,10 +557,11 @@ func (s *System) RunNvidiaTargetedPowerPack(ctx context.Context, baseDir string,
|
||||
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
|
||||
satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
|
||||
satJob{
|
||||
name: "03-dcgmi-targeted-power.log",
|
||||
cmd: nvidiaDCGMNamedDiagCommand("targeted_power", normalizeNvidiaBurnDuration(durationSec), selected),
|
||||
collectGPU: true,
|
||||
gpuIndices: selected,
|
||||
name: "03-dcgmi-targeted-power.log",
|
||||
cmd: nvidiaDCGMNamedDiagCommand("targeted_power", normalizeNvidiaBurnDuration(durationSec), selected),
|
||||
collectGPU: true,
|
||||
gpuIndices: selected,
|
||||
syncBracket: true,
|
||||
},
|
||||
satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
|
||||
), logFunc)
|
||||
@@ -553,10 +583,11 @@ func (s *System) RunNvidiaPulseTestPack(ctx context.Context, baseDir string, dur
|
||||
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
|
||||
satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
|
||||
satJob{
|
||||
name: "03-dcgmi-pulse-test.log",
|
||||
cmd: nvidiaDCGMNamedDiagCommand("pulse_test", normalizeNvidiaBurnDuration(durationSec), selected),
|
||||
collectGPU: true,
|
||||
gpuIndices: selected,
|
||||
name: "03-dcgmi-pulse-test.log",
|
||||
cmd: nvidiaDCGMNamedDiagCommand("pulse_test", normalizeNvidiaBurnDuration(durationSec), selected),
|
||||
collectGPU: true,
|
||||
gpuIndices: selected,
|
||||
syncBracket: true,
|
||||
},
|
||||
satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
|
||||
), logFunc)
|
||||
@@ -591,27 +622,30 @@ func (s *System) RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpu
|
||||
socketGroups := gpuBandwidthSocketGroups(selected, logFunc)
|
||||
if len(socketGroups) <= 1 {
|
||||
jobs = append(jobs, satJob{
|
||||
name: fmt.Sprintf("%02d-dcgmi-nvbandwidth.log", step),
|
||||
cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, selected),
|
||||
collectGPU: true,
|
||||
gpuIndices: selected,
|
||||
name: fmt.Sprintf("%02d-dcgmi-nvbandwidth.log", step),
|
||||
cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, selected),
|
||||
collectGPU: true,
|
||||
gpuIndices: selected,
|
||||
syncBracket: true,
|
||||
})
|
||||
step++
|
||||
} else {
|
||||
for i, group := range socketGroups {
|
||||
jobs = append(jobs, satJob{
|
||||
name: fmt.Sprintf("%02d-dcgmi-nvbandwidth-socket%d.log", step, i),
|
||||
cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, group),
|
||||
collectGPU: true,
|
||||
gpuIndices: group,
|
||||
name: fmt.Sprintf("%02d-dcgmi-nvbandwidth-socket%d.log", step, i),
|
||||
cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, group),
|
||||
collectGPU: true,
|
||||
gpuIndices: group,
|
||||
syncBracket: true,
|
||||
})
|
||||
step++
|
||||
}
|
||||
jobs = append(jobs, satJob{
|
||||
name: fmt.Sprintf("%02d-dcgmi-nvbandwidth-all.log", step),
|
||||
cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, selected),
|
||||
collectGPU: true,
|
||||
gpuIndices: selected,
|
||||
name: fmt.Sprintf("%02d-dcgmi-nvbandwidth-all.log", step),
|
||||
cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, selected),
|
||||
collectGPU: true,
|
||||
gpuIndices: selected,
|
||||
syncBracket: true,
|
||||
})
|
||||
step++
|
||||
}
|
||||
@@ -655,10 +689,11 @@ func (s *System) RunNvidiaTargetedStressValidatePack(ctx context.Context, baseDi
|
||||
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
|
||||
satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
|
||||
satJob{
|
||||
name: "03-dcgmi-targeted-stress.log",
|
||||
cmd: nvidiaDCGMNamedDiagCommand("targeted_stress", normalizeNvidiaBurnDuration(durationSec), selected),
|
||||
collectGPU: true,
|
||||
gpuIndices: selected,
|
||||
name: "03-dcgmi-targeted-stress.log",
|
||||
cmd: nvidiaDCGMNamedDiagCommand("targeted_stress", normalizeNvidiaBurnDuration(durationSec), selected),
|
||||
collectGPU: true,
|
||||
gpuIndices: selected,
|
||||
syncBracket: true,
|
||||
},
|
||||
satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
|
||||
), logFunc)
|
||||
@@ -716,7 +751,7 @@ func (s *System) RunMemoryAcceptancePack(ctx context.Context, baseDir string, si
|
||||
}
|
||||
return runAcceptancePackCtx(ctx, baseDir, "memory", []satJob{
|
||||
{name: "01-free-before.log", cmd: []string{"free", "-h"}},
|
||||
{name: "02-memtester.log", cmd: []string{"timeout", fmt.Sprintf("%d", timeoutSec), "memtester", fmt.Sprintf("%dM", sizeMB), fmt.Sprintf("%d", passes)}},
|
||||
{name: "02-memtester.log", cmd: []string{"timeout", fmt.Sprintf("%d", timeoutSec), "memtester", fmt.Sprintf("%dM", sizeMB), fmt.Sprintf("%d", passes)}, syncBracket: true},
|
||||
{name: "03-free-after.log", cmd: []string{"free", "-h"}},
|
||||
}, logFunc)
|
||||
}
|
||||
@@ -737,7 +772,7 @@ func (s *System) RunMemoryStressPack(ctx context.Context, baseDir string, durati
|
||||
"--vm-method", "all",
|
||||
"--timeout", fmt.Sprintf("%d", seconds),
|
||||
"--metrics-brief",
|
||||
}},
|
||||
}, syncBracket: true},
|
||||
{name: "03-free-after.log", cmd: []string{"free", "-h"}},
|
||||
}, logFunc)
|
||||
}
|
||||
@@ -765,7 +800,7 @@ func (s *System) RunCPUAcceptancePack(ctx context.Context, baseDir string, durat
|
||||
return runAcceptancePackCtx(ctx, baseDir, "cpu", []satJob{
|
||||
{name: "01-lscpu.log", cmd: []string{"lscpu"}},
|
||||
{name: "02-sensors-before.log", cmd: []string{"sensors"}},
|
||||
{name: "03-stress-ng.log", cmd: []string{"stress-ng", "--cpu", "0", "--cpu-method", "all", "--timeout", fmt.Sprintf("%d", durationSec)}},
|
||||
{name: "03-stress-ng.log", cmd: []string{"stress-ng", "--cpu", "0", "--cpu-method", "all", "--timeout", fmt.Sprintf("%d", durationSec)}, syncBracket: true},
|
||||
{name: "04-sensors-after.log", cmd: []string{"sensors"}},
|
||||
}, logFunc)
|
||||
}
|
||||
@@ -819,6 +854,7 @@ func (s *System) RunStorageAcceptancePack(ctx context.Context, baseDir string, e
|
||||
}
|
||||
name := fmt.Sprintf("%s-%02d-%s.log", prefix, cmdIndex+1, job.name)
|
||||
livePath := filepath.Join(runDir, name)
|
||||
runSyncBracketHook(job, "before", logFunc)
|
||||
out, err := runSATCommandCtx(ctx, verboseLog, job.name, job.cmd, nil, logFunc, livePath)
|
||||
deviceOutputs[job.name] = out
|
||||
if writeErr := os.WriteFile(livePath, out, 0644); writeErr != nil {
|
||||
@@ -827,17 +863,23 @@ func (s *System) RunStorageAcceptancePack(ctx context.Context, baseDir string, e
|
||||
if satJobBoundaryHook != nil {
|
||||
satJobBoundaryHook(name)
|
||||
}
|
||||
// smartctl -t short only launches the self-test on the drive firmware and
|
||||
// returns immediately ("Testing has begun"); unlike `nvme device-self-test
|
||||
// --wait`, smartctl has no blocking mode, so we must poll the drive
|
||||
// ourselves until the self-test actually finishes. Hold the "after" sync
|
||||
// until that poll completes — the self-test itself, not just its launch,
|
||||
// is the load worth having durable evidence of.
|
||||
deferSyncBracketAfter := job.name == "smartctl-self-test-short" && err == nil
|
||||
if !deferSyncBracketAfter {
|
||||
runSyncBracketHook(job, "after", logFunc)
|
||||
}
|
||||
status, rc := classifySATResult(job.name, out, err)
|
||||
stats.Add(status)
|
||||
key := filepath.Base(devPath) + "_" + strings.ReplaceAll(job.name, "-", "_")
|
||||
fmt.Fprintf(&summary, "%s_rc=%d\n", key, rc)
|
||||
fmt.Fprintf(&summary, "%s_status=%s\n", key, status)
|
||||
|
||||
// smartctl -t short only launches the self-test on the drive firmware and
|
||||
// returns immediately ("Testing has begun"); unlike `nvme device-self-test
|
||||
// --wait`, smartctl has no blocking mode, so we must poll the drive
|
||||
// ourselves until the self-test actually finishes.
|
||||
if job.name == "smartctl-self-test-short" && err == nil {
|
||||
if deferSyncBracketAfter {
|
||||
statusName := "smartctl-self-test-status"
|
||||
statusOut := waitForSmartctlSelfTest(ctx, verboseLog, devPath, logFunc)
|
||||
deviceOutputs[statusName] = statusOut
|
||||
@@ -845,6 +887,7 @@ func (s *System) RunStorageAcceptancePack(ctx context.Context, baseDir string, e
|
||||
if writeErr := os.WriteFile(filepath.Join(runDir, statusFile), statusOut, 0644); writeErr != nil {
|
||||
return "", writeErr
|
||||
}
|
||||
runSyncBracketHook(job, "after", logFunc)
|
||||
sStatus, sRC := classifySATResult(statusName, statusOut, nil)
|
||||
stats.Add(sStatus)
|
||||
sKey := filepath.Base(devPath) + "_" + strings.ReplaceAll(statusName, "-", "_")
|
||||
@@ -878,6 +921,11 @@ type satJob struct {
|
||||
// retries is the number of extra attempts (with a short backoff) if the
|
||||
// job's first run fails. Used for jobs racing nv-hostengine startup.
|
||||
retries int
|
||||
// syncBracket marks a job as the actual load step of a pack (as opposed
|
||||
// to the cheap inventory/discovery steps around it) — see
|
||||
// satSyncBracketHook. Set this on the command that can hang or crash the
|
||||
// host, not on nvidia-smi/dcgmi discovery calls.
|
||||
syncBracket bool
|
||||
}
|
||||
|
||||
type satStats struct {
|
||||
@@ -903,7 +951,7 @@ func nvidiaSATJobs() []satJob {
|
||||
satJob{name: "02-dmidecode-baseboard.log", cmd: []string{"dmidecode", "-t", "baseboard"}},
|
||||
satJob{name: "03-dmidecode-system.log", cmd: []string{"dmidecode", "-t", "system"}},
|
||||
satJob{name: "04-nvidia-bug-report.log", cmd: []string{"nvidia-bug-report.sh", "--output-file", "{{run_dir}}/nvidia-bug-report.log"}},
|
||||
satJob{name: "05-bee-gpu-burn.log", cmd: []string{"bee-gpu-burn", "--seconds", "5", "--size-mb", "64"}},
|
||||
satJob{name: "05-bee-gpu-burn.log", cmd: []string{"bee-gpu-burn", "--seconds", "5", "--size-mb", "64"}, syncBracket: true},
|
||||
)
|
||||
}
|
||||
|
||||
@@ -924,7 +972,7 @@ func nvidiaDCGMJobs(diagLevel int, gpuIndices []int) []satJob {
|
||||
satJob{name: "02-dmidecode-baseboard.log", cmd: []string{"dmidecode", "-t", "baseboard"}},
|
||||
satJob{name: "03-dmidecode-system.log", cmd: []string{"dmidecode", "-t", "system"}},
|
||||
satJob{name: "04-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
|
||||
satJob{name: "05-dcgmi-diag.log", cmd: diagArgs, gpuIndices: gpuIndices},
|
||||
satJob{name: "05-dcgmi-diag.log", cmd: diagArgs, gpuIndices: gpuIndices, syncBracket: true},
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1016,6 +1064,7 @@ func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []sa
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
runSyncBracketHook(job, "before", logFunc)
|
||||
for attempt := 0; ; attempt++ {
|
||||
if job.collectGPU {
|
||||
out, err = runSATCommandWithMetrics(ctx, verboseLog, job.name, cmd, job.env, job.gpuIndices, runDir, logFunc)
|
||||
@@ -1053,6 +1102,7 @@ func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []sa
|
||||
if satJobBoundaryHook != nil {
|
||||
satJobBoundaryHook(job.name)
|
||||
}
|
||||
runSyncBracketHook(job, "after", logFunc)
|
||||
if ctx.Err() != nil {
|
||||
return "", ctx.Err()
|
||||
}
|
||||
@@ -1383,7 +1433,7 @@ func storageSATCommands(devPath string, extended bool) []satJob {
|
||||
{name: "nvme-smart-log", cmd: []string{"nvme", "smart-log", devPath, "-o", "json"}},
|
||||
}
|
||||
if extended {
|
||||
jobs = append(jobs, satJob{name: "nvme-device-self-test", cmd: []string{"nvme", "device-self-test", devPath, "-s", "1", "--wait"}})
|
||||
jobs = append(jobs, satJob{name: "nvme-device-self-test", cmd: []string{"nvme", "device-self-test", devPath, "-s", "1", "--wait"}, syncBracket: true})
|
||||
}
|
||||
return jobs
|
||||
}
|
||||
@@ -1391,7 +1441,7 @@ func storageSATCommands(devPath string, extended bool) []satJob {
|
||||
{name: "smartctl-health", cmd: []string{"smartctl", "-H", "-A", "-i", devPath}},
|
||||
}
|
||||
if extended {
|
||||
jobs = append(jobs, satJob{name: "smartctl-self-test-short", cmd: []string{"smartctl", "-t", "short", devPath}})
|
||||
jobs = append(jobs, satJob{name: "smartctl-self-test-short", cmd: []string{"smartctl", "-t", "short", devPath}, syncBracket: true})
|
||||
}
|
||||
return jobs
|
||||
}
|
||||
|
||||
@@ -2,9 +2,11 @@ package platform
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@@ -95,3 +97,56 @@ func TestRunAcceptancePackCtxInvokesJobBoundaryHook(t *testing.T) {
|
||||
t.Fatalf("seen=%v want [01-a.log 02-b.log]", seen)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunAcceptancePackCtxInvokesSyncBracketHookOnlyForMarkedJobs(t *testing.T) {
|
||||
old := satSyncBracketHook
|
||||
t.Cleanup(func() { satSyncBracketHook = old })
|
||||
|
||||
var calls []string
|
||||
SetSyncBracketHook(func(jobName, phase string) error {
|
||||
calls = append(calls, jobName+":"+phase)
|
||||
return nil
|
||||
})
|
||||
|
||||
oldExecCommand := satExecCommand
|
||||
satExecCommand = func(name string, args ...string) *exec.Cmd {
|
||||
return exec.Command(name, args...)
|
||||
}
|
||||
t.Cleanup(func() { satExecCommand = oldExecCommand })
|
||||
|
||||
dir := t.TempDir()
|
||||
_, err := runAcceptancePackCtx(context.Background(), dir, "test-pack", []satJob{
|
||||
{name: "01-inventory.log", cmd: []string{"printf", "a\n"}},
|
||||
{name: "02-load.log", cmd: []string{"printf", "b\n"}, syncBracket: true},
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("runAcceptancePackCtx error: %v", err)
|
||||
}
|
||||
|
||||
want := []string{"02-load.log:before", "02-load.log:after"}
|
||||
if len(calls) != len(want) {
|
||||
t.Fatalf("calls=%v want %v", calls, want)
|
||||
}
|
||||
for i := range want {
|
||||
if calls[i] != want[i] {
|
||||
t.Fatalf("calls=%v want %v", calls, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSyncBracketHookLogsErrorWithoutFailingJob(t *testing.T) {
|
||||
old := satSyncBracketHook
|
||||
t.Cleanup(func() { satSyncBracketHook = old })
|
||||
SetSyncBracketHook(func(jobName, phase string) error {
|
||||
return fmt.Errorf("blackbox target unreachable")
|
||||
})
|
||||
|
||||
var logged []string
|
||||
runSyncBracketHook(satJob{name: "02-load.log", syncBracket: true}, "before", func(line string) {
|
||||
logged = append(logged, line)
|
||||
})
|
||||
|
||||
if len(logged) != 1 || !strings.Contains(logged[0], "02-load.log") || !strings.Contains(logged[0], "blackbox target unreachable") {
|
||||
t.Fatalf("logged=%v want a single line naming the job and the error", logged)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user