platform/app: fix nvbandwidth split fallback, blackbox discovery churn, and add blocking sync brackets around load steps
- gpuBandwidthSocketGroups: a single GPU whose NUMA node fails to resolve no longer collapses the whole per-socket nvbandwidth split into one fallback pass — it now folds into the last resolved group instead, preserving isolation for the sockets that did resolve. - blackbox discoverMarkedTargets: skip mounting/unmounting devices that already have a running worker on every 2s discovery tick. This was observed hammering the same USB target continuously (mount+unmount every ~2s for the whole session) and contending with the worker's own sync cycle, plausibly explaining multi-minute sync cycles seen on a real crash bundle. - syncFilesystem now calls syscall.Sync() directly instead of spawning /bin/sync per copied file; blackbox mounts removable targets with -o sync so writes are durable without relying on the app-level sync as the primary mechanism. - New platform.SetSyncBracketHook / satJob.syncBracket: blocks (with a bounded timeout) on blackbox actually reaching removable media right before and right after a diagnostic's real load step (nvbandwidth, memtester, stress-ng, dcgmi diag, nccl, smartctl/nvme self-test...), instead of only firing a fire-and-forget kick after the job's own log file is written. A crash mid-load now has durable evidence the load started, not just whatever streamed to the RAM-backed export dir before blackbox's next scheduled cycle. Found investigating a real support bundle where blackbox's last successful sync (19:55:25) predated both the previous job finishing and the crashing nvbandwidth job starting (19:56:57) — none of the crash window ever reached durable media. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
6d13c17d36
commit
e03267a72f
@@ -2,9 +2,11 @@ package platform
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@@ -95,3 +97,56 @@ func TestRunAcceptancePackCtxInvokesJobBoundaryHook(t *testing.T) {
|
||||
t.Fatalf("seen=%v want [01-a.log 02-b.log]", seen)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunAcceptancePackCtxInvokesSyncBracketHookOnlyForMarkedJobs(t *testing.T) {
|
||||
old := satSyncBracketHook
|
||||
t.Cleanup(func() { satSyncBracketHook = old })
|
||||
|
||||
var calls []string
|
||||
SetSyncBracketHook(func(jobName, phase string) error {
|
||||
calls = append(calls, jobName+":"+phase)
|
||||
return nil
|
||||
})
|
||||
|
||||
oldExecCommand := satExecCommand
|
||||
satExecCommand = func(name string, args ...string) *exec.Cmd {
|
||||
return exec.Command(name, args...)
|
||||
}
|
||||
t.Cleanup(func() { satExecCommand = oldExecCommand })
|
||||
|
||||
dir := t.TempDir()
|
||||
_, err := runAcceptancePackCtx(context.Background(), dir, "test-pack", []satJob{
|
||||
{name: "01-inventory.log", cmd: []string{"printf", "a\n"}},
|
||||
{name: "02-load.log", cmd: []string{"printf", "b\n"}, syncBracket: true},
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("runAcceptancePackCtx error: %v", err)
|
||||
}
|
||||
|
||||
want := []string{"02-load.log:before", "02-load.log:after"}
|
||||
if len(calls) != len(want) {
|
||||
t.Fatalf("calls=%v want %v", calls, want)
|
||||
}
|
||||
for i := range want {
|
||||
if calls[i] != want[i] {
|
||||
t.Fatalf("calls=%v want %v", calls, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSyncBracketHookLogsErrorWithoutFailingJob(t *testing.T) {
|
||||
old := satSyncBracketHook
|
||||
t.Cleanup(func() { satSyncBracketHook = old })
|
||||
SetSyncBracketHook(func(jobName, phase string) error {
|
||||
return fmt.Errorf("blackbox target unreachable")
|
||||
})
|
||||
|
||||
var logged []string
|
||||
runSyncBracketHook(satJob{name: "02-load.log", syncBracket: true}, "before", func(line string) {
|
||||
logged = append(logged, line)
|
||||
})
|
||||
|
||||
if len(logged) != 1 || !strings.Contains(logged[0], "02-load.log") || !strings.Contains(logged[0], "blackbox target unreachable") {
|
||||
t.Fatalf("logged=%v want a single line naming the job and the error", logged)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user