- gpuBandwidthSocketGroups: a single GPU whose NUMA node fails to resolve no longer collapses the whole per-socket nvbandwidth split into one fallback pass — it now folds into the last resolved group instead, preserving isolation for the sockets that did resolve. - blackbox discoverMarkedTargets: skip mounting/unmounting devices that already have a running worker on every 2s discovery tick. This was observed hammering the same USB target continuously (mount+unmount every ~2s for the whole session) and contending with the worker's own sync cycle, plausibly explaining multi-minute sync cycles seen on a real crash bundle. - syncFilesystem now calls syscall.Sync() directly instead of spawning /bin/sync per copied file; blackbox mounts removable targets with -o sync so writes are durable without relying on the app-level sync as the primary mechanism. - New platform.SetSyncBracketHook / satJob.syncBracket: blocks (with a bounded timeout) on blackbox actually reaching removable media right before and right after a diagnostic's real load step (nvbandwidth, memtester, stress-ng, dcgmi diag, nccl, smartctl/nvme self-test...), instead of only firing a fire-and-forget kick after the job's own log file is written. A crash mid-load now has durable evidence the load started, not just whatever streamed to the RAM-backed export dir before blackbox's next scheduled cycle. Found investigating a real support bundle where blackbox's last successful sync (19:55:25) predated both the previous job finishing and the crashing nvbandwidth job starting (19:56:57) — none of the crash window ever reached durable media. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
153 lines
4.3 KiB
Go
153 lines
4.3 KiB
Go
package platform
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestStreamExecOutputWritesLiveFileIncrementally(t *testing.T) {
|
|
dir := t.TempDir()
|
|
livePath := filepath.Join(dir, "job.log")
|
|
|
|
cmd := satExecCommand("printf", "line1\nline2\n")
|
|
out, err := streamExecOutput(cmd, nil, livePath)
|
|
if err != nil {
|
|
t.Fatalf("streamExecOutput error: %v", err)
|
|
}
|
|
if string(out) != "line1\nline2\n" {
|
|
t.Fatalf("out=%q want %q", out, "line1\nline2\n")
|
|
}
|
|
got, err := os.ReadFile(livePath)
|
|
if err != nil {
|
|
t.Fatalf("ReadFile(livePath): %v", err)
|
|
}
|
|
if string(got) != "line1\nline2\n" {
|
|
t.Fatalf("livePath content=%q want %q", got, "line1\nline2\n")
|
|
}
|
|
}
|
|
|
|
func TestStreamExecOutputSkipsLiveFileWhenPathEmpty(t *testing.T) {
|
|
cmd := satExecCommand("printf", "line1\n")
|
|
out, err := streamExecOutput(cmd, nil, "")
|
|
if err != nil {
|
|
t.Fatalf("streamExecOutput error: %v", err)
|
|
}
|
|
if string(out) != "line1\n" {
|
|
t.Fatalf("out=%q want %q", out, "line1\n")
|
|
}
|
|
}
|
|
|
|
func TestRunSATCommandCtxWritesLivePath(t *testing.T) {
|
|
dir := t.TempDir()
|
|
verboseLog := filepath.Join(dir, "verbose.log")
|
|
livePath := filepath.Join(dir, "job.log")
|
|
|
|
oldExecCommand := satExecCommand
|
|
satExecCommand = func(name string, args ...string) *exec.Cmd {
|
|
return exec.Command(name, args...)
|
|
}
|
|
t.Cleanup(func() { satExecCommand = oldExecCommand })
|
|
|
|
out, err := runSATCommandCtx(context.Background(), verboseLog, "job", []string{"printf", "hello\n"}, nil, nil, livePath)
|
|
if err != nil {
|
|
t.Fatalf("runSATCommandCtx error: %v", err)
|
|
}
|
|
if string(out) != "hello\n" {
|
|
t.Fatalf("out=%q want %q", out, "hello\n")
|
|
}
|
|
got, err := os.ReadFile(livePath)
|
|
if err != nil {
|
|
t.Fatalf("ReadFile(livePath): %v", err)
|
|
}
|
|
if string(got) != "hello\n" {
|
|
t.Fatalf("livePath content=%q want %q", got, "hello\n")
|
|
}
|
|
}
|
|
|
|
func TestRunAcceptancePackCtxInvokesJobBoundaryHook(t *testing.T) {
|
|
old := satJobBoundaryHook
|
|
t.Cleanup(func() { satJobBoundaryHook = old })
|
|
|
|
var seen []string
|
|
SetJobBoundaryHook(func(jobName string) {
|
|
seen = append(seen, jobName)
|
|
})
|
|
|
|
oldExecCommand := satExecCommand
|
|
satExecCommand = func(name string, args ...string) *exec.Cmd {
|
|
return exec.Command(name, args...)
|
|
}
|
|
t.Cleanup(func() { satExecCommand = oldExecCommand })
|
|
|
|
dir := t.TempDir()
|
|
_, err := runAcceptancePackCtx(context.Background(), dir, "test-pack", []satJob{
|
|
{name: "01-a.log", cmd: []string{"printf", "a\n"}},
|
|
{name: "02-b.log", cmd: []string{"printf", "b\n"}},
|
|
}, nil)
|
|
if err != nil {
|
|
t.Fatalf("runAcceptancePackCtx error: %v", err)
|
|
}
|
|
|
|
if len(seen) != 2 || seen[0] != "01-a.log" || seen[1] != "02-b.log" {
|
|
t.Fatalf("seen=%v want [01-a.log 02-b.log]", seen)
|
|
}
|
|
}
|
|
|
|
func TestRunAcceptancePackCtxInvokesSyncBracketHookOnlyForMarkedJobs(t *testing.T) {
|
|
old := satSyncBracketHook
|
|
t.Cleanup(func() { satSyncBracketHook = old })
|
|
|
|
var calls []string
|
|
SetSyncBracketHook(func(jobName, phase string) error {
|
|
calls = append(calls, jobName+":"+phase)
|
|
return nil
|
|
})
|
|
|
|
oldExecCommand := satExecCommand
|
|
satExecCommand = func(name string, args ...string) *exec.Cmd {
|
|
return exec.Command(name, args...)
|
|
}
|
|
t.Cleanup(func() { satExecCommand = oldExecCommand })
|
|
|
|
dir := t.TempDir()
|
|
_, err := runAcceptancePackCtx(context.Background(), dir, "test-pack", []satJob{
|
|
{name: "01-inventory.log", cmd: []string{"printf", "a\n"}},
|
|
{name: "02-load.log", cmd: []string{"printf", "b\n"}, syncBracket: true},
|
|
}, nil)
|
|
if err != nil {
|
|
t.Fatalf("runAcceptancePackCtx error: %v", err)
|
|
}
|
|
|
|
want := []string{"02-load.log:before", "02-load.log:after"}
|
|
if len(calls) != len(want) {
|
|
t.Fatalf("calls=%v want %v", calls, want)
|
|
}
|
|
for i := range want {
|
|
if calls[i] != want[i] {
|
|
t.Fatalf("calls=%v want %v", calls, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRunSyncBracketHookLogsErrorWithoutFailingJob(t *testing.T) {
|
|
old := satSyncBracketHook
|
|
t.Cleanup(func() { satSyncBracketHook = old })
|
|
SetSyncBracketHook(func(jobName, phase string) error {
|
|
return fmt.Errorf("blackbox target unreachable")
|
|
})
|
|
|
|
var logged []string
|
|
runSyncBracketHook(satJob{name: "02-load.log", syncBracket: true}, "before", func(line string) {
|
|
logged = append(logged, line)
|
|
})
|
|
|
|
if len(logged) != 1 || !strings.Contains(logged[0], "02-load.log") || !strings.Contains(logged[0], "blackbox target unreachable") {
|
|
t.Fatalf("logged=%v want a single line naming the job and the error", logged)
|
|
}
|
|
}
|