- gpuBandwidthSocketGroups: a single GPU whose NUMA node fails to resolve no longer collapses the whole per-socket nvbandwidth split into one fallback pass — it now folds into the last resolved group instead, preserving isolation for the sockets that did resolve. - blackbox discoverMarkedTargets: skip mounting/unmounting devices that already have a running worker on every 2s discovery tick. This was observed hammering the same USB target continuously (mount+unmount every ~2s for the whole session) and contending with the worker's own sync cycle, plausibly explaining multi-minute sync cycles seen on a real crash bundle. - syncFilesystem now calls syscall.Sync() directly instead of spawning /bin/sync per copied file; blackbox mounts removable targets with -o sync so writes are durable without relying on the app-level sync as the primary mechanism. - New platform.SetSyncBracketHook / satJob.syncBracket: blocks (with a bounded timeout) on blackbox actually reaching removable media right before and right after a diagnostic's real load step (nvbandwidth, memtester, stress-ng, dcgmi diag, nccl, smartctl/nvme self-test...), instead of only firing a fire-and-forget kick after the job's own log file is written. A crash mid-load now has durable evidence the load started, not just whatever streamed to the RAM-backed export dir before blackbox's next scheduled cycle. Found investigating a real support bundle where blackbox's last successful sync (19:55:25) predated both the previous job finishing and the crashing nvbandwidth job starting (19:56:57) — none of the crash window ever reached durable media. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
108 lines
3.7 KiB
Go
108 lines
3.7 KiB
Go
package app
|
|
|
|
import (
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestBlackboxStateCaughtUp(t *testing.T) {
|
|
requestedAt := time.Date(2026, 7, 27, 12, 0, 0, 0, time.UTC)
|
|
|
|
t.Run("no targets enrolled", func(t *testing.T) {
|
|
ok, pending := blackboxStateCaughtUp(BlackboxState{}, requestedAt)
|
|
if !ok || pending != "" {
|
|
t.Fatalf("ok=%v pending=%q, want ok=true pending=\"\"", ok, pending)
|
|
}
|
|
})
|
|
|
|
t.Run("target synced after request", func(t *testing.T) {
|
|
state := BlackboxState{Targets: []BlackboxTargetStatus{
|
|
{EnrollmentID: "bb-1", Status: "running", LastSyncAtUTC: requestedAt.Add(time.Second).Format(time.RFC3339)},
|
|
}}
|
|
ok, _ := blackboxStateCaughtUp(state, requestedAt)
|
|
if !ok {
|
|
t.Fatalf("want caught up")
|
|
}
|
|
})
|
|
|
|
t.Run("target synced before request is not caught up", func(t *testing.T) {
|
|
state := BlackboxState{Targets: []BlackboxTargetStatus{
|
|
{EnrollmentID: "bb-1", Status: "running", LastSyncAtUTC: requestedAt.Add(-time.Second).Format(time.RFC3339)},
|
|
}}
|
|
ok, pending := blackboxStateCaughtUp(state, requestedAt)
|
|
if ok || pending != "bb-1" {
|
|
t.Fatalf("ok=%v pending=%q, want ok=false pending=bb-1", ok, pending)
|
|
}
|
|
})
|
|
|
|
t.Run("degraded target is skipped, not waited on", func(t *testing.T) {
|
|
state := BlackboxState{Targets: []BlackboxTargetStatus{
|
|
{EnrollmentID: "bb-1", Status: "degraded", LastSyncAtUTC: ""},
|
|
}}
|
|
ok, _ := blackboxStateCaughtUp(state, requestedAt)
|
|
if !ok {
|
|
t.Fatalf("want a degraded target to not block catch-up")
|
|
}
|
|
})
|
|
|
|
t.Run("one caught up, one not — overall not caught up", func(t *testing.T) {
|
|
state := BlackboxState{Targets: []BlackboxTargetStatus{
|
|
{EnrollmentID: "bb-1", Status: "running", LastSyncAtUTC: requestedAt.Add(time.Second).Format(time.RFC3339)},
|
|
{EnrollmentID: "bb-2", Status: "running", LastSyncAtUTC: requestedAt.Add(-time.Second).Format(time.RFC3339)},
|
|
}}
|
|
ok, pending := blackboxStateCaughtUp(state, requestedAt)
|
|
if ok || pending != "bb-2" {
|
|
t.Fatalf("ok=%v pending=%q, want ok=false pending=bb-2", ok, pending)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestRequestBlackboxSyncAndWaitReturnsOnceStateCatchesUp(t *testing.T) {
|
|
exportDir := t.TempDir()
|
|
statePath := filepath.Join(t.TempDir(), "blackbox-state.json")
|
|
|
|
old := blackboxSyncWaitPollInterval
|
|
blackboxSyncWaitPollInterval = 5 * time.Millisecond
|
|
t.Cleanup(func() { blackboxSyncWaitPollInterval = old })
|
|
|
|
// No state file yet — simulates blackbox not running / nothing enrolled.
|
|
// Write a caught-up state shortly after the call starts, from another
|
|
// goroutine, mimicking a worker finishing a sync cycle mid-wait.
|
|
go func() {
|
|
time.Sleep(20 * time.Millisecond)
|
|
state := BlackboxState{Targets: []BlackboxTargetStatus{
|
|
{EnrollmentID: "bb-1", Status: "running", LastSyncAtUTC: blackboxNow().Format(time.RFC3339Nano)},
|
|
}}
|
|
_ = writeJSONAtomic(statePath, state)
|
|
}()
|
|
|
|
err := requestBlackboxSyncAndWait(exportDir, statePath, time.Second)
|
|
if err != nil {
|
|
t.Fatalf("requestBlackboxSyncAndWait error: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestRequestBlackboxSyncAndWaitTimesOut(t *testing.T) {
|
|
exportDir := t.TempDir()
|
|
statePath := filepath.Join(t.TempDir(), "blackbox-state.json")
|
|
|
|
old := blackboxSyncWaitPollInterval
|
|
blackboxSyncWaitPollInterval = 5 * time.Millisecond
|
|
t.Cleanup(func() { blackboxSyncWaitPollInterval = old })
|
|
|
|
// State never catches up (stuck in the past) — the target is not
|
|
// degraded, so this must time out rather than return immediately.
|
|
state := BlackboxState{Targets: []BlackboxTargetStatus{
|
|
{EnrollmentID: "bb-1", Status: "running", LastSyncAtUTC: time.Unix(0, 0).UTC().Format(time.RFC3339)},
|
|
}}
|
|
if err := writeJSONAtomic(statePath, state); err != nil {
|
|
t.Fatalf("writeJSONAtomic: %v", err)
|
|
}
|
|
|
|
err := requestBlackboxSyncAndWait(exportDir, statePath, 30*time.Millisecond)
|
|
if err == nil {
|
|
t.Fatalf("want a timeout error")
|
|
}
|
|
}
|