platform/app: fix nvbandwidth split fallback, blackbox discovery churn, and add blocking sync brackets around load steps

- gpuBandwidthSocketGroups: a single GPU whose NUMA node fails to resolve
  no longer collapses the whole per-socket nvbandwidth split into one
  fallback pass — it now folds into the last resolved group instead,
  preserving isolation for the sockets that did resolve.
- blackbox discoverMarkedTargets: skip mounting/unmounting devices that
  already have a running worker on every 2s discovery tick. This was
  observed hammering the same USB target continuously (mount+unmount
  every ~2s for the whole session) and contending with the worker's own
  sync cycle, plausibly explaining multi-minute sync cycles seen on a
  real crash bundle.
- syncFilesystem now calls syscall.Sync() directly instead of spawning
  /bin/sync per copied file; blackbox mounts removable targets with
  -o sync so writes are durable without relying on the app-level sync as
  the primary mechanism.
- New platform.SetSyncBracketHook / satJob.syncBracket: blocks (with a
  bounded timeout) on blackbox actually reaching removable media right
  before and right after a diagnostic's real load step (nvbandwidth,
  memtester, stress-ng, dcgmi diag, nccl, smartctl/nvme self-test...),
  instead of only firing a fire-and-forget kick after the job's own log
  file is written. A crash mid-load now has durable evidence the load
  started, not just whatever streamed to the RAM-backed export dir before
  blackbox's next scheduled cycle.

Found investigating a real support bundle where blackbox's last
successful sync (19:55:25) predated both the previous job finishing and
the crashing nvbandwidth job starting (19:56:57) — none of the crash
window ever reached durable media.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

audit/internal/app/blackbox_sync_wait_test.go

@@ -0,0 +1,107 @@
+package app
+
+import (
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func TestBlackboxStateCaughtUp(t *testing.T) {
+	requestedAt := time.Date(2026, 7, 27, 12, 0, 0, 0, time.UTC)
+
+	t.Run("no targets enrolled", func(t *testing.T) {
+		ok, pending := blackboxStateCaughtUp(BlackboxState{}, requestedAt)
+		if !ok || pending != "" {
+			t.Fatalf("ok=%v pending=%q, want ok=true pending=\"\"", ok, pending)
+		}
+	})
+
+	t.Run("target synced after request", func(t *testing.T) {
+		state := BlackboxState{Targets: []BlackboxTargetStatus{
+			{EnrollmentID: "bb-1", Status: "running", LastSyncAtUTC: requestedAt.Add(time.Second).Format(time.RFC3339)},
+		}}
+		ok, _ := blackboxStateCaughtUp(state, requestedAt)
+		if !ok {
+			t.Fatalf("want caught up")
+		}
+	})
+
+	t.Run("target synced before request is not caught up", func(t *testing.T) {
+		state := BlackboxState{Targets: []BlackboxTargetStatus{
+			{EnrollmentID: "bb-1", Status: "running", LastSyncAtUTC: requestedAt.Add(-time.Second).Format(time.RFC3339)},
+		}}
+		ok, pending := blackboxStateCaughtUp(state, requestedAt)
+		if ok || pending != "bb-1" {
+			t.Fatalf("ok=%v pending=%q, want ok=false pending=bb-1", ok, pending)
+		}
+	})
+
+	t.Run("degraded target is skipped, not waited on", func(t *testing.T) {
+		state := BlackboxState{Targets: []BlackboxTargetStatus{
+			{EnrollmentID: "bb-1", Status: "degraded", LastSyncAtUTC: ""},
+		}}
+		ok, _ := blackboxStateCaughtUp(state, requestedAt)
+		if !ok {
+			t.Fatalf("want a degraded target to not block catch-up")
+		}
+	})
+
+	t.Run("one caught up, one not — overall not caught up", func(t *testing.T) {
+		state := BlackboxState{Targets: []BlackboxTargetStatus{
+			{EnrollmentID: "bb-1", Status: "running", LastSyncAtUTC: requestedAt.Add(time.Second).Format(time.RFC3339)},
+			{EnrollmentID: "bb-2", Status: "running", LastSyncAtUTC: requestedAt.Add(-time.Second).Format(time.RFC3339)},
+		}}
+		ok, pending := blackboxStateCaughtUp(state, requestedAt)
+		if ok || pending != "bb-2" {
+			t.Fatalf("ok=%v pending=%q, want ok=false pending=bb-2", ok, pending)
+		}
+	})
+}
+
+func TestRequestBlackboxSyncAndWaitReturnsOnceStateCatchesUp(t *testing.T) {
+	exportDir := t.TempDir()
+	statePath := filepath.Join(t.TempDir(), "blackbox-state.json")
+
+	old := blackboxSyncWaitPollInterval
+	blackboxSyncWaitPollInterval = 5 * time.Millisecond
+	t.Cleanup(func() { blackboxSyncWaitPollInterval = old })
+
+	// No state file yet — simulates blackbox not running / nothing enrolled.
+	// Write a caught-up state shortly after the call starts, from another
+	// goroutine, mimicking a worker finishing a sync cycle mid-wait.
+	go func() {
+		time.Sleep(20 * time.Millisecond)
+		state := BlackboxState{Targets: []BlackboxTargetStatus{
+			{EnrollmentID: "bb-1", Status: "running", LastSyncAtUTC: blackboxNow().Format(time.RFC3339Nano)},
+		}}
+		_ = writeJSONAtomic(statePath, state)
+	}()
+
+	err := requestBlackboxSyncAndWait(exportDir, statePath, time.Second)
+	if err != nil {
+		t.Fatalf("requestBlackboxSyncAndWait error: %v", err)
+	}
+}
+
+func TestRequestBlackboxSyncAndWaitTimesOut(t *testing.T) {
+	exportDir := t.TempDir()
+	statePath := filepath.Join(t.TempDir(), "blackbox-state.json")
+
+	old := blackboxSyncWaitPollInterval
+	blackboxSyncWaitPollInterval = 5 * time.Millisecond
+	t.Cleanup(func() { blackboxSyncWaitPollInterval = old })
+
+	// State never catches up (stuck in the past) — the target is not
+	// degraded, so this must time out rather than return immediately.
+	state := BlackboxState{Targets: []BlackboxTargetStatus{
+		{EnrollmentID: "bb-1", Status: "running", LastSyncAtUTC: time.Unix(0, 0).UTC().Format(time.RFC3339)},
+	}}
+	if err := writeJSONAtomic(statePath, state); err != nil {
+		t.Fatalf("writeJSONAtomic: %v", err)
+	}
+
+	err := requestBlackboxSyncAndWait(exportDir, statePath, 30*time.Millisecond)
+	if err == nil {
+		t.Fatalf("want a timeout error")
+	}
+}