Files
bee/audit/internal/app/blackbox_sync_wait_test.go
Mikhail ChusavitinandClaude Sonnet 5 e03267a72f platform/app: fix nvbandwidth split fallback, blackbox discovery churn, and add blocking sync brackets around load steps
- gpuBandwidthSocketGroups: a single GPU whose NUMA node fails to resolve
  no longer collapses the whole per-socket nvbandwidth split into one
  fallback pass — it now folds into the last resolved group instead,
  preserving isolation for the sockets that did resolve.
- blackbox discoverMarkedTargets: skip mounting/unmounting devices that
  already have a running worker on every 2s discovery tick. This was
  observed hammering the same USB target continuously (mount+unmount
  every ~2s for the whole session) and contending with the worker's own
  sync cycle, plausibly explaining multi-minute sync cycles seen on a
  real crash bundle.
- syncFilesystem now calls syscall.Sync() directly instead of spawning
  /bin/sync per copied file; blackbox mounts removable targets with
  -o sync so writes are durable without relying on the app-level sync as
  the primary mechanism.
- New platform.SetSyncBracketHook / satJob.syncBracket: blocks (with a
  bounded timeout) on blackbox actually reaching removable media right
  before and right after a diagnostic's real load step (nvbandwidth,
  memtester, stress-ng, dcgmi diag, nccl, smartctl/nvme self-test...),
  instead of only firing a fire-and-forget kick after the job's own log
  file is written. A crash mid-load now has durable evidence the load
  started, not just whatever streamed to the RAM-backed export dir before
  blackbox's next scheduled cycle.

Found investigating a real support bundle where blackbox's last
successful sync (19:55:25) predated both the previous job finishing and
the crashing nvbandwidth job starting (19:56:57) — none of the crash
window ever reached durable media.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-07-28 14:22:05 +03:00

108 lines
3.7 KiB
Go

package app
import (
"path/filepath"
"testing"
"time"
)
func TestBlackboxStateCaughtUp(t *testing.T) {
requestedAt := time.Date(2026, 7, 27, 12, 0, 0, 0, time.UTC)
t.Run("no targets enrolled", func(t *testing.T) {
ok, pending := blackboxStateCaughtUp(BlackboxState{}, requestedAt)
if !ok || pending != "" {
t.Fatalf("ok=%v pending=%q, want ok=true pending=\"\"", ok, pending)
}
})
t.Run("target synced after request", func(t *testing.T) {
state := BlackboxState{Targets: []BlackboxTargetStatus{
{EnrollmentID: "bb-1", Status: "running", LastSyncAtUTC: requestedAt.Add(time.Second).Format(time.RFC3339)},
}}
ok, _ := blackboxStateCaughtUp(state, requestedAt)
if !ok {
t.Fatalf("want caught up")
}
})
t.Run("target synced before request is not caught up", func(t *testing.T) {
state := BlackboxState{Targets: []BlackboxTargetStatus{
{EnrollmentID: "bb-1", Status: "running", LastSyncAtUTC: requestedAt.Add(-time.Second).Format(time.RFC3339)},
}}
ok, pending := blackboxStateCaughtUp(state, requestedAt)
if ok || pending != "bb-1" {
t.Fatalf("ok=%v pending=%q, want ok=false pending=bb-1", ok, pending)
}
})
t.Run("degraded target is skipped, not waited on", func(t *testing.T) {
state := BlackboxState{Targets: []BlackboxTargetStatus{
{EnrollmentID: "bb-1", Status: "degraded", LastSyncAtUTC: ""},
}}
ok, _ := blackboxStateCaughtUp(state, requestedAt)
if !ok {
t.Fatalf("want a degraded target to not block catch-up")
}
})
t.Run("one caught up, one not — overall not caught up", func(t *testing.T) {
state := BlackboxState{Targets: []BlackboxTargetStatus{
{EnrollmentID: "bb-1", Status: "running", LastSyncAtUTC: requestedAt.Add(time.Second).Format(time.RFC3339)},
{EnrollmentID: "bb-2", Status: "running", LastSyncAtUTC: requestedAt.Add(-time.Second).Format(time.RFC3339)},
}}
ok, pending := blackboxStateCaughtUp(state, requestedAt)
if ok || pending != "bb-2" {
t.Fatalf("ok=%v pending=%q, want ok=false pending=bb-2", ok, pending)
}
})
}
func TestRequestBlackboxSyncAndWaitReturnsOnceStateCatchesUp(t *testing.T) {
exportDir := t.TempDir()
statePath := filepath.Join(t.TempDir(), "blackbox-state.json")
old := blackboxSyncWaitPollInterval
blackboxSyncWaitPollInterval = 5 * time.Millisecond
t.Cleanup(func() { blackboxSyncWaitPollInterval = old })
// No state file yet — simulates blackbox not running / nothing enrolled.
// Write a caught-up state shortly after the call starts, from another
// goroutine, mimicking a worker finishing a sync cycle mid-wait.
go func() {
time.Sleep(20 * time.Millisecond)
state := BlackboxState{Targets: []BlackboxTargetStatus{
{EnrollmentID: "bb-1", Status: "running", LastSyncAtUTC: blackboxNow().Format(time.RFC3339Nano)},
}}
_ = writeJSONAtomic(statePath, state)
}()
err := requestBlackboxSyncAndWait(exportDir, statePath, time.Second)
if err != nil {
t.Fatalf("requestBlackboxSyncAndWait error: %v", err)
}
}
func TestRequestBlackboxSyncAndWaitTimesOut(t *testing.T) {
exportDir := t.TempDir()
statePath := filepath.Join(t.TempDir(), "blackbox-state.json")
old := blackboxSyncWaitPollInterval
blackboxSyncWaitPollInterval = 5 * time.Millisecond
t.Cleanup(func() { blackboxSyncWaitPollInterval = old })
// State never catches up (stuck in the past) — the target is not
// degraded, so this must time out rather than return immediately.
state := BlackboxState{Targets: []BlackboxTargetStatus{
{EnrollmentID: "bb-1", Status: "running", LastSyncAtUTC: time.Unix(0, 0).UTC().Format(time.RFC3339)},
}}
if err := writeJSONAtomic(statePath, state); err != nil {
t.Fatalf("writeJSONAtomic: %v", err)
}
err := requestBlackboxSyncAndWait(exportDir, statePath, 30*time.Millisecond)
if err == nil {
t.Fatalf("want a timeout error")
}
}