Files
bee/audit/internal/platform/sat_live_output_test.go
T
Mikhail ChusavitinandClaude Sonnet 5 e03267a72f platform/app: fix nvbandwidth split fallback, blackbox discovery churn, and add blocking sync brackets around load steps
- gpuBandwidthSocketGroups: a single GPU whose NUMA node fails to resolve
  no longer collapses the whole per-socket nvbandwidth split into one
  fallback pass — it now folds into the last resolved group instead,
  preserving isolation for the sockets that did resolve.
- blackbox discoverMarkedTargets: skip mounting/unmounting devices that
  already have a running worker on every 2s discovery tick. This was
  observed hammering the same USB target continuously (mount+unmount
  every ~2s for the whole session) and contending with the worker's own
  sync cycle, plausibly explaining multi-minute sync cycles seen on a
  real crash bundle.
- syncFilesystem now calls syscall.Sync() directly instead of spawning
  /bin/sync per copied file; blackbox mounts removable targets with
  -o sync so writes are durable without relying on the app-level sync as
  the primary mechanism.
- New platform.SetSyncBracketHook / satJob.syncBracket: blocks (with a
  bounded timeout) on blackbox actually reaching removable media right
  before and right after a diagnostic's real load step (nvbandwidth,
  memtester, stress-ng, dcgmi diag, nccl, smartctl/nvme self-test...),
  instead of only firing a fire-and-forget kick after the job's own log
  file is written. A crash mid-load now has durable evidence the load
  started, not just whatever streamed to the RAM-backed export dir before
  blackbox's next scheduled cycle.

Found investigating a real support bundle where blackbox's last
successful sync (19:55:25) predated both the previous job finishing and
the crashing nvbandwidth job starting (19:56:57) — none of the crash
window ever reached durable media.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-07-28 14:22:05 +03:00

153 lines
4.3 KiB
Go

package platform
import (
"context"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
)
func TestStreamExecOutputWritesLiveFileIncrementally(t *testing.T) {
dir := t.TempDir()
livePath := filepath.Join(dir, "job.log")
cmd := satExecCommand("printf", "line1\nline2\n")
out, err := streamExecOutput(cmd, nil, livePath)
if err != nil {
t.Fatalf("streamExecOutput error: %v", err)
}
if string(out) != "line1\nline2\n" {
t.Fatalf("out=%q want %q", out, "line1\nline2\n")
}
got, err := os.ReadFile(livePath)
if err != nil {
t.Fatalf("ReadFile(livePath): %v", err)
}
if string(got) != "line1\nline2\n" {
t.Fatalf("livePath content=%q want %q", got, "line1\nline2\n")
}
}
func TestStreamExecOutputSkipsLiveFileWhenPathEmpty(t *testing.T) {
cmd := satExecCommand("printf", "line1\n")
out, err := streamExecOutput(cmd, nil, "")
if err != nil {
t.Fatalf("streamExecOutput error: %v", err)
}
if string(out) != "line1\n" {
t.Fatalf("out=%q want %q", out, "line1\n")
}
}
func TestRunSATCommandCtxWritesLivePath(t *testing.T) {
dir := t.TempDir()
verboseLog := filepath.Join(dir, "verbose.log")
livePath := filepath.Join(dir, "job.log")
oldExecCommand := satExecCommand
satExecCommand = func(name string, args ...string) *exec.Cmd {
return exec.Command(name, args...)
}
t.Cleanup(func() { satExecCommand = oldExecCommand })
out, err := runSATCommandCtx(context.Background(), verboseLog, "job", []string{"printf", "hello\n"}, nil, nil, livePath)
if err != nil {
t.Fatalf("runSATCommandCtx error: %v", err)
}
if string(out) != "hello\n" {
t.Fatalf("out=%q want %q", out, "hello\n")
}
got, err := os.ReadFile(livePath)
if err != nil {
t.Fatalf("ReadFile(livePath): %v", err)
}
if string(got) != "hello\n" {
t.Fatalf("livePath content=%q want %q", got, "hello\n")
}
}
func TestRunAcceptancePackCtxInvokesJobBoundaryHook(t *testing.T) {
old := satJobBoundaryHook
t.Cleanup(func() { satJobBoundaryHook = old })
var seen []string
SetJobBoundaryHook(func(jobName string) {
seen = append(seen, jobName)
})
oldExecCommand := satExecCommand
satExecCommand = func(name string, args ...string) *exec.Cmd {
return exec.Command(name, args...)
}
t.Cleanup(func() { satExecCommand = oldExecCommand })
dir := t.TempDir()
_, err := runAcceptancePackCtx(context.Background(), dir, "test-pack", []satJob{
{name: "01-a.log", cmd: []string{"printf", "a\n"}},
{name: "02-b.log", cmd: []string{"printf", "b\n"}},
}, nil)
if err != nil {
t.Fatalf("runAcceptancePackCtx error: %v", err)
}
if len(seen) != 2 || seen[0] != "01-a.log" || seen[1] != "02-b.log" {
t.Fatalf("seen=%v want [01-a.log 02-b.log]", seen)
}
}
func TestRunAcceptancePackCtxInvokesSyncBracketHookOnlyForMarkedJobs(t *testing.T) {
old := satSyncBracketHook
t.Cleanup(func() { satSyncBracketHook = old })
var calls []string
SetSyncBracketHook(func(jobName, phase string) error {
calls = append(calls, jobName+":"+phase)
return nil
})
oldExecCommand := satExecCommand
satExecCommand = func(name string, args ...string) *exec.Cmd {
return exec.Command(name, args...)
}
t.Cleanup(func() { satExecCommand = oldExecCommand })
dir := t.TempDir()
_, err := runAcceptancePackCtx(context.Background(), dir, "test-pack", []satJob{
{name: "01-inventory.log", cmd: []string{"printf", "a\n"}},
{name: "02-load.log", cmd: []string{"printf", "b\n"}, syncBracket: true},
}, nil)
if err != nil {
t.Fatalf("runAcceptancePackCtx error: %v", err)
}
want := []string{"02-load.log:before", "02-load.log:after"}
if len(calls) != len(want) {
t.Fatalf("calls=%v want %v", calls, want)
}
for i := range want {
if calls[i] != want[i] {
t.Fatalf("calls=%v want %v", calls, want)
}
}
}
func TestRunSyncBracketHookLogsErrorWithoutFailingJob(t *testing.T) {
old := satSyncBracketHook
t.Cleanup(func() { satSyncBracketHook = old })
SetSyncBracketHook(func(jobName, phase string) error {
return fmt.Errorf("blackbox target unreachable")
})
var logged []string
runSyncBracketHook(satJob{name: "02-load.log", syncBracket: true}, "before", func(line string) {
logged = append(logged, line)
})
if len(logged) != 1 || !strings.Contains(logged[0], "02-load.log") || !strings.Contains(logged[0], "blackbox target unreachable") {
t.Fatalf("logged=%v want a single line naming the job and the error", logged)
}
}