346 lines
14 KiB
Go
346 lines
14 KiB
Go
package app
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"bee/audit/internal/schema"
|
|
)
|
|
|
|
func TestExtractArchivePath(t *testing.T) {
|
|
cases := map[string]string{
|
|
"/appdata/bee/export/bee-sat/gpu-nvidia-20260706-174722": "/appdata/bee/export/bee-sat/gpu-nvidia-20260706-174722",
|
|
"Archive written to /appdata/bee/export/bee-sat/gpu-nvidia-20260706-174722": "/appdata/bee/export/bee-sat/gpu-nvidia-20260706-174722",
|
|
"Archive written to /path/with spaces/foo.tar.gz": "/path/with spaces/foo.tar.gz",
|
|
" Archive written to /path/foo.tar.gz ": "/path/foo.tar.gz",
|
|
}
|
|
for in, want := range cases {
|
|
if got := ExtractArchivePath(in); got != want {
|
|
t.Errorf("ExtractArchivePath(%q) = %q, want %q", in, got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestReadSATOverallStatus_HandlesActionResultPrefix(t *testing.T) {
|
|
runDir := t.TempDir()
|
|
summary := "run_at_utc=2026-07-06T17:47:22Z\noverall_status=FAILED\n"
|
|
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary), 0644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Regression: RunNvidiaAcceptancePackWithOptions wraps the bare run dir as
|
|
// "Archive written to <dir>" before it reaches ReadSATOverallStatus. If the
|
|
// prefix isn't stripped, the summary.txt lookup silently fails and a FAILED
|
|
// sub-job never surfaces as a task failure.
|
|
wrapped := "Archive written to " + runDir
|
|
if got := ReadSATOverallStatus(ExtractArchivePath(wrapped)); got != "FAILED" {
|
|
t.Errorf("ReadSATOverallStatus(wrapped) = %q, want FAILED", got)
|
|
}
|
|
|
|
if got := ReadSATOverallStatus(ExtractArchivePath(runDir)); got != "FAILED" {
|
|
t.Errorf("ReadSATOverallStatus(bare) = %q, want FAILED", got)
|
|
}
|
|
}
|
|
|
|
func writeSATSummary(t *testing.T, overall string) string {
|
|
t.Helper()
|
|
runDir := t.TempDir()
|
|
summary := "run_at_utc=2026-07-06T17:47:22Z\noverall_status=" + overall + "\n"
|
|
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary), 0644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return runDir
|
|
}
|
|
|
|
func TestApplySATResultToDBNormalizesGPUKeyByVendor(t *testing.T) {
|
|
// "nvidia" (Check tier) and "nvidia-stress" (Load/Burn tier) exercise the
|
|
// same physical GPUs and must collapse onto one component key so a later
|
|
// clean Check run can't erase an earlier Load-tier failure.
|
|
db, err := OpenComponentStatusDB(filepath.Join(t.TempDir(), "component-status.json"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
ApplySATResultToDB(db, "nvidia-stress", writeSATSummary(t, "FAILED"))
|
|
ApplySATResultToDB(db, "nvidia", writeSATSummary(t, "OK"))
|
|
|
|
rec, ok := db.Get("pcie:gpu:nvidia")
|
|
if !ok {
|
|
t.Fatalf("expected pcie:gpu:nvidia record to exist")
|
|
}
|
|
if rec.Status != "Warning" {
|
|
t.Fatalf("status=%q, want Warning (FAILED) to survive the later OK Check run", rec.Status)
|
|
}
|
|
}
|
|
|
|
func TestApplyComponentStatusDBMatchesGPUByVendor(t *testing.T) {
|
|
db, err := OpenComponentStatusDB(filepath.Join(t.TempDir(), "component-status.json"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
db.Record("pcie:gpu:nvidia", "sat:nvidia-stress", "Critical", "nvidia-stress SAT: FAILED")
|
|
|
|
class := "VideoController"
|
|
vendor := 0x10de // collector.NvidiaVendorID
|
|
snap := &schema.HardwareSnapshot{
|
|
PCIeDevices: []schema.HardwarePCIeDevice{
|
|
{DeviceClass: &class, VendorID: &vendor, BDF: strPtr("0000:c8:00.0")},
|
|
},
|
|
}
|
|
|
|
applyComponentStatusDB(snap, db)
|
|
|
|
if snap.PCIeDevices[0].Status == nil || *snap.PCIeDevices[0].Status != "Critical" {
|
|
t.Fatalf("expected GPU device status Critical, got %v", snap.PCIeDevices[0].Status)
|
|
}
|
|
}
|
|
|
|
func strPtr(s string) *string { return &s }
|
|
|
|
// TestSATFailureDetailPrefersWarningsField guards the real bug this
|
|
// exercises: an engineer looking at a failed nvidia-config task used to see
|
|
// only "SAT overall_status=FAILED (see summary.txt)" — no indication of
|
|
// which GPU or NVLink pair was the problem. nvidia-config's summary.txt
|
|
// (nvidia_config_check.go's renderNvidiaConfigCheckSummary) writes the
|
|
// specific reason into a "warnings" field; SATFailureDetail must surface it
|
|
// verbatim instead of falling through to the generic sub-job scan.
|
|
func TestSATFailureDetailPrefersWarningsField(t *testing.T) {
|
|
runDir := t.TempDir()
|
|
summary := "run_at_utc=2026-07-09T18:40:57Z\n" +
|
|
"nvlink_pairs_checked=1\n" +
|
|
"nvlink_pairs_with_issues=1\n" +
|
|
"overall_status=FAILED\n" +
|
|
"warnings=NVLink GPU0<->GPU1: 2/36 NVLinks inactive on a bonded pair\n"
|
|
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary), 0644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
want := "NVLink GPU0<->GPU1: 2/36 NVLinks inactive on a bonded pair"
|
|
if got := SATFailureDetail(runDir); got != want {
|
|
t.Fatalf("SATFailureDetail() = %q, want %q", got, want)
|
|
}
|
|
}
|
|
|
|
// TestSATFailureDetailFallsBackToFailedSubJobs guards generic SAT acceptance
|
|
// packs (nvidia, memory, cpu, storage, ...), which have no "warnings" field
|
|
// — they record one "<job>_status"/"<job>_rc" pair per sub-job instead.
|
|
// SATFailureDetail must name the specific job(s) that failed rather than
|
|
// telling the reader nothing beyond "see summary.txt".
|
|
func TestSATFailureDetailFallsBackToFailedSubJobs(t *testing.T) {
|
|
runDir := t.TempDir()
|
|
summary := "run_at_utc=2026-07-09T18:40:57Z\n" +
|
|
"nvidia-smi-q_rc=0\n" +
|
|
"nvidia-smi-q_status=OK\n" +
|
|
"bee-gpu-burn_rc=1\n" +
|
|
"bee-gpu-burn_status=FAILED\n" +
|
|
"job_ok=1\n" +
|
|
"job_failed=1\n" +
|
|
"overall_status=FAILED\n"
|
|
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary), 0644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
want := "failed sub-job(s): bee-gpu-burn=FAILED (rc=1)"
|
|
if got := SATFailureDetail(runDir); got != want {
|
|
t.Fatalf("SATFailureDetail() = %q, want %q", got, want)
|
|
}
|
|
}
|
|
|
|
func TestSATFailureDetailIncludesValidatorDetail(t *testing.T) {
|
|
runDir := t.TempDir()
|
|
summary := "2-all-reduce-perf_rc=0\n" +
|
|
"2-all-reduce-perf_status=FAILED\n" +
|
|
"2-all-reduce-perf_detail=NVLink state does not match selected topology: GPU0<->GPU1\n" +
|
|
"overall_status=FAILED\n"
|
|
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary), 0644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got := SATFailureDetail(runDir)
|
|
if !strings.Contains(got, "NVLink state does not match selected topology") {
|
|
t.Fatalf("SATFailureDetail() = %q, want validator detail", got)
|
|
}
|
|
}
|
|
|
|
// TestSATFailureDetailEmptyWhenNoReasonFound guards the ultimate fallback:
|
|
// when summary.txt carries no identifiable per-job or warnings detail (e.g.
|
|
// unreadable or from an older binary version), callers must get "" so they
|
|
// know to fall back to their own generic message rather than silently
|
|
// printing an empty explanation.
|
|
func TestSATFailureDetailEmptyWhenNoReasonFound(t *testing.T) {
|
|
if got := SATFailureDetail(filepath.Join(t.TempDir(), "does-not-exist")); got != "" {
|
|
t.Fatalf("SATFailureDetail() = %q, want empty for missing summary.txt", got)
|
|
}
|
|
}
|
|
|
|
// TestApplySATResultToDBCoversAllHealthCheckTargets guards against a target
|
|
// silently falling through ApplySATResultToDB's switch with no matching
|
|
// case — exactly what happened to the old "confidential-computing" target
|
|
// before it was folded into "nvidia-config" (task ran, produced a valid
|
|
// summary.txt, but no component-status.json record was ever written for it,
|
|
// so a /topo card or any other consumer had no way to know the check had
|
|
// even run). Every target below is a real health/acceptance check target
|
|
// dispatched by task_runner.go's executeTaskWithOptions that produces a
|
|
// summary.txt with overall_status; each must land somewhere in the DB.
|
|
func TestApplySATResultToDBCoversAllHealthCheckTargets(t *testing.T) {
|
|
cases := []struct {
|
|
target string
|
|
wantKey string
|
|
}{
|
|
{"nvidia", "pcie:gpu:nvidia"},
|
|
{"nvidia-stress", "pcie:gpu:nvidia"},
|
|
{"nvidia-targeted-stress", "pcie:gpu:nvidia"},
|
|
{"nvidia-compute", "pcie:gpu:nvidia"},
|
|
{"nvidia-targeted-power", "pcie:gpu:nvidia"},
|
|
{"nvidia-pulse", "pcie:gpu:nvidia"},
|
|
{"nvidia-interconnect", "pcie:gpu:nvidia"},
|
|
{"nvidia-bandwidth", "pcie:gpu:nvidia"},
|
|
{"nvidia-config", "pcie:gpu:nvidia"},
|
|
{"amd", "pcie:gpu:amd"},
|
|
{"amd-stress", "pcie:gpu:amd"},
|
|
{"amd-mem", "pcie:gpu:amd"},
|
|
{"amd-bandwidth", "pcie:gpu:amd"},
|
|
{"memory", "memory:all"},
|
|
{"memory-stress", "memory:all"},
|
|
{"sat-stress", "memory:all"},
|
|
{"cpu", "cpu:all"},
|
|
{"platform-stress", "cpu:all"},
|
|
{"storage", "storage:all"},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.target, func(t *testing.T) {
|
|
db, err := OpenComponentStatusDB(filepath.Join(t.TempDir(), "component-status.json"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ApplySATResultToDB(db, tc.target, writeSATSummary(t, "OK"))
|
|
rec, ok := db.Get(tc.wantKey)
|
|
if !ok {
|
|
t.Fatalf("target %q wrote no record under key %q — falls through the switch silently", tc.target, tc.wantKey)
|
|
}
|
|
if rec.Status != "OK" {
|
|
t.Fatalf("target %q key %q status=%q want OK", tc.target, tc.wantKey, rec.Status)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestApplySATResultToDBNvidiaConfigSharesGPUKeyWithOtherNvidiaTargets
|
|
// confirms "nvidia-config" (GPU config/NVLink/CC check — confidential
|
|
// computing readiness folded in here rather than run as its own standalone
|
|
// SAT target) reports under the same pcie:gpu:nvidia key as every other
|
|
// nvidia-* target, so a config/NVLink FAILED result isn't invisible next to
|
|
// the plain stress-test results, and a clean run afterward doesn't silently
|
|
// erase an earlier failure (severity merge, same as
|
|
// TestApplySATResultToDBNormalizesGPUKeyByVendor above).
|
|
func TestApplySATResultToDBNvidiaConfigSharesGPUKeyWithOtherNvidiaTargets(t *testing.T) {
|
|
db, err := OpenComponentStatusDB(filepath.Join(t.TempDir(), "component-status.json"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ApplySATResultToDB(db, "nvidia-config", writeSATSummary(t, "FAILED"))
|
|
ApplySATResultToDB(db, "nvidia", writeSATSummary(t, "OK"))
|
|
|
|
rec, ok := db.Get("pcie:gpu:nvidia")
|
|
if !ok {
|
|
t.Fatalf("expected pcie:gpu:nvidia record to exist")
|
|
}
|
|
if rec.Status != "Warning" {
|
|
t.Fatalf("status=%q, want Warning (nvidia-config FAILED must survive the later OK nvidia run)", rec.Status)
|
|
}
|
|
}
|
|
|
|
func TestApplySATResultToDBNvidiaInterconnectFailureReachesGPUComponent(t *testing.T) {
|
|
db, err := OpenComponentStatusDB(filepath.Join(t.TempDir(), "component-status.json"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ApplySATResultToDB(db, "nvidia-interconnect", writeSATSummary(t, "FAILED"))
|
|
rec, ok := db.Get("pcie:gpu:nvidia")
|
|
if !ok {
|
|
t.Fatal("nvidia-interconnect failure wrote no pcie:gpu:nvidia record")
|
|
}
|
|
if rec.Status != "Warning" {
|
|
t.Fatalf("status=%q want Warning", rec.Status)
|
|
}
|
|
}
|
|
|
|
// TestRecordDeduplicatesRepeatedIdenticalStatusFromSameSource guards the
|
|
// hardware-ingest-contract.md rule that status_history is a transition log
|
|
// ("История переходов статусов"), not a per-poll journal. A component
|
|
// checked continuously (the PSU watchdog, every 60s indefinitely) must not
|
|
// grow one History entry per poll while its status stays unchanged — that
|
|
// is exactly what made component-status.json grow without bound in
|
|
// practice.
|
|
func TestRecordDeduplicatesRepeatedIdenticalStatusFromSameSource(t *testing.T) {
|
|
db, err := OpenComponentStatusDB(filepath.Join(t.TempDir(), "component-status.json"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for i := 0; i < 50; i++ {
|
|
db.Record("psu:0", "watchdog:psu", "OK", "")
|
|
}
|
|
rec, ok := db.Get("psu:0")
|
|
if !ok {
|
|
t.Fatalf("expected psu:0 record")
|
|
}
|
|
if len(rec.History) != 1 {
|
|
t.Fatalf("history len=%d want 1 after 50 identical polls (dedup by transition)", len(rec.History))
|
|
}
|
|
|
|
// A real transition must still be recorded, and dedup resumes at the
|
|
// new status.
|
|
db.Record("psu:0", "watchdog:psu", "Critical", "PSU sensor reported non-OK state")
|
|
db.Record("psu:0", "watchdog:psu", "Critical", "PSU sensor reported non-OK state")
|
|
rec, _ = db.Get("psu:0")
|
|
if len(rec.History) != 2 {
|
|
t.Fatalf("history len=%d want 2 (OK, then Critical, second Critical deduped)", len(rec.History))
|
|
}
|
|
if rec.Status != "Critical" {
|
|
t.Fatalf("status=%q want Critical", rec.Status)
|
|
}
|
|
}
|
|
|
|
// TestRecordDoesNotClobberConcurrentWriterFromAnotherProcess reproduces the
|
|
// bug behind a real support bundle where many GPU/CPU/memory SAT tasks had
|
|
// completed successfully but component-status.json only ever held PSU
|
|
// records: bee-web keeps one ComponentStatusDB open for its whole process
|
|
// lifetime (writing PSU/kmsg watchdog records ~every 60s via a long-running
|
|
// health poller), while each SAT task runs as a short-lived "bee bee-worker"
|
|
// subprocess that opens its own separate ComponentStatusDB instance backed
|
|
// by the same file. Without a reload-before-write, the long-lived process's
|
|
// stale in-memory snapshot (which never learned about the subprocess's
|
|
// write) overwrites the whole file on its next save and erases it.
|
|
func TestRecordDoesNotClobberConcurrentWriterFromAnotherProcess(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), "component-status.json")
|
|
|
|
// bee-web's long-lived DB instance, opened once at process start.
|
|
webDB, err := OpenComponentStatusDB(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// A "bee bee-worker" subprocess for a completed GPU SAT task opens its
|
|
// own instance backed by the same file and records the result.
|
|
workerDB, err := OpenComponentStatusDB(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
workerDB.Record("pcie:gpu:nvidia", "sat:nvidia", "OK", "nvidia SAT: OK")
|
|
|
|
// bee-web's health poller ticks next, using its own (older) in-memory
|
|
// view, and records a PSU status it polled independently.
|
|
webDB.Record("psu:0", "watchdog:psu", "OK", "")
|
|
|
|
// The GPU record written by the worker subprocess must survive.
|
|
onDisk, err := OpenComponentStatusDB(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if rec, ok := onDisk.Get("pcie:gpu:nvidia"); !ok || rec.Status != "OK" {
|
|
t.Fatalf("pcie:gpu:nvidia record lost after concurrent PSU write: ok=%v rec=%+v", ok, rec)
|
|
}
|
|
if _, ok := onDisk.Get("psu:0"); !ok {
|
|
t.Fatalf("psu:0 record missing")
|
|
}
|
|
}
|