package platform import ( "context" "os" "path/filepath" "strings" "testing" ) const ncclValidOutput = `# size count type redop root time algbw busbw #wrong time algbw busbw #wrong 536870912 134217728 float sum -1 19547 27.47 48.07 0 19512 27.52 48.15 0 # Out of bounds values : 0 OK # Avg bus bandwidth : 48.1908 ` func TestValidateNCCLAllReduceOutputHealthyNVLinkPair(t *testing.T) { runDir := t.TempDir() writeNCCLValidationArtifact(t, runDir, "01-nvidia-smi-topo-m.log", "\tGPU0\tGPU1\nGPU0\tX\tNV2\nGPU1\tNV2\tX\n") writeNCCLValidationArtifact(t, runDir, "01-nvidia-smi-nvlink-s.log", `GPU 0: H100 Link 0: 26.562 GB/s Link 1: 26.562 GB/s GPU 1: H100 Link 0: 26.562 GB/s Link 1: 26.562 GB/s `) status, detail := validateNCCLAllReduceOutput(runDir, []byte(ncclValidOutput), []int{0, 1}) if status != "OK" { t.Fatalf("status=%q detail=%q want OK", status, detail) } } func TestValidateNCCLAllReduceOutputFailsInactiveNVLinkEndpoint(t *testing.T) { runDir := t.TempDir() writeNCCLValidationArtifact(t, runDir, "01-nvidia-smi-topo-m.log", "\tGPU0\tGPU1\nGPU0\tX\tNV2\nGPU1\tNV2\tX\n") writeNCCLValidationArtifact(t, runDir, "01-nvidia-smi-nvlink-s.log", `GPU 0: H100 Link 0: 26.562 GB/s Link 1: 26.562 GB/s GPU 1: H100 NVML: Unable to retrieve NVLink information as all links are inActive `) status, detail := validateNCCLAllReduceOutput(runDir, []byte(ncclValidOutput), []int{0, 1}) if status != "FAILED" || !strings.Contains(detail, "GPU1 0/2") { t.Fatalf("status=%q detail=%q want FAILED with GPU1 0/2", status, detail) } } func TestValidateNCCLAllReduceOutputRejectsWrongValues(t *testing.T) { bad := strings.Replace(ncclValidOutput, "48.07 0", "48.07 2", 1) status, detail := validateNCCLAllReduceOutput(t.TempDir(), []byte(bad), []int{0}) if status != "FAILED" || !strings.Contains(detail, "2 wrong") { t.Fatalf("status=%q detail=%q want wrong-value failure", status, detail) } } func TestValidateNCCLAllReduceOutputPCIeOnlyDoesNotRequireNVLink(t *testing.T) { runDir := t.TempDir() writeNCCLValidationArtifact(t, runDir, "01-nvidia-smi-topo-m.log", "\tGPU0\tGPU1\nGPU0\tX\tPIX\nGPU1\tPIX\tX\n") status, detail := validateNCCLAllReduceOutput(runDir, []byte(ncclValidOutput), []int{0, 1}) if status != "OK" || !strings.Contains(detail, "no NVLink-bonded pair") { t.Fatalf("status=%q detail=%q want PCIe-only OK", status, detail) } } func TestValidateNCCLAllReduceOutputUnknownFormatIsUnsupported(t *testing.T) { status, _ := validateNCCLAllReduceOutput(t.TempDir(), []byte("all done\n"), []int{0}) if status != "UNSUPPORTED" { t.Fatalf("status=%q want UNSUPPORTED", status) } } func TestRunAcceptancePackAppliesOutputValidator(t *testing.T) { runDir, err := runAcceptancePackCtx(context.Background(), t.TempDir(), "validator-test", []satJob{{ name: "01-result.log", cmd: []string{"sh", "-c", "printf command-ok"}, validate: func(string, []byte) (string, string) { return "FAILED", "documented output check failed" }, }}, nil) if err != nil { t.Fatal(err) } summary, err := os.ReadFile(filepath.Join(runDir, "summary.txt")) if err != nil { t.Fatal(err) } if !strings.Contains(string(summary), "overall_status=FAILED") || !strings.Contains(string(summary), "1-result_detail=documented output check failed") { t.Fatalf("summary did not retain validator failure:\n%s", summary) } logData, err := os.ReadFile(filepath.Join(runDir, "01-result.log")) if err != nil { t.Fatal(err) } if !strings.Contains(string(logData), "[bee-validator] FAILED: documented output check failed") { t.Fatalf("job log did not retain validator result:\n%s", logData) } } func TestNCCLJobHealthAndDebugConfiguration(t *testing.T) { jobs := ncclSATJobs([]int{0, 2}) var job satJob for _, candidate := range jobs { if candidate.name == "02-all-reduce-perf.log" { job = candidate break } } if job.name == "" || job.validate == nil { t.Fatal("all_reduce_perf job or its output validator is missing") } if !nvidiaJobNeedsHealthCheck(job) { t.Fatal("all_reduce_perf job must run NVIDIA health checks") } if !isNvidiaAcceptancePack("nccl-tests") { t.Fatal("nccl-tests must produce NVIDIA per-GPU status files") } env := strings.Join(job.env, "\n") for _, expected := range []string{ "CUDA_DEVICE_ORDER=PCI_BUS_ID", "CUDA_VISIBLE_DEVICES=0,2", "NCCL_DEBUG=INFO", "NCCL_DEBUG_SUBSYS=INIT,GRAPH", } { if !strings.Contains(env, expected) { t.Fatalf("job env=%q missing %q", env, expected) } } } func writeNCCLValidationArtifact(t *testing.T, dir, name, body string) { t.Helper() if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0644); err != nil { t.Fatal(err) } }