137 lines
4.6 KiB
Go
137 lines
4.6 KiB
Go
package platform
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
const ncclValidOutput = `# size count type redop root time algbw busbw #wrong time algbw busbw #wrong
|
|
536870912 134217728 float sum -1 19547 27.47 48.07 0 19512 27.52 48.15 0
|
|
# Out of bounds values : 0 OK
|
|
# Avg bus bandwidth : 48.1908
|
|
`
|
|
|
|
func TestValidateNCCLAllReduceOutputHealthyNVLinkPair(t *testing.T) {
|
|
runDir := t.TempDir()
|
|
writeNCCLValidationArtifact(t, runDir, "01-nvidia-smi-topo-m.log", "\tGPU0\tGPU1\nGPU0\tX\tNV2\nGPU1\tNV2\tX\n")
|
|
writeNCCLValidationArtifact(t, runDir, "01-nvidia-smi-nvlink-s.log", `GPU 0: H100
|
|
Link 0: 26.562 GB/s
|
|
Link 1: 26.562 GB/s
|
|
GPU 1: H100
|
|
Link 0: 26.562 GB/s
|
|
Link 1: 26.562 GB/s
|
|
`)
|
|
status, detail := validateNCCLAllReduceOutput(runDir, []byte(ncclValidOutput), []int{0, 1})
|
|
if status != "OK" {
|
|
t.Fatalf("status=%q detail=%q want OK", status, detail)
|
|
}
|
|
}
|
|
|
|
func TestValidateNCCLAllReduceOutputFailsInactiveNVLinkEndpoint(t *testing.T) {
|
|
runDir := t.TempDir()
|
|
writeNCCLValidationArtifact(t, runDir, "01-nvidia-smi-topo-m.log", "\tGPU0\tGPU1\nGPU0\tX\tNV2\nGPU1\tNV2\tX\n")
|
|
writeNCCLValidationArtifact(t, runDir, "01-nvidia-smi-nvlink-s.log", `GPU 0: H100
|
|
Link 0: 26.562 GB/s
|
|
Link 1: 26.562 GB/s
|
|
GPU 1: H100
|
|
NVML: Unable to retrieve NVLink information as all links are inActive
|
|
`)
|
|
status, detail := validateNCCLAllReduceOutput(runDir, []byte(ncclValidOutput), []int{0, 1})
|
|
if status != "FAILED" || !strings.Contains(detail, "GPU1 0/2") {
|
|
t.Fatalf("status=%q detail=%q want FAILED with GPU1 0/2", status, detail)
|
|
}
|
|
}
|
|
|
|
func TestValidateNCCLAllReduceOutputRejectsWrongValues(t *testing.T) {
|
|
bad := strings.Replace(ncclValidOutput, "48.07 0", "48.07 2", 1)
|
|
status, detail := validateNCCLAllReduceOutput(t.TempDir(), []byte(bad), []int{0})
|
|
if status != "FAILED" || !strings.Contains(detail, "2 wrong") {
|
|
t.Fatalf("status=%q detail=%q want wrong-value failure", status, detail)
|
|
}
|
|
}
|
|
|
|
func TestValidateNCCLAllReduceOutputPCIeOnlyDoesNotRequireNVLink(t *testing.T) {
|
|
runDir := t.TempDir()
|
|
writeNCCLValidationArtifact(t, runDir, "01-nvidia-smi-topo-m.log", "\tGPU0\tGPU1\nGPU0\tX\tPIX\nGPU1\tPIX\tX\n")
|
|
status, detail := validateNCCLAllReduceOutput(runDir, []byte(ncclValidOutput), []int{0, 1})
|
|
if status != "OK" || !strings.Contains(detail, "no NVLink-bonded pair") {
|
|
t.Fatalf("status=%q detail=%q want PCIe-only OK", status, detail)
|
|
}
|
|
}
|
|
|
|
func TestValidateNCCLAllReduceOutputUnknownFormatIsUnsupported(t *testing.T) {
|
|
status, _ := validateNCCLAllReduceOutput(t.TempDir(), []byte("all done\n"), []int{0})
|
|
if status != "UNSUPPORTED" {
|
|
t.Fatalf("status=%q want UNSUPPORTED", status)
|
|
}
|
|
}
|
|
|
|
func TestRunAcceptancePackAppliesOutputValidator(t *testing.T) {
|
|
runDir, err := runAcceptancePackCtx(context.Background(), t.TempDir(), "validator-test", []satJob{{
|
|
name: "01-result.log",
|
|
cmd: []string{"sh", "-c", "printf command-ok"},
|
|
validate: func(string, []byte) (string, string) {
|
|
return "FAILED", "documented output check failed"
|
|
},
|
|
}}, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
summary, err := os.ReadFile(filepath.Join(runDir, "summary.txt"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !strings.Contains(string(summary), "overall_status=FAILED") ||
|
|
!strings.Contains(string(summary), "1-result_detail=documented output check failed") {
|
|
t.Fatalf("summary did not retain validator failure:\n%s", summary)
|
|
}
|
|
logData, err := os.ReadFile(filepath.Join(runDir, "01-result.log"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !strings.Contains(string(logData), "[bee-validator] FAILED: documented output check failed") {
|
|
t.Fatalf("job log did not retain validator result:\n%s", logData)
|
|
}
|
|
}
|
|
|
|
func TestNCCLJobHealthAndDebugConfiguration(t *testing.T) {
|
|
jobs := ncclSATJobs([]int{0, 2})
|
|
var job satJob
|
|
for _, candidate := range jobs {
|
|
if candidate.name == "02-all-reduce-perf.log" {
|
|
job = candidate
|
|
break
|
|
}
|
|
}
|
|
if job.name == "" || job.validate == nil {
|
|
t.Fatal("all_reduce_perf job or its output validator is missing")
|
|
}
|
|
if !nvidiaJobNeedsHealthCheck(job) {
|
|
t.Fatal("all_reduce_perf job must run NVIDIA health checks")
|
|
}
|
|
if !isNvidiaAcceptancePack("nccl-tests") {
|
|
t.Fatal("nccl-tests must produce NVIDIA per-GPU status files")
|
|
}
|
|
env := strings.Join(job.env, "\n")
|
|
for _, expected := range []string{
|
|
"CUDA_DEVICE_ORDER=PCI_BUS_ID",
|
|
"CUDA_VISIBLE_DEVICES=0,2",
|
|
"NCCL_DEBUG=INFO",
|
|
"NCCL_DEBUG_SUBSYS=INIT,GRAPH",
|
|
} {
|
|
if !strings.Contains(env, expected) {
|
|
t.Fatalf("job env=%q missing %q", env, expected)
|
|
}
|
|
}
|
|
}
|
|
|
|
func writeNCCLValidationArtifact(t *testing.T, dir, name, body string) {
|
|
t.Helper()
|
|
if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|