refactor: harden diagnostics and consolidate runtime code

2026-09-01 13:01:28 +03:00
parent ac4bc0b2b7
commit 0a6ca8ba0f
49 changed files with 1441 additions and 837 deletions
@@ -0,0 +1,136 @@
+package platform
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+const ncclValidOutput = `# size count type redop root time algbw busbw #wrong time algbw busbw #wrong
+536870912 134217728 float sum -1 19547 27.47 48.07 0 19512 27.52 48.15 0
+# Out of bounds values : 0 OK
+# Avg bus bandwidth    : 48.1908
+`
+
+func TestValidateNCCLAllReduceOutputHealthyNVLinkPair(t *testing.T) {
+	runDir := t.TempDir()
+	writeNCCLValidationArtifact(t, runDir, "01-nvidia-smi-topo-m.log", "\tGPU0\tGPU1\nGPU0\tX\tNV2\nGPU1\tNV2\tX\n")
+	writeNCCLValidationArtifact(t, runDir, "01-nvidia-smi-nvlink-s.log", `GPU 0: H100
+Link 0: 26.562 GB/s
+Link 1: 26.562 GB/s
+GPU 1: H100
+Link 0: 26.562 GB/s
+Link 1: 26.562 GB/s
+`)
+	status, detail := validateNCCLAllReduceOutput(runDir, []byte(ncclValidOutput), []int{0, 1})
+	if status != "OK" {
+		t.Fatalf("status=%q detail=%q want OK", status, detail)
+	}
+}
+
+func TestValidateNCCLAllReduceOutputFailsInactiveNVLinkEndpoint(t *testing.T) {
+	runDir := t.TempDir()
+	writeNCCLValidationArtifact(t, runDir, "01-nvidia-smi-topo-m.log", "\tGPU0\tGPU1\nGPU0\tX\tNV2\nGPU1\tNV2\tX\n")
+	writeNCCLValidationArtifact(t, runDir, "01-nvidia-smi-nvlink-s.log", `GPU 0: H100
+Link 0: 26.562 GB/s
+Link 1: 26.562 GB/s
+GPU 1: H100
+NVML: Unable to retrieve NVLink information as all links are inActive
+`)
+	status, detail := validateNCCLAllReduceOutput(runDir, []byte(ncclValidOutput), []int{0, 1})
+	if status != "FAILED" || !strings.Contains(detail, "GPU1 0/2") {
+		t.Fatalf("status=%q detail=%q want FAILED with GPU1 0/2", status, detail)
+	}
+}
+
+func TestValidateNCCLAllReduceOutputRejectsWrongValues(t *testing.T) {
+	bad := strings.Replace(ncclValidOutput, "48.07 0", "48.07 2", 1)
+	status, detail := validateNCCLAllReduceOutput(t.TempDir(), []byte(bad), []int{0})
+	if status != "FAILED" || !strings.Contains(detail, "2 wrong") {
+		t.Fatalf("status=%q detail=%q want wrong-value failure", status, detail)
+	}
+}
+
+func TestValidateNCCLAllReduceOutputPCIeOnlyDoesNotRequireNVLink(t *testing.T) {
+	runDir := t.TempDir()
+	writeNCCLValidationArtifact(t, runDir, "01-nvidia-smi-topo-m.log", "\tGPU0\tGPU1\nGPU0\tX\tPIX\nGPU1\tPIX\tX\n")
+	status, detail := validateNCCLAllReduceOutput(runDir, []byte(ncclValidOutput), []int{0, 1})
+	if status != "OK" || !strings.Contains(detail, "no NVLink-bonded pair") {
+		t.Fatalf("status=%q detail=%q want PCIe-only OK", status, detail)
+	}
+}
+
+func TestValidateNCCLAllReduceOutputUnknownFormatIsUnsupported(t *testing.T) {
+	status, _ := validateNCCLAllReduceOutput(t.TempDir(), []byte("all done\n"), []int{0})
+	if status != "UNSUPPORTED" {
+		t.Fatalf("status=%q want UNSUPPORTED", status)
+	}
+}
+
+func TestRunAcceptancePackAppliesOutputValidator(t *testing.T) {
+	runDir, err := runAcceptancePackCtx(context.Background(), t.TempDir(), "validator-test", []satJob{{
+		name: "01-result.log",
+		cmd:  []string{"sh", "-c", "printf command-ok"},
+		validate: func(string, []byte) (string, string) {
+			return "FAILED", "documented output check failed"
+		},
+	}}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	summary, err := os.ReadFile(filepath.Join(runDir, "summary.txt"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(string(summary), "overall_status=FAILED") ||
+		!strings.Contains(string(summary), "1-result_detail=documented output check failed") {
+		t.Fatalf("summary did not retain validator failure:\n%s", summary)
+	}
+	logData, err := os.ReadFile(filepath.Join(runDir, "01-result.log"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(string(logData), "[bee-validator] FAILED: documented output check failed") {
+		t.Fatalf("job log did not retain validator result:\n%s", logData)
+	}
+}
+
+func TestNCCLJobHealthAndDebugConfiguration(t *testing.T) {
+	jobs := ncclSATJobs([]int{0, 2})
+	var job satJob
+	for _, candidate := range jobs {
+		if candidate.name == "02-all-reduce-perf.log" {
+			job = candidate
+			break
+		}
+	}
+	if job.name == "" || job.validate == nil {
+		t.Fatal("all_reduce_perf job or its output validator is missing")
+	}
+	if !nvidiaJobNeedsHealthCheck(job) {
+		t.Fatal("all_reduce_perf job must run NVIDIA health checks")
+	}
+	if !isNvidiaAcceptancePack("nccl-tests") {
+		t.Fatal("nccl-tests must produce NVIDIA per-GPU status files")
+	}
+	env := strings.Join(job.env, "\n")
+	for _, expected := range []string{
+		"CUDA_DEVICE_ORDER=PCI_BUS_ID",
+		"CUDA_VISIBLE_DEVICES=0,2",
+		"NCCL_DEBUG=INFO",
+		"NCCL_DEBUG_SUBSYS=INIT,GRAPH",
+	} {
+		if !strings.Contains(env, expected) {
+			t.Fatalf("job env=%q missing %q", env, expected)
+		}
+	}
+}
+
+func writeNCCLValidationArtifact(t *testing.T, dir, name, body string) {
+	t.Helper()
+	if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0644); err != nil {
+		t.Fatal(err)
+	}
+}