refactor: harden diagnostics and consolidate runtime code

audit/internal/platform/sat_nccl_validation.go

@@ -0,0 +1,123 @@
+package platform
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+var (
+	ncclOutOfBoundsRE = regexp.MustCompile(`(?im)^#\s*Out of bounds values\s*:\s*(\d+)(?:\s+\S+)?\s*$`)
+	ncclAvgBusBWRE    = regexp.MustCompile(`(?im)^#\s*Avg bus bandwidth\s*:\s*([0-9]+(?:\.[0-9]+)?)\s*$`)
+	ncclGPUHeaderRE   = regexp.MustCompile(`(?i)\bGPU\d+\b`)
+)
+
+// validateNCCLAllReduceOutput validates only facts that nccl-tests reports
+// directly. It deliberately does not apply a model-independent bandwidth
+// threshold: busbw depends on GPU model, topology, rank count, message size,
+// and NCCL algorithm. The measured value is retained in detail for diagnosis.
+func validateNCCLAllReduceOutput(runDir string, out []byte, selectedGPUIndices []int) (string, string) {
+	text := string(out)
+	oobMatch := ncclOutOfBoundsRE.FindStringSubmatch(text)
+	if len(oobMatch) != 2 {
+		return "UNSUPPORTED", "nccl-tests output has no recognized Out of bounds values summary"
+	}
+	oob, err := strconv.ParseUint(oobMatch[1], 10, 64)
+	if err != nil {
+		return "UNSUPPORTED", "nccl-tests Out of bounds values is not an unsigned integer"
+	}
+	if oob != 0 {
+		return "FAILED", fmt.Sprintf("nccl-tests reported %d out-of-bounds value(s)", oob)
+	}
+
+	rows, wrong, rowsValid := parseNCCLWrongCounts(text)
+	if !rowsValid || rows == 0 {
+		return "UNSUPPORTED", "nccl-tests output has no recognized all_reduce_perf result rows"
+	}
+	if wrong != 0 {
+		return "FAILED", fmt.Sprintf("nccl-tests result rows reported %d wrong value(s)", wrong)
+	}
+
+	avgMatch := ncclAvgBusBWRE.FindStringSubmatch(text)
+	if len(avgMatch) != 2 {
+		return "UNSUPPORTED", "nccl-tests output has no recognized Avg bus bandwidth value"
+	}
+	avgBusBW, err := strconv.ParseFloat(avgMatch[1], 64)
+	if err != nil {
+		return "UNSUPPORTED", "nccl-tests Avg bus bandwidth is not numeric"
+	}
+
+	detail := fmt.Sprintf("validation passed; avg_bus_bandwidth=%.4g GB/s (diagnostic only, no unverified threshold)", avgBusBW)
+	if len(selectedGPUIndices) < 2 {
+		return "OK", detail
+	}
+
+	topoRaw, err := os.ReadFile(filepath.Join(runDir, "01-nvidia-smi-topo-m.log"))
+	if err != nil || !ncclGPUHeaderRE.Match(topoRaw) {
+		return "UNSUPPORTED", "NVIDIA topology output unavailable or unrecognized; cannot validate expected NVLink pairs"
+	}
+	pairs := selectedNvidiaNVLinkPairs(ParseNvidiaNVLinkBondedPairs(string(topoRaw)), selectedGPUIndices)
+	if len(pairs) == 0 {
+		return "OK", detail + "; selected topology has no NVLink-bonded pair"
+	}
+
+	statusRaw, err := os.ReadFile(filepath.Join(runDir, "01-nvidia-smi-nvlink-s.log"))
+	if err != nil {
+		return "UNSUPPORTED", "NVLink status output unavailable for selected NVLink-bonded pair"
+	}
+	linkStatus := parseNvidiaNVLinkStatus(string(statusRaw))
+	var issues []string
+	for _, pair := range pairs {
+		finding := evaluateNvidiaNVLinkPair(pair, linkStatus, nil)
+		for _, issue := range finding.Issues {
+			issues = append(issues, fmt.Sprintf("GPU%d<->GPU%d: %s", pair.GPUA, pair.GPUB, issue))
+		}
+	}
+	if len(issues) != 0 {
+		return "FAILED", "NVLink state does not match selected topology: " + strings.Join(issues, "; ")
+	}
+	return "OK", detail + "; selected NVLink endpoints match topology"
+}
+
+func parseNCCLWrongCounts(raw string) (rows int, wrong uint64, valid bool) {
+	for _, line := range strings.Split(strings.ReplaceAll(raw, "\r\n", "\n"), "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		fields := strings.Fields(line)
+		if len(fields) < 13 {
+			continue
+		}
+		if _, err := strconv.ParseUint(fields[0], 10, 64); err != nil {
+			continue
+		}
+		outOfPlaceWrong, errA := strconv.ParseUint(fields[8], 10, 64)
+		inPlaceWrong, errB := strconv.ParseUint(fields[12], 10, 64)
+		if errA != nil || errB != nil {
+			return rows, wrong, false
+		}
+		rows++
+		wrong += outOfPlaceWrong + inPlaceWrong
+	}
+	return rows, wrong, true
+}
+
+func selectedNvidiaNVLinkPairs(pairs []NvidiaNVLinkBondedPair, selected []int) []NvidiaNVLinkBondedPair {
+	selectedSet := make(map[int]struct{}, len(selected))
+	for _, index := range selected {
+		selectedSet[index] = struct{}{}
+	}
+	filtered := make([]NvidiaNVLinkBondedPair, 0, len(pairs))
+	for _, pair := range pairs {
+		_, aSelected := selectedSet[pair.GPUA]
+		_, bSelected := selectedSet[pair.GPUB]
+		if aSelected && bSelected {
+			filtered = append(filtered, pair)
+		}
+	}
+	return filtered
+}