package platform import ( "fmt" "os" "path/filepath" "regexp" "strconv" "strings" ) var ( ncclOutOfBoundsRE = regexp.MustCompile(`(?im)^#\s*Out of bounds values\s*:\s*(\d+)(?:\s+\S+)?\s*$`) ncclAvgBusBWRE = regexp.MustCompile(`(?im)^#\s*Avg bus bandwidth\s*:\s*([0-9]+(?:\.[0-9]+)?)\s*$`) ncclGPUHeaderRE = regexp.MustCompile(`(?i)\bGPU\d+\b`) ) // validateNCCLAllReduceOutput validates only facts that nccl-tests reports // directly. It deliberately does not apply a model-independent bandwidth // threshold: busbw depends on GPU model, topology, rank count, message size, // and NCCL algorithm. The measured value is retained in detail for diagnosis. func validateNCCLAllReduceOutput(runDir string, out []byte, selectedGPUIndices []int) (string, string) { text := string(out) oobMatch := ncclOutOfBoundsRE.FindStringSubmatch(text) if len(oobMatch) != 2 { return "UNSUPPORTED", "nccl-tests output has no recognized Out of bounds values summary" } oob, err := strconv.ParseUint(oobMatch[1], 10, 64) if err != nil { return "UNSUPPORTED", "nccl-tests Out of bounds values is not an unsigned integer" } if oob != 0 { return "FAILED", fmt.Sprintf("nccl-tests reported %d out-of-bounds value(s)", oob) } rows, wrong, rowsValid := parseNCCLWrongCounts(text) if !rowsValid || rows == 0 { return "UNSUPPORTED", "nccl-tests output has no recognized all_reduce_perf result rows" } if wrong != 0 { return "FAILED", fmt.Sprintf("nccl-tests result rows reported %d wrong value(s)", wrong) } avgMatch := ncclAvgBusBWRE.FindStringSubmatch(text) if len(avgMatch) != 2 { return "UNSUPPORTED", "nccl-tests output has no recognized Avg bus bandwidth value" } avgBusBW, err := strconv.ParseFloat(avgMatch[1], 64) if err != nil { return "UNSUPPORTED", "nccl-tests Avg bus bandwidth is not numeric" } detail := fmt.Sprintf("validation passed; avg_bus_bandwidth=%.4g GB/s (diagnostic only, no unverified threshold)", avgBusBW) if len(selectedGPUIndices) < 2 { return "OK", detail } topoRaw, err := os.ReadFile(filepath.Join(runDir, "01-nvidia-smi-topo-m.log")) if err != nil || !ncclGPUHeaderRE.Match(topoRaw) { return "UNSUPPORTED", "NVIDIA topology output unavailable or unrecognized; cannot validate expected NVLink pairs" } pairs := selectedNvidiaNVLinkPairs(ParseNvidiaNVLinkBondedPairs(string(topoRaw)), selectedGPUIndices) if len(pairs) == 0 { return "OK", detail + "; selected topology has no NVLink-bonded pair" } statusRaw, err := os.ReadFile(filepath.Join(runDir, "01-nvidia-smi-nvlink-s.log")) if err != nil { return "UNSUPPORTED", "NVLink status output unavailable for selected NVLink-bonded pair" } linkStatus := parseNvidiaNVLinkStatus(string(statusRaw)) var issues []string for _, pair := range pairs { finding := evaluateNvidiaNVLinkPair(pair, linkStatus, nil) for _, issue := range finding.Issues { issues = append(issues, fmt.Sprintf("GPU%d<->GPU%d: %s", pair.GPUA, pair.GPUB, issue)) } } if len(issues) != 0 { return "FAILED", "NVLink state does not match selected topology: " + strings.Join(issues, "; ") } return "OK", detail + "; selected NVLink endpoints match topology" } func parseNCCLWrongCounts(raw string) (rows int, wrong uint64, valid bool) { for _, line := range strings.Split(strings.ReplaceAll(raw, "\r\n", "\n"), "\n") { line = strings.TrimSpace(line) if line == "" || strings.HasPrefix(line, "#") { continue } fields := strings.Fields(line) if len(fields) < 13 { continue } if _, err := strconv.ParseUint(fields[0], 10, 64); err != nil { continue } outOfPlaceWrong, errA := strconv.ParseUint(fields[8], 10, 64) inPlaceWrong, errB := strconv.ParseUint(fields[12], 10, 64) if errA != nil || errB != nil { return rows, wrong, false } rows++ wrong += outOfPlaceWrong + inPlaceWrong } return rows, wrong, true } func selectedNvidiaNVLinkPairs(pairs []NvidiaNVLinkBondedPair, selected []int) []NvidiaNVLinkBondedPair { selectedSet := make(map[int]struct{}, len(selected)) for _, index := range selected { selectedSet[index] = struct{}{} } filtered := make([]NvidiaNVLinkBondedPair, 0, len(pairs)) for _, pair := range pairs { _, aSelected := selectedSet[pair.GPUA] _, bSelected := selectedSet[pair.GPUB] if aSelected && bSelected { filtered = append(filtered, pair) } } return filtered }