124 lines
4.3 KiB
Go
124 lines
4.3 KiB
Go
package platform
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
var (
|
|
ncclOutOfBoundsRE = regexp.MustCompile(`(?im)^#\s*Out of bounds values\s*:\s*(\d+)(?:\s+\S+)?\s*$`)
|
|
ncclAvgBusBWRE = regexp.MustCompile(`(?im)^#\s*Avg bus bandwidth\s*:\s*([0-9]+(?:\.[0-9]+)?)\s*$`)
|
|
ncclGPUHeaderRE = regexp.MustCompile(`(?i)\bGPU\d+\b`)
|
|
)
|
|
|
|
// validateNCCLAllReduceOutput validates only facts that nccl-tests reports
|
|
// directly. It deliberately does not apply a model-independent bandwidth
|
|
// threshold: busbw depends on GPU model, topology, rank count, message size,
|
|
// and NCCL algorithm. The measured value is retained in detail for diagnosis.
|
|
func validateNCCLAllReduceOutput(runDir string, out []byte, selectedGPUIndices []int) (string, string) {
|
|
text := string(out)
|
|
oobMatch := ncclOutOfBoundsRE.FindStringSubmatch(text)
|
|
if len(oobMatch) != 2 {
|
|
return "UNSUPPORTED", "nccl-tests output has no recognized Out of bounds values summary"
|
|
}
|
|
oob, err := strconv.ParseUint(oobMatch[1], 10, 64)
|
|
if err != nil {
|
|
return "UNSUPPORTED", "nccl-tests Out of bounds values is not an unsigned integer"
|
|
}
|
|
if oob != 0 {
|
|
return "FAILED", fmt.Sprintf("nccl-tests reported %d out-of-bounds value(s)", oob)
|
|
}
|
|
|
|
rows, wrong, rowsValid := parseNCCLWrongCounts(text)
|
|
if !rowsValid || rows == 0 {
|
|
return "UNSUPPORTED", "nccl-tests output has no recognized all_reduce_perf result rows"
|
|
}
|
|
if wrong != 0 {
|
|
return "FAILED", fmt.Sprintf("nccl-tests result rows reported %d wrong value(s)", wrong)
|
|
}
|
|
|
|
avgMatch := ncclAvgBusBWRE.FindStringSubmatch(text)
|
|
if len(avgMatch) != 2 {
|
|
return "UNSUPPORTED", "nccl-tests output has no recognized Avg bus bandwidth value"
|
|
}
|
|
avgBusBW, err := strconv.ParseFloat(avgMatch[1], 64)
|
|
if err != nil {
|
|
return "UNSUPPORTED", "nccl-tests Avg bus bandwidth is not numeric"
|
|
}
|
|
|
|
detail := fmt.Sprintf("validation passed; avg_bus_bandwidth=%.4g GB/s (diagnostic only, no unverified threshold)", avgBusBW)
|
|
if len(selectedGPUIndices) < 2 {
|
|
return "OK", detail
|
|
}
|
|
|
|
topoRaw, err := os.ReadFile(filepath.Join(runDir, "01-nvidia-smi-topo-m.log"))
|
|
if err != nil || !ncclGPUHeaderRE.Match(topoRaw) {
|
|
return "UNSUPPORTED", "NVIDIA topology output unavailable or unrecognized; cannot validate expected NVLink pairs"
|
|
}
|
|
pairs := selectedNvidiaNVLinkPairs(ParseNvidiaNVLinkBondedPairs(string(topoRaw)), selectedGPUIndices)
|
|
if len(pairs) == 0 {
|
|
return "OK", detail + "; selected topology has no NVLink-bonded pair"
|
|
}
|
|
|
|
statusRaw, err := os.ReadFile(filepath.Join(runDir, "01-nvidia-smi-nvlink-s.log"))
|
|
if err != nil {
|
|
return "UNSUPPORTED", "NVLink status output unavailable for selected NVLink-bonded pair"
|
|
}
|
|
linkStatus := parseNvidiaNVLinkStatus(string(statusRaw))
|
|
var issues []string
|
|
for _, pair := range pairs {
|
|
finding := evaluateNvidiaNVLinkPair(pair, linkStatus, nil)
|
|
for _, issue := range finding.Issues {
|
|
issues = append(issues, fmt.Sprintf("GPU%d<->GPU%d: %s", pair.GPUA, pair.GPUB, issue))
|
|
}
|
|
}
|
|
if len(issues) != 0 {
|
|
return "FAILED", "NVLink state does not match selected topology: " + strings.Join(issues, "; ")
|
|
}
|
|
return "OK", detail + "; selected NVLink endpoints match topology"
|
|
}
|
|
|
|
func parseNCCLWrongCounts(raw string) (rows int, wrong uint64, valid bool) {
|
|
for _, line := range strings.Split(strings.ReplaceAll(raw, "\r\n", "\n"), "\n") {
|
|
line = strings.TrimSpace(line)
|
|
if line == "" || strings.HasPrefix(line, "#") {
|
|
continue
|
|
}
|
|
fields := strings.Fields(line)
|
|
if len(fields) < 13 {
|
|
continue
|
|
}
|
|
if _, err := strconv.ParseUint(fields[0], 10, 64); err != nil {
|
|
continue
|
|
}
|
|
outOfPlaceWrong, errA := strconv.ParseUint(fields[8], 10, 64)
|
|
inPlaceWrong, errB := strconv.ParseUint(fields[12], 10, 64)
|
|
if errA != nil || errB != nil {
|
|
return rows, wrong, false
|
|
}
|
|
rows++
|
|
wrong += outOfPlaceWrong + inPlaceWrong
|
|
}
|
|
return rows, wrong, true
|
|
}
|
|
|
|
func selectedNvidiaNVLinkPairs(pairs []NvidiaNVLinkBondedPair, selected []int) []NvidiaNVLinkBondedPair {
|
|
selectedSet := make(map[int]struct{}, len(selected))
|
|
for _, index := range selected {
|
|
selectedSet[index] = struct{}{}
|
|
}
|
|
filtered := make([]NvidiaNVLinkBondedPair, 0, len(pairs))
|
|
for _, pair := range pairs {
|
|
_, aSelected := selectedSet[pair.GPUA]
|
|
_, bSelected := selectedSet[pair.GPUB]
|
|
if aSelected && bSelected {
|
|
filtered = append(filtered, pair)
|
|
}
|
|
}
|
|
return filtered
|
|
}
|