reanimator/bee
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

refactor: harden diagnostics and consolidate runtime code

Browse Source
This commit is contained in:
Mikhail Chusavitin
2026-09-01 13:01:28 +03:00
parent ac4bc0b2b7
commit 0a6ca8ba0f
49 changed files with 1441 additions and 837 deletions
Download Patch File Download Diff File Expand all files Collapse all files
audit/internal/platform/nvidia_config_check.go
+146 -47
View File
@@ -7,6 +7,7 @@ import (
"os"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"time"
@@ -71,9 +72,9 @@ func (s *System) RunNvidiaConfigCheckPack(ctx context.Context, baseDir string, l
errOut, _ := runSATCommandCtx(ctx, verboseLog, "nvidia-smi-nvlink-e", []string{"nvidia-smi", "nvlink", "-e"}, nil, logFunc)
_ = os.WriteFile(filepath.Join(runDir, "03-nvidia-smi-nvlink-e.log"), errOut, 0644)
pairs := parseNvidiaNVLinkBondedPairs(string(topoOut))
pairs := ParseNvidiaNVLinkBondedPairs(string(topoOut))
linkStatus := parseNvidiaNVLinkStatus(string(statusOut))
linkErrors := parseNvidiaNVLinkErrors(string(errOut))
linkErrors := ParseNvidiaNVLinkErrors(string(errOut))
for _, pair := range pairs {
f := evaluateNvidiaNVLinkPair(pair, linkStatus, linkErrors)
status.NVLinkPairs = append(status.NVLinkPairs, f)
@@ -96,6 +97,11 @@ func (s *System) RunNvidiaConfigCheckPack(ctx context.Context, baseDir string, l
dmesgOut, _ := runSATCommandCtx(ctx, verboseLog, "dmesg", []string{"dmesg"}, nil, nil)
ccDmesgLines := filterConfComputeDmesgLines(dmesgOut)
_ = os.WriteFile(filepath.Join(runDir, "05-dmesg-cc-relevant.log"), []byte(strings.Join(ccDmesgLines, "\n")+"\n"), 0644)
nvlinkDegraded := parseNvidiaNVLinkDegradedDmesg(dmesgOut)
_ = os.WriteFile(filepath.Join(runDir, "06-dmesg-nvlink-degraded.log"), []byte(renderNvidiaNVLinkDegradedLog(nvlinkDegraded)), 0644)
for _, event := range nvlinkDegraded {
status.Warnings = append(status.Warnings, event.Warning())
}
lowerDmesg := strings.ToLower(strings.Join(ccDmesgLines, "\n"))
status.HostAMDSEVSNPActive = strings.Contains(lowerDmesg, "sev-snp enabled")
status.HostIntelTDXActive = strings.Contains(lowerDmesg, "tdx module") && strings.Contains(lowerDmesg, "module initialized") ||
@@ -166,7 +172,11 @@ type NvidiaNVLinkPairFinding struct {
GPUA int `json:"gpu_a"`
GPUB int `json:"gpu_b"`
ExpectedLinks int `json:"expected_links"` // from "nvidia-smi topo -m"'s NVx cell
ActiveLinks int `json:"active_links"` // from "nvidia-smi nvlink -s"
ActiveLinksA int `json:"active_links_a"` // per endpoint from "nvidia-smi nvlink -s"
ActiveLinksB int `json:"active_links_b"`
TotalLinksA int `json:"total_links_a"`
TotalLinksB int `json:"total_links_b"`
ActiveLinks int `json:"active_links"` // aggregate retained for report compatibility
TotalLinks int `json:"total_links"`
ErrorLinks int `json:"error_links"` // links with a nonzero replay/recovery/CRC counter
Issues []string `json:"issues,omitempty"`
@@ -197,28 +207,57 @@ func evaluateNvidiaGPUConfig(g NvidiaGPUSetting) []string {
// evaluateNvidiaNVLinkPair compares a bonded pair's actual link state
// against what the topology matrix says should be there.
func evaluateNvidiaNVLinkPair(pair nvidiaNVLinkBondedPair, status map[int][]nvidiaNVLinkPort, errors map[int]map[int][3]int64) NvidiaNVLinkPairFinding {
f := NvidiaNVLinkPairFinding{GPUA: pair.gpuA, GPUB: pair.gpuB, ExpectedLinks: pair.links}
func evaluateNvidiaNVLinkPair(pair NvidiaNVLinkBondedPair, status map[int][]nvidiaNVLinkPort, errors map[int]map[int][3]int64) NvidiaNVLinkPairFinding {
f := NvidiaNVLinkPairFinding{GPUA: pair.GPUA, GPUB: pair.GPUB, ExpectedLinks: pair.NVLinks}
active, total, errLinks := 0, 0, 0
for _, gpu := range []int{pair.gpuA, pair.gpuB} {
for _, port := range status[gpu] {
countPorts := func(gpu int) (active, total int, present bool) {
ports, present := status[gpu]
for _, port := range ports {
total++
if port.active {
active++
}
}
return active, total, present
}
f.ActiveLinksA, f.TotalLinksA, _ = countPorts(pair.GPUA)
f.ActiveLinksB, f.TotalLinksB, _ = countPorts(pair.GPUB)
f.ActiveLinks = f.ActiveLinksA + f.ActiveLinksB
f.TotalLinks = f.TotalLinksA + f.TotalLinksB
_, statusASeen := status[pair.GPUA]
_, statusBSeen := status[pair.GPUB]
if !statusASeen || !statusBSeen {
var missing []string
if !statusASeen {
missing = append(missing, fmt.Sprintf("GPU%d", pair.GPUA))
}
if !statusBSeen {
missing = append(missing, fmt.Sprintf("GPU%d", pair.GPUB))
}
f.Issues = append(f.Issues, "NVLink status unavailable for "+strings.Join(missing, ", "))
}
if pair.NVLinks > 0 && (f.ActiveLinksA < pair.NVLinks || f.ActiveLinksB < pair.NVLinks) {
f.Issues = append(f.Issues, fmt.Sprintf(
"GPU%d %d/%d, GPU%d %d/%d active links (topo NV%d)",
pair.GPUA, f.ActiveLinksA, pair.NVLinks,
pair.GPUB, f.ActiveLinksB, pair.NVLinks,
pair.NVLinks,
))
}
if f.TotalLinks > 0 && f.ActiveLinks < f.TotalLinks {
f.Issues = append(f.Issues, fmt.Sprintf("%d/%d reported NVLinks inactive on a bonded pair", f.TotalLinks-f.ActiveLinks, f.TotalLinks))
}
errLinks := 0
for _, gpu := range []int{pair.GPUA, pair.GPUB} {
for _, counters := range errors[gpu] {
if counters[0] != 0 || counters[1] != 0 || counters[2] != 0 {
errLinks++
}
}
}
f.ActiveLinks, f.TotalLinks, f.ErrorLinks = active, total, errLinks
if total > 0 && active < total {
f.Issues = append(f.Issues, fmt.Sprintf("%d/%d NVLinks inactive on a bonded pair", total-active, total))
}
f.ErrorLinks = errLinks
if errLinks > 0 {
f.Issues = append(f.Issues, fmt.Sprintf("%d link(s) reporting replay/recovery/CRC errors", errLinks))
}
@@ -226,17 +265,14 @@ func evaluateNvidiaNVLinkPair(pair nvidiaNVLinkBondedPair, status map[int][]nvid
}
// ---------------------------------------------------------------------------
// NVLink parsing — deliberately self-contained rather than importing
// internal/collector or internal/webui's equivalent parsers (isNVLinkBridgeCandidate
// et al. and parseGPUPairAdjacency/parseTopoNVLinkStatus/parseTopoNVLinkErrors,
// respectively), matching this codebase's existing precedent of small,
// package-local duplication over cross-package coupling for narrow parsing
// helpers (see webui/page_topo.go's isNICDeviceClassDev/isRAIDControllerClass).
// NVLink parsing shared by diagnostics and the read-only topology page.
// ---------------------------------------------------------------------------
type nvidiaNVLinkBondedPair struct {
gpuA, gpuB int
links int
// NvidiaNVLinkBondedPair identifies two GPU indices and their negotiated
// NVLink bond width as reported by nvidia-smi topo -m.
type NvidiaNVLinkBondedPair struct {
GPUA, GPUB int
NVLinks int
}
var (
@@ -244,37 +280,31 @@ var (
nvidiaNVLinkNVRe = regexp.MustCompile(`(?i)^NV(\d+)$`)
)
// parseNvidiaNVLinkBondedPairs returns every GPU pair with a nonzero NVLink
// ParseNvidiaNVLinkBondedPairs returns every GPU pair with a nonzero NVLink
// bond count from a "nvidia-smi topo -m" matrix, deduplicated (A,B) == (B,A).
func parseNvidiaNVLinkBondedPairs(raw string) []nvidiaNVLinkBondedPair {
func ParseNvidiaNVLinkBondedPairs(raw string) []NvidiaNVLinkBondedPair {
lines := strings.Split(nvidiaNVLinkANSIRe.ReplaceAllString(raw, ""), "\n")
headerIdx := -1
var gpuColIndices []int
gpuColumns := map[int]int{}
for i, line := range lines {
trimmed := strings.TrimSpace(line)
if strings.HasPrefix(trimmed, "GPU0") {
parts := strings.Fields(trimmed)
for j, col := range parts {
if strings.HasPrefix(col, "GPU") {
gpuColIndices = append(gpuColIndices, j)
}
}
if len(gpuColIndices) >= 2 {
headerIdx = i
columns := map[int]int{}
for columnIndex, label := range strings.Fields(strings.TrimSpace(line)) {
gpuIndex, err := strconv.Atoi(strings.TrimPrefix(label, "GPU"))
if err == nil && strings.HasPrefix(label, "GPU") {
columns[columnIndex] = gpuIndex
}
}
if len(columns) >= 2 {
headerIdx = i
gpuColumns = columns
break
}
}
if headerIdx < 0 {
return nil
}
colIdxToGPU := make(map[int]int, len(gpuColIndices))
for gpuIdx, colIdx := range gpuColIndices {
colIdxToGPU[colIdx] = gpuIdx
}
seen := map[[2]int]bool{}
var pairs []nvidiaNVLinkBondedPair
var pairs []NvidiaNVLinkBondedPair
for _, line := range lines[headerIdx+1:] {
trimmed := strings.TrimSpace(line)
if !strings.HasPrefix(trimmed, "GPU") {
@@ -288,7 +318,7 @@ func parseNvidiaNVLinkBondedPairs(raw string) []nvidiaNVLinkBondedPair {
if err != nil {
continue
}
for colIdx, colGPU := range colIdxToGPU {
for colIdx, colGPU := range gpuColumns {
if colGPU == rowGPU {
continue
}
@@ -313,9 +343,15 @@ func parseNvidiaNVLinkBondedPairs(raw string) []nvidiaNVLinkBondedPair {
continue
}
seen[key] = true
pairs = append(pairs, nvidiaNVLinkBondedPair{gpuA: a, gpuB: b, links: nv})
pairs = append(pairs, NvidiaNVLinkBondedPair{GPUA: a, GPUB: b, NVLinks: nv})
}
}
sort.Slice(pairs, func(i, j int) bool {
if pairs[i].GPUA != pairs[j].GPUA {
return pairs[i].GPUA < pairs[j].GPUA
}
return pairs[i].GPUB < pairs[j].GPUB
})
return pairs
}
@@ -339,6 +375,11 @@ func parseNvidiaNVLinkStatus(raw string) map[int][]nvidiaNVLinkPort {
trimmed := strings.TrimSpace(line)
if m := nvidiaNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
currentGPU, _ = strconv.Atoi(m[1])
if _, exists := result[currentGPU]; !exists {
// Preserve the distinction between a present GPU with no active
// links and a GPU whose block is absent from the command output.
result[currentGPU] = nil
}
continue
}
if currentGPU < 0 {
@@ -355,9 +396,67 @@ func parseNvidiaNVLinkStatus(raw string) map[int][]nvidiaNVLinkPort {
return result
}
// parseNvidiaNVLinkErrors parses "nvidia-smi nvlink -e" output into, per GPU
type nvidiaNVLinkDegradedEvent struct {
GPUIndex int
LinkID int
RawLine string
}
func (e nvidiaNVLinkDegradedEvent) Warning() string {
switch {
case e.GPUIndex >= 0 && e.LinkID >= 0:
return fmt.Sprintf("GPU%d NVLink degraded mode (linkId %d): %s", e.GPUIndex, e.LinkID, e.RawLine)
case e.GPUIndex >= 0:
return fmt.Sprintf("GPU%d NVLink degraded mode: %s", e.GPUIndex, e.RawLine)
default:
return "NVIDIA NVLink degraded mode: " + e.RawLine
}
}
var (
nvidiaNVLinkDegradedGPURe = regexp.MustCompile(`(?i)\bGPU\s*(\d+)\b`)
nvidiaNVLinkDegradedLinkRe = regexp.MustCompile(`(?i)\blinkId\s*[:=]?\s*(\d+)\b`)
)
func parseNvidiaNVLinkDegradedDmesg(raw []byte) []nvidiaNVLinkDegradedEvent {
var events []nvidiaNVLinkDegradedEvent
for _, lineBytes := range bytes.Split(raw, []byte("\n")) {
line := strings.TrimSpace(string(lineBytes))
lower := strings.ToLower(line)
explicitNVLinkMarker := strings.Contains(lower, "knvlinksetdegradedmode") ||
strings.Contains(lower, "nvlink_is_gpu_degraded") ||
strings.Contains(lower, "error originated on linkid")
contextualDegradedMarker := strings.Contains(lower, "marked degraded") &&
(strings.Contains(lower, "nvlink") || strings.Contains(lower, "nvrm") || nvidiaNVLinkDegradedGPURe.MatchString(line))
if line == "" || (!explicitNVLinkMarker && !contextualDegradedMarker) {
continue
}
event := nvidiaNVLinkDegradedEvent{GPUIndex: -1, LinkID: -1, RawLine: line}
if match := nvidiaNVLinkDegradedGPURe.FindStringSubmatch(line); len(match) == 2 {
event.GPUIndex, _ = strconv.Atoi(match[1])
}
if match := nvidiaNVLinkDegradedLinkRe.FindStringSubmatch(line); len(match) == 2 {
event.LinkID, _ = strconv.Atoi(match[1])
}
events = append(events, event)
}
return events
}
func renderNvidiaNVLinkDegradedLog(events []nvidiaNVLinkDegradedEvent) string {
if len(events) == 0 {
return ""
}
lines := make([]string, len(events))
for i, event := range events {
lines[i] = event.RawLine
}
return strings.Join(lines, "\n") + "\n"
}
// ParseNvidiaNVLinkErrors parses "nvidia-smi nvlink -e" output into, per GPU
// then link index, [replay, recovery, crc] error counts.
func parseNvidiaNVLinkErrors(raw string) map[int]map[int][3]int64 {
func ParseNvidiaNVLinkErrors(raw string) map[int]map[int][3]int64 {
result := map[int]map[int][3]int64{}
currentGPU := -1
for _, line := range strings.Split(raw, "\n") {
@@ -516,8 +615,8 @@ func renderNvidiaConfigCheckReport(status NvidiaConfigCheckStatus) string {
if len(p.Issues) > 0 {
verdict = "ISSUES FOUND"
}
fmt.Fprintf(&b, " GPU%d <-> GPU%d: %d/%d links active (topology expects %d): %s\n",
p.GPUA, p.GPUB, p.ActiveLinks, p.TotalLinks, p.ExpectedLinks, verdict)
fmt.Fprintf(&b, " GPU%d <-> GPU%d: endpoints %d/%d and %d/%d active (topology expects %d each): %s\n",
p.GPUA, p.GPUB, p.ActiveLinksA, p.TotalLinksA, p.ActiveLinksB, p.TotalLinksB, p.ExpectedLinks, verdict)
for _, issue := range p.Issues {
fmt.Fprintf(&b, " - %s\n", issue)
}