652 lines
24 KiB
Go
652 lines
24 KiB
Go
package platform
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// RunNvidiaConfigCheckPack runs a read-only check of NVIDIA GPU
|
|
// configuration and NVLink topology that DCGM diag does not cover: DCGM's
|
|
// diagnostic levels stress-test compute/memory/power/bandwidth, but never
|
|
// assert that persistent config (ECC, MIG, power limit) matches the factory
|
|
// default, and NVIDIA's own DGX BasePOD deployment guide calls out
|
|
// topology/NVLink validation as a distinct manual step outside DCGM diag.
|
|
// Confidential Computing readiness — previously its own standalone SAT
|
|
// target — is folded in here as one more read-only fact about GPU state
|
|
// rather than a dedicated test, since none of these are stress tests; they
|
|
// are one-shot state/config reads that all answer "is this GPU configured
|
|
// the way we expect", not "does it survive load".
|
|
func (s *System) RunNvidiaConfigCheckPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
|
|
if ctx == nil {
|
|
ctx = context.Background()
|
|
}
|
|
if baseDir == "" {
|
|
baseDir = "/var/log/bee-sat"
|
|
}
|
|
ts := time.Now().UTC().Format("20060102-150405")
|
|
runDir := filepath.Join(baseDir, "nvidia-config-"+ts)
|
|
if err := os.MkdirAll(runDir, 0755); err != nil {
|
|
return "", err
|
|
}
|
|
verboseLog := filepath.Join(runDir, "verbose.log")
|
|
|
|
status := NvidiaConfigCheckStatus{CollectedAt: time.Now().UTC()}
|
|
|
|
// -- GPU config compliance (ECC/MIG/power limit) -----------------------
|
|
if settings, err := s.ListNvidiaGPUSettings(); err != nil {
|
|
status.Notes = append(status.Notes, "nvidia-smi GPU settings unavailable (no driver, or no GPU present): "+err.Error())
|
|
} else {
|
|
if len(settings) == 0 {
|
|
status.NoGPUs = true
|
|
}
|
|
for _, g := range settings {
|
|
f := NvidiaGPUConfigFinding{
|
|
Index: g.Index, Name: g.Name,
|
|
ECCCurrent: g.ECCCurrent, MIGCurrent: g.MIGCurrent, MIGPending: g.MIGPending,
|
|
PowerLimitW: g.PowerLimitW, PowerDefaultLimitW: g.PowerDefaultLimitW,
|
|
Issues: evaluateNvidiaGPUConfig(g),
|
|
}
|
|
status.GPUs = append(status.GPUs, f)
|
|
for _, issue := range f.Issues {
|
|
status.Warnings = append(status.Warnings, fmt.Sprintf("GPU %d (%s): %s", g.Index, g.Name, issue))
|
|
}
|
|
if strings.EqualFold(g.CCState, "on") {
|
|
status.CCState = "ON"
|
|
}
|
|
}
|
|
}
|
|
|
|
// -- NVLink topology (bonded pairs, active links, error counters) ------
|
|
topoOut, _ := runSATCommandCtx(ctx, verboseLog, "nvidia-smi-topo-m", []string{"nvidia-smi", "topo", "-m"}, nil, logFunc)
|
|
_ = os.WriteFile(filepath.Join(runDir, "01-nvidia-smi-topo-m.log"), topoOut, 0644)
|
|
statusOut, _ := runSATCommandCtx(ctx, verboseLog, "nvidia-smi-nvlink-s", []string{"nvidia-smi", "nvlink", "-s"}, nil, logFunc)
|
|
_ = os.WriteFile(filepath.Join(runDir, "02-nvidia-smi-nvlink-s.log"), statusOut, 0644)
|
|
errOut, _ := runSATCommandCtx(ctx, verboseLog, "nvidia-smi-nvlink-e", []string{"nvidia-smi", "nvlink", "-e"}, nil, logFunc)
|
|
_ = os.WriteFile(filepath.Join(runDir, "03-nvidia-smi-nvlink-e.log"), errOut, 0644)
|
|
|
|
pairs := ParseNvidiaNVLinkBondedPairs(string(topoOut))
|
|
linkStatus := parseNvidiaNVLinkStatus(string(statusOut))
|
|
linkErrors := ParseNvidiaNVLinkErrors(string(errOut))
|
|
for _, pair := range pairs {
|
|
f := evaluateNvidiaNVLinkPair(pair, linkStatus, linkErrors)
|
|
status.NVLinkPairs = append(status.NVLinkPairs, f)
|
|
for _, issue := range f.Issues {
|
|
status.Warnings = append(status.Warnings, fmt.Sprintf("NVLink GPU%d<->GPU%d: %s", f.GPUA, f.GPUB, issue))
|
|
}
|
|
}
|
|
|
|
// -- Confidential Computing readiness (informational only) -------------
|
|
ccOut, ccErr := runSATCommandCtx(ctx, verboseLog, "nvidia-smi-conf-compute-q", []string{"nvidia-smi", "conf-compute", "-q"}, nil, logFunc)
|
|
_ = os.WriteFile(filepath.Join(runDir, "04-nvidia-smi-conf-compute-q.log"), ccOut, 0644)
|
|
if ccErr == nil {
|
|
fields := parseConfComputeFields(ccOut)
|
|
if status.CCState == "" {
|
|
status.CCState = fields["CC State"]
|
|
}
|
|
status.CPUCCCapability = fields["CPU CC Capabilities"]
|
|
status.GPUCCCapability = fields["GPU CC Capabilities"]
|
|
}
|
|
dmesgOut, _ := runSATCommandCtx(ctx, verboseLog, "dmesg", []string{"dmesg"}, nil, nil)
|
|
ccDmesgLines := filterConfComputeDmesgLines(dmesgOut)
|
|
_ = os.WriteFile(filepath.Join(runDir, "05-dmesg-cc-relevant.log"), []byte(strings.Join(ccDmesgLines, "\n")+"\n"), 0644)
|
|
nvlinkDegraded := parseNvidiaNVLinkDegradedDmesg(dmesgOut)
|
|
_ = os.WriteFile(filepath.Join(runDir, "06-dmesg-nvlink-degraded.log"), []byte(renderNvidiaNVLinkDegradedLog(nvlinkDegraded)), 0644)
|
|
for _, event := range nvlinkDegraded {
|
|
status.Warnings = append(status.Warnings, event.Warning())
|
|
}
|
|
lowerDmesg := strings.ToLower(strings.Join(ccDmesgLines, "\n"))
|
|
status.HostAMDSEVSNPActive = strings.Contains(lowerDmesg, "sev-snp enabled")
|
|
status.HostIntelTDXActive = strings.Contains(lowerDmesg, "tdx module") && strings.Contains(lowerDmesg, "module initialized") ||
|
|
strings.Contains(lowerDmesg, "virt/tdx: module initialized")
|
|
for _, path := range []string{"/sys/module/kvm_amd/parameters/sev_snp"} {
|
|
if out, err := runSATCommandCtx(ctx, verboseLog, "sysfs-sev-snp", []string{"cat", path}, nil, nil); err == nil &&
|
|
strings.EqualFold(strings.TrimSpace(string(out)), "Y") {
|
|
status.HostAMDSEVSNPActive = true
|
|
}
|
|
}
|
|
gpuCanRunCC := strings.EqualFold(strings.TrimSpace(status.GPUCCCapability), "CC Capable")
|
|
cpuCapReported := strings.TrimSpace(status.CPUCCCapability)
|
|
cpuCanRunCC := status.HostAMDSEVSNPActive || status.HostIntelTDXActive ||
|
|
(cpuCapReported != "" && !strings.EqualFold(cpuCapReported, "NONE"))
|
|
status.CCReady = cpuCanRunCC && gpuCanRunCC
|
|
|
|
summary := renderNvidiaConfigCheckSummary(status)
|
|
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary), 0644); err != nil {
|
|
return "", err
|
|
}
|
|
report := renderNvidiaConfigCheckReport(status)
|
|
if err := os.WriteFile(filepath.Join(runDir, "nvidia-config-report.txt"), []byte(report), 0644); err != nil {
|
|
return "", err
|
|
}
|
|
|
|
return runDir, nil
|
|
}
|
|
|
|
// NvidiaConfigCheckStatus is the result of RunNvidiaConfigCheckPack.
|
|
type NvidiaConfigCheckStatus struct {
|
|
CollectedAt time.Time `json:"collected_at"`
|
|
|
|
GPUs []NvidiaGPUConfigFinding `json:"gpus,omitempty"`
|
|
NVLinkPairs []NvidiaNVLinkPairFinding `json:"nvlink_pairs,omitempty"`
|
|
NoGPUs bool `json:"no_gpus"`
|
|
|
|
// Confidential Computing readiness — informational only, does not gate
|
|
// overall_status: an unconfigured/NOT_READY CC state is a
|
|
// feature-readiness fact on most fleets (CC is opt-in), not a fault.
|
|
CCReady bool `json:"cc_ready"`
|
|
CCState string `json:"cc_state,omitempty"`
|
|
CPUCCCapability string `json:"cpu_cc_capability,omitempty"`
|
|
GPUCCCapability string `json:"gpu_cc_capability,omitempty"`
|
|
|
|
HostAMDSEVSNPActive bool `json:"host_amd_sev_snp_active"`
|
|
HostIntelTDXActive bool `json:"host_intel_tdx_active"`
|
|
|
|
// Warnings drive overall_status: any entry here fails the check.
|
|
Warnings []string `json:"warnings,omitempty"`
|
|
Notes []string `json:"notes,omitempty"`
|
|
}
|
|
|
|
// NvidiaGPUConfigFinding is one GPU's config-compliance result.
|
|
type NvidiaGPUConfigFinding struct {
|
|
Index int `json:"index"`
|
|
Name string `json:"name"`
|
|
ECCCurrent string `json:"ecc_current"`
|
|
MIGCurrent string `json:"mig_current"`
|
|
MIGPending string `json:"mig_pending"`
|
|
PowerLimitW float64 `json:"power_limit_w"`
|
|
PowerDefaultLimitW float64 `json:"power_default_limit_w"`
|
|
Issues []string `json:"issues,omitempty"`
|
|
}
|
|
|
|
// NvidiaNVLinkPairFinding is one NVLink-bonded GPU pair's link-count/error
|
|
// validation result.
|
|
type NvidiaNVLinkPairFinding struct {
|
|
GPUA int `json:"gpu_a"`
|
|
GPUB int `json:"gpu_b"`
|
|
ExpectedLinks int `json:"expected_links"` // from "nvidia-smi topo -m"'s NVx cell
|
|
ActiveLinksA int `json:"active_links_a"` // per endpoint from "nvidia-smi nvlink -s"
|
|
ActiveLinksB int `json:"active_links_b"`
|
|
TotalLinksA int `json:"total_links_a"`
|
|
TotalLinksB int `json:"total_links_b"`
|
|
ActiveLinks int `json:"active_links"` // aggregate retained for report compatibility
|
|
TotalLinks int `json:"total_links"`
|
|
ErrorLinks int `json:"error_links"` // links with a nonzero replay/recovery/CRC counter
|
|
Issues []string `json:"issues,omitempty"`
|
|
}
|
|
|
|
// evaluateNvidiaGPUConfig flags config drift that DCGM diag does not check:
|
|
// ECC disabled (NVIDIA data-center GPUs ship with ECC on), a MIG mode
|
|
// change stuck pending a reset/reboot, and a power limit capped meaningfully
|
|
// below the card's own factory default (a silent throttle that a stress
|
|
// test would only surface as "lower than expected performance", with no
|
|
// clear cause).
|
|
func evaluateNvidiaGPUConfig(g NvidiaGPUSetting) []string {
|
|
var issues []string
|
|
if strings.EqualFold(g.ECCCurrent, "disabled") {
|
|
issues = append(issues, "ECC is disabled")
|
|
}
|
|
if g.MIGCurrent != "" && g.MIGPending != "" && !strings.EqualFold(g.MIGCurrent, g.MIGPending) {
|
|
issues = append(issues, fmt.Sprintf("MIG mode change pending (current=%s, pending=%s) — apply with a GPU reset or reboot", g.MIGCurrent, g.MIGPending))
|
|
}
|
|
if g.PowerDefaultLimitW > 0 && g.PowerLimitW > 0 {
|
|
deficit := (g.PowerDefaultLimitW - g.PowerLimitW) / g.PowerDefaultLimitW
|
|
if deficit > 0.05 {
|
|
issues = append(issues, fmt.Sprintf("power limit %.0fW is %.0f%% below factory default %.0fW", g.PowerLimitW, deficit*100, g.PowerDefaultLimitW))
|
|
}
|
|
}
|
|
return issues
|
|
}
|
|
|
|
// evaluateNvidiaNVLinkPair compares a bonded pair's actual link state
|
|
// against what the topology matrix says should be there.
|
|
func evaluateNvidiaNVLinkPair(pair NvidiaNVLinkBondedPair, status map[int][]nvidiaNVLinkPort, errors map[int]map[int][3]int64) NvidiaNVLinkPairFinding {
|
|
f := NvidiaNVLinkPairFinding{GPUA: pair.GPUA, GPUB: pair.GPUB, ExpectedLinks: pair.NVLinks}
|
|
|
|
countPorts := func(gpu int) (active, total int, present bool) {
|
|
ports, present := status[gpu]
|
|
for _, port := range ports {
|
|
total++
|
|
if port.active {
|
|
active++
|
|
}
|
|
}
|
|
return active, total, present
|
|
}
|
|
f.ActiveLinksA, f.TotalLinksA, _ = countPorts(pair.GPUA)
|
|
f.ActiveLinksB, f.TotalLinksB, _ = countPorts(pair.GPUB)
|
|
f.ActiveLinks = f.ActiveLinksA + f.ActiveLinksB
|
|
f.TotalLinks = f.TotalLinksA + f.TotalLinksB
|
|
|
|
_, statusASeen := status[pair.GPUA]
|
|
_, statusBSeen := status[pair.GPUB]
|
|
if !statusASeen || !statusBSeen {
|
|
var missing []string
|
|
if !statusASeen {
|
|
missing = append(missing, fmt.Sprintf("GPU%d", pair.GPUA))
|
|
}
|
|
if !statusBSeen {
|
|
missing = append(missing, fmt.Sprintf("GPU%d", pair.GPUB))
|
|
}
|
|
f.Issues = append(f.Issues, "NVLink status unavailable for "+strings.Join(missing, ", "))
|
|
}
|
|
if pair.NVLinks > 0 && (f.ActiveLinksA < pair.NVLinks || f.ActiveLinksB < pair.NVLinks) {
|
|
f.Issues = append(f.Issues, fmt.Sprintf(
|
|
"GPU%d %d/%d, GPU%d %d/%d active links (topo NV%d)",
|
|
pair.GPUA, f.ActiveLinksA, pair.NVLinks,
|
|
pair.GPUB, f.ActiveLinksB, pair.NVLinks,
|
|
pair.NVLinks,
|
|
))
|
|
}
|
|
if f.TotalLinks > 0 && f.ActiveLinks < f.TotalLinks {
|
|
f.Issues = append(f.Issues, fmt.Sprintf("%d/%d reported NVLinks inactive on a bonded pair", f.TotalLinks-f.ActiveLinks, f.TotalLinks))
|
|
}
|
|
|
|
errLinks := 0
|
|
for _, gpu := range []int{pair.GPUA, pair.GPUB} {
|
|
for _, counters := range errors[gpu] {
|
|
if counters[0] != 0 || counters[1] != 0 || counters[2] != 0 {
|
|
errLinks++
|
|
}
|
|
}
|
|
}
|
|
f.ErrorLinks = errLinks
|
|
if errLinks > 0 {
|
|
f.Issues = append(f.Issues, fmt.Sprintf("%d link(s) reporting replay/recovery/CRC errors", errLinks))
|
|
}
|
|
return f
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// NVLink parsing shared by diagnostics and the read-only topology page.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// NvidiaNVLinkBondedPair identifies two GPU indices and their negotiated
|
|
// NVLink bond width as reported by nvidia-smi topo -m.
|
|
type NvidiaNVLinkBondedPair struct {
|
|
GPUA, GPUB int
|
|
NVLinks int
|
|
}
|
|
|
|
var (
|
|
nvidiaNVLinkANSIRe = regexp.MustCompile("\x1b\\[[0-9;]*[A-Za-z]")
|
|
nvidiaNVLinkNVRe = regexp.MustCompile(`(?i)^NV(\d+)$`)
|
|
)
|
|
|
|
// ParseNvidiaNVLinkBondedPairs returns every GPU pair with a nonzero NVLink
|
|
// bond count from a "nvidia-smi topo -m" matrix, deduplicated (A,B) == (B,A).
|
|
func ParseNvidiaNVLinkBondedPairs(raw string) []NvidiaNVLinkBondedPair {
|
|
lines := strings.Split(nvidiaNVLinkANSIRe.ReplaceAllString(raw, ""), "\n")
|
|
headerIdx := -1
|
|
gpuColumns := map[int]int{}
|
|
for i, line := range lines {
|
|
columns := map[int]int{}
|
|
for columnIndex, label := range strings.Fields(strings.TrimSpace(line)) {
|
|
gpuIndex, err := strconv.Atoi(strings.TrimPrefix(label, "GPU"))
|
|
if err == nil && strings.HasPrefix(label, "GPU") {
|
|
columns[columnIndex] = gpuIndex
|
|
}
|
|
}
|
|
if len(columns) >= 2 {
|
|
headerIdx = i
|
|
gpuColumns = columns
|
|
break
|
|
}
|
|
}
|
|
if headerIdx < 0 {
|
|
return nil
|
|
}
|
|
seen := map[[2]int]bool{}
|
|
var pairs []NvidiaNVLinkBondedPair
|
|
for _, line := range lines[headerIdx+1:] {
|
|
trimmed := strings.TrimSpace(line)
|
|
if !strings.HasPrefix(trimmed, "GPU") {
|
|
continue
|
|
}
|
|
cells := strings.Fields(trimmed)
|
|
if len(cells) == 0 {
|
|
continue
|
|
}
|
|
rowGPU, err := strconv.Atoi(strings.TrimPrefix(cells[0], "GPU"))
|
|
if err != nil {
|
|
continue
|
|
}
|
|
for colIdx, colGPU := range gpuColumns {
|
|
if colGPU == rowGPU {
|
|
continue
|
|
}
|
|
dataIdx := colIdx + 1
|
|
if dataIdx >= len(cells) {
|
|
continue
|
|
}
|
|
m := nvidiaNVLinkNVRe.FindStringSubmatch(cells[dataIdx])
|
|
if len(m) != 2 {
|
|
continue
|
|
}
|
|
nv, err := strconv.Atoi(m[1])
|
|
if err != nil || nv <= 0 {
|
|
continue
|
|
}
|
|
a, b := rowGPU, colGPU
|
|
if a > b {
|
|
a, b = b, a
|
|
}
|
|
key := [2]int{a, b}
|
|
if seen[key] {
|
|
continue
|
|
}
|
|
seen[key] = true
|
|
pairs = append(pairs, NvidiaNVLinkBondedPair{GPUA: a, GPUB: b, NVLinks: nv})
|
|
}
|
|
}
|
|
sort.Slice(pairs, func(i, j int) bool {
|
|
if pairs[i].GPUA != pairs[j].GPUA {
|
|
return pairs[i].GPUA < pairs[j].GPUA
|
|
}
|
|
return pairs[i].GPUB < pairs[j].GPUB
|
|
})
|
|
return pairs
|
|
}
|
|
|
|
type nvidiaNVLinkPort struct {
|
|
active bool
|
|
}
|
|
|
|
var (
|
|
nvidiaNVLinkGPUHeaderRe = regexp.MustCompile(`^GPU (\d+):`)
|
|
nvidiaNVLinkSpeedLineRe = regexp.MustCompile(`^Link (\d+):\s*[\d.]+\s*GB/s`)
|
|
nvidiaNVLinkInactiveRe = regexp.MustCompile(`^Link (\d+):\s*<inactive>`)
|
|
nvidiaNVLinkErrorLineRe = regexp.MustCompile(`^Link (\d+):\s*(Replay|Recovery|CRC) Errors:\s*(\d+)`)
|
|
)
|
|
|
|
// parseNvidiaNVLinkStatus parses "nvidia-smi nvlink -s" output into per-GPU
|
|
// per-link active/inactive state.
|
|
func parseNvidiaNVLinkStatus(raw string) map[int][]nvidiaNVLinkPort {
|
|
result := map[int][]nvidiaNVLinkPort{}
|
|
currentGPU := -1
|
|
for _, line := range strings.Split(raw, "\n") {
|
|
trimmed := strings.TrimSpace(line)
|
|
if m := nvidiaNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
|
|
currentGPU, _ = strconv.Atoi(m[1])
|
|
if _, exists := result[currentGPU]; !exists {
|
|
// Preserve the distinction between a present GPU with no active
|
|
// links and a GPU whose block is absent from the command output.
|
|
result[currentGPU] = nil
|
|
}
|
|
continue
|
|
}
|
|
if currentGPU < 0 {
|
|
continue
|
|
}
|
|
if nvidiaNVLinkInactiveRe.MatchString(trimmed) {
|
|
result[currentGPU] = append(result[currentGPU], nvidiaNVLinkPort{active: false})
|
|
continue
|
|
}
|
|
if nvidiaNVLinkSpeedLineRe.MatchString(trimmed) {
|
|
result[currentGPU] = append(result[currentGPU], nvidiaNVLinkPort{active: true})
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
type nvidiaNVLinkDegradedEvent struct {
|
|
GPUIndex int
|
|
LinkID int
|
|
RawLine string
|
|
}
|
|
|
|
func (e nvidiaNVLinkDegradedEvent) Warning() string {
|
|
switch {
|
|
case e.GPUIndex >= 0 && e.LinkID >= 0:
|
|
return fmt.Sprintf("GPU%d NVLink degraded mode (linkId %d): %s", e.GPUIndex, e.LinkID, e.RawLine)
|
|
case e.GPUIndex >= 0:
|
|
return fmt.Sprintf("GPU%d NVLink degraded mode: %s", e.GPUIndex, e.RawLine)
|
|
default:
|
|
return "NVIDIA NVLink degraded mode: " + e.RawLine
|
|
}
|
|
}
|
|
|
|
var (
|
|
nvidiaNVLinkDegradedGPURe = regexp.MustCompile(`(?i)\bGPU\s*(\d+)\b`)
|
|
nvidiaNVLinkDegradedLinkRe = regexp.MustCompile(`(?i)\blinkId\s*[:=]?\s*(\d+)\b`)
|
|
)
|
|
|
|
func parseNvidiaNVLinkDegradedDmesg(raw []byte) []nvidiaNVLinkDegradedEvent {
|
|
var events []nvidiaNVLinkDegradedEvent
|
|
for _, lineBytes := range bytes.Split(raw, []byte("\n")) {
|
|
line := strings.TrimSpace(string(lineBytes))
|
|
lower := strings.ToLower(line)
|
|
explicitNVLinkMarker := strings.Contains(lower, "knvlinksetdegradedmode") ||
|
|
strings.Contains(lower, "nvlink_is_gpu_degraded") ||
|
|
strings.Contains(lower, "error originated on linkid")
|
|
contextualDegradedMarker := strings.Contains(lower, "marked degraded") &&
|
|
(strings.Contains(lower, "nvlink") || strings.Contains(lower, "nvrm") || nvidiaNVLinkDegradedGPURe.MatchString(line))
|
|
if line == "" || (!explicitNVLinkMarker && !contextualDegradedMarker) {
|
|
continue
|
|
}
|
|
event := nvidiaNVLinkDegradedEvent{GPUIndex: -1, LinkID: -1, RawLine: line}
|
|
if match := nvidiaNVLinkDegradedGPURe.FindStringSubmatch(line); len(match) == 2 {
|
|
event.GPUIndex, _ = strconv.Atoi(match[1])
|
|
}
|
|
if match := nvidiaNVLinkDegradedLinkRe.FindStringSubmatch(line); len(match) == 2 {
|
|
event.LinkID, _ = strconv.Atoi(match[1])
|
|
}
|
|
events = append(events, event)
|
|
}
|
|
return events
|
|
}
|
|
|
|
func renderNvidiaNVLinkDegradedLog(events []nvidiaNVLinkDegradedEvent) string {
|
|
if len(events) == 0 {
|
|
return ""
|
|
}
|
|
lines := make([]string, len(events))
|
|
for i, event := range events {
|
|
lines[i] = event.RawLine
|
|
}
|
|
return strings.Join(lines, "\n") + "\n"
|
|
}
|
|
|
|
// ParseNvidiaNVLinkErrors parses "nvidia-smi nvlink -e" output into, per GPU
|
|
// then link index, [replay, recovery, crc] error counts.
|
|
func ParseNvidiaNVLinkErrors(raw string) map[int]map[int][3]int64 {
|
|
result := map[int]map[int][3]int64{}
|
|
currentGPU := -1
|
|
for _, line := range strings.Split(raw, "\n") {
|
|
trimmed := strings.TrimSpace(line)
|
|
if m := nvidiaNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
|
|
currentGPU, _ = strconv.Atoi(m[1])
|
|
continue
|
|
}
|
|
if currentGPU < 0 {
|
|
continue
|
|
}
|
|
m := nvidiaNVLinkErrorLineRe.FindStringSubmatch(trimmed)
|
|
if m == nil {
|
|
continue
|
|
}
|
|
linkIdx, _ := strconv.Atoi(m[1])
|
|
count, _ := strconv.ParseInt(m[3], 10, 64)
|
|
if result[currentGPU] == nil {
|
|
result[currentGPU] = map[int][3]int64{}
|
|
}
|
|
c := result[currentGPU][linkIdx]
|
|
switch m[2] {
|
|
case "Replay":
|
|
c[0] = count
|
|
case "Recovery":
|
|
c[1] = count
|
|
case "CRC":
|
|
c[2] = count
|
|
}
|
|
result[currentGPU][linkIdx] = c
|
|
}
|
|
return result
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Confidential Computing (folded in — see RunNvidiaConfigCheckPack doc)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// parseConfComputeFields parses the indented "Key : Value" block emitted by
|
|
// `nvidia-smi conf-compute -q`, e.g.:
|
|
//
|
|
// CC State : OFF
|
|
// Multi-GPU Mode : Protected PCIe
|
|
// CPU CC Capabilities : INTEL TDX
|
|
// GPU CC Capabilities : CC Capable
|
|
// CC GPUs Ready State : Not Ready
|
|
func parseConfComputeFields(out []byte) map[string]string {
|
|
fields := map[string]string{}
|
|
for _, line := range strings.Split(string(out), "\n") {
|
|
idx := strings.Index(line, ":")
|
|
if idx < 0 {
|
|
continue
|
|
}
|
|
key := strings.TrimSpace(line[:idx])
|
|
val := strings.TrimSpace(line[idx+1:])
|
|
if key == "" || val == "" {
|
|
continue
|
|
}
|
|
fields[key] = val
|
|
}
|
|
return fields
|
|
}
|
|
|
|
// filterConfComputeDmesgLines returns the dmesg lines relevant to CPU
|
|
// Confidential Computing support (AMD SEV/SEV-ES/SEV-SNP, Intel TDX).
|
|
func filterConfComputeDmesgLines(dmesgOut []byte) []string {
|
|
var lines []string
|
|
for _, raw := range bytes.Split(dmesgOut, []byte("\n")) {
|
|
lower := strings.ToLower(string(raw))
|
|
if strings.Contains(lower, "sev") || strings.Contains(lower, "tdx") {
|
|
lines = append(lines, string(raw))
|
|
}
|
|
}
|
|
return lines
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Report rendering
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func renderNvidiaConfigCheckSummary(status NvidiaConfigCheckStatus) string {
|
|
var b strings.Builder
|
|
fmt.Fprintf(&b, "run_at_utc=%s\n", status.CollectedAt.Format(time.RFC3339))
|
|
fmt.Fprintf(&b, "gpu_count=%d\n", len(status.GPUs))
|
|
eccIssues, migIssues, powerIssues := 0, 0, 0
|
|
for _, g := range status.GPUs {
|
|
for _, issue := range g.Issues {
|
|
switch {
|
|
case strings.Contains(issue, "ECC"):
|
|
eccIssues++
|
|
case strings.Contains(issue, "MIG"):
|
|
migIssues++
|
|
case strings.Contains(issue, "power limit"):
|
|
powerIssues++
|
|
}
|
|
}
|
|
}
|
|
fmt.Fprintf(&b, "gpu_ecc_issues=%d\n", eccIssues)
|
|
fmt.Fprintf(&b, "gpu_mig_pending_issues=%d\n", migIssues)
|
|
fmt.Fprintf(&b, "gpu_power_limit_issues=%d\n", powerIssues)
|
|
fmt.Fprintf(&b, "nvlink_pairs_checked=%d\n", len(status.NVLinkPairs))
|
|
nvlinkIssues := 0
|
|
for _, p := range status.NVLinkPairs {
|
|
if len(p.Issues) > 0 {
|
|
nvlinkIssues++
|
|
}
|
|
}
|
|
fmt.Fprintf(&b, "nvlink_pairs_with_issues=%d\n", nvlinkIssues)
|
|
fmt.Fprintf(&b, "cc_ready=%t\n", status.CCReady)
|
|
fmt.Fprintf(&b, "cc_state=%s\n", status.CCState)
|
|
fmt.Fprintf(&b, "cpu_cc_capability=%s\n", status.CPUCCCapability)
|
|
fmt.Fprintf(&b, "gpu_cc_capability=%s\n", status.GPUCCCapability)
|
|
if status.NoGPUs {
|
|
fmt.Fprintln(&b, "overall_status=UNSUPPORTED")
|
|
fmt.Fprintln(&b, "reason=no_nvidia_gpus_detected")
|
|
} else if len(status.Warnings) == 0 {
|
|
fmt.Fprintln(&b, "overall_status=OK")
|
|
} else {
|
|
fmt.Fprintln(&b, "overall_status=FAILED")
|
|
// warnings carries the actual reason(s) in prose so a reader of
|
|
// summary.txt (or anything parsing it, like sat.DescribeFailure)
|
|
// never has to fall back to "see the report" — each entry already
|
|
// names the specific GPU or NVLink pair and what's wrong with it.
|
|
fmt.Fprintf(&b, "warnings=%s\n", strings.Join(status.Warnings, "; "))
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
func renderNvidiaConfigCheckReport(status NvidiaConfigCheckStatus) string {
|
|
var b strings.Builder
|
|
line := strings.Repeat("=", 80)
|
|
b.WriteString(line + "\n")
|
|
b.WriteString("NVIDIA GPU Configuration & NVLink Topology Check\n")
|
|
b.WriteString(line + "\n\n")
|
|
|
|
b.WriteString("-- GPU Configuration ----------------------------------------------------------\n")
|
|
for _, g := range status.GPUs {
|
|
verdict := "OK"
|
|
if len(g.Issues) > 0 {
|
|
verdict = "ISSUES FOUND"
|
|
}
|
|
fmt.Fprintf(&b, " GPU %d (%s): %s\n", g.Index, g.Name, verdict)
|
|
fmt.Fprintf(&b, " ECC: %s | MIG: %s (pending %s) | Power: %.0fW / %.0fW default\n",
|
|
nonEmptyOr(g.ECCCurrent, "unknown"), nonEmptyOr(g.MIGCurrent, "unknown"), nonEmptyOr(g.MIGPending, "unknown"),
|
|
g.PowerLimitW, g.PowerDefaultLimitW)
|
|
for _, issue := range g.Issues {
|
|
fmt.Fprintf(&b, " - %s\n", issue)
|
|
}
|
|
}
|
|
b.WriteString("\n-- NVLink Topology --------------------------------------------------------------\n")
|
|
if len(status.NVLinkPairs) == 0 {
|
|
b.WriteString(" No NVLink-bonded GPU pairs found.\n")
|
|
}
|
|
for _, p := range status.NVLinkPairs {
|
|
verdict := "OK"
|
|
if len(p.Issues) > 0 {
|
|
verdict = "ISSUES FOUND"
|
|
}
|
|
fmt.Fprintf(&b, " GPU%d <-> GPU%d: endpoints %d/%d and %d/%d active (topology expects %d each): %s\n",
|
|
p.GPUA, p.GPUB, p.ActiveLinksA, p.TotalLinksA, p.ActiveLinksB, p.TotalLinksB, p.ExpectedLinks, verdict)
|
|
for _, issue := range p.Issues {
|
|
fmt.Fprintf(&b, " - %s\n", issue)
|
|
}
|
|
}
|
|
|
|
b.WriteString("\n-- Confidential Computing (informational) ---------------------------------------\n")
|
|
verdict := "NOT READY"
|
|
if status.CCReady {
|
|
verdict = "READY"
|
|
}
|
|
fmt.Fprintf(&b, " Verdict: %s\n", verdict)
|
|
fmt.Fprintf(&b, " CC State: %s | CPU CC: %s | GPU CC: %s\n",
|
|
nonEmptyOr(status.CCState, "unknown"), nonEmptyOr(status.CPUCCCapability, "unknown"), nonEmptyOr(status.GPUCCCapability, "unknown"))
|
|
|
|
if len(status.Notes) > 0 {
|
|
b.WriteString("\n-- Notes -------------------------------------------------------------------------\n")
|
|
for _, n := range status.Notes {
|
|
fmt.Fprintf(&b, " - %s\n", n)
|
|
}
|
|
}
|
|
|
|
fmt.Fprintf(&b, "\nCollected : %s\n", status.CollectedAt.Format("2006-01-02 15:04:05 UTC"))
|
|
b.WriteString(line + "\n")
|
|
return b.String()
|
|
}
|
|
|
|
func nonEmptyOr(v, fallback string) string {
|
|
if strings.TrimSpace(v) == "" {
|
|
return fallback
|
|
}
|
|
return v
|
|
}
|