package platform import ( "bytes" "context" "fmt" "os" "path/filepath" "regexp" "sort" "strconv" "strings" "time" ) // RunNvidiaConfigCheckPack runs a read-only check of NVIDIA GPU // configuration and NVLink topology that DCGM diag does not cover: DCGM's // diagnostic levels stress-test compute/memory/power/bandwidth, but never // assert that persistent config (ECC, MIG, power limit) matches the factory // default, and NVIDIA's own DGX BasePOD deployment guide calls out // topology/NVLink validation as a distinct manual step outside DCGM diag. // Confidential Computing readiness — previously its own standalone SAT // target — is folded in here as one more read-only fact about GPU state // rather than a dedicated test, since none of these are stress tests; they // are one-shot state/config reads that all answer "is this GPU configured // the way we expect", not "does it survive load". func (s *System) RunNvidiaConfigCheckPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) { if ctx == nil { ctx = context.Background() } if baseDir == "" { baseDir = "/var/log/bee-sat" } ts := time.Now().UTC().Format("20060102-150405") runDir := filepath.Join(baseDir, "nvidia-config-"+ts) if err := os.MkdirAll(runDir, 0755); err != nil { return "", err } verboseLog := filepath.Join(runDir, "verbose.log") status := NvidiaConfigCheckStatus{CollectedAt: time.Now().UTC()} // -- GPU config compliance (ECC/MIG/power limit) ----------------------- if settings, err := s.ListNvidiaGPUSettings(); err != nil { status.Notes = append(status.Notes, "nvidia-smi GPU settings unavailable (no driver, or no GPU present): "+err.Error()) } else { if len(settings) == 0 { status.NoGPUs = true } for _, g := range settings { f := NvidiaGPUConfigFinding{ Index: g.Index, Name: g.Name, ECCCurrent: g.ECCCurrent, MIGCurrent: g.MIGCurrent, MIGPending: g.MIGPending, PowerLimitW: g.PowerLimitW, PowerDefaultLimitW: g.PowerDefaultLimitW, Issues: evaluateNvidiaGPUConfig(g), } status.GPUs = append(status.GPUs, f) for _, issue := range f.Issues { status.Warnings = append(status.Warnings, fmt.Sprintf("GPU %d (%s): %s", g.Index, g.Name, issue)) } if strings.EqualFold(g.CCState, "on") { status.CCState = "ON" } } } // -- NVLink topology (bonded pairs, active links, error counters) ------ topoOut, _ := runSATCommandCtx(ctx, verboseLog, "nvidia-smi-topo-m", []string{"nvidia-smi", "topo", "-m"}, nil, logFunc) _ = os.WriteFile(filepath.Join(runDir, "01-nvidia-smi-topo-m.log"), topoOut, 0644) statusOut, _ := runSATCommandCtx(ctx, verboseLog, "nvidia-smi-nvlink-s", []string{"nvidia-smi", "nvlink", "-s"}, nil, logFunc) _ = os.WriteFile(filepath.Join(runDir, "02-nvidia-smi-nvlink-s.log"), statusOut, 0644) errOut, _ := runSATCommandCtx(ctx, verboseLog, "nvidia-smi-nvlink-e", []string{"nvidia-smi", "nvlink", "-e"}, nil, logFunc) _ = os.WriteFile(filepath.Join(runDir, "03-nvidia-smi-nvlink-e.log"), errOut, 0644) pairs := ParseNvidiaNVLinkBondedPairs(string(topoOut)) linkStatus := parseNvidiaNVLinkStatus(string(statusOut)) linkErrors := ParseNvidiaNVLinkErrors(string(errOut)) for _, pair := range pairs { f := evaluateNvidiaNVLinkPair(pair, linkStatus, linkErrors) status.NVLinkPairs = append(status.NVLinkPairs, f) for _, issue := range f.Issues { status.Warnings = append(status.Warnings, fmt.Sprintf("NVLink GPU%d<->GPU%d: %s", f.GPUA, f.GPUB, issue)) } } // -- Confidential Computing readiness (informational only) ------------- ccOut, ccErr := runSATCommandCtx(ctx, verboseLog, "nvidia-smi-conf-compute-q", []string{"nvidia-smi", "conf-compute", "-q"}, nil, logFunc) _ = os.WriteFile(filepath.Join(runDir, "04-nvidia-smi-conf-compute-q.log"), ccOut, 0644) if ccErr == nil { fields := parseConfComputeFields(ccOut) if status.CCState == "" { status.CCState = fields["CC State"] } status.CPUCCCapability = fields["CPU CC Capabilities"] status.GPUCCCapability = fields["GPU CC Capabilities"] } dmesgOut, _ := runSATCommandCtx(ctx, verboseLog, "dmesg", []string{"dmesg"}, nil, nil) ccDmesgLines := filterConfComputeDmesgLines(dmesgOut) _ = os.WriteFile(filepath.Join(runDir, "05-dmesg-cc-relevant.log"), []byte(strings.Join(ccDmesgLines, "\n")+"\n"), 0644) nvlinkDegraded := parseNvidiaNVLinkDegradedDmesg(dmesgOut) _ = os.WriteFile(filepath.Join(runDir, "06-dmesg-nvlink-degraded.log"), []byte(renderNvidiaNVLinkDegradedLog(nvlinkDegraded)), 0644) for _, event := range nvlinkDegraded { status.Warnings = append(status.Warnings, event.Warning()) } lowerDmesg := strings.ToLower(strings.Join(ccDmesgLines, "\n")) status.HostAMDSEVSNPActive = strings.Contains(lowerDmesg, "sev-snp enabled") status.HostIntelTDXActive = strings.Contains(lowerDmesg, "tdx module") && strings.Contains(lowerDmesg, "module initialized") || strings.Contains(lowerDmesg, "virt/tdx: module initialized") for _, path := range []string{"/sys/module/kvm_amd/parameters/sev_snp"} { if out, err := runSATCommandCtx(ctx, verboseLog, "sysfs-sev-snp", []string{"cat", path}, nil, nil); err == nil && strings.EqualFold(strings.TrimSpace(string(out)), "Y") { status.HostAMDSEVSNPActive = true } } gpuCanRunCC := strings.EqualFold(strings.TrimSpace(status.GPUCCCapability), "CC Capable") cpuCapReported := strings.TrimSpace(status.CPUCCCapability) cpuCanRunCC := status.HostAMDSEVSNPActive || status.HostIntelTDXActive || (cpuCapReported != "" && !strings.EqualFold(cpuCapReported, "NONE")) status.CCReady = cpuCanRunCC && gpuCanRunCC summary := renderNvidiaConfigCheckSummary(status) if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary), 0644); err != nil { return "", err } report := renderNvidiaConfigCheckReport(status) if err := os.WriteFile(filepath.Join(runDir, "nvidia-config-report.txt"), []byte(report), 0644); err != nil { return "", err } return runDir, nil } // NvidiaConfigCheckStatus is the result of RunNvidiaConfigCheckPack. type NvidiaConfigCheckStatus struct { CollectedAt time.Time `json:"collected_at"` GPUs []NvidiaGPUConfigFinding `json:"gpus,omitempty"` NVLinkPairs []NvidiaNVLinkPairFinding `json:"nvlink_pairs,omitempty"` NoGPUs bool `json:"no_gpus"` // Confidential Computing readiness — informational only, does not gate // overall_status: an unconfigured/NOT_READY CC state is a // feature-readiness fact on most fleets (CC is opt-in), not a fault. CCReady bool `json:"cc_ready"` CCState string `json:"cc_state,omitempty"` CPUCCCapability string `json:"cpu_cc_capability,omitempty"` GPUCCCapability string `json:"gpu_cc_capability,omitempty"` HostAMDSEVSNPActive bool `json:"host_amd_sev_snp_active"` HostIntelTDXActive bool `json:"host_intel_tdx_active"` // Warnings drive overall_status: any entry here fails the check. Warnings []string `json:"warnings,omitempty"` Notes []string `json:"notes,omitempty"` } // NvidiaGPUConfigFinding is one GPU's config-compliance result. type NvidiaGPUConfigFinding struct { Index int `json:"index"` Name string `json:"name"` ECCCurrent string `json:"ecc_current"` MIGCurrent string `json:"mig_current"` MIGPending string `json:"mig_pending"` PowerLimitW float64 `json:"power_limit_w"` PowerDefaultLimitW float64 `json:"power_default_limit_w"` Issues []string `json:"issues,omitempty"` } // NvidiaNVLinkPairFinding is one NVLink-bonded GPU pair's link-count/error // validation result. type NvidiaNVLinkPairFinding struct { GPUA int `json:"gpu_a"` GPUB int `json:"gpu_b"` ExpectedLinks int `json:"expected_links"` // from "nvidia-smi topo -m"'s NVx cell ActiveLinksA int `json:"active_links_a"` // per endpoint from "nvidia-smi nvlink -s" ActiveLinksB int `json:"active_links_b"` TotalLinksA int `json:"total_links_a"` TotalLinksB int `json:"total_links_b"` ActiveLinks int `json:"active_links"` // aggregate retained for report compatibility TotalLinks int `json:"total_links"` ErrorLinks int `json:"error_links"` // links with a nonzero replay/recovery/CRC counter Issues []string `json:"issues,omitempty"` } // evaluateNvidiaGPUConfig flags config drift that DCGM diag does not check: // ECC disabled (NVIDIA data-center GPUs ship with ECC on), a MIG mode // change stuck pending a reset/reboot, and a power limit capped meaningfully // below the card's own factory default (a silent throttle that a stress // test would only surface as "lower than expected performance", with no // clear cause). func evaluateNvidiaGPUConfig(g NvidiaGPUSetting) []string { var issues []string if strings.EqualFold(g.ECCCurrent, "disabled") { issues = append(issues, "ECC is disabled") } if g.MIGCurrent != "" && g.MIGPending != "" && !strings.EqualFold(g.MIGCurrent, g.MIGPending) { issues = append(issues, fmt.Sprintf("MIG mode change pending (current=%s, pending=%s) — apply with a GPU reset or reboot", g.MIGCurrent, g.MIGPending)) } if g.PowerDefaultLimitW > 0 && g.PowerLimitW > 0 { deficit := (g.PowerDefaultLimitW - g.PowerLimitW) / g.PowerDefaultLimitW if deficit > 0.05 { issues = append(issues, fmt.Sprintf("power limit %.0fW is %.0f%% below factory default %.0fW", g.PowerLimitW, deficit*100, g.PowerDefaultLimitW)) } } return issues } // evaluateNvidiaNVLinkPair compares a bonded pair's actual link state // against what the topology matrix says should be there. func evaluateNvidiaNVLinkPair(pair NvidiaNVLinkBondedPair, status map[int][]nvidiaNVLinkPort, errors map[int]map[int][3]int64) NvidiaNVLinkPairFinding { f := NvidiaNVLinkPairFinding{GPUA: pair.GPUA, GPUB: pair.GPUB, ExpectedLinks: pair.NVLinks} countPorts := func(gpu int) (active, total int, present bool) { ports, present := status[gpu] for _, port := range ports { total++ if port.active { active++ } } return active, total, present } f.ActiveLinksA, f.TotalLinksA, _ = countPorts(pair.GPUA) f.ActiveLinksB, f.TotalLinksB, _ = countPorts(pair.GPUB) f.ActiveLinks = f.ActiveLinksA + f.ActiveLinksB f.TotalLinks = f.TotalLinksA + f.TotalLinksB _, statusASeen := status[pair.GPUA] _, statusBSeen := status[pair.GPUB] if !statusASeen || !statusBSeen { var missing []string if !statusASeen { missing = append(missing, fmt.Sprintf("GPU%d", pair.GPUA)) } if !statusBSeen { missing = append(missing, fmt.Sprintf("GPU%d", pair.GPUB)) } f.Issues = append(f.Issues, "NVLink status unavailable for "+strings.Join(missing, ", ")) } if pair.NVLinks > 0 && (f.ActiveLinksA < pair.NVLinks || f.ActiveLinksB < pair.NVLinks) { f.Issues = append(f.Issues, fmt.Sprintf( "GPU%d %d/%d, GPU%d %d/%d active links (topo NV%d)", pair.GPUA, f.ActiveLinksA, pair.NVLinks, pair.GPUB, f.ActiveLinksB, pair.NVLinks, pair.NVLinks, )) } if f.TotalLinks > 0 && f.ActiveLinks < f.TotalLinks { f.Issues = append(f.Issues, fmt.Sprintf("%d/%d reported NVLinks inactive on a bonded pair", f.TotalLinks-f.ActiveLinks, f.TotalLinks)) } errLinks := 0 for _, gpu := range []int{pair.GPUA, pair.GPUB} { for _, counters := range errors[gpu] { if counters[0] != 0 || counters[1] != 0 || counters[2] != 0 { errLinks++ } } } f.ErrorLinks = errLinks if errLinks > 0 { f.Issues = append(f.Issues, fmt.Sprintf("%d link(s) reporting replay/recovery/CRC errors", errLinks)) } return f } // --------------------------------------------------------------------------- // NVLink parsing shared by diagnostics and the read-only topology page. // --------------------------------------------------------------------------- // NvidiaNVLinkBondedPair identifies two GPU indices and their negotiated // NVLink bond width as reported by nvidia-smi topo -m. type NvidiaNVLinkBondedPair struct { GPUA, GPUB int NVLinks int } var ( nvidiaNVLinkANSIRe = regexp.MustCompile("\x1b\\[[0-9;]*[A-Za-z]") nvidiaNVLinkNVRe = regexp.MustCompile(`(?i)^NV(\d+)$`) ) // ParseNvidiaNVLinkBondedPairs returns every GPU pair with a nonzero NVLink // bond count from a "nvidia-smi topo -m" matrix, deduplicated (A,B) == (B,A). func ParseNvidiaNVLinkBondedPairs(raw string) []NvidiaNVLinkBondedPair { lines := strings.Split(nvidiaNVLinkANSIRe.ReplaceAllString(raw, ""), "\n") headerIdx := -1 gpuColumns := map[int]int{} for i, line := range lines { columns := map[int]int{} for columnIndex, label := range strings.Fields(strings.TrimSpace(line)) { gpuIndex, err := strconv.Atoi(strings.TrimPrefix(label, "GPU")) if err == nil && strings.HasPrefix(label, "GPU") { columns[columnIndex] = gpuIndex } } if len(columns) >= 2 { headerIdx = i gpuColumns = columns break } } if headerIdx < 0 { return nil } seen := map[[2]int]bool{} var pairs []NvidiaNVLinkBondedPair for _, line := range lines[headerIdx+1:] { trimmed := strings.TrimSpace(line) if !strings.HasPrefix(trimmed, "GPU") { continue } cells := strings.Fields(trimmed) if len(cells) == 0 { continue } rowGPU, err := strconv.Atoi(strings.TrimPrefix(cells[0], "GPU")) if err != nil { continue } for colIdx, colGPU := range gpuColumns { if colGPU == rowGPU { continue } dataIdx := colIdx + 1 if dataIdx >= len(cells) { continue } m := nvidiaNVLinkNVRe.FindStringSubmatch(cells[dataIdx]) if len(m) != 2 { continue } nv, err := strconv.Atoi(m[1]) if err != nil || nv <= 0 { continue } a, b := rowGPU, colGPU if a > b { a, b = b, a } key := [2]int{a, b} if seen[key] { continue } seen[key] = true pairs = append(pairs, NvidiaNVLinkBondedPair{GPUA: a, GPUB: b, NVLinks: nv}) } } sort.Slice(pairs, func(i, j int) bool { if pairs[i].GPUA != pairs[j].GPUA { return pairs[i].GPUA < pairs[j].GPUA } return pairs[i].GPUB < pairs[j].GPUB }) return pairs } type nvidiaNVLinkPort struct { active bool } var ( nvidiaNVLinkGPUHeaderRe = regexp.MustCompile(`^GPU (\d+):`) nvidiaNVLinkSpeedLineRe = regexp.MustCompile(`^Link (\d+):\s*[\d.]+\s*GB/s`) nvidiaNVLinkInactiveRe = regexp.MustCompile(`^Link (\d+):\s*`) nvidiaNVLinkErrorLineRe = regexp.MustCompile(`^Link (\d+):\s*(Replay|Recovery|CRC) Errors:\s*(\d+)`) ) // parseNvidiaNVLinkStatus parses "nvidia-smi nvlink -s" output into per-GPU // per-link active/inactive state. func parseNvidiaNVLinkStatus(raw string) map[int][]nvidiaNVLinkPort { result := map[int][]nvidiaNVLinkPort{} currentGPU := -1 for _, line := range strings.Split(raw, "\n") { trimmed := strings.TrimSpace(line) if m := nvidiaNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil { currentGPU, _ = strconv.Atoi(m[1]) if _, exists := result[currentGPU]; !exists { // Preserve the distinction between a present GPU with no active // links and a GPU whose block is absent from the command output. result[currentGPU] = nil } continue } if currentGPU < 0 { continue } if nvidiaNVLinkInactiveRe.MatchString(trimmed) { result[currentGPU] = append(result[currentGPU], nvidiaNVLinkPort{active: false}) continue } if nvidiaNVLinkSpeedLineRe.MatchString(trimmed) { result[currentGPU] = append(result[currentGPU], nvidiaNVLinkPort{active: true}) } } return result } type nvidiaNVLinkDegradedEvent struct { GPUIndex int LinkID int RawLine string } func (e nvidiaNVLinkDegradedEvent) Warning() string { switch { case e.GPUIndex >= 0 && e.LinkID >= 0: return fmt.Sprintf("GPU%d NVLink degraded mode (linkId %d): %s", e.GPUIndex, e.LinkID, e.RawLine) case e.GPUIndex >= 0: return fmt.Sprintf("GPU%d NVLink degraded mode: %s", e.GPUIndex, e.RawLine) default: return "NVIDIA NVLink degraded mode: " + e.RawLine } } var ( nvidiaNVLinkDegradedGPURe = regexp.MustCompile(`(?i)\bGPU\s*(\d+)\b`) nvidiaNVLinkDegradedLinkRe = regexp.MustCompile(`(?i)\blinkId\s*[:=]?\s*(\d+)\b`) ) func parseNvidiaNVLinkDegradedDmesg(raw []byte) []nvidiaNVLinkDegradedEvent { var events []nvidiaNVLinkDegradedEvent for _, lineBytes := range bytes.Split(raw, []byte("\n")) { line := strings.TrimSpace(string(lineBytes)) lower := strings.ToLower(line) explicitNVLinkMarker := strings.Contains(lower, "knvlinksetdegradedmode") || strings.Contains(lower, "nvlink_is_gpu_degraded") || strings.Contains(lower, "error originated on linkid") contextualDegradedMarker := strings.Contains(lower, "marked degraded") && (strings.Contains(lower, "nvlink") || strings.Contains(lower, "nvrm") || nvidiaNVLinkDegradedGPURe.MatchString(line)) if line == "" || (!explicitNVLinkMarker && !contextualDegradedMarker) { continue } event := nvidiaNVLinkDegradedEvent{GPUIndex: -1, LinkID: -1, RawLine: line} if match := nvidiaNVLinkDegradedGPURe.FindStringSubmatch(line); len(match) == 2 { event.GPUIndex, _ = strconv.Atoi(match[1]) } if match := nvidiaNVLinkDegradedLinkRe.FindStringSubmatch(line); len(match) == 2 { event.LinkID, _ = strconv.Atoi(match[1]) } events = append(events, event) } return events } func renderNvidiaNVLinkDegradedLog(events []nvidiaNVLinkDegradedEvent) string { if len(events) == 0 { return "" } lines := make([]string, len(events)) for i, event := range events { lines[i] = event.RawLine } return strings.Join(lines, "\n") + "\n" } // ParseNvidiaNVLinkErrors parses "nvidia-smi nvlink -e" output into, per GPU // then link index, [replay, recovery, crc] error counts. func ParseNvidiaNVLinkErrors(raw string) map[int]map[int][3]int64 { result := map[int]map[int][3]int64{} currentGPU := -1 for _, line := range strings.Split(raw, "\n") { trimmed := strings.TrimSpace(line) if m := nvidiaNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil { currentGPU, _ = strconv.Atoi(m[1]) continue } if currentGPU < 0 { continue } m := nvidiaNVLinkErrorLineRe.FindStringSubmatch(trimmed) if m == nil { continue } linkIdx, _ := strconv.Atoi(m[1]) count, _ := strconv.ParseInt(m[3], 10, 64) if result[currentGPU] == nil { result[currentGPU] = map[int][3]int64{} } c := result[currentGPU][linkIdx] switch m[2] { case "Replay": c[0] = count case "Recovery": c[1] = count case "CRC": c[2] = count } result[currentGPU][linkIdx] = c } return result } // --------------------------------------------------------------------------- // Confidential Computing (folded in — see RunNvidiaConfigCheckPack doc) // --------------------------------------------------------------------------- // parseConfComputeFields parses the indented "Key : Value" block emitted by // `nvidia-smi conf-compute -q`, e.g.: // // CC State : OFF // Multi-GPU Mode : Protected PCIe // CPU CC Capabilities : INTEL TDX // GPU CC Capabilities : CC Capable // CC GPUs Ready State : Not Ready func parseConfComputeFields(out []byte) map[string]string { fields := map[string]string{} for _, line := range strings.Split(string(out), "\n") { idx := strings.Index(line, ":") if idx < 0 { continue } key := strings.TrimSpace(line[:idx]) val := strings.TrimSpace(line[idx+1:]) if key == "" || val == "" { continue } fields[key] = val } return fields } // filterConfComputeDmesgLines returns the dmesg lines relevant to CPU // Confidential Computing support (AMD SEV/SEV-ES/SEV-SNP, Intel TDX). func filterConfComputeDmesgLines(dmesgOut []byte) []string { var lines []string for _, raw := range bytes.Split(dmesgOut, []byte("\n")) { lower := strings.ToLower(string(raw)) if strings.Contains(lower, "sev") || strings.Contains(lower, "tdx") { lines = append(lines, string(raw)) } } return lines } // --------------------------------------------------------------------------- // Report rendering // --------------------------------------------------------------------------- func renderNvidiaConfigCheckSummary(status NvidiaConfigCheckStatus) string { var b strings.Builder fmt.Fprintf(&b, "run_at_utc=%s\n", status.CollectedAt.Format(time.RFC3339)) fmt.Fprintf(&b, "gpu_count=%d\n", len(status.GPUs)) eccIssues, migIssues, powerIssues := 0, 0, 0 for _, g := range status.GPUs { for _, issue := range g.Issues { switch { case strings.Contains(issue, "ECC"): eccIssues++ case strings.Contains(issue, "MIG"): migIssues++ case strings.Contains(issue, "power limit"): powerIssues++ } } } fmt.Fprintf(&b, "gpu_ecc_issues=%d\n", eccIssues) fmt.Fprintf(&b, "gpu_mig_pending_issues=%d\n", migIssues) fmt.Fprintf(&b, "gpu_power_limit_issues=%d\n", powerIssues) fmt.Fprintf(&b, "nvlink_pairs_checked=%d\n", len(status.NVLinkPairs)) nvlinkIssues := 0 for _, p := range status.NVLinkPairs { if len(p.Issues) > 0 { nvlinkIssues++ } } fmt.Fprintf(&b, "nvlink_pairs_with_issues=%d\n", nvlinkIssues) fmt.Fprintf(&b, "cc_ready=%t\n", status.CCReady) fmt.Fprintf(&b, "cc_state=%s\n", status.CCState) fmt.Fprintf(&b, "cpu_cc_capability=%s\n", status.CPUCCCapability) fmt.Fprintf(&b, "gpu_cc_capability=%s\n", status.GPUCCCapability) if status.NoGPUs { fmt.Fprintln(&b, "overall_status=UNSUPPORTED") fmt.Fprintln(&b, "reason=no_nvidia_gpus_detected") } else if len(status.Warnings) == 0 { fmt.Fprintln(&b, "overall_status=OK") } else { fmt.Fprintln(&b, "overall_status=FAILED") // warnings carries the actual reason(s) in prose so a reader of // summary.txt (or anything parsing it, like sat.DescribeFailure) // never has to fall back to "see the report" — each entry already // names the specific GPU or NVLink pair and what's wrong with it. fmt.Fprintf(&b, "warnings=%s\n", strings.Join(status.Warnings, "; ")) } return b.String() } func renderNvidiaConfigCheckReport(status NvidiaConfigCheckStatus) string { var b strings.Builder line := strings.Repeat("=", 80) b.WriteString(line + "\n") b.WriteString("NVIDIA GPU Configuration & NVLink Topology Check\n") b.WriteString(line + "\n\n") b.WriteString("-- GPU Configuration ----------------------------------------------------------\n") for _, g := range status.GPUs { verdict := "OK" if len(g.Issues) > 0 { verdict = "ISSUES FOUND" } fmt.Fprintf(&b, " GPU %d (%s): %s\n", g.Index, g.Name, verdict) fmt.Fprintf(&b, " ECC: %s | MIG: %s (pending %s) | Power: %.0fW / %.0fW default\n", nonEmptyOr(g.ECCCurrent, "unknown"), nonEmptyOr(g.MIGCurrent, "unknown"), nonEmptyOr(g.MIGPending, "unknown"), g.PowerLimitW, g.PowerDefaultLimitW) for _, issue := range g.Issues { fmt.Fprintf(&b, " - %s\n", issue) } } b.WriteString("\n-- NVLink Topology --------------------------------------------------------------\n") if len(status.NVLinkPairs) == 0 { b.WriteString(" No NVLink-bonded GPU pairs found.\n") } for _, p := range status.NVLinkPairs { verdict := "OK" if len(p.Issues) > 0 { verdict = "ISSUES FOUND" } fmt.Fprintf(&b, " GPU%d <-> GPU%d: endpoints %d/%d and %d/%d active (topology expects %d each): %s\n", p.GPUA, p.GPUB, p.ActiveLinksA, p.TotalLinksA, p.ActiveLinksB, p.TotalLinksB, p.ExpectedLinks, verdict) for _, issue := range p.Issues { fmt.Fprintf(&b, " - %s\n", issue) } } b.WriteString("\n-- Confidential Computing (informational) ---------------------------------------\n") verdict := "NOT READY" if status.CCReady { verdict = "READY" } fmt.Fprintf(&b, " Verdict: %s\n", verdict) fmt.Fprintf(&b, " CC State: %s | CPU CC: %s | GPU CC: %s\n", nonEmptyOr(status.CCState, "unknown"), nonEmptyOr(status.CPUCCCapability, "unknown"), nonEmptyOr(status.GPUCCCapability, "unknown")) if len(status.Notes) > 0 { b.WriteString("\n-- Notes -------------------------------------------------------------------------\n") for _, n := range status.Notes { fmt.Fprintf(&b, " - %s\n", n) } } fmt.Fprintf(&b, "\nCollected : %s\n", status.CollectedAt.Format("2006-01-02 15:04:05 UTC")) b.WriteString(line + "\n") return b.String() } func nonEmptyOr(v, fallback string) string { if strings.TrimSpace(v) == "" { return fallback } return v }