platform/webui: fold confidential-computing into a GPU config + NVLink check
The standalone "confidential-computing" SAT target only ever checked CC readiness, which most fleets never opt into (a NOT_READY verdict there isn't a fault). Meanwhile DCGM diag never asserts GPU config compliance (ECC/MIG/power-limit vs factory default) or NVLink topology (per NVIDIA's own DGX BasePOD deployment guide, this needs a separate validation step) — gaps confirmed against public DCGM docs and a real NV17-vs-expected-NV18 bonded pair found on a live bundle. Repurposes the routine into "nvidia-config": reuses the existing ListNvidiaGPUSettings() (already backing the GPU-settings page) to flag ECC disabled, a MIG mode change stuck pending a reset/reboot, and a power limit capped >5% below default; parses "nvidia-smi topo -m" bonded pairs against "nvlink -s/-e" to flag any inactive lane or nonzero replay/ recovery/CRC counter on an otherwise-active bond. CC readiness is folded in as one informational field (does not gate overall_status) rather than a dedicated test. Reports under the same pcie:gpu:nvidia severity key as every other nvidia-* SAT target instead of an isolated key, so a config/NVLink FAILED result isn't invisible next to stress-test results. Also fixes ApplySATResultToDB silently dropping any target with no matching switch case (exactly what the old confidential-computing target did) with a new coverage test enumerating every real SAT target. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
cc3997f7b1
commit
cfaa15ec7c
@@ -1,248 +0,0 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ConfidentialComputingStatus summarizes whether this server can run NVIDIA
|
||||
// Confidential Computing: CPU-side TEE support (Intel TDX / AMD SEV-SNP) and
|
||||
// GPU firmware CC capability, as reported by `nvidia-smi conf-compute -q`.
|
||||
type ConfidentialComputingStatus struct {
|
||||
CollectedAt time.Time `json:"collected_at"`
|
||||
|
||||
// GPU-reported fields, parsed from `nvidia-smi conf-compute -q`.
|
||||
NvidiaSMIAvailable bool `json:"nvidia_smi_available"`
|
||||
CCState string `json:"cc_state,omitempty"` // ON / OFF
|
||||
MultiGPUMode string `json:"multi_gpu_mode,omitempty"` // Protected PCIe / ...
|
||||
CPUCCCapability string `json:"cpu_cc_capability,omitempty"` // e.g. "INTEL TDX", "AMD SEV-SNP", "NONE"
|
||||
GPUCCCapability string `json:"gpu_cc_capability,omitempty"` // e.g. "CC Capable", "Not Capable"
|
||||
CCGPUsReadyState string `json:"cc_gpus_ready_state,omitempty"` // Ready / Not Ready
|
||||
|
||||
// Host-side evidence that the CPU's TEE is actually active in the running
|
||||
// kernel (BIOS + kernel cmdline + firmware), independent of what the GPU
|
||||
// driver reports. Used as a fallback when the NVIDIA driver isn't loaded.
|
||||
HostAMDSEVSupported bool `json:"host_amd_sev_supported"`
|
||||
HostAMDSEVESSupported bool `json:"host_amd_sev_es_supported"`
|
||||
HostAMDSEVSNPActive bool `json:"host_amd_sev_snp_active"`
|
||||
HostIntelTDXActive bool `json:"host_intel_tdx_active"`
|
||||
|
||||
// GPUCanRunCC is true when the GPU firmware reports CC-capable.
|
||||
GPUCanRunCC bool `json:"gpu_can_run_cc"`
|
||||
// CPUCanRunCC is true when either the GPU driver or the host kernel
|
||||
// reports an active/available CPU TEE (SEV-SNP or TDX).
|
||||
CPUCanRunCC bool `json:"cpu_can_run_cc"`
|
||||
// Ready is true when both the CPU and the GPU support Confidential
|
||||
// Computing, regardless of whether CC mode is currently enabled.
|
||||
Ready bool `json:"ready"`
|
||||
|
||||
Notes []string `json:"notes,omitempty"`
|
||||
}
|
||||
|
||||
// RunConfidentialComputingCheckPack runs a read-only check of whether this
|
||||
// server can run NVIDIA Confidential Computing: it queries the GPU driver
|
||||
// (`nvidia-smi conf-compute -q`) and inspects host kernel/dmesg evidence of
|
||||
// AMD SEV-SNP / Intel TDX support. It changes nothing on the system.
|
||||
func (s *System) RunConfidentialComputingCheckPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if baseDir == "" {
|
||||
baseDir = "/var/log/bee-sat"
|
||||
}
|
||||
ts := time.Now().UTC().Format("20060102-150405")
|
||||
runDir := filepath.Join(baseDir, "confidential-computing-"+ts)
|
||||
if err := os.MkdirAll(runDir, 0755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
verboseLog := filepath.Join(runDir, "verbose.log")
|
||||
|
||||
status := ConfidentialComputingStatus{CollectedAt: time.Now().UTC()}
|
||||
|
||||
// GPU firmware / driver state.
|
||||
ccOut, ccErr := runSATCommandCtx(ctx, verboseLog, "nvidia-smi-conf-compute-q", []string{"nvidia-smi", "conf-compute", "-q"}, nil, logFunc)
|
||||
_ = os.WriteFile(filepath.Join(runDir, "01-nvidia-smi-conf-compute-q.log"), ccOut, 0644)
|
||||
if ccErr == nil {
|
||||
status.NvidiaSMIAvailable = true
|
||||
fields := parseConfComputeFields(ccOut)
|
||||
status.CCState = fields["CC State"]
|
||||
status.MultiGPUMode = fields["Multi-GPU Mode"]
|
||||
status.CPUCCCapability = fields["CPU CC Capabilities"]
|
||||
status.GPUCCCapability = fields["GPU CC Capabilities"]
|
||||
status.CCGPUsReadyState = fields["CC GPUs Ready State"]
|
||||
} else {
|
||||
status.Notes = append(status.Notes, "nvidia-smi conf-compute -q unavailable (no NVIDIA driver, or GPU not present): "+firstLine(string(ccOut)))
|
||||
}
|
||||
|
||||
// Host kernel evidence, independent of the GPU driver.
|
||||
dmesgOut, _ := runSATCommandCtx(ctx, verboseLog, "dmesg", []string{"dmesg"}, nil, nil)
|
||||
ccDmesgLines := filterConfComputeDmesgLines(dmesgOut)
|
||||
_ = os.WriteFile(filepath.Join(runDir, "02-dmesg-cc-relevant.log"), []byte(strings.Join(ccDmesgLines, "\n")+"\n"), 0644)
|
||||
|
||||
lowerDmesg := strings.ToLower(strings.Join(ccDmesgLines, "\n"))
|
||||
status.HostAMDSEVSNPActive = strings.Contains(lowerDmesg, "sev-snp enabled")
|
||||
status.HostIntelTDXActive = strings.Contains(lowerDmesg, "tdx module") && strings.Contains(lowerDmesg, "module initialized") ||
|
||||
strings.Contains(lowerDmesg, "virt/tdx: module initialized")
|
||||
|
||||
for i, path := range []string{
|
||||
"/sys/module/kvm_amd/parameters/sev",
|
||||
"/sys/module/kvm_amd/parameters/sev_es",
|
||||
"/sys/module/kvm_amd/parameters/sev_snp",
|
||||
} {
|
||||
name := fmt.Sprintf("sysfs-%s", filepath.Base(path))
|
||||
out, err := runSATCommandCtx(ctx, verboseLog, name, []string{"cat", path}, nil, nil)
|
||||
_ = os.WriteFile(filepath.Join(runDir, fmt.Sprintf("03-%02d-%s.log", i+1, name)), out, 0644)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
val := strings.TrimSpace(string(out))
|
||||
switch filepath.Base(path) {
|
||||
case "sev":
|
||||
status.HostAMDSEVSupported = strings.EqualFold(val, "Y")
|
||||
case "sev_es":
|
||||
status.HostAMDSEVESSupported = strings.EqualFold(val, "Y")
|
||||
case "sev_snp":
|
||||
if strings.EqualFold(val, "Y") {
|
||||
status.HostAMDSEVSNPActive = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
status.GPUCanRunCC = strings.EqualFold(strings.TrimSpace(status.GPUCCCapability), "CC Capable")
|
||||
cpuCapReported := strings.TrimSpace(status.CPUCCCapability)
|
||||
status.CPUCanRunCC = status.HostAMDSEVSNPActive || status.HostIntelTDXActive ||
|
||||
(cpuCapReported != "" && !strings.EqualFold(cpuCapReported, "NONE"))
|
||||
status.Ready = status.CPUCanRunCC && status.GPUCanRunCC
|
||||
|
||||
if !status.NvidiaSMIAvailable {
|
||||
status.Notes = append(status.Notes, "GPU CC capability unknown — install the NVIDIA driver to query it with `nvidia-smi conf-compute -q`.")
|
||||
}
|
||||
|
||||
summary := renderConfidentialComputingSummary(status)
|
||||
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary), 0644); err != nil {
|
||||
return "", err
|
||||
}
|
||||
report := renderConfidentialComputingReport(status)
|
||||
if err := os.WriteFile(filepath.Join(runDir, "confidential-computing-report.txt"), []byte(report), 0644); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return runDir, nil
|
||||
}
|
||||
|
||||
// parseConfComputeFields parses the indented "Key : Value" block emitted by
|
||||
// `nvidia-smi conf-compute -q`, e.g.:
|
||||
//
|
||||
// CC State : OFF
|
||||
// Multi-GPU Mode : Protected PCIe
|
||||
// CPU CC Capabilities : INTEL TDX
|
||||
// GPU CC Capabilities : CC Capable
|
||||
// CC GPUs Ready State : Not Ready
|
||||
func parseConfComputeFields(out []byte) map[string]string {
|
||||
fields := map[string]string{}
|
||||
for _, line := range strings.Split(string(out), "\n") {
|
||||
idx := strings.Index(line, ":")
|
||||
if idx < 0 {
|
||||
continue
|
||||
}
|
||||
key := strings.TrimSpace(line[:idx])
|
||||
val := strings.TrimSpace(line[idx+1:])
|
||||
if key == "" || val == "" {
|
||||
continue
|
||||
}
|
||||
fields[key] = val
|
||||
}
|
||||
return fields
|
||||
}
|
||||
|
||||
// filterConfComputeDmesgLines returns the dmesg lines relevant to CPU
|
||||
// Confidential Computing support (AMD SEV/SEV-ES/SEV-SNP, Intel TDX).
|
||||
func filterConfComputeDmesgLines(dmesgOut []byte) []string {
|
||||
var lines []string
|
||||
scanner := bytes.Split(dmesgOut, []byte("\n"))
|
||||
for _, raw := range scanner {
|
||||
lower := strings.ToLower(string(raw))
|
||||
if strings.Contains(lower, "sev") || strings.Contains(lower, "tdx") {
|
||||
lines = append(lines, string(raw))
|
||||
}
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
func renderConfidentialComputingSummary(status ConfidentialComputingStatus) string {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "run_at_utc=%s\n", status.CollectedAt.Format(time.RFC3339))
|
||||
fmt.Fprintf(&b, "nvidia_smi_available=%t\n", status.NvidiaSMIAvailable)
|
||||
fmt.Fprintf(&b, "cc_state=%s\n", status.CCState)
|
||||
fmt.Fprintf(&b, "multi_gpu_mode=%s\n", status.MultiGPUMode)
|
||||
fmt.Fprintf(&b, "cpu_cc_capability=%s\n", status.CPUCCCapability)
|
||||
fmt.Fprintf(&b, "gpu_cc_capability=%s\n", status.GPUCCCapability)
|
||||
fmt.Fprintf(&b, "cc_gpus_ready_state=%s\n", status.CCGPUsReadyState)
|
||||
fmt.Fprintf(&b, "host_amd_sev_supported=%t\n", status.HostAMDSEVSupported)
|
||||
fmt.Fprintf(&b, "host_amd_sev_es_supported=%t\n", status.HostAMDSEVESSupported)
|
||||
fmt.Fprintf(&b, "host_amd_sev_snp_active=%t\n", status.HostAMDSEVSNPActive)
|
||||
fmt.Fprintf(&b, "host_intel_tdx_active=%t\n", status.HostIntelTDXActive)
|
||||
fmt.Fprintf(&b, "cpu_can_run_cc=%t\n", status.CPUCanRunCC)
|
||||
fmt.Fprintf(&b, "gpu_can_run_cc=%t\n", status.GPUCanRunCC)
|
||||
fmt.Fprintf(&b, "ready=%t\n", status.Ready)
|
||||
if status.Ready {
|
||||
fmt.Fprintln(&b, "overall_status=OK")
|
||||
} else {
|
||||
fmt.Fprintln(&b, "overall_status=NOT_READY")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func renderConfidentialComputingReport(status ConfidentialComputingStatus) string {
|
||||
var b strings.Builder
|
||||
line := strings.Repeat("=", 80)
|
||||
b.WriteString(line + "\n")
|
||||
b.WriteString("Confidential Computing Readiness\n")
|
||||
b.WriteString(line + "\n\n")
|
||||
|
||||
verdict := "NOT READY"
|
||||
if status.Ready {
|
||||
verdict = "READY"
|
||||
}
|
||||
fmt.Fprintf(&b, "Verdict: %s\n\n", verdict)
|
||||
|
||||
b.WriteString("-- CPU ----------------------------------------------------------------------\n")
|
||||
fmt.Fprintf(&b, " Reported by GPU driver : %s\n", nonEmptyOr(status.CPUCCCapability, "unknown"))
|
||||
fmt.Fprintf(&b, " AMD SEV supported : %t\n", status.HostAMDSEVSupported)
|
||||
fmt.Fprintf(&b, " AMD SEV-ES supported : %t\n", status.HostAMDSEVESSupported)
|
||||
fmt.Fprintf(&b, " AMD SEV-SNP active : %t\n", status.HostAMDSEVSNPActive)
|
||||
fmt.Fprintf(&b, " Intel TDX active : %t\n", status.HostIntelTDXActive)
|
||||
fmt.Fprintf(&b, " Can run CC : %t\n\n", status.CPUCanRunCC)
|
||||
|
||||
b.WriteString("-- GPU ----------------------------------------------------------------------\n")
|
||||
fmt.Fprintf(&b, " nvidia-smi available : %t\n", status.NvidiaSMIAvailable)
|
||||
fmt.Fprintf(&b, " GPU CC Capabilities : %s\n", nonEmptyOr(status.GPUCCCapability, "unknown"))
|
||||
fmt.Fprintf(&b, " CC State (current) : %s\n", nonEmptyOr(status.CCState, "unknown"))
|
||||
fmt.Fprintf(&b, " Multi-GPU Mode : %s\n", nonEmptyOr(status.MultiGPUMode, "unknown"))
|
||||
fmt.Fprintf(&b, " CC GPUs Ready State : %s\n", nonEmptyOr(status.CCGPUsReadyState, "unknown"))
|
||||
fmt.Fprintf(&b, " Can run CC : %t\n\n", status.GPUCanRunCC)
|
||||
|
||||
if len(status.Notes) > 0 {
|
||||
b.WriteString("-- Notes ----------------------------------------------------------------------\n")
|
||||
for _, n := range status.Notes {
|
||||
fmt.Fprintf(&b, " - %s\n", n)
|
||||
}
|
||||
b.WriteString("\n")
|
||||
}
|
||||
|
||||
fmt.Fprintf(&b, "Collected : %s\n", status.CollectedAt.Format("2006-01-02 15:04:05 UTC"))
|
||||
b.WriteString(line + "\n")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func nonEmptyOr(v, fallback string) string {
|
||||
if strings.TrimSpace(v) == "" {
|
||||
return fallback
|
||||
}
|
||||
return v
|
||||
}
|
||||
@@ -0,0 +1,540 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// RunNvidiaConfigCheckPack runs a read-only check of NVIDIA GPU
|
||||
// configuration and NVLink topology that DCGM diag does not cover: DCGM's
|
||||
// diagnostic levels stress-test compute/memory/power/bandwidth, but never
|
||||
// assert that persistent config (ECC, MIG, power limit) matches the factory
|
||||
// default, and NVIDIA's own DGX BasePOD deployment guide calls out
|
||||
// topology/NVLink validation as a distinct manual step outside DCGM diag.
|
||||
// Confidential Computing readiness — previously its own standalone SAT
|
||||
// target — is folded in here as one more read-only fact about GPU state
|
||||
// rather than a dedicated test, since none of these are stress tests; they
|
||||
// are one-shot state/config reads that all answer "is this GPU configured
|
||||
// the way we expect", not "does it survive load".
|
||||
func (s *System) RunNvidiaConfigCheckPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
if baseDir == "" {
|
||||
baseDir = "/var/log/bee-sat"
|
||||
}
|
||||
ts := time.Now().UTC().Format("20060102-150405")
|
||||
runDir := filepath.Join(baseDir, "nvidia-config-"+ts)
|
||||
if err := os.MkdirAll(runDir, 0755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
verboseLog := filepath.Join(runDir, "verbose.log")
|
||||
|
||||
status := NvidiaConfigCheckStatus{CollectedAt: time.Now().UTC()}
|
||||
|
||||
// -- GPU config compliance (ECC/MIG/power limit) -----------------------
|
||||
if settings, err := s.ListNvidiaGPUSettings(); err != nil {
|
||||
status.Notes = append(status.Notes, "nvidia-smi GPU settings unavailable (no driver, or no GPU present): "+err.Error())
|
||||
} else {
|
||||
for _, g := range settings {
|
||||
f := NvidiaGPUConfigFinding{
|
||||
Index: g.Index, Name: g.Name,
|
||||
ECCCurrent: g.ECCCurrent, MIGCurrent: g.MIGCurrent, MIGPending: g.MIGPending,
|
||||
PowerLimitW: g.PowerLimitW, PowerDefaultLimitW: g.PowerDefaultLimitW,
|
||||
Issues: evaluateNvidiaGPUConfig(g),
|
||||
}
|
||||
status.GPUs = append(status.GPUs, f)
|
||||
for _, issue := range f.Issues {
|
||||
status.Warnings = append(status.Warnings, fmt.Sprintf("GPU %d (%s): %s", g.Index, g.Name, issue))
|
||||
}
|
||||
if strings.EqualFold(g.CCState, "on") {
|
||||
status.CCState = "ON"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// -- NVLink topology (bonded pairs, active links, error counters) ------
|
||||
topoOut, _ := runSATCommandCtx(ctx, verboseLog, "nvidia-smi-topo-m", []string{"nvidia-smi", "topo", "-m"}, nil, logFunc)
|
||||
_ = os.WriteFile(filepath.Join(runDir, "01-nvidia-smi-topo-m.log"), topoOut, 0644)
|
||||
statusOut, _ := runSATCommandCtx(ctx, verboseLog, "nvidia-smi-nvlink-s", []string{"nvidia-smi", "nvlink", "-s"}, nil, logFunc)
|
||||
_ = os.WriteFile(filepath.Join(runDir, "02-nvidia-smi-nvlink-s.log"), statusOut, 0644)
|
||||
errOut, _ := runSATCommandCtx(ctx, verboseLog, "nvidia-smi-nvlink-e", []string{"nvidia-smi", "nvlink", "-e"}, nil, logFunc)
|
||||
_ = os.WriteFile(filepath.Join(runDir, "03-nvidia-smi-nvlink-e.log"), errOut, 0644)
|
||||
|
||||
pairs := parseNvidiaNVLinkBondedPairs(string(topoOut))
|
||||
linkStatus := parseNvidiaNVLinkStatus(string(statusOut))
|
||||
linkErrors := parseNvidiaNVLinkErrors(string(errOut))
|
||||
for _, pair := range pairs {
|
||||
f := evaluateNvidiaNVLinkPair(pair, linkStatus, linkErrors)
|
||||
status.NVLinkPairs = append(status.NVLinkPairs, f)
|
||||
for _, issue := range f.Issues {
|
||||
status.Warnings = append(status.Warnings, fmt.Sprintf("NVLink GPU%d<->GPU%d: %s", f.GPUA, f.GPUB, issue))
|
||||
}
|
||||
}
|
||||
|
||||
// -- Confidential Computing readiness (informational only) -------------
|
||||
ccOut, ccErr := runSATCommandCtx(ctx, verboseLog, "nvidia-smi-conf-compute-q", []string{"nvidia-smi", "conf-compute", "-q"}, nil, logFunc)
|
||||
_ = os.WriteFile(filepath.Join(runDir, "04-nvidia-smi-conf-compute-q.log"), ccOut, 0644)
|
||||
if ccErr == nil {
|
||||
fields := parseConfComputeFields(ccOut)
|
||||
if status.CCState == "" {
|
||||
status.CCState = fields["CC State"]
|
||||
}
|
||||
status.CPUCCCapability = fields["CPU CC Capabilities"]
|
||||
status.GPUCCCapability = fields["GPU CC Capabilities"]
|
||||
}
|
||||
dmesgOut, _ := runSATCommandCtx(ctx, verboseLog, "dmesg", []string{"dmesg"}, nil, nil)
|
||||
ccDmesgLines := filterConfComputeDmesgLines(dmesgOut)
|
||||
_ = os.WriteFile(filepath.Join(runDir, "05-dmesg-cc-relevant.log"), []byte(strings.Join(ccDmesgLines, "\n")+"\n"), 0644)
|
||||
lowerDmesg := strings.ToLower(strings.Join(ccDmesgLines, "\n"))
|
||||
status.HostAMDSEVSNPActive = strings.Contains(lowerDmesg, "sev-snp enabled")
|
||||
status.HostIntelTDXActive = strings.Contains(lowerDmesg, "tdx module") && strings.Contains(lowerDmesg, "module initialized") ||
|
||||
strings.Contains(lowerDmesg, "virt/tdx: module initialized")
|
||||
for _, path := range []string{"/sys/module/kvm_amd/parameters/sev_snp"} {
|
||||
if out, err := runSATCommandCtx(ctx, verboseLog, "sysfs-sev-snp", []string{"cat", path}, nil, nil); err == nil &&
|
||||
strings.EqualFold(strings.TrimSpace(string(out)), "Y") {
|
||||
status.HostAMDSEVSNPActive = true
|
||||
}
|
||||
}
|
||||
gpuCanRunCC := strings.EqualFold(strings.TrimSpace(status.GPUCCCapability), "CC Capable")
|
||||
cpuCapReported := strings.TrimSpace(status.CPUCCCapability)
|
||||
cpuCanRunCC := status.HostAMDSEVSNPActive || status.HostIntelTDXActive ||
|
||||
(cpuCapReported != "" && !strings.EqualFold(cpuCapReported, "NONE"))
|
||||
status.CCReady = cpuCanRunCC && gpuCanRunCC
|
||||
|
||||
summary := renderNvidiaConfigCheckSummary(status)
|
||||
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary), 0644); err != nil {
|
||||
return "", err
|
||||
}
|
||||
report := renderNvidiaConfigCheckReport(status)
|
||||
if err := os.WriteFile(filepath.Join(runDir, "nvidia-config-report.txt"), []byte(report), 0644); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return runDir, nil
|
||||
}
|
||||
|
||||
// NvidiaConfigCheckStatus is the result of RunNvidiaConfigCheckPack.
|
||||
type NvidiaConfigCheckStatus struct {
|
||||
CollectedAt time.Time `json:"collected_at"`
|
||||
|
||||
GPUs []NvidiaGPUConfigFinding `json:"gpus,omitempty"`
|
||||
NVLinkPairs []NvidiaNVLinkPairFinding `json:"nvlink_pairs,omitempty"`
|
||||
|
||||
// Confidential Computing readiness — informational only, does not gate
|
||||
// overall_status: an unconfigured/NOT_READY CC state is a
|
||||
// feature-readiness fact on most fleets (CC is opt-in), not a fault.
|
||||
CCReady bool `json:"cc_ready"`
|
||||
CCState string `json:"cc_state,omitempty"`
|
||||
CPUCCCapability string `json:"cpu_cc_capability,omitempty"`
|
||||
GPUCCCapability string `json:"gpu_cc_capability,omitempty"`
|
||||
|
||||
HostAMDSEVSNPActive bool `json:"host_amd_sev_snp_active"`
|
||||
HostIntelTDXActive bool `json:"host_intel_tdx_active"`
|
||||
|
||||
// Warnings drive overall_status: any entry here fails the check.
|
||||
Warnings []string `json:"warnings,omitempty"`
|
||||
Notes []string `json:"notes,omitempty"`
|
||||
}
|
||||
|
||||
// NvidiaGPUConfigFinding is one GPU's config-compliance result.
|
||||
type NvidiaGPUConfigFinding struct {
|
||||
Index int `json:"index"`
|
||||
Name string `json:"name"`
|
||||
ECCCurrent string `json:"ecc_current"`
|
||||
MIGCurrent string `json:"mig_current"`
|
||||
MIGPending string `json:"mig_pending"`
|
||||
PowerLimitW float64 `json:"power_limit_w"`
|
||||
PowerDefaultLimitW float64 `json:"power_default_limit_w"`
|
||||
Issues []string `json:"issues,omitempty"`
|
||||
}
|
||||
|
||||
// NvidiaNVLinkPairFinding is one NVLink-bonded GPU pair's link-count/error
|
||||
// validation result.
|
||||
type NvidiaNVLinkPairFinding struct {
|
||||
GPUA int `json:"gpu_a"`
|
||||
GPUB int `json:"gpu_b"`
|
||||
ExpectedLinks int `json:"expected_links"` // from "nvidia-smi topo -m"'s NVx cell
|
||||
ActiveLinks int `json:"active_links"` // from "nvidia-smi nvlink -s"
|
||||
TotalLinks int `json:"total_links"`
|
||||
ErrorLinks int `json:"error_links"` // links with a nonzero replay/recovery/CRC counter
|
||||
Issues []string `json:"issues,omitempty"`
|
||||
}
|
||||
|
||||
// evaluateNvidiaGPUConfig flags config drift that DCGM diag does not check:
|
||||
// ECC disabled (NVIDIA data-center GPUs ship with ECC on), a MIG mode
|
||||
// change stuck pending a reset/reboot, and a power limit capped meaningfully
|
||||
// below the card's own factory default (a silent throttle that a stress
|
||||
// test would only surface as "lower than expected performance", with no
|
||||
// clear cause).
|
||||
func evaluateNvidiaGPUConfig(g NvidiaGPUSetting) []string {
|
||||
var issues []string
|
||||
if strings.EqualFold(g.ECCCurrent, "disabled") {
|
||||
issues = append(issues, "ECC is disabled")
|
||||
}
|
||||
if g.MIGCurrent != "" && g.MIGPending != "" && !strings.EqualFold(g.MIGCurrent, g.MIGPending) {
|
||||
issues = append(issues, fmt.Sprintf("MIG mode change pending (current=%s, pending=%s) — apply with a GPU reset or reboot", g.MIGCurrent, g.MIGPending))
|
||||
}
|
||||
if g.PowerDefaultLimitW > 0 && g.PowerLimitW > 0 {
|
||||
deficit := (g.PowerDefaultLimitW - g.PowerLimitW) / g.PowerDefaultLimitW
|
||||
if deficit > 0.05 {
|
||||
issues = append(issues, fmt.Sprintf("power limit %.0fW is %.0f%% below factory default %.0fW", g.PowerLimitW, deficit*100, g.PowerDefaultLimitW))
|
||||
}
|
||||
}
|
||||
return issues
|
||||
}
|
||||
|
||||
// evaluateNvidiaNVLinkPair compares a bonded pair's actual link state
|
||||
// against what the topology matrix says should be there.
|
||||
func evaluateNvidiaNVLinkPair(pair nvidiaNVLinkBondedPair, status map[int][]nvidiaNVLinkPort, errors map[int]map[int][3]int64) NvidiaNVLinkPairFinding {
|
||||
f := NvidiaNVLinkPairFinding{GPUA: pair.gpuA, GPUB: pair.gpuB, ExpectedLinks: pair.links}
|
||||
|
||||
active, total, errLinks := 0, 0, 0
|
||||
for _, gpu := range []int{pair.gpuA, pair.gpuB} {
|
||||
for _, port := range status[gpu] {
|
||||
total++
|
||||
if port.active {
|
||||
active++
|
||||
}
|
||||
}
|
||||
for _, counters := range errors[gpu] {
|
||||
if counters[0] != 0 || counters[1] != 0 || counters[2] != 0 {
|
||||
errLinks++
|
||||
}
|
||||
}
|
||||
}
|
||||
f.ActiveLinks, f.TotalLinks, f.ErrorLinks = active, total, errLinks
|
||||
|
||||
if total > 0 && active < total {
|
||||
f.Issues = append(f.Issues, fmt.Sprintf("%d/%d NVLinks inactive on a bonded pair", total-active, total))
|
||||
}
|
||||
if errLinks > 0 {
|
||||
f.Issues = append(f.Issues, fmt.Sprintf("%d link(s) reporting replay/recovery/CRC errors", errLinks))
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// NVLink parsing — deliberately self-contained rather than importing
|
||||
// internal/collector or internal/webui's equivalent parsers (isNVLinkBridgeCandidate
|
||||
// et al. and parseGPUPairAdjacency/parseTopoNVLinkStatus/parseTopoNVLinkErrors,
|
||||
// respectively), matching this codebase's existing precedent of small,
|
||||
// package-local duplication over cross-package coupling for narrow parsing
|
||||
// helpers (see webui/page_topo.go's isNICDeviceClassDev/isRAIDControllerClass).
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type nvidiaNVLinkBondedPair struct {
|
||||
gpuA, gpuB int
|
||||
links int
|
||||
}
|
||||
|
||||
var (
|
||||
nvidiaNVLinkANSIRe = regexp.MustCompile("\x1b\\[[0-9;]*[A-Za-z]")
|
||||
nvidiaNVLinkNVRe = regexp.MustCompile(`(?i)^NV(\d+)$`)
|
||||
)
|
||||
|
||||
// parseNvidiaNVLinkBondedPairs returns every GPU pair with a nonzero NVLink
|
||||
// bond count from a "nvidia-smi topo -m" matrix, deduplicated (A,B) == (B,A).
|
||||
func parseNvidiaNVLinkBondedPairs(raw string) []nvidiaNVLinkBondedPair {
|
||||
lines := strings.Split(nvidiaNVLinkANSIRe.ReplaceAllString(raw, ""), "\n")
|
||||
headerIdx := -1
|
||||
var gpuColIndices []int
|
||||
for i, line := range lines {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if strings.HasPrefix(trimmed, "GPU0") {
|
||||
parts := strings.Fields(trimmed)
|
||||
for j, col := range parts {
|
||||
if strings.HasPrefix(col, "GPU") {
|
||||
gpuColIndices = append(gpuColIndices, j)
|
||||
}
|
||||
}
|
||||
if len(gpuColIndices) >= 2 {
|
||||
headerIdx = i
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
if headerIdx < 0 {
|
||||
return nil
|
||||
}
|
||||
colIdxToGPU := make(map[int]int, len(gpuColIndices))
|
||||
for gpuIdx, colIdx := range gpuColIndices {
|
||||
colIdxToGPU[colIdx] = gpuIdx
|
||||
}
|
||||
|
||||
seen := map[[2]int]bool{}
|
||||
var pairs []nvidiaNVLinkBondedPair
|
||||
for _, line := range lines[headerIdx+1:] {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if !strings.HasPrefix(trimmed, "GPU") {
|
||||
continue
|
||||
}
|
||||
cells := strings.Fields(trimmed)
|
||||
if len(cells) == 0 {
|
||||
continue
|
||||
}
|
||||
rowGPU, err := strconv.Atoi(strings.TrimPrefix(cells[0], "GPU"))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for colIdx, colGPU := range colIdxToGPU {
|
||||
if colGPU == rowGPU {
|
||||
continue
|
||||
}
|
||||
dataIdx := colIdx + 1
|
||||
if dataIdx >= len(cells) {
|
||||
continue
|
||||
}
|
||||
m := nvidiaNVLinkNVRe.FindStringSubmatch(cells[dataIdx])
|
||||
if len(m) != 2 {
|
||||
continue
|
||||
}
|
||||
nv, err := strconv.Atoi(m[1])
|
||||
if err != nil || nv <= 0 {
|
||||
continue
|
||||
}
|
||||
a, b := rowGPU, colGPU
|
||||
if a > b {
|
||||
a, b = b, a
|
||||
}
|
||||
key := [2]int{a, b}
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
pairs = append(pairs, nvidiaNVLinkBondedPair{gpuA: a, gpuB: b, links: nv})
|
||||
}
|
||||
}
|
||||
return pairs
|
||||
}
|
||||
|
||||
type nvidiaNVLinkPort struct {
|
||||
active bool
|
||||
}
|
||||
|
||||
var (
|
||||
nvidiaNVLinkGPUHeaderRe = regexp.MustCompile(`^GPU (\d+):`)
|
||||
nvidiaNVLinkSpeedLineRe = regexp.MustCompile(`^Link (\d+):\s*[\d.]+\s*GB/s`)
|
||||
nvidiaNVLinkInactiveRe = regexp.MustCompile(`^Link (\d+):\s*<inactive>`)
|
||||
nvidiaNVLinkErrorLineRe = regexp.MustCompile(`^Link (\d+):\s*(Replay|Recovery|CRC) Errors:\s*(\d+)`)
|
||||
)
|
||||
|
||||
// parseNvidiaNVLinkStatus parses "nvidia-smi nvlink -s" output into per-GPU
|
||||
// per-link active/inactive state.
|
||||
func parseNvidiaNVLinkStatus(raw string) map[int][]nvidiaNVLinkPort {
|
||||
result := map[int][]nvidiaNVLinkPort{}
|
||||
currentGPU := -1
|
||||
for _, line := range strings.Split(raw, "\n") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if m := nvidiaNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
|
||||
currentGPU, _ = strconv.Atoi(m[1])
|
||||
continue
|
||||
}
|
||||
if currentGPU < 0 {
|
||||
continue
|
||||
}
|
||||
if nvidiaNVLinkInactiveRe.MatchString(trimmed) {
|
||||
result[currentGPU] = append(result[currentGPU], nvidiaNVLinkPort{active: false})
|
||||
continue
|
||||
}
|
||||
if nvidiaNVLinkSpeedLineRe.MatchString(trimmed) {
|
||||
result[currentGPU] = append(result[currentGPU], nvidiaNVLinkPort{active: true})
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// parseNvidiaNVLinkErrors parses "nvidia-smi nvlink -e" output into, per GPU
|
||||
// then link index, [replay, recovery, crc] error counts.
|
||||
func parseNvidiaNVLinkErrors(raw string) map[int]map[int][3]int64 {
|
||||
result := map[int]map[int][3]int64{}
|
||||
currentGPU := -1
|
||||
for _, line := range strings.Split(raw, "\n") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if m := nvidiaNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
|
||||
currentGPU, _ = strconv.Atoi(m[1])
|
||||
continue
|
||||
}
|
||||
if currentGPU < 0 {
|
||||
continue
|
||||
}
|
||||
m := nvidiaNVLinkErrorLineRe.FindStringSubmatch(trimmed)
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
linkIdx, _ := strconv.Atoi(m[1])
|
||||
count, _ := strconv.ParseInt(m[3], 10, 64)
|
||||
if result[currentGPU] == nil {
|
||||
result[currentGPU] = map[int][3]int64{}
|
||||
}
|
||||
c := result[currentGPU][linkIdx]
|
||||
switch m[2] {
|
||||
case "Replay":
|
||||
c[0] = count
|
||||
case "Recovery":
|
||||
c[1] = count
|
||||
case "CRC":
|
||||
c[2] = count
|
||||
}
|
||||
result[currentGPU][linkIdx] = c
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Confidential Computing (folded in — see RunNvidiaConfigCheckPack doc)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// parseConfComputeFields parses the indented "Key : Value" block emitted by
|
||||
// `nvidia-smi conf-compute -q`, e.g.:
|
||||
//
|
||||
// CC State : OFF
|
||||
// Multi-GPU Mode : Protected PCIe
|
||||
// CPU CC Capabilities : INTEL TDX
|
||||
// GPU CC Capabilities : CC Capable
|
||||
// CC GPUs Ready State : Not Ready
|
||||
func parseConfComputeFields(out []byte) map[string]string {
|
||||
fields := map[string]string{}
|
||||
for _, line := range strings.Split(string(out), "\n") {
|
||||
idx := strings.Index(line, ":")
|
||||
if idx < 0 {
|
||||
continue
|
||||
}
|
||||
key := strings.TrimSpace(line[:idx])
|
||||
val := strings.TrimSpace(line[idx+1:])
|
||||
if key == "" || val == "" {
|
||||
continue
|
||||
}
|
||||
fields[key] = val
|
||||
}
|
||||
return fields
|
||||
}
|
||||
|
||||
// filterConfComputeDmesgLines returns the dmesg lines relevant to CPU
|
||||
// Confidential Computing support (AMD SEV/SEV-ES/SEV-SNP, Intel TDX).
|
||||
func filterConfComputeDmesgLines(dmesgOut []byte) []string {
|
||||
var lines []string
|
||||
for _, raw := range bytes.Split(dmesgOut, []byte("\n")) {
|
||||
lower := strings.ToLower(string(raw))
|
||||
if strings.Contains(lower, "sev") || strings.Contains(lower, "tdx") {
|
||||
lines = append(lines, string(raw))
|
||||
}
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Report rendering
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func renderNvidiaConfigCheckSummary(status NvidiaConfigCheckStatus) string {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "run_at_utc=%s\n", status.CollectedAt.Format(time.RFC3339))
|
||||
fmt.Fprintf(&b, "gpu_count=%d\n", len(status.GPUs))
|
||||
eccIssues, migIssues, powerIssues := 0, 0, 0
|
||||
for _, g := range status.GPUs {
|
||||
for _, issue := range g.Issues {
|
||||
switch {
|
||||
case strings.Contains(issue, "ECC"):
|
||||
eccIssues++
|
||||
case strings.Contains(issue, "MIG"):
|
||||
migIssues++
|
||||
case strings.Contains(issue, "power limit"):
|
||||
powerIssues++
|
||||
}
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(&b, "gpu_ecc_issues=%d\n", eccIssues)
|
||||
fmt.Fprintf(&b, "gpu_mig_pending_issues=%d\n", migIssues)
|
||||
fmt.Fprintf(&b, "gpu_power_limit_issues=%d\n", powerIssues)
|
||||
fmt.Fprintf(&b, "nvlink_pairs_checked=%d\n", len(status.NVLinkPairs))
|
||||
nvlinkIssues := 0
|
||||
for _, p := range status.NVLinkPairs {
|
||||
if len(p.Issues) > 0 {
|
||||
nvlinkIssues++
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(&b, "nvlink_pairs_with_issues=%d\n", nvlinkIssues)
|
||||
fmt.Fprintf(&b, "cc_ready=%t\n", status.CCReady)
|
||||
fmt.Fprintf(&b, "cc_state=%s\n", status.CCState)
|
||||
fmt.Fprintf(&b, "cpu_cc_capability=%s\n", status.CPUCCCapability)
|
||||
fmt.Fprintf(&b, "gpu_cc_capability=%s\n", status.GPUCCCapability)
|
||||
if len(status.Warnings) == 0 {
|
||||
fmt.Fprintln(&b, "overall_status=OK")
|
||||
} else {
|
||||
fmt.Fprintln(&b, "overall_status=FAILED")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func renderNvidiaConfigCheckReport(status NvidiaConfigCheckStatus) string {
|
||||
var b strings.Builder
|
||||
line := strings.Repeat("=", 80)
|
||||
b.WriteString(line + "\n")
|
||||
b.WriteString("NVIDIA GPU Configuration & NVLink Topology Check\n")
|
||||
b.WriteString(line + "\n\n")
|
||||
|
||||
b.WriteString("-- GPU Configuration ----------------------------------------------------------\n")
|
||||
for _, g := range status.GPUs {
|
||||
verdict := "OK"
|
||||
if len(g.Issues) > 0 {
|
||||
verdict = "ISSUES FOUND"
|
||||
}
|
||||
fmt.Fprintf(&b, " GPU %d (%s): %s\n", g.Index, g.Name, verdict)
|
||||
fmt.Fprintf(&b, " ECC: %s | MIG: %s (pending %s) | Power: %.0fW / %.0fW default\n",
|
||||
nonEmptyOr(g.ECCCurrent, "unknown"), nonEmptyOr(g.MIGCurrent, "unknown"), nonEmptyOr(g.MIGPending, "unknown"),
|
||||
g.PowerLimitW, g.PowerDefaultLimitW)
|
||||
for _, issue := range g.Issues {
|
||||
fmt.Fprintf(&b, " - %s\n", issue)
|
||||
}
|
||||
}
|
||||
b.WriteString("\n-- NVLink Topology --------------------------------------------------------------\n")
|
||||
if len(status.NVLinkPairs) == 0 {
|
||||
b.WriteString(" No NVLink-bonded GPU pairs found.\n")
|
||||
}
|
||||
for _, p := range status.NVLinkPairs {
|
||||
verdict := "OK"
|
||||
if len(p.Issues) > 0 {
|
||||
verdict = "ISSUES FOUND"
|
||||
}
|
||||
fmt.Fprintf(&b, " GPU%d <-> GPU%d: %d/%d links active (topology expects %d): %s\n",
|
||||
p.GPUA, p.GPUB, p.ActiveLinks, p.TotalLinks, p.ExpectedLinks, verdict)
|
||||
for _, issue := range p.Issues {
|
||||
fmt.Fprintf(&b, " - %s\n", issue)
|
||||
}
|
||||
}
|
||||
|
||||
b.WriteString("\n-- Confidential Computing (informational) ---------------------------------------\n")
|
||||
verdict := "NOT READY"
|
||||
if status.CCReady {
|
||||
verdict = "READY"
|
||||
}
|
||||
fmt.Fprintf(&b, " Verdict: %s\n", verdict)
|
||||
fmt.Fprintf(&b, " CC State: %s | CPU CC: %s | GPU CC: %s\n",
|
||||
nonEmptyOr(status.CCState, "unknown"), nonEmptyOr(status.CPUCCCapability, "unknown"), nonEmptyOr(status.GPUCCCapability, "unknown"))
|
||||
|
||||
if len(status.Notes) > 0 {
|
||||
b.WriteString("\n-- Notes -------------------------------------------------------------------------\n")
|
||||
for _, n := range status.Notes {
|
||||
fmt.Fprintf(&b, " - %s\n", n)
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Fprintf(&b, "\nCollected : %s\n", status.CollectedAt.Format("2006-01-02 15:04:05 UTC"))
|
||||
b.WriteString(line + "\n")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func nonEmptyOr(v, fallback string) string {
|
||||
if strings.TrimSpace(v) == "" {
|
||||
return fallback
|
||||
}
|
||||
return v
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseNvidiaNVLinkBondedPairsRealTwoGPUDump(t *testing.T) {
|
||||
// Real system/nvidia-smi-topo.txt shape: two H100s directly bridged
|
||||
// (NV17).
|
||||
input := "\tGPU0\tGPU1\tNIC0\tNIC1\tCPU Affinity\tNUMA Affinity\tGPU NUMA ID\n" +
|
||||
"GPU0\t X \tNV17\tSYS\tSYS\t0-23,48-71\t0\t\tN/A\n" +
|
||||
"GPU1\tNV17\t X \tNODE\tNODE\t24-47,72-95\t1\t\tN/A\n" +
|
||||
"NIC0\tSYS\tNODE\t X \tPIX\t\t\t\n" +
|
||||
"NIC1\tSYS\tNODE\tPIX\t X \t\t\t\n"
|
||||
|
||||
pairs := parseNvidiaNVLinkBondedPairs(input)
|
||||
if len(pairs) != 1 {
|
||||
t.Fatalf("pairs=%d want 1 (%#v)", len(pairs), pairs)
|
||||
}
|
||||
if pairs[0].gpuA != 0 || pairs[0].gpuB != 1 || pairs[0].links != 17 {
|
||||
t.Fatalf("pair=%#v want {0,1,17}", pairs[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseNvidiaNVLinkBondedPairsANSIUnderlinedHeader(t *testing.T) {
|
||||
// nvidia-smi underlines the topo -m header with ANSI CSI codes even when
|
||||
// writing to a file — the same real-world quirk fixed for the /topo page
|
||||
// parser must be handled here too, since this uses a fresh live query
|
||||
// rather than reading the persisted techdump.
|
||||
input := "\x1b[4m\tGPU0\tGPU1\tGPU2\tGPU3\tCPU Affinity\x1b[0m\n" +
|
||||
"GPU0\t X \tNV18\tPIX\tPIX\t0-31,64-95\n" +
|
||||
"GPU1\tNV18\t X \tPIX\tPIX\t0-31,64-95\n" +
|
||||
"GPU2\tPIX\tPIX\t X \tNV18\t0-31,64-95\n" +
|
||||
"GPU3\tPIX\tPIX\tNV18\t X \t0-31,64-95\n"
|
||||
|
||||
pairs := parseNvidiaNVLinkBondedPairs(input)
|
||||
if len(pairs) != 2 {
|
||||
t.Fatalf("pairs=%d want 2 (%#v)", len(pairs), pairs)
|
||||
}
|
||||
want := map[[2]int]int{{0, 1}: 18, {2, 3}: 18}
|
||||
for _, p := range pairs {
|
||||
if want[[2]int{p.gpuA, p.gpuB}] != p.links {
|
||||
t.Fatalf("unexpected pair %#v", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseNvidiaNVLinkStatusMarksInactiveLinks(t *testing.T) {
|
||||
input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140)
|
||||
Link 0: 26.562 GB/s
|
||||
Link 1: <inactive>
|
||||
GPU 1: NVIDIA H100 80GB HBM3 (UUID: GPU-603fe750-0516-9db5-86ec-ea61af3fce35)
|
||||
Link 0: 26.562 GB/s
|
||||
`
|
||||
got := parseNvidiaNVLinkStatus(input)
|
||||
if len(got[0]) != 2 || got[0][0].active != true || got[0][1].active != false {
|
||||
t.Fatalf("gpu0=%#v want [active, inactive]", got[0])
|
||||
}
|
||||
if len(got[1]) != 1 || !got[1][0].active {
|
||||
t.Fatalf("gpu1=%#v want [active]", got[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseNvidiaNVLinkErrors(t *testing.T) {
|
||||
input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140)
|
||||
Link 0: Replay Errors: 0
|
||||
Link 0: Recovery Errors: 0
|
||||
Link 0: CRC Errors: 0
|
||||
Link 1: Replay Errors: 3
|
||||
Link 1: Recovery Errors: 1
|
||||
Link 1: CRC Errors: 2
|
||||
`
|
||||
got := parseNvidiaNVLinkErrors(input)
|
||||
c := got[0][1]
|
||||
if c[0] != 3 || c[1] != 1 || c[2] != 2 {
|
||||
t.Fatalf("link1 counters=%#v want {3,1,2}", c)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluateNvidiaNVLinkPairFlagsInactiveLinksAndErrors(t *testing.T) {
|
||||
pair := nvidiaNVLinkBondedPair{gpuA: 0, gpuB: 1, links: 18}
|
||||
status := map[int][]nvidiaNVLinkPort{
|
||||
0: {{active: true}, {active: false}},
|
||||
1: {{active: true}, {active: true}},
|
||||
}
|
||||
errors := map[int]map[int][3]int64{
|
||||
0: {1: {3, 0, 0}},
|
||||
}
|
||||
|
||||
f := evaluateNvidiaNVLinkPair(pair, status, errors)
|
||||
if f.ActiveLinks != 3 || f.TotalLinks != 4 {
|
||||
t.Fatalf("active=%d total=%d want 3/4", f.ActiveLinks, f.TotalLinks)
|
||||
}
|
||||
if len(f.Issues) != 2 {
|
||||
t.Fatalf("issues=%#v want 2 (inactive link + error)", f.Issues)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluateNvidiaNVLinkPairHealthyBondHasNoIssues(t *testing.T) {
|
||||
pair := nvidiaNVLinkBondedPair{gpuA: 0, gpuB: 1, links: 18}
|
||||
status := map[int][]nvidiaNVLinkPort{
|
||||
0: {{active: true}},
|
||||
1: {{active: true}},
|
||||
}
|
||||
f := evaluateNvidiaNVLinkPair(pair, status, nil)
|
||||
if len(f.Issues) != 0 {
|
||||
t.Fatalf("issues=%#v want none for a fully active, error-free bond", f.Issues)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluateNvidiaGPUConfigFlagsECCDisabled(t *testing.T) {
|
||||
g := NvidiaGPUSetting{Index: 0, Name: "H100", ECCCurrent: "Disabled"}
|
||||
issues := evaluateNvidiaGPUConfig(g)
|
||||
if len(issues) != 1 {
|
||||
t.Fatalf("issues=%#v want 1 (ECC disabled)", issues)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluateNvidiaGPUConfigFlagsStuckMIGPending(t *testing.T) {
|
||||
g := NvidiaGPUSetting{Index: 0, Name: "H100", ECCCurrent: "Enabled", MIGCurrent: "Disabled", MIGPending: "Enabled"}
|
||||
issues := evaluateNvidiaGPUConfig(g)
|
||||
if len(issues) != 1 {
|
||||
t.Fatalf("issues=%#v want 1 (MIG pending != current)", issues)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluateNvidiaGPUConfigFlagsPowerLimitBelowDefault(t *testing.T) {
|
||||
g := NvidiaGPUSetting{Index: 0, Name: "H100", ECCCurrent: "Enabled", PowerLimitW: 300, PowerDefaultLimitW: 700}
|
||||
issues := evaluateNvidiaGPUConfig(g)
|
||||
if len(issues) != 1 {
|
||||
t.Fatalf("issues=%#v want 1 (power limit far below default)", issues)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluateNvidiaGPUConfigNominalHasNoIssues(t *testing.T) {
|
||||
g := NvidiaGPUSetting{
|
||||
Index: 0, Name: "H100", ECCCurrent: "Enabled",
|
||||
MIGCurrent: "Disabled", MIGPending: "Disabled",
|
||||
PowerLimitW: 700, PowerDefaultLimitW: 700,
|
||||
}
|
||||
issues := evaluateNvidiaGPUConfig(g)
|
||||
if len(issues) != 0 {
|
||||
t.Fatalf("issues=%#v want none for nominal config", issues)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluateNvidiaGPUConfigSmallPowerLimitDeviationIsNotFlagged(t *testing.T) {
|
||||
// Power limits set slightly below default (e.g. rounding, minor
|
||||
// operator tuning) should not be treated as an anomaly — only a
|
||||
// meaningful cap (>5%) is.
|
||||
g := NvidiaGPUSetting{Index: 0, Name: "H100", ECCCurrent: "Enabled", PowerLimitW: 690, PowerDefaultLimitW: 700}
|
||||
issues := evaluateNvidiaGPUConfig(g)
|
||||
if len(issues) != 0 {
|
||||
t.Fatalf("issues=%#v want none for a <2%% power-limit deviation", issues)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderNvidiaConfigCheckSummaryOverallStatus(t *testing.T) {
|
||||
clean := NvidiaConfigCheckStatus{}
|
||||
if got := renderNvidiaConfigCheckSummary(clean); !strings.Contains(got, "overall_status=OK") {
|
||||
t.Fatalf("clean status summary missing overall_status=OK:\n%s", got)
|
||||
}
|
||||
withWarning := NvidiaConfigCheckStatus{Warnings: []string{"GPU 0: ECC is disabled"}}
|
||||
if got := renderNvidiaConfigCheckSummary(withWarning); !strings.Contains(got, "overall_status=FAILED") {
|
||||
t.Fatalf("status with warnings missing overall_status=FAILED:\n%s", got)
|
||||
}
|
||||
}
|
||||
@@ -70,7 +70,7 @@ func (s *System) ListNvidiaGPUSettings() ([]NvidiaGPUSetting, error) {
|
||||
|
||||
// queryNvidiaGPUCCState reads the per-GPU Confidential Computing state via
|
||||
// `nvidia-smi conf-compute -i <index> -q`, reusing the same field-name
|
||||
// convention as RunConfidentialComputingCheckPack's parseConfComputeFields.
|
||||
// convention as RunNvidiaConfigCheckPack's parseConfComputeFields.
|
||||
// Returns "" if the driver/GPU doesn't support conf-compute at all.
|
||||
func (s *System) queryNvidiaGPUCCState(index int) string {
|
||||
out, err := satExecCommand("nvidia-smi", "conf-compute", "-i", strconv.Itoa(index), "-q").Output()
|
||||
|
||||
Reference in New Issue
Block a user