From cfaa15ec7c14a5c48feb11eb507775f0457c6631 Mon Sep 17 00:00:00 2001 From: Mikhail Chusavitin Date: Thu, 9 Jul 2026 11:36:52 +0300 Subject: [PATCH] platform/webui: fold confidential-computing into a GPU config + NVLink check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The standalone "confidential-computing" SAT target only ever checked CC readiness, which most fleets never opt into (a NOT_READY verdict there isn't a fault). Meanwhile DCGM diag never asserts GPU config compliance (ECC/MIG/power-limit vs factory default) or NVLink topology (per NVIDIA's own DGX BasePOD deployment guide, this needs a separate validation step) — gaps confirmed against public DCGM docs and a real NV17-vs-expected-NV18 bonded pair found on a live bundle. Repurposes the routine into "nvidia-config": reuses the existing ListNvidiaGPUSettings() (already backing the GPU-settings page) to flag ECC disabled, a MIG mode change stuck pending a reset/reboot, and a power limit capped >5% below default; parses "nvidia-smi topo -m" bonded pairs against "nvlink -s/-e" to flag any inactive lane or nonzero replay/ recovery/CRC counter on an otherwise-active bond. CC readiness is folded in as one informational field (does not gate overall_status) rather than a dedicated test. Reports under the same pcie:gpu:nvidia severity key as every other nvidia-* SAT target instead of an isolated key, so a config/NVLink FAILED result isn't invisible next to stress-test results. Also fixes ApplySATResultToDB silently dropping any target with no matching switch case (exactly what the old confidential-computing target did) with a new coverage test enumerating every real SAT target. Co-Authored-By: Claude Sonnet 5 --- audit/internal/app/app.go | 2 +- audit/internal/app/app_packs.go | 14 +- audit/internal/app/app_test.go | 2 +- audit/internal/app/component_status_db.go | 2 +- .../internal/app/component_status_db_test.go | 78 +++ .../internal/platform/confidential_compute.go | 248 -------- .../internal/platform/nvidia_config_check.go | 540 ++++++++++++++++++ .../platform/nvidia_config_check_test.go | 168 ++++++ audit/internal/platform/nvidia_settings.go | 2 +- audit/internal/webui/api.go | 2 +- audit/internal/webui/page_validate.go | 10 +- audit/internal/webui/server.go | 2 +- audit/internal/webui/task_runner.go | 4 +- audit/internal/webui/tasks.go | 10 +- 14 files changed, 811 insertions(+), 273 deletions(-) delete mode 100644 audit/internal/platform/confidential_compute.go create mode 100644 audit/internal/platform/nvidia_config_check.go create mode 100644 audit/internal/platform/nvidia_config_check_test.go diff --git a/audit/internal/app/app.go b/audit/internal/app/app.go index a63b105..387dbca 100644 --- a/audit/internal/app/app.go +++ b/audit/internal/app/app.go @@ -134,7 +134,7 @@ type satRunner interface { ResetNvidiaGPU(index int) (string, error) RunMemoryAcceptancePack(ctx context.Context, baseDir string, sizeMB, passes int, logFunc func(string)) (string, error) RunStorageAcceptancePack(ctx context.Context, baseDir string, extended bool, logFunc func(string)) (string, error) - RunConfidentialComputingCheckPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) + RunNvidiaConfigCheckPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) RunCPUAcceptancePack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) ListNvidiaGPUs() ([]platform.NvidiaGPU, error) ListNvidiaGPUSettings() ([]platform.NvidiaGPUSetting, error) diff --git a/audit/internal/app/app_packs.go b/audit/internal/app/app_packs.go index 13c4410..0886df5 100644 --- a/audit/internal/app/app_packs.go +++ b/audit/internal/app/app_packs.go @@ -247,20 +247,20 @@ func (a *App) RunStorageAcceptancePackResult(baseDir string) (ActionResult, erro return ActionResult{Title: "Storage SAT", Body: satResultBody(path)}, err } -func (a *App) RunConfidentialComputingCheckPackCtx(ctx context.Context, baseDir string, logFunc func(string)) (string, error) { +func (a *App) RunNvidiaConfigCheckPackCtx(ctx context.Context, baseDir string, logFunc func(string)) (string, error) { if strings.TrimSpace(baseDir) == "" { baseDir = DefaultSATBaseDir } - return a.sat.RunConfidentialComputingCheckPack(ctx, baseDir, logFunc) + return a.sat.RunNvidiaConfigCheckPack(ctx, baseDir, logFunc) } -func (a *App) RunConfidentialComputingCheckPack(baseDir string, logFunc func(string)) (string, error) { - return a.RunConfidentialComputingCheckPackCtx(context.Background(), baseDir, logFunc) +func (a *App) RunNvidiaConfigCheckPack(baseDir string, logFunc func(string)) (string, error) { + return a.RunNvidiaConfigCheckPackCtx(context.Background(), baseDir, logFunc) } -func (a *App) RunConfidentialComputingCheckPackResult(baseDir string) (ActionResult, error) { - path, err := a.RunConfidentialComputingCheckPack(baseDir, nil) - return ActionResult{Title: "Confidential Computing Check", Body: satResultBody(path)}, err +func (a *App) RunNvidiaConfigCheckPackResult(baseDir string) (ActionResult, error) { + path, err := a.RunNvidiaConfigCheckPack(baseDir, nil) + return ActionResult{Title: "GPU Config & NVLink Check", Body: satResultBody(path)}, err } func (a *App) DetectGPUVendor() string { diff --git a/audit/internal/app/app_test.go b/audit/internal/app/app_test.go index 7dabdba..d2cb1d2 100644 --- a/audit/internal/app/app_test.go +++ b/audit/internal/app/app_test.go @@ -267,7 +267,7 @@ func (f fakeSAT) RunStorageAcceptancePack(_ context.Context, baseDir string, _ b return f.runStorageFn(baseDir) } -func (f fakeSAT) RunConfidentialComputingCheckPack(_ context.Context, baseDir string, _ func(string)) (string, error) { +func (f fakeSAT) RunNvidiaConfigCheckPack(_ context.Context, baseDir string, _ func(string)) (string, error) { return "", nil } diff --git a/audit/internal/app/component_status_db.go b/audit/internal/app/component_status_db.go index 4b68b9a..5450aff 100644 --- a/audit/internal/app/component_status_db.go +++ b/audit/internal/app/component_status_db.go @@ -243,7 +243,7 @@ func ApplySATResultToDB(db *ComponentStatusDB, target, archivePath string) { // otherwise fails to match any real BDF. switch target { case "nvidia", "nvidia-targeted-stress", "nvidia-compute", "nvidia-targeted-power", "nvidia-pulse", - "nvidia-interconnect", "nvidia-bandwidth", "nvidia-stress": + "nvidia-interconnect", "nvidia-bandwidth", "nvidia-stress", "nvidia-config": db.Record("pcie:gpu:nvidia", source, dbStatus, target+" SAT: "+overall) case "amd", "amd-stress", "amd-mem", "amd-bandwidth": db.Record("pcie:gpu:amd", source, dbStatus, target+" SAT: "+overall) diff --git a/audit/internal/app/component_status_db_test.go b/audit/internal/app/component_status_db_test.go index e2a263c..81bd9ff 100644 --- a/audit/internal/app/component_status_db_test.go +++ b/audit/internal/app/component_status_db_test.go @@ -98,6 +98,84 @@ func TestApplyComponentStatusDBMatchesGPUByVendor(t *testing.T) { func strPtr(s string) *string { return &s } +// TestApplySATResultToDBCoversAllHealthCheckTargets guards against a target +// silently falling through ApplySATResultToDB's switch with no matching +// case — exactly what happened to the old "confidential-computing" target +// before it was folded into "nvidia-config" (task ran, produced a valid +// summary.txt, but no component-status.json record was ever written for it, +// so a /topo card or any other consumer had no way to know the check had +// even run). Every target below is a real health/acceptance check target +// dispatched by task_runner.go's executeTaskWithOptions that produces a +// summary.txt with overall_status; each must land somewhere in the DB. +func TestApplySATResultToDBCoversAllHealthCheckTargets(t *testing.T) { + cases := []struct { + target string + wantKey string + }{ + {"nvidia", "pcie:gpu:nvidia"}, + {"nvidia-stress", "pcie:gpu:nvidia"}, + {"nvidia-targeted-stress", "pcie:gpu:nvidia"}, + {"nvidia-compute", "pcie:gpu:nvidia"}, + {"nvidia-targeted-power", "pcie:gpu:nvidia"}, + {"nvidia-pulse", "pcie:gpu:nvidia"}, + {"nvidia-interconnect", "pcie:gpu:nvidia"}, + {"nvidia-bandwidth", "pcie:gpu:nvidia"}, + {"nvidia-config", "pcie:gpu:nvidia"}, + {"amd", "pcie:gpu:amd"}, + {"amd-stress", "pcie:gpu:amd"}, + {"amd-mem", "pcie:gpu:amd"}, + {"amd-bandwidth", "pcie:gpu:amd"}, + {"memory", "memory:all"}, + {"memory-stress", "memory:all"}, + {"sat-stress", "memory:all"}, + {"cpu", "cpu:all"}, + {"platform-stress", "cpu:all"}, + {"storage", "storage:all"}, + } + for _, tc := range cases { + t.Run(tc.target, func(t *testing.T) { + db, err := OpenComponentStatusDB(filepath.Join(t.TempDir(), "component-status.json")) + if err != nil { + t.Fatal(err) + } + ApplySATResultToDB(db, tc.target, writeSATSummary(t, "OK")) + rec, ok := db.Get(tc.wantKey) + if !ok { + t.Fatalf("target %q wrote no record under key %q — falls through the switch silently", tc.target, tc.wantKey) + } + if rec.Status != "OK" { + t.Fatalf("target %q key %q status=%q want OK", tc.target, tc.wantKey, rec.Status) + } + }) + } +} + +// TestApplySATResultToDBNvidiaConfigSharesGPUKeyWithOtherNvidiaTargets +// confirms "nvidia-config" (GPU config/NVLink/CC check — confidential +// computing readiness folded in here rather than run as its own standalone +// SAT target) reports under the same pcie:gpu:nvidia key as every other +// nvidia-* target, so a config/NVLink FAILED result isn't invisible next to +// the plain stress-test results, and a clean run afterward doesn't silently +// erase an earlier failure (severity merge, same as +// TestApplySATResultToDBNormalizesGPUKeyByVendor above). +func TestApplySATResultToDBNvidiaConfigSharesGPUKeyWithOtherNvidiaTargets(t *testing.T) { + db, err := OpenComponentStatusDB(filepath.Join(t.TempDir(), "component-status.json")) + if err != nil { + t.Fatal(err) + } + ApplySATResultToDB(db, "nvidia-config", writeSATSummary(t, "FAILED")) + ApplySATResultToDB(db, "nvidia", writeSATSummary(t, "OK")) + + rec, ok := db.Get("pcie:gpu:nvidia") + if !ok { + t.Fatalf("expected pcie:gpu:nvidia record to exist") + } + if rec.Status != "Warning" { + t.Fatalf("status=%q, want Warning (nvidia-config FAILED must survive the later OK nvidia run)", rec.Status) + } +} + + // TestRecordDeduplicatesRepeatedIdenticalStatusFromSameSource guards the // hardware-ingest-contract.md rule that status_history is a transition log // ("История переходов статусов"), not a per-poll journal. A component diff --git a/audit/internal/platform/confidential_compute.go b/audit/internal/platform/confidential_compute.go deleted file mode 100644 index 95551b9..0000000 --- a/audit/internal/platform/confidential_compute.go +++ /dev/null @@ -1,248 +0,0 @@ -package platform - -import ( - "bytes" - "context" - "fmt" - "os" - "path/filepath" - "strings" - "time" -) - -// ConfidentialComputingStatus summarizes whether this server can run NVIDIA -// Confidential Computing: CPU-side TEE support (Intel TDX / AMD SEV-SNP) and -// GPU firmware CC capability, as reported by `nvidia-smi conf-compute -q`. -type ConfidentialComputingStatus struct { - CollectedAt time.Time `json:"collected_at"` - - // GPU-reported fields, parsed from `nvidia-smi conf-compute -q`. - NvidiaSMIAvailable bool `json:"nvidia_smi_available"` - CCState string `json:"cc_state,omitempty"` // ON / OFF - MultiGPUMode string `json:"multi_gpu_mode,omitempty"` // Protected PCIe / ... - CPUCCCapability string `json:"cpu_cc_capability,omitempty"` // e.g. "INTEL TDX", "AMD SEV-SNP", "NONE" - GPUCCCapability string `json:"gpu_cc_capability,omitempty"` // e.g. "CC Capable", "Not Capable" - CCGPUsReadyState string `json:"cc_gpus_ready_state,omitempty"` // Ready / Not Ready - - // Host-side evidence that the CPU's TEE is actually active in the running - // kernel (BIOS + kernel cmdline + firmware), independent of what the GPU - // driver reports. Used as a fallback when the NVIDIA driver isn't loaded. - HostAMDSEVSupported bool `json:"host_amd_sev_supported"` - HostAMDSEVESSupported bool `json:"host_amd_sev_es_supported"` - HostAMDSEVSNPActive bool `json:"host_amd_sev_snp_active"` - HostIntelTDXActive bool `json:"host_intel_tdx_active"` - - // GPUCanRunCC is true when the GPU firmware reports CC-capable. - GPUCanRunCC bool `json:"gpu_can_run_cc"` - // CPUCanRunCC is true when either the GPU driver or the host kernel - // reports an active/available CPU TEE (SEV-SNP or TDX). - CPUCanRunCC bool `json:"cpu_can_run_cc"` - // Ready is true when both the CPU and the GPU support Confidential - // Computing, regardless of whether CC mode is currently enabled. - Ready bool `json:"ready"` - - Notes []string `json:"notes,omitempty"` -} - -// RunConfidentialComputingCheckPack runs a read-only check of whether this -// server can run NVIDIA Confidential Computing: it queries the GPU driver -// (`nvidia-smi conf-compute -q`) and inspects host kernel/dmesg evidence of -// AMD SEV-SNP / Intel TDX support. It changes nothing on the system. -func (s *System) RunConfidentialComputingCheckPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) { - if ctx == nil { - ctx = context.Background() - } - if baseDir == "" { - baseDir = "/var/log/bee-sat" - } - ts := time.Now().UTC().Format("20060102-150405") - runDir := filepath.Join(baseDir, "confidential-computing-"+ts) - if err := os.MkdirAll(runDir, 0755); err != nil { - return "", err - } - verboseLog := filepath.Join(runDir, "verbose.log") - - status := ConfidentialComputingStatus{CollectedAt: time.Now().UTC()} - - // GPU firmware / driver state. - ccOut, ccErr := runSATCommandCtx(ctx, verboseLog, "nvidia-smi-conf-compute-q", []string{"nvidia-smi", "conf-compute", "-q"}, nil, logFunc) - _ = os.WriteFile(filepath.Join(runDir, "01-nvidia-smi-conf-compute-q.log"), ccOut, 0644) - if ccErr == nil { - status.NvidiaSMIAvailable = true - fields := parseConfComputeFields(ccOut) - status.CCState = fields["CC State"] - status.MultiGPUMode = fields["Multi-GPU Mode"] - status.CPUCCCapability = fields["CPU CC Capabilities"] - status.GPUCCCapability = fields["GPU CC Capabilities"] - status.CCGPUsReadyState = fields["CC GPUs Ready State"] - } else { - status.Notes = append(status.Notes, "nvidia-smi conf-compute -q unavailable (no NVIDIA driver, or GPU not present): "+firstLine(string(ccOut))) - } - - // Host kernel evidence, independent of the GPU driver. - dmesgOut, _ := runSATCommandCtx(ctx, verboseLog, "dmesg", []string{"dmesg"}, nil, nil) - ccDmesgLines := filterConfComputeDmesgLines(dmesgOut) - _ = os.WriteFile(filepath.Join(runDir, "02-dmesg-cc-relevant.log"), []byte(strings.Join(ccDmesgLines, "\n")+"\n"), 0644) - - lowerDmesg := strings.ToLower(strings.Join(ccDmesgLines, "\n")) - status.HostAMDSEVSNPActive = strings.Contains(lowerDmesg, "sev-snp enabled") - status.HostIntelTDXActive = strings.Contains(lowerDmesg, "tdx module") && strings.Contains(lowerDmesg, "module initialized") || - strings.Contains(lowerDmesg, "virt/tdx: module initialized") - - for i, path := range []string{ - "/sys/module/kvm_amd/parameters/sev", - "/sys/module/kvm_amd/parameters/sev_es", - "/sys/module/kvm_amd/parameters/sev_snp", - } { - name := fmt.Sprintf("sysfs-%s", filepath.Base(path)) - out, err := runSATCommandCtx(ctx, verboseLog, name, []string{"cat", path}, nil, nil) - _ = os.WriteFile(filepath.Join(runDir, fmt.Sprintf("03-%02d-%s.log", i+1, name)), out, 0644) - if err != nil { - continue - } - val := strings.TrimSpace(string(out)) - switch filepath.Base(path) { - case "sev": - status.HostAMDSEVSupported = strings.EqualFold(val, "Y") - case "sev_es": - status.HostAMDSEVESSupported = strings.EqualFold(val, "Y") - case "sev_snp": - if strings.EqualFold(val, "Y") { - status.HostAMDSEVSNPActive = true - } - } - } - - status.GPUCanRunCC = strings.EqualFold(strings.TrimSpace(status.GPUCCCapability), "CC Capable") - cpuCapReported := strings.TrimSpace(status.CPUCCCapability) - status.CPUCanRunCC = status.HostAMDSEVSNPActive || status.HostIntelTDXActive || - (cpuCapReported != "" && !strings.EqualFold(cpuCapReported, "NONE")) - status.Ready = status.CPUCanRunCC && status.GPUCanRunCC - - if !status.NvidiaSMIAvailable { - status.Notes = append(status.Notes, "GPU CC capability unknown — install the NVIDIA driver to query it with `nvidia-smi conf-compute -q`.") - } - - summary := renderConfidentialComputingSummary(status) - if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary), 0644); err != nil { - return "", err - } - report := renderConfidentialComputingReport(status) - if err := os.WriteFile(filepath.Join(runDir, "confidential-computing-report.txt"), []byte(report), 0644); err != nil { - return "", err - } - - return runDir, nil -} - -// parseConfComputeFields parses the indented "Key : Value" block emitted by -// `nvidia-smi conf-compute -q`, e.g.: -// -// CC State : OFF -// Multi-GPU Mode : Protected PCIe -// CPU CC Capabilities : INTEL TDX -// GPU CC Capabilities : CC Capable -// CC GPUs Ready State : Not Ready -func parseConfComputeFields(out []byte) map[string]string { - fields := map[string]string{} - for _, line := range strings.Split(string(out), "\n") { - idx := strings.Index(line, ":") - if idx < 0 { - continue - } - key := strings.TrimSpace(line[:idx]) - val := strings.TrimSpace(line[idx+1:]) - if key == "" || val == "" { - continue - } - fields[key] = val - } - return fields -} - -// filterConfComputeDmesgLines returns the dmesg lines relevant to CPU -// Confidential Computing support (AMD SEV/SEV-ES/SEV-SNP, Intel TDX). -func filterConfComputeDmesgLines(dmesgOut []byte) []string { - var lines []string - scanner := bytes.Split(dmesgOut, []byte("\n")) - for _, raw := range scanner { - lower := strings.ToLower(string(raw)) - if strings.Contains(lower, "sev") || strings.Contains(lower, "tdx") { - lines = append(lines, string(raw)) - } - } - return lines -} - -func renderConfidentialComputingSummary(status ConfidentialComputingStatus) string { - var b strings.Builder - fmt.Fprintf(&b, "run_at_utc=%s\n", status.CollectedAt.Format(time.RFC3339)) - fmt.Fprintf(&b, "nvidia_smi_available=%t\n", status.NvidiaSMIAvailable) - fmt.Fprintf(&b, "cc_state=%s\n", status.CCState) - fmt.Fprintf(&b, "multi_gpu_mode=%s\n", status.MultiGPUMode) - fmt.Fprintf(&b, "cpu_cc_capability=%s\n", status.CPUCCCapability) - fmt.Fprintf(&b, "gpu_cc_capability=%s\n", status.GPUCCCapability) - fmt.Fprintf(&b, "cc_gpus_ready_state=%s\n", status.CCGPUsReadyState) - fmt.Fprintf(&b, "host_amd_sev_supported=%t\n", status.HostAMDSEVSupported) - fmt.Fprintf(&b, "host_amd_sev_es_supported=%t\n", status.HostAMDSEVESSupported) - fmt.Fprintf(&b, "host_amd_sev_snp_active=%t\n", status.HostAMDSEVSNPActive) - fmt.Fprintf(&b, "host_intel_tdx_active=%t\n", status.HostIntelTDXActive) - fmt.Fprintf(&b, "cpu_can_run_cc=%t\n", status.CPUCanRunCC) - fmt.Fprintf(&b, "gpu_can_run_cc=%t\n", status.GPUCanRunCC) - fmt.Fprintf(&b, "ready=%t\n", status.Ready) - if status.Ready { - fmt.Fprintln(&b, "overall_status=OK") - } else { - fmt.Fprintln(&b, "overall_status=NOT_READY") - } - return b.String() -} - -func renderConfidentialComputingReport(status ConfidentialComputingStatus) string { - var b strings.Builder - line := strings.Repeat("=", 80) - b.WriteString(line + "\n") - b.WriteString("Confidential Computing Readiness\n") - b.WriteString(line + "\n\n") - - verdict := "NOT READY" - if status.Ready { - verdict = "READY" - } - fmt.Fprintf(&b, "Verdict: %s\n\n", verdict) - - b.WriteString("-- CPU ----------------------------------------------------------------------\n") - fmt.Fprintf(&b, " Reported by GPU driver : %s\n", nonEmptyOr(status.CPUCCCapability, "unknown")) - fmt.Fprintf(&b, " AMD SEV supported : %t\n", status.HostAMDSEVSupported) - fmt.Fprintf(&b, " AMD SEV-ES supported : %t\n", status.HostAMDSEVESSupported) - fmt.Fprintf(&b, " AMD SEV-SNP active : %t\n", status.HostAMDSEVSNPActive) - fmt.Fprintf(&b, " Intel TDX active : %t\n", status.HostIntelTDXActive) - fmt.Fprintf(&b, " Can run CC : %t\n\n", status.CPUCanRunCC) - - b.WriteString("-- GPU ----------------------------------------------------------------------\n") - fmt.Fprintf(&b, " nvidia-smi available : %t\n", status.NvidiaSMIAvailable) - fmt.Fprintf(&b, " GPU CC Capabilities : %s\n", nonEmptyOr(status.GPUCCCapability, "unknown")) - fmt.Fprintf(&b, " CC State (current) : %s\n", nonEmptyOr(status.CCState, "unknown")) - fmt.Fprintf(&b, " Multi-GPU Mode : %s\n", nonEmptyOr(status.MultiGPUMode, "unknown")) - fmt.Fprintf(&b, " CC GPUs Ready State : %s\n", nonEmptyOr(status.CCGPUsReadyState, "unknown")) - fmt.Fprintf(&b, " Can run CC : %t\n\n", status.GPUCanRunCC) - - if len(status.Notes) > 0 { - b.WriteString("-- Notes ----------------------------------------------------------------------\n") - for _, n := range status.Notes { - fmt.Fprintf(&b, " - %s\n", n) - } - b.WriteString("\n") - } - - fmt.Fprintf(&b, "Collected : %s\n", status.CollectedAt.Format("2006-01-02 15:04:05 UTC")) - b.WriteString(line + "\n") - return b.String() -} - -func nonEmptyOr(v, fallback string) string { - if strings.TrimSpace(v) == "" { - return fallback - } - return v -} diff --git a/audit/internal/platform/nvidia_config_check.go b/audit/internal/platform/nvidia_config_check.go new file mode 100644 index 0000000..8f14f36 --- /dev/null +++ b/audit/internal/platform/nvidia_config_check.go @@ -0,0 +1,540 @@ +package platform + +import ( + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" +) + +// RunNvidiaConfigCheckPack runs a read-only check of NVIDIA GPU +// configuration and NVLink topology that DCGM diag does not cover: DCGM's +// diagnostic levels stress-test compute/memory/power/bandwidth, but never +// assert that persistent config (ECC, MIG, power limit) matches the factory +// default, and NVIDIA's own DGX BasePOD deployment guide calls out +// topology/NVLink validation as a distinct manual step outside DCGM diag. +// Confidential Computing readiness — previously its own standalone SAT +// target — is folded in here as one more read-only fact about GPU state +// rather than a dedicated test, since none of these are stress tests; they +// are one-shot state/config reads that all answer "is this GPU configured +// the way we expect", not "does it survive load". +func (s *System) RunNvidiaConfigCheckPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) { + if ctx == nil { + ctx = context.Background() + } + if baseDir == "" { + baseDir = "/var/log/bee-sat" + } + ts := time.Now().UTC().Format("20060102-150405") + runDir := filepath.Join(baseDir, "nvidia-config-"+ts) + if err := os.MkdirAll(runDir, 0755); err != nil { + return "", err + } + verboseLog := filepath.Join(runDir, "verbose.log") + + status := NvidiaConfigCheckStatus{CollectedAt: time.Now().UTC()} + + // -- GPU config compliance (ECC/MIG/power limit) ----------------------- + if settings, err := s.ListNvidiaGPUSettings(); err != nil { + status.Notes = append(status.Notes, "nvidia-smi GPU settings unavailable (no driver, or no GPU present): "+err.Error()) + } else { + for _, g := range settings { + f := NvidiaGPUConfigFinding{ + Index: g.Index, Name: g.Name, + ECCCurrent: g.ECCCurrent, MIGCurrent: g.MIGCurrent, MIGPending: g.MIGPending, + PowerLimitW: g.PowerLimitW, PowerDefaultLimitW: g.PowerDefaultLimitW, + Issues: evaluateNvidiaGPUConfig(g), + } + status.GPUs = append(status.GPUs, f) + for _, issue := range f.Issues { + status.Warnings = append(status.Warnings, fmt.Sprintf("GPU %d (%s): %s", g.Index, g.Name, issue)) + } + if strings.EqualFold(g.CCState, "on") { + status.CCState = "ON" + } + } + } + + // -- NVLink topology (bonded pairs, active links, error counters) ------ + topoOut, _ := runSATCommandCtx(ctx, verboseLog, "nvidia-smi-topo-m", []string{"nvidia-smi", "topo", "-m"}, nil, logFunc) + _ = os.WriteFile(filepath.Join(runDir, "01-nvidia-smi-topo-m.log"), topoOut, 0644) + statusOut, _ := runSATCommandCtx(ctx, verboseLog, "nvidia-smi-nvlink-s", []string{"nvidia-smi", "nvlink", "-s"}, nil, logFunc) + _ = os.WriteFile(filepath.Join(runDir, "02-nvidia-smi-nvlink-s.log"), statusOut, 0644) + errOut, _ := runSATCommandCtx(ctx, verboseLog, "nvidia-smi-nvlink-e", []string{"nvidia-smi", "nvlink", "-e"}, nil, logFunc) + _ = os.WriteFile(filepath.Join(runDir, "03-nvidia-smi-nvlink-e.log"), errOut, 0644) + + pairs := parseNvidiaNVLinkBondedPairs(string(topoOut)) + linkStatus := parseNvidiaNVLinkStatus(string(statusOut)) + linkErrors := parseNvidiaNVLinkErrors(string(errOut)) + for _, pair := range pairs { + f := evaluateNvidiaNVLinkPair(pair, linkStatus, linkErrors) + status.NVLinkPairs = append(status.NVLinkPairs, f) + for _, issue := range f.Issues { + status.Warnings = append(status.Warnings, fmt.Sprintf("NVLink GPU%d<->GPU%d: %s", f.GPUA, f.GPUB, issue)) + } + } + + // -- Confidential Computing readiness (informational only) ------------- + ccOut, ccErr := runSATCommandCtx(ctx, verboseLog, "nvidia-smi-conf-compute-q", []string{"nvidia-smi", "conf-compute", "-q"}, nil, logFunc) + _ = os.WriteFile(filepath.Join(runDir, "04-nvidia-smi-conf-compute-q.log"), ccOut, 0644) + if ccErr == nil { + fields := parseConfComputeFields(ccOut) + if status.CCState == "" { + status.CCState = fields["CC State"] + } + status.CPUCCCapability = fields["CPU CC Capabilities"] + status.GPUCCCapability = fields["GPU CC Capabilities"] + } + dmesgOut, _ := runSATCommandCtx(ctx, verboseLog, "dmesg", []string{"dmesg"}, nil, nil) + ccDmesgLines := filterConfComputeDmesgLines(dmesgOut) + _ = os.WriteFile(filepath.Join(runDir, "05-dmesg-cc-relevant.log"), []byte(strings.Join(ccDmesgLines, "\n")+"\n"), 0644) + lowerDmesg := strings.ToLower(strings.Join(ccDmesgLines, "\n")) + status.HostAMDSEVSNPActive = strings.Contains(lowerDmesg, "sev-snp enabled") + status.HostIntelTDXActive = strings.Contains(lowerDmesg, "tdx module") && strings.Contains(lowerDmesg, "module initialized") || + strings.Contains(lowerDmesg, "virt/tdx: module initialized") + for _, path := range []string{"/sys/module/kvm_amd/parameters/sev_snp"} { + if out, err := runSATCommandCtx(ctx, verboseLog, "sysfs-sev-snp", []string{"cat", path}, nil, nil); err == nil && + strings.EqualFold(strings.TrimSpace(string(out)), "Y") { + status.HostAMDSEVSNPActive = true + } + } + gpuCanRunCC := strings.EqualFold(strings.TrimSpace(status.GPUCCCapability), "CC Capable") + cpuCapReported := strings.TrimSpace(status.CPUCCCapability) + cpuCanRunCC := status.HostAMDSEVSNPActive || status.HostIntelTDXActive || + (cpuCapReported != "" && !strings.EqualFold(cpuCapReported, "NONE")) + status.CCReady = cpuCanRunCC && gpuCanRunCC + + summary := renderNvidiaConfigCheckSummary(status) + if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary), 0644); err != nil { + return "", err + } + report := renderNvidiaConfigCheckReport(status) + if err := os.WriteFile(filepath.Join(runDir, "nvidia-config-report.txt"), []byte(report), 0644); err != nil { + return "", err + } + + return runDir, nil +} + +// NvidiaConfigCheckStatus is the result of RunNvidiaConfigCheckPack. +type NvidiaConfigCheckStatus struct { + CollectedAt time.Time `json:"collected_at"` + + GPUs []NvidiaGPUConfigFinding `json:"gpus,omitempty"` + NVLinkPairs []NvidiaNVLinkPairFinding `json:"nvlink_pairs,omitempty"` + + // Confidential Computing readiness — informational only, does not gate + // overall_status: an unconfigured/NOT_READY CC state is a + // feature-readiness fact on most fleets (CC is opt-in), not a fault. + CCReady bool `json:"cc_ready"` + CCState string `json:"cc_state,omitempty"` + CPUCCCapability string `json:"cpu_cc_capability,omitempty"` + GPUCCCapability string `json:"gpu_cc_capability,omitempty"` + + HostAMDSEVSNPActive bool `json:"host_amd_sev_snp_active"` + HostIntelTDXActive bool `json:"host_intel_tdx_active"` + + // Warnings drive overall_status: any entry here fails the check. + Warnings []string `json:"warnings,omitempty"` + Notes []string `json:"notes,omitempty"` +} + +// NvidiaGPUConfigFinding is one GPU's config-compliance result. +type NvidiaGPUConfigFinding struct { + Index int `json:"index"` + Name string `json:"name"` + ECCCurrent string `json:"ecc_current"` + MIGCurrent string `json:"mig_current"` + MIGPending string `json:"mig_pending"` + PowerLimitW float64 `json:"power_limit_w"` + PowerDefaultLimitW float64 `json:"power_default_limit_w"` + Issues []string `json:"issues,omitempty"` +} + +// NvidiaNVLinkPairFinding is one NVLink-bonded GPU pair's link-count/error +// validation result. +type NvidiaNVLinkPairFinding struct { + GPUA int `json:"gpu_a"` + GPUB int `json:"gpu_b"` + ExpectedLinks int `json:"expected_links"` // from "nvidia-smi topo -m"'s NVx cell + ActiveLinks int `json:"active_links"` // from "nvidia-smi nvlink -s" + TotalLinks int `json:"total_links"` + ErrorLinks int `json:"error_links"` // links with a nonzero replay/recovery/CRC counter + Issues []string `json:"issues,omitempty"` +} + +// evaluateNvidiaGPUConfig flags config drift that DCGM diag does not check: +// ECC disabled (NVIDIA data-center GPUs ship with ECC on), a MIG mode +// change stuck pending a reset/reboot, and a power limit capped meaningfully +// below the card's own factory default (a silent throttle that a stress +// test would only surface as "lower than expected performance", with no +// clear cause). +func evaluateNvidiaGPUConfig(g NvidiaGPUSetting) []string { + var issues []string + if strings.EqualFold(g.ECCCurrent, "disabled") { + issues = append(issues, "ECC is disabled") + } + if g.MIGCurrent != "" && g.MIGPending != "" && !strings.EqualFold(g.MIGCurrent, g.MIGPending) { + issues = append(issues, fmt.Sprintf("MIG mode change pending (current=%s, pending=%s) — apply with a GPU reset or reboot", g.MIGCurrent, g.MIGPending)) + } + if g.PowerDefaultLimitW > 0 && g.PowerLimitW > 0 { + deficit := (g.PowerDefaultLimitW - g.PowerLimitW) / g.PowerDefaultLimitW + if deficit > 0.05 { + issues = append(issues, fmt.Sprintf("power limit %.0fW is %.0f%% below factory default %.0fW", g.PowerLimitW, deficit*100, g.PowerDefaultLimitW)) + } + } + return issues +} + +// evaluateNvidiaNVLinkPair compares a bonded pair's actual link state +// against what the topology matrix says should be there. +func evaluateNvidiaNVLinkPair(pair nvidiaNVLinkBondedPair, status map[int][]nvidiaNVLinkPort, errors map[int]map[int][3]int64) NvidiaNVLinkPairFinding { + f := NvidiaNVLinkPairFinding{GPUA: pair.gpuA, GPUB: pair.gpuB, ExpectedLinks: pair.links} + + active, total, errLinks := 0, 0, 0 + for _, gpu := range []int{pair.gpuA, pair.gpuB} { + for _, port := range status[gpu] { + total++ + if port.active { + active++ + } + } + for _, counters := range errors[gpu] { + if counters[0] != 0 || counters[1] != 0 || counters[2] != 0 { + errLinks++ + } + } + } + f.ActiveLinks, f.TotalLinks, f.ErrorLinks = active, total, errLinks + + if total > 0 && active < total { + f.Issues = append(f.Issues, fmt.Sprintf("%d/%d NVLinks inactive on a bonded pair", total-active, total)) + } + if errLinks > 0 { + f.Issues = append(f.Issues, fmt.Sprintf("%d link(s) reporting replay/recovery/CRC errors", errLinks)) + } + return f +} + +// --------------------------------------------------------------------------- +// NVLink parsing — deliberately self-contained rather than importing +// internal/collector or internal/webui's equivalent parsers (isNVLinkBridgeCandidate +// et al. and parseGPUPairAdjacency/parseTopoNVLinkStatus/parseTopoNVLinkErrors, +// respectively), matching this codebase's existing precedent of small, +// package-local duplication over cross-package coupling for narrow parsing +// helpers (see webui/page_topo.go's isNICDeviceClassDev/isRAIDControllerClass). +// --------------------------------------------------------------------------- + +type nvidiaNVLinkBondedPair struct { + gpuA, gpuB int + links int +} + +var ( + nvidiaNVLinkANSIRe = regexp.MustCompile("\x1b\\[[0-9;]*[A-Za-z]") + nvidiaNVLinkNVRe = regexp.MustCompile(`(?i)^NV(\d+)$`) +) + +// parseNvidiaNVLinkBondedPairs returns every GPU pair with a nonzero NVLink +// bond count from a "nvidia-smi topo -m" matrix, deduplicated (A,B) == (B,A). +func parseNvidiaNVLinkBondedPairs(raw string) []nvidiaNVLinkBondedPair { + lines := strings.Split(nvidiaNVLinkANSIRe.ReplaceAllString(raw, ""), "\n") + headerIdx := -1 + var gpuColIndices []int + for i, line := range lines { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "GPU0") { + parts := strings.Fields(trimmed) + for j, col := range parts { + if strings.HasPrefix(col, "GPU") { + gpuColIndices = append(gpuColIndices, j) + } + } + if len(gpuColIndices) >= 2 { + headerIdx = i + } + break + } + } + if headerIdx < 0 { + return nil + } + colIdxToGPU := make(map[int]int, len(gpuColIndices)) + for gpuIdx, colIdx := range gpuColIndices { + colIdxToGPU[colIdx] = gpuIdx + } + + seen := map[[2]int]bool{} + var pairs []nvidiaNVLinkBondedPair + for _, line := range lines[headerIdx+1:] { + trimmed := strings.TrimSpace(line) + if !strings.HasPrefix(trimmed, "GPU") { + continue + } + cells := strings.Fields(trimmed) + if len(cells) == 0 { + continue + } + rowGPU, err := strconv.Atoi(strings.TrimPrefix(cells[0], "GPU")) + if err != nil { + continue + } + for colIdx, colGPU := range colIdxToGPU { + if colGPU == rowGPU { + continue + } + dataIdx := colIdx + 1 + if dataIdx >= len(cells) { + continue + } + m := nvidiaNVLinkNVRe.FindStringSubmatch(cells[dataIdx]) + if len(m) != 2 { + continue + } + nv, err := strconv.Atoi(m[1]) + if err != nil || nv <= 0 { + continue + } + a, b := rowGPU, colGPU + if a > b { + a, b = b, a + } + key := [2]int{a, b} + if seen[key] { + continue + } + seen[key] = true + pairs = append(pairs, nvidiaNVLinkBondedPair{gpuA: a, gpuB: b, links: nv}) + } + } + return pairs +} + +type nvidiaNVLinkPort struct { + active bool +} + +var ( + nvidiaNVLinkGPUHeaderRe = regexp.MustCompile(`^GPU (\d+):`) + nvidiaNVLinkSpeedLineRe = regexp.MustCompile(`^Link (\d+):\s*[\d.]+\s*GB/s`) + nvidiaNVLinkInactiveRe = regexp.MustCompile(`^Link (\d+):\s*`) + nvidiaNVLinkErrorLineRe = regexp.MustCompile(`^Link (\d+):\s*(Replay|Recovery|CRC) Errors:\s*(\d+)`) +) + +// parseNvidiaNVLinkStatus parses "nvidia-smi nvlink -s" output into per-GPU +// per-link active/inactive state. +func parseNvidiaNVLinkStatus(raw string) map[int][]nvidiaNVLinkPort { + result := map[int][]nvidiaNVLinkPort{} + currentGPU := -1 + for _, line := range strings.Split(raw, "\n") { + trimmed := strings.TrimSpace(line) + if m := nvidiaNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil { + currentGPU, _ = strconv.Atoi(m[1]) + continue + } + if currentGPU < 0 { + continue + } + if nvidiaNVLinkInactiveRe.MatchString(trimmed) { + result[currentGPU] = append(result[currentGPU], nvidiaNVLinkPort{active: false}) + continue + } + if nvidiaNVLinkSpeedLineRe.MatchString(trimmed) { + result[currentGPU] = append(result[currentGPU], nvidiaNVLinkPort{active: true}) + } + } + return result +} + +// parseNvidiaNVLinkErrors parses "nvidia-smi nvlink -e" output into, per GPU +// then link index, [replay, recovery, crc] error counts. +func parseNvidiaNVLinkErrors(raw string) map[int]map[int][3]int64 { + result := map[int]map[int][3]int64{} + currentGPU := -1 + for _, line := range strings.Split(raw, "\n") { + trimmed := strings.TrimSpace(line) + if m := nvidiaNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil { + currentGPU, _ = strconv.Atoi(m[1]) + continue + } + if currentGPU < 0 { + continue + } + m := nvidiaNVLinkErrorLineRe.FindStringSubmatch(trimmed) + if m == nil { + continue + } + linkIdx, _ := strconv.Atoi(m[1]) + count, _ := strconv.ParseInt(m[3], 10, 64) + if result[currentGPU] == nil { + result[currentGPU] = map[int][3]int64{} + } + c := result[currentGPU][linkIdx] + switch m[2] { + case "Replay": + c[0] = count + case "Recovery": + c[1] = count + case "CRC": + c[2] = count + } + result[currentGPU][linkIdx] = c + } + return result +} + +// --------------------------------------------------------------------------- +// Confidential Computing (folded in — see RunNvidiaConfigCheckPack doc) +// --------------------------------------------------------------------------- + +// parseConfComputeFields parses the indented "Key : Value" block emitted by +// `nvidia-smi conf-compute -q`, e.g.: +// +// CC State : OFF +// Multi-GPU Mode : Protected PCIe +// CPU CC Capabilities : INTEL TDX +// GPU CC Capabilities : CC Capable +// CC GPUs Ready State : Not Ready +func parseConfComputeFields(out []byte) map[string]string { + fields := map[string]string{} + for _, line := range strings.Split(string(out), "\n") { + idx := strings.Index(line, ":") + if idx < 0 { + continue + } + key := strings.TrimSpace(line[:idx]) + val := strings.TrimSpace(line[idx+1:]) + if key == "" || val == "" { + continue + } + fields[key] = val + } + return fields +} + +// filterConfComputeDmesgLines returns the dmesg lines relevant to CPU +// Confidential Computing support (AMD SEV/SEV-ES/SEV-SNP, Intel TDX). +func filterConfComputeDmesgLines(dmesgOut []byte) []string { + var lines []string + for _, raw := range bytes.Split(dmesgOut, []byte("\n")) { + lower := strings.ToLower(string(raw)) + if strings.Contains(lower, "sev") || strings.Contains(lower, "tdx") { + lines = append(lines, string(raw)) + } + } + return lines +} + +// --------------------------------------------------------------------------- +// Report rendering +// --------------------------------------------------------------------------- + +func renderNvidiaConfigCheckSummary(status NvidiaConfigCheckStatus) string { + var b strings.Builder + fmt.Fprintf(&b, "run_at_utc=%s\n", status.CollectedAt.Format(time.RFC3339)) + fmt.Fprintf(&b, "gpu_count=%d\n", len(status.GPUs)) + eccIssues, migIssues, powerIssues := 0, 0, 0 + for _, g := range status.GPUs { + for _, issue := range g.Issues { + switch { + case strings.Contains(issue, "ECC"): + eccIssues++ + case strings.Contains(issue, "MIG"): + migIssues++ + case strings.Contains(issue, "power limit"): + powerIssues++ + } + } + } + fmt.Fprintf(&b, "gpu_ecc_issues=%d\n", eccIssues) + fmt.Fprintf(&b, "gpu_mig_pending_issues=%d\n", migIssues) + fmt.Fprintf(&b, "gpu_power_limit_issues=%d\n", powerIssues) + fmt.Fprintf(&b, "nvlink_pairs_checked=%d\n", len(status.NVLinkPairs)) + nvlinkIssues := 0 + for _, p := range status.NVLinkPairs { + if len(p.Issues) > 0 { + nvlinkIssues++ + } + } + fmt.Fprintf(&b, "nvlink_pairs_with_issues=%d\n", nvlinkIssues) + fmt.Fprintf(&b, "cc_ready=%t\n", status.CCReady) + fmt.Fprintf(&b, "cc_state=%s\n", status.CCState) + fmt.Fprintf(&b, "cpu_cc_capability=%s\n", status.CPUCCCapability) + fmt.Fprintf(&b, "gpu_cc_capability=%s\n", status.GPUCCCapability) + if len(status.Warnings) == 0 { + fmt.Fprintln(&b, "overall_status=OK") + } else { + fmt.Fprintln(&b, "overall_status=FAILED") + } + return b.String() +} + +func renderNvidiaConfigCheckReport(status NvidiaConfigCheckStatus) string { + var b strings.Builder + line := strings.Repeat("=", 80) + b.WriteString(line + "\n") + b.WriteString("NVIDIA GPU Configuration & NVLink Topology Check\n") + b.WriteString(line + "\n\n") + + b.WriteString("-- GPU Configuration ----------------------------------------------------------\n") + for _, g := range status.GPUs { + verdict := "OK" + if len(g.Issues) > 0 { + verdict = "ISSUES FOUND" + } + fmt.Fprintf(&b, " GPU %d (%s): %s\n", g.Index, g.Name, verdict) + fmt.Fprintf(&b, " ECC: %s | MIG: %s (pending %s) | Power: %.0fW / %.0fW default\n", + nonEmptyOr(g.ECCCurrent, "unknown"), nonEmptyOr(g.MIGCurrent, "unknown"), nonEmptyOr(g.MIGPending, "unknown"), + g.PowerLimitW, g.PowerDefaultLimitW) + for _, issue := range g.Issues { + fmt.Fprintf(&b, " - %s\n", issue) + } + } + b.WriteString("\n-- NVLink Topology --------------------------------------------------------------\n") + if len(status.NVLinkPairs) == 0 { + b.WriteString(" No NVLink-bonded GPU pairs found.\n") + } + for _, p := range status.NVLinkPairs { + verdict := "OK" + if len(p.Issues) > 0 { + verdict = "ISSUES FOUND" + } + fmt.Fprintf(&b, " GPU%d <-> GPU%d: %d/%d links active (topology expects %d): %s\n", + p.GPUA, p.GPUB, p.ActiveLinks, p.TotalLinks, p.ExpectedLinks, verdict) + for _, issue := range p.Issues { + fmt.Fprintf(&b, " - %s\n", issue) + } + } + + b.WriteString("\n-- Confidential Computing (informational) ---------------------------------------\n") + verdict := "NOT READY" + if status.CCReady { + verdict = "READY" + } + fmt.Fprintf(&b, " Verdict: %s\n", verdict) + fmt.Fprintf(&b, " CC State: %s | CPU CC: %s | GPU CC: %s\n", + nonEmptyOr(status.CCState, "unknown"), nonEmptyOr(status.CPUCCCapability, "unknown"), nonEmptyOr(status.GPUCCCapability, "unknown")) + + if len(status.Notes) > 0 { + b.WriteString("\n-- Notes -------------------------------------------------------------------------\n") + for _, n := range status.Notes { + fmt.Fprintf(&b, " - %s\n", n) + } + } + + fmt.Fprintf(&b, "\nCollected : %s\n", status.CollectedAt.Format("2006-01-02 15:04:05 UTC")) + b.WriteString(line + "\n") + return b.String() +} + +func nonEmptyOr(v, fallback string) string { + if strings.TrimSpace(v) == "" { + return fallback + } + return v +} diff --git a/audit/internal/platform/nvidia_config_check_test.go b/audit/internal/platform/nvidia_config_check_test.go new file mode 100644 index 0000000..ec5c4b2 --- /dev/null +++ b/audit/internal/platform/nvidia_config_check_test.go @@ -0,0 +1,168 @@ +package platform + +import ( + "strings" + "testing" +) + +func TestParseNvidiaNVLinkBondedPairsRealTwoGPUDump(t *testing.T) { + // Real system/nvidia-smi-topo.txt shape: two H100s directly bridged + // (NV17). + input := "\tGPU0\tGPU1\tNIC0\tNIC1\tCPU Affinity\tNUMA Affinity\tGPU NUMA ID\n" + + "GPU0\t X \tNV17\tSYS\tSYS\t0-23,48-71\t0\t\tN/A\n" + + "GPU1\tNV17\t X \tNODE\tNODE\t24-47,72-95\t1\t\tN/A\n" + + "NIC0\tSYS\tNODE\t X \tPIX\t\t\t\n" + + "NIC1\tSYS\tNODE\tPIX\t X \t\t\t\n" + + pairs := parseNvidiaNVLinkBondedPairs(input) + if len(pairs) != 1 { + t.Fatalf("pairs=%d want 1 (%#v)", len(pairs), pairs) + } + if pairs[0].gpuA != 0 || pairs[0].gpuB != 1 || pairs[0].links != 17 { + t.Fatalf("pair=%#v want {0,1,17}", pairs[0]) + } +} + +func TestParseNvidiaNVLinkBondedPairsANSIUnderlinedHeader(t *testing.T) { + // nvidia-smi underlines the topo -m header with ANSI CSI codes even when + // writing to a file — the same real-world quirk fixed for the /topo page + // parser must be handled here too, since this uses a fresh live query + // rather than reading the persisted techdump. + input := "\x1b[4m\tGPU0\tGPU1\tGPU2\tGPU3\tCPU Affinity\x1b[0m\n" + + "GPU0\t X \tNV18\tPIX\tPIX\t0-31,64-95\n" + + "GPU1\tNV18\t X \tPIX\tPIX\t0-31,64-95\n" + + "GPU2\tPIX\tPIX\t X \tNV18\t0-31,64-95\n" + + "GPU3\tPIX\tPIX\tNV18\t X \t0-31,64-95\n" + + pairs := parseNvidiaNVLinkBondedPairs(input) + if len(pairs) != 2 { + t.Fatalf("pairs=%d want 2 (%#v)", len(pairs), pairs) + } + want := map[[2]int]int{{0, 1}: 18, {2, 3}: 18} + for _, p := range pairs { + if want[[2]int{p.gpuA, p.gpuB}] != p.links { + t.Fatalf("unexpected pair %#v", p) + } + } +} + +func TestParseNvidiaNVLinkStatusMarksInactiveLinks(t *testing.T) { + input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140) + Link 0: 26.562 GB/s + Link 1: +GPU 1: NVIDIA H100 80GB HBM3 (UUID: GPU-603fe750-0516-9db5-86ec-ea61af3fce35) + Link 0: 26.562 GB/s +` + got := parseNvidiaNVLinkStatus(input) + if len(got[0]) != 2 || got[0][0].active != true || got[0][1].active != false { + t.Fatalf("gpu0=%#v want [active, inactive]", got[0]) + } + if len(got[1]) != 1 || !got[1][0].active { + t.Fatalf("gpu1=%#v want [active]", got[1]) + } +} + +func TestParseNvidiaNVLinkErrors(t *testing.T) { + input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140) + Link 0: Replay Errors: 0 + Link 0: Recovery Errors: 0 + Link 0: CRC Errors: 0 + Link 1: Replay Errors: 3 + Link 1: Recovery Errors: 1 + Link 1: CRC Errors: 2 +` + got := parseNvidiaNVLinkErrors(input) + c := got[0][1] + if c[0] != 3 || c[1] != 1 || c[2] != 2 { + t.Fatalf("link1 counters=%#v want {3,1,2}", c) + } +} + +func TestEvaluateNvidiaNVLinkPairFlagsInactiveLinksAndErrors(t *testing.T) { + pair := nvidiaNVLinkBondedPair{gpuA: 0, gpuB: 1, links: 18} + status := map[int][]nvidiaNVLinkPort{ + 0: {{active: true}, {active: false}}, + 1: {{active: true}, {active: true}}, + } + errors := map[int]map[int][3]int64{ + 0: {1: {3, 0, 0}}, + } + + f := evaluateNvidiaNVLinkPair(pair, status, errors) + if f.ActiveLinks != 3 || f.TotalLinks != 4 { + t.Fatalf("active=%d total=%d want 3/4", f.ActiveLinks, f.TotalLinks) + } + if len(f.Issues) != 2 { + t.Fatalf("issues=%#v want 2 (inactive link + error)", f.Issues) + } +} + +func TestEvaluateNvidiaNVLinkPairHealthyBondHasNoIssues(t *testing.T) { + pair := nvidiaNVLinkBondedPair{gpuA: 0, gpuB: 1, links: 18} + status := map[int][]nvidiaNVLinkPort{ + 0: {{active: true}}, + 1: {{active: true}}, + } + f := evaluateNvidiaNVLinkPair(pair, status, nil) + if len(f.Issues) != 0 { + t.Fatalf("issues=%#v want none for a fully active, error-free bond", f.Issues) + } +} + +func TestEvaluateNvidiaGPUConfigFlagsECCDisabled(t *testing.T) { + g := NvidiaGPUSetting{Index: 0, Name: "H100", ECCCurrent: "Disabled"} + issues := evaluateNvidiaGPUConfig(g) + if len(issues) != 1 { + t.Fatalf("issues=%#v want 1 (ECC disabled)", issues) + } +} + +func TestEvaluateNvidiaGPUConfigFlagsStuckMIGPending(t *testing.T) { + g := NvidiaGPUSetting{Index: 0, Name: "H100", ECCCurrent: "Enabled", MIGCurrent: "Disabled", MIGPending: "Enabled"} + issues := evaluateNvidiaGPUConfig(g) + if len(issues) != 1 { + t.Fatalf("issues=%#v want 1 (MIG pending != current)", issues) + } +} + +func TestEvaluateNvidiaGPUConfigFlagsPowerLimitBelowDefault(t *testing.T) { + g := NvidiaGPUSetting{Index: 0, Name: "H100", ECCCurrent: "Enabled", PowerLimitW: 300, PowerDefaultLimitW: 700} + issues := evaluateNvidiaGPUConfig(g) + if len(issues) != 1 { + t.Fatalf("issues=%#v want 1 (power limit far below default)", issues) + } +} + +func TestEvaluateNvidiaGPUConfigNominalHasNoIssues(t *testing.T) { + g := NvidiaGPUSetting{ + Index: 0, Name: "H100", ECCCurrent: "Enabled", + MIGCurrent: "Disabled", MIGPending: "Disabled", + PowerLimitW: 700, PowerDefaultLimitW: 700, + } + issues := evaluateNvidiaGPUConfig(g) + if len(issues) != 0 { + t.Fatalf("issues=%#v want none for nominal config", issues) + } +} + +func TestEvaluateNvidiaGPUConfigSmallPowerLimitDeviationIsNotFlagged(t *testing.T) { + // Power limits set slightly below default (e.g. rounding, minor + // operator tuning) should not be treated as an anomaly — only a + // meaningful cap (>5%) is. + g := NvidiaGPUSetting{Index: 0, Name: "H100", ECCCurrent: "Enabled", PowerLimitW: 690, PowerDefaultLimitW: 700} + issues := evaluateNvidiaGPUConfig(g) + if len(issues) != 0 { + t.Fatalf("issues=%#v want none for a <2%% power-limit deviation", issues) + } +} + +func TestRenderNvidiaConfigCheckSummaryOverallStatus(t *testing.T) { + clean := NvidiaConfigCheckStatus{} + if got := renderNvidiaConfigCheckSummary(clean); !strings.Contains(got, "overall_status=OK") { + t.Fatalf("clean status summary missing overall_status=OK:\n%s", got) + } + withWarning := NvidiaConfigCheckStatus{Warnings: []string{"GPU 0: ECC is disabled"}} + if got := renderNvidiaConfigCheckSummary(withWarning); !strings.Contains(got, "overall_status=FAILED") { + t.Fatalf("status with warnings missing overall_status=FAILED:\n%s", got) + } +} diff --git a/audit/internal/platform/nvidia_settings.go b/audit/internal/platform/nvidia_settings.go index 391dbcf..49a8483 100644 --- a/audit/internal/platform/nvidia_settings.go +++ b/audit/internal/platform/nvidia_settings.go @@ -70,7 +70,7 @@ func (s *System) ListNvidiaGPUSettings() ([]NvidiaGPUSetting, error) { // queryNvidiaGPUCCState reads the per-GPU Confidential Computing state via // `nvidia-smi conf-compute -i -q`, reusing the same field-name -// convention as RunConfidentialComputingCheckPack's parseConfComputeFields. +// convention as RunNvidiaConfigCheckPack's parseConfComputeFields. // Returns "" if the driver/GPU doesn't support conf-compute at all. func (s *System) queryNvidiaGPUCCState(index int) string { out, err := satExecCommand("nvidia-smi", "conf-compute", "-i", strconv.Itoa(index), "-q").Output() diff --git a/audit/internal/webui/api.go b/audit/internal/webui/api.go index 4ca831d..d072a49 100644 --- a/audit/internal/webui/api.go +++ b/audit/internal/webui/api.go @@ -135,7 +135,7 @@ func defaultTaskPriority(target string, params taskParams) int { return taskPriorityBurn case "nvidia", "nvidia-targeted-stress", "nvidia-targeted-power", "nvidia-pulse", "nvidia-interconnect", "nvidia-bandwidth", "memory", "storage", "cpu", - "amd", "amd-mem", "amd-bandwidth", "confidential-computing": + "amd", "amd-mem", "amd-bandwidth", "nvidia-config": if params.StressMode { return taskPriorityValidateStress } diff --git a/audit/internal/webui/page_validate.go b/audit/internal/webui/page_validate.go index 1423e4b..20b175a 100644 --- a/audit/internal/webui/page_validate.go +++ b/audit/internal/webui/page_validate.go @@ -676,10 +676,10 @@ func renderCheck(opts HandlerOptions) string { `lsblk; NVMe: nvme id-ctrl, nvme smart-log; SATA/SAS: smartctl -H -A`, `Seconds — instantaneous device query, no wear counters incremented.`, )) + - renderSATCard("confidential-computing", "Confidential Computing", "runSAT('confidential-computing')", "", renderValidateCardBody( + renderSATCard("nvidia-config", "GPU Config & NVLink", "runSAT('nvidia-config')", "", renderValidateCardBody( inv.NVIDIA, - `Checks whether this server can run NVIDIA Confidential Computing: CPU TEE support (Intel TDX / AMD SEV-SNP) and GPU firmware CC capability. Read-only — changes nothing.`, - `nvidia-smi conf-compute -q, dmesg, /sys/module/kvm_amd/parameters/*`, + `Checks GPU configuration and NVLink topology that DCGM diag does not cover: ECC/MIG/power-limit drift from factory default, NVLink-bonded pair link count and error counters, and (informational) NVIDIA Confidential Computing readiness (CPU TEE support + GPU firmware CC capability). Read-only — changes nothing.`, + `nvidia-smi --query-gpu=..., nvidia-smi topo -m, nvidia-smi nvlink -s/-e, nvidia-smi conf-compute -q, dmesg`, `Seconds — read-only query only.`, )) + ` @@ -743,7 +743,7 @@ func renderCheck(opts HandlerOptions) string {