platform/webui: fold confidential-computing into a GPU config + NVLink check

The standalone "confidential-computing" SAT target only ever checked CC readiness, which most fleets never opt into (a NOT_READY verdict there isn't a fault). Meanwhile DCGM diag never asserts GPU config compliance (ECC/MIG/power-limit vs factory default) or NVLink topology (per NVIDIA's own DGX BasePOD deployment guide, this needs a separate validation step) — gaps confirmed against public DCGM docs and a real NV17-vs-expected-NV18 bonded pair found on a live bundle. Repurposes the routine into "nvidia-config": reuses the existing ListNvidiaGPUSettings() (already backing the GPU-settings page) to flag ECC disabled, a MIG mode change stuck pending a reset/reboot, and a power limit capped >5% below default; parses "nvidia-smi topo -m" bonded pairs against "nvlink -s/-e" to flag any inactive lane or nonzero replay/ recovery/CRC counter on an otherwise-active bond. CC readiness is folded in as one informational field (does not gate overall_status) rather than a dedicated test. Reports under the same pcie:gpu:nvidia severity key as every other nvidia-* SAT target instead of an isolated key, so a config/NVLink FAILED result isn't invisible next to stress-test results. Also fixes ApplySATResultToDB silently dropping any target with no matching switch case (exactly what the old confidential-computing target did) with a new coverage test enumerating every real SAT target. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-07-09 11:36:52 +03:00
parent cc3997f7b1
commit cfaa15ec7c
14 changed files with 811 additions and 273 deletions
@@ -0,0 +1,168 @@
+package platform
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestParseNvidiaNVLinkBondedPairsRealTwoGPUDump(t *testing.T) {
+	// Real system/nvidia-smi-topo.txt shape: two H100s directly bridged
+	// (NV17).
+	input := "\tGPU0\tGPU1\tNIC0\tNIC1\tCPU Affinity\tNUMA Affinity\tGPU NUMA ID\n" +
+		"GPU0\t X \tNV17\tSYS\tSYS\t0-23,48-71\t0\t\tN/A\n" +
+		"GPU1\tNV17\t X \tNODE\tNODE\t24-47,72-95\t1\t\tN/A\n" +
+		"NIC0\tSYS\tNODE\t X \tPIX\t\t\t\n" +
+		"NIC1\tSYS\tNODE\tPIX\t X \t\t\t\n"
+
+	pairs := parseNvidiaNVLinkBondedPairs(input)
+	if len(pairs) != 1 {
+		t.Fatalf("pairs=%d want 1 (%#v)", len(pairs), pairs)
+	}
+	if pairs[0].gpuA != 0 || pairs[0].gpuB != 1 || pairs[0].links != 17 {
+		t.Fatalf("pair=%#v want {0,1,17}", pairs[0])
+	}
+}
+
+func TestParseNvidiaNVLinkBondedPairsANSIUnderlinedHeader(t *testing.T) {
+	// nvidia-smi underlines the topo -m header with ANSI CSI codes even when
+	// writing to a file — the same real-world quirk fixed for the /topo page
+	// parser must be handled here too, since this uses a fresh live query
+	// rather than reading the persisted techdump.
+	input := "\x1b[4m\tGPU0\tGPU1\tGPU2\tGPU3\tCPU Affinity\x1b[0m\n" +
+		"GPU0\t X \tNV18\tPIX\tPIX\t0-31,64-95\n" +
+		"GPU1\tNV18\t X \tPIX\tPIX\t0-31,64-95\n" +
+		"GPU2\tPIX\tPIX\t X \tNV18\t0-31,64-95\n" +
+		"GPU3\tPIX\tPIX\tNV18\t X \t0-31,64-95\n"
+
+	pairs := parseNvidiaNVLinkBondedPairs(input)
+	if len(pairs) != 2 {
+		t.Fatalf("pairs=%d want 2 (%#v)", len(pairs), pairs)
+	}
+	want := map[[2]int]int{{0, 1}: 18, {2, 3}: 18}
+	for _, p := range pairs {
+		if want[[2]int{p.gpuA, p.gpuB}] != p.links {
+			t.Fatalf("unexpected pair %#v", p)
+		}
+	}
+}
+
+func TestParseNvidiaNVLinkStatusMarksInactiveLinks(t *testing.T) {
+	input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140)
+	 Link 0: 26.562 GB/s
+	 Link 1: <inactive>
+GPU 1: NVIDIA H100 80GB HBM3 (UUID: GPU-603fe750-0516-9db5-86ec-ea61af3fce35)
+	 Link 0: 26.562 GB/s
+`
+	got := parseNvidiaNVLinkStatus(input)
+	if len(got[0]) != 2 || got[0][0].active != true || got[0][1].active != false {
+		t.Fatalf("gpu0=%#v want [active, inactive]", got[0])
+	}
+	if len(got[1]) != 1 || !got[1][0].active {
+		t.Fatalf("gpu1=%#v want [active]", got[1])
+	}
+}
+
+func TestParseNvidiaNVLinkErrors(t *testing.T) {
+	input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140)
+	 Link 0: Replay Errors: 0
+	 Link 0: Recovery Errors: 0
+	 Link 0: CRC Errors: 0
+	 Link 1: Replay Errors: 3
+	 Link 1: Recovery Errors: 1
+	 Link 1: CRC Errors: 2
+`
+	got := parseNvidiaNVLinkErrors(input)
+	c := got[0][1]
+	if c[0] != 3 || c[1] != 1 || c[2] != 2 {
+		t.Fatalf("link1 counters=%#v want {3,1,2}", c)
+	}
+}
+
+func TestEvaluateNvidiaNVLinkPairFlagsInactiveLinksAndErrors(t *testing.T) {
+	pair := nvidiaNVLinkBondedPair{gpuA: 0, gpuB: 1, links: 18}
+	status := map[int][]nvidiaNVLinkPort{
+		0: {{active: true}, {active: false}},
+		1: {{active: true}, {active: true}},
+	}
+	errors := map[int]map[int][3]int64{
+		0: {1: {3, 0, 0}},
+	}
+
+	f := evaluateNvidiaNVLinkPair(pair, status, errors)
+	if f.ActiveLinks != 3 || f.TotalLinks != 4 {
+		t.Fatalf("active=%d total=%d want 3/4", f.ActiveLinks, f.TotalLinks)
+	}
+	if len(f.Issues) != 2 {
+		t.Fatalf("issues=%#v want 2 (inactive link + error)", f.Issues)
+	}
+}
+
+func TestEvaluateNvidiaNVLinkPairHealthyBondHasNoIssues(t *testing.T) {
+	pair := nvidiaNVLinkBondedPair{gpuA: 0, gpuB: 1, links: 18}
+	status := map[int][]nvidiaNVLinkPort{
+		0: {{active: true}},
+		1: {{active: true}},
+	}
+	f := evaluateNvidiaNVLinkPair(pair, status, nil)
+	if len(f.Issues) != 0 {
+		t.Fatalf("issues=%#v want none for a fully active, error-free bond", f.Issues)
+	}
+}
+
+func TestEvaluateNvidiaGPUConfigFlagsECCDisabled(t *testing.T) {
+	g := NvidiaGPUSetting{Index: 0, Name: "H100", ECCCurrent: "Disabled"}
+	issues := evaluateNvidiaGPUConfig(g)
+	if len(issues) != 1 {
+		t.Fatalf("issues=%#v want 1 (ECC disabled)", issues)
+	}
+}
+
+func TestEvaluateNvidiaGPUConfigFlagsStuckMIGPending(t *testing.T) {
+	g := NvidiaGPUSetting{Index: 0, Name: "H100", ECCCurrent: "Enabled", MIGCurrent: "Disabled", MIGPending: "Enabled"}
+	issues := evaluateNvidiaGPUConfig(g)
+	if len(issues) != 1 {
+		t.Fatalf("issues=%#v want 1 (MIG pending != current)", issues)
+	}
+}
+
+func TestEvaluateNvidiaGPUConfigFlagsPowerLimitBelowDefault(t *testing.T) {
+	g := NvidiaGPUSetting{Index: 0, Name: "H100", ECCCurrent: "Enabled", PowerLimitW: 300, PowerDefaultLimitW: 700}
+	issues := evaluateNvidiaGPUConfig(g)
+	if len(issues) != 1 {
+		t.Fatalf("issues=%#v want 1 (power limit far below default)", issues)
+	}
+}
+
+func TestEvaluateNvidiaGPUConfigNominalHasNoIssues(t *testing.T) {
+	g := NvidiaGPUSetting{
+		Index: 0, Name: "H100", ECCCurrent: "Enabled",
+		MIGCurrent: "Disabled", MIGPending: "Disabled",
+		PowerLimitW: 700, PowerDefaultLimitW: 700,
+	}
+	issues := evaluateNvidiaGPUConfig(g)
+	if len(issues) != 0 {
+		t.Fatalf("issues=%#v want none for nominal config", issues)
+	}
+}
+
+func TestEvaluateNvidiaGPUConfigSmallPowerLimitDeviationIsNotFlagged(t *testing.T) {
+	// Power limits set slightly below default (e.g. rounding, minor
+	// operator tuning) should not be treated as an anomaly — only a
+	// meaningful cap (>5%) is.
+	g := NvidiaGPUSetting{Index: 0, Name: "H100", ECCCurrent: "Enabled", PowerLimitW: 690, PowerDefaultLimitW: 700}
+	issues := evaluateNvidiaGPUConfig(g)
+	if len(issues) != 0 {
+		t.Fatalf("issues=%#v want none for a <2%% power-limit deviation", issues)
+	}
+}
+
+func TestRenderNvidiaConfigCheckSummaryOverallStatus(t *testing.T) {
+	clean := NvidiaConfigCheckStatus{}
+	if got := renderNvidiaConfigCheckSummary(clean); !strings.Contains(got, "overall_status=OK") {
+		t.Fatalf("clean status summary missing overall_status=OK:\n%s", got)
+	}
+	withWarning := NvidiaConfigCheckStatus{Warnings: []string{"GPU 0: ECC is disabled"}}
+	if got := renderNvidiaConfigCheckSummary(withWarning); !strings.Contains(got, "overall_status=FAILED") {
+		t.Fatalf("status with warnings missing overall_status=FAILED:\n%s", got)
+	}
+}