platform/webui: fold confidential-computing into a GPU config + NVLink check

The standalone "confidential-computing" SAT target only ever checked CC
readiness, which most fleets never opt into (a NOT_READY verdict there
isn't a fault). Meanwhile DCGM diag never asserts GPU config compliance
(ECC/MIG/power-limit vs factory default) or NVLink topology (per NVIDIA's
own DGX BasePOD deployment guide, this needs a separate validation step)
— gaps confirmed against public DCGM docs and a real NV17-vs-expected-NV18
bonded pair found on a live bundle.

Repurposes the routine into "nvidia-config": reuses the existing
ListNvidiaGPUSettings() (already backing the GPU-settings page) to flag
ECC disabled, a MIG mode change stuck pending a reset/reboot, and a power
limit capped >5% below default; parses "nvidia-smi topo -m" bonded pairs
against "nvlink -s/-e" to flag any inactive lane or nonzero replay/
recovery/CRC counter on an otherwise-active bond. CC readiness is folded
in as one informational field (does not gate overall_status) rather than
a dedicated test. Reports under the same pcie:gpu:nvidia severity key as
every other nvidia-* SAT target instead of an isolated key, so a
config/NVLink FAILED result isn't invisible next to stress-test results.

Also fixes ApplySATResultToDB silently dropping any target with no
matching switch case (exactly what the old confidential-computing target
did) with a new coverage test enumerating every real SAT target.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-07-09 11:36:52 +03:00
co-authored by Claude Sonnet 5
parent cc3997f7b1
commit cfaa15ec7c
14 changed files with 811 additions and 273 deletions
@@ -98,6 +98,84 @@ func TestApplyComponentStatusDBMatchesGPUByVendor(t *testing.T) {
func strPtr(s string) *string { return &s }
// TestApplySATResultToDBCoversAllHealthCheckTargets guards against a target
// silently falling through ApplySATResultToDB's switch with no matching
// case — exactly what happened to the old "confidential-computing" target
// before it was folded into "nvidia-config" (task ran, produced a valid
// summary.txt, but no component-status.json record was ever written for it,
// so a /topo card or any other consumer had no way to know the check had
// even run). Every target below is a real health/acceptance check target
// dispatched by task_runner.go's executeTaskWithOptions that produces a
// summary.txt with overall_status; each must land somewhere in the DB.
func TestApplySATResultToDBCoversAllHealthCheckTargets(t *testing.T) {
cases := []struct {
target string
wantKey string
}{
{"nvidia", "pcie:gpu:nvidia"},
{"nvidia-stress", "pcie:gpu:nvidia"},
{"nvidia-targeted-stress", "pcie:gpu:nvidia"},
{"nvidia-compute", "pcie:gpu:nvidia"},
{"nvidia-targeted-power", "pcie:gpu:nvidia"},
{"nvidia-pulse", "pcie:gpu:nvidia"},
{"nvidia-interconnect", "pcie:gpu:nvidia"},
{"nvidia-bandwidth", "pcie:gpu:nvidia"},
{"nvidia-config", "pcie:gpu:nvidia"},
{"amd", "pcie:gpu:amd"},
{"amd-stress", "pcie:gpu:amd"},
{"amd-mem", "pcie:gpu:amd"},
{"amd-bandwidth", "pcie:gpu:amd"},
{"memory", "memory:all"},
{"memory-stress", "memory:all"},
{"sat-stress", "memory:all"},
{"cpu", "cpu:all"},
{"platform-stress", "cpu:all"},
{"storage", "storage:all"},
}
for _, tc := range cases {
t.Run(tc.target, func(t *testing.T) {
db, err := OpenComponentStatusDB(filepath.Join(t.TempDir(), "component-status.json"))
if err != nil {
t.Fatal(err)
}
ApplySATResultToDB(db, tc.target, writeSATSummary(t, "OK"))
rec, ok := db.Get(tc.wantKey)
if !ok {
t.Fatalf("target %q wrote no record under key %q — falls through the switch silently", tc.target, tc.wantKey)
}
if rec.Status != "OK" {
t.Fatalf("target %q key %q status=%q want OK", tc.target, tc.wantKey, rec.Status)
}
})
}
}
// TestApplySATResultToDBNvidiaConfigSharesGPUKeyWithOtherNvidiaTargets
// confirms "nvidia-config" (GPU config/NVLink/CC check — confidential
// computing readiness folded in here rather than run as its own standalone
// SAT target) reports under the same pcie:gpu:nvidia key as every other
// nvidia-* target, so a config/NVLink FAILED result isn't invisible next to
// the plain stress-test results, and a clean run afterward doesn't silently
// erase an earlier failure (severity merge, same as
// TestApplySATResultToDBNormalizesGPUKeyByVendor above).
func TestApplySATResultToDBNvidiaConfigSharesGPUKeyWithOtherNvidiaTargets(t *testing.T) {
db, err := OpenComponentStatusDB(filepath.Join(t.TempDir(), "component-status.json"))
if err != nil {
t.Fatal(err)
}
ApplySATResultToDB(db, "nvidia-config", writeSATSummary(t, "FAILED"))
ApplySATResultToDB(db, "nvidia", writeSATSummary(t, "OK"))
rec, ok := db.Get("pcie:gpu:nvidia")
if !ok {
t.Fatalf("expected pcie:gpu:nvidia record to exist")
}
if rec.Status != "Warning" {
t.Fatalf("status=%q, want Warning (nvidia-config FAILED must survive the later OK nvidia run)", rec.Status)
}
}
// TestRecordDeduplicatesRepeatedIdenticalStatusFromSameSource guards the
// hardware-ingest-contract.md rule that status_history is a transition log
// ("История переходов статусов"), not a per-poll journal. A component