platform/webui: fold confidential-computing into a GPU config + NVLink check
The standalone "confidential-computing" SAT target only ever checked CC readiness, which most fleets never opt into (a NOT_READY verdict there isn't a fault). Meanwhile DCGM diag never asserts GPU config compliance (ECC/MIG/power-limit vs factory default) or NVLink topology (per NVIDIA's own DGX BasePOD deployment guide, this needs a separate validation step) — gaps confirmed against public DCGM docs and a real NV17-vs-expected-NV18 bonded pair found on a live bundle. Repurposes the routine into "nvidia-config": reuses the existing ListNvidiaGPUSettings() (already backing the GPU-settings page) to flag ECC disabled, a MIG mode change stuck pending a reset/reboot, and a power limit capped >5% below default; parses "nvidia-smi topo -m" bonded pairs against "nvlink -s/-e" to flag any inactive lane or nonzero replay/ recovery/CRC counter on an otherwise-active bond. CC readiness is folded in as one informational field (does not gate overall_status) rather than a dedicated test. Reports under the same pcie:gpu:nvidia severity key as every other nvidia-* SAT target instead of an isolated key, so a config/NVLink FAILED result isn't invisible next to stress-test results. Also fixes ApplySATResultToDB silently dropping any target with no matching switch case (exactly what the old confidential-computing target did) with a new coverage test enumerating every real SAT target. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
cc3997f7b1
commit
cfaa15ec7c
@@ -98,6 +98,84 @@ func TestApplyComponentStatusDBMatchesGPUByVendor(t *testing.T) {
|
||||
|
||||
func strPtr(s string) *string { return &s }
|
||||
|
||||
// TestApplySATResultToDBCoversAllHealthCheckTargets guards against a target
|
||||
// silently falling through ApplySATResultToDB's switch with no matching
|
||||
// case — exactly what happened to the old "confidential-computing" target
|
||||
// before it was folded into "nvidia-config" (task ran, produced a valid
|
||||
// summary.txt, but no component-status.json record was ever written for it,
|
||||
// so a /topo card or any other consumer had no way to know the check had
|
||||
// even run). Every target below is a real health/acceptance check target
|
||||
// dispatched by task_runner.go's executeTaskWithOptions that produces a
|
||||
// summary.txt with overall_status; each must land somewhere in the DB.
|
||||
func TestApplySATResultToDBCoversAllHealthCheckTargets(t *testing.T) {
|
||||
cases := []struct {
|
||||
target string
|
||||
wantKey string
|
||||
}{
|
||||
{"nvidia", "pcie:gpu:nvidia"},
|
||||
{"nvidia-stress", "pcie:gpu:nvidia"},
|
||||
{"nvidia-targeted-stress", "pcie:gpu:nvidia"},
|
||||
{"nvidia-compute", "pcie:gpu:nvidia"},
|
||||
{"nvidia-targeted-power", "pcie:gpu:nvidia"},
|
||||
{"nvidia-pulse", "pcie:gpu:nvidia"},
|
||||
{"nvidia-interconnect", "pcie:gpu:nvidia"},
|
||||
{"nvidia-bandwidth", "pcie:gpu:nvidia"},
|
||||
{"nvidia-config", "pcie:gpu:nvidia"},
|
||||
{"amd", "pcie:gpu:amd"},
|
||||
{"amd-stress", "pcie:gpu:amd"},
|
||||
{"amd-mem", "pcie:gpu:amd"},
|
||||
{"amd-bandwidth", "pcie:gpu:amd"},
|
||||
{"memory", "memory:all"},
|
||||
{"memory-stress", "memory:all"},
|
||||
{"sat-stress", "memory:all"},
|
||||
{"cpu", "cpu:all"},
|
||||
{"platform-stress", "cpu:all"},
|
||||
{"storage", "storage:all"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.target, func(t *testing.T) {
|
||||
db, err := OpenComponentStatusDB(filepath.Join(t.TempDir(), "component-status.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ApplySATResultToDB(db, tc.target, writeSATSummary(t, "OK"))
|
||||
rec, ok := db.Get(tc.wantKey)
|
||||
if !ok {
|
||||
t.Fatalf("target %q wrote no record under key %q — falls through the switch silently", tc.target, tc.wantKey)
|
||||
}
|
||||
if rec.Status != "OK" {
|
||||
t.Fatalf("target %q key %q status=%q want OK", tc.target, tc.wantKey, rec.Status)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestApplySATResultToDBNvidiaConfigSharesGPUKeyWithOtherNvidiaTargets
|
||||
// confirms "nvidia-config" (GPU config/NVLink/CC check — confidential
|
||||
// computing readiness folded in here rather than run as its own standalone
|
||||
// SAT target) reports under the same pcie:gpu:nvidia key as every other
|
||||
// nvidia-* target, so a config/NVLink FAILED result isn't invisible next to
|
||||
// the plain stress-test results, and a clean run afterward doesn't silently
|
||||
// erase an earlier failure (severity merge, same as
|
||||
// TestApplySATResultToDBNormalizesGPUKeyByVendor above).
|
||||
func TestApplySATResultToDBNvidiaConfigSharesGPUKeyWithOtherNvidiaTargets(t *testing.T) {
|
||||
db, err := OpenComponentStatusDB(filepath.Join(t.TempDir(), "component-status.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ApplySATResultToDB(db, "nvidia-config", writeSATSummary(t, "FAILED"))
|
||||
ApplySATResultToDB(db, "nvidia", writeSATSummary(t, "OK"))
|
||||
|
||||
rec, ok := db.Get("pcie:gpu:nvidia")
|
||||
if !ok {
|
||||
t.Fatalf("expected pcie:gpu:nvidia record to exist")
|
||||
}
|
||||
if rec.Status != "Warning" {
|
||||
t.Fatalf("status=%q, want Warning (nvidia-config FAILED must survive the later OK nvidia run)", rec.Status)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// TestRecordDeduplicatesRepeatedIdenticalStatusFromSameSource guards the
|
||||
// hardware-ingest-contract.md rule that status_history is a transition log
|
||||
// ("История переходов статусов"), not a per-poll journal. A component
|
||||
|
||||
Reference in New Issue
Block a user