platform/webui: fold confidential-computing into a GPU config + NVLink check

The standalone "confidential-computing" SAT target only ever checked CC
readiness, which most fleets never opt into (a NOT_READY verdict there
isn't a fault). Meanwhile DCGM diag never asserts GPU config compliance
(ECC/MIG/power-limit vs factory default) or NVLink topology (per NVIDIA's
own DGX BasePOD deployment guide, this needs a separate validation step)
— gaps confirmed against public DCGM docs and a real NV17-vs-expected-NV18
bonded pair found on a live bundle.

Repurposes the routine into "nvidia-config": reuses the existing
ListNvidiaGPUSettings() (already backing the GPU-settings page) to flag
ECC disabled, a MIG mode change stuck pending a reset/reboot, and a power
limit capped >5% below default; parses "nvidia-smi topo -m" bonded pairs
against "nvlink -s/-e" to flag any inactive lane or nonzero replay/
recovery/CRC counter on an otherwise-active bond. CC readiness is folded
in as one informational field (does not gate overall_status) rather than
a dedicated test. Reports under the same pcie:gpu:nvidia severity key as
every other nvidia-* SAT target instead of an isolated key, so a
config/NVLink FAILED result isn't invisible next to stress-test results.

Also fixes ApplySATResultToDB silently dropping any target with no
matching switch case (exactly what the old confidential-computing target
did) with a new coverage test enumerating every real SAT target.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-07-09 11:36:52 +03:00
co-authored by Claude Sonnet 5
parent cc3997f7b1
commit cfaa15ec7c
14 changed files with 811 additions and 273 deletions
+1 -1
View File
@@ -134,7 +134,7 @@ type satRunner interface {
ResetNvidiaGPU(index int) (string, error)
RunMemoryAcceptancePack(ctx context.Context, baseDir string, sizeMB, passes int, logFunc func(string)) (string, error)
RunStorageAcceptancePack(ctx context.Context, baseDir string, extended bool, logFunc func(string)) (string, error)
RunConfidentialComputingCheckPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error)
RunNvidiaConfigCheckPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error)
RunCPUAcceptancePack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error)
ListNvidiaGPUs() ([]platform.NvidiaGPU, error)
ListNvidiaGPUSettings() ([]platform.NvidiaGPUSetting, error)
+7 -7
View File
@@ -247,20 +247,20 @@ func (a *App) RunStorageAcceptancePackResult(baseDir string) (ActionResult, erro
return ActionResult{Title: "Storage SAT", Body: satResultBody(path)}, err
}
func (a *App) RunConfidentialComputingCheckPackCtx(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
func (a *App) RunNvidiaConfigCheckPackCtx(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
if strings.TrimSpace(baseDir) == "" {
baseDir = DefaultSATBaseDir
}
return a.sat.RunConfidentialComputingCheckPack(ctx, baseDir, logFunc)
return a.sat.RunNvidiaConfigCheckPack(ctx, baseDir, logFunc)
}
func (a *App) RunConfidentialComputingCheckPack(baseDir string, logFunc func(string)) (string, error) {
return a.RunConfidentialComputingCheckPackCtx(context.Background(), baseDir, logFunc)
func (a *App) RunNvidiaConfigCheckPack(baseDir string, logFunc func(string)) (string, error) {
return a.RunNvidiaConfigCheckPackCtx(context.Background(), baseDir, logFunc)
}
func (a *App) RunConfidentialComputingCheckPackResult(baseDir string) (ActionResult, error) {
path, err := a.RunConfidentialComputingCheckPack(baseDir, nil)
return ActionResult{Title: "Confidential Computing Check", Body: satResultBody(path)}, err
func (a *App) RunNvidiaConfigCheckPackResult(baseDir string) (ActionResult, error) {
path, err := a.RunNvidiaConfigCheckPack(baseDir, nil)
return ActionResult{Title: "GPU Config & NVLink Check", Body: satResultBody(path)}, err
}
func (a *App) DetectGPUVendor() string {
+1 -1
View File
@@ -267,7 +267,7 @@ func (f fakeSAT) RunStorageAcceptancePack(_ context.Context, baseDir string, _ b
return f.runStorageFn(baseDir)
}
func (f fakeSAT) RunConfidentialComputingCheckPack(_ context.Context, baseDir string, _ func(string)) (string, error) {
func (f fakeSAT) RunNvidiaConfigCheckPack(_ context.Context, baseDir string, _ func(string)) (string, error) {
return "", nil
}
+1 -1
View File
@@ -243,7 +243,7 @@ func ApplySATResultToDB(db *ComponentStatusDB, target, archivePath string) {
// otherwise fails to match any real BDF.
switch target {
case "nvidia", "nvidia-targeted-stress", "nvidia-compute", "nvidia-targeted-power", "nvidia-pulse",
"nvidia-interconnect", "nvidia-bandwidth", "nvidia-stress":
"nvidia-interconnect", "nvidia-bandwidth", "nvidia-stress", "nvidia-config":
db.Record("pcie:gpu:nvidia", source, dbStatus, target+" SAT: "+overall)
case "amd", "amd-stress", "amd-mem", "amd-bandwidth":
db.Record("pcie:gpu:amd", source, dbStatus, target+" SAT: "+overall)
@@ -98,6 +98,84 @@ func TestApplyComponentStatusDBMatchesGPUByVendor(t *testing.T) {
func strPtr(s string) *string { return &s }
// TestApplySATResultToDBCoversAllHealthCheckTargets guards against a target
// silently falling through ApplySATResultToDB's switch with no matching
// case — exactly what happened to the old "confidential-computing" target
// before it was folded into "nvidia-config" (task ran, produced a valid
// summary.txt, but no component-status.json record was ever written for it,
// so a /topo card or any other consumer had no way to know the check had
// even run). Every target below is a real health/acceptance check target
// dispatched by task_runner.go's executeTaskWithOptions that produces a
// summary.txt with overall_status; each must land somewhere in the DB.
func TestApplySATResultToDBCoversAllHealthCheckTargets(t *testing.T) {
cases := []struct {
target string
wantKey string
}{
{"nvidia", "pcie:gpu:nvidia"},
{"nvidia-stress", "pcie:gpu:nvidia"},
{"nvidia-targeted-stress", "pcie:gpu:nvidia"},
{"nvidia-compute", "pcie:gpu:nvidia"},
{"nvidia-targeted-power", "pcie:gpu:nvidia"},
{"nvidia-pulse", "pcie:gpu:nvidia"},
{"nvidia-interconnect", "pcie:gpu:nvidia"},
{"nvidia-bandwidth", "pcie:gpu:nvidia"},
{"nvidia-config", "pcie:gpu:nvidia"},
{"amd", "pcie:gpu:amd"},
{"amd-stress", "pcie:gpu:amd"},
{"amd-mem", "pcie:gpu:amd"},
{"amd-bandwidth", "pcie:gpu:amd"},
{"memory", "memory:all"},
{"memory-stress", "memory:all"},
{"sat-stress", "memory:all"},
{"cpu", "cpu:all"},
{"platform-stress", "cpu:all"},
{"storage", "storage:all"},
}
for _, tc := range cases {
t.Run(tc.target, func(t *testing.T) {
db, err := OpenComponentStatusDB(filepath.Join(t.TempDir(), "component-status.json"))
if err != nil {
t.Fatal(err)
}
ApplySATResultToDB(db, tc.target, writeSATSummary(t, "OK"))
rec, ok := db.Get(tc.wantKey)
if !ok {
t.Fatalf("target %q wrote no record under key %q — falls through the switch silently", tc.target, tc.wantKey)
}
if rec.Status != "OK" {
t.Fatalf("target %q key %q status=%q want OK", tc.target, tc.wantKey, rec.Status)
}
})
}
}
// TestApplySATResultToDBNvidiaConfigSharesGPUKeyWithOtherNvidiaTargets
// confirms "nvidia-config" (GPU config/NVLink/CC check — confidential
// computing readiness folded in here rather than run as its own standalone
// SAT target) reports under the same pcie:gpu:nvidia key as every other
// nvidia-* target, so a config/NVLink FAILED result isn't invisible next to
// the plain stress-test results, and a clean run afterward doesn't silently
// erase an earlier failure (severity merge, same as
// TestApplySATResultToDBNormalizesGPUKeyByVendor above).
func TestApplySATResultToDBNvidiaConfigSharesGPUKeyWithOtherNvidiaTargets(t *testing.T) {
db, err := OpenComponentStatusDB(filepath.Join(t.TempDir(), "component-status.json"))
if err != nil {
t.Fatal(err)
}
ApplySATResultToDB(db, "nvidia-config", writeSATSummary(t, "FAILED"))
ApplySATResultToDB(db, "nvidia", writeSATSummary(t, "OK"))
rec, ok := db.Get("pcie:gpu:nvidia")
if !ok {
t.Fatalf("expected pcie:gpu:nvidia record to exist")
}
if rec.Status != "Warning" {
t.Fatalf("status=%q, want Warning (nvidia-config FAILED must survive the later OK nvidia run)", rec.Status)
}
}
// TestRecordDeduplicatesRepeatedIdenticalStatusFromSameSource guards the
// hardware-ingest-contract.md rule that status_history is a transition log
// ("История переходов статусов"), not a per-poll journal. A component