platform/webui: fold confidential-computing into a GPU config + NVLink check
The standalone "confidential-computing" SAT target only ever checked CC readiness, which most fleets never opt into (a NOT_READY verdict there isn't a fault). Meanwhile DCGM diag never asserts GPU config compliance (ECC/MIG/power-limit vs factory default) or NVLink topology (per NVIDIA's own DGX BasePOD deployment guide, this needs a separate validation step) — gaps confirmed against public DCGM docs and a real NV17-vs-expected-NV18 bonded pair found on a live bundle. Repurposes the routine into "nvidia-config": reuses the existing ListNvidiaGPUSettings() (already backing the GPU-settings page) to flag ECC disabled, a MIG mode change stuck pending a reset/reboot, and a power limit capped >5% below default; parses "nvidia-smi topo -m" bonded pairs against "nvlink -s/-e" to flag any inactive lane or nonzero replay/ recovery/CRC counter on an otherwise-active bond. CC readiness is folded in as one informational field (does not gate overall_status) rather than a dedicated test. Reports under the same pcie:gpu:nvidia severity key as every other nvidia-* SAT target instead of an isolated key, so a config/NVLink FAILED result isn't invisible next to stress-test results. Also fixes ApplySATResultToDB silently dropping any target with no matching switch case (exactly what the old confidential-computing target did) with a new coverage test enumerating every real SAT target. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
cc3997f7b1
commit
cfaa15ec7c
@@ -135,7 +135,7 @@ func defaultTaskPriority(target string, params taskParams) int {
|
||||
return taskPriorityBurn
|
||||
case "nvidia", "nvidia-targeted-stress", "nvidia-targeted-power", "nvidia-pulse",
|
||||
"nvidia-interconnect", "nvidia-bandwidth", "memory", "storage", "cpu",
|
||||
"amd", "amd-mem", "amd-bandwidth", "confidential-computing":
|
||||
"amd", "amd-mem", "amd-bandwidth", "nvidia-config":
|
||||
if params.StressMode {
|
||||
return taskPriorityValidateStress
|
||||
}
|
||||
|
||||
@@ -676,10 +676,10 @@ func renderCheck(opts HandlerOptions) string {
|
||||
`<code>lsblk</code>; NVMe: <code>nvme id-ctrl</code>, <code>nvme smart-log</code>; SATA/SAS: <code>smartctl -H -A</code>`,
|
||||
`Seconds — instantaneous device query, no wear counters incremented.`,
|
||||
)) +
|
||||
renderSATCard("confidential-computing", "Confidential Computing", "runSAT('confidential-computing')", "", renderValidateCardBody(
|
||||
renderSATCard("nvidia-config", "GPU Config & NVLink", "runSAT('nvidia-config')", "", renderValidateCardBody(
|
||||
inv.NVIDIA,
|
||||
`Checks whether this server can run NVIDIA Confidential Computing: CPU TEE support (Intel TDX / AMD SEV-SNP) and GPU firmware CC capability. Read-only — changes nothing.`,
|
||||
`<code>nvidia-smi conf-compute -q</code>, <code>dmesg</code>, <code>/sys/module/kvm_amd/parameters/*</code>`,
|
||||
`Checks GPU configuration and NVLink topology that DCGM diag does not cover: ECC/MIG/power-limit drift from factory default, NVLink-bonded pair link count and error counters, and (informational) NVIDIA Confidential Computing readiness (CPU TEE support + GPU firmware CC capability). Read-only — changes nothing.`,
|
||||
`<code>nvidia-smi --query-gpu=...</code>, <code>nvidia-smi topo -m</code>, <code>nvidia-smi nvlink -s/-e</code>, <code>nvidia-smi conf-compute -q</code>, <code>dmesg</code>`,
|
||||
`Seconds — read-only query only.`,
|
||||
)) +
|
||||
`</div>
|
||||
@@ -743,7 +743,7 @@ func renderCheck(opts HandlerOptions) string {
|
||||
<script>
|
||||
let satES = null;
|
||||
function satLabels() {
|
||||
return {nvidia:'Check GPU (DCGM L2)', 'nvidia-interconnect':'NVIDIA Interconnect (NCCL)', 'nvidia-bandwidth':'NVIDIA Bandwidth (NVBandwidth)', memory:'Check Memory', storage:'Check Storage', cpu:'Check CPU', amd:'Check AMD GPU', 'amd-mem':'AMD GPU MEM Integrity', 'amd-bandwidth':'AMD GPU MEM Bandwidth', 'confidential-computing':'Check Confidential Computing'};
|
||||
return {nvidia:'Check GPU (DCGM L2)', 'nvidia-interconnect':'NVIDIA Interconnect (NCCL)', 'nvidia-bandwidth':'NVIDIA Bandwidth (NVBandwidth)', memory:'Check Memory', storage:'Check Storage', cpu:'Check CPU', amd:'Check AMD GPU', 'amd-mem':'AMD GPU MEM Integrity', 'amd-bandwidth':'AMD GPU MEM Bandwidth', 'nvidia-config':'Check GPU Config & NVLink'};
|
||||
}
|
||||
let satNvidiaGPUsPromise = null;
|
||||
function loadSatNvidiaGPUs() {
|
||||
@@ -879,7 +879,7 @@ function runAllCheckSAT() {
|
||||
status.textContent = 'Enqueuing...';
|
||||
const nvidiaIndices = satSelectedGPUIndices();
|
||||
const nvidiaAllTargets = ['nvidia', 'nvidia-interconnect', 'nvidia-bandwidth'];
|
||||
const baseTargets = ['cpu', 'memory', 'storage', 'confidential-computing'];
|
||||
const baseTargets = ['cpu', 'memory', 'storage', 'nvidia-config'];
|
||||
const amdTargets = selectedAMDValidateTargets();
|
||||
const expanded = [];
|
||||
baseTargets.forEach(t => expanded.push({target: t}));
|
||||
|
||||
@@ -264,7 +264,7 @@ func NewHandler(opts HandlerOptions) http.Handler {
|
||||
mux.HandleFunc("POST /api/sat/nvidia-stress/run", h.handleAPISATRun("nvidia-stress"))
|
||||
mux.HandleFunc("POST /api/sat/memory/run", h.handleAPISATRun("memory"))
|
||||
mux.HandleFunc("POST /api/sat/storage/run", h.handleAPISATRun("storage"))
|
||||
mux.HandleFunc("POST /api/sat/confidential-computing/run", h.handleAPISATRun("confidential-computing"))
|
||||
mux.HandleFunc("POST /api/sat/nvidia-config/run", h.handleAPISATRun("nvidia-config"))
|
||||
mux.HandleFunc("POST /api/sat/cpu/run", h.handleAPISATRun("cpu"))
|
||||
mux.HandleFunc("POST /api/sat/amd/run", h.handleAPISATRun("amd"))
|
||||
mux.HandleFunc("POST /api/sat/amd-mem/run", h.handleAPISATRun("amd-mem"))
|
||||
|
||||
@@ -272,12 +272,12 @@ func executeTaskWithOptions(opts *HandlerOptions, t *Task, j *jobState, ctx cont
|
||||
break
|
||||
}
|
||||
archive, err = runStorageAcceptancePackCtx(a, ctx, "", t.params.StressMode, j.append)
|
||||
case "confidential-computing":
|
||||
case "nvidia-config":
|
||||
if a == nil {
|
||||
err = fmt.Errorf("app not configured")
|
||||
break
|
||||
}
|
||||
archive, err = runConfidentialComputingCheckPackCtx(a, ctx, "", j.append)
|
||||
archive, err = runNvidiaConfigCheckPackCtx(a, ctx, "", j.append)
|
||||
case "cpu":
|
||||
if a == nil {
|
||||
err = fmt.Errorf("app not configured")
|
||||
|
||||
@@ -45,7 +45,7 @@ var taskNames = map[string]string{
|
||||
"nvidia-stress": "NVIDIA GPU Stress",
|
||||
"memory": "Memory SAT",
|
||||
"storage": "Storage SAT",
|
||||
"confidential-computing": "Confidential Computing Check",
|
||||
"nvidia-config": "GPU Config & NVLink Check",
|
||||
"cpu": "CPU SAT",
|
||||
"amd": "AMD GPU SAT",
|
||||
"amd-mem": "AMD GPU MEM Integrity",
|
||||
@@ -313,8 +313,8 @@ var (
|
||||
runStorageAcceptancePackCtx = func(a *app.App, ctx context.Context, baseDir string, extended bool, logFunc func(string)) (string, error) {
|
||||
return a.RunStorageAcceptancePackCtx(ctx, baseDir, extended, logFunc)
|
||||
}
|
||||
runConfidentialComputingCheckPackCtx = func(a *app.App, ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
|
||||
return a.RunConfidentialComputingCheckPackCtx(ctx, baseDir, logFunc)
|
||||
runNvidiaConfigCheckPackCtx = func(a *app.App, ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
|
||||
return a.RunNvidiaConfigCheckPackCtx(ctx, baseDir, logFunc)
|
||||
}
|
||||
runCPUAcceptancePackCtx = func(a *app.App, ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
|
||||
return a.RunCPUAcceptancePackCtx(ctx, baseDir, durationSec, logFunc)
|
||||
@@ -1029,12 +1029,12 @@ func (q *taskQueue) runTask(t *Task, j *jobState, ctx context.Context) {
|
||||
break
|
||||
}
|
||||
archive, err = runStorageAcceptancePackCtx(a, ctx, "", t.params.StressMode, j.append)
|
||||
case "confidential-computing":
|
||||
case "nvidia-config":
|
||||
if a == nil {
|
||||
err = fmt.Errorf("app not configured")
|
||||
break
|
||||
}
|
||||
archive, err = runConfidentialComputingCheckPackCtx(a, ctx, "", j.append)
|
||||
archive, err = runNvidiaConfigCheckPackCtx(a, ctx, "", j.append)
|
||||
case "cpu":
|
||||
if a == nil {
|
||||
err = fmt.Errorf("app not configured")
|
||||
|
||||
Reference in New Issue
Block a user