fix(webui): keep Check-only tasks out of Load

This commit is contained in:
Mikhail Chusavitin
2026-09-03 15:41:18 +03:00
parent a9a9b18a8c
commit 677f4f4006
4 changed files with 75 additions and 15 deletions
+14 -9
View File
@@ -103,18 +103,21 @@ func (h *handler) planSATRunAll(ctx context.Context, req satRunAllRequest) ([]sa
satRunAllSpec{target: "storage", params: taskParams{StressMode: req.StressMode}}, satRunAllSpec{target: "storage", params: taskParams{StressMode: req.StressMode}},
) )
if h.opts.App.TPMPresent() { if !req.StressMode {
specs = append(specs, satRunAllSpec{target: "tpm", params: taskParams{}}) if h.opts.App.TPMPresent() {
} else { specs = append(specs, satRunAllSpec{target: "tpm", params: taskParams{}})
skip("TPM: no TPM device on this host; check skipped") } else {
skip("TPM: no TPM device on this host; check skipped")
}
} }
gp := h.opts.App.DetectGPUPresence() gp := h.opts.App.DetectGPUPresence()
if gp.Nvidia || gp.NvidiaInitializing { if gp.Nvidia || gp.NvidiaInitializing {
// nvidia-config only collects inventory and NVLink state; safe to run // nvidia-config is a read-only Check task, not a load test.
// even while the compute stack is still coming up. if !req.StressMode {
specs = append(specs, satRunAllSpec{target: "nvidia-config", params: taskParams{}}) specs = append(specs, satRunAllSpec{target: "nvidia-config", params: taskParams{}})
}
health, gpus, ready := h.waitForNvidiaReady(ctx) health, gpus, ready := h.waitForNvidiaReady(ctx)
switch { switch {
@@ -138,10 +141,12 @@ func (h *handler) planSATRunAll(ctx context.Context, req satRunAllRequest) ([]sa
if !health.CUDAReady { if !health.CUDAReady {
notes = append(notes, "NVIDIA: CUDA runtime not confirmed ready; GPU tests queued anyway") notes = append(notes, "NVIDIA: CUDA runtime not confirmed ready; GPU tests queued anyway")
} }
gpuTargets := []string{"nvidia", "nvidia-interconnect", "nvidia-bandwidth"} gpuTargets := []string{"nvidia", "nvidia-bandwidth"}
if req.StressMode { if req.StressMode {
// Stress tier adds the targeted dcgmi diag load tests. // Stress tier adds the targeted dcgmi diag load tests.
gpuTargets = append(gpuTargets, "nvidia-targeted-stress", "nvidia-targeted-power", "nvidia-pulse") gpuTargets = append(gpuTargets, "nvidia-targeted-stress", "nvidia-targeted-power", "nvidia-pulse")
} else {
gpuTargets = append(gpuTargets, "nvidia-interconnect")
} }
for _, target := range gpuTargets { for _, target := range gpuTargets {
specs = append(specs, satRunAllSpec{ specs = append(specs, satRunAllSpec{
@@ -152,7 +157,7 @@ func (h *handler) planSATRunAll(ctx context.Context, req satRunAllRequest) ([]sa
} }
} }
if gp.AMD { if gp.AMD && !req.StressMode {
for _, target := range req.AMDTargets { for _, target := range req.AMDTargets {
switch target { switch target {
case "amd", "amd-mem", "amd-bandwidth": case "amd", "amd-mem", "amd-bandwidth":
@@ -3,6 +3,7 @@ package webui
import ( import (
"context" "context"
"reflect" "reflect"
"strings"
"testing" "testing"
"time" "time"
@@ -88,3 +89,21 @@ func TestPlanSATRunAllNoAcceleratorNoTPM(t *testing.T) {
t.Fatalf("expected a note about TPM being skipped") t.Fatalf("expected a note about TPM being skipped")
} }
} }
func TestPlanSATRunAllLoadOmitsReadOnlyTPMCheck(t *testing.T) {
h := &handler{opts: HandlerOptions{App: app.New(&platform.System{})}}
specs, notes := h.planSATRunAll(context.Background(), satRunAllRequest{StressMode: true})
var targets []string
for _, s := range specs {
targets = append(targets, s.target)
}
if want := []string{"cpu", "memory", "storage"}; !reflect.DeepEqual(targets, want) {
t.Fatalf("targets=%v want %v", targets, want)
}
for _, note := range notes {
if strings.Contains(note, "TPM") {
t.Fatalf("load plan must not inspect or report TPM: notes=%v", notes)
}
}
}
+13 -6
View File
@@ -149,12 +149,12 @@ func renderValidateMode(opts HandlerOptions, stressDefault bool) string {
`<code>lsblk</code>; NVMe: <code>nvme id-ctrl</code>, <code>nvme smart-log</code>, <code>nvme device-self-test -s 1</code>; SATA/SAS: <code>smartctl -H -A</code>, <code>smartctl -t short</code>`, `<code>lsblk</code>; NVMe: <code>nvme id-ctrl</code>, <code>nvme smart-log</code>, <code>nvme device-self-test -s 1</code>; SATA/SAS: <code>smartctl -H -A</code>, <code>smartctl -t short</code>`,
`~2 min per device (NVMe short self-test; SATA/SAS short self-test — duration device-dependent).`, `~2 min per device (NVMe short self-test; SATA/SAS short self-test — duration device-dependent).`,
)) + )) +
renderSATCard("tpm", "TPM", "runSAT('tpm')", "", renderValidateCardBody( renderCheckOnlySATCard(stressDefault, renderSATCard("tpm", "TPM", "runSAT('tpm')", "", renderValidateCardBody(
inv.TPM, inv.TPM,
`Checks TPM 2.0 communication and reports its fixed properties, allocated PCR banks, current PCR values, and the result of self-tests already performed by the TPM. It does not start a new self-test or change TPM state.`, `Checks TPM 2.0 communication and reports its fixed properties, allocated PCR banks, current PCR values, and the result of self-tests already performed by the TPM. It does not start a new self-test or change TPM state.`,
`<code>tpm2_getcap properties-fixed</code>, <code>tpm2_getcap pcrs</code>, <code>tpm2_pcrread</code>, <code>tpm2_gettestresult</code>`, `<code>tpm2_getcap properties-fixed</code>, <code>tpm2_getcap pcrs</code>, <code>tpm2_pcrread</code>, <code>tpm2_gettestresult</code>`,
`Seconds - read-only queries; no ownership, NV, PCR, or key changes.`, `Seconds - read-only queries; no ownership, NV, PCR, or key changes.`,
)) + ))) +
`</div> `</div>
<div style="height:1px;background:var(--border);margin:16px 0"></div> <div style="height:1px;background:var(--border);margin:16px 0"></div>
<div class="card" style="margin-bottom:16px"> <div class="card" style="margin-bottom:16px">
@@ -183,12 +183,12 @@ func renderValidateMode(opts HandlerOptions, stressDefault bool) string {
validateFmtDur(platform.SATEstimatedNvidiaGPUStressSec)), validateFmtDur(platform.SATEstimatedNvidiaGPUStressSec)),
)) + )) +
stressOnlyCards + stressOnlyCards +
renderSATCard("nvidia-interconnect", "NVIDIA Interconnect (NCCL)", "runNvidiaFabricValidate('nvidia-interconnect')", "", renderValidateCardBody( renderCheckOnlySATCard(stressDefault, renderSATCard("nvidia-interconnect", "NVIDIA Interconnect (NCCL)", "runNvidiaFabricValidate('nvidia-interconnect')", "", renderValidateCardBody(
inv.NVIDIA, inv.NVIDIA,
`Verifies NVLink/NVSwitch fabric bandwidth using NCCL all_reduce_perf across all selected GPUs. Pass/fail based on achieved bandwidth vs. theoretical.`, `Verifies NVLink/NVSwitch fabric bandwidth using NCCL all_reduce_perf across all selected GPUs. Pass/fail based on achieved bandwidth vs. theoretical.`,
`<code>all_reduce_perf</code> (NCCL tests)`, `<code>all_reduce_perf</code> (NCCL tests)`,
validateFmtDur(platform.SATEstimatedNvidiaInterconnectSec)+` (all GPUs simultaneously, requires ≥2).`, validateFmtDur(platform.SATEstimatedNvidiaInterconnectSec)+` (all GPUs simultaneously, requires ≥2).`,
)) + ))) +
renderSATCard("nvidia-bandwidth", "NVIDIA Bandwidth + PCIe Link", "runNvidiaFabricValidate('nvidia-bandwidth')", "", renderValidateCardBody( renderSATCard("nvidia-bandwidth", "NVIDIA Bandwidth + PCIe Link", "runNvidiaFabricValidate('nvidia-bandwidth')", "", renderValidateCardBody(
inv.NVIDIA, inv.NVIDIA,
`Validates GPU memory copy and peer-to-peer bandwidth paths, then samples each GPU's negotiated PCIe speed and width immediately after real traffic. Idle or forced-retrain Gen1 readings do not fail the test.`, `Validates GPU memory copy and peer-to-peer bandwidth paths, then samples each GPU's negotiated PCIe speed and width immediately after real traffic. Idle or forced-retrain Gen1 readings do not fail the test.`,
@@ -197,12 +197,12 @@ func renderValidateMode(opts HandlerOptions, stressDefault bool) string {
)) + )) +
`</div> `</div>
<div class="grid3" style="margin-top:16px"> <div class="grid3" style="margin-top:16px">
` + renderSATCard("amd", "AMD GPU", "runAMDValidateSet()", "", renderValidateCardBody( ` + renderCheckOnlySATCard(stressDefault, renderSATCard("amd", "AMD GPU", "runAMDValidateSet()", "", renderValidateCardBody(
inv.AMD, inv.AMD,
`Runs the selected AMD checks only. GPU Validate collects inventory; MEM Integrity uses the RVS MEM module; MEM Bandwidth uses rocm-bandwidth-test and the RVS BABEL module.`, `Runs the selected AMD checks only. GPU Validate collects inventory; MEM Integrity uses the RVS MEM module; MEM Bandwidth uses rocm-bandwidth-test and the RVS BABEL module.`,
`GPU Validate: <code>rocm-smi</code>, <code>dmidecode</code>; MEM Integrity: <code>rvs mem</code>; MEM Bandwidth: <code>rocm-bandwidth-test</code>, <code>rvs babel</code>`, `GPU Validate: <code>rocm-smi</code>, <code>dmidecode</code>; MEM Integrity: <code>rvs mem</code>; MEM Bandwidth: <code>rocm-bandwidth-test</code>, <code>rvs babel</code>`,
`<div style="display:flex;flex-direction:column;gap:4px"><label class="cb-row"><input type="checkbox" id="sat-amd-target" checked><span>GPU Validate</span></label><label class="cb-row"><input type="checkbox" id="sat-amd-mem-target" checked><span>MEM Integrity</span></label><label class="cb-row"><input type="checkbox" id="sat-amd-bandwidth-target" checked><span>MEM Bandwidth</span></label></div>`, `<div style="display:flex;flex-direction:column;gap:4px"><label class="cb-row"><input type="checkbox" id="sat-amd-target" checked><span>GPU Validate</span></label><label class="cb-row"><input type="checkbox" id="sat-amd-mem-target" checked><span>MEM Integrity</span></label><label class="cb-row"><input type="checkbox" id="sat-amd-bandwidth-target" checked><span>MEM Bandwidth</span></label></div>`,
)) + ))) +
`</div> `</div>
<div id="sat-output" style="display:none;margin-top:16px" class="card"> <div id="sat-output" style="display:none;margin-top:16px" class="card">
<div class="card-head">Test Output <span id="sat-title"></span></div> <div class="card-head">Test Output <span id="sat-title"></span></div>
@@ -620,6 +620,13 @@ func validateIsVendorGPU(dev schema.HardwarePCIeDevice, vendor string) bool {
} }
} }
func renderCheckOnlySATCard(stressMode bool, card string) string {
if stressMode {
return ""
}
return card
}
// renderCheck renders the non-destructive Check page (step 2). // renderCheck renders the non-destructive Check page (step 2).
// Shows validate-mode tests only: CPU, Memory, Storage, NVIDIA L2, NCCL, NVBandwidth, AMD. // Shows validate-mode tests only: CPU, Memory, Storage, NVIDIA L2, NCCL, NVBandwidth, AMD.
// Stress-mode tests (targeted-stress, targeted-power, pulse) are on the Load page. // Stress-mode tests (targeted-stress, targeted-power, pulse) are on the Load page.
@@ -46,3 +46,32 @@ func TestRenderCheckIncludesReadOnlyTPMValidation(t *testing.T) {
t.Fatal("check page must not offer tpm2_selftest") t.Fatal("check page must not offer tpm2_selftest")
} }
} }
func TestRenderLoadOmitsExactCheckDuplicates(t *testing.T) {
t.Parallel()
page := renderValidateMode(HandlerOptions{}, true)
for _, duplicateID := range []string{
`id="sat-btn-tpm"`,
`id="sat-btn-nvidia-interconnect"`,
`id="sat-btn-amd"`,
} {
if strings.Contains(page, duplicateID) {
t.Fatalf("load page contains Check-only card %q", duplicateID)
}
}
for _, loadID := range []string{
`id="sat-btn-cpu"`,
`id="sat-btn-memory"`,
`id="sat-btn-storage"`,
`id="sat-btn-nvidia"`,
`id="sat-btn-nvidia-bandwidth"`,
`id="sat-btn-nvidia-targeted-stress"`,
`id="sat-btn-nvidia-targeted-power"`,
`id="sat-btn-nvidia-pulse"`,
} {
if !strings.Contains(page, loadID) {
t.Fatalf("load page is missing load-specific card %q", loadID)
}
}
}