fix(webui): keep Check-only tasks out of Load
This commit is contained in:
@@ -103,18 +103,21 @@ func (h *handler) planSATRunAll(ctx context.Context, req satRunAllRequest) ([]sa
|
|||||||
satRunAllSpec{target: "storage", params: taskParams{StressMode: req.StressMode}},
|
satRunAllSpec{target: "storage", params: taskParams{StressMode: req.StressMode}},
|
||||||
)
|
)
|
||||||
|
|
||||||
if h.opts.App.TPMPresent() {
|
if !req.StressMode {
|
||||||
specs = append(specs, satRunAllSpec{target: "tpm", params: taskParams{}})
|
if h.opts.App.TPMPresent() {
|
||||||
} else {
|
specs = append(specs, satRunAllSpec{target: "tpm", params: taskParams{}})
|
||||||
skip("TPM: no TPM device on this host; check skipped")
|
} else {
|
||||||
|
skip("TPM: no TPM device on this host; check skipped")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
gp := h.opts.App.DetectGPUPresence()
|
gp := h.opts.App.DetectGPUPresence()
|
||||||
|
|
||||||
if gp.Nvidia || gp.NvidiaInitializing {
|
if gp.Nvidia || gp.NvidiaInitializing {
|
||||||
// nvidia-config only collects inventory and NVLink state; safe to run
|
// nvidia-config is a read-only Check task, not a load test.
|
||||||
// even while the compute stack is still coming up.
|
if !req.StressMode {
|
||||||
specs = append(specs, satRunAllSpec{target: "nvidia-config", params: taskParams{}})
|
specs = append(specs, satRunAllSpec{target: "nvidia-config", params: taskParams{}})
|
||||||
|
}
|
||||||
|
|
||||||
health, gpus, ready := h.waitForNvidiaReady(ctx)
|
health, gpus, ready := h.waitForNvidiaReady(ctx)
|
||||||
switch {
|
switch {
|
||||||
@@ -138,10 +141,12 @@ func (h *handler) planSATRunAll(ctx context.Context, req satRunAllRequest) ([]sa
|
|||||||
if !health.CUDAReady {
|
if !health.CUDAReady {
|
||||||
notes = append(notes, "NVIDIA: CUDA runtime not confirmed ready; GPU tests queued anyway")
|
notes = append(notes, "NVIDIA: CUDA runtime not confirmed ready; GPU tests queued anyway")
|
||||||
}
|
}
|
||||||
gpuTargets := []string{"nvidia", "nvidia-interconnect", "nvidia-bandwidth"}
|
gpuTargets := []string{"nvidia", "nvidia-bandwidth"}
|
||||||
if req.StressMode {
|
if req.StressMode {
|
||||||
// Stress tier adds the targeted dcgmi diag load tests.
|
// Stress tier adds the targeted dcgmi diag load tests.
|
||||||
gpuTargets = append(gpuTargets, "nvidia-targeted-stress", "nvidia-targeted-power", "nvidia-pulse")
|
gpuTargets = append(gpuTargets, "nvidia-targeted-stress", "nvidia-targeted-power", "nvidia-pulse")
|
||||||
|
} else {
|
||||||
|
gpuTargets = append(gpuTargets, "nvidia-interconnect")
|
||||||
}
|
}
|
||||||
for _, target := range gpuTargets {
|
for _, target := range gpuTargets {
|
||||||
specs = append(specs, satRunAllSpec{
|
specs = append(specs, satRunAllSpec{
|
||||||
@@ -152,7 +157,7 @@ func (h *handler) planSATRunAll(ctx context.Context, req satRunAllRequest) ([]sa
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if gp.AMD {
|
if gp.AMD && !req.StressMode {
|
||||||
for _, target := range req.AMDTargets {
|
for _, target := range req.AMDTargets {
|
||||||
switch target {
|
switch target {
|
||||||
case "amd", "amd-mem", "amd-bandwidth":
|
case "amd", "amd-mem", "amd-bandwidth":
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package webui
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"reflect"
|
"reflect"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -88,3 +89,21 @@ func TestPlanSATRunAllNoAcceleratorNoTPM(t *testing.T) {
|
|||||||
t.Fatalf("expected a note about TPM being skipped")
|
t.Fatalf("expected a note about TPM being skipped")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPlanSATRunAllLoadOmitsReadOnlyTPMCheck(t *testing.T) {
|
||||||
|
h := &handler{opts: HandlerOptions{App: app.New(&platform.System{})}}
|
||||||
|
specs, notes := h.planSATRunAll(context.Background(), satRunAllRequest{StressMode: true})
|
||||||
|
|
||||||
|
var targets []string
|
||||||
|
for _, s := range specs {
|
||||||
|
targets = append(targets, s.target)
|
||||||
|
}
|
||||||
|
if want := []string{"cpu", "memory", "storage"}; !reflect.DeepEqual(targets, want) {
|
||||||
|
t.Fatalf("targets=%v want %v", targets, want)
|
||||||
|
}
|
||||||
|
for _, note := range notes {
|
||||||
|
if strings.Contains(note, "TPM") {
|
||||||
|
t.Fatalf("load plan must not inspect or report TPM: notes=%v", notes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -149,12 +149,12 @@ func renderValidateMode(opts HandlerOptions, stressDefault bool) string {
|
|||||||
`<code>lsblk</code>; NVMe: <code>nvme id-ctrl</code>, <code>nvme smart-log</code>, <code>nvme device-self-test -s 1</code>; SATA/SAS: <code>smartctl -H -A</code>, <code>smartctl -t short</code>`,
|
`<code>lsblk</code>; NVMe: <code>nvme id-ctrl</code>, <code>nvme smart-log</code>, <code>nvme device-self-test -s 1</code>; SATA/SAS: <code>smartctl -H -A</code>, <code>smartctl -t short</code>`,
|
||||||
`~2 min per device (NVMe short self-test; SATA/SAS short self-test — duration device-dependent).`,
|
`~2 min per device (NVMe short self-test; SATA/SAS short self-test — duration device-dependent).`,
|
||||||
)) +
|
)) +
|
||||||
renderSATCard("tpm", "TPM", "runSAT('tpm')", "", renderValidateCardBody(
|
renderCheckOnlySATCard(stressDefault, renderSATCard("tpm", "TPM", "runSAT('tpm')", "", renderValidateCardBody(
|
||||||
inv.TPM,
|
inv.TPM,
|
||||||
`Checks TPM 2.0 communication and reports its fixed properties, allocated PCR banks, current PCR values, and the result of self-tests already performed by the TPM. It does not start a new self-test or change TPM state.`,
|
`Checks TPM 2.0 communication and reports its fixed properties, allocated PCR banks, current PCR values, and the result of self-tests already performed by the TPM. It does not start a new self-test or change TPM state.`,
|
||||||
`<code>tpm2_getcap properties-fixed</code>, <code>tpm2_getcap pcrs</code>, <code>tpm2_pcrread</code>, <code>tpm2_gettestresult</code>`,
|
`<code>tpm2_getcap properties-fixed</code>, <code>tpm2_getcap pcrs</code>, <code>tpm2_pcrread</code>, <code>tpm2_gettestresult</code>`,
|
||||||
`Seconds - read-only queries; no ownership, NV, PCR, or key changes.`,
|
`Seconds - read-only queries; no ownership, NV, PCR, or key changes.`,
|
||||||
)) +
|
))) +
|
||||||
`</div>
|
`</div>
|
||||||
<div style="height:1px;background:var(--border);margin:16px 0"></div>
|
<div style="height:1px;background:var(--border);margin:16px 0"></div>
|
||||||
<div class="card" style="margin-bottom:16px">
|
<div class="card" style="margin-bottom:16px">
|
||||||
@@ -183,12 +183,12 @@ func renderValidateMode(opts HandlerOptions, stressDefault bool) string {
|
|||||||
validateFmtDur(platform.SATEstimatedNvidiaGPUStressSec)),
|
validateFmtDur(platform.SATEstimatedNvidiaGPUStressSec)),
|
||||||
)) +
|
)) +
|
||||||
stressOnlyCards +
|
stressOnlyCards +
|
||||||
renderSATCard("nvidia-interconnect", "NVIDIA Interconnect (NCCL)", "runNvidiaFabricValidate('nvidia-interconnect')", "", renderValidateCardBody(
|
renderCheckOnlySATCard(stressDefault, renderSATCard("nvidia-interconnect", "NVIDIA Interconnect (NCCL)", "runNvidiaFabricValidate('nvidia-interconnect')", "", renderValidateCardBody(
|
||||||
inv.NVIDIA,
|
inv.NVIDIA,
|
||||||
`Verifies NVLink/NVSwitch fabric bandwidth using NCCL all_reduce_perf across all selected GPUs. Pass/fail based on achieved bandwidth vs. theoretical.`,
|
`Verifies NVLink/NVSwitch fabric bandwidth using NCCL all_reduce_perf across all selected GPUs. Pass/fail based on achieved bandwidth vs. theoretical.`,
|
||||||
`<code>all_reduce_perf</code> (NCCL tests)`,
|
`<code>all_reduce_perf</code> (NCCL tests)`,
|
||||||
validateFmtDur(platform.SATEstimatedNvidiaInterconnectSec)+` (all GPUs simultaneously, requires ≥2).`,
|
validateFmtDur(platform.SATEstimatedNvidiaInterconnectSec)+` (all GPUs simultaneously, requires ≥2).`,
|
||||||
)) +
|
))) +
|
||||||
renderSATCard("nvidia-bandwidth", "NVIDIA Bandwidth + PCIe Link", "runNvidiaFabricValidate('nvidia-bandwidth')", "", renderValidateCardBody(
|
renderSATCard("nvidia-bandwidth", "NVIDIA Bandwidth + PCIe Link", "runNvidiaFabricValidate('nvidia-bandwidth')", "", renderValidateCardBody(
|
||||||
inv.NVIDIA,
|
inv.NVIDIA,
|
||||||
`Validates GPU memory copy and peer-to-peer bandwidth paths, then samples each GPU's negotiated PCIe speed and width immediately after real traffic. Idle or forced-retrain Gen1 readings do not fail the test.`,
|
`Validates GPU memory copy and peer-to-peer bandwidth paths, then samples each GPU's negotiated PCIe speed and width immediately after real traffic. Idle or forced-retrain Gen1 readings do not fail the test.`,
|
||||||
@@ -197,12 +197,12 @@ func renderValidateMode(opts HandlerOptions, stressDefault bool) string {
|
|||||||
)) +
|
)) +
|
||||||
`</div>
|
`</div>
|
||||||
<div class="grid3" style="margin-top:16px">
|
<div class="grid3" style="margin-top:16px">
|
||||||
` + renderSATCard("amd", "AMD GPU", "runAMDValidateSet()", "", renderValidateCardBody(
|
` + renderCheckOnlySATCard(stressDefault, renderSATCard("amd", "AMD GPU", "runAMDValidateSet()", "", renderValidateCardBody(
|
||||||
inv.AMD,
|
inv.AMD,
|
||||||
`Runs the selected AMD checks only. GPU Validate collects inventory; MEM Integrity uses the RVS MEM module; MEM Bandwidth uses rocm-bandwidth-test and the RVS BABEL module.`,
|
`Runs the selected AMD checks only. GPU Validate collects inventory; MEM Integrity uses the RVS MEM module; MEM Bandwidth uses rocm-bandwidth-test and the RVS BABEL module.`,
|
||||||
`GPU Validate: <code>rocm-smi</code>, <code>dmidecode</code>; MEM Integrity: <code>rvs mem</code>; MEM Bandwidth: <code>rocm-bandwidth-test</code>, <code>rvs babel</code>`,
|
`GPU Validate: <code>rocm-smi</code>, <code>dmidecode</code>; MEM Integrity: <code>rvs mem</code>; MEM Bandwidth: <code>rocm-bandwidth-test</code>, <code>rvs babel</code>`,
|
||||||
`<div style="display:flex;flex-direction:column;gap:4px"><label class="cb-row"><input type="checkbox" id="sat-amd-target" checked><span>GPU Validate</span></label><label class="cb-row"><input type="checkbox" id="sat-amd-mem-target" checked><span>MEM Integrity</span></label><label class="cb-row"><input type="checkbox" id="sat-amd-bandwidth-target" checked><span>MEM Bandwidth</span></label></div>`,
|
`<div style="display:flex;flex-direction:column;gap:4px"><label class="cb-row"><input type="checkbox" id="sat-amd-target" checked><span>GPU Validate</span></label><label class="cb-row"><input type="checkbox" id="sat-amd-mem-target" checked><span>MEM Integrity</span></label><label class="cb-row"><input type="checkbox" id="sat-amd-bandwidth-target" checked><span>MEM Bandwidth</span></label></div>`,
|
||||||
)) +
|
))) +
|
||||||
`</div>
|
`</div>
|
||||||
<div id="sat-output" style="display:none;margin-top:16px" class="card">
|
<div id="sat-output" style="display:none;margin-top:16px" class="card">
|
||||||
<div class="card-head">Test Output <span id="sat-title"></span></div>
|
<div class="card-head">Test Output <span id="sat-title"></span></div>
|
||||||
@@ -620,6 +620,13 @@ func validateIsVendorGPU(dev schema.HardwarePCIeDevice, vendor string) bool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func renderCheckOnlySATCard(stressMode bool, card string) string {
|
||||||
|
if stressMode {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return card
|
||||||
|
}
|
||||||
|
|
||||||
// renderCheck renders the non-destructive Check page (step 2).
|
// renderCheck renders the non-destructive Check page (step 2).
|
||||||
// Shows validate-mode tests only: CPU, Memory, Storage, NVIDIA L2, NCCL, NVBandwidth, AMD.
|
// Shows validate-mode tests only: CPU, Memory, Storage, NVIDIA L2, NCCL, NVBandwidth, AMD.
|
||||||
// Stress-mode tests (targeted-stress, targeted-power, pulse) are on the Load page.
|
// Stress-mode tests (targeted-stress, targeted-power, pulse) are on the Load page.
|
||||||
|
|||||||
@@ -46,3 +46,32 @@ func TestRenderCheckIncludesReadOnlyTPMValidation(t *testing.T) {
|
|||||||
t.Fatal("check page must not offer tpm2_selftest")
|
t.Fatal("check page must not offer tpm2_selftest")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRenderLoadOmitsExactCheckDuplicates(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
page := renderValidateMode(HandlerOptions{}, true)
|
||||||
|
for _, duplicateID := range []string{
|
||||||
|
`id="sat-btn-tpm"`,
|
||||||
|
`id="sat-btn-nvidia-interconnect"`,
|
||||||
|
`id="sat-btn-amd"`,
|
||||||
|
} {
|
||||||
|
if strings.Contains(page, duplicateID) {
|
||||||
|
t.Fatalf("load page contains Check-only card %q", duplicateID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, loadID := range []string{
|
||||||
|
`id="sat-btn-cpu"`,
|
||||||
|
`id="sat-btn-memory"`,
|
||||||
|
`id="sat-btn-storage"`,
|
||||||
|
`id="sat-btn-nvidia"`,
|
||||||
|
`id="sat-btn-nvidia-bandwidth"`,
|
||||||
|
`id="sat-btn-nvidia-targeted-stress"`,
|
||||||
|
`id="sat-btn-nvidia-targeted-power"`,
|
||||||
|
`id="sat-btn-nvidia-pulse"`,
|
||||||
|
} {
|
||||||
|
if !strings.Contains(page, loadID) {
|
||||||
|
t.Fatalf("load page is missing load-specific card %q", loadID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user