reanimator/bee
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(pcie): verify GPU links under real bandwidth load

Browse Source
This commit is contained in:
Mikhail Chusavitin
2026-09-03 10:23:39 +03:00
parent 642e68631d
commit 347bc8310a
27 changed files with 384 additions and 902 deletions
Download Patch File Download Diff File Expand all files Collapse all files
audit/internal/webui/api.go
+1 -1
View File
@@ -132,7 +132,7 @@ func defaultTaskPriority(target string, params taskParams) int {
return taskPriorityBurn
case "nvidia", "nvidia-targeted-stress", "nvidia-targeted-power", "nvidia-pulse",
"nvidia-interconnect", "nvidia-bandwidth", "memory", "storage", "cpu",
"amd", "amd-mem", "amd-bandwidth", "nvidia-config", "pcie-link", "nvidia-pcie-bandwidth":
"amd", "amd-mem", "amd-bandwidth", "nvidia-config":
if params.StressMode {
return taskPriorityValidateStress
}
audit/internal/webui/api_sat_runall.go
+1 -2
View File
@@ -101,7 +101,6 @@ func (h *handler) planSATRunAll(ctx context.Context, req satRunAllRequest) ([]sa
satRunAllSpec{target: "cpu", params: taskParams{Duration: cpuDur, StressMode: req.StressMode}},
satRunAllSpec{target: "memory", params: taskParams{StressMode: req.StressMode}},
satRunAllSpec{target: "storage", params: taskParams{StressMode: req.StressMode}},
satRunAllSpec{target: "pcie-link", params: taskParams{}},
)
if h.opts.App.TPMPresent() {
@@ -139,7 +138,7 @@ func (h *handler) planSATRunAll(ctx context.Context, req satRunAllRequest) ([]sa
if !health.CUDAReady {
notes = append(notes, "NVIDIA: CUDA runtime not confirmed ready; GPU tests queued anyway")
}
gpuTargets := []string{"nvidia", "nvidia-interconnect", "nvidia-bandwidth", "nvidia-pcie-bandwidth"}
gpuTargets := []string{"nvidia", "nvidia-interconnect", "nvidia-bandwidth"}
if req.StressMode {
// Stress tier adds the targeted dcgmi diag load tests.
gpuTargets = append(gpuTargets, "nvidia-targeted-stress", "nvidia-targeted-power", "nvidia-pulse")
audit/internal/webui/api_sat_runall_test.go
+1 -1
View File
@@ -80,7 +80,7 @@ func TestPlanSATRunAllNoAcceleratorNoTPM(t *testing.T) {
for _, s := range specs {
targets = append(targets, s.target)
}
want := []string{"cpu", "memory", "storage", "pcie-link"}
want := []string{"cpu", "memory", "storage"}
if !reflect.DeepEqual(targets, want) {
t.Fatalf("targets=%v want %v", targets, want)
}
audit/internal/webui/page_validate.go
+8 -20
View File
@@ -189,11 +189,11 @@ func renderValidateMode(opts HandlerOptions, stressDefault bool) string {
`<code>all_reduce_perf</code> (NCCL tests)`,
validateFmtDur(platform.SATEstimatedNvidiaInterconnectSec)+` (all GPUs simultaneously, requires ≥2).`,
)) +
renderSATCard("nvidia-bandwidth", "NVIDIA Bandwidth (NVBandwidth)", "runNvidiaFabricValidate('nvidia-bandwidth')", "", renderValidateCardBody(
renderSATCard("nvidia-bandwidth", "NVIDIA Bandwidth + PCIe Link", "runNvidiaFabricValidate('nvidia-bandwidth')", "", renderValidateCardBody(
inv.NVIDIA,
`Validates GPU memory copy and peer-to-peer bandwidth paths using NVBandwidth.`,
`<code>nvbandwidth</code>`,
validateFmtDur(platform.SATEstimatedNvidiaBandwidthSec)+` (all GPUs simultaneously; nvbandwidth runs all built-in tests without a time limit — duration set by the tool).`,
`Validates GPU memory copy and peer-to-peer bandwidth paths, then samples each GPU's negotiated PCIe speed and width immediately after real traffic. Idle or forced-retrain Gen1 readings do not fail the test.`,
`<code>dcgmi diag -r nvbandwidth</code>, sysfs PCIe link attributes`,
validateFmtDur(platform.SATEstimatedNvidiaBandwidthSec)+` (all GPUs simultaneously; nvbandwidth runs all built-in tests without a time limit - duration set by the tool).`,
)) +
`</div>
<div class="grid3" style="margin-top:16px">
@@ -669,18 +669,6 @@ func renderCheck(opts HandlerOptions) string {
`<code>nvidia-smi --query-gpu=...</code>, <code>nvidia-smi topo -m</code>, <code>nvidia-smi nvlink -s/-e</code>, <code>nvidia-smi conf-compute -q</code>, <code>dmesg</code>`,
`Seconds — read-only query only.`,
)) +
renderSATCard("pcie-link", "PCIe Link Check", "runSAT('pcie-link')", "", renderValidateCardBody(
`Every enabled PCIe device in the machine (GPUs, NICs, RAID/HBA controllers, PCIe switches).`,
`Forces every enabled PCIe device to retrain its link (PCIe spec Link Control "Retrain Link" bit) and compares the negotiated speed against the device's own maximum. An idle sysfs reading alone can't tell a real degraded slot/riser from a device that's simply power-managed down at idle (GPUs do this routinely); a forced retrain settles that without needing a device-specific load generator, so this one check covers non-GPU PCIe hardware too, not just GPUs.`,
`<code>setpci</code> (Link Control/Link Status registers), <code>lspci</code>`,
`Seconds per device — brief link retrain, no sustained traffic.`,
)) +
renderSATCard("nvidia-pcie-bandwidth", "GPU PCIe Bandwidth", "runSAT('nvidia-pcie-bandwidth')", "", renderValidateCardBody(
inv.NVIDIA,
`Drives real host&lt;-&gt;device traffic across each GPU's PCIe link and resamples link speed immediately after, to confirm the link actually trains to its negotiated maximum under real load — the load-bearing counterpart to PCIe Link Check for GPUs specifically.`,
`<code>dcgmi diag -r nvbandwidth</code>, sysfs link-speed resample`,
`Depends on nvbandwidth's built-in test duration.`,
)) +
`</div>
<div style="height:1px;background:var(--border);margin:16px 0"></div>
<div class="card" style="margin-bottom:16px">
@@ -711,10 +699,10 @@ func renderCheck(opts HandlerOptions) string {
`<code>all_reduce_perf</code> (NCCL tests)`,
validateFmtDur(platform.SATEstimatedNvidiaInterconnectSec)+` (all GPUs simultaneously, requires ≥2).`,
)) +
renderSATCard("nvidia-bandwidth", "NVIDIA Bandwidth (NVBandwidth)", "runNvidiaFabricValidate('nvidia-bandwidth')", "", renderValidateCardBody(
renderSATCard("nvidia-bandwidth", "NVIDIA Bandwidth + PCIe Link", "runNvidiaFabricValidate('nvidia-bandwidth')", "", renderValidateCardBody(
inv.NVIDIA,
`Validates GPU memory copy and peer-to-peer bandwidth paths using NVBandwidth.`,
`<code>nvbandwidth</code>`,
`Validates GPU memory copy and peer-to-peer bandwidth paths, then samples each GPU's negotiated PCIe speed and width immediately after real traffic.`,
`<code>dcgmi diag -r nvbandwidth</code>, sysfs PCIe link attributes`,
validateFmtDur(platform.SATEstimatedNvidiaBandwidthSec)+` (all GPUs simultaneously).`,
)) +
`</div>
@@ -742,7 +730,7 @@ func renderCheck(opts HandlerOptions) string {
<script>
let satES = null;
function satLabels() {
return {nvidia:'Check GPU (DCGM L2)', 'nvidia-interconnect':'NVIDIA Interconnect (NCCL)', 'nvidia-bandwidth':'NVIDIA Bandwidth (NVBandwidth)', memory:'Check Memory', storage:'Check Storage', tpm:'Check TPM (read-only)', cpu:'Check CPU', amd:'Check AMD GPU', 'amd-mem':'AMD GPU MEM Integrity', 'amd-bandwidth':'AMD GPU MEM Bandwidth', 'nvidia-config':'Check GPU Config & NVLink', 'pcie-link':'PCIe Link Check', 'nvidia-pcie-bandwidth':'GPU PCIe Bandwidth Check'};
return {nvidia:'Check GPU (DCGM L2)', 'nvidia-interconnect':'NVIDIA Interconnect (NCCL)', 'nvidia-bandwidth':'NVIDIA Bandwidth + PCIe Link Under Load', memory:'Check Memory', storage:'Check Storage', tpm:'Check TPM (read-only)', cpu:'Check CPU', amd:'Check AMD GPU', 'amd-mem':'AMD GPU MEM Integrity', 'amd-bandwidth':'AMD GPU MEM Bandwidth', 'nvidia-config':'Check GPU Config & NVLink'};
}
let satNvidiaGPUsPromise = null;
function loadSatNvidiaGPUs() {
audit/internal/webui/server.go
-2
View File
@@ -264,8 +264,6 @@ func NewHandler(opts HandlerOptions) http.Handler {
mux.HandleFunc("POST /api/sat/storage/run", h.handleAPISATRun("storage"))
mux.HandleFunc("POST /api/sat/tpm/run", h.handleAPISATRun("tpm"))
mux.HandleFunc("POST /api/sat/nvidia-config/run", h.handleAPISATRun("nvidia-config"))
mux.HandleFunc("POST /api/sat/pcie-link/run", h.handleAPISATRun("pcie-link"))
mux.HandleFunc("POST /api/sat/nvidia-pcie-bandwidth/run", h.handleAPISATRun("nvidia-pcie-bandwidth"))
mux.HandleFunc("POST /api/sat/cpu/run", h.handleAPISATRun("cpu"))
mux.HandleFunc("POST /api/sat/amd/run", h.handleAPISATRun("amd"))
mux.HandleFunc("POST /api/sat/amd-mem/run", h.handleAPISATRun("amd-mem"))
audit/internal/webui/server_test.go
+2 -2
View File
@@ -815,7 +815,7 @@ func TestCheckPageRendersGPUSelectionAndNvidiaCards(t *testing.T) {
`Select All`,
`id="sat-btn-nvidia"`,
`NVIDIA Interconnect (NCCL)`,
`NVIDIA Bandwidth (NVBandwidth)`,
`NVIDIA Bandwidth + PCIe Link`,
`Non-destructive`,
} {
if !strings.Contains(body, needle) {
@@ -834,7 +834,7 @@ func TestCheckPageRendersNvidiaFabricCards(t *testing.T) {
body := rec.Body.String()
for _, needle := range []string{
`NVIDIA Interconnect (NCCL)`,
`NVIDIA Bandwidth (NVBandwidth)`,
`NVIDIA Bandwidth + PCIe Link`,
`nvbandwidth`,
`all_reduce_perf`,
} {
audit/internal/webui/task_runner.go
-12
View File
@@ -303,18 +303,6 @@ func executeTaskWithOptions(opts *HandlerOptions, t *Task, j *jobState, ctx cont
break
}
archive, err = runNvidiaConfigCheckPackCtx(a, ctx, "", j.append)
case "pcie-link":
if a == nil {
err = fmt.Errorf("app not configured")
break
}
archive, err = runPCIeLinkCheckPackCtx(a, ctx, "", j.append)
case "nvidia-pcie-bandwidth":
if a == nil {
err = fmt.Errorf("app not configured")
break
}
archive, err = runNvidiaPCIeBandwidthPackCtx(a, ctx, "", t.params.GPUIndices, j.append)
case "cpu":
if a == nil {
err = fmt.Errorf("app not configured")
audit/internal/webui/tasks.go
-8
View File
@@ -45,8 +45,6 @@ var taskNames = map[string]string{
"storage": "Storage SAT",
"tpm": "TPM Read-only Check",
"nvidia-config": "GPU Config & NVLink Check",
"pcie-link": "PCIe Link Check (forced retrain)",
"nvidia-pcie-bandwidth": "NVIDIA GPU PCIe Bandwidth Check",
"cpu": "CPU SAT",
"amd": "AMD GPU SAT",
"amd-mem": "AMD GPU MEM Integrity",
@@ -321,12 +319,6 @@ var (
runNvidiaConfigCheckPackCtx = func(a *app.App, ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
return a.RunNvidiaConfigCheckPackCtx(ctx, baseDir, logFunc)
}
runPCIeLinkCheckPackCtx = func(a *app.App, ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
return a.RunPCIeLinkCheckPackCtx(ctx, baseDir, logFunc)
}
runNvidiaPCIeBandwidthPackCtx = func(a *app.App, ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) {
return a.RunNvidiaPCIeBandwidthPackCtx(ctx, baseDir, gpuIndices, logFunc)
}
runCPUAcceptancePackCtx = func(a *app.App, ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
return a.RunCPUAcceptancePackCtx(ctx, baseDir, durationSec, logFunc)
}