fix(collector): stop pinning GPU PCIe status on an unverified idle reading
NVIDIA GPUs deliberately downclock PCIe to Gen1 at idle for power saving, and applyPCIeLinkSpeedWarning fired on every idle collector pass regardless - since component-status DB records never downgrade (Record() only ever raises severity), one boot-time idle sample permanently pinned pcie:gpu:nvidia to Warning for the rest of the session even after every load-bearing GPU SAT test passed clean. Two prior fixes (nvidia-smi-sourced link speed, pcie_aspm=off boot flag) didn't hold up against this hardware/driver combination - see bible-local/decisions/2026-08-24-pcie-gpu-gen1-idle-warning.md for the full history. Rather than add a downgrade path, stop writing an unverified status in the first place: parseLspciDevice no longer calls applyPCIeLinkSpeedWarning on the idle path. LinkSpeed/MaxLinkSpeed stay populated as plain descriptive fields; only a verified-under-load caller may now turn them into a status verdict. Two new SAT targets provide that verified signal: - pcie-link (platform/pcie_link_check.go): forces every enabled PCIe device - not just GPUs - to retrain via the PCIe spec's Link Control "Retrain Link" bit, then compares the negotiated speed against the device's max. Covers NICs/HBAs/switches that have no bee-gpu-burn equivalent load tool. Classifies by PCI class code + vendor ID, not name substrings. Routes gpu_nvidia/gpu_amd/other sub-verdicts into their own component-status keys so a degraded NIC never reads as a GPU fault. - nvidia-pcie-bandwidth (platform/nvidia_pcie_bandwidth.go): drives real host<->device traffic via dcgmi diag -r nvbandwidth and resamples link speed immediately after, independent of nvbandwidth's own pass/fail. Both wired into the task queue/webui the same way as nvidia-config (routes, dispatch, priority, Validate page cards, Run All Check SAT). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
7aa276320b
commit
b11018ac5e
@@ -0,0 +1,66 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestClassifyGPUFromVendorClass(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
vendor string
|
||||
class string
|
||||
wantIsGPU bool
|
||||
wantVendor string
|
||||
}{
|
||||
{"nvidia GPU", "0x10de", "0x030200", true, "nvidia"},
|
||||
{"amd GPU", "0x1002", "0x030000", true, "amd"},
|
||||
{"nvidia NIC (same vendor, not display class)", "0x10de", "0x020000", false, ""},
|
||||
{"intel NIC", "0x8086", "0x020000", false, ""},
|
||||
{"unrelated display-class vendor", "0x1234", "0x030000", false, ""},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
gotIsGPU, gotVendor := classifyGPUFromVendorClass(tc.vendor, tc.class)
|
||||
if gotIsGPU != tc.wantIsGPU || gotVendor != tc.wantVendor {
|
||||
t.Fatalf("classifyGPUFromVendorClass(%q,%q) = (%v,%q), want (%v,%q)",
|
||||
tc.vendor, tc.class, gotIsGPU, gotVendor, tc.wantIsGPU, tc.wantVendor)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderPCIeLinkCheckSummaryDegradedGPU(t *testing.T) {
|
||||
findings := []pcieLinkFinding{
|
||||
{BDF: "0000:0d:00.0", Description: "NVIDIA GPU", IsGPU: true, GPUVendor: "nvidia",
|
||||
BeforeSpeed: "Gen1", AfterSpeed: "Gen1", MaxSpeed: "Gen5", Degraded: true},
|
||||
{BDF: "0000:37:00.0", Description: "NVIDIA GPU", IsGPU: true, GPUVendor: "nvidia",
|
||||
BeforeSpeed: "Gen1", AfterSpeed: "Gen5", MaxSpeed: "Gen5", Degraded: false},
|
||||
{BDF: "0000:01:00.0", Description: "Mellanox NIC", IsGPU: false,
|
||||
BeforeSpeed: "Gen3", AfterSpeed: "Gen4", MaxSpeed: "Gen4", Degraded: false},
|
||||
}
|
||||
summary := renderPCIeLinkCheckSummary(findings)
|
||||
if !strings.Contains(summary, "gpu_nvidia_status=FAILED") {
|
||||
t.Fatalf("expected gpu_nvidia_status=FAILED (one degraded GPU should fail the vendor group), got:\n%s", summary)
|
||||
}
|
||||
if !strings.Contains(summary, "other_status=OK") {
|
||||
t.Fatalf("expected other_status=OK, got:\n%s", summary)
|
||||
}
|
||||
if !strings.Contains(summary, "overall_status=FAILED") {
|
||||
t.Fatalf("expected overall_status=FAILED, got:\n%s", summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderPCIeLinkCheckSummaryAllClean(t *testing.T) {
|
||||
findings := []pcieLinkFinding{
|
||||
{BDF: "0000:0d:00.0", IsGPU: true, GPUVendor: "nvidia", BeforeSpeed: "Gen1", AfterSpeed: "Gen5", MaxSpeed: "Gen5"},
|
||||
{BDF: "0000:2c:00.0", Skipped: "device disabled (no data traffic; link state has no operational impact)"},
|
||||
}
|
||||
summary := renderPCIeLinkCheckSummary(findings)
|
||||
if !strings.Contains(summary, "overall_status=OK") {
|
||||
t.Fatalf("expected overall_status=OK, got:\n%s", summary)
|
||||
}
|
||||
if strings.Contains(summary, "other_status=") {
|
||||
t.Fatalf("disabled/skipped-only device should not produce an other_status line, got:\n%s", summary)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user