fix(collector): surface PCIe link-speed degradation in component-status DB, sample it under load

pcie:gpu:<vendor> in the component-status DB (what the Hardware
Summary/webui "check passed" status reads) was only ever written by SAT
diag results, and none of the nvidia/nvidia-config/nvidia-interconnect/
nvidia-bandwidth SAT jobs check PCIe link speed. So a real Gen1/Gen4
degradation the collector already flagged as Warning in the hardware
snapshot never reached the DB-backed status, and the audit kept
reporting "OK" despite GPUs training at Gen1 with BMC showing x16.

Add writePCIeGPUStatusesToDB, mirroring the existing PSU write-through,
so RunAudit pushes the collector's PCIe status into the DB alongside
SAT results.

Also add export/gpu/pcie-nvidia-link-under-load.txt to the support
bundle: NVIDIA drivers deliberately downclock PCIe at idle to save
power and re-train to full speed under load, so an idle Gen1 reading
alone can't distinguish real hardware/riser degradation from normal
power management. Resample the same sysfs link attributes while
bee-gpu-burn is actively loading the GPUs so both bundles ship
together.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-08-06 13:51:14 +03:00
co-authored by Claude Sonnet 5
parent 3ec7ca08da
commit b1f165edb3
4 changed files with 130 additions and 1 deletions
+39
View File
@@ -352,6 +352,45 @@ func (f fakeSAT) RunScenario(_ context.Context, baseDir string, _ platform.Scena
return "", nil
}
// TestWritePCIeGPUStatusesToDBSurfacesLinkSpeedDegradation guards the bug
// where a PCIe link-speed degradation (e.g. Gen1 instead of Gen4) that the
// collector already flags as Warning on the hardware snapshot never reached
// the component-status DB: only SAT diag results wrote pcie:gpu:<vendor>
// there, and none of the nvidia SAT jobs check PCIe link speed, so the
// DB-backed "check passed" status stayed OK despite the degraded link.
func TestWritePCIeGPUStatusesToDBSurfacesLinkSpeedDegradation(t *testing.T) {
db, err := OpenComponentStatusDB(filepath.Join(t.TempDir(), "component-status.json"))
if err != nil {
t.Fatal(err)
}
class := "VideoController"
vendor := 0x10de // collector.NvidiaVendorID
status := "Warning"
desc := "PCIe link speed degraded: running at Gen1, capable of Gen4"
devices := []schema.HardwarePCIeDevice{
{
HardwareComponentStatus: schema.HardwareComponentStatus{Status: &status, ErrorDescription: &desc},
DeviceClass: &class,
VendorID: &vendor,
BDF: strPtr("0000:01:00.0"),
},
}
writePCIeGPUStatusesToDB(db, devices)
rec, ok := db.Get("pcie:gpu:nvidia")
if !ok {
t.Fatal("expected pcie:gpu:nvidia record to be written")
}
if rec.Status != "Warning" {
t.Fatalf("expected Warning, got %q", rec.Status)
}
if rec.ErrorSummary != desc {
t.Fatalf("expected error summary %q, got %q", desc, rec.ErrorSummary)
}
}
func TestRunNCCLTestsPassesSelectedGPUs(t *testing.T) {
t.Parallel()