fix(collector): surface PCIe link-speed degradation in component-status DB, sample it under load
pcie:gpu:<vendor> in the component-status DB (what the Hardware Summary/webui "check passed" status reads) was only ever written by SAT diag results, and none of the nvidia/nvidia-config/nvidia-interconnect/ nvidia-bandwidth SAT jobs check PCIe link speed. So a real Gen1/Gen4 degradation the collector already flagged as Warning in the hardware snapshot never reached the DB-backed status, and the audit kept reporting "OK" despite GPUs training at Gen1 with BMC showing x16. Add writePCIeGPUStatusesToDB, mirroring the existing PSU write-through, so RunAudit pushes the collector's PCIe status into the DB alongside SAT results. Also add export/gpu/pcie-nvidia-link-under-load.txt to the support bundle: NVIDIA drivers deliberately downclock PCIe at idle to save power and re-train to full speed under load, so an idle Gen1 reading alone can't distinguish real hardware/riser degradation from normal power management. Resample the same sysfs link attributes while bee-gpu-burn is actively loading the GPUs so both bundles ship together. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
3ec7ca08da
commit
b1f165edb3
@@ -352,6 +352,45 @@ func (f fakeSAT) RunScenario(_ context.Context, baseDir string, _ platform.Scena
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// TestWritePCIeGPUStatusesToDBSurfacesLinkSpeedDegradation guards the bug
|
||||
// where a PCIe link-speed degradation (e.g. Gen1 instead of Gen4) that the
|
||||
// collector already flags as Warning on the hardware snapshot never reached
|
||||
// the component-status DB: only SAT diag results wrote pcie:gpu:<vendor>
|
||||
// there, and none of the nvidia SAT jobs check PCIe link speed, so the
|
||||
// DB-backed "check passed" status stayed OK despite the degraded link.
|
||||
func TestWritePCIeGPUStatusesToDBSurfacesLinkSpeedDegradation(t *testing.T) {
|
||||
db, err := OpenComponentStatusDB(filepath.Join(t.TempDir(), "component-status.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
class := "VideoController"
|
||||
vendor := 0x10de // collector.NvidiaVendorID
|
||||
status := "Warning"
|
||||
desc := "PCIe link speed degraded: running at Gen1, capable of Gen4"
|
||||
devices := []schema.HardwarePCIeDevice{
|
||||
{
|
||||
HardwareComponentStatus: schema.HardwareComponentStatus{Status: &status, ErrorDescription: &desc},
|
||||
DeviceClass: &class,
|
||||
VendorID: &vendor,
|
||||
BDF: strPtr("0000:01:00.0"),
|
||||
},
|
||||
}
|
||||
|
||||
writePCIeGPUStatusesToDB(db, devices)
|
||||
|
||||
rec, ok := db.Get("pcie:gpu:nvidia")
|
||||
if !ok {
|
||||
t.Fatal("expected pcie:gpu:nvidia record to be written")
|
||||
}
|
||||
if rec.Status != "Warning" {
|
||||
t.Fatalf("expected Warning, got %q", rec.Status)
|
||||
}
|
||||
if rec.ErrorSummary != desc {
|
||||
t.Fatalf("expected error summary %q, got %q", desc, rec.ErrorSummary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunNCCLTestsPassesSelectedGPUs(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user