From b1f165edb380d723cfe82d8cdff32107063fc112 Mon Sep 17 00:00:00 2001 From: Mikhail Chusavitin Date: Thu, 6 Aug 2026 13:51:14 +0300 Subject: [PATCH] fix(collector): surface PCIe link-speed degradation in component-status DB, sample it under load pcie:gpu: in the component-status DB (what the Hardware Summary/webui "check passed" status reads) was only ever written by SAT diag results, and none of the nvidia/nvidia-config/nvidia-interconnect/ nvidia-bandwidth SAT jobs check PCIe link speed. So a real Gen1/Gen4 degradation the collector already flagged as Warning in the hardware snapshot never reached the DB-backed status, and the audit kept reporting "OK" despite GPUs training at Gen1 with BMC showing x16. Add writePCIeGPUStatusesToDB, mirroring the existing PSU write-through, so RunAudit pushes the collector's PCIe status into the DB alongside SAT results. Also add export/gpu/pcie-nvidia-link-under-load.txt to the support bundle: NVIDIA drivers deliberately downclock PCIe at idle to save power and re-train to full speed under load, so an idle Gen1 reading alone can't distinguish real hardware/riser degradation from normal power management. Resample the same sysfs link attributes while bee-gpu-burn is actively loading the GPUs so both bundles ship together. Co-Authored-By: Claude Sonnet 5 --- audit/internal/app/app.go | 53 ++++++++++++++++++++++++++++ audit/internal/app/app_test.go | 39 ++++++++++++++++++++ audit/internal/app/assets/README.md | 7 +++- audit/internal/app/support_bundle.go | 32 +++++++++++++++++ 4 files changed, 130 insertions(+), 1 deletion(-) diff --git a/audit/internal/app/app.go b/audit/internal/app/app.go index 7a04082..4cbb5f7 100644 --- a/audit/internal/app/app.go +++ b/audit/internal/app/app.go @@ -227,6 +227,7 @@ func (a *App) RunAudit(runtimeMode runtimeenv.Mode, output string) (string, erro result := collector.Run(runtimeMode) applyLatestSATStatuses(&result.Hardware, DefaultSATBaseDir, a.StatusDB) writePSUStatusesToDB(a.StatusDB, result.Hardware.PowerSupplies) + writePCIeGPUStatusesToDB(a.StatusDB, result.Hardware.PCIeDevices) if health, err := ReadRuntimeHealth(DefaultRuntimeJSONPath); err == nil { result.Runtime = &health } @@ -450,6 +451,58 @@ func writePSUStatusesToDB(db *ComponentStatusDB, psus []schema.HardwarePowerSupp db.Record("psu:all", source, worstStatus, "") } +// writePCIeGPUStatusesToDB records GPU PCIe-link statuses (e.g. link speed +// degraded to Gen1 while capable of Gen4) collected during audit into the +// component-status DB. Without this, the DB's pcie:gpu: key is only +// ever written by SAT diag results (ApplySATResultToDB), and none of the +// nvidia/nvidia-config/nvidia-interconnect/nvidia-bandwidth SAT jobs check +// PCIe link speed — so a real Gen1/Gen4 degradation the collector already +// flagged as Warning in the hardware snapshot never surfaces in the DB-backed +// "check passed" status the Hardware Summary/webui reads. +func writePCIeGPUStatusesToDB(db *ComponentStatusDB, devices []schema.HardwarePCIeDevice) { + if db == nil || len(devices) == 0 { + return + } + const source = "audit:pcie" + worst := map[string]string{} + detail := map[string]string{} + for _, dev := range devices { + if dev.Status == nil { + continue + } + st := *dev.Status + for _, vendor := range []string{"nvidia", "amd"} { + if !matchesGPUVendor(dev, vendor) { + continue + } + if _, seen := worst[vendor]; !seen { + worst[vendor] = "OK" + } + switch st { + case "Critical": + worst[vendor] = "Critical" + if dev.ErrorDescription != nil { + detail[vendor] = *dev.ErrorDescription + } + case "Warning": + if worst[vendor] != "Critical" { + worst[vendor] = "Warning" + if dev.ErrorDescription != nil { + detail[vendor] = *dev.ErrorDescription + } + } + } + } + } + for _, vendor := range []string{"nvidia", "amd"} { + status, present := worst[vendor] + if !present { + continue + } + db.Record("pcie:gpu:"+vendor, source, status, detail[vendor]) + } +} + func ReadRuntimeHealth(path string) (schema.RuntimeHealth, error) { raw, err := os.ReadFile(path) if err != nil { diff --git a/audit/internal/app/app_test.go b/audit/internal/app/app_test.go index d9aeb07..5c35337 100644 --- a/audit/internal/app/app_test.go +++ b/audit/internal/app/app_test.go @@ -352,6 +352,45 @@ func (f fakeSAT) RunScenario(_ context.Context, baseDir string, _ platform.Scena return "", nil } +// TestWritePCIeGPUStatusesToDBSurfacesLinkSpeedDegradation guards the bug +// where a PCIe link-speed degradation (e.g. Gen1 instead of Gen4) that the +// collector already flags as Warning on the hardware snapshot never reached +// the component-status DB: only SAT diag results wrote pcie:gpu: +// there, and none of the nvidia SAT jobs check PCIe link speed, so the +// DB-backed "check passed" status stayed OK despite the degraded link. +func TestWritePCIeGPUStatusesToDBSurfacesLinkSpeedDegradation(t *testing.T) { + db, err := OpenComponentStatusDB(filepath.Join(t.TempDir(), "component-status.json")) + if err != nil { + t.Fatal(err) + } + + class := "VideoController" + vendor := 0x10de // collector.NvidiaVendorID + status := "Warning" + desc := "PCIe link speed degraded: running at Gen1, capable of Gen4" + devices := []schema.HardwarePCIeDevice{ + { + HardwareComponentStatus: schema.HardwareComponentStatus{Status: &status, ErrorDescription: &desc}, + DeviceClass: &class, + VendorID: &vendor, + BDF: strPtr("0000:01:00.0"), + }, + } + + writePCIeGPUStatusesToDB(db, devices) + + rec, ok := db.Get("pcie:gpu:nvidia") + if !ok { + t.Fatal("expected pcie:gpu:nvidia record to be written") + } + if rec.Status != "Warning" { + t.Fatalf("expected Warning, got %q", rec.Status) + } + if rec.ErrorSummary != desc { + t.Fatalf("expected error summary %q, got %q", desc, rec.ErrorSummary) + } +} + func TestRunNCCLTestsPassesSelectedGPUs(t *testing.T) { t.Parallel() diff --git a/audit/internal/app/assets/README.md b/audit/internal/app/assets/README.md index 5912659..367f7da 100644 --- a/audit/internal/app/assets/README.md +++ b/audit/internal/app/assets/README.md @@ -162,7 +162,12 @@ Vendor-neutral: nothing here assumes familiarity with bee. | `cpu/` | `lscpu.txt`, `dmidecode-type4.txt` (Processor Information). | | `memory/` | `dmidecode-type17.txt` (Memory Device, one entry per DIMM). | | `storage/` | `smartctl-.json`, `nvme-id-ctrl-.json`, `nvme-smart-log-.json`, `nvme-list.json`, `storcli64-drives.json`, `storcli2-show-all.json`, `lsblk.json`. | -| `gpu/` | `nvidia-smi-*.txt/.csv` (state/topology/NVLink; `-fresh` variants are recaptured live at bundle-build time — see "GPU topology" above), `dcgmi-nvlink-status.txt`, `nvidia-bug-report.txt`, `nvidia-dcgm.service`/`nvidia-fabricmanager.service` status+journal, `fabric-manager-paths.txt`, `fabricmanager.log`/`nvlsm.log`, `lspci-nvidia-bridges-vv.txt`, `pcie-nvidia-link.txt`, `pcie-aer-sysfs.txt`, `kernel-aer-nvidia.txt` (AER/NVRM/Xid-filtered dmesg), `lspci-video-vv.txt`, `systemctl-nvidia-units.txt`. AMD systems get `rocm-smi*.txt` here instead. | +| `gpu/` | `nvidia-smi-*.txt/.csv` (state/topology/NVLink; `-fresh` variants are recaptured live at bundle-build time — see "GPU topology" above), `dcgmi-nvlink-status.txt`, `nvidia-bug-report.txt`, `nvidia-dcgm.service`/`nvidia-fabricmanager.service` status+journal, `fabric-manager-paths.txt`, `fabricmanager.log`/`nvlsm.log`, `lspci-nvidia-bridges-vv.txt`, `pcie-nvidia-link.txt` (idle sample — a Gen1 +reading here can be normal driver power management, not a fault), +`pcie-nvidia-link-under-load.txt` (same sysfs attributes resampled while +`bee-gpu-burn` is actively loading the GPUs; if the link trains up to full +speed here, the idle Gen1 reading was power saving — if it stays at Gen1 +under load, that's a real link/riser/slot degradation), `pcie-aer-sysfs.txt`, `kernel-aer-nvidia.txt` (AER/NVRM/Xid-filtered dmesg), `lspci-video-vv.txt`, `systemctl-nvidia-units.txt`. AMD systems get `rocm-smi*.txt` here instead. | | `network/` | `ethtool-{info,link,module}.txt` (per-NIC), `mstflint-query.txt` (Mellanox/NVIDIA NICs). | | `platform/` | `dmidecode-type{0,1,2}.txt` (BIOS/System/Baseboard), `ipmitool-{fru,sdr,sensor,sel,sel-time}.txt` (BMC), `sensors.json`, `lspci-{nn,vmm,vvv}.txt`. | diff --git a/audit/internal/app/support_bundle.go b/audit/internal/app/support_bundle.go index 410f86a..ca4402c 100644 --- a/audit/internal/app/support_bundle.go +++ b/audit/internal/app/support_bundle.go @@ -308,6 +308,38 @@ for d in /sys/bus/pci/devices/*/; do printf " %-22s %s\n" "$f" "$(cat "$d/$f" 2>/dev/null)" done done +`}}, + // A Gen1-vs-Gen4 link speed reading at idle is ambiguous: NVIDIA drivers + // deliberately downclock PCIe in low power states and re-train to full + // speed under load, so pcie-nvidia-link.txt alone can't tell a real + // hardware/riser fault from normal power management. Re-sample the same + // sysfs attributes while bee-gpu-burn is actively loading the GPUs — if + // the link comes up here, the idle Gen1 reading above was power saving, + // not a fault. + {name: "export/gpu/pcie-nvidia-link-under-load.txt", cmd: []string{"sh", "-c", ` +if ! command -v bee-gpu-burn >/dev/null 2>&1; then + echo "bee-gpu-burn not found; cannot sample PCIe link speed under load" + exit 0 +fi +bee-gpu-burn --seconds 8 --size-mb 64 >/tmp/bee-pcie-load-burn.log 2>&1 & +burn_pid=$! +sleep 3 +for d in /sys/bus/pci/devices/*/; do + vendor=$(cat "$d/vendor" 2>/dev/null) + [ "$vendor" = "0x10de" ] || continue + class=$(cat "$d/class" 2>/dev/null) + case "$class" in + 0x030000|0x030200) ;; + *) continue ;; + esac + dev=$(basename "$d") + echo "=== $dev ===" + for f in current_link_speed current_link_width max_link_speed max_link_width; do + printf " %-22s %s\n" "$f" "$(cat "$d/$f" 2>/dev/null)" + done +done +wait "$burn_pid" 2>/dev/null || true +rm -f /tmp/bee-pcie-load-burn.log `}}, {name: "export/gpu/pcie-aer-sysfs.txt", cmd: []string{"sh", "-c", ` found=0