fix(collector): surface PCIe link-speed degradation in component-status DB, sample it under load

pcie:gpu:<vendor> in the component-status DB (what the Hardware
Summary/webui "check passed" status reads) was only ever written by SAT
diag results, and none of the nvidia/nvidia-config/nvidia-interconnect/
nvidia-bandwidth SAT jobs check PCIe link speed. So a real Gen1/Gen4
degradation the collector already flagged as Warning in the hardware
snapshot never reached the DB-backed status, and the audit kept
reporting "OK" despite GPUs training at Gen1 with BMC showing x16.

Add writePCIeGPUStatusesToDB, mirroring the existing PSU write-through,
so RunAudit pushes the collector's PCIe status into the DB alongside
SAT results.

Also add export/gpu/pcie-nvidia-link-under-load.txt to the support
bundle: NVIDIA drivers deliberately downclock PCIe at idle to save
power and re-train to full speed under load, so an idle Gen1 reading
alone can't distinguish real hardware/riser degradation from normal
power management. Resample the same sysfs link attributes while
bee-gpu-burn is actively loading the GPUs so both bundles ship
together.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

audit/internal/app/app.go

@@ -227,6 +227,7 @@ func (a *App) RunAudit(runtimeMode runtimeenv.Mode, output string) (string, erro
 	result := collector.Run(runtimeMode)
 	applyLatestSATStatuses(&result.Hardware, DefaultSATBaseDir, a.StatusDB)
 	writePSUStatusesToDB(a.StatusDB, result.Hardware.PowerSupplies)
+	writePCIeGPUStatusesToDB(a.StatusDB, result.Hardware.PCIeDevices)
 	if health, err := ReadRuntimeHealth(DefaultRuntimeJSONPath); err == nil {
 		result.Runtime = &health
 	}
@@ -450,6 +451,58 @@ func writePSUStatusesToDB(db *ComponentStatusDB, psus []schema.HardwarePowerSupp
 	db.Record("psu:all", source, worstStatus, "")
 }
 
+// writePCIeGPUStatusesToDB records GPU PCIe-link statuses (e.g. link speed
+// degraded to Gen1 while capable of Gen4) collected during audit into the
+// component-status DB. Without this, the DB's pcie:gpu:<vendor> key is only
+// ever written by SAT diag results (ApplySATResultToDB), and none of the
+// nvidia/nvidia-config/nvidia-interconnect/nvidia-bandwidth SAT jobs check
+// PCIe link speed — so a real Gen1/Gen4 degradation the collector already
+// flagged as Warning in the hardware snapshot never surfaces in the DB-backed
+// "check passed" status the Hardware Summary/webui reads.
+func writePCIeGPUStatusesToDB(db *ComponentStatusDB, devices []schema.HardwarePCIeDevice) {
+	if db == nil || len(devices) == 0 {
+		return
+	}
+	const source = "audit:pcie"
+	worst := map[string]string{}
+	detail := map[string]string{}
+	for _, dev := range devices {
+		if dev.Status == nil {
+			continue
+		}
+		st := *dev.Status
+		for _, vendor := range []string{"nvidia", "amd"} {
+			if !matchesGPUVendor(dev, vendor) {
+				continue
+			}
+			if _, seen := worst[vendor]; !seen {
+				worst[vendor] = "OK"
+			}
+			switch st {
+			case "Critical":
+				worst[vendor] = "Critical"
+				if dev.ErrorDescription != nil {
+					detail[vendor] = *dev.ErrorDescription
+				}
+			case "Warning":
+				if worst[vendor] != "Critical" {
+					worst[vendor] = "Warning"
+					if dev.ErrorDescription != nil {
+						detail[vendor] = *dev.ErrorDescription
+					}
+				}
+			}
+		}
+	}
+	for _, vendor := range []string{"nvidia", "amd"} {
+		status, present := worst[vendor]
+		if !present {
+			continue
+		}
+		db.Record("pcie:gpu:"+vendor, source, status, detail[vendor])
+	}
+}
+
 func ReadRuntimeHealth(path string) (schema.RuntimeHealth, error) {
 	raw, err := os.ReadFile(path)
 	if err != nil {

audit/internal/app/app_test.go

@@ -352,6 +352,45 @@ func (f fakeSAT) RunScenario(_ context.Context, baseDir string, _ platform.Scena
 	return "", nil
 }
 
+// TestWritePCIeGPUStatusesToDBSurfacesLinkSpeedDegradation guards the bug
+// where a PCIe link-speed degradation (e.g. Gen1 instead of Gen4) that the
+// collector already flags as Warning on the hardware snapshot never reached
+// the component-status DB: only SAT diag results wrote pcie:gpu:<vendor>
+// there, and none of the nvidia SAT jobs check PCIe link speed, so the
+// DB-backed "check passed" status stayed OK despite the degraded link.
+func TestWritePCIeGPUStatusesToDBSurfacesLinkSpeedDegradation(t *testing.T) {
+	db, err := OpenComponentStatusDB(filepath.Join(t.TempDir(), "component-status.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	class := "VideoController"
+	vendor := 0x10de // collector.NvidiaVendorID
+	status := "Warning"
+	desc := "PCIe link speed degraded: running at Gen1, capable of Gen4"
+	devices := []schema.HardwarePCIeDevice{
+		{
+			HardwareComponentStatus: schema.HardwareComponentStatus{Status: &status, ErrorDescription: &desc},
+			DeviceClass:             &class,
+			VendorID:                &vendor,
+			BDF:                     strPtr("0000:01:00.0"),
+		},
+	}
+
+	writePCIeGPUStatusesToDB(db, devices)
+
+	rec, ok := db.Get("pcie:gpu:nvidia")
+	if !ok {
+		t.Fatal("expected pcie:gpu:nvidia record to be written")
+	}
+	if rec.Status != "Warning" {
+		t.Fatalf("expected Warning, got %q", rec.Status)
+	}
+	if rec.ErrorSummary != desc {
+		t.Fatalf("expected error summary %q, got %q", desc, rec.ErrorSummary)
+	}
+}
+
 func TestRunNCCLTestsPassesSelectedGPUs(t *testing.T) {
 	t.Parallel()
 

audit/internal/app/assets/README.md

@@ -162,7 +162,12 @@ Vendor-neutral: nothing here assumes familiarity with bee.
 | `cpu/` | `lscpu.txt`, `dmidecode-type4.txt` (Processor Information). |
 | `memory/` | `dmidecode-type17.txt` (Memory Device, one entry per DIMM). |
 | `storage/` | `smartctl-<dev>.json`, `nvme-id-ctrl-<dev>.json`, `nvme-smart-log-<dev>.json`, `nvme-list.json`, `storcli64-drives.json`, `storcli2-show-all.json`, `lsblk.json`. |
-| `gpu/` | `nvidia-smi-*.txt/.csv` (state/topology/NVLink; `-fresh` variants are recaptured live at bundle-build time — see "GPU topology" above), `dcgmi-nvlink-status.txt`, `nvidia-bug-report.txt`, `nvidia-dcgm.service`/`nvidia-fabricmanager.service` status+journal, `fabric-manager-paths.txt`, `fabricmanager.log`/`nvlsm.log`, `lspci-nvidia-bridges-vv.txt`, `pcie-nvidia-link.txt`, `pcie-aer-sysfs.txt`, `kernel-aer-nvidia.txt` (AER/NVRM/Xid-filtered dmesg), `lspci-video-vv.txt`, `systemctl-nvidia-units.txt`. AMD systems get `rocm-smi*.txt` here instead. |
+| `gpu/` | `nvidia-smi-*.txt/.csv` (state/topology/NVLink; `-fresh` variants are recaptured live at bundle-build time — see "GPU topology" above), `dcgmi-nvlink-status.txt`, `nvidia-bug-report.txt`, `nvidia-dcgm.service`/`nvidia-fabricmanager.service` status+journal, `fabric-manager-paths.txt`, `fabricmanager.log`/`nvlsm.log`, `lspci-nvidia-bridges-vv.txt`, `pcie-nvidia-link.txt` (idle sample — a Gen1
+reading here can be normal driver power management, not a fault),
+`pcie-nvidia-link-under-load.txt` (same sysfs attributes resampled while
+`bee-gpu-burn` is actively loading the GPUs; if the link trains up to full
+speed here, the idle Gen1 reading was power saving — if it stays at Gen1
+under load, that's a real link/riser/slot degradation), `pcie-aer-sysfs.txt`, `kernel-aer-nvidia.txt` (AER/NVRM/Xid-filtered dmesg), `lspci-video-vv.txt`, `systemctl-nvidia-units.txt`. AMD systems get `rocm-smi*.txt` here instead. |
 | `network/` | `ethtool-{info,link,module}.txt` (per-NIC), `mstflint-query.txt` (Mellanox/NVIDIA NICs). |
 | `platform/` | `dmidecode-type{0,1,2}.txt` (BIOS/System/Baseboard), `ipmitool-{fru,sdr,sensor,sel,sel-time}.txt` (BMC), `sensors.json`, `lspci-{nn,vmm,vvv}.txt`. |
 

audit/internal/app/support_bundle.go

@@ -308,6 +308,38 @@ for d in /sys/bus/pci/devices/*/; do
     printf "  %-22s %s\n" "$f" "$(cat "$d/$f" 2>/dev/null)"
   done
 done
+`}},
+	// A Gen1-vs-Gen4 link speed reading at idle is ambiguous: NVIDIA drivers
+	// deliberately downclock PCIe in low power states and re-train to full
+	// speed under load, so pcie-nvidia-link.txt alone can't tell a real
+	// hardware/riser fault from normal power management. Re-sample the same
+	// sysfs attributes while bee-gpu-burn is actively loading the GPUs — if
+	// the link comes up here, the idle Gen1 reading above was power saving,
+	// not a fault.
+	{name: "export/gpu/pcie-nvidia-link-under-load.txt", cmd: []string{"sh", "-c", `
+if ! command -v bee-gpu-burn >/dev/null 2>&1; then
+  echo "bee-gpu-burn not found; cannot sample PCIe link speed under load"
+  exit 0
+fi
+bee-gpu-burn --seconds 8 --size-mb 64 >/tmp/bee-pcie-load-burn.log 2>&1 &
+burn_pid=$!
+sleep 3
+for d in /sys/bus/pci/devices/*/; do
+  vendor=$(cat "$d/vendor" 2>/dev/null)
+  [ "$vendor" = "0x10de" ] || continue
+  class=$(cat "$d/class" 2>/dev/null)
+  case "$class" in
+    0x030000|0x030200) ;;
+    *) continue ;;
+  esac
+  dev=$(basename "$d")
+  echo "=== $dev ==="
+  for f in current_link_speed current_link_width max_link_speed max_link_width; do
+    printf "  %-22s %s\n" "$f" "$(cat "$d/$f" 2>/dev/null)"
+  done
+done
+wait "$burn_pid" 2>/dev/null || true
+rm -f /tmp/bee-pcie-load-burn.log
 `}},
 	{name: "export/gpu/pcie-aer-sysfs.txt", cmd: []string{"sh", "-c", `
 found=0