fix(collector): surface PCIe link-speed degradation in component-status DB, sample it under load

pcie:gpu:<vendor> in the component-status DB (what the Hardware
Summary/webui "check passed" status reads) was only ever written by SAT
diag results, and none of the nvidia/nvidia-config/nvidia-interconnect/
nvidia-bandwidth SAT jobs check PCIe link speed. So a real Gen1/Gen4
degradation the collector already flagged as Warning in the hardware
snapshot never reached the DB-backed status, and the audit kept
reporting "OK" despite GPUs training at Gen1 with BMC showing x16.

Add writePCIeGPUStatusesToDB, mirroring the existing PSU write-through,
so RunAudit pushes the collector's PCIe status into the DB alongside
SAT results.

Also add export/gpu/pcie-nvidia-link-under-load.txt to the support
bundle: NVIDIA drivers deliberately downclock PCIe at idle to save
power and re-train to full speed under load, so an idle Gen1 reading
alone can't distinguish real hardware/riser degradation from normal
power management. Resample the same sysfs link attributes while
bee-gpu-burn is actively loading the GPUs so both bundles ship
together.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-08-06 13:51:14 +03:00
co-authored by Claude Sonnet 5
parent 3ec7ca08da
commit b1f165edb3
4 changed files with 130 additions and 1 deletions
+53
View File
@@ -227,6 +227,7 @@ func (a *App) RunAudit(runtimeMode runtimeenv.Mode, output string) (string, erro
result := collector.Run(runtimeMode)
applyLatestSATStatuses(&result.Hardware, DefaultSATBaseDir, a.StatusDB)
writePSUStatusesToDB(a.StatusDB, result.Hardware.PowerSupplies)
writePCIeGPUStatusesToDB(a.StatusDB, result.Hardware.PCIeDevices)
if health, err := ReadRuntimeHealth(DefaultRuntimeJSONPath); err == nil {
result.Runtime = &health
}
@@ -450,6 +451,58 @@ func writePSUStatusesToDB(db *ComponentStatusDB, psus []schema.HardwarePowerSupp
db.Record("psu:all", source, worstStatus, "")
}
// writePCIeGPUStatusesToDB records GPU PCIe-link statuses (e.g. link speed
// degraded to Gen1 while capable of Gen4) collected during audit into the
// component-status DB. Without this, the DB's pcie:gpu:<vendor> key is only
// ever written by SAT diag results (ApplySATResultToDB), and none of the
// nvidia/nvidia-config/nvidia-interconnect/nvidia-bandwidth SAT jobs check
// PCIe link speed — so a real Gen1/Gen4 degradation the collector already
// flagged as Warning in the hardware snapshot never surfaces in the DB-backed
// "check passed" status the Hardware Summary/webui reads.
func writePCIeGPUStatusesToDB(db *ComponentStatusDB, devices []schema.HardwarePCIeDevice) {
if db == nil || len(devices) == 0 {
return
}
const source = "audit:pcie"
worst := map[string]string{}
detail := map[string]string{}
for _, dev := range devices {
if dev.Status == nil {
continue
}
st := *dev.Status
for _, vendor := range []string{"nvidia", "amd"} {
if !matchesGPUVendor(dev, vendor) {
continue
}
if _, seen := worst[vendor]; !seen {
worst[vendor] = "OK"
}
switch st {
case "Critical":
worst[vendor] = "Critical"
if dev.ErrorDescription != nil {
detail[vendor] = *dev.ErrorDescription
}
case "Warning":
if worst[vendor] != "Critical" {
worst[vendor] = "Warning"
if dev.ErrorDescription != nil {
detail[vendor] = *dev.ErrorDescription
}
}
}
}
}
for _, vendor := range []string{"nvidia", "amd"} {
status, present := worst[vendor]
if !present {
continue
}
db.Record("pcie:gpu:"+vendor, source, status, detail[vendor])
}
}
func ReadRuntimeHealth(path string) (schema.RuntimeHealth, error) {
raw, err := os.ReadFile(path)
if err != nil {