fix(collector): surface PCIe link-speed degradation in component-status DB, sample it under load
pcie:gpu:<vendor> in the component-status DB (what the Hardware Summary/webui "check passed" status reads) was only ever written by SAT diag results, and none of the nvidia/nvidia-config/nvidia-interconnect/ nvidia-bandwidth SAT jobs check PCIe link speed. So a real Gen1/Gen4 degradation the collector already flagged as Warning in the hardware snapshot never reached the DB-backed status, and the audit kept reporting "OK" despite GPUs training at Gen1 with BMC showing x16. Add writePCIeGPUStatusesToDB, mirroring the existing PSU write-through, so RunAudit pushes the collector's PCIe status into the DB alongside SAT results. Also add export/gpu/pcie-nvidia-link-under-load.txt to the support bundle: NVIDIA drivers deliberately downclock PCIe at idle to save power and re-train to full speed under load, so an idle Gen1 reading alone can't distinguish real hardware/riser degradation from normal power management. Resample the same sysfs link attributes while bee-gpu-burn is actively loading the GPUs so both bundles ship together. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
3ec7ca08da
commit
b1f165edb3
@@ -227,6 +227,7 @@ func (a *App) RunAudit(runtimeMode runtimeenv.Mode, output string) (string, erro
|
||||
result := collector.Run(runtimeMode)
|
||||
applyLatestSATStatuses(&result.Hardware, DefaultSATBaseDir, a.StatusDB)
|
||||
writePSUStatusesToDB(a.StatusDB, result.Hardware.PowerSupplies)
|
||||
writePCIeGPUStatusesToDB(a.StatusDB, result.Hardware.PCIeDevices)
|
||||
if health, err := ReadRuntimeHealth(DefaultRuntimeJSONPath); err == nil {
|
||||
result.Runtime = &health
|
||||
}
|
||||
@@ -450,6 +451,58 @@ func writePSUStatusesToDB(db *ComponentStatusDB, psus []schema.HardwarePowerSupp
|
||||
db.Record("psu:all", source, worstStatus, "")
|
||||
}
|
||||
|
||||
// writePCIeGPUStatusesToDB records GPU PCIe-link statuses (e.g. link speed
|
||||
// degraded to Gen1 while capable of Gen4) collected during audit into the
|
||||
// component-status DB. Without this, the DB's pcie:gpu:<vendor> key is only
|
||||
// ever written by SAT diag results (ApplySATResultToDB), and none of the
|
||||
// nvidia/nvidia-config/nvidia-interconnect/nvidia-bandwidth SAT jobs check
|
||||
// PCIe link speed — so a real Gen1/Gen4 degradation the collector already
|
||||
// flagged as Warning in the hardware snapshot never surfaces in the DB-backed
|
||||
// "check passed" status the Hardware Summary/webui reads.
|
||||
func writePCIeGPUStatusesToDB(db *ComponentStatusDB, devices []schema.HardwarePCIeDevice) {
|
||||
if db == nil || len(devices) == 0 {
|
||||
return
|
||||
}
|
||||
const source = "audit:pcie"
|
||||
worst := map[string]string{}
|
||||
detail := map[string]string{}
|
||||
for _, dev := range devices {
|
||||
if dev.Status == nil {
|
||||
continue
|
||||
}
|
||||
st := *dev.Status
|
||||
for _, vendor := range []string{"nvidia", "amd"} {
|
||||
if !matchesGPUVendor(dev, vendor) {
|
||||
continue
|
||||
}
|
||||
if _, seen := worst[vendor]; !seen {
|
||||
worst[vendor] = "OK"
|
||||
}
|
||||
switch st {
|
||||
case "Critical":
|
||||
worst[vendor] = "Critical"
|
||||
if dev.ErrorDescription != nil {
|
||||
detail[vendor] = *dev.ErrorDescription
|
||||
}
|
||||
case "Warning":
|
||||
if worst[vendor] != "Critical" {
|
||||
worst[vendor] = "Warning"
|
||||
if dev.ErrorDescription != nil {
|
||||
detail[vendor] = *dev.ErrorDescription
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, vendor := range []string{"nvidia", "amd"} {
|
||||
status, present := worst[vendor]
|
||||
if !present {
|
||||
continue
|
||||
}
|
||||
db.Record("pcie:gpu:"+vendor, source, status, detail[vendor])
|
||||
}
|
||||
}
|
||||
|
||||
func ReadRuntimeHealth(path string) (schema.RuntimeHealth, error) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user