fix(collector): stop NVIDIA enrichment from clobbering PCIe status

enrichPCIeWithNVIDIAData unconditionally overwrote dev.Status after
collectPCIe() had already flagged a Warning/Critical (e.g. PCIe link
speed degraded), so a clean ECC/remap/reset readout silently downgraded
that finding back to OK while leaving the stale ErrorDescription behind.

Add a severity-ordered merge (OK/Unknown < Warning < Critical) shared
via mergeDeviceStatus in contract.go, and route both the NVIDIA status
calculation and the driver-unavailable fallback through it.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-08-04 17:28:47 +03:00
co-authored by Claude Sonnet 5
parent 6c8be629d2
commit a34e823f82
3 changed files with 153 additions and 8 deletions
+102
View File
@@ -192,6 +192,108 @@ func TestEnrichPCIeWithNVIDIAData_remapFailureCritical(t *testing.T) {
}
}
func TestEnrichPCIeWithNVIDIAData_preservesExistingPCIeLinkWarning(t *testing.T) {
vendorID := NvidiaVendorID
bdf := "0000:65:00.0"
status := statusWarning
desc := "PCIe link speed degraded: running at Gen1, capable of Gen4"
devices := []schema.HardwarePCIeDevice{
{
HardwareComponentStatus: schema.HardwareComponentStatus{Status: &status, ErrorDescription: &desc},
VendorID: &vendorID,
BDF: &bdf,
},
}
// Clean ECC/remap/reset: NVIDIA enrichment itself sees nothing wrong.
byBDF := map[string]nvidiaGPUInfo{
"0000:65:00.0": {},
}
out := enrichPCIeWithNVIDIAData(devices, byBDF, true)
if out[0].Status == nil || *out[0].Status != statusWarning {
t.Fatalf("status: got %v, want %v (must not be overwritten by clean NVIDIA telemetry)", out[0].Status, statusWarning)
}
if out[0].ErrorDescription == nil || *out[0].ErrorDescription != desc {
t.Fatalf("error_description: got %v, want %q", out[0].ErrorDescription, desc)
}
}
func TestEnrichPCIeWithNVIDIAData_eccWarningDoesNotDowngradeExistingWarning(t *testing.T) {
vendorID := NvidiaVendorID
bdf := "0000:65:00.0"
status := statusWarning
desc := "PCIe link speed degraded: running at Gen1, capable of Gen4"
devices := []schema.HardwarePCIeDevice{
{
HardwareComponentStatus: schema.HardwareComponentStatus{Status: &status, ErrorDescription: &desc},
VendorID: &vendorID,
BDF: &bdf,
},
}
byBDF := map[string]nvidiaGPUInfo{
"0000:65:00.0": {ECCUncorrected: ptrInt64(1)},
}
out := enrichPCIeWithNVIDIAData(devices, byBDF, true)
// ECCUncorrected only raises Warning (same severity as the pre-existing
// PCIe-link Warning), so the merge must not lose the already-detected
// (and equally severe) status/description.
if out[0].Status == nil || *out[0].Status != statusWarning {
t.Fatalf("status: got %v, want %v", out[0].Status, statusWarning)
}
if out[0].ErrorDescription == nil || *out[0].ErrorDescription != desc {
t.Fatalf("error_description: got %v, want %q (equal-severity ECC finding must not clobber it)", out[0].ErrorDescription, desc)
}
}
func TestEnrichPCIeWithNVIDIAData_resetRequiredEscalatesOverExistingWarning(t *testing.T) {
vendorID := NvidiaVendorID
bdf := "0000:65:00.0"
status := statusWarning
desc := "PCIe link speed degraded: running at Gen1, capable of Gen4"
devices := []schema.HardwarePCIeDevice{
{
HardwareComponentStatus: schema.HardwareComponentStatus{Status: &status, ErrorDescription: &desc},
VendorID: &vendorID,
BDF: &bdf,
},
}
byBDF := map[string]nvidiaGPUInfo{
"0000:65:00.0": {ResetRequired: ptrBool(true)},
}
out := enrichPCIeWithNVIDIAData(devices, byBDF, true)
if out[0].Status == nil || *out[0].Status != statusCritical {
t.Fatalf("status: got %v, want %v (reset-required Critical must escalate over the PCIe-link Warning)", out[0].Status, statusCritical)
}
if out[0].ErrorDescription == nil || *out[0].ErrorDescription != "GPU requires a reset" {
t.Fatalf("error_description: got %v", out[0].ErrorDescription)
}
}
func TestEnrichPCIeWithNVIDIAData_regressionNoOverride(t *testing.T) {
vendorID := NvidiaVendorID
bdf := "0000:65:00.0"
devices := []schema.HardwarePCIeDevice{
{VendorID: &vendorID, BDF: &bdf},
}
byBDF := map[string]nvidiaGPUInfo{
"0000:65:00.0": {},
}
out := enrichPCIeWithNVIDIAData(devices, byBDF, true)
if out[0].Status == nil || *out[0].Status != statusOK {
t.Fatalf("status: got %v, want %v", out[0].Status, statusOK)
}
if out[0].ErrorDescription != nil {
t.Fatalf("error_description: got %v, want nil", out[0].ErrorDescription)
}
}
func ptrInt64(v int64) *int64 { return &v }
func ptrFloat(v float64) *float64 { return &v }
func ptrBool(v bool) *bool { return &v }