fix(collector): stop NVIDIA enrichment from clobbering PCIe status
enrichPCIeWithNVIDIAData unconditionally overwrote dev.Status after collectPCIe() had already flagged a Warning/Critical (e.g. PCIe link speed degraded), so a clean ECC/remap/reset readout silently downgraded that finding back to OK while leaving the stale ErrorDescription behind. Add a severity-ordered merge (OK/Unknown < Warning < Critical) shared via mergeDeviceStatus in contract.go, and route both the NVIDIA status calculation and the driver-unavailable fallback through it. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
6c8be629d2
commit
a34e823f82
@@ -192,6 +192,108 @@ func TestEnrichPCIeWithNVIDIAData_remapFailureCritical(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnrichPCIeWithNVIDIAData_preservesExistingPCIeLinkWarning(t *testing.T) {
|
||||
vendorID := NvidiaVendorID
|
||||
bdf := "0000:65:00.0"
|
||||
status := statusWarning
|
||||
desc := "PCIe link speed degraded: running at Gen1, capable of Gen4"
|
||||
devices := []schema.HardwarePCIeDevice{
|
||||
{
|
||||
HardwareComponentStatus: schema.HardwareComponentStatus{Status: &status, ErrorDescription: &desc},
|
||||
VendorID: &vendorID,
|
||||
BDF: &bdf,
|
||||
},
|
||||
}
|
||||
|
||||
// Clean ECC/remap/reset: NVIDIA enrichment itself sees nothing wrong.
|
||||
byBDF := map[string]nvidiaGPUInfo{
|
||||
"0000:65:00.0": {},
|
||||
}
|
||||
|
||||
out := enrichPCIeWithNVIDIAData(devices, byBDF, true)
|
||||
if out[0].Status == nil || *out[0].Status != statusWarning {
|
||||
t.Fatalf("status: got %v, want %v (must not be overwritten by clean NVIDIA telemetry)", out[0].Status, statusWarning)
|
||||
}
|
||||
if out[0].ErrorDescription == nil || *out[0].ErrorDescription != desc {
|
||||
t.Fatalf("error_description: got %v, want %q", out[0].ErrorDescription, desc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnrichPCIeWithNVIDIAData_eccWarningDoesNotDowngradeExistingWarning(t *testing.T) {
|
||||
vendorID := NvidiaVendorID
|
||||
bdf := "0000:65:00.0"
|
||||
status := statusWarning
|
||||
desc := "PCIe link speed degraded: running at Gen1, capable of Gen4"
|
||||
devices := []schema.HardwarePCIeDevice{
|
||||
{
|
||||
HardwareComponentStatus: schema.HardwareComponentStatus{Status: &status, ErrorDescription: &desc},
|
||||
VendorID: &vendorID,
|
||||
BDF: &bdf,
|
||||
},
|
||||
}
|
||||
|
||||
byBDF := map[string]nvidiaGPUInfo{
|
||||
"0000:65:00.0": {ECCUncorrected: ptrInt64(1)},
|
||||
}
|
||||
|
||||
out := enrichPCIeWithNVIDIAData(devices, byBDF, true)
|
||||
// ECCUncorrected only raises Warning (same severity as the pre-existing
|
||||
// PCIe-link Warning), so the merge must not lose the already-detected
|
||||
// (and equally severe) status/description.
|
||||
if out[0].Status == nil || *out[0].Status != statusWarning {
|
||||
t.Fatalf("status: got %v, want %v", out[0].Status, statusWarning)
|
||||
}
|
||||
if out[0].ErrorDescription == nil || *out[0].ErrorDescription != desc {
|
||||
t.Fatalf("error_description: got %v, want %q (equal-severity ECC finding must not clobber it)", out[0].ErrorDescription, desc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnrichPCIeWithNVIDIAData_resetRequiredEscalatesOverExistingWarning(t *testing.T) {
|
||||
vendorID := NvidiaVendorID
|
||||
bdf := "0000:65:00.0"
|
||||
status := statusWarning
|
||||
desc := "PCIe link speed degraded: running at Gen1, capable of Gen4"
|
||||
devices := []schema.HardwarePCIeDevice{
|
||||
{
|
||||
HardwareComponentStatus: schema.HardwareComponentStatus{Status: &status, ErrorDescription: &desc},
|
||||
VendorID: &vendorID,
|
||||
BDF: &bdf,
|
||||
},
|
||||
}
|
||||
|
||||
byBDF := map[string]nvidiaGPUInfo{
|
||||
"0000:65:00.0": {ResetRequired: ptrBool(true)},
|
||||
}
|
||||
|
||||
out := enrichPCIeWithNVIDIAData(devices, byBDF, true)
|
||||
if out[0].Status == nil || *out[0].Status != statusCritical {
|
||||
t.Fatalf("status: got %v, want %v (reset-required Critical must escalate over the PCIe-link Warning)", out[0].Status, statusCritical)
|
||||
}
|
||||
if out[0].ErrorDescription == nil || *out[0].ErrorDescription != "GPU requires a reset" {
|
||||
t.Fatalf("error_description: got %v", out[0].ErrorDescription)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnrichPCIeWithNVIDIAData_regressionNoOverride(t *testing.T) {
|
||||
vendorID := NvidiaVendorID
|
||||
bdf := "0000:65:00.0"
|
||||
devices := []schema.HardwarePCIeDevice{
|
||||
{VendorID: &vendorID, BDF: &bdf},
|
||||
}
|
||||
|
||||
byBDF := map[string]nvidiaGPUInfo{
|
||||
"0000:65:00.0": {},
|
||||
}
|
||||
|
||||
out := enrichPCIeWithNVIDIAData(devices, byBDF, true)
|
||||
if out[0].Status == nil || *out[0].Status != statusOK {
|
||||
t.Fatalf("status: got %v, want %v", out[0].Status, statusOK)
|
||||
}
|
||||
if out[0].ErrorDescription != nil {
|
||||
t.Fatalf("error_description: got %v, want nil", out[0].ErrorDescription)
|
||||
}
|
||||
}
|
||||
|
||||
func ptrInt64(v int64) *int64 { return &v }
|
||||
func ptrFloat(v float64) *float64 { return &v }
|
||||
func ptrBool(v bool) *bool { return &v }
|
||||
|
||||
Reference in New Issue
Block a user