gpu: collect reset-required/row-remap status and classify Xid codes by severity
nvidia-smi exposes reset_status.reset_required and remapped_rows.* only on newer drivers for Ampere+ GPUs; queried via a separate exec call since an unrecognized field name fails the whole --query-gpu command and would have wiped out unrelated telemetry (temp/ECC/power) on older drivers otherwise. Also refines Xid severity in both the ingest dmesg collector and the always-on kmsg watcher: Xid 64 (row-remap InfoROM write failure) now escalates to Critical instead of the generic warning every other Xid got, and fixes the SAT-window flush path which previously hardcoded "Warning" and ignored pattern severity entirely. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
d850d4fc3a
commit
1175e6ccd8
@@ -124,5 +124,74 @@ func TestEnrichPCIeWithNVIDIAData_driverMissingFallback(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseNVIDIAReliability(t *testing.T) {
|
||||
raw := "0000:65:00.0, No, 0, 2, No, Yes\n"
|
||||
byBDF, err := parseNVIDIAReliabilityCSV(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("parse failed: %v", err)
|
||||
}
|
||||
|
||||
gpu, ok := byBDF["0000:65:00.0"]
|
||||
if !ok {
|
||||
t.Fatalf("gpu by normalized bdf not found")
|
||||
}
|
||||
if gpu.ResetRequired == nil || *gpu.ResetRequired {
|
||||
t.Fatalf("reset_required: got %v, want false", gpu.ResetRequired)
|
||||
}
|
||||
if gpu.RemapCorrectable == nil || *gpu.RemapCorrectable != 0 {
|
||||
t.Fatalf("remap correctable: got %v", gpu.RemapCorrectable)
|
||||
}
|
||||
if gpu.RemapUncorrectable == nil || *gpu.RemapUncorrectable != 2 {
|
||||
t.Fatalf("remap uncorrectable: got %v", gpu.RemapUncorrectable)
|
||||
}
|
||||
if gpu.RemapPending == nil || *gpu.RemapPending {
|
||||
t.Fatalf("remap pending: got %v, want false", gpu.RemapPending)
|
||||
}
|
||||
if gpu.RemapFailure == nil || !*gpu.RemapFailure {
|
||||
t.Fatalf("remap failure: got %v, want true", gpu.RemapFailure)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnrichPCIeWithNVIDIAData_resetRequiredCritical(t *testing.T) {
|
||||
vendorID := NvidiaVendorID
|
||||
bdf := "0000:65:00.0"
|
||||
devices := []schema.HardwarePCIeDevice{
|
||||
{VendorID: &vendorID, BDF: &bdf},
|
||||
}
|
||||
|
||||
byBDF := map[string]nvidiaGPUInfo{
|
||||
"0000:65:00.0": {ResetRequired: ptrBool(true)},
|
||||
}
|
||||
|
||||
out := enrichPCIeWithNVIDIAData(devices, byBDF, true)
|
||||
if out[0].Status == nil || *out[0].Status != statusCritical {
|
||||
t.Fatalf("status: got %v, want %v", out[0].Status, statusCritical)
|
||||
}
|
||||
if out[0].ResetRequired == nil || !*out[0].ResetRequired {
|
||||
t.Fatalf("reset_required: got %v", out[0].ResetRequired)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnrichPCIeWithNVIDIAData_remapFailureCritical(t *testing.T) {
|
||||
vendorID := NvidiaVendorID
|
||||
bdf := "0000:65:00.0"
|
||||
devices := []schema.HardwarePCIeDevice{
|
||||
{VendorID: &vendorID, BDF: &bdf},
|
||||
}
|
||||
|
||||
byBDF := map[string]nvidiaGPUInfo{
|
||||
"0000:65:00.0": {RemapFailure: ptrBool(true)},
|
||||
}
|
||||
|
||||
out := enrichPCIeWithNVIDIAData(devices, byBDF, true)
|
||||
if out[0].Status == nil || *out[0].Status != statusCritical {
|
||||
t.Fatalf("status: got %v, want %v", out[0].Status, statusCritical)
|
||||
}
|
||||
if out[0].RemappedRowsFailure == nil || !*out[0].RemappedRowsFailure {
|
||||
t.Fatalf("remapped_rows_failure: got %v", out[0].RemappedRowsFailure)
|
||||
}
|
||||
}
|
||||
|
||||
func ptrInt64(v int64) *int64 { return &v }
|
||||
func ptrFloat(v float64) *float64 { return &v }
|
||||
func ptrBool(v bool) *bool { return &v }
|
||||
|
||||
Reference in New Issue
Block a user