gpu: collect reset-required/row-remap status and classify Xid codes by severity

nvidia-smi exposes reset_status.reset_required and remapped_rows.* only on
newer drivers for Ampere+ GPUs; queried via a separate exec call since an
unrecognized field name fails the whole --query-gpu command and would have
wiped out unrelated telemetry (temp/ECC/power) on older drivers otherwise.

Also refines Xid severity in both the ingest dmesg collector and the
always-on kmsg watcher: Xid 64 (row-remap InfoROM write failure) now
escalates to Critical instead of the generic warning every other Xid got,
and fixes the SAT-window flush path which previously hardcoded "Warning"
and ignored pattern severity entirely.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-07-07 19:28:59 +03:00
co-authored by Claude Sonnet 5
parent d850d4fc3a
commit 1175e6ccd8
10 changed files with 477 additions and 69 deletions
+69
View File
@@ -124,5 +124,74 @@ func TestEnrichPCIeWithNVIDIAData_driverMissingFallback(t *testing.T) {
}
}
func TestParseNVIDIAReliability(t *testing.T) {
raw := "0000:65:00.0, No, 0, 2, No, Yes\n"
byBDF, err := parseNVIDIAReliabilityCSV(raw)
if err != nil {
t.Fatalf("parse failed: %v", err)
}
gpu, ok := byBDF["0000:65:00.0"]
if !ok {
t.Fatalf("gpu by normalized bdf not found")
}
if gpu.ResetRequired == nil || *gpu.ResetRequired {
t.Fatalf("reset_required: got %v, want false", gpu.ResetRequired)
}
if gpu.RemapCorrectable == nil || *gpu.RemapCorrectable != 0 {
t.Fatalf("remap correctable: got %v", gpu.RemapCorrectable)
}
if gpu.RemapUncorrectable == nil || *gpu.RemapUncorrectable != 2 {
t.Fatalf("remap uncorrectable: got %v", gpu.RemapUncorrectable)
}
if gpu.RemapPending == nil || *gpu.RemapPending {
t.Fatalf("remap pending: got %v, want false", gpu.RemapPending)
}
if gpu.RemapFailure == nil || !*gpu.RemapFailure {
t.Fatalf("remap failure: got %v, want true", gpu.RemapFailure)
}
}
func TestEnrichPCIeWithNVIDIAData_resetRequiredCritical(t *testing.T) {
vendorID := NvidiaVendorID
bdf := "0000:65:00.0"
devices := []schema.HardwarePCIeDevice{
{VendorID: &vendorID, BDF: &bdf},
}
byBDF := map[string]nvidiaGPUInfo{
"0000:65:00.0": {ResetRequired: ptrBool(true)},
}
out := enrichPCIeWithNVIDIAData(devices, byBDF, true)
if out[0].Status == nil || *out[0].Status != statusCritical {
t.Fatalf("status: got %v, want %v", out[0].Status, statusCritical)
}
if out[0].ResetRequired == nil || !*out[0].ResetRequired {
t.Fatalf("reset_required: got %v", out[0].ResetRequired)
}
}
func TestEnrichPCIeWithNVIDIAData_remapFailureCritical(t *testing.T) {
vendorID := NvidiaVendorID
bdf := "0000:65:00.0"
devices := []schema.HardwarePCIeDevice{
{VendorID: &vendorID, BDF: &bdf},
}
byBDF := map[string]nvidiaGPUInfo{
"0000:65:00.0": {RemapFailure: ptrBool(true)},
}
out := enrichPCIeWithNVIDIAData(devices, byBDF, true)
if out[0].Status == nil || *out[0].Status != statusCritical {
t.Fatalf("status: got %v, want %v", out[0].Status, statusCritical)
}
if out[0].RemappedRowsFailure == nil || !*out[0].RemappedRowsFailure {
t.Fatalf("remapped_rows_failure: got %v", out[0].RemappedRowsFailure)
}
}
func ptrInt64(v int64) *int64 { return &v }
func ptrFloat(v float64) *float64 { return &v }
func ptrBool(v bool) *bool { return &v }