gpu: collect reset-required/row-remap status and classify Xid codes by severity

nvidia-smi exposes reset_status.reset_required and remapped_rows.* only on
newer drivers for Ampere+ GPUs; queried via a separate exec call since an
unrecognized field name fails the whole --query-gpu command and would have
wiped out unrelated telemetry (temp/ECC/power) on older drivers otherwise.

Also refines Xid severity in both the ingest dmesg collector and the
always-on kmsg watcher: Xid 64 (row-remap InfoROM write failure) now
escalates to Critical instead of the generic warning every other Xid got,
and fixes the SAT-window flush path which previously hardcoded "Warning"
and ignored pattern severity entirely.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

audit/internal/collector/dmesg_events.go

@@ -109,6 +109,12 @@ func matchesAny(s string, patterns []*regexp.Regexp) bool {
 }
 
 func dmesgSeverity(msg string) string {
+	if sev, ok := XidSeverity(msg); ok {
+		if sev == "critical" {
+			return statusCritical
+		}
+		return statusWarning
+	}
 	lower := strings.ToLower(msg)
 	switch {
 	case strings.Contains(lower, "panic") ||

audit/internal/collector/dmesg_events_test.go

@@ -0,0 +1,46 @@
+package collector
+
+import "testing"
+
+func TestDmesgSeverity_xidCodes(t *testing.T) {
+	tests := []struct {
+		name string
+		msg  string
+		want string
+	}{
+		{
+			name: "xid 64 remap write failure escalates to critical",
+			msg:  "NVRM: Xid (PCI:0000:65:00): 64, pid=1234, name=python",
+			want: statusCritical,
+		},
+		{
+			name: "xid 94 contained ecc is downgraded to warning",
+			msg:  "NVRM: Xid (PCI:0000:65:00): 94, pid=1234, name=python",
+			want: statusWarning,
+		},
+		{
+			name: "xid 63 remap committed is downgraded to warning",
+			msg:  "NVRM: Xid (PCI:0000:65:00): 63, pid=1234, Class 0x90a0",
+			want: statusWarning,
+		},
+		{
+			name: "unrecognized xid code falls back to generic critical",
+			msg:  "NVRM: Xid (PCI:0000:65:00): 79, pid=1234, GPU has fallen off the bus",
+			want: statusCritical,
+		},
+		{
+			name: "non-xid generic error keyword",
+			msg:  "blk_update_request: I/O error, dev sda",
+			want: statusCritical,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := dmesgSeverity(tt.msg)
+			if got != tt.want {
+				t.Fatalf("dmesgSeverity(%q) = %q, want %q", tt.msg, got, tt.want)
+			}
+		})
+	}
+}

audit/internal/collector/nvidia.go

@@ -25,6 +25,11 @@ type nvidiaGPUInfo struct {
 	PCIeLinkGenMax     *int
 	PCIeLinkWidthCur   *int
 	PCIeLinkWidthMax   *int
+	ResetRequired      *bool
+	RemapCorrectable   *int64
+	RemapUncorrectable *int64
+	RemapPending       *bool
+	RemapFailure       *bool
 }
 
 // enrichPCIeWithNVIDIA enriches NVIDIA PCIe devices with data from nvidia-smi.
@@ -87,6 +92,18 @@ func enrichPCIeWithNVIDIAData(devs []schema.HardwarePCIeDevice, gpuByBDF map[str
 			status = statusWarning
 			devs[i].ErrorDescription = stringPtr("GPU reports uncorrected ECC errors")
 		}
+		if info.RemapUncorrectable != nil && *info.RemapUncorrectable > 0 {
+			status = statusWarning
+			devs[i].ErrorDescription = stringPtr("GPU has uncorrectable row remap events (bad HBM cell repair scheduled)")
+		}
+		if info.RemapFailure != nil && *info.RemapFailure {
+			status = statusCritical
+			devs[i].ErrorDescription = stringPtr("GPU row remap failed to commit to InfoROM (XID 64)")
+		}
+		if info.ResetRequired != nil && *info.ResetRequired {
+			status = statusCritical
+			devs[i].ErrorDescription = stringPtr("GPU requires a reset")
+		}
 		devs[i].Status = &status
 		injectNVIDIATelemetry(&devs[i], info)
 		enriched++
@@ -107,7 +124,74 @@ func queryNVIDIAGPUs() (map[string]nvidiaGPUInfo, error) {
 	if err != nil {
 		return nil, err
 	}
-	return parseNVIDIASMIQuery(string(out))
+	result, err := parseNVIDIASMIQuery(string(out))
+	if err != nil {
+		return nil, err
+	}
+
+	// reset_status.* and remapped_rows.* are only recognized by newer drivers
+	// on Ampere+ GPUs; an unrecognized field name fails the whole nvidia-smi
+	// call, so this is queried separately to avoid losing the fields above.
+	if reliability, err := queryNVIDIAReliability(); err != nil {
+		slog.Info("nvidia: reliability fields skipped", "err", err)
+	} else {
+		for bdf, r := range reliability {
+			if info, ok := result[bdf]; ok {
+				info.ResetRequired = r.ResetRequired
+				info.RemapCorrectable = r.RemapCorrectable
+				info.RemapUncorrectable = r.RemapUncorrectable
+				info.RemapPending = r.RemapPending
+				info.RemapFailure = r.RemapFailure
+				result[bdf] = info
+			}
+		}
+	}
+
+	return result, nil
+}
+
+func queryNVIDIAReliability() (map[string]nvidiaGPUInfo, error) {
+	out, err := exec.Command(
+		"nvidia-smi",
+		"--query-gpu=pci.bus_id,reset_status.reset_required,remapped_rows.correctable,remapped_rows.uncorrectable,remapped_rows.pending,remapped_rows.failure",
+		"--format=csv,noheader,nounits",
+	).Output()
+	if err != nil {
+		return nil, err
+	}
+	return parseNVIDIAReliabilityCSV(string(out))
+}
+
+func parseNVIDIAReliabilityCSV(raw string) (map[string]nvidiaGPUInfo, error) {
+	r := csv.NewReader(strings.NewReader(raw))
+	r.TrimLeadingSpace = true
+	r.FieldsPerRecord = -1
+	records, err := r.ReadAll()
+	if err != nil {
+		return nil, err
+	}
+
+	result := make(map[string]nvidiaGPUInfo)
+	for _, rec := range records {
+		if len(rec) == 0 {
+			continue
+		}
+		if len(rec) < 6 {
+			return nil, fmt.Errorf("unexpected nvidia-smi reliability columns: got %d, want 6", len(rec))
+		}
+		bdf := normalizePCIeBDF(rec[0])
+		if bdf == "" {
+			continue
+		}
+		result[bdf] = nvidiaGPUInfo{
+			ResetRequired:      parseMaybeBool(rec[1]),
+			RemapCorrectable:   parseMaybeInt64(rec[2]),
+			RemapUncorrectable: parseMaybeInt64(rec[3]),
+			RemapPending:       parseMaybeBool(rec[4]),
+			RemapFailure:       parseMaybeBool(rec[5]),
+		}
+	}
+	return result, nil
 }
 
 func parseNVIDIASMIQuery(raw string) (map[string]nvidiaGPUInfo, error) {
@@ -207,10 +291,10 @@ func pcieLinkGenLabel(gen int) string {
 func parseMaybeBool(v string) *bool {
 	v = strings.TrimSpace(strings.ToLower(v))
 	switch v {
-	case "active", "enabled", "true", "1":
+	case "active", "enabled", "true", "1", "yes":
 		b := true
 		return &b
-	case "not active", "disabled", "false", "0":
+	case "not active", "disabled", "false", "0", "no":
 		b := false
 		return &b
 	default:
@@ -266,6 +350,21 @@ func injectNVIDIATelemetry(dev *schema.HardwarePCIeDevice, info nvidiaGPUInfo) {
 	if info.HWSlowdown != nil {
 		dev.HWSlowdown = info.HWSlowdown
 	}
+	if info.ResetRequired != nil {
+		dev.ResetRequired = info.ResetRequired
+	}
+	if info.RemapCorrectable != nil {
+		dev.RemappedRowsCorrectable = info.RemapCorrectable
+	}
+	if info.RemapUncorrectable != nil {
+		dev.RemappedRowsUncorrectable = info.RemapUncorrectable
+	}
+	if info.RemapPending != nil {
+		dev.RemappedRowsPending = info.RemapPending
+	}
+	if info.RemapFailure != nil {
+		dev.RemappedRowsFailure = info.RemapFailure
+	}
 	// Override PCIe link speed/width with nvidia-smi driver values.
 	// sysfs current_link_speed reflects the instantaneous physical link state and
 	// can show Gen1 when the GPU is idle due to ASPM power management. The driver

audit/internal/collector/nvidia_test.go

@@ -124,5 +124,74 @@ func TestEnrichPCIeWithNVIDIAData_driverMissingFallback(t *testing.T) {
 	}
 }
 
+func TestParseNVIDIAReliability(t *testing.T) {
+	raw := "0000:65:00.0, No, 0, 2, No, Yes\n"
+	byBDF, err := parseNVIDIAReliabilityCSV(raw)
+	if err != nil {
+		t.Fatalf("parse failed: %v", err)
+	}
+
+	gpu, ok := byBDF["0000:65:00.0"]
+	if !ok {
+		t.Fatalf("gpu by normalized bdf not found")
+	}
+	if gpu.ResetRequired == nil || *gpu.ResetRequired {
+		t.Fatalf("reset_required: got %v, want false", gpu.ResetRequired)
+	}
+	if gpu.RemapCorrectable == nil || *gpu.RemapCorrectable != 0 {
+		t.Fatalf("remap correctable: got %v", gpu.RemapCorrectable)
+	}
+	if gpu.RemapUncorrectable == nil || *gpu.RemapUncorrectable != 2 {
+		t.Fatalf("remap uncorrectable: got %v", gpu.RemapUncorrectable)
+	}
+	if gpu.RemapPending == nil || *gpu.RemapPending {
+		t.Fatalf("remap pending: got %v, want false", gpu.RemapPending)
+	}
+	if gpu.RemapFailure == nil || !*gpu.RemapFailure {
+		t.Fatalf("remap failure: got %v, want true", gpu.RemapFailure)
+	}
+}
+
+func TestEnrichPCIeWithNVIDIAData_resetRequiredCritical(t *testing.T) {
+	vendorID := NvidiaVendorID
+	bdf := "0000:65:00.0"
+	devices := []schema.HardwarePCIeDevice{
+		{VendorID: &vendorID, BDF: &bdf},
+	}
+
+	byBDF := map[string]nvidiaGPUInfo{
+		"0000:65:00.0": {ResetRequired: ptrBool(true)},
+	}
+
+	out := enrichPCIeWithNVIDIAData(devices, byBDF, true)
+	if out[0].Status == nil || *out[0].Status != statusCritical {
+		t.Fatalf("status: got %v, want %v", out[0].Status, statusCritical)
+	}
+	if out[0].ResetRequired == nil || !*out[0].ResetRequired {
+		t.Fatalf("reset_required: got %v", out[0].ResetRequired)
+	}
+}
+
+func TestEnrichPCIeWithNVIDIAData_remapFailureCritical(t *testing.T) {
+	vendorID := NvidiaVendorID
+	bdf := "0000:65:00.0"
+	devices := []schema.HardwarePCIeDevice{
+		{VendorID: &vendorID, BDF: &bdf},
+	}
+
+	byBDF := map[string]nvidiaGPUInfo{
+		"0000:65:00.0": {RemapFailure: ptrBool(true)},
+	}
+
+	out := enrichPCIeWithNVIDIAData(devices, byBDF, true)
+	if out[0].Status == nil || *out[0].Status != statusCritical {
+		t.Fatalf("status: got %v, want %v", out[0].Status, statusCritical)
+	}
+	if out[0].RemappedRowsFailure == nil || !*out[0].RemappedRowsFailure {
+		t.Fatalf("remapped_rows_failure: got %v", out[0].RemappedRowsFailure)
+	}
+}
+
 func ptrInt64(v int64) *int64     { return &v }
 func ptrFloat(v float64) *float64 { return &v }
+func ptrBool(v bool) *bool        { return &v }

audit/internal/collector/xid.go

@@ -0,0 +1,53 @@
+package collector
+
+import "regexp"
+
+// xidCodeParenRE extracts the NVIDIA Xid error code from the common
+// "Xid (PCI:0000:65:00): 64, ..." / "Xid (0000:65:00.0): 64, ..." form, where
+// the BDF inside the parens contains digits that a naive "next number" regex
+// would grab instead of the actual code.
+var xidCodeParenRE = regexp.MustCompile(`(?i)Xid\s*\([^)]*\)\s*:?\s*(\d+)`)
+
+// xidCodeColonRE handles the older "Xid: 64, ..." form with no BDF parens.
+var xidCodeColonRE = regexp.MustCompile(`(?i)\bXid\s*:\s*(\d+)`)
+
+// xidCodeSeverity maps NVIDIA Xid codes relevant to GPU HBM/ECC health to a
+// severity, refining the generic "nvidia-xid" kernel-log pattern's default
+// "warning". Xid 64 is the same InfoROM row-remap-write failure surfaced by
+// this package's remapped_rows_failure field (see nvidia.go), so it must
+// escalate to critical rather than the generic warning every other Xid gets.
+// Codes not listed here keep the caller's default severity.
+// Source: NVIDIA GPU Memory Error Management docs + field experience (Xid 48
+// uncorrectable ECC, 63 remap committed, 64 remap write failed, 94 contained
+// ECC, 95 uncontained ECC, 160 memory marked for repair).
+var xidCodeSeverity = map[string]string{
+	"48":  "critical",
+	"64":  "critical",
+	"95":  "critical",
+	"63":  "warning",
+	"94":  "warning",
+	"160": "warning",
+}
+
+// XidSeverity returns the refined severity ("critical"/"warning") for a kernel
+// log line containing an NVIDIA Xid error, if the specific code is known. ok
+// is false when no Xid code could be extracted or the code isn't in
+// xidCodeSeverity, in which case callers should fall back to their own default.
+func XidSeverity(line string) (severity string, ok bool) {
+	code, ok := extractXidCode(line)
+	if !ok {
+		return "", false
+	}
+	sev, ok := xidCodeSeverity[code]
+	return sev, ok
+}
+
+func extractXidCode(line string) (string, bool) {
+	if m := xidCodeParenRE.FindStringSubmatch(line); m != nil {
+		return m[1], true
+	}
+	if m := xidCodeColonRE.FindStringSubmatch(line); m != nil {
+		return m[1], true
+	}
+	return "", false
+}

audit/internal/collector/xid_test.go

@@ -0,0 +1,53 @@
+package collector
+
+import "testing"
+
+func TestXidSeverity(t *testing.T) {
+	tests := []struct {
+		name    string
+		line    string
+		wantSev string
+		wantOK  bool
+	}{
+		{
+			name:    "xid 64 remap write failure is critical",
+			line:    "NVRM: Xid (PCI:0000:65:00): 64, pid=1234, name=python, Row Remapper: New row remapping failed",
+			wantSev: "critical",
+			wantOK:  true,
+		},
+		{
+			name:    "xid 48 uncorrectable ecc is critical",
+			line:    "NVRM: Xid (PCI:0000:65:00): 48, pid=1234, name=python",
+			wantSev: "critical",
+			wantOK:  true,
+		},
+		{
+			name:    "xid 63 remap committed is warning",
+			line:    "NVRM: Xid (PCI:0000:65:00): 63, pid=1234, Class 0x90a0",
+			wantSev: "warning",
+			wantOK:  true,
+		},
+		{
+			name:   "xid 79 unknown code falls back to caller default",
+			line:   "NVRM: Xid (PCI:0000:65:00): 79, pid=1234, GPU has fallen off the bus",
+			wantOK: false,
+		},
+		{
+			name:   "no xid in line",
+			line:   "NVRM: GPU 0000:65:00.0: RmInitAdapter failed",
+			wantOK: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			sev, ok := XidSeverity(tt.line)
+			if ok != tt.wantOK {
+				t.Fatalf("ok: got %v, want %v", ok, tt.wantOK)
+			}
+			if ok && sev != tt.wantSev {
+				t.Fatalf("severity: got %q, want %q", sev, tt.wantSev)
+			}
+		})
+	}
+}