gpu: collect reset-required/row-remap status and classify Xid codes by severity

nvidia-smi exposes reset_status.reset_required and remapped_rows.* only on
newer drivers for Ampere+ GPUs; queried via a separate exec call since an
unrecognized field name fails the whole --query-gpu command and would have
wiped out unrelated telemetry (temp/ECC/power) on older drivers otherwise.

Also refines Xid severity in both the ingest dmesg collector and the
always-on kmsg watcher: Xid 64 (row-remap InfoROM write failure) now
escalates to Critical instead of the generic warning every other Xid got,
and fixes the SAT-window flush path which previously hardcoded "Warning"
and ignored pattern severity entirely.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

audit/internal/collector/nvidia.go

@@ -25,6 +25,11 @@ type nvidiaGPUInfo struct {
 	PCIeLinkGenMax     *int
 	PCIeLinkWidthCur   *int
 	PCIeLinkWidthMax   *int
+	ResetRequired      *bool
+	RemapCorrectable   *int64
+	RemapUncorrectable *int64
+	RemapPending       *bool
+	RemapFailure       *bool
 }
 
 // enrichPCIeWithNVIDIA enriches NVIDIA PCIe devices with data from nvidia-smi.
@@ -87,6 +92,18 @@ func enrichPCIeWithNVIDIAData(devs []schema.HardwarePCIeDevice, gpuByBDF map[str
 			status = statusWarning
 			devs[i].ErrorDescription = stringPtr("GPU reports uncorrected ECC errors")
 		}
+		if info.RemapUncorrectable != nil && *info.RemapUncorrectable > 0 {
+			status = statusWarning
+			devs[i].ErrorDescription = stringPtr("GPU has uncorrectable row remap events (bad HBM cell repair scheduled)")
+		}
+		if info.RemapFailure != nil && *info.RemapFailure {
+			status = statusCritical
+			devs[i].ErrorDescription = stringPtr("GPU row remap failed to commit to InfoROM (XID 64)")
+		}
+		if info.ResetRequired != nil && *info.ResetRequired {
+			status = statusCritical
+			devs[i].ErrorDescription = stringPtr("GPU requires a reset")
+		}
 		devs[i].Status = &status
 		injectNVIDIATelemetry(&devs[i], info)
 		enriched++
@@ -107,7 +124,74 @@ func queryNVIDIAGPUs() (map[string]nvidiaGPUInfo, error) {
 	if err != nil {
 		return nil, err
 	}
-	return parseNVIDIASMIQuery(string(out))
+	result, err := parseNVIDIASMIQuery(string(out))
+	if err != nil {
+		return nil, err
+	}
+
+	// reset_status.* and remapped_rows.* are only recognized by newer drivers
+	// on Ampere+ GPUs; an unrecognized field name fails the whole nvidia-smi
+	// call, so this is queried separately to avoid losing the fields above.
+	if reliability, err := queryNVIDIAReliability(); err != nil {
+		slog.Info("nvidia: reliability fields skipped", "err", err)
+	} else {
+		for bdf, r := range reliability {
+			if info, ok := result[bdf]; ok {
+				info.ResetRequired = r.ResetRequired
+				info.RemapCorrectable = r.RemapCorrectable
+				info.RemapUncorrectable = r.RemapUncorrectable
+				info.RemapPending = r.RemapPending
+				info.RemapFailure = r.RemapFailure
+				result[bdf] = info
+			}
+		}
+	}
+
+	return result, nil
+}
+
+func queryNVIDIAReliability() (map[string]nvidiaGPUInfo, error) {
+	out, err := exec.Command(
+		"nvidia-smi",
+		"--query-gpu=pci.bus_id,reset_status.reset_required,remapped_rows.correctable,remapped_rows.uncorrectable,remapped_rows.pending,remapped_rows.failure",
+		"--format=csv,noheader,nounits",
+	).Output()
+	if err != nil {
+		return nil, err
+	}
+	return parseNVIDIAReliabilityCSV(string(out))
+}
+
+func parseNVIDIAReliabilityCSV(raw string) (map[string]nvidiaGPUInfo, error) {
+	r := csv.NewReader(strings.NewReader(raw))
+	r.TrimLeadingSpace = true
+	r.FieldsPerRecord = -1
+	records, err := r.ReadAll()
+	if err != nil {
+		return nil, err
+	}
+
+	result := make(map[string]nvidiaGPUInfo)
+	for _, rec := range records {
+		if len(rec) == 0 {
+			continue
+		}
+		if len(rec) < 6 {
+			return nil, fmt.Errorf("unexpected nvidia-smi reliability columns: got %d, want 6", len(rec))
+		}
+		bdf := normalizePCIeBDF(rec[0])
+		if bdf == "" {
+			continue
+		}
+		result[bdf] = nvidiaGPUInfo{
+			ResetRequired:      parseMaybeBool(rec[1]),
+			RemapCorrectable:   parseMaybeInt64(rec[2]),
+			RemapUncorrectable: parseMaybeInt64(rec[3]),
+			RemapPending:       parseMaybeBool(rec[4]),
+			RemapFailure:       parseMaybeBool(rec[5]),
+		}
+	}
+	return result, nil
 }
 
 func parseNVIDIASMIQuery(raw string) (map[string]nvidiaGPUInfo, error) {
@@ -207,10 +291,10 @@ func pcieLinkGenLabel(gen int) string {
 func parseMaybeBool(v string) *bool {
 	v = strings.TrimSpace(strings.ToLower(v))
 	switch v {
-	case "active", "enabled", "true", "1":
+	case "active", "enabled", "true", "1", "yes":
 		b := true
 		return &b
-	case "not active", "disabled", "false", "0":
+	case "not active", "disabled", "false", "0", "no":
 		b := false
 		return &b
 	default:
@@ -266,6 +350,21 @@ func injectNVIDIATelemetry(dev *schema.HardwarePCIeDevice, info nvidiaGPUInfo) {
 	if info.HWSlowdown != nil {
 		dev.HWSlowdown = info.HWSlowdown
 	}
+	if info.ResetRequired != nil {
+		dev.ResetRequired = info.ResetRequired
+	}
+	if info.RemapCorrectable != nil {
+		dev.RemappedRowsCorrectable = info.RemapCorrectable
+	}
+	if info.RemapUncorrectable != nil {
+		dev.RemappedRowsUncorrectable = info.RemapUncorrectable
+	}
+	if info.RemapPending != nil {
+		dev.RemappedRowsPending = info.RemapPending
+	}
+	if info.RemapFailure != nil {
+		dev.RemappedRowsFailure = info.RemapFailure
+	}
 	// Override PCIe link speed/width with nvidia-smi driver values.
 	// sysfs current_link_speed reflects the instantaneous physical link state and
 	// can show Gen1 when the GPU is idle due to ASPM power management. The driver