gpu: collect reset-required/row-remap status and classify Xid codes by severity

nvidia-smi exposes reset_status.reset_required and remapped_rows.* only on
newer drivers for Ampere+ GPUs; queried via a separate exec call since an
unrecognized field name fails the whole --query-gpu command and would have
wiped out unrelated telemetry (temp/ECC/power) on older drivers otherwise.

Also refines Xid severity in both the ingest dmesg collector and the
always-on kmsg watcher: Xid 64 (row-remap InfoROM write failure) now
escalates to Critical instead of the generic warning every other Xid got,
and fixes the SAT-window flush path which previously hardcoded "Warning"
and ignored pattern severity entirely.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

audit/internal/collector/dmesg_events.go

@@ -109,6 +109,12 @@ func matchesAny(s string, patterns []*regexp.Regexp) bool {
 }
 
 func dmesgSeverity(msg string) string {
+	if sev, ok := XidSeverity(msg); ok {
+		if sev == "critical" {
+			return statusCritical
+		}
+		return statusWarning
+	}
 	lower := strings.ToLower(msg)
 	switch {
 	case strings.Contains(lower, "panic") ||

audit/internal/collector/dmesg_events_test.go

@@ -0,0 +1,46 @@
+package collector
+
+import "testing"
+
+func TestDmesgSeverity_xidCodes(t *testing.T) {
+	tests := []struct {
+		name string
+		msg  string
+		want string
+	}{
+		{
+			name: "xid 64 remap write failure escalates to critical",
+			msg:  "NVRM: Xid (PCI:0000:65:00): 64, pid=1234, name=python",
+			want: statusCritical,
+		},
+		{
+			name: "xid 94 contained ecc is downgraded to warning",
+			msg:  "NVRM: Xid (PCI:0000:65:00): 94, pid=1234, name=python",
+			want: statusWarning,
+		},
+		{
+			name: "xid 63 remap committed is downgraded to warning",
+			msg:  "NVRM: Xid (PCI:0000:65:00): 63, pid=1234, Class 0x90a0",
+			want: statusWarning,
+		},
+		{
+			name: "unrecognized xid code falls back to generic critical",
+			msg:  "NVRM: Xid (PCI:0000:65:00): 79, pid=1234, GPU has fallen off the bus",
+			want: statusCritical,
+		},
+		{
+			name: "non-xid generic error keyword",
+			msg:  "blk_update_request: I/O error, dev sda",
+			want: statusCritical,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := dmesgSeverity(tt.msg)
+			if got != tt.want {
+				t.Fatalf("dmesgSeverity(%q) = %q, want %q", tt.msg, got, tt.want)
+			}
+		})
+	}
+}

audit/internal/collector/nvidia.go

@@ -25,6 +25,11 @@ type nvidiaGPUInfo struct {
 	PCIeLinkGenMax     *int
 	PCIeLinkWidthCur   *int
 	PCIeLinkWidthMax   *int
+	ResetRequired      *bool
+	RemapCorrectable   *int64
+	RemapUncorrectable *int64
+	RemapPending       *bool
+	RemapFailure       *bool
 }
 
 // enrichPCIeWithNVIDIA enriches NVIDIA PCIe devices with data from nvidia-smi.
@@ -87,6 +92,18 @@ func enrichPCIeWithNVIDIAData(devs []schema.HardwarePCIeDevice, gpuByBDF map[str
 			status = statusWarning
 			devs[i].ErrorDescription = stringPtr("GPU reports uncorrected ECC errors")
 		}
+		if info.RemapUncorrectable != nil && *info.RemapUncorrectable > 0 {
+			status = statusWarning
+			devs[i].ErrorDescription = stringPtr("GPU has uncorrectable row remap events (bad HBM cell repair scheduled)")
+		}
+		if info.RemapFailure != nil && *info.RemapFailure {
+			status = statusCritical
+			devs[i].ErrorDescription = stringPtr("GPU row remap failed to commit to InfoROM (XID 64)")
+		}
+		if info.ResetRequired != nil && *info.ResetRequired {
+			status = statusCritical
+			devs[i].ErrorDescription = stringPtr("GPU requires a reset")
+		}
 		devs[i].Status = &status
 		injectNVIDIATelemetry(&devs[i], info)
 		enriched++
@@ -107,7 +124,74 @@ func queryNVIDIAGPUs() (map[string]nvidiaGPUInfo, error) {
 	if err != nil {
 		return nil, err
 	}
-	return parseNVIDIASMIQuery(string(out))
+	result, err := parseNVIDIASMIQuery(string(out))
+	if err != nil {
+		return nil, err
+	}
+
+	// reset_status.* and remapped_rows.* are only recognized by newer drivers
+	// on Ampere+ GPUs; an unrecognized field name fails the whole nvidia-smi
+	// call, so this is queried separately to avoid losing the fields above.
+	if reliability, err := queryNVIDIAReliability(); err != nil {
+		slog.Info("nvidia: reliability fields skipped", "err", err)
+	} else {
+		for bdf, r := range reliability {
+			if info, ok := result[bdf]; ok {
+				info.ResetRequired = r.ResetRequired
+				info.RemapCorrectable = r.RemapCorrectable
+				info.RemapUncorrectable = r.RemapUncorrectable
+				info.RemapPending = r.RemapPending
+				info.RemapFailure = r.RemapFailure
+				result[bdf] = info
+			}
+		}
+	}
+
+	return result, nil
+}
+
+func queryNVIDIAReliability() (map[string]nvidiaGPUInfo, error) {
+	out, err := exec.Command(
+		"nvidia-smi",
+		"--query-gpu=pci.bus_id,reset_status.reset_required,remapped_rows.correctable,remapped_rows.uncorrectable,remapped_rows.pending,remapped_rows.failure",
+		"--format=csv,noheader,nounits",
+	).Output()
+	if err != nil {
+		return nil, err
+	}
+	return parseNVIDIAReliabilityCSV(string(out))
+}
+
+func parseNVIDIAReliabilityCSV(raw string) (map[string]nvidiaGPUInfo, error) {
+	r := csv.NewReader(strings.NewReader(raw))
+	r.TrimLeadingSpace = true
+	r.FieldsPerRecord = -1
+	records, err := r.ReadAll()
+	if err != nil {
+		return nil, err
+	}
+
+	result := make(map[string]nvidiaGPUInfo)
+	for _, rec := range records {
+		if len(rec) == 0 {
+			continue
+		}
+		if len(rec) < 6 {
+			return nil, fmt.Errorf("unexpected nvidia-smi reliability columns: got %d, want 6", len(rec))
+		}
+		bdf := normalizePCIeBDF(rec[0])
+		if bdf == "" {
+			continue
+		}
+		result[bdf] = nvidiaGPUInfo{
+			ResetRequired:      parseMaybeBool(rec[1]),
+			RemapCorrectable:   parseMaybeInt64(rec[2]),
+			RemapUncorrectable: parseMaybeInt64(rec[3]),
+			RemapPending:       parseMaybeBool(rec[4]),
+			RemapFailure:       parseMaybeBool(rec[5]),
+		}
+	}
+	return result, nil
 }
 
 func parseNVIDIASMIQuery(raw string) (map[string]nvidiaGPUInfo, error) {
@@ -207,10 +291,10 @@ func pcieLinkGenLabel(gen int) string {
 func parseMaybeBool(v string) *bool {
 	v = strings.TrimSpace(strings.ToLower(v))
 	switch v {
-	case "active", "enabled", "true", "1":
+	case "active", "enabled", "true", "1", "yes":
 		b := true
 		return &b
-	case "not active", "disabled", "false", "0":
+	case "not active", "disabled", "false", "0", "no":
 		b := false
 		return &b
 	default:
@@ -266,6 +350,21 @@ func injectNVIDIATelemetry(dev *schema.HardwarePCIeDevice, info nvidiaGPUInfo) {
 	if info.HWSlowdown != nil {
 		dev.HWSlowdown = info.HWSlowdown
 	}
+	if info.ResetRequired != nil {
+		dev.ResetRequired = info.ResetRequired
+	}
+	if info.RemapCorrectable != nil {
+		dev.RemappedRowsCorrectable = info.RemapCorrectable
+	}
+	if info.RemapUncorrectable != nil {
+		dev.RemappedRowsUncorrectable = info.RemapUncorrectable
+	}
+	if info.RemapPending != nil {
+		dev.RemappedRowsPending = info.RemapPending
+	}
+	if info.RemapFailure != nil {
+		dev.RemappedRowsFailure = info.RemapFailure
+	}
 	// Override PCIe link speed/width with nvidia-smi driver values.
 	// sysfs current_link_speed reflects the instantaneous physical link state and
 	// can show Gen1 when the GPU is idle due to ASPM power management. The driver

audit/internal/collector/nvidia_test.go

@@ -124,5 +124,74 @@ func TestEnrichPCIeWithNVIDIAData_driverMissingFallback(t *testing.T) {
 	}
 }
 
+func TestParseNVIDIAReliability(t *testing.T) {
+	raw := "0000:65:00.0, No, 0, 2, No, Yes\n"
+	byBDF, err := parseNVIDIAReliabilityCSV(raw)
+	if err != nil {
+		t.Fatalf("parse failed: %v", err)
+	}
+
+	gpu, ok := byBDF["0000:65:00.0"]
+	if !ok {
+		t.Fatalf("gpu by normalized bdf not found")
+	}
+	if gpu.ResetRequired == nil || *gpu.ResetRequired {
+		t.Fatalf("reset_required: got %v, want false", gpu.ResetRequired)
+	}
+	if gpu.RemapCorrectable == nil || *gpu.RemapCorrectable != 0 {
+		t.Fatalf("remap correctable: got %v", gpu.RemapCorrectable)
+	}
+	if gpu.RemapUncorrectable == nil || *gpu.RemapUncorrectable != 2 {
+		t.Fatalf("remap uncorrectable: got %v", gpu.RemapUncorrectable)
+	}
+	if gpu.RemapPending == nil || *gpu.RemapPending {
+		t.Fatalf("remap pending: got %v, want false", gpu.RemapPending)
+	}
+	if gpu.RemapFailure == nil || !*gpu.RemapFailure {
+		t.Fatalf("remap failure: got %v, want true", gpu.RemapFailure)
+	}
+}
+
+func TestEnrichPCIeWithNVIDIAData_resetRequiredCritical(t *testing.T) {
+	vendorID := NvidiaVendorID
+	bdf := "0000:65:00.0"
+	devices := []schema.HardwarePCIeDevice{
+		{VendorID: &vendorID, BDF: &bdf},
+	}
+
+	byBDF := map[string]nvidiaGPUInfo{
+		"0000:65:00.0": {ResetRequired: ptrBool(true)},
+	}
+
+	out := enrichPCIeWithNVIDIAData(devices, byBDF, true)
+	if out[0].Status == nil || *out[0].Status != statusCritical {
+		t.Fatalf("status: got %v, want %v", out[0].Status, statusCritical)
+	}
+	if out[0].ResetRequired == nil || !*out[0].ResetRequired {
+		t.Fatalf("reset_required: got %v", out[0].ResetRequired)
+	}
+}
+
+func TestEnrichPCIeWithNVIDIAData_remapFailureCritical(t *testing.T) {
+	vendorID := NvidiaVendorID
+	bdf := "0000:65:00.0"
+	devices := []schema.HardwarePCIeDevice{
+		{VendorID: &vendorID, BDF: &bdf},
+	}
+
+	byBDF := map[string]nvidiaGPUInfo{
+		"0000:65:00.0": {RemapFailure: ptrBool(true)},
+	}
+
+	out := enrichPCIeWithNVIDIAData(devices, byBDF, true)
+	if out[0].Status == nil || *out[0].Status != statusCritical {
+		t.Fatalf("status: got %v, want %v", out[0].Status, statusCritical)
+	}
+	if out[0].RemappedRowsFailure == nil || !*out[0].RemappedRowsFailure {
+		t.Fatalf("remapped_rows_failure: got %v", out[0].RemappedRowsFailure)
+	}
+}
+
 func ptrInt64(v int64) *int64     { return &v }
 func ptrFloat(v float64) *float64 { return &v }
+func ptrBool(v bool) *bool        { return &v }

audit/internal/collector/xid.go

@@ -0,0 +1,53 @@
+package collector
+
+import "regexp"
+
+// xidCodeParenRE extracts the NVIDIA Xid error code from the common
+// "Xid (PCI:0000:65:00): 64, ..." / "Xid (0000:65:00.0): 64, ..." form, where
+// the BDF inside the parens contains digits that a naive "next number" regex
+// would grab instead of the actual code.
+var xidCodeParenRE = regexp.MustCompile(`(?i)Xid\s*\([^)]*\)\s*:?\s*(\d+)`)
+
+// xidCodeColonRE handles the older "Xid: 64, ..." form with no BDF parens.
+var xidCodeColonRE = regexp.MustCompile(`(?i)\bXid\s*:\s*(\d+)`)
+
+// xidCodeSeverity maps NVIDIA Xid codes relevant to GPU HBM/ECC health to a
+// severity, refining the generic "nvidia-xid" kernel-log pattern's default
+// "warning". Xid 64 is the same InfoROM row-remap-write failure surfaced by
+// this package's remapped_rows_failure field (see nvidia.go), so it must
+// escalate to critical rather than the generic warning every other Xid gets.
+// Codes not listed here keep the caller's default severity.
+// Source: NVIDIA GPU Memory Error Management docs + field experience (Xid 48
+// uncorrectable ECC, 63 remap committed, 64 remap write failed, 94 contained
+// ECC, 95 uncontained ECC, 160 memory marked for repair).
+var xidCodeSeverity = map[string]string{
+	"48":  "critical",
+	"64":  "critical",
+	"95":  "critical",
+	"63":  "warning",
+	"94":  "warning",
+	"160": "warning",
+}
+
+// XidSeverity returns the refined severity ("critical"/"warning") for a kernel
+// log line containing an NVIDIA Xid error, if the specific code is known. ok
+// is false when no Xid code could be extracted or the code isn't in
+// xidCodeSeverity, in which case callers should fall back to their own default.
+func XidSeverity(line string) (severity string, ok bool) {
+	code, ok := extractXidCode(line)
+	if !ok {
+		return "", false
+	}
+	sev, ok := xidCodeSeverity[code]
+	return sev, ok
+}
+
+func extractXidCode(line string) (string, bool) {
+	if m := xidCodeParenRE.FindStringSubmatch(line); m != nil {
+		return m[1], true
+	}
+	if m := xidCodeColonRE.FindStringSubmatch(line); m != nil {
+		return m[1], true
+	}
+	return "", false
+}

audit/internal/collector/xid_test.go

@@ -0,0 +1,53 @@
+package collector
+
+import "testing"
+
+func TestXidSeverity(t *testing.T) {
+	tests := []struct {
+		name    string
+		line    string
+		wantSev string
+		wantOK  bool
+	}{
+		{
+			name:    "xid 64 remap write failure is critical",
+			line:    "NVRM: Xid (PCI:0000:65:00): 64, pid=1234, name=python, Row Remapper: New row remapping failed",
+			wantSev: "critical",
+			wantOK:  true,
+		},
+		{
+			name:    "xid 48 uncorrectable ecc is critical",
+			line:    "NVRM: Xid (PCI:0000:65:00): 48, pid=1234, name=python",
+			wantSev: "critical",
+			wantOK:  true,
+		},
+		{
+			name:    "xid 63 remap committed is warning",
+			line:    "NVRM: Xid (PCI:0000:65:00): 63, pid=1234, Class 0x90a0",
+			wantSev: "warning",
+			wantOK:  true,
+		},
+		{
+			name:   "xid 79 unknown code falls back to caller default",
+			line:   "NVRM: Xid (PCI:0000:65:00): 79, pid=1234, GPU has fallen off the bus",
+			wantOK: false,
+		},
+		{
+			name:   "no xid in line",
+			line:   "NVRM: GPU 0000:65:00.0: RmInitAdapter failed",
+			wantOK: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			sev, ok := XidSeverity(tt.line)
+			if ok != tt.wantOK {
+				t.Fatalf("ok: got %v, want %v", ok, tt.wantOK)
+			}
+			if ok && sev != tt.wantSev {
+				t.Fatalf("severity: got %q, want %q", sev, tt.wantSev)
+			}
+		})
+	}
+}

audit/internal/schema/hardware.go

@@ -178,48 +178,53 @@ type HardwareStorage struct {
 
 type HardwarePCIeDevice struct {
 	HardwareComponentStatus
-	Slot                   *string        `json:"slot,omitempty"`
-	VendorID               *int           `json:"vendor_id,omitempty"`
-	DeviceID               *int           `json:"device_id,omitempty"`
-	NUMANode               *int           `json:"numa_node,omitempty"`
-	TemperatureC           *float64       `json:"temperature_c,omitempty"`
-	PowerW                 *float64       `json:"power_w,omitempty"`
-	LifeRemainingPct       *float64       `json:"life_remaining_pct,omitempty"`
-	LifeUsedPct            *float64       `json:"life_used_pct,omitempty"`
-	ECCCorrectedTotal      *int64         `json:"ecc_corrected_total,omitempty"`
-	ECCUncorrectedTotal    *int64         `json:"ecc_uncorrected_total,omitempty"`
-	HWSlowdown             *bool          `json:"hw_slowdown,omitempty"`
-	BatteryChargePct       *float64       `json:"battery_charge_pct,omitempty"`
-	BatteryHealthPct       *float64       `json:"battery_health_pct,omitempty"`
-	BatteryTemperatureC    *float64       `json:"battery_temperature_c,omitempty"`
-	BatteryVoltageV        *float64       `json:"battery_voltage_v,omitempty"`
-	BatteryReplaceRequired *bool          `json:"battery_replace_required,omitempty"`
-	SFPPresent             *bool          `json:"sfp_present,omitempty"`
-	SFPIdentifier          *string        `json:"sfp_identifier,omitempty"`
-	SFPConnector           *string        `json:"sfp_connector,omitempty"`
-	SFPVendor              *string        `json:"sfp_vendor,omitempty"`
-	SFPPartNumber          *string        `json:"sfp_part_number,omitempty"`
-	SFPSerialNumber        *string        `json:"sfp_serial_number,omitempty"`
-	SFPWavelengthNM        *float64       `json:"sfp_wavelength_nm,omitempty"`
-	SFPTemperatureC        *float64       `json:"sfp_temperature_c,omitempty"`
-	SFPTXPowerDBM          *float64       `json:"sfp_tx_power_dbm,omitempty"`
-	SFPRXPowerDBM          *float64       `json:"sfp_rx_power_dbm,omitempty"`
-	SFPVoltageV            *float64       `json:"sfp_voltage_v,omitempty"`
-	SFPBiasMA              *float64       `json:"sfp_bias_ma,omitempty"`
-	BDF                    *string        `json:"-"`
-	DeviceClass            *string        `json:"device_class,omitempty"`
-	Manufacturer           *string        `json:"manufacturer,omitempty"`
-	Model                  *string        `json:"model,omitempty"`
-	LinkWidth              *int           `json:"link_width,omitempty"`
-	LinkSpeed              *string        `json:"link_speed,omitempty"`
-	MaxLinkWidth           *int           `json:"max_link_width,omitempty"`
-	MaxLinkSpeed           *string        `json:"max_link_speed,omitempty"`
-	SerialNumber           *string        `json:"serial_number,omitempty"`
-	Firmware               *string        `json:"firmware,omitempty"`
-	MacAddresses           []string       `json:"mac_addresses,omitempty"`
-	Present                *bool          `json:"present,omitempty"`
-	IOMMUGroup             *int           `json:"iommu_group,omitempty"`
-	Telemetry              map[string]any `json:"-"`
+	Slot                      *string        `json:"slot,omitempty"`
+	VendorID                  *int           `json:"vendor_id,omitempty"`
+	DeviceID                  *int           `json:"device_id,omitempty"`
+	NUMANode                  *int           `json:"numa_node,omitempty"`
+	TemperatureC              *float64       `json:"temperature_c,omitempty"`
+	PowerW                    *float64       `json:"power_w,omitempty"`
+	LifeRemainingPct          *float64       `json:"life_remaining_pct,omitempty"`
+	LifeUsedPct               *float64       `json:"life_used_pct,omitempty"`
+	ECCCorrectedTotal         *int64         `json:"ecc_corrected_total,omitempty"`
+	ECCUncorrectedTotal       *int64         `json:"ecc_uncorrected_total,omitempty"`
+	HWSlowdown                *bool          `json:"hw_slowdown,omitempty"`
+	ResetRequired             *bool          `json:"reset_required,omitempty"`
+	RemappedRowsCorrectable   *int64         `json:"remapped_rows_correctable,omitempty"`
+	RemappedRowsUncorrectable *int64         `json:"remapped_rows_uncorrectable,omitempty"`
+	RemappedRowsPending       *bool          `json:"remapped_rows_pending,omitempty"`
+	RemappedRowsFailure       *bool          `json:"remapped_rows_failure,omitempty"`
+	BatteryChargePct          *float64       `json:"battery_charge_pct,omitempty"`
+	BatteryHealthPct          *float64       `json:"battery_health_pct,omitempty"`
+	BatteryTemperatureC       *float64       `json:"battery_temperature_c,omitempty"`
+	BatteryVoltageV           *float64       `json:"battery_voltage_v,omitempty"`
+	BatteryReplaceRequired    *bool          `json:"battery_replace_required,omitempty"`
+	SFPPresent                *bool          `json:"sfp_present,omitempty"`
+	SFPIdentifier             *string        `json:"sfp_identifier,omitempty"`
+	SFPConnector              *string        `json:"sfp_connector,omitempty"`
+	SFPVendor                 *string        `json:"sfp_vendor,omitempty"`
+	SFPPartNumber             *string        `json:"sfp_part_number,omitempty"`
+	SFPSerialNumber           *string        `json:"sfp_serial_number,omitempty"`
+	SFPWavelengthNM           *float64       `json:"sfp_wavelength_nm,omitempty"`
+	SFPTemperatureC           *float64       `json:"sfp_temperature_c,omitempty"`
+	SFPTXPowerDBM             *float64       `json:"sfp_tx_power_dbm,omitempty"`
+	SFPRXPowerDBM             *float64       `json:"sfp_rx_power_dbm,omitempty"`
+	SFPVoltageV               *float64       `json:"sfp_voltage_v,omitempty"`
+	SFPBiasMA                 *float64       `json:"sfp_bias_ma,omitempty"`
+	BDF                       *string        `json:"-"`
+	DeviceClass               *string        `json:"device_class,omitempty"`
+	Manufacturer              *string        `json:"manufacturer,omitempty"`
+	Model                     *string        `json:"model,omitempty"`
+	LinkWidth                 *int           `json:"link_width,omitempty"`
+	LinkSpeed                 *string        `json:"link_speed,omitempty"`
+	MaxLinkWidth              *int           `json:"max_link_width,omitempty"`
+	MaxLinkSpeed              *string        `json:"max_link_speed,omitempty"`
+	SerialNumber              *string        `json:"serial_number,omitempty"`
+	Firmware                  *string        `json:"firmware,omitempty"`
+	MacAddresses              []string       `json:"mac_addresses,omitempty"`
+	Present                   *bool          `json:"present,omitempty"`
+	IOMMUGroup                *int           `json:"iommu_group,omitempty"`
+	Telemetry                 map[string]any `json:"-"`
 }
 
 type HardwarePowerSupply struct {

audit/internal/webui/kmsg_watcher.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"bee/audit/internal/app"
+	"bee/audit/internal/collector"
 	"bee/audit/internal/platform"
 )
 
@@ -40,6 +41,7 @@ type kmsgEvent struct {
 	raw       string
 	ids       []string // BDF addresses or device names extracted
 	category  string
+	severity  string // "warning" or "critical", resolved once at parse time
 }
 
 func newKmsgWatcher(statusDB *app.ComponentStatusDB) *kmsgWatcher {
@@ -148,8 +150,18 @@ func (w *kmsgWatcher) flushWindow(window *kmsgWindow) {
 		return
 	}
 	source := "watchdog:kmsg"
-	// Collect unique component keys from events.
-	seen := map[string]string{} // componentKey → first raw line
+	// Collect unique component keys from events, escalating to Critical if any
+	// matching event for that component was critical (e.g. Xid 64 remap failure).
+	seen := map[string]string{}       // componentKey → first raw line
+	severities := map[string]string{} // componentKey → worst severity seen
+	record := func(key string, evt kmsgEvent) {
+		if _, exists := seen[key]; !exists {
+			seen[key] = evt.raw
+			severities[key] = capitalizeSeverity(evt.severity)
+		} else if evt.severity == "critical" {
+			severities[key] = "Critical"
+		}
+	}
 	for _, evt := range window.events {
 		if len(evt.ids) == 0 {
 			// MCE or un-identified error.
@@ -157,9 +169,7 @@ func (w *kmsgWatcher) flushWindow(window *kmsgWindow) {
 			if evt.category == "memory" {
 				key = "memory:all"
 			}
-			if _, exists := seen[key]; !exists {
-				seen[key] = evt.raw
-			}
+			record(key, evt)
 			continue
 		}
 		for _, id := range evt.ids {
@@ -174,14 +184,12 @@ func (w *kmsgWatcher) flushWindow(window *kmsgWindow) {
 			default:
 				key = "pcie:" + normalizeBDF(id)
 			}
-			if _, exists := seen[key]; !exists {
-				seen[key] = evt.raw
-			}
+			record(key, evt)
 		}
 	}
 	for key, detail := range seen {
 		detail = "kernel error during SAT (" + strings.Join(window.targets, ",") + "): " + truncate(detail, 120)
-		w.statusDB.Record(key, source, "Warning", detail)
+		w.statusDB.Record(key, source, severities[key], detail)
 	}
 }
 
@@ -193,21 +201,7 @@ func (w *kmsgWatcher) flushImmediate(evt kmsgEvent) {
 	}
 	const source = "watchdog:kmsg"
 	detail := "kernel: " + truncate(evt.raw, 120)
-
-	var severity string
-	for _, p := range platform.HardwareErrorPatterns {
-		if p.Re.MatchString(evt.raw) {
-			if p.Severity == "critical" {
-				severity = "Critical"
-			} else {
-				severity = "Warning"
-			}
-			break
-		}
-	}
-	if severity == "" {
-		severity = "Warning"
-	}
+	severity := capitalizeSeverity(evt.severity)
 
 	if len(evt.ids) == 0 {
 		key := "cpu:all"
@@ -250,10 +244,15 @@ func parseKmsgLine(raw string) (kmsgEvent, bool) {
 		if m == nil {
 			continue
 		}
+		severity := p.Severity
+		if sev, ok := collector.XidSeverity(msg); ok {
+			severity = sev
+		}
 		evt := kmsgEvent{
 			timestamp: time.Now(),
 			raw:       msg,
 			category:  p.Category,
+			severity:  severity,
 		}
 		if p.BDFGroup > 0 && p.BDFGroup < len(m) {
 			evt.ids = append(evt.ids, normalizeBDF(m[p.BDFGroup]))
@@ -266,6 +265,15 @@ func parseKmsgLine(raw string) (kmsgEvent, bool) {
 	return kmsgEvent{}, false
 }
 
+// capitalizeSeverity converts an ErrorPattern severity ("warning"/"critical")
+// to the Title-case convention ComponentStatusDB.Record expects.
+func capitalizeSeverity(s string) string {
+	if s == "critical" {
+		return "Critical"
+	}
+	return "Warning"
+}
+
 // normalizeBDF normalizes a PCIe BDF to the 4-part form "0000:c8:00.0".
 func normalizeBDF(bdf string) string {
 	bdf = strings.ToLower(strings.TrimSpace(bdf))

audit/internal/webui/kmsg_watcher_test.go

@@ -0,0 +1,62 @@
+package webui
+
+import (
+	"path/filepath"
+	"testing"
+	"time"
+
+	"bee/audit/internal/app"
+)
+
+func TestParseKmsgLine_xid64EscalatesToCritical(t *testing.T) {
+	line := "6,1234,555555,-;NVRM: Xid (PCI:0000:65:00.0): 64, pid=1234, name=python, Row Remapper: New row remapping failed"
+	evt, ok := parseKmsgLine(line)
+	if !ok {
+		t.Fatalf("expected line to match a pattern")
+	}
+	if evt.severity != "critical" {
+		t.Fatalf("severity: got %q, want %q", evt.severity, "critical")
+	}
+	if len(evt.ids) != 1 || evt.ids[0] != "0000:65:00.0" {
+		t.Fatalf("ids: got %v", evt.ids)
+	}
+}
+
+func TestParseKmsgLine_xid63StaysWarning(t *testing.T) {
+	line := "6,1234,555555,-;NVRM: Xid (PCI:0000:65:00.0): 63, pid=1234, Class 0x90a0"
+	evt, ok := parseKmsgLine(line)
+	if !ok {
+		t.Fatalf("expected line to match a pattern")
+	}
+	if evt.severity != "warning" {
+		t.Fatalf("severity: got %q, want %q", evt.severity, "warning")
+	}
+}
+
+func TestFlushWindow_escalatesToCriticalOnXid64(t *testing.T) {
+	dbPath := filepath.Join(t.TempDir(), "status.json")
+	db, err := app.OpenComponentStatusDB(dbPath)
+	if err != nil {
+		t.Fatalf("open status db: %v", err)
+	}
+	w := newKmsgWatcher(db)
+
+	window := &kmsgWindow{
+		targets:   []string{"gpu-burn"},
+		startedAt: time.Now(),
+		events: []kmsgEvent{
+			{raw: "NVRM: GPU 0000:65:00.0: RmInitAdapter failed", ids: []string{"0000:65:00.0"}, category: "gpu", severity: "warning"},
+			{raw: "NVRM: Xid (PCI:0000:65:00): 64, pid=1234, name=python", ids: []string{"0000:65:00.0"}, category: "gpu", severity: "critical"},
+		},
+	}
+
+	w.flushWindow(window)
+
+	rec, ok := db.Get("pcie:gpu:0000:65:00.0")
+	if !ok {
+		t.Fatalf("expected a status record for the gpu")
+	}
+	if rec.Status != "Critical" {
+		t.Fatalf("status: got %q, want %q", rec.Status, "Critical")
+	}
+}

bible-local/docs/hardware-ingest-contract.md

@@ -418,6 +418,11 @@ GET /ingest/hardware/jobs/{job_id}
 | `ecc_corrected_total` | int64 | нет | Всего корректируемых ECC-ошибок |
 | `ecc_uncorrected_total` | int64 | нет | Всего некорректируемых ECC-ошибок |
 | `hw_slowdown` | bool | нет | Устройство вошло в hardware slowdown / protective mode |
+| `reset_required` | bool | нет | GPU требует reset (nvidia-smi `reset_status.reset_required`); драйвер/hardware state повреждён, требуется reset или reboot узла |
+| `remapped_rows_correctable` | int64 | нет | Число строк HBM, переназначенных из-за корректируемых ECC-ошибок (nvidia-smi `remapped_rows.correctable`) |
+| `remapped_rows_uncorrectable` | int64 | нет | Число строк HBM, переназначенных из-за некорректируемых ECC-ошибок; >0 означает как минимум одну аппаратно неисправную ячейку памяти (nvidia-smi `remapped_rows.uncorrectable`) |
+| `remapped_rows_pending` | bool | нет | Remap запланирован, но требует полного power cycle узла, чтобы вступить в силу (nvidia-smi `remapped_rows.pending`) |
+| `remapped_rows_failure` | bool | нет | Попытка remap не смогла записаться в InfoROM (XID 64) — серьёзная неисправность (nvidia-smi `remapped_rows.failure`) |
 | `battery_charge_pct` | float | нет | Заряд батареи / supercap, % |
 | `battery_health_pct` | float | нет | Состояние батареи / supercap, % |
 | `battery_temperature_c` | float | нет | Температура батареи / supercap, °C |
@@ -443,6 +448,8 @@ GET /ingest/hardware/jobs/{job_id}
 | `present` | bool | нет | Наличие (по умолчанию `true`) |
 | + общие поля статуса | | | см. раздел выше |
 
+`reset_required` и `remapped_rows_*` доступны только для NVIDIA datacenter GPU (Ampere и новее) на драйверах, поддерживающих поля `reset_status.*`/`remapped_rows.*` в `nvidia-smi --query-gpu`. На старых драйверах или GPU без row remapping (HBM ECC) поля просто отсутствуют в payload — это не ошибка сборщика.
+
 `numa_node` передавайте для NIC / InfiniBand / RAID / GPU, когда источник знает CPU/NUMA affinity. Поле сохраняется в snapshot-атрибутах PCIe-компонента и дублируется в telemetry для topology use cases.
 Поля `temperature_c` и `power_w` используйте для device-level telemetry GPU / accelerator / smart PCIe devices. Они не влияют на идентификацию компонента.