From 1175e6ccd8be373b4915bed1fb337abf7ae292c8 Mon Sep 17 00:00:00 2001 From: Mikhail Chusavitin Date: Tue, 7 Jul 2026 19:28:59 +0300 Subject: [PATCH] gpu: collect reset-required/row-remap status and classify Xid codes by severity nvidia-smi exposes reset_status.reset_required and remapped_rows.* only on newer drivers for Ampere+ GPUs; queried via a separate exec call since an unrecognized field name fails the whole --query-gpu command and would have wiped out unrelated telemetry (temp/ECC/power) on older drivers otherwise. Also refines Xid severity in both the ingest dmesg collector and the always-on kmsg watcher: Xid 64 (row-remap InfoROM write failure) now escalates to Critical instead of the generic warning every other Xid got, and fixes the SAT-window flush path which previously hardcoded "Warning" and ignored pattern severity entirely. Co-Authored-By: Claude Sonnet 5 --- audit/internal/collector/dmesg_events.go | 6 + audit/internal/collector/dmesg_events_test.go | 46 ++++++++ audit/internal/collector/nvidia.go | 105 +++++++++++++++++- audit/internal/collector/nvidia_test.go | 69 ++++++++++++ audit/internal/collector/xid.go | 53 +++++++++ audit/internal/collector/xid_test.go | 53 +++++++++ audit/internal/schema/hardware.go | 89 ++++++++------- audit/internal/webui/kmsg_watcher.go | 56 ++++++---- audit/internal/webui/kmsg_watcher_test.go | 62 +++++++++++ bible-local/docs/hardware-ingest-contract.md | 7 ++ 10 files changed, 477 insertions(+), 69 deletions(-) create mode 100644 audit/internal/collector/dmesg_events_test.go create mode 100644 audit/internal/collector/xid.go create mode 100644 audit/internal/collector/xid_test.go create mode 100644 audit/internal/webui/kmsg_watcher_test.go diff --git a/audit/internal/collector/dmesg_events.go b/audit/internal/collector/dmesg_events.go index 7956ed5..6415f7a 100644 --- a/audit/internal/collector/dmesg_events.go +++ b/audit/internal/collector/dmesg_events.go @@ -109,6 +109,12 @@ func matchesAny(s string, patterns []*regexp.Regexp) bool { } func dmesgSeverity(msg string) string { + if sev, ok := XidSeverity(msg); ok { + if sev == "critical" { + return statusCritical + } + return statusWarning + } lower := strings.ToLower(msg) switch { case strings.Contains(lower, "panic") || diff --git a/audit/internal/collector/dmesg_events_test.go b/audit/internal/collector/dmesg_events_test.go new file mode 100644 index 0000000..d46de2c --- /dev/null +++ b/audit/internal/collector/dmesg_events_test.go @@ -0,0 +1,46 @@ +package collector + +import "testing" + +func TestDmesgSeverity_xidCodes(t *testing.T) { + tests := []struct { + name string + msg string + want string + }{ + { + name: "xid 64 remap write failure escalates to critical", + msg: "NVRM: Xid (PCI:0000:65:00): 64, pid=1234, name=python", + want: statusCritical, + }, + { + name: "xid 94 contained ecc is downgraded to warning", + msg: "NVRM: Xid (PCI:0000:65:00): 94, pid=1234, name=python", + want: statusWarning, + }, + { + name: "xid 63 remap committed is downgraded to warning", + msg: "NVRM: Xid (PCI:0000:65:00): 63, pid=1234, Class 0x90a0", + want: statusWarning, + }, + { + name: "unrecognized xid code falls back to generic critical", + msg: "NVRM: Xid (PCI:0000:65:00): 79, pid=1234, GPU has fallen off the bus", + want: statusCritical, + }, + { + name: "non-xid generic error keyword", + msg: "blk_update_request: I/O error, dev sda", + want: statusCritical, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := dmesgSeverity(tt.msg) + if got != tt.want { + t.Fatalf("dmesgSeverity(%q) = %q, want %q", tt.msg, got, tt.want) + } + }) + } +} diff --git a/audit/internal/collector/nvidia.go b/audit/internal/collector/nvidia.go index 25639d5..b6625c3 100644 --- a/audit/internal/collector/nvidia.go +++ b/audit/internal/collector/nvidia.go @@ -25,6 +25,11 @@ type nvidiaGPUInfo struct { PCIeLinkGenMax *int PCIeLinkWidthCur *int PCIeLinkWidthMax *int + ResetRequired *bool + RemapCorrectable *int64 + RemapUncorrectable *int64 + RemapPending *bool + RemapFailure *bool } // enrichPCIeWithNVIDIA enriches NVIDIA PCIe devices with data from nvidia-smi. @@ -87,6 +92,18 @@ func enrichPCIeWithNVIDIAData(devs []schema.HardwarePCIeDevice, gpuByBDF map[str status = statusWarning devs[i].ErrorDescription = stringPtr("GPU reports uncorrected ECC errors") } + if info.RemapUncorrectable != nil && *info.RemapUncorrectable > 0 { + status = statusWarning + devs[i].ErrorDescription = stringPtr("GPU has uncorrectable row remap events (bad HBM cell repair scheduled)") + } + if info.RemapFailure != nil && *info.RemapFailure { + status = statusCritical + devs[i].ErrorDescription = stringPtr("GPU row remap failed to commit to InfoROM (XID 64)") + } + if info.ResetRequired != nil && *info.ResetRequired { + status = statusCritical + devs[i].ErrorDescription = stringPtr("GPU requires a reset") + } devs[i].Status = &status injectNVIDIATelemetry(&devs[i], info) enriched++ @@ -107,7 +124,74 @@ func queryNVIDIAGPUs() (map[string]nvidiaGPUInfo, error) { if err != nil { return nil, err } - return parseNVIDIASMIQuery(string(out)) + result, err := parseNVIDIASMIQuery(string(out)) + if err != nil { + return nil, err + } + + // reset_status.* and remapped_rows.* are only recognized by newer drivers + // on Ampere+ GPUs; an unrecognized field name fails the whole nvidia-smi + // call, so this is queried separately to avoid losing the fields above. + if reliability, err := queryNVIDIAReliability(); err != nil { + slog.Info("nvidia: reliability fields skipped", "err", err) + } else { + for bdf, r := range reliability { + if info, ok := result[bdf]; ok { + info.ResetRequired = r.ResetRequired + info.RemapCorrectable = r.RemapCorrectable + info.RemapUncorrectable = r.RemapUncorrectable + info.RemapPending = r.RemapPending + info.RemapFailure = r.RemapFailure + result[bdf] = info + } + } + } + + return result, nil +} + +func queryNVIDIAReliability() (map[string]nvidiaGPUInfo, error) { + out, err := exec.Command( + "nvidia-smi", + "--query-gpu=pci.bus_id,reset_status.reset_required,remapped_rows.correctable,remapped_rows.uncorrectable,remapped_rows.pending,remapped_rows.failure", + "--format=csv,noheader,nounits", + ).Output() + if err != nil { + return nil, err + } + return parseNVIDIAReliabilityCSV(string(out)) +} + +func parseNVIDIAReliabilityCSV(raw string) (map[string]nvidiaGPUInfo, error) { + r := csv.NewReader(strings.NewReader(raw)) + r.TrimLeadingSpace = true + r.FieldsPerRecord = -1 + records, err := r.ReadAll() + if err != nil { + return nil, err + } + + result := make(map[string]nvidiaGPUInfo) + for _, rec := range records { + if len(rec) == 0 { + continue + } + if len(rec) < 6 { + return nil, fmt.Errorf("unexpected nvidia-smi reliability columns: got %d, want 6", len(rec)) + } + bdf := normalizePCIeBDF(rec[0]) + if bdf == "" { + continue + } + result[bdf] = nvidiaGPUInfo{ + ResetRequired: parseMaybeBool(rec[1]), + RemapCorrectable: parseMaybeInt64(rec[2]), + RemapUncorrectable: parseMaybeInt64(rec[3]), + RemapPending: parseMaybeBool(rec[4]), + RemapFailure: parseMaybeBool(rec[5]), + } + } + return result, nil } func parseNVIDIASMIQuery(raw string) (map[string]nvidiaGPUInfo, error) { @@ -207,10 +291,10 @@ func pcieLinkGenLabel(gen int) string { func parseMaybeBool(v string) *bool { v = strings.TrimSpace(strings.ToLower(v)) switch v { - case "active", "enabled", "true", "1": + case "active", "enabled", "true", "1", "yes": b := true return &b - case "not active", "disabled", "false", "0": + case "not active", "disabled", "false", "0", "no": b := false return &b default: @@ -266,6 +350,21 @@ func injectNVIDIATelemetry(dev *schema.HardwarePCIeDevice, info nvidiaGPUInfo) { if info.HWSlowdown != nil { dev.HWSlowdown = info.HWSlowdown } + if info.ResetRequired != nil { + dev.ResetRequired = info.ResetRequired + } + if info.RemapCorrectable != nil { + dev.RemappedRowsCorrectable = info.RemapCorrectable + } + if info.RemapUncorrectable != nil { + dev.RemappedRowsUncorrectable = info.RemapUncorrectable + } + if info.RemapPending != nil { + dev.RemappedRowsPending = info.RemapPending + } + if info.RemapFailure != nil { + dev.RemappedRowsFailure = info.RemapFailure + } // Override PCIe link speed/width with nvidia-smi driver values. // sysfs current_link_speed reflects the instantaneous physical link state and // can show Gen1 when the GPU is idle due to ASPM power management. The driver diff --git a/audit/internal/collector/nvidia_test.go b/audit/internal/collector/nvidia_test.go index 781d574..44bd0ea 100644 --- a/audit/internal/collector/nvidia_test.go +++ b/audit/internal/collector/nvidia_test.go @@ -124,5 +124,74 @@ func TestEnrichPCIeWithNVIDIAData_driverMissingFallback(t *testing.T) { } } +func TestParseNVIDIAReliability(t *testing.T) { + raw := "0000:65:00.0, No, 0, 2, No, Yes\n" + byBDF, err := parseNVIDIAReliabilityCSV(raw) + if err != nil { + t.Fatalf("parse failed: %v", err) + } + + gpu, ok := byBDF["0000:65:00.0"] + if !ok { + t.Fatalf("gpu by normalized bdf not found") + } + if gpu.ResetRequired == nil || *gpu.ResetRequired { + t.Fatalf("reset_required: got %v, want false", gpu.ResetRequired) + } + if gpu.RemapCorrectable == nil || *gpu.RemapCorrectable != 0 { + t.Fatalf("remap correctable: got %v", gpu.RemapCorrectable) + } + if gpu.RemapUncorrectable == nil || *gpu.RemapUncorrectable != 2 { + t.Fatalf("remap uncorrectable: got %v", gpu.RemapUncorrectable) + } + if gpu.RemapPending == nil || *gpu.RemapPending { + t.Fatalf("remap pending: got %v, want false", gpu.RemapPending) + } + if gpu.RemapFailure == nil || !*gpu.RemapFailure { + t.Fatalf("remap failure: got %v, want true", gpu.RemapFailure) + } +} + +func TestEnrichPCIeWithNVIDIAData_resetRequiredCritical(t *testing.T) { + vendorID := NvidiaVendorID + bdf := "0000:65:00.0" + devices := []schema.HardwarePCIeDevice{ + {VendorID: &vendorID, BDF: &bdf}, + } + + byBDF := map[string]nvidiaGPUInfo{ + "0000:65:00.0": {ResetRequired: ptrBool(true)}, + } + + out := enrichPCIeWithNVIDIAData(devices, byBDF, true) + if out[0].Status == nil || *out[0].Status != statusCritical { + t.Fatalf("status: got %v, want %v", out[0].Status, statusCritical) + } + if out[0].ResetRequired == nil || !*out[0].ResetRequired { + t.Fatalf("reset_required: got %v", out[0].ResetRequired) + } +} + +func TestEnrichPCIeWithNVIDIAData_remapFailureCritical(t *testing.T) { + vendorID := NvidiaVendorID + bdf := "0000:65:00.0" + devices := []schema.HardwarePCIeDevice{ + {VendorID: &vendorID, BDF: &bdf}, + } + + byBDF := map[string]nvidiaGPUInfo{ + "0000:65:00.0": {RemapFailure: ptrBool(true)}, + } + + out := enrichPCIeWithNVIDIAData(devices, byBDF, true) + if out[0].Status == nil || *out[0].Status != statusCritical { + t.Fatalf("status: got %v, want %v", out[0].Status, statusCritical) + } + if out[0].RemappedRowsFailure == nil || !*out[0].RemappedRowsFailure { + t.Fatalf("remapped_rows_failure: got %v", out[0].RemappedRowsFailure) + } +} + func ptrInt64(v int64) *int64 { return &v } func ptrFloat(v float64) *float64 { return &v } +func ptrBool(v bool) *bool { return &v } diff --git a/audit/internal/collector/xid.go b/audit/internal/collector/xid.go new file mode 100644 index 0000000..8497b01 --- /dev/null +++ b/audit/internal/collector/xid.go @@ -0,0 +1,53 @@ +package collector + +import "regexp" + +// xidCodeParenRE extracts the NVIDIA Xid error code from the common +// "Xid (PCI:0000:65:00): 64, ..." / "Xid (0000:65:00.0): 64, ..." form, where +// the BDF inside the parens contains digits that a naive "next number" regex +// would grab instead of the actual code. +var xidCodeParenRE = regexp.MustCompile(`(?i)Xid\s*\([^)]*\)\s*:?\s*(\d+)`) + +// xidCodeColonRE handles the older "Xid: 64, ..." form with no BDF parens. +var xidCodeColonRE = regexp.MustCompile(`(?i)\bXid\s*:\s*(\d+)`) + +// xidCodeSeverity maps NVIDIA Xid codes relevant to GPU HBM/ECC health to a +// severity, refining the generic "nvidia-xid" kernel-log pattern's default +// "warning". Xid 64 is the same InfoROM row-remap-write failure surfaced by +// this package's remapped_rows_failure field (see nvidia.go), so it must +// escalate to critical rather than the generic warning every other Xid gets. +// Codes not listed here keep the caller's default severity. +// Source: NVIDIA GPU Memory Error Management docs + field experience (Xid 48 +// uncorrectable ECC, 63 remap committed, 64 remap write failed, 94 contained +// ECC, 95 uncontained ECC, 160 memory marked for repair). +var xidCodeSeverity = map[string]string{ + "48": "critical", + "64": "critical", + "95": "critical", + "63": "warning", + "94": "warning", + "160": "warning", +} + +// XidSeverity returns the refined severity ("critical"/"warning") for a kernel +// log line containing an NVIDIA Xid error, if the specific code is known. ok +// is false when no Xid code could be extracted or the code isn't in +// xidCodeSeverity, in which case callers should fall back to their own default. +func XidSeverity(line string) (severity string, ok bool) { + code, ok := extractXidCode(line) + if !ok { + return "", false + } + sev, ok := xidCodeSeverity[code] + return sev, ok +} + +func extractXidCode(line string) (string, bool) { + if m := xidCodeParenRE.FindStringSubmatch(line); m != nil { + return m[1], true + } + if m := xidCodeColonRE.FindStringSubmatch(line); m != nil { + return m[1], true + } + return "", false +} diff --git a/audit/internal/collector/xid_test.go b/audit/internal/collector/xid_test.go new file mode 100644 index 0000000..15c707a --- /dev/null +++ b/audit/internal/collector/xid_test.go @@ -0,0 +1,53 @@ +package collector + +import "testing" + +func TestXidSeverity(t *testing.T) { + tests := []struct { + name string + line string + wantSev string + wantOK bool + }{ + { + name: "xid 64 remap write failure is critical", + line: "NVRM: Xid (PCI:0000:65:00): 64, pid=1234, name=python, Row Remapper: New row remapping failed", + wantSev: "critical", + wantOK: true, + }, + { + name: "xid 48 uncorrectable ecc is critical", + line: "NVRM: Xid (PCI:0000:65:00): 48, pid=1234, name=python", + wantSev: "critical", + wantOK: true, + }, + { + name: "xid 63 remap committed is warning", + line: "NVRM: Xid (PCI:0000:65:00): 63, pid=1234, Class 0x90a0", + wantSev: "warning", + wantOK: true, + }, + { + name: "xid 79 unknown code falls back to caller default", + line: "NVRM: Xid (PCI:0000:65:00): 79, pid=1234, GPU has fallen off the bus", + wantOK: false, + }, + { + name: "no xid in line", + line: "NVRM: GPU 0000:65:00.0: RmInitAdapter failed", + wantOK: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sev, ok := XidSeverity(tt.line) + if ok != tt.wantOK { + t.Fatalf("ok: got %v, want %v", ok, tt.wantOK) + } + if ok && sev != tt.wantSev { + t.Fatalf("severity: got %q, want %q", sev, tt.wantSev) + } + }) + } +} diff --git a/audit/internal/schema/hardware.go b/audit/internal/schema/hardware.go index ceb99b6..ed7911c 100644 --- a/audit/internal/schema/hardware.go +++ b/audit/internal/schema/hardware.go @@ -178,48 +178,53 @@ type HardwareStorage struct { type HardwarePCIeDevice struct { HardwareComponentStatus - Slot *string `json:"slot,omitempty"` - VendorID *int `json:"vendor_id,omitempty"` - DeviceID *int `json:"device_id,omitempty"` - NUMANode *int `json:"numa_node,omitempty"` - TemperatureC *float64 `json:"temperature_c,omitempty"` - PowerW *float64 `json:"power_w,omitempty"` - LifeRemainingPct *float64 `json:"life_remaining_pct,omitempty"` - LifeUsedPct *float64 `json:"life_used_pct,omitempty"` - ECCCorrectedTotal *int64 `json:"ecc_corrected_total,omitempty"` - ECCUncorrectedTotal *int64 `json:"ecc_uncorrected_total,omitempty"` - HWSlowdown *bool `json:"hw_slowdown,omitempty"` - BatteryChargePct *float64 `json:"battery_charge_pct,omitempty"` - BatteryHealthPct *float64 `json:"battery_health_pct,omitempty"` - BatteryTemperatureC *float64 `json:"battery_temperature_c,omitempty"` - BatteryVoltageV *float64 `json:"battery_voltage_v,omitempty"` - BatteryReplaceRequired *bool `json:"battery_replace_required,omitempty"` - SFPPresent *bool `json:"sfp_present,omitempty"` - SFPIdentifier *string `json:"sfp_identifier,omitempty"` - SFPConnector *string `json:"sfp_connector,omitempty"` - SFPVendor *string `json:"sfp_vendor,omitempty"` - SFPPartNumber *string `json:"sfp_part_number,omitempty"` - SFPSerialNumber *string `json:"sfp_serial_number,omitempty"` - SFPWavelengthNM *float64 `json:"sfp_wavelength_nm,omitempty"` - SFPTemperatureC *float64 `json:"sfp_temperature_c,omitempty"` - SFPTXPowerDBM *float64 `json:"sfp_tx_power_dbm,omitempty"` - SFPRXPowerDBM *float64 `json:"sfp_rx_power_dbm,omitempty"` - SFPVoltageV *float64 `json:"sfp_voltage_v,omitempty"` - SFPBiasMA *float64 `json:"sfp_bias_ma,omitempty"` - BDF *string `json:"-"` - DeviceClass *string `json:"device_class,omitempty"` - Manufacturer *string `json:"manufacturer,omitempty"` - Model *string `json:"model,omitempty"` - LinkWidth *int `json:"link_width,omitempty"` - LinkSpeed *string `json:"link_speed,omitempty"` - MaxLinkWidth *int `json:"max_link_width,omitempty"` - MaxLinkSpeed *string `json:"max_link_speed,omitempty"` - SerialNumber *string `json:"serial_number,omitempty"` - Firmware *string `json:"firmware,omitempty"` - MacAddresses []string `json:"mac_addresses,omitempty"` - Present *bool `json:"present,omitempty"` - IOMMUGroup *int `json:"iommu_group,omitempty"` - Telemetry map[string]any `json:"-"` + Slot *string `json:"slot,omitempty"` + VendorID *int `json:"vendor_id,omitempty"` + DeviceID *int `json:"device_id,omitempty"` + NUMANode *int `json:"numa_node,omitempty"` + TemperatureC *float64 `json:"temperature_c,omitempty"` + PowerW *float64 `json:"power_w,omitempty"` + LifeRemainingPct *float64 `json:"life_remaining_pct,omitempty"` + LifeUsedPct *float64 `json:"life_used_pct,omitempty"` + ECCCorrectedTotal *int64 `json:"ecc_corrected_total,omitempty"` + ECCUncorrectedTotal *int64 `json:"ecc_uncorrected_total,omitempty"` + HWSlowdown *bool `json:"hw_slowdown,omitempty"` + ResetRequired *bool `json:"reset_required,omitempty"` + RemappedRowsCorrectable *int64 `json:"remapped_rows_correctable,omitempty"` + RemappedRowsUncorrectable *int64 `json:"remapped_rows_uncorrectable,omitempty"` + RemappedRowsPending *bool `json:"remapped_rows_pending,omitempty"` + RemappedRowsFailure *bool `json:"remapped_rows_failure,omitempty"` + BatteryChargePct *float64 `json:"battery_charge_pct,omitempty"` + BatteryHealthPct *float64 `json:"battery_health_pct,omitempty"` + BatteryTemperatureC *float64 `json:"battery_temperature_c,omitempty"` + BatteryVoltageV *float64 `json:"battery_voltage_v,omitempty"` + BatteryReplaceRequired *bool `json:"battery_replace_required,omitempty"` + SFPPresent *bool `json:"sfp_present,omitempty"` + SFPIdentifier *string `json:"sfp_identifier,omitempty"` + SFPConnector *string `json:"sfp_connector,omitempty"` + SFPVendor *string `json:"sfp_vendor,omitempty"` + SFPPartNumber *string `json:"sfp_part_number,omitempty"` + SFPSerialNumber *string `json:"sfp_serial_number,omitempty"` + SFPWavelengthNM *float64 `json:"sfp_wavelength_nm,omitempty"` + SFPTemperatureC *float64 `json:"sfp_temperature_c,omitempty"` + SFPTXPowerDBM *float64 `json:"sfp_tx_power_dbm,omitempty"` + SFPRXPowerDBM *float64 `json:"sfp_rx_power_dbm,omitempty"` + SFPVoltageV *float64 `json:"sfp_voltage_v,omitempty"` + SFPBiasMA *float64 `json:"sfp_bias_ma,omitempty"` + BDF *string `json:"-"` + DeviceClass *string `json:"device_class,omitempty"` + Manufacturer *string `json:"manufacturer,omitempty"` + Model *string `json:"model,omitempty"` + LinkWidth *int `json:"link_width,omitempty"` + LinkSpeed *string `json:"link_speed,omitempty"` + MaxLinkWidth *int `json:"max_link_width,omitempty"` + MaxLinkSpeed *string `json:"max_link_speed,omitempty"` + SerialNumber *string `json:"serial_number,omitempty"` + Firmware *string `json:"firmware,omitempty"` + MacAddresses []string `json:"mac_addresses,omitempty"` + Present *bool `json:"present,omitempty"` + IOMMUGroup *int `json:"iommu_group,omitempty"` + Telemetry map[string]any `json:"-"` } type HardwarePowerSupply struct { diff --git a/audit/internal/webui/kmsg_watcher.go b/audit/internal/webui/kmsg_watcher.go index cfc8903..214b1f2 100644 --- a/audit/internal/webui/kmsg_watcher.go +++ b/audit/internal/webui/kmsg_watcher.go @@ -10,6 +10,7 @@ import ( "time" "bee/audit/internal/app" + "bee/audit/internal/collector" "bee/audit/internal/platform" ) @@ -40,6 +41,7 @@ type kmsgEvent struct { raw string ids []string // BDF addresses or device names extracted category string + severity string // "warning" or "critical", resolved once at parse time } func newKmsgWatcher(statusDB *app.ComponentStatusDB) *kmsgWatcher { @@ -148,8 +150,18 @@ func (w *kmsgWatcher) flushWindow(window *kmsgWindow) { return } source := "watchdog:kmsg" - // Collect unique component keys from events. - seen := map[string]string{} // componentKey → first raw line + // Collect unique component keys from events, escalating to Critical if any + // matching event for that component was critical (e.g. Xid 64 remap failure). + seen := map[string]string{} // componentKey → first raw line + severities := map[string]string{} // componentKey → worst severity seen + record := func(key string, evt kmsgEvent) { + if _, exists := seen[key]; !exists { + seen[key] = evt.raw + severities[key] = capitalizeSeverity(evt.severity) + } else if evt.severity == "critical" { + severities[key] = "Critical" + } + } for _, evt := range window.events { if len(evt.ids) == 0 { // MCE or un-identified error. @@ -157,9 +169,7 @@ func (w *kmsgWatcher) flushWindow(window *kmsgWindow) { if evt.category == "memory" { key = "memory:all" } - if _, exists := seen[key]; !exists { - seen[key] = evt.raw - } + record(key, evt) continue } for _, id := range evt.ids { @@ -174,14 +184,12 @@ func (w *kmsgWatcher) flushWindow(window *kmsgWindow) { default: key = "pcie:" + normalizeBDF(id) } - if _, exists := seen[key]; !exists { - seen[key] = evt.raw - } + record(key, evt) } } for key, detail := range seen { detail = "kernel error during SAT (" + strings.Join(window.targets, ",") + "): " + truncate(detail, 120) - w.statusDB.Record(key, source, "Warning", detail) + w.statusDB.Record(key, source, severities[key], detail) } } @@ -193,21 +201,7 @@ func (w *kmsgWatcher) flushImmediate(evt kmsgEvent) { } const source = "watchdog:kmsg" detail := "kernel: " + truncate(evt.raw, 120) - - var severity string - for _, p := range platform.HardwareErrorPatterns { - if p.Re.MatchString(evt.raw) { - if p.Severity == "critical" { - severity = "Critical" - } else { - severity = "Warning" - } - break - } - } - if severity == "" { - severity = "Warning" - } + severity := capitalizeSeverity(evt.severity) if len(evt.ids) == 0 { key := "cpu:all" @@ -250,10 +244,15 @@ func parseKmsgLine(raw string) (kmsgEvent, bool) { if m == nil { continue } + severity := p.Severity + if sev, ok := collector.XidSeverity(msg); ok { + severity = sev + } evt := kmsgEvent{ timestamp: time.Now(), raw: msg, category: p.Category, + severity: severity, } if p.BDFGroup > 0 && p.BDFGroup < len(m) { evt.ids = append(evt.ids, normalizeBDF(m[p.BDFGroup])) @@ -266,6 +265,15 @@ func parseKmsgLine(raw string) (kmsgEvent, bool) { return kmsgEvent{}, false } +// capitalizeSeverity converts an ErrorPattern severity ("warning"/"critical") +// to the Title-case convention ComponentStatusDB.Record expects. +func capitalizeSeverity(s string) string { + if s == "critical" { + return "Critical" + } + return "Warning" +} + // normalizeBDF normalizes a PCIe BDF to the 4-part form "0000:c8:00.0". func normalizeBDF(bdf string) string { bdf = strings.ToLower(strings.TrimSpace(bdf)) diff --git a/audit/internal/webui/kmsg_watcher_test.go b/audit/internal/webui/kmsg_watcher_test.go new file mode 100644 index 0000000..f4a2e75 --- /dev/null +++ b/audit/internal/webui/kmsg_watcher_test.go @@ -0,0 +1,62 @@ +package webui + +import ( + "path/filepath" + "testing" + "time" + + "bee/audit/internal/app" +) + +func TestParseKmsgLine_xid64EscalatesToCritical(t *testing.T) { + line := "6,1234,555555,-;NVRM: Xid (PCI:0000:65:00.0): 64, pid=1234, name=python, Row Remapper: New row remapping failed" + evt, ok := parseKmsgLine(line) + if !ok { + t.Fatalf("expected line to match a pattern") + } + if evt.severity != "critical" { + t.Fatalf("severity: got %q, want %q", evt.severity, "critical") + } + if len(evt.ids) != 1 || evt.ids[0] != "0000:65:00.0" { + t.Fatalf("ids: got %v", evt.ids) + } +} + +func TestParseKmsgLine_xid63StaysWarning(t *testing.T) { + line := "6,1234,555555,-;NVRM: Xid (PCI:0000:65:00.0): 63, pid=1234, Class 0x90a0" + evt, ok := parseKmsgLine(line) + if !ok { + t.Fatalf("expected line to match a pattern") + } + if evt.severity != "warning" { + t.Fatalf("severity: got %q, want %q", evt.severity, "warning") + } +} + +func TestFlushWindow_escalatesToCriticalOnXid64(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "status.json") + db, err := app.OpenComponentStatusDB(dbPath) + if err != nil { + t.Fatalf("open status db: %v", err) + } + w := newKmsgWatcher(db) + + window := &kmsgWindow{ + targets: []string{"gpu-burn"}, + startedAt: time.Now(), + events: []kmsgEvent{ + {raw: "NVRM: GPU 0000:65:00.0: RmInitAdapter failed", ids: []string{"0000:65:00.0"}, category: "gpu", severity: "warning"}, + {raw: "NVRM: Xid (PCI:0000:65:00): 64, pid=1234, name=python", ids: []string{"0000:65:00.0"}, category: "gpu", severity: "critical"}, + }, + } + + w.flushWindow(window) + + rec, ok := db.Get("pcie:gpu:0000:65:00.0") + if !ok { + t.Fatalf("expected a status record for the gpu") + } + if rec.Status != "Critical" { + t.Fatalf("status: got %q, want %q", rec.Status, "Critical") + } +} diff --git a/bible-local/docs/hardware-ingest-contract.md b/bible-local/docs/hardware-ingest-contract.md index b99654a..44e4694 100644 --- a/bible-local/docs/hardware-ingest-contract.md +++ b/bible-local/docs/hardware-ingest-contract.md @@ -418,6 +418,11 @@ GET /ingest/hardware/jobs/{job_id} | `ecc_corrected_total` | int64 | нет | Всего корректируемых ECC-ошибок | | `ecc_uncorrected_total` | int64 | нет | Всего некорректируемых ECC-ошибок | | `hw_slowdown` | bool | нет | Устройство вошло в hardware slowdown / protective mode | +| `reset_required` | bool | нет | GPU требует reset (nvidia-smi `reset_status.reset_required`); драйвер/hardware state повреждён, требуется reset или reboot узла | +| `remapped_rows_correctable` | int64 | нет | Число строк HBM, переназначенных из-за корректируемых ECC-ошибок (nvidia-smi `remapped_rows.correctable`) | +| `remapped_rows_uncorrectable` | int64 | нет | Число строк HBM, переназначенных из-за некорректируемых ECC-ошибок; >0 означает как минимум одну аппаратно неисправную ячейку памяти (nvidia-smi `remapped_rows.uncorrectable`) | +| `remapped_rows_pending` | bool | нет | Remap запланирован, но требует полного power cycle узла, чтобы вступить в силу (nvidia-smi `remapped_rows.pending`) | +| `remapped_rows_failure` | bool | нет | Попытка remap не смогла записаться в InfoROM (XID 64) — серьёзная неисправность (nvidia-smi `remapped_rows.failure`) | | `battery_charge_pct` | float | нет | Заряд батареи / supercap, % | | `battery_health_pct` | float | нет | Состояние батареи / supercap, % | | `battery_temperature_c` | float | нет | Температура батареи / supercap, °C | @@ -443,6 +448,8 @@ GET /ingest/hardware/jobs/{job_id} | `present` | bool | нет | Наличие (по умолчанию `true`) | | + общие поля статуса | | | см. раздел выше | +`reset_required` и `remapped_rows_*` доступны только для NVIDIA datacenter GPU (Ampere и новее) на драйверах, поддерживающих поля `reset_status.*`/`remapped_rows.*` в `nvidia-smi --query-gpu`. На старых драйверах или GPU без row remapping (HBM ECC) поля просто отсутствуют в payload — это не ошибка сборщика. + `numa_node` передавайте для NIC / InfiniBand / RAID / GPU, когда источник знает CPU/NUMA affinity. Поле сохраняется в snapshot-атрибутах PCIe-компонента и дублируется в telemetry для topology use cases. Поля `temperature_c` и `power_w` используйте для device-level telemetry GPU / accelerator / smart PCIe devices. Они не влияют на идентификацию компонента.