gpu: collect reset-required/row-remap status and classify Xid codes by severity

nvidia-smi exposes reset_status.reset_required and remapped_rows.* only on
newer drivers for Ampere+ GPUs; queried via a separate exec call since an
unrecognized field name fails the whole --query-gpu command and would have
wiped out unrelated telemetry (temp/ECC/power) on older drivers otherwise.

Also refines Xid severity in both the ingest dmesg collector and the
always-on kmsg watcher: Xid 64 (row-remap InfoROM write failure) now
escalates to Critical instead of the generic warning every other Xid got,
and fixes the SAT-window flush path which previously hardcoded "Warning"
and ignored pattern severity entirely.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-07-07 19:28:59 +03:00
co-authored by Claude Sonnet 5
parent d850d4fc3a
commit 1175e6ccd8
10 changed files with 477 additions and 69 deletions
+53
View File
@@ -0,0 +1,53 @@
package collector
import "regexp"
// xidCodeParenRE extracts the NVIDIA Xid error code from the common
// "Xid (PCI:0000:65:00): 64, ..." / "Xid (0000:65:00.0): 64, ..." form, where
// the BDF inside the parens contains digits that a naive "next number" regex
// would grab instead of the actual code.
var xidCodeParenRE = regexp.MustCompile(`(?i)Xid\s*\([^)]*\)\s*:?\s*(\d+)`)
// xidCodeColonRE handles the older "Xid: 64, ..." form with no BDF parens.
var xidCodeColonRE = regexp.MustCompile(`(?i)\bXid\s*:\s*(\d+)`)
// xidCodeSeverity maps NVIDIA Xid codes relevant to GPU HBM/ECC health to a
// severity, refining the generic "nvidia-xid" kernel-log pattern's default
// "warning". Xid 64 is the same InfoROM row-remap-write failure surfaced by
// this package's remapped_rows_failure field (see nvidia.go), so it must
// escalate to critical rather than the generic warning every other Xid gets.
// Codes not listed here keep the caller's default severity.
// Source: NVIDIA GPU Memory Error Management docs + field experience (Xid 48
// uncorrectable ECC, 63 remap committed, 64 remap write failed, 94 contained
// ECC, 95 uncontained ECC, 160 memory marked for repair).
var xidCodeSeverity = map[string]string{
"48": "critical",
"64": "critical",
"95": "critical",
"63": "warning",
"94": "warning",
"160": "warning",
}
// XidSeverity returns the refined severity ("critical"/"warning") for a kernel
// log line containing an NVIDIA Xid error, if the specific code is known. ok
// is false when no Xid code could be extracted or the code isn't in
// xidCodeSeverity, in which case callers should fall back to their own default.
func XidSeverity(line string) (severity string, ok bool) {
code, ok := extractXidCode(line)
if !ok {
return "", false
}
sev, ok := xidCodeSeverity[code]
return sev, ok
}
func extractXidCode(line string) (string, bool) {
if m := xidCodeParenRE.FindStringSubmatch(line); m != nil {
return m[1], true
}
if m := xidCodeColonRE.FindStringSubmatch(line); m != nil {
return m[1], true
}
return "", false
}