gpu: collect reset-required/row-remap status and classify Xid codes by severity
nvidia-smi exposes reset_status.reset_required and remapped_rows.* only on newer drivers for Ampere+ GPUs; queried via a separate exec call since an unrecognized field name fails the whole --query-gpu command and would have wiped out unrelated telemetry (temp/ECC/power) on older drivers otherwise. Also refines Xid severity in both the ingest dmesg collector and the always-on kmsg watcher: Xid 64 (row-remap InfoROM write failure) now escalates to Critical instead of the generic warning every other Xid got, and fixes the SAT-window flush path which previously hardcoded "Warning" and ignored pattern severity entirely. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
d850d4fc3a
commit
1175e6ccd8
@@ -0,0 +1,53 @@
|
||||
package collector
|
||||
|
||||
import "regexp"
|
||||
|
||||
// xidCodeParenRE extracts the NVIDIA Xid error code from the common
|
||||
// "Xid (PCI:0000:65:00): 64, ..." / "Xid (0000:65:00.0): 64, ..." form, where
|
||||
// the BDF inside the parens contains digits that a naive "next number" regex
|
||||
// would grab instead of the actual code.
|
||||
var xidCodeParenRE = regexp.MustCompile(`(?i)Xid\s*\([^)]*\)\s*:?\s*(\d+)`)
|
||||
|
||||
// xidCodeColonRE handles the older "Xid: 64, ..." form with no BDF parens.
|
||||
var xidCodeColonRE = regexp.MustCompile(`(?i)\bXid\s*:\s*(\d+)`)
|
||||
|
||||
// xidCodeSeverity maps NVIDIA Xid codes relevant to GPU HBM/ECC health to a
|
||||
// severity, refining the generic "nvidia-xid" kernel-log pattern's default
|
||||
// "warning". Xid 64 is the same InfoROM row-remap-write failure surfaced by
|
||||
// this package's remapped_rows_failure field (see nvidia.go), so it must
|
||||
// escalate to critical rather than the generic warning every other Xid gets.
|
||||
// Codes not listed here keep the caller's default severity.
|
||||
// Source: NVIDIA GPU Memory Error Management docs + field experience (Xid 48
|
||||
// uncorrectable ECC, 63 remap committed, 64 remap write failed, 94 contained
|
||||
// ECC, 95 uncontained ECC, 160 memory marked for repair).
|
||||
var xidCodeSeverity = map[string]string{
|
||||
"48": "critical",
|
||||
"64": "critical",
|
||||
"95": "critical",
|
||||
"63": "warning",
|
||||
"94": "warning",
|
||||
"160": "warning",
|
||||
}
|
||||
|
||||
// XidSeverity returns the refined severity ("critical"/"warning") for a kernel
|
||||
// log line containing an NVIDIA Xid error, if the specific code is known. ok
|
||||
// is false when no Xid code could be extracted or the code isn't in
|
||||
// xidCodeSeverity, in which case callers should fall back to their own default.
|
||||
func XidSeverity(line string) (severity string, ok bool) {
|
||||
code, ok := extractXidCode(line)
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
sev, ok := xidCodeSeverity[code]
|
||||
return sev, ok
|
||||
}
|
||||
|
||||
func extractXidCode(line string) (string, bool) {
|
||||
if m := xidCodeParenRE.FindStringSubmatch(line); m != nil {
|
||||
return m[1], true
|
||||
}
|
||||
if m := xidCodeColonRE.FindStringSubmatch(line); m != nil {
|
||||
return m[1], true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
Reference in New Issue
Block a user