Mikhail ChusavitinandClaude Sonnet 5
1175e6ccd8
nvidia-smi exposes reset_status.reset_required and remapped_rows.* only on newer drivers for Ampere+ GPUs; queried via a separate exec call since an unrecognized field name fails the whole --query-gpu command and would have wiped out unrelated telemetry (temp/ECC/power) on older drivers otherwise. Also refines Xid severity in both the ingest dmesg collector and the always-on kmsg watcher: Xid 64 (row-remap InfoROM write failure) now escalates to Critical instead of the generic warning every other Xid got, and fixes the SAT-window flush path which previously hardcoded "Warning" and ignored pattern severity entirely. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
54 lines
2.0 KiB
Go
54 lines
2.0 KiB
Go
package collector
|
|
|
|
import "regexp"
|
|
|
|
// xidCodeParenRE extracts the NVIDIA Xid error code from the common
|
|
// "Xid (PCI:0000:65:00): 64, ..." / "Xid (0000:65:00.0): 64, ..." form, where
|
|
// the BDF inside the parens contains digits that a naive "next number" regex
|
|
// would grab instead of the actual code.
|
|
var xidCodeParenRE = regexp.MustCompile(`(?i)Xid\s*\([^)]*\)\s*:?\s*(\d+)`)
|
|
|
|
// xidCodeColonRE handles the older "Xid: 64, ..." form with no BDF parens.
|
|
var xidCodeColonRE = regexp.MustCompile(`(?i)\bXid\s*:\s*(\d+)`)
|
|
|
|
// xidCodeSeverity maps NVIDIA Xid codes relevant to GPU HBM/ECC health to a
|
|
// severity, refining the generic "nvidia-xid" kernel-log pattern's default
|
|
// "warning". Xid 64 is the same InfoROM row-remap-write failure surfaced by
|
|
// this package's remapped_rows_failure field (see nvidia.go), so it must
|
|
// escalate to critical rather than the generic warning every other Xid gets.
|
|
// Codes not listed here keep the caller's default severity.
|
|
// Source: NVIDIA GPU Memory Error Management docs + field experience (Xid 48
|
|
// uncorrectable ECC, 63 remap committed, 64 remap write failed, 94 contained
|
|
// ECC, 95 uncontained ECC, 160 memory marked for repair).
|
|
var xidCodeSeverity = map[string]string{
|
|
"48": "critical",
|
|
"64": "critical",
|
|
"95": "critical",
|
|
"63": "warning",
|
|
"94": "warning",
|
|
"160": "warning",
|
|
}
|
|
|
|
// XidSeverity returns the refined severity ("critical"/"warning") for a kernel
|
|
// log line containing an NVIDIA Xid error, if the specific code is known. ok
|
|
// is false when no Xid code could be extracted or the code isn't in
|
|
// xidCodeSeverity, in which case callers should fall back to their own default.
|
|
func XidSeverity(line string) (severity string, ok bool) {
|
|
code, ok := extractXidCode(line)
|
|
if !ok {
|
|
return "", false
|
|
}
|
|
sev, ok := xidCodeSeverity[code]
|
|
return sev, ok
|
|
}
|
|
|
|
func extractXidCode(line string) (string, bool) {
|
|
if m := xidCodeParenRE.FindStringSubmatch(line); m != nil {
|
|
return m[1], true
|
|
}
|
|
if m := xidCodeColonRE.FindStringSubmatch(line); m != nil {
|
|
return m[1], true
|
|
}
|
|
return "", false
|
|
}
|