Mikhail ChusavitinandClaude Sonnet 5
b2b3f86c8d
nvidia-bug-report.sh appends .gz to --output-file when gzip is available, which silently produced empty nvidia-bug-report.txt in support bundles (cat looked for the uncompressed name that never existed). Also: a GPU that falls off the PCIe/NVLink bus (Xid 79) or gets flagged for Node Reboot Required (Xid 154) mid-SAT-run left every downstream test failing with generic, unrelated-looking errors (CUDA "unknown error", "unable to determine device handle") with no indication the GPU needed a physical power-cycle to recover. Detect these codes from SAT run logs and surface a plain-English "physical reboot required" message in the task's failure detail, the persisted component-status DB, a dashboard banner on the Hardware Summary card, and topology diagram GPU-node severity. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
86 lines
3.4 KiB
Go
86 lines
3.4 KiB
Go
package collector
|
|
|
|
import "regexp"
|
|
|
|
// xidCodeParenRE extracts the NVIDIA Xid error code from the common
|
|
// "Xid (PCI:0000:65:00): 64, ..." / "Xid (0000:65:00.0): 64, ..." form, where
|
|
// the BDF inside the parens contains digits that a naive "next number" regex
|
|
// would grab instead of the actual code.
|
|
var xidCodeParenRE = regexp.MustCompile(`(?i)Xid\s*\([^)]*\)\s*:?\s*(\d+)`)
|
|
|
|
// xidCodeColonRE handles the older "Xid: 64, ..." form with no BDF parens.
|
|
var xidCodeColonRE = regexp.MustCompile(`(?i)\bXid\s*:\s*(\d+)`)
|
|
|
|
// xidCodeBareRE handles dcgmi diag's own report wording, e.g.
|
|
// "Detected XID 79 for GPU 1" — no colon or parens around the code.
|
|
var xidCodeBareRE = regexp.MustCompile(`(?i)\bXid\s+(\d+)\b`)
|
|
|
|
// xidCodeSeverity maps NVIDIA Xid codes relevant to GPU HBM/ECC health to a
|
|
// severity, refining the generic "nvidia-xid" kernel-log pattern's default
|
|
// "warning". Xid 64 is the same InfoROM row-remap-write failure surfaced by
|
|
// this package's remapped_rows_failure field (see nvidia.go), so it must
|
|
// escalate to critical rather than the generic warning every other Xid gets.
|
|
// Codes not listed here keep the caller's default severity.
|
|
// Source: NVIDIA GPU Memory Error Management docs + field experience (Xid 48
|
|
// uncorrectable ECC, 63 remap committed, 64 remap write failed, 94 contained
|
|
// ECC, 95 uncontained ECC, 160 memory marked for repair).
|
|
var xidCodeSeverity = map[string]string{
|
|
"48": "critical",
|
|
"64": "critical",
|
|
"79": "critical",
|
|
"95": "critical",
|
|
"154": "critical",
|
|
"63": "warning",
|
|
"94": "warning",
|
|
"160": "warning",
|
|
}
|
|
|
|
// XidSeverity returns the refined severity ("critical"/"warning") for a kernel
|
|
// log line containing an NVIDIA Xid error, if the specific code is known. ok
|
|
// is false when no Xid code could be extracted or the code isn't in
|
|
// xidCodeSeverity, in which case callers should fall back to their own default.
|
|
func XidSeverity(line string) (severity string, ok bool) {
|
|
code, ok := extractXidCode(line)
|
|
if !ok {
|
|
return "", false
|
|
}
|
|
sev, ok := xidCodeSeverity[code]
|
|
return sev, ok
|
|
}
|
|
|
|
// xidHardwareFaultMessages maps NVIDIA Xid codes that mean the GPU has left
|
|
// the bus and cannot self-recover to a plain-English explanation, so an
|
|
// operator sees "physical reboot required" directly instead of having to
|
|
// look up what "Xid 79" means in NVIDIA's docs.
|
|
// Source: NVIDIA Xid error docs (79 = GPU has fallen off the bus, 154 = GPU
|
|
// recovery action escalated to Node Reboot Required).
|
|
var xidHardwareFaultMessages = map[string]string{
|
|
"79": "GPU has fallen off the PCIe bus (Xid 79) — hardware fault, will not recover without a physical reboot/power-cycle",
|
|
"154": "NVIDIA driver flagged this GPU for a required node reboot (Xid 154: recovery action = Node Reboot Required)",
|
|
}
|
|
|
|
// XidHardwareFaultMessage returns a plain-English explanation for a log line
|
|
// carrying an NVIDIA Xid code that requires a physical reboot to clear, if
|
|
// the code is recognized. ok is false otherwise.
|
|
func XidHardwareFaultMessage(line string) (message string, ok bool) {
|
|
code, ok := extractXidCode(line)
|
|
if !ok {
|
|
return "", false
|
|
}
|
|
msg, ok := xidHardwareFaultMessages[code]
|
|
return msg, ok
|
|
}
|
|
|
|
func extractXidCode(line string) (string, bool) {
|
|
if m := xidCodeParenRE.FindStringSubmatch(line); m != nil {
|
|
return m[1], true
|
|
}
|
|
if m := xidCodeColonRE.FindStringSubmatch(line); m != nil {
|
|
return m[1], true
|
|
}
|
|
if m := xidCodeBareRE.FindStringSubmatch(line); m != nil {
|
|
return m[1], true
|
|
}
|
|
return "", false
|
|
}
|