fix(nvidia): surface Xid 79/154 GPU bus-fall-off as a physical-reboot-required signal
nvidia-bug-report.sh appends .gz to --output-file when gzip is available, which silently produced empty nvidia-bug-report.txt in support bundles (cat looked for the uncompressed name that never existed). Also: a GPU that falls off the PCIe/NVLink bus (Xid 79) or gets flagged for Node Reboot Required (Xid 154) mid-SAT-run left every downstream test failing with generic, unrelated-looking errors (CUDA "unknown error", "unable to determine device handle") with no indication the GPU needed a physical power-cycle to recover. Detect these codes from SAT run logs and surface a plain-English "physical reboot required" message in the task's failure detail, the persisted component-status DB, a dashboard banner on the Hardware Summary card, and topology diagram GPU-node severity. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
a34e823f82
commit
b2b3f86c8d
@@ -11,6 +11,10 @@ var xidCodeParenRE = regexp.MustCompile(`(?i)Xid\s*\([^)]*\)\s*:?\s*(\d+)`)
|
||||
// xidCodeColonRE handles the older "Xid: 64, ..." form with no BDF parens.
|
||||
var xidCodeColonRE = regexp.MustCompile(`(?i)\bXid\s*:\s*(\d+)`)
|
||||
|
||||
// xidCodeBareRE handles dcgmi diag's own report wording, e.g.
|
||||
// "Detected XID 79 for GPU 1" — no colon or parens around the code.
|
||||
var xidCodeBareRE = regexp.MustCompile(`(?i)\bXid\s+(\d+)\b`)
|
||||
|
||||
// xidCodeSeverity maps NVIDIA Xid codes relevant to GPU HBM/ECC health to a
|
||||
// severity, refining the generic "nvidia-xid" kernel-log pattern's default
|
||||
// "warning". Xid 64 is the same InfoROM row-remap-write failure surfaced by
|
||||
@@ -23,7 +27,9 @@ var xidCodeColonRE = regexp.MustCompile(`(?i)\bXid\s*:\s*(\d+)`)
|
||||
var xidCodeSeverity = map[string]string{
|
||||
"48": "critical",
|
||||
"64": "critical",
|
||||
"79": "critical",
|
||||
"95": "critical",
|
||||
"154": "critical",
|
||||
"63": "warning",
|
||||
"94": "warning",
|
||||
"160": "warning",
|
||||
@@ -42,6 +48,29 @@ func XidSeverity(line string) (severity string, ok bool) {
|
||||
return sev, ok
|
||||
}
|
||||
|
||||
// xidHardwareFaultMessages maps NVIDIA Xid codes that mean the GPU has left
|
||||
// the bus and cannot self-recover to a plain-English explanation, so an
|
||||
// operator sees "physical reboot required" directly instead of having to
|
||||
// look up what "Xid 79" means in NVIDIA's docs.
|
||||
// Source: NVIDIA Xid error docs (79 = GPU has fallen off the bus, 154 = GPU
|
||||
// recovery action escalated to Node Reboot Required).
|
||||
var xidHardwareFaultMessages = map[string]string{
|
||||
"79": "GPU has fallen off the PCIe bus (Xid 79) — hardware fault, will not recover without a physical reboot/power-cycle",
|
||||
"154": "NVIDIA driver flagged this GPU for a required node reboot (Xid 154: recovery action = Node Reboot Required)",
|
||||
}
|
||||
|
||||
// XidHardwareFaultMessage returns a plain-English explanation for a log line
|
||||
// carrying an NVIDIA Xid code that requires a physical reboot to clear, if
|
||||
// the code is recognized. ok is false otherwise.
|
||||
func XidHardwareFaultMessage(line string) (message string, ok bool) {
|
||||
code, ok := extractXidCode(line)
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
msg, ok := xidHardwareFaultMessages[code]
|
||||
return msg, ok
|
||||
}
|
||||
|
||||
func extractXidCode(line string) (string, bool) {
|
||||
if m := xidCodeParenRE.FindStringSubmatch(line); m != nil {
|
||||
return m[1], true
|
||||
@@ -49,5 +78,8 @@ func extractXidCode(line string) (string, bool) {
|
||||
if m := xidCodeColonRE.FindStringSubmatch(line); m != nil {
|
||||
return m[1], true
|
||||
}
|
||||
if m := xidCodeBareRE.FindStringSubmatch(line); m != nil {
|
||||
return m[1], true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
@@ -28,9 +28,10 @@ func TestXidSeverity(t *testing.T) {
|
||||
wantOK: true,
|
||||
},
|
||||
{
|
||||
name: "xid 79 unknown code falls back to caller default",
|
||||
line: "NVRM: Xid (PCI:0000:65:00): 79, pid=1234, GPU has fallen off the bus",
|
||||
wantOK: false,
|
||||
name: "xid 79 GPU fell off the bus is critical",
|
||||
line: "NVRM: Xid (PCI:0000:65:00): 79, pid=1234, GPU has fallen off the bus",
|
||||
wantSev: "critical",
|
||||
wantOK: true,
|
||||
},
|
||||
{
|
||||
name: "no xid in line",
|
||||
|
||||
Reference in New Issue
Block a user