fix(nvidia): surface Xid 79/154 GPU bus-fall-off as a physical-reboot-required signal
nvidia-bug-report.sh appends .gz to --output-file when gzip is available, which silently produced empty nvidia-bug-report.txt in support bundles (cat looked for the uncompressed name that never existed). Also: a GPU that falls off the PCIe/NVLink bus (Xid 79) or gets flagged for Node Reboot Required (Xid 154) mid-SAT-run left every downstream test failing with generic, unrelated-looking errors (CUDA "unknown error", "unable to determine device handle") with no indication the GPU needed a physical power-cycle to recover. Detect these codes from SAT run logs and surface a plain-English "physical reboot required" message in the task's failure detail, the persisted component-status DB, a dashboard banner on the Hardware Summary card, and topology diagram GPU-node severity. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
a34e823f82
commit
b2b3f86c8d
@@ -587,6 +587,19 @@ type topoEdge struct {
|
||||
}
|
||||
|
||||
func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string {
|
||||
// A GPU hardware fault that needs a physical reboot (Xid 79 "fallen off
|
||||
// the bus", Xid 154 "Node Reboot Required" — see gpuNeedsPhysicalReboot)
|
||||
// isn't visible in dev.Status: the collector only sets that from PCIe
|
||||
// link-speed checks, not from SAT/kmsg results. Without this, a GPU that
|
||||
// dropped off the bus mid-test still renders green here even though the
|
||||
// Hardware Summary card is showing a critical banner for it.
|
||||
gpuHardwareFault := false
|
||||
if db, err := app.OpenComponentStatusDB(filepath.Join(exportDir, "component-status.json")); err == nil {
|
||||
if _, needsReboot := gpuNeedsPhysicalReboot(matchedRecords(db.All(), nil, []string{"pcie:gpu:"})); needsReboot {
|
||||
gpuHardwareFault = true
|
||||
}
|
||||
}
|
||||
|
||||
socketIdx := buildSocketIndex(hw.CPUs)
|
||||
numCols := len(hw.CPUs)
|
||||
if numCols == 0 {
|
||||
@@ -810,6 +823,9 @@ func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string
|
||||
if kind == "gpu" && crossNUMAWarnBDF[p.bdf] && sev < 2 {
|
||||
sev = 2
|
||||
}
|
||||
if kind == "gpu" && gpuHardwareFault && sev < 3 {
|
||||
sev = 3
|
||||
}
|
||||
tally.add(sev)
|
||||
if i == 0 && p.dev.Model != nil {
|
||||
model = *p.dev.Model
|
||||
|
||||
Reference in New Issue
Block a user