fix(nvidia): surface Xid 79/154 GPU bus-fall-off as a physical-reboot-required signal
nvidia-bug-report.sh appends .gz to --output-file when gzip is available, which silently produced empty nvidia-bug-report.txt in support bundles (cat looked for the uncompressed name that never existed). Also: a GPU that falls off the PCIe/NVLink bus (Xid 79) or gets flagged for Node Reboot Required (Xid 154) mid-SAT-run left every downstream test failing with generic, unrelated-looking errors (CUDA "unknown error", "unable to determine device handle") with no indication the GPU needed a physical power-cycle to recover. Detect these codes from SAT run logs and surface a plain-English "physical reboot required" message in the task's failure detail, the persisted component-status DB, a dashboard banner on the Hardware Summary card, and topology diagram GPU-node severity. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
a34e823f82
commit
b2b3f86c8d
@@ -9,6 +9,8 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"bee/audit/internal/collector"
|
||||
)
|
||||
|
||||
// ComponentStatusDB is a persistent, append-only store of hardware component health records.
|
||||
@@ -235,7 +237,7 @@ func ApplySATResultToDB(db *ComponentStatusDB, target, archivePath string) {
|
||||
dbStatus := satStatusToDBStatus(overall)
|
||||
detail := target + " SAT: " + overall
|
||||
if overall != "OK" {
|
||||
if reason := satFailureDetailFromKV(kv); reason != "" {
|
||||
if reason := prependHardwareFaultBanner(runDir, satFailureDetailFromKV(kv)); reason != "" {
|
||||
detail += " — " + reason
|
||||
}
|
||||
}
|
||||
@@ -340,7 +342,59 @@ func SATFailureDetail(archivePath string) string {
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return satFailureDetailFromKV(parseSATKV(string(data)))
|
||||
reason := satFailureDetailFromKV(parseSATKV(string(data)))
|
||||
return prependHardwareFaultBanner(runDir, reason)
|
||||
}
|
||||
|
||||
// prependHardwareFaultBanner checks the SAT run directory's captured logs
|
||||
// for a known GPU hardware fault (e.g. Xid 79 "fallen off the bus") and, if
|
||||
// found, puts a plain-English banner in front of reason — so the task's
|
||||
// error message reads "GPU fell off the bus, reboot required" directly
|
||||
// instead of just "failed sub-job(s): ...", which tells an engineer nothing
|
||||
// without opening the run directory and cross-referencing Xid codes by hand.
|
||||
func prependHardwareFaultBanner(runDir, reason string) string {
|
||||
banner := gpuHardwareFaultBanner(runDir)
|
||||
switch {
|
||||
case banner == "":
|
||||
return reason
|
||||
case reason == "":
|
||||
return banner
|
||||
default:
|
||||
return banner + " (" + reason + ")"
|
||||
}
|
||||
}
|
||||
|
||||
// gpuHardwareFaultBanner scans a SAT run directory's captured *.log files
|
||||
// for NVIDIA Xid codes that mean the GPU cannot recover without a physical
|
||||
// reboot, returning a de-duplicated, human-readable summary.
|
||||
func gpuHardwareFaultBanner(runDir string) string {
|
||||
entries, err := os.ReadDir(runDir)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
var messages []string
|
||||
for _, e := range entries {
|
||||
if e.IsDir() || !strings.HasSuffix(e.Name(), ".log") {
|
||||
continue
|
||||
}
|
||||
data, err := readFileLimited(filepath.Join(runDir, e.Name()), 2<<20)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
msg, ok := collector.XidHardwareFaultMessage(line)
|
||||
if !ok || seen[msg] {
|
||||
continue
|
||||
}
|
||||
seen[msg] = true
|
||||
messages = append(messages, msg)
|
||||
}
|
||||
}
|
||||
if len(messages) == 0 {
|
||||
return ""
|
||||
}
|
||||
return strings.Join(messages, "; ")
|
||||
}
|
||||
|
||||
// satFailureDetailFromKV inspects an already-parsed summary.txt for the
|
||||
|
||||
@@ -227,9 +227,16 @@ fi
|
||||
`}},
|
||||
{name: "export/gpu/nvidia-bug-report.txt", cmd: []string{"sh", "-c", `
|
||||
if command -v nvidia-bug-report.sh >/dev/null 2>&1; then
|
||||
nvidia-bug-report.sh --output-file /tmp/bee-nvidia-bug-report.log >/dev/null 2>&1 \
|
||||
&& cat /tmp/bee-nvidia-bug-report.log \
|
||||
&& rm -f /tmp/bee-nvidia-bug-report.log
|
||||
rm -f /tmp/bee-nvidia-bug-report.log /tmp/bee-nvidia-bug-report.log.gz
|
||||
nvidia-bug-report.sh --output-file /tmp/bee-nvidia-bug-report.log >/dev/null 2>&1
|
||||
if [ -f /tmp/bee-nvidia-bug-report.log.gz ]; then
|
||||
gzip -dc /tmp/bee-nvidia-bug-report.log.gz
|
||||
elif [ -f /tmp/bee-nvidia-bug-report.log ]; then
|
||||
cat /tmp/bee-nvidia-bug-report.log
|
||||
else
|
||||
echo "nvidia-bug-report.sh produced no output file"
|
||||
fi
|
||||
rm -f /tmp/bee-nvidia-bug-report.log /tmp/bee-nvidia-bug-report.log.gz
|
||||
else
|
||||
echo "nvidia-bug-report.sh not found"
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user