diff --git a/audit/internal/app/component_status_db.go b/audit/internal/app/component_status_db.go index 8eab2b1..74586b6 100644 --- a/audit/internal/app/component_status_db.go +++ b/audit/internal/app/component_status_db.go @@ -9,6 +9,8 @@ import ( "strings" "sync" "time" + + "bee/audit/internal/collector" ) // ComponentStatusDB is a persistent, append-only store of hardware component health records. @@ -235,7 +237,7 @@ func ApplySATResultToDB(db *ComponentStatusDB, target, archivePath string) { dbStatus := satStatusToDBStatus(overall) detail := target + " SAT: " + overall if overall != "OK" { - if reason := satFailureDetailFromKV(kv); reason != "" { + if reason := prependHardwareFaultBanner(runDir, satFailureDetailFromKV(kv)); reason != "" { detail += " — " + reason } } @@ -340,7 +342,59 @@ func SATFailureDetail(archivePath string) string { if err != nil { return "" } - return satFailureDetailFromKV(parseSATKV(string(data))) + reason := satFailureDetailFromKV(parseSATKV(string(data))) + return prependHardwareFaultBanner(runDir, reason) +} + +// prependHardwareFaultBanner checks the SAT run directory's captured logs +// for a known GPU hardware fault (e.g. Xid 79 "fallen off the bus") and, if +// found, puts a plain-English banner in front of reason — so the task's +// error message reads "GPU fell off the bus, reboot required" directly +// instead of just "failed sub-job(s): ...", which tells an engineer nothing +// without opening the run directory and cross-referencing Xid codes by hand. +func prependHardwareFaultBanner(runDir, reason string) string { + banner := gpuHardwareFaultBanner(runDir) + switch { + case banner == "": + return reason + case reason == "": + return banner + default: + return banner + " (" + reason + ")" + } +} + +// gpuHardwareFaultBanner scans a SAT run directory's captured *.log files +// for NVIDIA Xid codes that mean the GPU cannot recover without a physical +// reboot, returning a de-duplicated, human-readable summary. +func gpuHardwareFaultBanner(runDir string) string { + entries, err := os.ReadDir(runDir) + if err != nil { + return "" + } + seen := map[string]bool{} + var messages []string + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".log") { + continue + } + data, err := readFileLimited(filepath.Join(runDir, e.Name()), 2<<20) + if err != nil { + continue + } + for _, line := range strings.Split(string(data), "\n") { + msg, ok := collector.XidHardwareFaultMessage(line) + if !ok || seen[msg] { + continue + } + seen[msg] = true + messages = append(messages, msg) + } + } + if len(messages) == 0 { + return "" + } + return strings.Join(messages, "; ") } // satFailureDetailFromKV inspects an already-parsed summary.txt for the diff --git a/audit/internal/app/support_bundle.go b/audit/internal/app/support_bundle.go index 216b890..410f86a 100644 --- a/audit/internal/app/support_bundle.go +++ b/audit/internal/app/support_bundle.go @@ -227,9 +227,16 @@ fi `}}, {name: "export/gpu/nvidia-bug-report.txt", cmd: []string{"sh", "-c", ` if command -v nvidia-bug-report.sh >/dev/null 2>&1; then - nvidia-bug-report.sh --output-file /tmp/bee-nvidia-bug-report.log >/dev/null 2>&1 \ - && cat /tmp/bee-nvidia-bug-report.log \ - && rm -f /tmp/bee-nvidia-bug-report.log + rm -f /tmp/bee-nvidia-bug-report.log /tmp/bee-nvidia-bug-report.log.gz + nvidia-bug-report.sh --output-file /tmp/bee-nvidia-bug-report.log >/dev/null 2>&1 + if [ -f /tmp/bee-nvidia-bug-report.log.gz ]; then + gzip -dc /tmp/bee-nvidia-bug-report.log.gz + elif [ -f /tmp/bee-nvidia-bug-report.log ]; then + cat /tmp/bee-nvidia-bug-report.log + else + echo "nvidia-bug-report.sh produced no output file" + fi + rm -f /tmp/bee-nvidia-bug-report.log /tmp/bee-nvidia-bug-report.log.gz else echo "nvidia-bug-report.sh not found" fi diff --git a/audit/internal/collector/xid.go b/audit/internal/collector/xid.go index 8497b01..dd7a33e 100644 --- a/audit/internal/collector/xid.go +++ b/audit/internal/collector/xid.go @@ -11,6 +11,10 @@ var xidCodeParenRE = regexp.MustCompile(`(?i)Xid\s*\([^)]*\)\s*:?\s*(\d+)`) // xidCodeColonRE handles the older "Xid: 64, ..." form with no BDF parens. var xidCodeColonRE = regexp.MustCompile(`(?i)\bXid\s*:\s*(\d+)`) +// xidCodeBareRE handles dcgmi diag's own report wording, e.g. +// "Detected XID 79 for GPU 1" — no colon or parens around the code. +var xidCodeBareRE = regexp.MustCompile(`(?i)\bXid\s+(\d+)\b`) + // xidCodeSeverity maps NVIDIA Xid codes relevant to GPU HBM/ECC health to a // severity, refining the generic "nvidia-xid" kernel-log pattern's default // "warning". Xid 64 is the same InfoROM row-remap-write failure surfaced by @@ -23,7 +27,9 @@ var xidCodeColonRE = regexp.MustCompile(`(?i)\bXid\s*:\s*(\d+)`) var xidCodeSeverity = map[string]string{ "48": "critical", "64": "critical", + "79": "critical", "95": "critical", + "154": "critical", "63": "warning", "94": "warning", "160": "warning", @@ -42,6 +48,29 @@ func XidSeverity(line string) (severity string, ok bool) { return sev, ok } +// xidHardwareFaultMessages maps NVIDIA Xid codes that mean the GPU has left +// the bus and cannot self-recover to a plain-English explanation, so an +// operator sees "physical reboot required" directly instead of having to +// look up what "Xid 79" means in NVIDIA's docs. +// Source: NVIDIA Xid error docs (79 = GPU has fallen off the bus, 154 = GPU +// recovery action escalated to Node Reboot Required). +var xidHardwareFaultMessages = map[string]string{ + "79": "GPU has fallen off the PCIe bus (Xid 79) — hardware fault, will not recover without a physical reboot/power-cycle", + "154": "NVIDIA driver flagged this GPU for a required node reboot (Xid 154: recovery action = Node Reboot Required)", +} + +// XidHardwareFaultMessage returns a plain-English explanation for a log line +// carrying an NVIDIA Xid code that requires a physical reboot to clear, if +// the code is recognized. ok is false otherwise. +func XidHardwareFaultMessage(line string) (message string, ok bool) { + code, ok := extractXidCode(line) + if !ok { + return "", false + } + msg, ok := xidHardwareFaultMessages[code] + return msg, ok +} + func extractXidCode(line string) (string, bool) { if m := xidCodeParenRE.FindStringSubmatch(line); m != nil { return m[1], true @@ -49,5 +78,8 @@ func extractXidCode(line string) (string, bool) { if m := xidCodeColonRE.FindStringSubmatch(line); m != nil { return m[1], true } + if m := xidCodeBareRE.FindStringSubmatch(line); m != nil { + return m[1], true + } return "", false } diff --git a/audit/internal/collector/xid_test.go b/audit/internal/collector/xid_test.go index 15c707a..b9294c5 100644 --- a/audit/internal/collector/xid_test.go +++ b/audit/internal/collector/xid_test.go @@ -28,9 +28,10 @@ func TestXidSeverity(t *testing.T) { wantOK: true, }, { - name: "xid 79 unknown code falls back to caller default", - line: "NVRM: Xid (PCI:0000:65:00): 79, pid=1234, GPU has fallen off the bus", - wantOK: false, + name: "xid 79 GPU fell off the bus is critical", + line: "NVRM: Xid (PCI:0000:65:00): 79, pid=1234, GPU has fallen off the bus", + wantSev: "critical", + wantOK: true, }, { name: "no xid in line", diff --git a/audit/internal/platform/sat.go b/audit/internal/platform/sat.go index 0f38dfc..fe66d0c 100644 --- a/audit/internal/platform/sat.go +++ b/audit/internal/platform/sat.go @@ -940,7 +940,9 @@ func nvidiaSATJobs() []satJob { satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}}, satJob{name: "02-dmidecode-baseboard.log", cmd: []string{"dmidecode", "-t", "baseboard"}}, satJob{name: "03-dmidecode-system.log", cmd: []string{"dmidecode", "-t", "system"}}, - satJob{name: "04-nvidia-bug-report.log", cmd: []string{"nvidia-bug-report.sh", "--output-file", "{{run_dir}}/nvidia-bug-report.log"}}, + // nvidia-bug-report.sh appends .gz to --output-file whenever gzip is + // available, so the artifact actually lands at nvidia-bug-report.log.gz. + satJob{name: "04-nvidia-bug-report.log.gz", cmd: []string{"nvidia-bug-report.sh", "--output-file", "{{run_dir}}/nvidia-bug-report.log"}}, satJob{name: "05-bee-gpu-burn.log", cmd: []string{"bee-gpu-burn", "--seconds", "5", "--size-mb", "64"}, syncBracket: true}, ) } diff --git a/audit/internal/webui/layout.go b/audit/internal/webui/layout.go index 5db779f..04eb932 100644 --- a/audit/internal/webui/layout.go +++ b/audit/internal/webui/layout.go @@ -90,6 +90,7 @@ tbody tr:hover td{background:rgba(0,0,0,.03)} .alert{padding:10px 14px;border-radius:4px;font-size:13px;margin-bottom:14px} .alert-info{background:#dff0ff;border:1px solid #a9d4f5;color:#1e3a5f} .alert-warn{background:var(--warn-bg);border:1px solid #c9ba9b;color:var(--warn-fg)} +.alert-crit{background:var(--crit-bg);border:1px solid var(--crit-border);color:var(--crit-fg);font-weight:700} diff --git a/audit/internal/webui/page_topo.go b/audit/internal/webui/page_topo.go index 8fa4f58..ee8c69b 100644 --- a/audit/internal/webui/page_topo.go +++ b/audit/internal/webui/page_topo.go @@ -587,6 +587,19 @@ type topoEdge struct { } func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string { + // A GPU hardware fault that needs a physical reboot (Xid 79 "fallen off + // the bus", Xid 154 "Node Reboot Required" — see gpuNeedsPhysicalReboot) + // isn't visible in dev.Status: the collector only sets that from PCIe + // link-speed checks, not from SAT/kmsg results. Without this, a GPU that + // dropped off the bus mid-test still renders green here even though the + // Hardware Summary card is showing a critical banner for it. + gpuHardwareFault := false + if db, err := app.OpenComponentStatusDB(filepath.Join(exportDir, "component-status.json")); err == nil { + if _, needsReboot := gpuNeedsPhysicalReboot(matchedRecords(db.All(), nil, []string{"pcie:gpu:"})); needsReboot { + gpuHardwareFault = true + } + } + socketIdx := buildSocketIndex(hw.CPUs) numCols := len(hw.CPUs) if numCols == 0 { @@ -810,6 +823,9 @@ func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string if kind == "gpu" && crossNUMAWarnBDF[p.bdf] && sev < 2 { sev = 2 } + if kind == "gpu" && gpuHardwareFault && sev < 3 { + sev = 3 + } tally.add(sev) if i == 0 && p.dev.Model != nil { model = *p.dev.Model diff --git a/audit/internal/webui/pages.go b/audit/internal/webui/pages.go index d3f07da..ef7e6d5 100644 --- a/audit/internal/webui/pages.go +++ b/audit/internal/webui/pages.go @@ -138,6 +138,7 @@ function openComponentDetail(type) { func renderDashboard(opts HandlerOptions) string { var b strings.Builder b.WriteString(renderAuditStatusBanner(opts)) + b.WriteString(renderTimeSyncCard()) b.WriteString(renderHardwareSummaryCard(opts)) b.WriteString(renderHealthCard(opts)) b.WriteString(renderMetrics()) @@ -152,6 +153,45 @@ setInterval(function(){ return b.String() } +// renderTimeSyncCard shows the server's current time and a button that syncs +// the host clock and timezone to whatever the client's browser reports. +func renderTimeSyncCard() string { + return `
+
+ System Time +
+ +
+
+
+ +
+
+` +} + // renderAuditStatusBanner shows a live progress banner when an audit task is // running and auto-reloads the page when it completes. func renderAuditStatusBanner(opts HandlerOptions) string { @@ -242,10 +282,16 @@ func renderHardwareSummaryCard(opts HandlerOptions) string { if db, err := app.OpenComponentStatusDB(filepath.Join(opts.ExportDir, "component-status.json")); err == nil { records = db.All() } + gpuRecords := matchedRecords(records, nil, []string{"pcie:gpu:"}) var b strings.Builder b.WriteString(`
Hardware Summary
`) + if reason, needsReboot := gpuNeedsPhysicalReboot(gpuRecords); needsReboot { + fmt.Fprintf(&b, `
⚠ GPU needs a physical reboot/power-cycle — further GPU tests will keep failing until then: %s
`, + html.EscapeString(reason)) + } + // Server identity block above the component table. { var model, serial string @@ -297,8 +343,7 @@ func renderHardwareSummaryCard(opts HandlerOptions) string { writeRow("Storage", hwDescribeStorage(hw), renderComponentChips(matchedRecords(records, []string{"storage:all"}, []string{"storage:"})), "storage") - writeRow("GPU", hwDescribeGPU(hw), - renderComponentChips(matchedRecords(records, nil, []string{"pcie:gpu:"})), "gpu") + writeRow("GPU", hwDescribeGPU(hw), renderComponentChips(gpuRecords), "gpu") psuMatched := matchedRecords(records, nil, []string{"psu:"}) if len(psuMatched) == 0 && len(hw.PowerSupplies) > 0 { @@ -986,6 +1031,25 @@ func runtimeIssueDescriptions(issues []schema.RuntimeIssue, codes ...string) str return strings.Join(messages, "; ") } +// gpuNeedsPhysicalReboot reports whether any GPU component record carries a +// hardware-fault reason that a driver reset can't clear (e.g. Xid 79 "GPU +// has fallen off the bus", Xid 154 "Node Reboot Required" — see +// collector.xidHardwareFaultMessages). Those errors mean every subsequent +// GPU SAT job will keep failing until the node is physically power-cycled, +// so this drives a dashboard banner that says so up front instead of making +// an operator burn another test cycle to rediscover it. +func gpuNeedsPhysicalReboot(records []app.ComponentStatusRecord) (reason string, needsReboot bool) { + for _, rec := range records { + if !strings.EqualFold(strings.TrimSpace(rec.Status), "Critical") { + continue + } + if strings.Contains(strings.ToLower(rec.ErrorSummary), "reboot") { + return rec.ErrorSummary, true + } + } + return "", false +} + // chipLetterClass maps a component status to a single display letter and CSS class. func chipLetterClass(status string) (letter, cls string) { switch strings.ToUpper(strings.TrimSpace(status)) {