fix(nvidia): surface Xid 79/154 GPU bus-fall-off as a physical-reboot-required signal

nvidia-bug-report.sh appends .gz to --output-file when gzip is available,
which silently produced empty nvidia-bug-report.txt in support bundles
(cat looked for the uncompressed name that never existed).

Also: a GPU that falls off the PCIe/NVLink bus (Xid 79) or gets flagged
for Node Reboot Required (Xid 154) mid-SAT-run left every downstream test
failing with generic, unrelated-looking errors (CUDA "unknown error",
"unable to determine device handle") with no indication the GPU needed a
physical power-cycle to recover. Detect these codes from SAT run logs and
surface a plain-English "physical reboot required" message in the task's
failure detail, the persisted component-status DB, a dashboard banner on
the Hardware Summary card, and topology diagram GPU-node severity.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

audit/internal/app/component_status_db.go

@@ -9,6 +9,8 @@ import (
 	"strings"
 	"sync"
 	"time"
+
+	"bee/audit/internal/collector"
 )
 
 // ComponentStatusDB is a persistent, append-only store of hardware component health records.
@@ -235,7 +237,7 @@ func ApplySATResultToDB(db *ComponentStatusDB, target, archivePath string) {
 	dbStatus := satStatusToDBStatus(overall)
 	detail := target + " SAT: " + overall
 	if overall != "OK" {
-		if reason := satFailureDetailFromKV(kv); reason != "" {
+		if reason := prependHardwareFaultBanner(runDir, satFailureDetailFromKV(kv)); reason != "" {
 			detail += " — " + reason
 		}
 	}
@@ -340,7 +342,59 @@ func SATFailureDetail(archivePath string) string {
 	if err != nil {
 		return ""
 	}
-	return satFailureDetailFromKV(parseSATKV(string(data)))
+	reason := satFailureDetailFromKV(parseSATKV(string(data)))
+	return prependHardwareFaultBanner(runDir, reason)
+}
+
+// prependHardwareFaultBanner checks the SAT run directory's captured logs
+// for a known GPU hardware fault (e.g. Xid 79 "fallen off the bus") and, if
+// found, puts a plain-English banner in front of reason — so the task's
+// error message reads "GPU fell off the bus, reboot required" directly
+// instead of just "failed sub-job(s): ...", which tells an engineer nothing
+// without opening the run directory and cross-referencing Xid codes by hand.
+func prependHardwareFaultBanner(runDir, reason string) string {
+	banner := gpuHardwareFaultBanner(runDir)
+	switch {
+	case banner == "":
+		return reason
+	case reason == "":
+		return banner
+	default:
+		return banner + " (" + reason + ")"
+	}
+}
+
+// gpuHardwareFaultBanner scans a SAT run directory's captured *.log files
+// for NVIDIA Xid codes that mean the GPU cannot recover without a physical
+// reboot, returning a de-duplicated, human-readable summary.
+func gpuHardwareFaultBanner(runDir string) string {
+	entries, err := os.ReadDir(runDir)
+	if err != nil {
+		return ""
+	}
+	seen := map[string]bool{}
+	var messages []string
+	for _, e := range entries {
+		if e.IsDir() || !strings.HasSuffix(e.Name(), ".log") {
+			continue
+		}
+		data, err := readFileLimited(filepath.Join(runDir, e.Name()), 2<<20)
+		if err != nil {
+			continue
+		}
+		for _, line := range strings.Split(string(data), "\n") {
+			msg, ok := collector.XidHardwareFaultMessage(line)
+			if !ok || seen[msg] {
+				continue
+			}
+			seen[msg] = true
+			messages = append(messages, msg)
+		}
+	}
+	if len(messages) == 0 {
+		return ""
+	}
+	return strings.Join(messages, "; ")
 }
 
 // satFailureDetailFromKV inspects an already-parsed summary.txt for the

audit/internal/app/support_bundle.go

@@ -227,9 +227,16 @@ fi
 `}},
 	{name: "export/gpu/nvidia-bug-report.txt", cmd: []string{"sh", "-c", `
 if command -v nvidia-bug-report.sh >/dev/null 2>&1; then
-  nvidia-bug-report.sh --output-file /tmp/bee-nvidia-bug-report.log >/dev/null 2>&1 \
+  rm -f /tmp/bee-nvidia-bug-report.log /tmp/bee-nvidia-bug-report.log.gz
-    && cat /tmp/bee-nvidia-bug-report.log \
+  nvidia-bug-report.sh --output-file /tmp/bee-nvidia-bug-report.log >/dev/null 2>&1
-    && rm -f /tmp/bee-nvidia-bug-report.log
+  if [ -f /tmp/bee-nvidia-bug-report.log.gz ]; then
+    gzip -dc /tmp/bee-nvidia-bug-report.log.gz
+  elif [ -f /tmp/bee-nvidia-bug-report.log ]; then
+    cat /tmp/bee-nvidia-bug-report.log
+  else
+    echo "nvidia-bug-report.sh produced no output file"
+  fi
+  rm -f /tmp/bee-nvidia-bug-report.log /tmp/bee-nvidia-bug-report.log.gz
 else
   echo "nvidia-bug-report.sh not found"
 fi

audit/internal/collector/xid.go

@@ -11,6 +11,10 @@ var xidCodeParenRE = regexp.MustCompile(`(?i)Xid\s*\([^)]*\)\s*:?\s*(\d+)`)
 // xidCodeColonRE handles the older "Xid: 64, ..." form with no BDF parens.
 var xidCodeColonRE = regexp.MustCompile(`(?i)\bXid\s*:\s*(\d+)`)
 
+// xidCodeBareRE handles dcgmi diag's own report wording, e.g.
+// "Detected XID 79 for GPU 1" — no colon or parens around the code.
+var xidCodeBareRE = regexp.MustCompile(`(?i)\bXid\s+(\d+)\b`)
+
 // xidCodeSeverity maps NVIDIA Xid codes relevant to GPU HBM/ECC health to a
 // severity, refining the generic "nvidia-xid" kernel-log pattern's default
 // "warning". Xid 64 is the same InfoROM row-remap-write failure surfaced by
@@ -23,7 +27,9 @@ var xidCodeColonRE = regexp.MustCompile(`(?i)\bXid\s*:\s*(\d+)`)
 var xidCodeSeverity = map[string]string{
 	"48":  "critical",
 	"64":  "critical",
+	"79":  "critical",
 	"95":  "critical",
+	"154": "critical",
 	"63":  "warning",
 	"94":  "warning",
 	"160": "warning",
@@ -42,6 +48,29 @@ func XidSeverity(line string) (severity string, ok bool) {
 	return sev, ok
 }
 
+// xidHardwareFaultMessages maps NVIDIA Xid codes that mean the GPU has left
+// the bus and cannot self-recover to a plain-English explanation, so an
+// operator sees "physical reboot required" directly instead of having to
+// look up what "Xid 79" means in NVIDIA's docs.
+// Source: NVIDIA Xid error docs (79 = GPU has fallen off the bus, 154 = GPU
+// recovery action escalated to Node Reboot Required).
+var xidHardwareFaultMessages = map[string]string{
+	"79":  "GPU has fallen off the PCIe bus (Xid 79) — hardware fault, will not recover without a physical reboot/power-cycle",
+	"154": "NVIDIA driver flagged this GPU for a required node reboot (Xid 154: recovery action = Node Reboot Required)",
+}
+
+// XidHardwareFaultMessage returns a plain-English explanation for a log line
+// carrying an NVIDIA Xid code that requires a physical reboot to clear, if
+// the code is recognized. ok is false otherwise.
+func XidHardwareFaultMessage(line string) (message string, ok bool) {
+	code, ok := extractXidCode(line)
+	if !ok {
+		return "", false
+	}
+	msg, ok := xidHardwareFaultMessages[code]
+	return msg, ok
+}
+
 func extractXidCode(line string) (string, bool) {
 	if m := xidCodeParenRE.FindStringSubmatch(line); m != nil {
 		return m[1], true
@@ -49,5 +78,8 @@ func extractXidCode(line string) (string, bool) {
 	if m := xidCodeColonRE.FindStringSubmatch(line); m != nil {
 		return m[1], true
 	}
+	if m := xidCodeBareRE.FindStringSubmatch(line); m != nil {
+		return m[1], true
+	}
 	return "", false
 }

audit/internal/collector/xid_test.go

@@ -28,9 +28,10 @@ func TestXidSeverity(t *testing.T) {
 			wantOK:  true,
 		},
 		{
-			name:   "xid 79 unknown code falls back to caller default",
+			name:    "xid 79 GPU fell off the bus is critical",
-			line:   "NVRM: Xid (PCI:0000:65:00): 79, pid=1234, GPU has fallen off the bus",
+			line:    "NVRM: Xid (PCI:0000:65:00): 79, pid=1234, GPU has fallen off the bus",
-			wantOK: false,
+			wantSev: "critical",
+			wantOK:  true,
 		},
 		{
 			name:   "no xid in line",

audit/internal/platform/sat.go

@@ -940,7 +940,9 @@ func nvidiaSATJobs() []satJob {
 		satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
 		satJob{name: "02-dmidecode-baseboard.log", cmd: []string{"dmidecode", "-t", "baseboard"}},
 		satJob{name: "03-dmidecode-system.log", cmd: []string{"dmidecode", "-t", "system"}},
-		satJob{name: "04-nvidia-bug-report.log", cmd: []string{"nvidia-bug-report.sh", "--output-file", "{{run_dir}}/nvidia-bug-report.log"}},
+		// nvidia-bug-report.sh appends .gz to --output-file whenever gzip is
+		// available, so the artifact actually lands at nvidia-bug-report.log.gz.
+		satJob{name: "04-nvidia-bug-report.log.gz", cmd: []string{"nvidia-bug-report.sh", "--output-file", "{{run_dir}}/nvidia-bug-report.log"}},
 		satJob{name: "05-bee-gpu-burn.log", cmd: []string{"bee-gpu-burn", "--seconds", "5", "--size-mb", "64"}, syncBracket: true},
 	)
 }

audit/internal/webui/layout.go

@@ -90,6 +90,7 @@ tbody tr:hover td{background:rgba(0,0,0,.03)}
 .alert{padding:10px 14px;border-radius:4px;font-size:13px;margin-bottom:14px}
 .alert-info{background:#dff0ff;border:1px solid #a9d4f5;color:#1e3a5f}
 .alert-warn{background:var(--warn-bg);border:1px solid #c9ba9b;color:var(--warn-fg)}
+.alert-crit{background:var(--crit-bg);border:1px solid var(--crit-border);color:var(--crit-fg);font-weight:700}
 </style>
 </head>
 <body>

audit/internal/webui/page_topo.go

@@ -587,6 +587,19 @@ type topoEdge struct {
 }
 
 func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string {
+	// A GPU hardware fault that needs a physical reboot (Xid 79 "fallen off
+	// the bus", Xid 154 "Node Reboot Required" — see gpuNeedsPhysicalReboot)
+	// isn't visible in dev.Status: the collector only sets that from PCIe
+	// link-speed checks, not from SAT/kmsg results. Without this, a GPU that
+	// dropped off the bus mid-test still renders green here even though the
+	// Hardware Summary card is showing a critical banner for it.
+	gpuHardwareFault := false
+	if db, err := app.OpenComponentStatusDB(filepath.Join(exportDir, "component-status.json")); err == nil {
+		if _, needsReboot := gpuNeedsPhysicalReboot(matchedRecords(db.All(), nil, []string{"pcie:gpu:"})); needsReboot {
+			gpuHardwareFault = true
+		}
+	}
+
 	socketIdx := buildSocketIndex(hw.CPUs)
 	numCols := len(hw.CPUs)
 	if numCols == 0 {
@@ -810,6 +823,9 @@ func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string
 				if kind == "gpu" && crossNUMAWarnBDF[p.bdf] && sev < 2 {
 					sev = 2
 				}
+				if kind == "gpu" && gpuHardwareFault && sev < 3 {
+					sev = 3
+				}
 				tally.add(sev)
 				if i == 0 && p.dev.Model != nil {
 					model = *p.dev.Model

audit/internal/webui/pages.go

@@ -138,6 +138,7 @@ function openComponentDetail(type) {
 func renderDashboard(opts HandlerOptions) string {
 	var b strings.Builder
 	b.WriteString(renderAuditStatusBanner(opts))
+	b.WriteString(renderTimeSyncCard())
 	b.WriteString(renderHardwareSummaryCard(opts))
 	b.WriteString(renderHealthCard(opts))
 	b.WriteString(renderMetrics())
@@ -152,6 +153,45 @@ setInterval(function(){
 	return b.String()
 }
 
+// renderTimeSyncCard shows the server's current time and a button that syncs
+// the host clock and timezone to whatever the client's browser reports.
+func renderTimeSyncCard() string {
+	return `<div class="card" style="margin-bottom:16px">
+  <div class="card-head card-head-actions">
+    <span>System Time</span>
+    <div class="card-head-buttons">
+      <button id="time-sync-btn" class="btn btn-primary btn-sm" onclick="timeSyncRun()">&#8635; Sync with this browser</button>
+    </div>
+  </div>
+  <div class="card-body">
+    <span id="time-sync-status" style="font-size:13px;color:var(--muted)"></span>
+  </div>
+</div>
+<script>
+function timeSyncRun() {
+  var btn = document.getElementById('time-sync-btn');
+  var status = document.getElementById('time-sync-status');
+  btn.disabled = true;
+  var tz = Intl.DateTimeFormat().resolvedOptions().timeZone || '';
+  fetch('/api/system/time-sync', {
+    method: 'POST',
+    headers: {'Content-Type': 'application/json'},
+    body: JSON.stringify({timezone: tz, epoch_ms: Date.now()})
+  })
+    .then(function(r) { if (!r.ok) return r.text().then(function(t){throw new Error(t || r.statusText);}); return r.json(); })
+    .then(function(d) {
+      status.style.color = 'var(--ok-fg,#2c662d)';
+      status.textContent = '✓ Synced to ' + tz + ' at ' + new Date().toLocaleString();
+    })
+    .catch(function(err) {
+      status.style.color = 'var(--crit-fg,#9f3a38)';
+      status.textContent = '✗ Sync failed: ' + err.message;
+    })
+    .finally(function() { btn.disabled = false; });
+}
+</script>`
+}
+
 // renderAuditStatusBanner shows a live progress banner when an audit task is
 // running and auto-reloads the page when it completes.
 func renderAuditStatusBanner(opts HandlerOptions) string {
@@ -242,10 +282,16 @@ func renderHardwareSummaryCard(opts HandlerOptions) string {
 	if db, err := app.OpenComponentStatusDB(filepath.Join(opts.ExportDir, "component-status.json")); err == nil {
 		records = db.All()
 	}
+	gpuRecords := matchedRecords(records, nil, []string{"pcie:gpu:"})
 
 	var b strings.Builder
 	b.WriteString(`<div class="card"` + cardID + `><div class="card-head">Hardware Summary</div><div class="card-body">`)
 
+	if reason, needsReboot := gpuNeedsPhysicalReboot(gpuRecords); needsReboot {
+		fmt.Fprintf(&b, `<div class="alert alert-crit">&#9888; GPU needs a physical reboot/power-cycle — further GPU tests will keep failing until then: %s</div>`,
+			html.EscapeString(reason))
+	}
+
 	// Server identity block above the component table.
 	{
 		var model, serial string
@@ -297,8 +343,7 @@ func renderHardwareSummaryCard(opts HandlerOptions) string {
 	writeRow("Storage", hwDescribeStorage(hw),
 		renderComponentChips(matchedRecords(records, []string{"storage:all"}, []string{"storage:"})), "storage")
 
-	writeRow("GPU", hwDescribeGPU(hw),
+	writeRow("GPU", hwDescribeGPU(hw), renderComponentChips(gpuRecords), "gpu")
-		renderComponentChips(matchedRecords(records, nil, []string{"pcie:gpu:"})), "gpu")
 
 	psuMatched := matchedRecords(records, nil, []string{"psu:"})
 	if len(psuMatched) == 0 && len(hw.PowerSupplies) > 0 {
@@ -986,6 +1031,25 @@ func runtimeIssueDescriptions(issues []schema.RuntimeIssue, codes ...string) str
 	return strings.Join(messages, "; ")
 }
 
+// gpuNeedsPhysicalReboot reports whether any GPU component record carries a
+// hardware-fault reason that a driver reset can't clear (e.g. Xid 79 "GPU
+// has fallen off the bus", Xid 154 "Node Reboot Required" — see
+// collector.xidHardwareFaultMessages). Those errors mean every subsequent
+// GPU SAT job will keep failing until the node is physically power-cycled,
+// so this drives a dashboard banner that says so up front instead of making
+// an operator burn another test cycle to rediscover it.
+func gpuNeedsPhysicalReboot(records []app.ComponentStatusRecord) (reason string, needsReboot bool) {
+	for _, rec := range records {
+		if !strings.EqualFold(strings.TrimSpace(rec.Status), "Critical") {
+			continue
+		}
+		if strings.Contains(strings.ToLower(rec.ErrorSummary), "reboot") {
+			return rec.ErrorSummary, true
+		}
+	}
+	return "", false
+}
+
 // chipLetterClass maps a component status to a single display letter and CSS class.
 func chipLetterClass(status string) (letter, cls string) {
 	switch strings.ToUpper(strings.TrimSpace(status)) {