reanimator/bee
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(nvidia): surface Xid 79/154 GPU bus-fall-off as a physical-reboot-required signal

Browse Source 
nvidia-bug-report.sh appends .gz to --output-file when gzip is available,
which silently produced empty nvidia-bug-report.txt in support bundles
(cat looked for the uncompressed name that never existed).

Also: a GPU that falls off the PCIe/NVLink bus (Xid 79) or gets flagged
for Node Reboot Required (Xid 154) mid-SAT-run left every downstream test
failing with generic, unrelated-looking errors (CUDA "unknown error",
"unable to determine device handle") with no indication the GPU needed a
physical power-cycle to recover. Detect these codes from SAT run logs and
surface a plain-English "physical reboot required" message in the task's
failure detail, the persisted component-status DB, a dashboard banner on
the Hardware Summary card, and topology diagram GPU-node severity.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-08-06 12:28:34 +03:00
co-authored by Claude Sonnet 5
parent a34e823f82
commit b2b3f86c8d
8 changed files with 188 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
audit/internal/webui/layout.go
+1
View File
@@ -90,6 +90,7 @@ tbody tr:hover td{background:rgba(0,0,0,.03)}
.alert{padding:10px 14px;border-radius:4px;font-size:13px;margin-bottom:14px}
.alert-info{background:#dff0ff;border:1px solid #a9d4f5;color:#1e3a5f}
.alert-warn{background:var(--warn-bg);border:1px solid #c9ba9b;color:var(--warn-fg)}
.alert-crit{background:var(--crit-bg);border:1px solid var(--crit-border);color:var(--crit-fg);font-weight:700}
</style>
</head>
<body>
audit/internal/webui/page_topo.go
+16
View File
@@ -587,6 +587,19 @@ type topoEdge struct {
}
func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string {
// A GPU hardware fault that needs a physical reboot (Xid 79 "fallen off
// the bus", Xid 154 "Node Reboot Required" — see gpuNeedsPhysicalReboot)
// isn't visible in dev.Status: the collector only sets that from PCIe
// link-speed checks, not from SAT/kmsg results. Without this, a GPU that
// dropped off the bus mid-test still renders green here even though the
// Hardware Summary card is showing a critical banner for it.
gpuHardwareFault := false
if db, err := app.OpenComponentStatusDB(filepath.Join(exportDir, "component-status.json")); err == nil {
if _, needsReboot := gpuNeedsPhysicalReboot(matchedRecords(db.All(), nil, []string{"pcie:gpu:"})); needsReboot {
gpuHardwareFault = true
}
}
socketIdx := buildSocketIndex(hw.CPUs)
numCols := len(hw.CPUs)
if numCols == 0 {
@@ -810,6 +823,9 @@ func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string
if kind == "gpu" && crossNUMAWarnBDF[p.bdf] && sev < 2 {
sev = 2
}
if kind == "gpu" && gpuHardwareFault && sev < 3 {
sev = 3
}
tally.add(sev)
if i == 0 && p.dev.Model != nil {
model = *p.dev.Model
audit/internal/webui/pages.go
+66 -2
View File
@@ -138,6 +138,7 @@ function openComponentDetail(type) {
func renderDashboard(opts HandlerOptions) string {
var b strings.Builder
b.WriteString(renderAuditStatusBanner(opts))
b.WriteString(renderTimeSyncCard())
b.WriteString(renderHardwareSummaryCard(opts))
b.WriteString(renderHealthCard(opts))
b.WriteString(renderMetrics())
@@ -152,6 +153,45 @@ setInterval(function(){
return b.String()
}
// renderTimeSyncCard shows the server's current time and a button that syncs
// the host clock and timezone to whatever the client's browser reports.
func renderTimeSyncCard() string {
return `<div class="card" style="margin-bottom:16px">
<div class="card-head card-head-actions">
<span>System Time</span>
<div class="card-head-buttons">
<button id="time-sync-btn" class="btn btn-primary btn-sm" onclick="timeSyncRun()">&#8635; Sync with this browser</button>
</div>
</div>
<div class="card-body">
<span id="time-sync-status" style="font-size:13px;color:var(--muted)"></span>
</div>
</div>
<script>
function timeSyncRun() {
var btn = document.getElementById('time-sync-btn');
var status = document.getElementById('time-sync-status');
btn.disabled = true;
var tz = Intl.DateTimeFormat().resolvedOptions().timeZone || '';
fetch('/api/system/time-sync', {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({timezone: tz, epoch_ms: Date.now()})
})
.then(function(r) { if (!r.ok) return r.text().then(function(t){throw new Error(t || r.statusText);}); return r.json(); })
.then(function(d) {
status.style.color = 'var(--ok-fg,#2c662d)';
status.textContent = '✓ Synced to ' + tz + ' at ' + new Date().toLocaleString();
})
.catch(function(err) {
status.style.color = 'var(--crit-fg,#9f3a38)';
status.textContent = '✗ Sync failed: ' + err.message;
})
.finally(function() { btn.disabled = false; });
}
</script>`
}
// renderAuditStatusBanner shows a live progress banner when an audit task is
// running and auto-reloads the page when it completes.
func renderAuditStatusBanner(opts HandlerOptions) string {
@@ -242,10 +282,16 @@ func renderHardwareSummaryCard(opts HandlerOptions) string {
if db, err := app.OpenComponentStatusDB(filepath.Join(opts.ExportDir, "component-status.json")); err == nil {
records = db.All()
}
gpuRecords := matchedRecords(records, nil, []string{"pcie:gpu:"})
var b strings.Builder
b.WriteString(`<div class="card"` + cardID + `><div class="card-head">Hardware Summary</div><div class="card-body">`)
if reason, needsReboot := gpuNeedsPhysicalReboot(gpuRecords); needsReboot {
fmt.Fprintf(&b, `<div class="alert alert-crit">&#9888; GPU needs a physical reboot/power-cycle — further GPU tests will keep failing until then: %s</div>`,
html.EscapeString(reason))
}
// Server identity block above the component table.
{
var model, serial string
@@ -297,8 +343,7 @@ func renderHardwareSummaryCard(opts HandlerOptions) string {
writeRow("Storage", hwDescribeStorage(hw),
renderComponentChips(matchedRecords(records, []string{"storage:all"}, []string{"storage:"})), "storage")
writeRow("GPU", hwDescribeGPU(hw),
renderComponentChips(matchedRecords(records, nil, []string{"pcie:gpu:"})), "gpu")
writeRow("GPU", hwDescribeGPU(hw), renderComponentChips(gpuRecords), "gpu")
psuMatched := matchedRecords(records, nil, []string{"psu:"})
if len(psuMatched) == 0 && len(hw.PowerSupplies) > 0 {
@@ -986,6 +1031,25 @@ func runtimeIssueDescriptions(issues []schema.RuntimeIssue, codes ...string) str
return strings.Join(messages, "; ")
}
// gpuNeedsPhysicalReboot reports whether any GPU component record carries a
// hardware-fault reason that a driver reset can't clear (e.g. Xid 79 "GPU
// has fallen off the bus", Xid 154 "Node Reboot Required" — see
// collector.xidHardwareFaultMessages). Those errors mean every subsequent
// GPU SAT job will keep failing until the node is physically power-cycled,
// so this drives a dashboard banner that says so up front instead of making
// an operator burn another test cycle to rediscover it.
func gpuNeedsPhysicalReboot(records []app.ComponentStatusRecord) (reason string, needsReboot bool) {
for _, rec := range records {
if !strings.EqualFold(strings.TrimSpace(rec.Status), "Critical") {
continue
}
if strings.Contains(strings.ToLower(rec.ErrorSummary), "reboot") {
return rec.ErrorSummary, true
}
}
return "", false
}
// chipLetterClass maps a component status to a single display letter and CSS class.
func chipLetterClass(status string) (letter, cls string) {
switch strings.ToUpper(strings.TrimSpace(status)) {