fix(nvidia): surface Xid 79/154 GPU bus-fall-off as a physical-reboot-required signal
nvidia-bug-report.sh appends .gz to --output-file when gzip is available, which silently produced empty nvidia-bug-report.txt in support bundles (cat looked for the uncompressed name that never existed). Also: a GPU that falls off the PCIe/NVLink bus (Xid 79) or gets flagged for Node Reboot Required (Xid 154) mid-SAT-run left every downstream test failing with generic, unrelated-looking errors (CUDA "unknown error", "unable to determine device handle") with no indication the GPU needed a physical power-cycle to recover. Detect these codes from SAT run logs and surface a plain-English "physical reboot required" message in the task's failure detail, the persisted component-status DB, a dashboard banner on the Hardware Summary card, and topology diagram GPU-node severity. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
a34e823f82
commit
b2b3f86c8d
@@ -138,6 +138,7 @@ function openComponentDetail(type) {
|
||||
func renderDashboard(opts HandlerOptions) string {
|
||||
var b strings.Builder
|
||||
b.WriteString(renderAuditStatusBanner(opts))
|
||||
b.WriteString(renderTimeSyncCard())
|
||||
b.WriteString(renderHardwareSummaryCard(opts))
|
||||
b.WriteString(renderHealthCard(opts))
|
||||
b.WriteString(renderMetrics())
|
||||
@@ -152,6 +153,45 @@ setInterval(function(){
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// renderTimeSyncCard shows the server's current time and a button that syncs
|
||||
// the host clock and timezone to whatever the client's browser reports.
|
||||
func renderTimeSyncCard() string {
|
||||
return `<div class="card" style="margin-bottom:16px">
|
||||
<div class="card-head card-head-actions">
|
||||
<span>System Time</span>
|
||||
<div class="card-head-buttons">
|
||||
<button id="time-sync-btn" class="btn btn-primary btn-sm" onclick="timeSyncRun()">↻ Sync with this browser</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<span id="time-sync-status" style="font-size:13px;color:var(--muted)"></span>
|
||||
</div>
|
||||
</div>
|
||||
<script>
|
||||
function timeSyncRun() {
|
||||
var btn = document.getElementById('time-sync-btn');
|
||||
var status = document.getElementById('time-sync-status');
|
||||
btn.disabled = true;
|
||||
var tz = Intl.DateTimeFormat().resolvedOptions().timeZone || '';
|
||||
fetch('/api/system/time-sync', {
|
||||
method: 'POST',
|
||||
headers: {'Content-Type': 'application/json'},
|
||||
body: JSON.stringify({timezone: tz, epoch_ms: Date.now()})
|
||||
})
|
||||
.then(function(r) { if (!r.ok) return r.text().then(function(t){throw new Error(t || r.statusText);}); return r.json(); })
|
||||
.then(function(d) {
|
||||
status.style.color = 'var(--ok-fg,#2c662d)';
|
||||
status.textContent = '✓ Synced to ' + tz + ' at ' + new Date().toLocaleString();
|
||||
})
|
||||
.catch(function(err) {
|
||||
status.style.color = 'var(--crit-fg,#9f3a38)';
|
||||
status.textContent = '✗ Sync failed: ' + err.message;
|
||||
})
|
||||
.finally(function() { btn.disabled = false; });
|
||||
}
|
||||
</script>`
|
||||
}
|
||||
|
||||
// renderAuditStatusBanner shows a live progress banner when an audit task is
|
||||
// running and auto-reloads the page when it completes.
|
||||
func renderAuditStatusBanner(opts HandlerOptions) string {
|
||||
@@ -242,10 +282,16 @@ func renderHardwareSummaryCard(opts HandlerOptions) string {
|
||||
if db, err := app.OpenComponentStatusDB(filepath.Join(opts.ExportDir, "component-status.json")); err == nil {
|
||||
records = db.All()
|
||||
}
|
||||
gpuRecords := matchedRecords(records, nil, []string{"pcie:gpu:"})
|
||||
|
||||
var b strings.Builder
|
||||
b.WriteString(`<div class="card"` + cardID + `><div class="card-head">Hardware Summary</div><div class="card-body">`)
|
||||
|
||||
if reason, needsReboot := gpuNeedsPhysicalReboot(gpuRecords); needsReboot {
|
||||
fmt.Fprintf(&b, `<div class="alert alert-crit">⚠ GPU needs a physical reboot/power-cycle — further GPU tests will keep failing until then: %s</div>`,
|
||||
html.EscapeString(reason))
|
||||
}
|
||||
|
||||
// Server identity block above the component table.
|
||||
{
|
||||
var model, serial string
|
||||
@@ -297,8 +343,7 @@ func renderHardwareSummaryCard(opts HandlerOptions) string {
|
||||
writeRow("Storage", hwDescribeStorage(hw),
|
||||
renderComponentChips(matchedRecords(records, []string{"storage:all"}, []string{"storage:"})), "storage")
|
||||
|
||||
writeRow("GPU", hwDescribeGPU(hw),
|
||||
renderComponentChips(matchedRecords(records, nil, []string{"pcie:gpu:"})), "gpu")
|
||||
writeRow("GPU", hwDescribeGPU(hw), renderComponentChips(gpuRecords), "gpu")
|
||||
|
||||
psuMatched := matchedRecords(records, nil, []string{"psu:"})
|
||||
if len(psuMatched) == 0 && len(hw.PowerSupplies) > 0 {
|
||||
@@ -986,6 +1031,25 @@ func runtimeIssueDescriptions(issues []schema.RuntimeIssue, codes ...string) str
|
||||
return strings.Join(messages, "; ")
|
||||
}
|
||||
|
||||
// gpuNeedsPhysicalReboot reports whether any GPU component record carries a
|
||||
// hardware-fault reason that a driver reset can't clear (e.g. Xid 79 "GPU
|
||||
// has fallen off the bus", Xid 154 "Node Reboot Required" — see
|
||||
// collector.xidHardwareFaultMessages). Those errors mean every subsequent
|
||||
// GPU SAT job will keep failing until the node is physically power-cycled,
|
||||
// so this drives a dashboard banner that says so up front instead of making
|
||||
// an operator burn another test cycle to rediscover it.
|
||||
func gpuNeedsPhysicalReboot(records []app.ComponentStatusRecord) (reason string, needsReboot bool) {
|
||||
for _, rec := range records {
|
||||
if !strings.EqualFold(strings.TrimSpace(rec.Status), "Critical") {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(strings.ToLower(rec.ErrorSummary), "reboot") {
|
||||
return rec.ErrorSummary, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// chipLetterClass maps a component status to a single display letter and CSS class.
|
||||
func chipLetterClass(status string) (letter, cls string) {
|
||||
switch strings.ToUpper(strings.TrimSpace(status)) {
|
||||
|
||||
Reference in New Issue
Block a user