reanimator/bee
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

app/webui: replace generic "see summary.txt" SAT failure text with the real reason

Browse Source 
Every place that surfaced a FAILED SAT result — task error messages,
component-status.json detail, and the hardware snapshot's ErrorDescription/
StatusHistory — used to say only "SAT overall_status=FAILED (see
summary.txt)" or "<label> failed", forcing an engineer to go dig through the
run directory to find out what actually broke.

nvidia-config's summary.txt now carries a "warnings" field with the specific
GPU/NVLink finding. A new SATFailureDetail/satFailureDetailFromKV in
component_status_db.go reads that field, or falls back to naming whichever
generic SAT sub-job(s) reported non-OK/UNSUPPORTED status along with their
exit code. This feeds both the task-runner error message and the component
status DB. sat_overlay.go's satKeyStatus (which drives ErrorDescription on
the exported hardware snapshot) now does the same, with storage kept
per-device so one drive's rc doesn't get attributed to another's card.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-07-09 15:48:12 +03:00
co-authored by Claude Sonnet 5
parent 5aee146903
commit b30d34199a
8 changed files with 265 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
audit/internal/app/sat_overlay.go
+27 -5
View File
@@ -1,6 +1,7 @@
package app
import (
"fmt"
"os"
"path/filepath"
"sort"
@@ -61,7 +62,7 @@ func applyNvidiaPerGPUStatus(devs []schema.HardwarePCIeDevice, baseDir string) {
if !ok {
continue
}
status, description, ok := satKeyStatus(st.runStatus, firstNonEmpty(strings.TrimSpace(st.reason), "nvidia GPU SAT"))
status, description, ok := satKeyStatus(st.runStatus, firstNonEmpty(strings.TrimSpace(st.reason), "nvidia GPU SAT"), nil)
if !ok {
continue
}
@@ -221,10 +222,20 @@ func parseStorageSATStatus(summary satSummary) map[string]satStatusResult {
}
devName := base[:idx]
step := strings.ReplaceAll(base[idx+1:], "_", "-")
stepStatus, desc, ok := satKeyStatus(strings.ToUpper(strings.TrimSpace(value)), "storage "+step)
label := "storage " + step
// Storage steps are per-device, per-job — satFailureDetailFromKV(summary.kv)
// would combine every device's failures into one description here.
// Build the reason from this one job's own rc instead so a failing
// nvme1n1 self-test doesn't get another drive's error attributed to it.
stepStatus, desc, ok := satKeyStatus(strings.ToUpper(strings.TrimSpace(value)), label, nil)
if !ok {
continue
}
if stepStatus == "Critical" {
if rc, hasRC := summary.kv[base+"_rc"]; hasRC && strings.TrimSpace(rc) != "" {
desc = fmt.Sprintf("%s failed (rc=%s)", label, rc)
}
}
current := result[devName]
if !current.ok || statusSeverity(stepStatus) > statusSeverity(current.status) {
result[devName] = satStatusResult{status: stepStatus, description: desc, ok: true}
@@ -234,10 +245,17 @@ func parseStorageSATStatus(summary satSummary) map[string]satStatusResult {
}
func satSummaryStatus(summary satSummary, label string) (string, string, bool) {
return satKeyStatus(summary.overall, label)
return satKeyStatus(summary.overall, label, summary.kv)
}
func satKeyStatus(rawStatus, label string) (string, string, bool) {
// satKeyStatus maps a raw SAT overall_status to a hardware component status
// and a human-readable description. For FAILED, "<label> failed" alone
// tells an engineer nothing they didn't already know from the Critical
// badge — kv (the parsed summary.txt) is consulted via
// satFailureDetailFromKV for the actual reason (which GPU/NVLink pair, or
// which sub-job and its exit code) so ErrorDescription/StatusHistory carry
// something actionable instead of a dead end.
func satKeyStatus(rawStatus, label string, kv map[string]string) (string, string, bool) {
switch strings.ToUpper(strings.TrimSpace(rawStatus)) {
case "OK":
// No error description on success — error_description is for problems only.
@@ -246,7 +264,11 @@ func satKeyStatus(rawStatus, label string) (string, string, bool) {
// Tool couldn't run or test was incomplete — we can't assert hardware health.
return "Unknown", "", true
case "FAILED":
return "Critical", label + " failed", true
desc := label + " failed"
if reason := satFailureDetailFromKV(kv); reason != "" {
desc = label + " failed: " + reason
}
return "Critical", desc, true
default:
return "", "", false
}