app/webui: replace generic "see summary.txt" SAT failure text with the real reason

Every place that surfaced a FAILED SAT result — task error messages,
component-status.json detail, and the hardware snapshot's ErrorDescription/
StatusHistory — used to say only "SAT overall_status=FAILED (see
summary.txt)" or "<label> failed", forcing an engineer to go dig through the
run directory to find out what actually broke.

nvidia-config's summary.txt now carries a "warnings" field with the specific
GPU/NVLink finding. A new SATFailureDetail/satFailureDetailFromKV in
component_status_db.go reads that field, or falls back to naming whichever
generic SAT sub-job(s) reported non-OK/UNSUPPORTED status along with their
exit code. This feeds both the task-runner error message and the component
status DB. sat_overlay.go's satKeyStatus (which drives ErrorDescription on
the exported hardware snapshot) now does the same, with storage kept
per-device so one drive's rc doesn't get attributed to another's card.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-07-09 15:48:12 +03:00
co-authored by Claude Sonnet 5
parent 5aee146903
commit b30d34199a
8 changed files with 265 additions and 13 deletions
+82 -6
View File
@@ -2,8 +2,10 @@ package app
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"time"
@@ -231,6 +233,12 @@ func ApplySATResultToDB(db *ComponentStatusDB, target, archivePath string) {
source := "sat:" + target
dbStatus := satStatusToDBStatus(overall)
detail := target + " SAT: " + overall
if overall != "OK" {
if reason := satFailureDetailFromKV(kv); reason != "" {
detail += " — " + reason
}
}
// Map SAT target to component keys. GPU targets are keyed by vendor, not
// by the raw target string: "nvidia" (Check tier) and "nvidia-stress" /
@@ -244,13 +252,13 @@ func ApplySATResultToDB(db *ComponentStatusDB, target, archivePath string) {
switch target {
case "nvidia", "nvidia-targeted-stress", "nvidia-compute", "nvidia-targeted-power", "nvidia-pulse",
"nvidia-interconnect", "nvidia-bandwidth", "nvidia-stress", "nvidia-config":
db.Record("pcie:gpu:nvidia", source, dbStatus, target+" SAT: "+overall)
db.Record("pcie:gpu:nvidia", source, dbStatus, detail)
case "amd", "amd-stress", "amd-mem", "amd-bandwidth":
db.Record("pcie:gpu:amd", source, dbStatus, target+" SAT: "+overall)
db.Record("pcie:gpu:amd", source, dbStatus, detail)
case "memory", "memory-stress", "sat-stress":
db.Record("memory:all", source, dbStatus, target+" SAT: "+overall)
db.Record("memory:all", source, dbStatus, detail)
case "cpu", "platform-stress":
db.Record("cpu:all", source, dbStatus, target+" SAT: "+overall)
db.Record("cpu:all", source, dbStatus, detail)
case "storage":
// Try to record per-device if available in summary.
recordedAny := false
@@ -265,11 +273,19 @@ func ApplySATResultToDB(db *ComponentStatusDB, target, archivePath string) {
}
devName := base[:idx]
devStatus := satStatusToDBStatus(strings.ToUpper(strings.TrimSpace(val)))
db.Record("storage:"+devName, source, devStatus, "storage SAT: "+val)
devDetail := "storage SAT: " + val
if strings.ToUpper(strings.TrimSpace(val)) != "OK" {
if rc, ok := kv[base+"_rc"]; ok {
devDetail = fmt.Sprintf("storage SAT job %q: %s (rc=%s)", base, val, rc)
} else {
devDetail = fmt.Sprintf("storage SAT job %q: %s", base, val)
}
}
db.Record("storage:"+devName, source, devStatus, devDetail)
recordedAny = true
}
if !recordedAny {
db.Record("storage:all", source, dbStatus, "storage SAT: "+overall)
db.Record("storage:all", source, dbStatus, detail)
}
}
}
@@ -309,6 +325,66 @@ func ReadSATOverallStatus(archivePath string) string {
return strings.ToUpper(strings.TrimSpace(kv["overall_status"]))
}
// SATFailureDetail explains *why* a SAT run's overall_status isn't OK, read
// from the summary.txt next to archivePath. "SAT overall_status=FAILED (see
// summary.txt)" tells an engineer nothing without opening the run directory
// themselves; this pulls the specific reason out so it can be surfaced
// directly in the task's error message and in the component status DB.
// Returns "" if summary.txt is unreadable or carries no identifiable reason.
func SATFailureDetail(archivePath string) string {
if strings.TrimSpace(archivePath) == "" {
return ""
}
runDir := strings.TrimSuffix(extractArchivePath(archivePath), ".tar.gz")
data, err := os.ReadFile(filepath.Join(runDir, "summary.txt"))
if err != nil {
return ""
}
return satFailureDetailFromKV(parseSATKV(string(data)))
}
// satFailureDetailFromKV inspects an already-parsed summary.txt for the
// reason behind a non-OK overall_status.
//
// - Checks that write structured findings (nvidia-config's GPU config /
// NVLink topology check) put the human-readable reason straight into a
// "warnings" field — return that verbatim.
// - Generic SAT acceptance packs (nvidia, amd, memory, cpu, storage, ...)
// instead record one "<job>_status"/"<job>_rc" pair per sub-job; walk
// those and report whichever job(s) didn't come back OK/UNSUPPORTED.
func satFailureDetailFromKV(kv map[string]string) string {
if w := strings.TrimSpace(kv["warnings"]); w != "" {
return w
}
keys := make([]string, 0, len(kv))
for k := range kv {
keys = append(keys, k)
}
sort.Strings(keys)
var failed []string
for _, k := range keys {
if k == "overall_status" || !strings.HasSuffix(k, "_status") {
continue
}
v := strings.ToUpper(strings.TrimSpace(kv[k]))
if v == "" || v == "OK" || v == "UNSUPPORTED" {
continue
}
job := strings.TrimSuffix(k, "_status")
if rc, ok := kv[job+"_rc"]; ok && strings.TrimSpace(rc) != "" {
failed = append(failed, fmt.Sprintf("%s=%s (rc=%s)", job, v, rc))
} else {
failed = append(failed, fmt.Sprintf("%s=%s", job, v))
}
}
if len(failed) == 0 {
return ""
}
return "failed sub-job(s): " + strings.Join(failed, ", ")
}
func extractArchivePath(s string) string {
s = strings.TrimSpace(s)
if rest, ok := strings.CutPrefix(s, "Archive written to "); ok {