app/webui: replace generic "see summary.txt" SAT failure text with the real reason
Every place that surfaced a FAILED SAT result — task error messages, component-status.json detail, and the hardware snapshot's ErrorDescription/ StatusHistory — used to say only "SAT overall_status=FAILED (see summary.txt)" or "<label> failed", forcing an engineer to go dig through the run directory to find out what actually broke. nvidia-config's summary.txt now carries a "warnings" field with the specific GPU/NVLink finding. A new SATFailureDetail/satFailureDetailFromKV in component_status_db.go reads that field, or falls back to naming whichever generic SAT sub-job(s) reported non-OK/UNSUPPORTED status along with their exit code. This feeds both the task-runner error message and the component status DB. sat_overlay.go's satKeyStatus (which drives ErrorDescription on the exported hardware snapshot) now does the same, with storage kept per-device so one drive's rc doesn't get attributed to another's card. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
5aee146903
commit
b30d34199a
@@ -2,8 +2,10 @@ package app
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -231,6 +233,12 @@ func ApplySATResultToDB(db *ComponentStatusDB, target, archivePath string) {
|
||||
|
||||
source := "sat:" + target
|
||||
dbStatus := satStatusToDBStatus(overall)
|
||||
detail := target + " SAT: " + overall
|
||||
if overall != "OK" {
|
||||
if reason := satFailureDetailFromKV(kv); reason != "" {
|
||||
detail += " — " + reason
|
||||
}
|
||||
}
|
||||
|
||||
// Map SAT target to component keys. GPU targets are keyed by vendor, not
|
||||
// by the raw target string: "nvidia" (Check tier) and "nvidia-stress" /
|
||||
@@ -244,13 +252,13 @@ func ApplySATResultToDB(db *ComponentStatusDB, target, archivePath string) {
|
||||
switch target {
|
||||
case "nvidia", "nvidia-targeted-stress", "nvidia-compute", "nvidia-targeted-power", "nvidia-pulse",
|
||||
"nvidia-interconnect", "nvidia-bandwidth", "nvidia-stress", "nvidia-config":
|
||||
db.Record("pcie:gpu:nvidia", source, dbStatus, target+" SAT: "+overall)
|
||||
db.Record("pcie:gpu:nvidia", source, dbStatus, detail)
|
||||
case "amd", "amd-stress", "amd-mem", "amd-bandwidth":
|
||||
db.Record("pcie:gpu:amd", source, dbStatus, target+" SAT: "+overall)
|
||||
db.Record("pcie:gpu:amd", source, dbStatus, detail)
|
||||
case "memory", "memory-stress", "sat-stress":
|
||||
db.Record("memory:all", source, dbStatus, target+" SAT: "+overall)
|
||||
db.Record("memory:all", source, dbStatus, detail)
|
||||
case "cpu", "platform-stress":
|
||||
db.Record("cpu:all", source, dbStatus, target+" SAT: "+overall)
|
||||
db.Record("cpu:all", source, dbStatus, detail)
|
||||
case "storage":
|
||||
// Try to record per-device if available in summary.
|
||||
recordedAny := false
|
||||
@@ -265,11 +273,19 @@ func ApplySATResultToDB(db *ComponentStatusDB, target, archivePath string) {
|
||||
}
|
||||
devName := base[:idx]
|
||||
devStatus := satStatusToDBStatus(strings.ToUpper(strings.TrimSpace(val)))
|
||||
db.Record("storage:"+devName, source, devStatus, "storage SAT: "+val)
|
||||
devDetail := "storage SAT: " + val
|
||||
if strings.ToUpper(strings.TrimSpace(val)) != "OK" {
|
||||
if rc, ok := kv[base+"_rc"]; ok {
|
||||
devDetail = fmt.Sprintf("storage SAT job %q: %s (rc=%s)", base, val, rc)
|
||||
} else {
|
||||
devDetail = fmt.Sprintf("storage SAT job %q: %s", base, val)
|
||||
}
|
||||
}
|
||||
db.Record("storage:"+devName, source, devStatus, devDetail)
|
||||
recordedAny = true
|
||||
}
|
||||
if !recordedAny {
|
||||
db.Record("storage:all", source, dbStatus, "storage SAT: "+overall)
|
||||
db.Record("storage:all", source, dbStatus, detail)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -309,6 +325,66 @@ func ReadSATOverallStatus(archivePath string) string {
|
||||
return strings.ToUpper(strings.TrimSpace(kv["overall_status"]))
|
||||
}
|
||||
|
||||
// SATFailureDetail explains *why* a SAT run's overall_status isn't OK, read
|
||||
// from the summary.txt next to archivePath. "SAT overall_status=FAILED (see
|
||||
// summary.txt)" tells an engineer nothing without opening the run directory
|
||||
// themselves; this pulls the specific reason out so it can be surfaced
|
||||
// directly in the task's error message and in the component status DB.
|
||||
// Returns "" if summary.txt is unreadable or carries no identifiable reason.
|
||||
func SATFailureDetail(archivePath string) string {
|
||||
if strings.TrimSpace(archivePath) == "" {
|
||||
return ""
|
||||
}
|
||||
runDir := strings.TrimSuffix(extractArchivePath(archivePath), ".tar.gz")
|
||||
data, err := os.ReadFile(filepath.Join(runDir, "summary.txt"))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return satFailureDetailFromKV(parseSATKV(string(data)))
|
||||
}
|
||||
|
||||
// satFailureDetailFromKV inspects an already-parsed summary.txt for the
|
||||
// reason behind a non-OK overall_status.
|
||||
//
|
||||
// - Checks that write structured findings (nvidia-config's GPU config /
|
||||
// NVLink topology check) put the human-readable reason straight into a
|
||||
// "warnings" field — return that verbatim.
|
||||
// - Generic SAT acceptance packs (nvidia, amd, memory, cpu, storage, ...)
|
||||
// instead record one "<job>_status"/"<job>_rc" pair per sub-job; walk
|
||||
// those and report whichever job(s) didn't come back OK/UNSUPPORTED.
|
||||
func satFailureDetailFromKV(kv map[string]string) string {
|
||||
if w := strings.TrimSpace(kv["warnings"]); w != "" {
|
||||
return w
|
||||
}
|
||||
|
||||
keys := make([]string, 0, len(kv))
|
||||
for k := range kv {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
var failed []string
|
||||
for _, k := range keys {
|
||||
if k == "overall_status" || !strings.HasSuffix(k, "_status") {
|
||||
continue
|
||||
}
|
||||
v := strings.ToUpper(strings.TrimSpace(kv[k]))
|
||||
if v == "" || v == "OK" || v == "UNSUPPORTED" {
|
||||
continue
|
||||
}
|
||||
job := strings.TrimSuffix(k, "_status")
|
||||
if rc, ok := kv[job+"_rc"]; ok && strings.TrimSpace(rc) != "" {
|
||||
failed = append(failed, fmt.Sprintf("%s=%s (rc=%s)", job, v, rc))
|
||||
} else {
|
||||
failed = append(failed, fmt.Sprintf("%s=%s", job, v))
|
||||
}
|
||||
}
|
||||
if len(failed) == 0 {
|
||||
return ""
|
||||
}
|
||||
return "failed sub-job(s): " + strings.Join(failed, ", ")
|
||||
}
|
||||
|
||||
func extractArchivePath(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if rest, ok := strings.CutPrefix(s, "Archive written to "); ok {
|
||||
|
||||
Reference in New Issue
Block a user