app/webui: replace generic "see summary.txt" SAT failure text with the real reason
Every place that surfaced a FAILED SAT result — task error messages, component-status.json detail, and the hardware snapshot's ErrorDescription/ StatusHistory — used to say only "SAT overall_status=FAILED (see summary.txt)" or "<label> failed", forcing an engineer to go dig through the run directory to find out what actually broke. nvidia-config's summary.txt now carries a "warnings" field with the specific GPU/NVLink finding. A new SATFailureDetail/satFailureDetailFromKV in component_status_db.go reads that field, or falls back to naming whichever generic SAT sub-job(s) reported non-OK/UNSUPPORTED status along with their exit code. This feeds both the task-runner error message and the component status DB. sat_overlay.go's satKeyStatus (which drives ErrorDescription on the exported hardware snapshot) now does the same, with storage kept per-device so one drive's rc doesn't get attributed to another's card. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
5aee146903
commit
b30d34199a
@@ -470,6 +470,11 @@ func renderNvidiaConfigCheckSummary(status NvidiaConfigCheckStatus) string {
|
||||
fmt.Fprintln(&b, "overall_status=OK")
|
||||
} else {
|
||||
fmt.Fprintln(&b, "overall_status=FAILED")
|
||||
// warnings carries the actual reason(s) in prose so a reader of
|
||||
// summary.txt (or anything parsing it, like sat.DescribeFailure)
|
||||
// never has to fall back to "see the report" — each entry already
|
||||
// names the specific GPU or NVLink pair and what's wrong with it.
|
||||
fmt.Fprintf(&b, "warnings=%s\n", strings.Join(status.Warnings, "; "))
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
@@ -166,3 +166,25 @@ func TestRenderNvidiaConfigCheckSummaryOverallStatus(t *testing.T) {
|
||||
t.Fatalf("status with warnings missing overall_status=FAILED:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRenderNvidiaConfigCheckSummaryIncludesWarningsField guards that a
|
||||
// failed run's summary.txt carries the actual reason in prose (a "warnings"
|
||||
// field), not just a bare overall_status=FAILED — app.SATFailureDetail reads
|
||||
// this field so a failed task's error message names the specific GPU/NVLink
|
||||
// problem instead of telling the engineer to go read summary.txt themselves.
|
||||
func TestRenderNvidiaConfigCheckSummaryIncludesWarningsField(t *testing.T) {
|
||||
status := NvidiaConfigCheckStatus{Warnings: []string{
|
||||
"GPU 0 (H100): ECC is disabled",
|
||||
"NVLink GPU0<->GPU1: 2/36 NVLinks inactive on a bonded pair",
|
||||
}}
|
||||
got := renderNvidiaConfigCheckSummary(status)
|
||||
want := "warnings=GPU 0 (H100): ECC is disabled; NVLink GPU0<->GPU1: 2/36 NVLinks inactive on a bonded pair\n"
|
||||
if !strings.Contains(got, want) {
|
||||
t.Fatalf("summary missing combined warnings field:\ngot:\n%s\nwant substring:\n%s", got, want)
|
||||
}
|
||||
|
||||
clean := NvidiaConfigCheckStatus{}
|
||||
if got := renderNvidiaConfigCheckSummary(clean); strings.Contains(got, "warnings=") {
|
||||
t.Fatalf("clean status summary should omit warnings field entirely:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user