app/webui: replace generic "see summary.txt" SAT failure text with the real reason

Every place that surfaced a FAILED SAT result — task error messages,
component-status.json detail, and the hardware snapshot's ErrorDescription/
StatusHistory — used to say only "SAT overall_status=FAILED (see
summary.txt)" or "<label> failed", forcing an engineer to go dig through the
run directory to find out what actually broke.

nvidia-config's summary.txt now carries a "warnings" field with the specific
GPU/NVLink finding. A new SATFailureDetail/satFailureDetailFromKV in
component_status_db.go reads that field, or falls back to naming whichever
generic SAT sub-job(s) reported non-OK/UNSUPPORTED status along with their
exit code. This feeds both the task-runner error message and the component
status DB. sat_overlay.go's satKeyStatus (which drives ErrorDescription on
the exported hardware snapshot) now does the same, with storage kept
per-device so one drive's rc doesn't get attributed to another's card.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-07-09 15:48:12 +03:00
co-authored by Claude Sonnet 5
parent 5aee146903
commit b30d34199a
8 changed files with 265 additions and 13 deletions
+61
View File
@@ -3,6 +3,7 @@ package app
import (
"os"
"path/filepath"
"strings"
"testing"
"bee/audit/internal/collector"
@@ -63,6 +64,66 @@ func TestApplyLatestSATStatusesMarksAMDGPUs(t *testing.T) {
}
}
// TestApplyLatestSATStatusesGPUFailureDescriptionNamesFailingSubJob guards
// against ErrorDescription regressing to a bare "amd GPU SAT failed" — an
// engineer reading the hardware snapshot (e.g. in a support bundle) needs to
// know which sub-job failed and its exit code, not just that something did.
func TestApplyLatestSATStatusesGPUFailureDescriptionNamesFailingSubJob(t *testing.T) {
baseDir := t.TempDir()
runDir := filepath.Join(baseDir, "gpu-amd-20260325-161436")
if err := os.MkdirAll(runDir, 0755); err != nil {
t.Fatal(err)
}
raw := "run_at_utc=2026-03-25T16:14:36Z\n" +
"rocm-bandwidth-test_rc=1\n" +
"rocm-bandwidth-test_status=FAILED\n" +
"overall_status=FAILED\n"
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(raw), 0644); err != nil {
t.Fatal(err)
}
class := "DisplayController"
amdVendorID := collector.AMDVendorID
snap := schema.HardwareSnapshot{
PCIeDevices: []schema.HardwarePCIeDevice{{DeviceClass: &class, VendorID: &amdVendorID}},
}
applyLatestSATStatuses(&snap, baseDir, nil)
desc := snap.PCIeDevices[0].ErrorDescription
if desc == nil || !strings.Contains(*desc, "rocm-bandwidth-test=FAILED (rc=1)") {
t.Fatalf("ErrorDescription=%v, want it to name the failing sub-job and rc", desc)
}
}
// TestApplyLatestSATStatusesStorageFailureDescriptionIncludesRC guards the
// per-device storage path, which must attribute the rc of *that device's*
// own failing job rather than any other device's.
func TestApplyLatestSATStatusesStorageFailureDescriptionIncludesRC(t *testing.T) {
baseDir := t.TempDir()
runDir := filepath.Join(baseDir, "storage-20260325-161151")
if err := os.MkdirAll(runDir, 0755); err != nil {
t.Fatal(err)
}
raw := "run_at_utc=2026-03-25T16:11:51Z\n" +
"sda_smartctl_health_rc=2\n" +
"sda_smartctl_health_status=FAILED\n" +
"overall_status=FAILED\n"
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(raw), 0644); err != nil {
t.Fatal(err)
}
sda := schema.HardwareStorage{Telemetry: map[string]any{"linux_device": "/dev/sda"}}
snap := schema.HardwareSnapshot{Storage: []schema.HardwareStorage{sda}}
applyLatestSATStatuses(&snap, baseDir, nil)
desc := snap.Storage[0].ErrorDescription
if desc == nil || !strings.Contains(*desc, "rc=2") {
t.Fatalf("ErrorDescription=%v, want it to include the failing job's rc", desc)
}
}
func TestApplyLatestSATStatusesMarksNvidiaGPUByPerGPUStatusFile(t *testing.T) {
baseDir := t.TempDir()
runDir := filepath.Join(baseDir, "gpu-nvidia-20260407-162123")