app/webui: replace generic "see summary.txt" SAT failure text with the real reason
Every place that surfaced a FAILED SAT result — task error messages, component-status.json detail, and the hardware snapshot's ErrorDescription/ StatusHistory — used to say only "SAT overall_status=FAILED (see summary.txt)" or "<label> failed", forcing an engineer to go dig through the run directory to find out what actually broke. nvidia-config's summary.txt now carries a "warnings" field with the specific GPU/NVLink finding. A new SATFailureDetail/satFailureDetailFromKV in component_status_db.go reads that field, or falls back to naming whichever generic SAT sub-job(s) reported non-OK/UNSUPPORTED status along with their exit code. This feeds both the task-runner error message and the component status DB. sat_overlay.go's satKeyStatus (which drives ErrorDescription on the exported hardware snapshot) now does the same, with storage kept per-device so one drive's rc doesn't get attributed to another's card. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
5aee146903
commit
b30d34199a
@@ -3,6 +3,7 @@ package app
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"bee/audit/internal/collector"
|
||||
@@ -63,6 +64,66 @@ func TestApplyLatestSATStatusesMarksAMDGPUs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestApplyLatestSATStatusesGPUFailureDescriptionNamesFailingSubJob guards
|
||||
// against ErrorDescription regressing to a bare "amd GPU SAT failed" — an
|
||||
// engineer reading the hardware snapshot (e.g. in a support bundle) needs to
|
||||
// know which sub-job failed and its exit code, not just that something did.
|
||||
func TestApplyLatestSATStatusesGPUFailureDescriptionNamesFailingSubJob(t *testing.T) {
|
||||
baseDir := t.TempDir()
|
||||
runDir := filepath.Join(baseDir, "gpu-amd-20260325-161436")
|
||||
if err := os.MkdirAll(runDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw := "run_at_utc=2026-03-25T16:14:36Z\n" +
|
||||
"rocm-bandwidth-test_rc=1\n" +
|
||||
"rocm-bandwidth-test_status=FAILED\n" +
|
||||
"overall_status=FAILED\n"
|
||||
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(raw), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
class := "DisplayController"
|
||||
amdVendorID := collector.AMDVendorID
|
||||
snap := schema.HardwareSnapshot{
|
||||
PCIeDevices: []schema.HardwarePCIeDevice{{DeviceClass: &class, VendorID: &amdVendorID}},
|
||||
}
|
||||
|
||||
applyLatestSATStatuses(&snap, baseDir, nil)
|
||||
|
||||
desc := snap.PCIeDevices[0].ErrorDescription
|
||||
if desc == nil || !strings.Contains(*desc, "rocm-bandwidth-test=FAILED (rc=1)") {
|
||||
t.Fatalf("ErrorDescription=%v, want it to name the failing sub-job and rc", desc)
|
||||
}
|
||||
}
|
||||
|
||||
// TestApplyLatestSATStatusesStorageFailureDescriptionIncludesRC guards the
|
||||
// per-device storage path, which must attribute the rc of *that device's*
|
||||
// own failing job rather than any other device's.
|
||||
func TestApplyLatestSATStatusesStorageFailureDescriptionIncludesRC(t *testing.T) {
|
||||
baseDir := t.TempDir()
|
||||
runDir := filepath.Join(baseDir, "storage-20260325-161151")
|
||||
if err := os.MkdirAll(runDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw := "run_at_utc=2026-03-25T16:11:51Z\n" +
|
||||
"sda_smartctl_health_rc=2\n" +
|
||||
"sda_smartctl_health_status=FAILED\n" +
|
||||
"overall_status=FAILED\n"
|
||||
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(raw), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
sda := schema.HardwareStorage{Telemetry: map[string]any{"linux_device": "/dev/sda"}}
|
||||
snap := schema.HardwareSnapshot{Storage: []schema.HardwareStorage{sda}}
|
||||
|
||||
applyLatestSATStatuses(&snap, baseDir, nil)
|
||||
|
||||
desc := snap.Storage[0].ErrorDescription
|
||||
if desc == nil || !strings.Contains(*desc, "rc=2") {
|
||||
t.Fatalf("ErrorDescription=%v, want it to include the failing job's rc", desc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyLatestSATStatusesMarksNvidiaGPUByPerGPUStatusFile(t *testing.T) {
|
||||
baseDir := t.TempDir()
|
||||
runDir := filepath.Join(baseDir, "gpu-nvidia-20260407-162123")
|
||||
|
||||
Reference in New Issue
Block a user