package app import ( "fmt" "os" "path/filepath" "sort" "strconv" "strings" "bee/audit/internal/collector" "bee/audit/internal/schema" ) func applyLatestSATStatuses(snap *schema.HardwareSnapshot, baseDir string, db *ComponentStatusDB) { if snap == nil || strings.TrimSpace(baseDir) == "" { return } if summary, ok := loadLatestSATSummary(baseDir, "gpu-amd-"); ok { applyGPUVendorSAT(snap.PCIeDevices, "amd", summary) } if summary, ok := loadLatestSATSummary(baseDir, "gpu-nvidia-"); ok { applyGPUVendorSAT(snap.PCIeDevices, "nvidia", summary) applyNvidiaPerGPUStatus(snap.PCIeDevices, baseDir) } if summary, ok := loadLatestSATSummary(baseDir, "memory-"); ok { applyMemorySAT(snap.Memory, summary) } if summary, ok := loadLatestSATSummary(baseDir, "cpu-"); ok { applyCPUSAT(snap.CPUs, summary) } if summary, ok := loadLatestSATSummary(baseDir, "storage-"); ok { applyStorageSAT(snap.Storage, summary) } // Apply unified component status DB — overlaid last so it can only upgrade severity. applyComponentStatusDB(snap, db) } type nvidiaPerGPUStatus struct { runStatus string reason string } func applyNvidiaPerGPUStatus(devs []schema.HardwarePCIeDevice, baseDir string) { statusByIndex, ts, ok := loadLatestNvidiaPerGPUStatus(baseDir) if !ok { return } for i := range devs { if devs[i].Telemetry == nil { continue } rawIdx, ok := devs[i].Telemetry["nvidia_gpu_index"] if !ok { continue } idx, ok := telemetryInt(rawIdx) if !ok { continue } st, ok := statusByIndex[idx] if !ok { continue } status, description, ok := satKeyStatus(st.runStatus, firstNonEmpty(strings.TrimSpace(st.reason), "nvidia GPU SAT"), nil) if !ok { continue } mergeComponentStatusPreferDetail(&devs[i].HardwareComponentStatus, ts, status, description) } } func loadLatestNvidiaPerGPUStatus(baseDir string) (map[int]nvidiaPerGPUStatus, string, bool) { matches, err := filepath.Glob(filepath.Join(baseDir, "gpu-nvidia-*")) if err != nil || len(matches) == 0 { return nil, "", false } sort.Strings(matches) runDir := matches[len(matches)-1] summaryRaw, err := os.ReadFile(filepath.Join(runDir, "summary.txt")) if err != nil { return nil, "", false } summaryKV := parseKeyValueSummary(string(summaryRaw)) runAtUTC := strings.TrimSpace(summaryKV["run_at_utc"]) files, err := filepath.Glob(filepath.Join(runDir, "gpu-*-status.txt")) if err != nil || len(files) == 0 { return nil, "", false } out := make(map[int]nvidiaPerGPUStatus, len(files)) for _, file := range files { raw, err := os.ReadFile(file) if err != nil { continue } kv := parseKeyValueSummary(string(raw)) idx, err := strconv.Atoi(strings.TrimSpace(kv["gpu_index"])) if err != nil { continue } out[idx] = nvidiaPerGPUStatus{ runStatus: strings.ToUpper(strings.TrimSpace(kv["run_status"])), reason: strings.TrimSpace(kv["reason"]), } } if len(out) == 0 { return nil, "", false } return out, runAtUTC, true } func telemetryInt(v any) (int, bool) { switch value := v.(type) { case int: return value, true case int32: return int(value), true case int64: return int(value), true case float64: return int(value), true case string: n, err := strconv.Atoi(strings.TrimSpace(value)) if err != nil { return 0, false } return n, true default: return 0, false } } type satSummary struct { runAtUTC string overall string kv map[string]string } func loadLatestSATSummary(baseDir, prefix string) (satSummary, bool) { matches, err := filepath.Glob(filepath.Join(baseDir, prefix+"*/summary.txt")) if err != nil || len(matches) == 0 { return satSummary{}, false } sort.Strings(matches) raw, err := os.ReadFile(matches[len(matches)-1]) if err != nil { return satSummary{}, false } kv := parseKeyValueSummary(string(raw)) return satSummary{ runAtUTC: strings.TrimSpace(kv["run_at_utc"]), overall: strings.ToUpper(strings.TrimSpace(kv["overall_status"])), kv: kv, }, true } func applyGPUVendorSAT(devs []schema.HardwarePCIeDevice, vendor string, summary satSummary) { status, description, ok := satSummaryStatus(summary, vendor+" GPU SAT") if !ok { return } for i := range devs { if !matchesGPUVendor(devs[i], vendor) { continue } mergeComponentStatus(&devs[i].HardwareComponentStatus, summary.runAtUTC, status, description) } } func applyMemorySAT(dimms []schema.HardwareMemory, summary satSummary) { status, description, ok := satSummaryStatus(summary, "memory SAT") if !ok { return } for i := range dimms { mergeComponentStatus(&dimms[i].HardwareComponentStatus, summary.runAtUTC, status, description) } } func applyCPUSAT(cpus []schema.HardwareCPU, summary satSummary) { status, description, ok := satSummaryStatus(summary, "CPU SAT") if !ok { return } for i := range cpus { mergeComponentStatus(&cpus[i].HardwareComponentStatus, summary.runAtUTC, status, description) } } func applyStorageSAT(disks []schema.HardwareStorage, summary satSummary) { byDevice := parseStorageSATStatus(summary) for i := range disks { devPath, _ := disks[i].Telemetry["linux_device"].(string) devName := filepath.Base(strings.TrimSpace(devPath)) if devName == "" { continue } result, ok := byDevice[devName] if !ok { continue } mergeComponentStatus(&disks[i].HardwareComponentStatus, summary.runAtUTC, result.status, result.description) } } type satStatusResult struct { status string description string ok bool } func parseStorageSATStatus(summary satSummary) map[string]satStatusResult { result := map[string]satStatusResult{} for key, value := range summary.kv { if !strings.HasSuffix(key, "_status") || key == "overall_status" { continue } base := strings.TrimSuffix(key, "_status") idx := strings.Index(base, "_") if idx <= 0 { continue } devName := base[:idx] step := strings.ReplaceAll(base[idx+1:], "_", "-") label := "storage " + step // Storage steps are per-device, per-job — satFailureDetailFromKV(summary.kv) // would combine every device's failures into one description here. // Build the reason from this one job's own rc instead so a failing // nvme1n1 self-test doesn't get another drive's error attributed to it. stepStatus, desc, ok := satKeyStatus(strings.ToUpper(strings.TrimSpace(value)), label, nil) if !ok { continue } if stepStatus == "Critical" { if rc, hasRC := summary.kv[base+"_rc"]; hasRC && strings.TrimSpace(rc) != "" { desc = fmt.Sprintf("%s failed (rc=%s)", label, rc) } } current := result[devName] if !current.ok || statusSeverity(stepStatus) > statusSeverity(current.status) { result[devName] = satStatusResult{status: stepStatus, description: desc, ok: true} } } return result } func satSummaryStatus(summary satSummary, label string) (string, string, bool) { return satKeyStatus(summary.overall, label, summary.kv) } // satKeyStatus maps a raw SAT overall_status to a hardware component status // and a human-readable description. For FAILED, "