package webui import ( "encoding/json" "fmt" "html" "path/filepath" "regexp" "sort" "strconv" "strings" "bee/audit/internal/app" "bee/audit/internal/schema" ) func renderHealthCard(opts HandlerOptions) string { data, err := loadSnapshot(filepath.Join(opts.ExportDir, "runtime-health.json")) if err != nil { return `
Runtime Health
No data
` } var health schema.RuntimeHealth if err := json.Unmarshal(data, &health); err != nil { return `
Runtime Health
Parse error
` } status := strings.TrimSpace(health.Status) if status == "" { status = "UNKNOWN" } badge := "badge-ok" if status == "PARTIAL" { badge = "badge-warn" } else if status == "FAIL" || status == "FAILED" { badge = "badge-err" } var b strings.Builder b.WriteString(`
Runtime Health
`) b.WriteString(fmt.Sprintf(`
%s
`, badge, html.EscapeString(status))) if checkedAt := strings.TrimSpace(health.CheckedAt); checkedAt != "" { b.WriteString(`
Checked at: ` + html.EscapeString(checkedAt) + `
`) } rows := []runtimeHealthRow{ buildRuntimeExportRow(health), buildRuntimeNetworkRow(health), buildRuntimeDriverRow(health), buildRuntimeAccelerationRow(health), buildRuntimeToolsRow(health), buildRuntimeServicesRow(health), buildRuntimeUSBExportRow(health), buildRuntimeToRAMRow(health), } b.WriteString(``) for _, row := range rows { b.WriteString(``) } b.WriteString(`
CheckStatusSourceIssue
` + html.EscapeString(row.Title) + `` + runtimeStatusBadge(row.Status) + `` + html.EscapeString(row.Source) + `` + rowIssueHTML(row.Issue) + `
`) b.WriteString(`
`) return b.String() } type runtimeHealthRow struct { Title string Status string Source string Issue string } func buildRuntimeExportRow(health schema.RuntimeHealth) runtimeHealthRow { issue := runtimeIssueDescriptions(health.Issues, "export_dir_unavailable") status := "UNKNOWN" switch { case issue != "": status = "FAILED" case strings.TrimSpace(health.ExportDir) != "": status = "OK" } source := "os.MkdirAll" if dir := strings.TrimSpace(health.ExportDir); dir != "" { source += " " + dir } return runtimeHealthRow{Title: "Export Directory", Status: status, Source: source, Issue: issue} } func buildRuntimeNetworkRow(health schema.RuntimeHealth) runtimeHealthRow { status := strings.TrimSpace(health.NetworkStatus) if status == "" { status = "UNKNOWN" } issue := runtimeIssueDescriptions(health.Issues, "dhcp_failed") return runtimeHealthRow{Title: "Network", Status: status, Source: "ListInterfaces / DHCP", Issue: issue} } func buildRuntimeDriverRow(health schema.RuntimeHealth) runtimeHealthRow { issue := runtimeIssueDescriptions(health.Issues, "nvidia_kernel_module_missing", "nvidia_modeset_failed", "amdgpu_kernel_module_missing") status := "UNKNOWN" switch { case health.DriverReady && issue == "": status = "OK" case health.DriverReady: status = "PARTIAL" case issue != "": status = "FAILED" } return runtimeHealthRow{Title: "NVIDIA/AMD Driver", Status: status, Source: "lsmod / vendor probe", Issue: issue} } func buildRuntimeAccelerationRow(health schema.RuntimeHealth) runtimeHealthRow { issue := runtimeIssueDescriptions(health.Issues, "cuda_runtime_not_ready", "rocm_smi_unavailable") status := "UNKNOWN" switch { case health.CUDAReady && issue == "": status = "OK" case health.CUDAReady: status = "PARTIAL" case issue != "": status = "FAILED" } return runtimeHealthRow{Title: "CUDA / ROCm", Status: status, Source: "bee-gpu-burn / rocm-smi", Issue: issue} } func buildRuntimeToolsRow(health schema.RuntimeHealth) runtimeHealthRow { if len(health.Tools) == 0 { return runtimeHealthRow{Title: "Required Utilities", Status: "UNKNOWN", Source: "CheckTools", Issue: "No tool status data."} } missing := make([]string, 0) for _, tool := range health.Tools { if !tool.OK { missing = append(missing, tool.Name) } } status := "OK" issue := "" if len(missing) > 0 { status = "PARTIAL" issue = "Missing: " + strings.Join(missing, ", ") } return runtimeHealthRow{Title: "Required Utilities", Status: status, Source: "CheckTools", Issue: issue} } func buildRuntimeServicesRow(health schema.RuntimeHealth) runtimeHealthRow { if len(health.Services) == 0 { return runtimeHealthRow{Title: "Bee Services", Status: "UNKNOWN", Source: "systemctl is-active", Issue: "No service status data."} } nonActive := make([]string, 0) for _, svc := range health.Services { state := strings.TrimSpace(strings.ToLower(svc.Status)) // "inactive" is OK for oneshot services that have completed successfully // (bee-sshsetup, bee-preflight, bee-audit, bee-network, etc.). // Only "failed" is a genuine problem. switch state { case "active", "activating", "deactivating", "reloading", "inactive": // OK — service is running, transitioning normally, or completed successfully default: nonActive = append(nonActive, svc.Name+"="+svc.Status) } } status := "OK" issue := "" if len(nonActive) > 0 { status = "PARTIAL" issue = strings.Join(nonActive, ", ") } return runtimeHealthRow{Title: "Bee Services", Status: status, Source: "ServiceState", Issue: issue} } func buildRuntimeUSBExportRow(health schema.RuntimeHealth) runtimeHealthRow { path := strings.TrimSpace(health.USBExportPath) if path != "" { return runtimeHealthRow{ Title: "USB Export Drive", Status: "OK", Source: "/proc/mounts + lsblk", Issue: path, } } return runtimeHealthRow{ Title: "USB Export Drive", Status: "WARNING", Source: "/proc/mounts + lsblk", Issue: "No writable USB drive mounted. Plug in a USB drive to enable log export.", } } func buildRuntimeToRAMRow(health schema.RuntimeHealth) runtimeHealthRow { switch strings.ToLower(strings.TrimSpace(health.ToRAMStatus)) { case "ok": return runtimeHealthRow{ Title: "LiveCD in RAM", Status: "OK", Source: "live-boot / /proc/mounts", Issue: "", } case "partial": return runtimeHealthRow{ Title: "LiveCD in RAM", Status: "WARNING", Source: "live-boot / /proc/mounts / /dev/shm/bee-live", Issue: "Partial or staged RAM copy detected. System is not fully running from RAM; Copy to RAM can be retried.", } case "failed": return runtimeHealthRow{ Title: "LiveCD in RAM", Status: "FAILED", Source: "live-boot / /proc/mounts", Issue: "toram boot parameter set but ISO is not mounted from RAM. Copy may have failed.", } default: // toram not active — ISO still on original boot media (USB/CD) return runtimeHealthRow{ Title: "LiveCD in RAM", Status: "WARNING", Source: "live-boot / /proc/mounts", Issue: "ISO not copied to RAM. Use \u201cCopy to RAM\u201d to free the boot drive and improve performance.", } } } // matchedRecords returns all ComponentStatusRecord entries whose key matches // any exact key or any of the given prefixes. Used for per-device chip rendering. func firstNonEmpty(vals ...string) string { for _, v := range vals { if v != "" { return v } } return "" } func matchedRecords(records []app.ComponentStatusRecord, exact []string, prefixes []string) []app.ComponentStatusRecord { var matched []app.ComponentStatusRecord for _, rec := range records { key := strings.TrimSpace(rec.ComponentKey) if key == "" { continue } if containsExactKey(key, exact) || hasAnyPrefix(key, prefixes) { matched = append(matched, rec) } } return matched } func containsExactKey(key string, exact []string) bool { for _, candidate := range exact { if key == candidate { return true } } return false } func hasAnyPrefix(key string, prefixes []string) bool { for _, prefix := range prefixes { if strings.HasPrefix(key, prefix) { return true } } return false } func runtimeIssueDescriptions(issues []schema.RuntimeIssue, codes ...string) string { if len(issues) == 0 || len(codes) == 0 { return "" } allowed := make(map[string]struct{}, len(codes)) for _, code := range codes { allowed[code] = struct{}{} } messages := make([]string, 0) for _, issue := range issues { if _, ok := allowed[issue.Code]; !ok { continue } desc := strings.TrimSpace(issue.Description) if desc == "" { desc = issue.Code } messages = append(messages, desc) } return strings.Join(messages, "; ") } // gpuNeedsPhysicalReboot reports whether any GPU component record carries a // hardware-fault reason that a driver reset can't clear (e.g. Xid 79 "GPU // has fallen off the bus", Xid 154 "Node Reboot Required" — see // collector.xidHardwareFaultMessages). Those errors mean every subsequent // GPU SAT job will keep failing until the node is physically power-cycled, // so this drives a dashboard banner that says so up front instead of making // an operator burn another test cycle to rediscover it. func gpuNeedsPhysicalReboot(records []app.ComponentStatusRecord) (reason string, needsReboot bool) { for _, rec := range records { if !strings.EqualFold(strings.TrimSpace(rec.Status), "Critical") { continue } if strings.Contains(strings.ToLower(rec.ErrorSummary), "reboot") { return rec.ErrorSummary, true } } return "", false } // chipLetterClass maps a component status to a single display letter and CSS class. func chipLetterClass(status string) (letter, cls string) { switch strings.ToUpper(strings.TrimSpace(status)) { case "OK": return "O", "chip-ok" case "WARNING", "WARN", "PARTIAL": return "W", "chip-warn" case "CRITICAL", "FAIL", "FAILED", "ERROR": return "F", "chip-fail" default: return "?", "chip-unknown" } } // renderComponentChips renders one 20×20 chip per ComponentStatusRecord. // Hover tooltip shows component key, status, error summary and last check time. // Falls back to a single unknown chip when no records are available. func renderComponentChips(matched []app.ComponentStatusRecord) string { if len(matched) == 0 { return `?` } sort.Slice(matched, func(i, j int) bool { return matched[i].ComponentKey < matched[j].ComponentKey }) var b strings.Builder b.WriteString(``) for _, rec := range matched { letter, cls := chipLetterClass(rec.Status) var tooltip strings.Builder tooltip.WriteString(rec.ComponentKey) tooltip.WriteString(": ") tooltip.WriteString(firstNonEmpty(rec.Status, "UNKNOWN")) if rec.ErrorSummary != "" { tooltip.WriteString(" — ") tooltip.WriteString(rec.ErrorSummary) } if !rec.LastCheckedAt.IsZero() { fmt.Fprintf(&tooltip, " (checked %s)", rec.LastCheckedAt.Format("15:04:05")) } fmt.Fprintf(&b, `%s`, cls, html.EscapeString(tooltip.String()), letter) } b.WriteString(``) return b.String() } func runtimeStatusBadge(status string) string { status = strings.ToUpper(strings.TrimSpace(status)) badge := "badge-unknown" switch status { case "OK": badge = "badge-ok" case "PARTIAL", "WARNING", "WARN": badge = "badge-warn" case "FAIL", "FAILED", "CRITICAL": badge = "badge-err" } return `` + html.EscapeString(status) + `` } func rowIssueHTML(issue string) string { issue = strings.TrimSpace(issue) if issue == "" { return `` } return html.EscapeString(issue) } var aerStatusRe = regexp.MustCompile(`aer_status:\s*0x([0-9a-fA-F]{1,8})`) // decodeAERStatus parses an AER status hex value from a kernel error detail string // and returns a human-readable list of set bit names with correctable/uncorrectable label, // or "" if no AER status is found. func decodeAERStatus(detail string) string { m := aerStatusRe.FindStringSubmatch(detail) if m == nil { return "" } v64, err := strconv.ParseUint(m[1], 16, 32) if err != nil { return "" } val := uint32(v64) type bitDef struct { bit uint32 name string } corrBits := []bitDef{ {0, "Receiver Error"}, {6, "Replay Timer Timeout"}, {7, "Advisory Non-Fatal"}, {8, "Corrected Internal Error"}, {9, "Header Log Overflow"}, {13, "Replay Num Rollover"}, {14, "Bad DLLP"}, {15, "Bad TLP"}, } uncorrBits := []bitDef{ {4, "Data Link Protocol Error"}, {5, "Surprise Down Error"}, {12, "Poisoned TLP Received"}, {13, "Flow Control Protocol Error"}, {14, "Completion Timeout"}, {15, "Completer Abort"}, {16, "Unexpected Completion"}, {17, "Receiver Overflow"}, {18, "Malformed TLP"}, {19, "ECRC Error"}, {20, "Unsupported Request Error"}, {21, "ACS Violation"}, {22, "Uncorrectable Internal Error"}, } var corrNames, uncorrNames []string for _, b := range corrBits { if val&(1<= len(uncorrNames) && len(corrNames) > 0 { return strings.Join(corrNames, ", ") + " (correctable)" } if len(uncorrNames) > 0 { return strings.Join(uncorrNames, ", ") + " (uncorrectable)" } return fmt.Sprintf("unknown bits: 0x%08x", val) } // renderSparkline returns a small inline SVG showing non-OK events over time. // Events are positioned proportionally along the time axis; if all share the same // timestamp they are spaced evenly. Width is always 100px. func renderSparkline(history []app.ComponentStatusEntry) string { const ( svgW = 100 svgH = 20 barW = 3 barH = 14 ) var events []app.ComponentStatusEntry for _, e := range history { if e.Status != "OK" { events = append(events, e) } } if len(events) == 0 { return "" } n := len(events) barColor := func(status string) string { if status == "Critical" { return "#c0392b" } return "#d97706" } yTop := (svgH - barH) / 2 var bars strings.Builder if n == 1 { x := (svgW - barW) / 2 fmt.Fprintf(&bars, ``, x, yTop, barW, barH, barColor(events[0].Status)) } else { minT := events[0].At maxT := events[n-1].At dur := maxT.Sub(minT).Seconds() for i, e := range events { var x int if dur <= 0 { step := svgW / n x = i*step + (step-barW)/2 } else { frac := e.At.Sub(minT).Seconds() / dur x = int(frac * float64(svgW-barW)) } fmt.Fprintf(&bars, ``, x, yTop, barW, barH, barColor(e.Status)) } } return fmt.Sprintf( ``+ `%s`, svgW, svgH, svgW, svgH, bars.String()) } // renderComponentDetail renders a modal content fragment for one component type. // Called by handleAPIComponentDetail and displayed inside #component-detail-dialog. // fromInventory marks that records were synthesized from the audit inventory // snapshot (no ComponentStatusDB history yet) rather than real SAT/watchdog // observations — see inventoryFallbackRecords. func renderComponentDetail(title string, records []app.ComponentStatusRecord, fromInventory bool) string { var b strings.Builder fmt.Fprintf(&b, `
`) fmt.Fprintf(&b, `
`) fmt.Fprintf(&b, `%s — Status Detail`, html.EscapeString(title)) b.WriteString(``) b.WriteString(`
`) if len(records) == 0 { b.WriteString(`

No status data recorded yet for this component type.

`) b.WriteString(`
`) return b.String() } if fromInventory { b.WriteString(`

No SAT-test history yet — showing latest inventory snapshot.

`) } sort.Slice(records, func(i, j int) bool { return records[i].ComponentKey < records[j].ComponentKey }) for _, rec := range records { letter, cls := chipLetterClass(rec.Status) // Count non-OK events across the full history for the badge + sparkline. warnCount := 0 for _, e := range rec.History { if e.Status != "OK" { warnCount++ } } fmt.Fprintf(&b, `
`) fmt.Fprintf(&b, `
`) fmt.Fprintf(&b, `%s`, cls, letter) fmt.Fprintf(&b, `%s`, html.EscapeString(rec.ComponentKey)) if !rec.LastCheckedAt.IsZero() { fmt.Fprintf(&b, `checked %s`, rec.LastCheckedAt.Format("2006-01-02 15:04:05")) } if warnCount > 0 { noun := "events" if warnCount == 1 { noun = "event" } fmt.Fprintf(&b, `%d %s`, warnCount, noun) b.WriteString(renderSparkline(rec.History)) } b.WriteString(`
`) if rec.ErrorSummary != "" { fmt.Fprintf(&b, `
%s
`, html.EscapeString(rec.ErrorSummary)) if decoded := decodeAERStatus(rec.ErrorSummary); decoded != "" { fmt.Fprintf(&b, `
AER: %s
`, html.EscapeString(decoded)) } } // History table — newest first, cap at 20 entries. history := rec.History if len(history) > 20 { history = history[len(history)-20:] } b.WriteString(``) b.WriteString(``) for i := len(history) - 1; i >= 0; i-- { e := history[i] eLetter, eCls := chipLetterClass(e.Status) detail := e.Detail if detail == "" { detail = "—" } fmt.Fprintf(&b, ``, html.EscapeString(e.At.Format("2006-01-02 15:04:05")), eCls, eLetter, html.EscapeString(e.Source), html.EscapeString(detail), ) } b.WriteString(`
TimeStatusSourceDetail
%s%s%s%s
`) b.WriteString(`
`) } b.WriteString(``) return b.String() }