Add continuous hardware health monitoring and component detail view

- kmsg watcher now records kernel errors (GPU Xid, MCE, EDAC, storage I/O) at all times,
  not only during SAT tasks; flushImmediate writes directly to ComponentStatusDB
- New health_poller: polls ipmitool sdr every 60s for PSU health (watchdog:psu source)
- Hardware Summary card auto-refreshes every 30s via htmx without page reload
- Component rows (CPU/Memory/Storage/GPU/PSU) are now clickable -- opens a modal
  with per-component status, source, timestamp and last 20 history entries

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

audit/internal/webui/api.go

@@ -1679,6 +1679,56 @@ func (h *handler) handleAPIBenchmarkResults(w http.ResponseWriter, r *http.Reque
 	fmt.Fprint(w, renderBenchmarkResultsCard(h.opts.ExportDir))
 }
 
+// ── Hardware summary / component detail ──────────────────────────────────────
+
+// handleAPIHardwareSummary returns the hardware summary card HTML fragment for
+// htmx polling (hx-get="/api/hardware-summary" hx-swap="outerHTML").
+func (h *handler) handleAPIHardwareSummary(w http.ResponseWriter, _ *http.Request) {
+	w.Header().Set("Content-Type", "text/html; charset=utf-8")
+	w.Header().Set("Cache-Control", "no-store")
+	fmt.Fprint(w, renderHardwareSummaryCard(h.opts))
+}
+
+// handleAPIComponentDetail returns an HTML fragment describing the current and
+// historical status for one component type (cpu, memory, storage, gpu, psu).
+func (h *handler) handleAPIComponentDetail(w http.ResponseWriter, r *http.Request) {
+	compType := r.PathValue("type")
+	var exact, prefixes []string
+	var title string
+	switch compType {
+	case "cpu":
+		title = "CPU"
+		exact = []string{"cpu:all"}
+	case "memory":
+		title = "Memory"
+		exact = []string{"memory:all"}
+		prefixes = []string{"memory:"}
+	case "storage":
+		title = "Storage"
+		exact = []string{"storage:all"}
+		prefixes = []string{"storage:"}
+	case "gpu":
+		title = "GPU"
+		prefixes = []string{"pcie:gpu:"}
+	case "psu":
+		title = "PSU"
+		prefixes = []string{"psu:"}
+	default:
+		http.NotFound(w, r)
+		return
+	}
+
+	var records []app.ComponentStatusRecord
+	if h.opts.App != nil && h.opts.App.StatusDB != nil {
+		all := h.opts.App.StatusDB.All()
+		records = matchedRecords(all, exact, prefixes)
+	}
+
+	w.Header().Set("Content-Type", "text/html; charset=utf-8")
+	w.Header().Set("Cache-Control", "no-store")
+	fmt.Fprint(w, renderComponentDetail(title, records))
+}
+
 func (h *handler) rollbackPendingNetworkChange() error {
 	h.pendingNetMu.Lock()
 	pnc := h.pendingNet

audit/internal/webui/health_poller.go

@@ -0,0 +1,76 @@
+package webui
+
+import (
+	"bytes"
+	"context"
+	"log/slog"
+	"os/exec"
+	"time"
+
+	"bee/audit/internal/app"
+	"bee/audit/internal/collector"
+)
+
+const healthPollInterval = 60 * time.Second
+const psuIPMITimeout = 15 * time.Second
+
+// healthPoller runs periodic health checks for hardware components that do not
+// emit kernel log events (e.g. PSU). Results are written to ComponentStatusDB.
+type healthPoller struct {
+	statusDB *app.ComponentStatusDB
+}
+
+func newHealthPoller(statusDB *app.ComponentStatusDB) *healthPoller {
+	return &healthPoller{statusDB: statusDB}
+}
+
+func (p *healthPoller) start() {
+	goRecoverLoop("health poller", 5*time.Second, p.run)
+}
+
+func (p *healthPoller) run() {
+	ticker := time.NewTicker(healthPollInterval)
+	defer ticker.Stop()
+	for range ticker.C {
+		p.pollPSU()
+	}
+}
+
+func (p *healthPoller) pollPSU() {
+	if p.statusDB == nil {
+		return
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), psuIPMITimeout)
+	defer cancel()
+
+	cmd := exec.CommandContext(ctx, "ipmitool", "sdr")
+	var out bytes.Buffer
+	cmd.Stdout = &out
+	if err := cmd.Run(); err != nil {
+		// IPMI not available or not a server — skip silently.
+		slog.Debug("health poller: ipmitool sdr unavailable", "err", err)
+		return
+	}
+
+	slots := collector.PSUSlotsFromSDR(out.String())
+	if len(slots) == 0 {
+		return
+	}
+
+	const source = "watchdog:psu"
+	for slot, psu := range slots {
+		key := "psu:" + slot
+		status := psu.Status
+		if status == "" {
+			status = "Unknown"
+		}
+		detail := ""
+		switch status {
+		case "Critical":
+			detail = "PSU sensor reported non-OK state"
+		case "Warning":
+			detail = "PSU sensor in warning state"
+		}
+		p.statusDB.Record(key, source, status, detail)
+	}
+}

audit/internal/webui/kmsg_watcher.go

@@ -73,6 +73,9 @@ func (w *kmsgWatcher) run() {
 			w.mu.Lock()
 			if w.window != nil {
 				w.recordEvent(evt)
+			} else {
+				evtCopy := evt
+				goRecoverOnce("kmsg flush immediate", func() { w.flushImmediate(evtCopy) })
 			}
 			w.mu.Unlock()
 		}
@@ -180,6 +183,52 @@ func (w *kmsgWatcher) flushWindow(window *kmsgWindow) {
 	}
 }
 
+// flushImmediate writes a single kmsg event directly to the status DB without a SAT window.
+// Called when an error is detected outside of any SAT task (always-on watching).
+func (w *kmsgWatcher) flushImmediate(evt kmsgEvent) {
+	if w.statusDB == nil {
+		return
+	}
+	const source = "watchdog:kmsg"
+	detail := "kernel: " + truncate(evt.raw, 120)
+
+	var severity string
+	for _, p := range platform.HardwareErrorPatterns {
+		if p.Re.MatchString(evt.raw) {
+			if p.Severity == "critical" {
+				severity = "Critical"
+			} else {
+				severity = "Warning"
+			}
+			break
+		}
+	}
+	if severity == "" {
+		severity = "Warning"
+	}
+
+	if len(evt.ids) == 0 {
+		key := "cpu:all"
+		if evt.category == "memory" {
+			key = "memory:all"
+		}
+		w.statusDB.Record(key, source, severity, detail)
+		return
+	}
+	for _, id := range evt.ids {
+		var key string
+		switch evt.category {
+		case "gpu", "pcie":
+			key = "pcie:" + normalizeBDF(id)
+		case "storage":
+			key = "storage:" + id
+		default:
+			key = "pcie:" + normalizeBDF(id)
+		}
+		w.statusDB.Record(key, source, severity, detail)
+	}
+}
+
 // parseKmsgLine parses a single /dev/kmsg line and returns an event if it matches
 // any pattern in platform.HardwareErrorPatterns.
 // kmsg format: "<priority>,<sequence>,<timestamp_usec>,-;message text"

audit/internal/webui/pages.go

@@ -85,6 +85,7 @@ func renderPage(page string, opts HandlerOptions) string {
 		body +
 		`</div></div>` +
 		renderAuditModal() +
+		`<dialog id="component-detail-dialog" style="min-width:600px;max-width:900px;width:90vw;padding:0;border:1px solid var(--border);border-radius:8px;background:var(--surface)"><div id="component-detail-body" style="padding-bottom:20px"></div></dialog>` +
 		`<script>
 // Add copy button to every .terminal on the page
 document.querySelectorAll('.terminal').forEach(function(t){
@@ -184,13 +185,14 @@ func renderAudit() string {
 }
 
 func renderHardwareSummaryCard(opts HandlerOptions) string {
+	const cardAttrs = ` hx-get="/api/hardware-summary" hx-trigger="every 30s" hx-swap="outerHTML"`
 	data, err := loadSnapshot(opts.AuditPath)
 	if err != nil {
-		return `<div class="card"><div class="card-head card-head-actions"><span>Hardware Summary</span><div class="card-head-buttons"><button class="btn btn-primary btn-sm" onclick="auditModalRun()">Run audit</button></div></div><div class="card-body"></div></div>`
+		return `<div class="card"` + cardAttrs + `><div class="card-head card-head-actions"><span>Hardware Summary</span><div class="card-head-buttons"><button class="btn btn-primary btn-sm" onclick="auditModalRun()">Run audit</button></div></div><div class="card-body"></div></div>`
 	}
 	var ingest schema.HardwareIngestRequest
 	if err := json.Unmarshal(data, &ingest); err != nil {
-		return `<div class="card"><div class="card-head">Hardware Summary</div><div class="card-body"><span class="badge badge-err">Parse error</span></div></div>`
+		return `<div class="card"` + cardAttrs + `><div class="card-head">Hardware Summary</div><div class="card-body"><span class="badge badge-err">Parse error</span></div></div>`
 	}
 	hw := ingest.Hardware
 
@@ -200,7 +202,7 @@ func renderHardwareSummaryCard(opts HandlerOptions) string {
 	}
 
 	var b strings.Builder
-	b.WriteString(`<div class="card"><div class="card-head">Hardware Summary</div><div class="card-body">`)
+	b.WriteString(`<div class="card"` + cardAttrs + `><div class="card-head">Hardware Summary</div><div class="card-body">`)
 
 	// Server identity block above the component table.
 	{
@@ -229,22 +231,32 @@ func renderHardwareSummaryCard(opts HandlerOptions) string {
 	}
 
 	b.WriteString(`<table style="width:auto">`)
-	writeRow := func(label, value, badgeHTML string) {
+	// writeRow renders one component row. compType is the URL path segment for the detail
-		b.WriteString(fmt.Sprintf(`<tr><td style="padding:6px 14px 6px 0;font-weight:700;white-space:nowrap">%s</td><td style="padding:6px 0;color:var(--muted);font-size:13px">%s</td><td style="padding:6px 0 6px 12px">%s</td></tr>`,
+	// endpoint (e.g. "cpu"). Pass "" for rows that have no detail view.
-			html.EscapeString(label), html.EscapeString(value), badgeHTML))
+	writeRow := func(label, value, badgeHTML, compType string) {
+		var labelHTML string
+		if compType != "" {
+			labelHTML = fmt.Sprintf(
+				`<span style="cursor:pointer;text-decoration:underline dotted;text-underline-offset:3px" hx-get="/api/components/%s" hx-target="#component-detail-body" hx-swap="innerHTML" onclick="document.getElementById('component-detail-dialog').showModal()">%s</span>`,
+				compType, html.EscapeString(label))
+		} else {
+			labelHTML = html.EscapeString(label)
+		}
+		fmt.Fprintf(&b, `<tr><td style="padding:6px 14px 6px 0;font-weight:700;white-space:nowrap">%s</td><td style="padding:6px 0;color:var(--muted);font-size:13px">%s</td><td style="padding:6px 0 6px 12px">%s</td></tr>`,
+			labelHTML, html.EscapeString(value), badgeHTML)
 	}
 
 	writeRow("CPU", hwDescribeCPU(hw),
-		renderComponentChips(matchedRecords(records, []string{"cpu:all"}, nil)))
+		renderComponentChips(matchedRecords(records, []string{"cpu:all"}, nil)), "cpu")
 
 	writeRow("Memory", hwDescribeMemory(hw),
-		renderComponentChips(matchedRecords(records, []string{"memory:all"}, []string{"memory:"})))
+		renderComponentChips(matchedRecords(records, []string{"memory:all"}, []string{"memory:"})), "memory")
 
 	writeRow("Storage", hwDescribeStorage(hw),
-		renderComponentChips(matchedRecords(records, []string{"storage:all"}, []string{"storage:"})))
+		renderComponentChips(matchedRecords(records, []string{"storage:all"}, []string{"storage:"})), "storage")
 
 	writeRow("GPU", hwDescribeGPU(hw),
-		renderComponentChips(matchedRecords(records, nil, []string{"pcie:gpu:"})))
+		renderComponentChips(matchedRecords(records, nil, []string{"pcie:gpu:"})), "gpu")
 
 	psuMatched := matchedRecords(records, nil, []string{"psu:"})
 	if len(psuMatched) == 0 && len(hw.PowerSupplies) > 0 {
@@ -252,10 +264,10 @@ func renderHardwareSummaryCard(opts HandlerOptions) string {
 		psuStatus := hwPSUStatus(hw.PowerSupplies)
 		psuMatched = []app.ComponentStatusRecord{{ComponentKey: "psu:ipmi", Status: psuStatus}}
 	}
-	writeRow("PSU", hwDescribePSU(hw), renderComponentChips(psuMatched))
+	writeRow("PSU", hwDescribePSU(hw), renderComponentChips(psuMatched), "psu")
 
 	if nicDesc := hwDescribeNIC(hw); nicDesc != "" {
-		writeRow("Network", nicDesc, "")
+		writeRow("Network", nicDesc, "", "")
 	}
 
 	b.WriteString(`</table>`)
@@ -999,3 +1011,67 @@ func rowIssueHTML(issue string) string {
 	}
 	return html.EscapeString(issue)
 }
+
+// renderComponentDetail renders a modal content fragment for one component type.
+// Called by handleAPIComponentDetail and displayed inside #component-detail-dialog.
+func renderComponentDetail(title string, records []app.ComponentStatusRecord) string {
+	var b strings.Builder
+	fmt.Fprintf(&b, `<div style="padding:20px 24px 0">`)
+	fmt.Fprintf(&b, `<div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:16px">`)
+	fmt.Fprintf(&b, `<span style="font-size:16px;font-weight:700">%s — Status Detail</span>`, html.EscapeString(title))
+	b.WriteString(`<button class="btn btn-sm btn-secondary" onclick="document.getElementById('component-detail-dialog').close()">Close</button>`)
+	b.WriteString(`</div>`)
+
+	if len(records) == 0 {
+		b.WriteString(`<p style="color:var(--muted)">No status data recorded yet for this component type.</p>`)
+		b.WriteString(`</div>`)
+		return b.String()
+	}
+
+	sort.Slice(records, func(i, j int) bool {
+		return records[i].ComponentKey < records[j].ComponentKey
+	})
+
+	for _, rec := range records {
+		letter, cls := chipLetterClass(rec.Status)
+		fmt.Fprintf(&b, `<div style="margin-bottom:20px">`)
+		fmt.Fprintf(&b, `<div style="display:flex;align-items:center;gap:8px;margin-bottom:8px">`)
+		fmt.Fprintf(&b, `<span class="chip %s">%s</span>`, cls, letter)
+		fmt.Fprintf(&b, `<span style="font-weight:700;font-size:13px">%s</span>`, html.EscapeString(rec.ComponentKey))
+		if !rec.LastCheckedAt.IsZero() {
+			fmt.Fprintf(&b, `<span style="color:var(--muted);font-size:12px">checked %s</span>`, rec.LastCheckedAt.Format("2006-01-02 15:04:05"))
+		}
+		b.WriteString(`</div>`)
+		if rec.ErrorSummary != "" {
+			fmt.Fprintf(&b, `<div style="font-size:12px;margin-bottom:8px;color:var(--muted)">%s</div>`, html.EscapeString(rec.ErrorSummary))
+		}
+
+		// History table — newest first, cap at 20 entries.
+		history := rec.History
+		if len(history) > 20 {
+			history = history[len(history)-20:]
+		}
+		b.WriteString(`<table style="width:100%;font-size:12px;border-collapse:collapse">`)
+		b.WriteString(`<tr style="color:var(--muted)"><th style="text-align:left;padding:2px 10px 2px 0;white-space:nowrap">Time</th><th style="text-align:left;padding:2px 10px 2px 0">Status</th><th style="text-align:left;padding:2px 10px 2px 0">Source</th><th style="text-align:left;padding:2px 0">Detail</th></tr>`)
+		for i := len(history) - 1; i >= 0; i-- {
+			e := history[i]
+			eLetter, eCls := chipLetterClass(e.Status)
+			detail := e.Detail
+			if detail == "" {
+				detail = "—"
+			}
+			fmt.Fprintf(&b,
+				`<tr><td style="padding:3px 10px 3px 0;white-space:nowrap;color:var(--muted)">%s</td><td style="padding:3px 10px 3px 0"><span class="chip %s" style="font-size:10px;width:16px;height:16px">%s</span></td><td style="padding:3px 10px 3px 0;white-space:nowrap">%s</td><td style="padding:3px 0;color:var(--muted)">%s</td></tr>`,
+				html.EscapeString(e.At.Format("2006-01-02 15:04:05")),
+				eCls, eLetter,
+				html.EscapeString(e.Source),
+				html.EscapeString(detail),
+			)
+		}
+		b.WriteString(`</table>`)
+		b.WriteString(`</div>`)
+	}
+
+	b.WriteString(`</div>`)
+	return b.String()
+}

audit/internal/webui/server.go

@@ -221,6 +221,11 @@ func NewHandler(opts HandlerOptions) http.Handler {
 		h.kmsg = newKmsgWatcher(opts.App.StatusDB)
 		h.kmsg.start()
 		globalQueue.kmsgWatcher = h.kmsg
+
+		// Start periodic health poller for components that don't emit kernel log events (e.g. PSU).
+		if opts.App.StatusDB != nil {
+			newHealthPoller(opts.App.StatusDB).start()
+		}
 	}
 
 	globalQueue.startWorker(&opts)
@@ -328,6 +333,10 @@ func NewHandler(opts HandlerOptions) http.Handler {
 	mux.HandleFunc("GET /api/install/disks", h.handleAPIInstallDisks)
 	mux.HandleFunc("POST /api/install/run", h.handleAPIInstallRun)
 
+	// Hardware component detail (fragment for modal in Hardware Summary card)
+	mux.HandleFunc("GET /api/hardware-summary", h.handleAPIHardwareSummary)
+	mux.HandleFunc("GET /api/components/{type}", h.handleAPIComponentDetail)
+
 	// Metrics — SSE stream of live sensor data + server-side SVG charts + CSV export
 	mux.HandleFunc("GET /api/metrics/stream", h.handleAPIMetricsStream)
 	mux.HandleFunc("GET /api/metrics/latest", h.handleAPIMetricsLatest)