refactor: modularize audit and harden build validation

This commit is contained in:
Mikhail Chusavitin
2026-08-31 21:22:16 +03:00
parent bb22ccfafe
commit ac4bc0b2b7
78 changed files with 13598 additions and 13130 deletions
-684
View File
@@ -5,9 +5,6 @@ import (
"fmt"
"html"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"bee/audit/internal/app"
@@ -645,684 +642,3 @@ function auditModalRun() {
}
</script>`
}
func renderHealthCard(opts HandlerOptions) string {
data, err := loadSnapshot(filepath.Join(opts.ExportDir, "runtime-health.json"))
if err != nil {
return `<div class="card"><div class="card-head">Runtime Health</div><div class="card-body"><span class="badge badge-unknown">No data</span></div></div>`
}
var health schema.RuntimeHealth
if err := json.Unmarshal(data, &health); err != nil {
return `<div class="card"><div class="card-head">Runtime Health</div><div class="card-body"><span class="badge badge-err">Parse error</span></div></div>`
}
status := strings.TrimSpace(health.Status)
if status == "" {
status = "UNKNOWN"
}
badge := "badge-ok"
if status == "PARTIAL" {
badge = "badge-warn"
} else if status == "FAIL" || status == "FAILED" {
badge = "badge-err"
}
var b strings.Builder
b.WriteString(`<div class="card"><div class="card-head">Runtime Health</div><div class="card-body">`)
b.WriteString(fmt.Sprintf(`<div style="margin-bottom:10px"><span class="badge %s">%s</span></div>`, badge, html.EscapeString(status)))
if checkedAt := strings.TrimSpace(health.CheckedAt); checkedAt != "" {
b.WriteString(`<div style="font-size:12px;color:var(--muted);margin-bottom:12px">Checked at: ` + html.EscapeString(checkedAt) + `</div>`)
}
rows := []runtimeHealthRow{
buildRuntimeExportRow(health),
buildRuntimeNetworkRow(health),
buildRuntimeDriverRow(health),
buildRuntimeAccelerationRow(health),
buildRuntimeToolsRow(health),
buildRuntimeServicesRow(health),
buildRuntimeUSBExportRow(health),
buildRuntimeToRAMRow(health),
}
b.WriteString(`<table><thead><tr><th>Check</th><th>Status</th><th>Source</th><th>Issue</th></tr></thead><tbody>`)
for _, row := range rows {
b.WriteString(`<tr><td>` + html.EscapeString(row.Title) + `</td><td>` + runtimeStatusBadge(row.Status) + `</td><td>` + html.EscapeString(row.Source) + `</td><td>` + rowIssueHTML(row.Issue) + `</td></tr>`)
}
b.WriteString(`</tbody></table>`)
b.WriteString(`</div></div>`)
return b.String()
}
type runtimeHealthRow struct {
Title string
Status string
Source string
Issue string
}
func buildRuntimeExportRow(health schema.RuntimeHealth) runtimeHealthRow {
issue := runtimeIssueDescriptions(health.Issues, "export_dir_unavailable")
status := "UNKNOWN"
switch {
case issue != "":
status = "FAILED"
case strings.TrimSpace(health.ExportDir) != "":
status = "OK"
}
source := "os.MkdirAll"
if dir := strings.TrimSpace(health.ExportDir); dir != "" {
source += " " + dir
}
return runtimeHealthRow{Title: "Export Directory", Status: status, Source: source, Issue: issue}
}
func buildRuntimeNetworkRow(health schema.RuntimeHealth) runtimeHealthRow {
status := strings.TrimSpace(health.NetworkStatus)
if status == "" {
status = "UNKNOWN"
}
issue := runtimeIssueDescriptions(health.Issues, "dhcp_failed")
return runtimeHealthRow{Title: "Network", Status: status, Source: "ListInterfaces / DHCP", Issue: issue}
}
func buildRuntimeDriverRow(health schema.RuntimeHealth) runtimeHealthRow {
issue := runtimeIssueDescriptions(health.Issues, "nvidia_kernel_module_missing", "nvidia_modeset_failed", "amdgpu_kernel_module_missing")
status := "UNKNOWN"
switch {
case health.DriverReady && issue == "":
status = "OK"
case health.DriverReady:
status = "PARTIAL"
case issue != "":
status = "FAILED"
}
return runtimeHealthRow{Title: "NVIDIA/AMD Driver", Status: status, Source: "lsmod / vendor probe", Issue: issue}
}
func buildRuntimeAccelerationRow(health schema.RuntimeHealth) runtimeHealthRow {
issue := runtimeIssueDescriptions(health.Issues, "cuda_runtime_not_ready", "rocm_smi_unavailable")
status := "UNKNOWN"
switch {
case health.CUDAReady && issue == "":
status = "OK"
case health.CUDAReady:
status = "PARTIAL"
case issue != "":
status = "FAILED"
}
return runtimeHealthRow{Title: "CUDA / ROCm", Status: status, Source: "bee-gpu-burn / rocm-smi", Issue: issue}
}
func buildRuntimeToolsRow(health schema.RuntimeHealth) runtimeHealthRow {
if len(health.Tools) == 0 {
return runtimeHealthRow{Title: "Required Utilities", Status: "UNKNOWN", Source: "CheckTools", Issue: "No tool status data."}
}
missing := make([]string, 0)
for _, tool := range health.Tools {
if !tool.OK {
missing = append(missing, tool.Name)
}
}
status := "OK"
issue := ""
if len(missing) > 0 {
status = "PARTIAL"
issue = "Missing: " + strings.Join(missing, ", ")
}
return runtimeHealthRow{Title: "Required Utilities", Status: status, Source: "CheckTools", Issue: issue}
}
func buildRuntimeServicesRow(health schema.RuntimeHealth) runtimeHealthRow {
if len(health.Services) == 0 {
return runtimeHealthRow{Title: "Bee Services", Status: "UNKNOWN", Source: "systemctl is-active", Issue: "No service status data."}
}
nonActive := make([]string, 0)
for _, svc := range health.Services {
state := strings.TrimSpace(strings.ToLower(svc.Status))
// "inactive" is OK for oneshot services that have completed successfully
// (bee-sshsetup, bee-preflight, bee-audit, bee-network, etc.).
// Only "failed" is a genuine problem.
switch state {
case "active", "activating", "deactivating", "reloading", "inactive":
// OK — service is running, transitioning normally, or completed successfully
default:
nonActive = append(nonActive, svc.Name+"="+svc.Status)
}
}
status := "OK"
issue := ""
if len(nonActive) > 0 {
status = "PARTIAL"
issue = strings.Join(nonActive, ", ")
}
return runtimeHealthRow{Title: "Bee Services", Status: status, Source: "ServiceState", Issue: issue}
}
func buildRuntimeUSBExportRow(health schema.RuntimeHealth) runtimeHealthRow {
path := strings.TrimSpace(health.USBExportPath)
if path != "" {
return runtimeHealthRow{
Title: "USB Export Drive",
Status: "OK",
Source: "/proc/mounts + lsblk",
Issue: path,
}
}
return runtimeHealthRow{
Title: "USB Export Drive",
Status: "WARNING",
Source: "/proc/mounts + lsblk",
Issue: "No writable USB drive mounted. Plug in a USB drive to enable log export.",
}
}
func buildRuntimeToRAMRow(health schema.RuntimeHealth) runtimeHealthRow {
switch strings.ToLower(strings.TrimSpace(health.ToRAMStatus)) {
case "ok":
return runtimeHealthRow{
Title: "LiveCD in RAM",
Status: "OK",
Source: "live-boot / /proc/mounts",
Issue: "",
}
case "partial":
return runtimeHealthRow{
Title: "LiveCD in RAM",
Status: "WARNING",
Source: "live-boot / /proc/mounts / /dev/shm/bee-live",
Issue: "Partial or staged RAM copy detected. System is not fully running from RAM; Copy to RAM can be retried.",
}
case "failed":
return runtimeHealthRow{
Title: "LiveCD in RAM",
Status: "FAILED",
Source: "live-boot / /proc/mounts",
Issue: "toram boot parameter set but ISO is not mounted from RAM. Copy may have failed.",
}
default:
// toram not active — ISO still on original boot media (USB/CD)
return runtimeHealthRow{
Title: "LiveCD in RAM",
Status: "WARNING",
Source: "live-boot / /proc/mounts",
Issue: "ISO not copied to RAM. Use \u201cCopy to RAM\u201d to free the boot drive and improve performance.",
}
}
}
func buildHardwareComponentRows(exportDir string) []runtimeHealthRow {
path := filepath.Join(exportDir, "component-status.json")
db, err := app.OpenComponentStatusDB(path)
if err != nil {
return []runtimeHealthRow{
{Title: "CPU Component Health", Status: "UNKNOWN", Source: "component-status.json", Issue: "Component status DB not available."},
{Title: "Memory Component Health", Status: "UNKNOWN", Source: "component-status.json", Issue: "Component status DB not available."},
{Title: "Storage Component Health", Status: "UNKNOWN", Source: "component-status.json", Issue: "Component status DB not available."},
{Title: "GPU Component Health", Status: "UNKNOWN", Source: "component-status.json", Issue: "Component status DB not available."},
{Title: "PSU Component Health", Status: "UNKNOWN", Source: "component-status.json", Issue: "No PSU component checks recorded."},
}
}
records := db.All()
return []runtimeHealthRow{
aggregateComponentStatus("CPU", records, []string{"cpu:all"}, nil),
aggregateComponentStatus("Memory", records, []string{"memory:all"}, []string{"memory:"}),
aggregateComponentStatus("Storage", records, []string{"storage:all"}, []string{"storage:"}),
aggregateComponentStatus("GPU", records, nil, []string{"pcie:gpu:"}),
aggregateComponentStatus("PSU", records, nil, []string{"psu:"}),
}
}
// matchedRecords returns all ComponentStatusRecord entries whose key matches
// any exact key or any of the given prefixes. Used for per-device chip rendering.
func firstNonEmpty(vals ...string) string {
for _, v := range vals {
if v != "" {
return v
}
}
return ""
}
func matchedRecords(records []app.ComponentStatusRecord, exact []string, prefixes []string) []app.ComponentStatusRecord {
var matched []app.ComponentStatusRecord
for _, rec := range records {
key := strings.TrimSpace(rec.ComponentKey)
if key == "" {
continue
}
if containsExactKey(key, exact) || hasAnyPrefix(key, prefixes) {
matched = append(matched, rec)
}
}
return matched
}
func aggregateComponentStatus(title string, records []app.ComponentStatusRecord, exact []string, prefixes []string) runtimeHealthRow {
matched := make([]app.ComponentStatusRecord, 0)
for _, rec := range records {
key := strings.TrimSpace(rec.ComponentKey)
if key == "" {
continue
}
if containsExactKey(key, exact) || hasAnyPrefix(key, prefixes) {
matched = append(matched, rec)
}
}
if len(matched) == 0 {
return runtimeHealthRow{Title: title, Status: "UNKNOWN", Source: "component-status.json", Issue: "No component status data."}
}
maxSev := -1
for _, rec := range matched {
if sev := runtimeComponentSeverity(rec.Status); sev > maxSev {
maxSev = sev
}
}
status := "UNKNOWN"
switch maxSev {
case 3:
status = "CRITICAL"
case 2:
status = "WARNING"
case 1:
status = "OK"
}
sources := make([]string, 0)
sourceSeen := map[string]struct{}{}
issues := make([]string, 0)
issueSeen := map[string]struct{}{}
for _, rec := range matched {
if runtimeComponentSeverity(rec.Status) != maxSev {
continue
}
source := latestComponentSource(rec)
if source == "" {
source = "component-status.json"
}
if _, ok := sourceSeen[source]; !ok {
sourceSeen[source] = struct{}{}
sources = append(sources, source)
}
issue := strings.TrimSpace(rec.ErrorSummary)
if issue == "" {
issue = latestComponentDetail(rec)
}
if issue == "" {
continue
}
if _, ok := issueSeen[issue]; ok {
continue
}
issueSeen[issue] = struct{}{}
issues = append(issues, issue)
}
if len(sources) == 0 {
sources = append(sources, "component-status.json")
}
issue := strings.Join(issues, "; ")
if issue == "" {
issue = "—"
}
return runtimeHealthRow{
Title: title,
Status: status,
Source: strings.Join(sources, ", "),
Issue: issue,
}
}
func containsExactKey(key string, exact []string) bool {
for _, candidate := range exact {
if key == candidate {
return true
}
}
return false
}
func hasAnyPrefix(key string, prefixes []string) bool {
for _, prefix := range prefixes {
if strings.HasPrefix(key, prefix) {
return true
}
}
return false
}
func runtimeComponentSeverity(status string) int {
switch strings.TrimSpace(strings.ToLower(status)) {
case "critical":
return 3
case "warning":
return 2
case "ok":
return 1
default:
return 0
}
}
func latestComponentSource(rec app.ComponentStatusRecord) string {
if len(rec.History) == 0 {
return ""
}
return strings.TrimSpace(rec.History[len(rec.History)-1].Source)
}
func latestComponentDetail(rec app.ComponentStatusRecord) string {
if len(rec.History) == 0 {
return ""
}
return strings.TrimSpace(rec.History[len(rec.History)-1].Detail)
}
func runtimeIssueDescriptions(issues []schema.RuntimeIssue, codes ...string) string {
if len(issues) == 0 || len(codes) == 0 {
return ""
}
allowed := make(map[string]struct{}, len(codes))
for _, code := range codes {
allowed[code] = struct{}{}
}
messages := make([]string, 0)
for _, issue := range issues {
if _, ok := allowed[issue.Code]; !ok {
continue
}
desc := strings.TrimSpace(issue.Description)
if desc == "" {
desc = issue.Code
}
messages = append(messages, desc)
}
return strings.Join(messages, "; ")
}
// gpuNeedsPhysicalReboot reports whether any GPU component record carries a
// hardware-fault reason that a driver reset can't clear (e.g. Xid 79 "GPU
// has fallen off the bus", Xid 154 "Node Reboot Required" — see
// collector.xidHardwareFaultMessages). Those errors mean every subsequent
// GPU SAT job will keep failing until the node is physically power-cycled,
// so this drives a dashboard banner that says so up front instead of making
// an operator burn another test cycle to rediscover it.
func gpuNeedsPhysicalReboot(records []app.ComponentStatusRecord) (reason string, needsReboot bool) {
for _, rec := range records {
if !strings.EqualFold(strings.TrimSpace(rec.Status), "Critical") {
continue
}
if strings.Contains(strings.ToLower(rec.ErrorSummary), "reboot") {
return rec.ErrorSummary, true
}
}
return "", false
}
// chipLetterClass maps a component status to a single display letter and CSS class.
func chipLetterClass(status string) (letter, cls string) {
switch strings.ToUpper(strings.TrimSpace(status)) {
case "OK":
return "O", "chip-ok"
case "WARNING", "WARN", "PARTIAL":
return "W", "chip-warn"
case "CRITICAL", "FAIL", "FAILED", "ERROR":
return "F", "chip-fail"
default:
return "?", "chip-unknown"
}
}
// renderComponentChips renders one 20×20 chip per ComponentStatusRecord.
// Hover tooltip shows component key, status, error summary and last check time.
// Falls back to a single unknown chip when no records are available.
func renderComponentChips(matched []app.ComponentStatusRecord) string {
if len(matched) == 0 {
return `<span class="chips"><span class="chip chip-unknown" title="No data">?</span></span>`
}
sort.Slice(matched, func(i, j int) bool {
return matched[i].ComponentKey < matched[j].ComponentKey
})
var b strings.Builder
b.WriteString(`<span class="chips">`)
for _, rec := range matched {
letter, cls := chipLetterClass(rec.Status)
var tooltip strings.Builder
tooltip.WriteString(rec.ComponentKey)
tooltip.WriteString(": ")
tooltip.WriteString(firstNonEmpty(rec.Status, "UNKNOWN"))
if rec.ErrorSummary != "" {
tooltip.WriteString(" — ")
tooltip.WriteString(rec.ErrorSummary)
}
if !rec.LastCheckedAt.IsZero() {
fmt.Fprintf(&tooltip, " (checked %s)", rec.LastCheckedAt.Format("15:04:05"))
}
fmt.Fprintf(&b, `<span class="chip %s" title="%s">%s</span>`,
cls, html.EscapeString(tooltip.String()), letter)
}
b.WriteString(`</span>`)
return b.String()
}
func runtimeStatusBadge(status string) string {
status = strings.ToUpper(strings.TrimSpace(status))
badge := "badge-unknown"
switch status {
case "OK":
badge = "badge-ok"
case "PARTIAL", "WARNING", "WARN":
badge = "badge-warn"
case "FAIL", "FAILED", "CRITICAL":
badge = "badge-err"
}
return `<span class="badge ` + badge + `">` + html.EscapeString(status) + `</span>`
}
func rowIssueHTML(issue string) string {
issue = strings.TrimSpace(issue)
if issue == "" {
return `<span style="color:var(--muted)">—</span>`
}
return html.EscapeString(issue)
}
var aerStatusRe = regexp.MustCompile(`aer_status:\s*0x([0-9a-fA-F]{1,8})`)
// decodeAERStatus parses an AER status hex value from a kernel error detail string
// and returns a human-readable list of set bit names with correctable/uncorrectable label,
// or "" if no AER status is found.
func decodeAERStatus(detail string) string {
m := aerStatusRe.FindStringSubmatch(detail)
if m == nil {
return ""
}
v64, err := strconv.ParseUint(m[1], 16, 32)
if err != nil {
return ""
}
val := uint32(v64)
type bitDef struct {
bit uint32
name string
}
corrBits := []bitDef{
{0, "Receiver Error"}, {6, "Replay Timer Timeout"}, {7, "Advisory Non-Fatal"},
{8, "Corrected Internal Error"}, {9, "Header Log Overflow"},
{13, "Replay Num Rollover"}, {14, "Bad DLLP"}, {15, "Bad TLP"},
}
uncorrBits := []bitDef{
{4, "Data Link Protocol Error"}, {5, "Surprise Down Error"},
{12, "Poisoned TLP Received"}, {13, "Flow Control Protocol Error"},
{14, "Completion Timeout"}, {15, "Completer Abort"}, {16, "Unexpected Completion"},
{17, "Receiver Overflow"}, {18, "Malformed TLP"}, {19, "ECRC Error"},
{20, "Unsupported Request Error"}, {21, "ACS Violation"}, {22, "Uncorrectable Internal Error"},
}
var corrNames, uncorrNames []string
for _, b := range corrBits {
if val&(1<<b.bit) != 0 {
corrNames = append(corrNames, b.name)
}
}
for _, b := range uncorrBits {
if val&(1<<b.bit) != 0 {
uncorrNames = append(uncorrNames, b.name)
}
}
if len(corrNames) >= len(uncorrNames) && len(corrNames) > 0 {
return strings.Join(corrNames, ", ") + " (correctable)"
}
if len(uncorrNames) > 0 {
return strings.Join(uncorrNames, ", ") + " (uncorrectable)"
}
return fmt.Sprintf("unknown bits: 0x%08x", val)
}
// renderSparkline returns a small inline SVG showing non-OK events over time.
// Events are positioned proportionally along the time axis; if all share the same
// timestamp they are spaced evenly. Width is always 100px.
func renderSparkline(history []app.ComponentStatusEntry) string {
const (
svgW = 100
svgH = 20
barW = 3
barH = 14
)
var events []app.ComponentStatusEntry
for _, e := range history {
if e.Status != "OK" {
events = append(events, e)
}
}
if len(events) == 0 {
return ""
}
n := len(events)
barColor := func(status string) string {
if status == "Critical" {
return "#c0392b"
}
return "#d97706"
}
yTop := (svgH - barH) / 2
var bars strings.Builder
if n == 1 {
x := (svgW - barW) / 2
fmt.Fprintf(&bars, `<rect x="%d" y="%d" width="%d" height="%d" fill="%s" rx="1"/>`,
x, yTop, barW, barH, barColor(events[0].Status))
} else {
minT := events[0].At
maxT := events[n-1].At
dur := maxT.Sub(minT).Seconds()
for i, e := range events {
var x int
if dur <= 0 {
step := svgW / n
x = i*step + (step-barW)/2
} else {
frac := e.At.Sub(minT).Seconds() / dur
x = int(frac * float64(svgW-barW))
}
fmt.Fprintf(&bars, `<rect x="%d" y="%d" width="%d" height="%d" fill="%s" rx="1"/>`,
x, yTop, barW, barH, barColor(e.Status))
}
}
return fmt.Sprintf(
`<svg width="%d" height="%d" style="display:inline-block;vertical-align:middle;margin-left:6px;flex-shrink:0" xmlns="http://www.w3.org/2000/svg">`+
`<rect x="0" y="0" width="%d" height="%d" fill="var(--surface-alt,#ebebeb)" rx="3"/>%s</svg>`,
svgW, svgH, svgW, svgH, bars.String())
}
// renderComponentDetail renders a modal content fragment for one component type.
// Called by handleAPIComponentDetail and displayed inside #component-detail-dialog.
// fromInventory marks that records were synthesized from the audit inventory
// snapshot (no ComponentStatusDB history yet) rather than real SAT/watchdog
// observations — see inventoryFallbackRecords.
func renderComponentDetail(title string, records []app.ComponentStatusRecord, fromInventory bool) string {
var b strings.Builder
fmt.Fprintf(&b, `<div style="padding:20px 24px 0">`)
fmt.Fprintf(&b, `<div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:16px">`)
fmt.Fprintf(&b, `<span style="font-size:16px;font-weight:700">%s — Status Detail</span>`, html.EscapeString(title))
b.WriteString(`<button class="btn btn-sm btn-secondary" onclick="document.getElementById('component-detail-dialog').close()">Close</button>`)
b.WriteString(`</div>`)
if len(records) == 0 {
b.WriteString(`<p style="color:var(--muted)">No status data recorded yet for this component type.</p>`)
b.WriteString(`</div>`)
return b.String()
}
if fromInventory {
b.WriteString(`<p style="color:var(--muted);font-size:12px;margin-top:-8px;margin-bottom:16px">No SAT-test history yet — showing latest inventory snapshot.</p>`)
}
sort.Slice(records, func(i, j int) bool {
return records[i].ComponentKey < records[j].ComponentKey
})
for _, rec := range records {
letter, cls := chipLetterClass(rec.Status)
// Count non-OK events across the full history for the badge + sparkline.
warnCount := 0
for _, e := range rec.History {
if e.Status != "OK" {
warnCount++
}
}
fmt.Fprintf(&b, `<div style="margin-bottom:20px">`)
fmt.Fprintf(&b, `<div style="display:flex;align-items:center;gap:8px;margin-bottom:8px;flex-wrap:wrap">`)
fmt.Fprintf(&b, `<span class="chip %s">%s</span>`, cls, letter)
fmt.Fprintf(&b, `<span style="font-weight:700;font-size:13px">%s</span>`, html.EscapeString(rec.ComponentKey))
if !rec.LastCheckedAt.IsZero() {
fmt.Fprintf(&b, `<span style="color:var(--muted);font-size:12px">checked %s</span>`, rec.LastCheckedAt.Format("2006-01-02 15:04:05"))
}
if warnCount > 0 {
noun := "events"
if warnCount == 1 {
noun = "event"
}
fmt.Fprintf(&b,
`<span style="font-size:11px;background:var(--warn-bg,#fffbeb);color:var(--warn-fg,#92400e);border:1px solid var(--warn-border,#fde68a);border-radius:10px;padding:1px 7px;white-space:nowrap">%d %s</span>`,
warnCount, noun)
b.WriteString(renderSparkline(rec.History))
}
b.WriteString(`</div>`)
if rec.ErrorSummary != "" {
fmt.Fprintf(&b, `<div style="font-size:12px;margin-bottom:4px;color:var(--muted)">%s</div>`, html.EscapeString(rec.ErrorSummary))
if decoded := decodeAERStatus(rec.ErrorSummary); decoded != "" {
fmt.Fprintf(&b,
`<div style="font-size:12px;margin-bottom:8px;color:var(--muted)"><span style="background:var(--surface-alt,#f5f5f5);border-radius:4px;padding:1px 6px;font-family:monospace">AER: %s</span></div>`,
html.EscapeString(decoded))
}
}
// History table — newest first, cap at 20 entries.
history := rec.History
if len(history) > 20 {
history = history[len(history)-20:]
}
b.WriteString(`<table style="width:100%;font-size:12px;border-collapse:collapse">`)
b.WriteString(`<tr style="color:var(--muted)"><th style="text-align:left;padding:2px 10px 2px 0;white-space:nowrap">Time</th><th style="text-align:left;padding:2px 10px 2px 0">Status</th><th style="text-align:left;padding:2px 10px 2px 0">Source</th><th style="text-align:left;padding:2px 0">Detail</th></tr>`)
for i := len(history) - 1; i >= 0; i-- {
e := history[i]
eLetter, eCls := chipLetterClass(e.Status)
detail := e.Detail
if detail == "" {
detail = "—"
}
fmt.Fprintf(&b,
`<tr><td style="padding:3px 10px 3px 0;white-space:nowrap;color:var(--muted)">%s</td><td style="padding:3px 10px 3px 0"><span class="chip %s" style="font-size:10px;width:16px;height:16px">%s</span></td><td style="padding:3px 10px 3px 0;white-space:nowrap">%s</td><td style="padding:3px 0;color:var(--muted)">%s</td></tr>`,
html.EscapeString(e.At.Format("2006-01-02 15:04:05")),
eCls, eLetter,
html.EscapeString(e.Source),
html.EscapeString(detail),
)
}
b.WriteString(`</table>`)
b.WriteString(`</div>`)
}
b.WriteString(`</div>`)
return b.String()
}