feat(sat): fan ceiling check + topology fan tiles

Repurpose the previously-unwired RunFanStressTest into RunFanCheck, a
Load-tier SAT test that drives stressapptest (CPU+memory) and, when a GPU
is present, a GPU burn to 100% simultaneously, then watches every fan
until none has climbed for ~60s. The observed peak RPM per fan is the
"ceiling"; it is persisted through the existing fan-observation store.

MSI G4201 / AMI MegaRAC exposes no host-side fan force (every OEM IPMI
command returns 0xc1; Redfish Thermal is GET-only), so load-driven ramp
is the closest safe equivalent. See
bible-local/decisions/2026-09-04-fan-ceiling-check.md.

- platform.ResolveFanMaxRPM: per-fan max with fallback (persisted peak ->
  peer peak -> current RPM), resolved in platform, not the view.
- platform.ErrTestNotApplicable: no load source or no fan sensors ->
  task lands as cancelled ("not applicable"), never failed, so an
  engineer never sees a false red. executeTaskWithOptions maps the
  sentinel; finalizeTaskForResult honours a pre-set TaskCancelled.
- Verdict FAIL only for a fan at 0 RPM / IPMI cr-nr under load.
- /topo: one small spinning square per fan, sized by RPM / resolved max,
  clickable through to a new "fan" component-detail type; per-fan status
  recorded to the component-status DB from the fan SAT summary.
- Wiring: /api/sat/fan/run route, "fan" task target, Load-page card,
  stress-mode Run All.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019VHG21rgTUiR1G3qFHTVmN
This commit is contained in:
Mikhail Chusavitin
2026-09-04 10:35:34 +03:00
co-authored by Claude Sonnet 5
parent e4f7519ef3
commit bb2a501a28
22 changed files with 798 additions and 222 deletions
+1 -1
View File
@@ -128,7 +128,7 @@ func defaultTaskPriority(target string, params taskParams) int {
return taskPriorityAudit
case "nvidia-bench-perf", "nvidia-bench-power", "nvidia-bench-autotune":
return taskPriorityBenchmark
case "nvidia-stress", "amd-stress", "memory-stress", "sat-stress", "platform-stress", "nvidia-compute", "scenario":
case "nvidia-stress", "amd-stress", "memory-stress", "sat-stress", "platform-stress", "fan", "nvidia-compute", "scenario":
return taskPriorityBurn
case "nvidia", "nvidia-targeted-stress", "nvidia-targeted-power", "nvidia-pulse",
"nvidia-interconnect", "nvidia-bandwidth", "memory", "storage", "cpu",
+4 -1
View File
@@ -289,7 +289,7 @@ func (h *handler) handleAPIHardwareSummary(w http.ResponseWriter, _ *http.Reques
}
// handleAPIComponentDetail returns an HTML fragment describing the current and
// historical status for one component type (cpu, memory, storage, gpu, psu).
// historical status for one component type (cpu, memory, storage, gpu, psu, fan).
func (h *handler) handleAPIComponentDetail(w http.ResponseWriter, r *http.Request) {
compType := r.PathValue("type")
var exact, prefixes []string
@@ -315,6 +315,9 @@ func (h *handler) handleAPIComponentDetail(w http.ResponseWriter, r *http.Reques
case "psu":
title = "PSU"
prefixes = []string{"psu:"}
case "fan":
title = "Fans"
prefixes = []string{"fan:"}
case "raid":
title = "RAID"
prefixes = []string{"pcie:raid:"}
+5
View File
@@ -109,6 +109,11 @@ func (h *handler) planSATRunAll(ctx context.Context, req satRunAllRequest) ([]sa
} else {
skip("TPM: no TPM device on this host; check skipped")
}
} else {
// Fan ceiling check runs on the Load tier only. It self-cancels as
// "not applicable" on a host with no fan sensors or no way to load
// the CPU/GPU, so it is safe to queue unconditionally here.
specs = append(specs, satRunAllSpec{target: "fan", params: taskParams{StressMode: true}})
}
gp := h.opts.App.DetectGPUPresence()
+1 -1
View File
@@ -98,7 +98,7 @@ func TestPlanSATRunAllLoadOmitsReadOnlyTPMCheck(t *testing.T) {
for _, s := range specs {
targets = append(targets, s.target)
}
if want := []string{"cpu", "memory", "storage"}; !reflect.DeepEqual(targets, want) {
if want := []string{"cpu", "memory", "storage", "fan"}; !reflect.DeepEqual(targets, want) {
t.Fatalf("targets=%v want %v", targets, want)
}
for _, note := range notes {
+1 -1
View File
@@ -296,7 +296,7 @@ func isSATTarget(target string) bool {
case "nvidia", "nvidia-targeted-stress", "nvidia-bench-perf", "nvidia-bench-power", "nvidia-compute", "nvidia-targeted-power", "nvidia-pulse",
"nvidia-interconnect", "nvidia-bandwidth", "nvidia-stress", "memory", "memory-stress", "storage",
"cpu", "sat-stress", "amd", "amd-mem", "amd-bandwidth", "amd-stress",
"platform-stress":
"platform-stress", "fan":
return true
}
return false
+139 -2
View File
@@ -614,9 +614,138 @@ func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string
}}))
}
// Cooling fans — one small square per fan (no PCIe/CPU affinity, arbitrary
// count, so a wrapping flex row like PSUs rather than SVG boxes). Each
// square is sized by rpm / observed-max-rpm and carries a fan glyph that
// spins via CSS — faster when the fan is spinning faster.
if fans := dedupeFansByName(hw.Sensors); len(fans) > 0 {
current := map[string]float64{}
for _, f := range fans {
if f.RPM != nil {
current[strings.TrimSpace(f.Name)] = float64(*f.RPM)
}
}
b.WriteString(renderTopoFanRow(fans, platform.ResolveFanMaxRPM(current)))
}
return topoCard("Topology", b.String())
}
// renderTopoFanRow renders the COOLING row: one clickable square per fan,
// side length scaled by rpm/maxRPM and a fan glyph whose spin rate tracks the
// same ratio. maxByName comes from platform.ResolveFanMaxRPM — it already has
// an entry for every fan (persisted peak, else peer peak, else current RPM),
// so no fallback logic lives here.
func renderTopoFanRow(fans []schema.HardwareFanSensor, maxByName map[string]float64) string {
const (
fanTileMin = 30 // px, a stalled / slowest fan
fanTileMax = 58 // px, a fan at its ceiling
)
var tally topoStatusTally
for _, f := range fans {
tally.add(classifyTopoSeverity(f.Status))
}
var b strings.Builder
b.WriteString(topoRowHeading("Cooling"))
b.WriteString(topoFanSpinStyle())
b.WriteString(`<div style="display:flex;flex-wrap:wrap;gap:6px;align-items:flex-end">`)
for _, f := range fans {
name := strings.TrimSpace(f.Name)
fill, stroke, text := topoSeverityColors(classifyTopoSeverity(f.Status))
denom := maxByName[name]
ratio := 0.0
title := name
if f.RPM != nil {
if denom > 0 {
ratio = float64(*f.RPM) / denom
}
if ratio < 0 {
ratio = 0
}
if ratio > 1 {
ratio = 1
}
if denom > float64(*f.RPM) {
title = fmt.Sprintf("%s · %d RPM (max %d)", name, *f.RPM, int(denom))
} else {
title = fmt.Sprintf("%s · %d RPM", name, *f.RPM)
}
} else {
title = name + " · no reading"
}
side := fanTileMin + int(float64(fanTileMax-fanTileMin)*ratio+0.5)
// Spin period: 2.6s at rest down to 0.5s at the observed peak. A fan
// with no reading doesn't spin.
spin := ""
if f.RPM != nil && *f.RPM > 0 {
period := 2.6 - 2.1*ratio
spin = fmt.Sprintf(`<svg class="topo-fan-spin" style="animation-duration:%.2fs" width="%d" height="%d" viewBox="0 0 24 24" fill="currentColor" aria-hidden="true">`,
period, side*7/16, side*7/16) + topoFanGlyphPaths() + `</svg>`
} else {
spin = fmt.Sprintf(`<svg width="%d" height="%d" viewBox="0 0 24 24" fill="currentColor" style="opacity:.4" aria-hidden="true">`,
side*7/16, side*7/16) + topoFanGlyphPaths() + `</svg>`
}
fmt.Fprintf(&b, `<div title="%s" onclick="openComponentDetail('fan')" `+
`style="width:%dpx;height:%dpx;display:flex;align-items:center;justify-content:center;`+
`border-radius:5px;background:%s;border:1px solid %s;color:%s;cursor:pointer">%s</div>`,
html.EscapeString(title), side, side, fill, stroke, text, spin)
}
b.WriteString(`</div>`)
fmt.Fprintf(&b, `<div style="font-size:11px;color:var(--muted);margin-top:6px">%d fans · %s · tile size ∝ RPM / observed max</div>`,
len(fans), html.EscapeString(tally.line()))
return b.String()
}
// topoFanSpinStyle emits the keyframes + base class for the spinning fan
// glyph once per row. A repeated identical <style> is harmless.
func topoFanSpinStyle() string {
return `<style>@keyframes topoFanSpin{to{transform:rotate(360deg)}}` +
`.topo-fan-spin{transform-box:fill-box;transform-origin:center;` +
`animation-name:topoFanSpin;animation-timing-function:linear;animation-iteration-count:infinite}` +
`@media (prefers-reduced-motion:reduce){.topo-fan-spin{animation:none}}</style>`
}
// topoFanGlyphPaths is the fan-blade drawing shared by every fan square,
// designed on a 24×24 viewBox.
func topoFanGlyphPaths() string {
return `<ellipse cx="12" cy="6.5" rx="3.1" ry="5.2"/>` +
`<ellipse cx="12" cy="6.5" rx="3.1" ry="5.2" transform="rotate(120 12 12)"/>` +
`<ellipse cx="12" cy="6.5" rx="3.1" ry="5.2" transform="rotate(240 12 12)"/>` +
`<circle cx="12" cy="12" r="2.3"/>`
}
// dedupeFansByName returns the fan sensors from a snapshot with duplicate
// names collapsed to their first occurrence, matching the ingest contract's
// "(sensor_type, name) — first wins" rule and skipping unnamed sensors.
func dedupeFansByName(sensors *schema.HardwareSensors) []schema.HardwareFanSensor {
if sensors == nil {
return nil
}
seen := map[string]bool{}
var out []schema.HardwareFanSensor
for _, f := range sensors.Fans {
name := strings.TrimSpace(f.Name)
if name == "" || seen[name] {
continue
}
seen[name] = true
out = append(out, f)
}
return out
}
// topoRowHeading renders the small uppercase section label shared by the
// flex rows below the SVG diagram (Firmware / Power Supplies / Cooling / ...).
func topoRowHeading(title string) string {
return fmt.Sprintf(`<div style="font-size:11px;color:var(--muted);text-transform:uppercase;letter-spacing:.05em;margin:16px 0 6px">%s</div>`,
html.EscapeString(title))
}
// renderTopoFlexRow renders a labeled, wrapping row of component cards.
// Returns "" if items is empty (e.g. no PSU data in this audit).
func renderTopoFlexRow(title string, items []topoCardInfo) string {
@@ -624,8 +753,7 @@ func renderTopoFlexRow(title string, items []topoCardInfo) string {
return ""
}
var b strings.Builder
fmt.Fprintf(&b, `<div style="font-size:11px;color:var(--muted);text-transform:uppercase;letter-spacing:.05em;margin:16px 0 6px">%s</div>`,
html.EscapeString(title))
b.WriteString(topoRowHeading(title))
b.WriteString(`<div style="display:flex;flex-wrap:wrap;gap:10px">`)
for _, item := range items {
onclick := ""
@@ -1064,6 +1192,15 @@ func inventoryFallbackRecords(compType string, opts HandlerOptions) []app.Compon
}
records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(p.Status)})
}
case "fan":
for i, f := range dedupeFansByName(hw.Sensors) {
name := strings.TrimSpace(f.Name)
key := fmt.Sprintf("fan:%d", i)
if name != "" {
key = "fan:" + name
}
records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(f.Status)})
}
case "gpu", "nic", "raid":
for i, dev := range hw.PCIeDevices {
if pcieDeviceKind(dev) != compType {
+65
View File
@@ -152,6 +152,71 @@ func TestTopoPageRendersArbitraryPSUAndFirmwareCountsAsFlexRows(t *testing.T) {
}
}
func TestTopoPageRendersCoolingFansAsFlexRow(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "audit.json")
ok, warn := "OK", "Warning"
rpm := func(v int) *int { return &v }
ingest := schema.HardwareIngestRequest{
CollectedAt: "2026-03-15T00:00:00Z",
Hardware: schema.HardwareSnapshot{
Sensors: &schema.HardwareSensors{
Fans: []schema.HardwareFanSensor{
{Name: "FAN1", RPM: rpm(4200), Status: &ok},
{Name: "FAN2", RPM: rpm(15000), Status: &warn},
{Name: "FAN2", RPM: rpm(15000), Status: &warn}, // dup name, first wins
},
},
},
}
data, err := json.Marshal(ingest)
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, data, 0644); err != nil {
t.Fatal(err)
}
handler := NewHandler(HandlerOptions{AuditPath: path})
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/topo", nil))
if rec.Code != http.StatusOK {
t.Fatalf("status=%d", rec.Code)
}
body := rec.Body.String()
// One clickable square per fan (2 after dedup by name), each with a
// spinning glyph, under a COOLING heading.
if !strings.Contains(body, "Cooling") {
t.Fatalf("topo page missing Cooling heading: %s", body)
}
if n := strings.Count(body, `onclick="openComponentDetail('fan')"`) +
strings.Count(body, `onclick="openComponentDetail(&#39;fan&#39;)"`); n != 2 {
t.Fatalf("expected one clickable square per fan (2), got %d: %s", n, body)
}
if n := strings.Count(body, `class="topo-fan-spin"`); n != 2 {
t.Fatalf("expected 2 spinning fan glyphs, got %d: %s", n, body)
}
if !strings.Contains(body, "FAN1 · 4200 RPM") || !strings.Contains(body, "FAN2 · 15000 RPM") {
t.Fatalf("topo page missing per-fan RPM tooltips: %s", body)
}
if !strings.Contains(body, "2 fans") {
t.Fatalf("topo page missing fan count caption: %s", body)
}
// Component-detail fallback endpoint must resolve the "fan" type.
rec2 := httptest.NewRecorder()
handler.ServeHTTP(rec2, httptest.NewRequest(http.MethodGet, "/api/components/fan", nil))
if rec2.Code != http.StatusOK {
t.Fatalf("/api/components/fan status=%d", rec2.Code)
}
if b := rec2.Body.String(); !strings.Contains(b, "FAN1") || !strings.Contains(b, "FAN2") {
t.Fatalf("fan component detail missing fan names: %s", b)
}
}
func TestTopoPageRendersStorageDisksGroupedByType(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "audit.json")
+6
View File
@@ -114,6 +114,12 @@ func renderValidateMode(opts HandlerOptions, stressDefault bool) string {
`Tests power supply transient response by pulsing all GPUs simultaneously between idle and full load. Synchronous pulses across all GPUs create worst-case PSU load spikes — running per-GPU would miss PSU-level failures.`,
`<code>dcgmi diag pulse_test</code>`,
validateFmtDur(platform.SATEstimatedNvidiaPulseTestSec)+` (all GPUs simultaneously; measured on 8-GPU system).`,
)) +
renderSATCard("fan", "Fan Ceiling Check", "runSAT('fan')", "", renderValidateCardBody(
"All system fans reported over IPMI / lm-sensors.",
`Drives CPU (+memory) and, when a GPU is present, GPU load to 100% at the same time and watches every fan until none has climbed for ~1 min. The peak RPM reached is recorded as each fan's ceiling and is what the Topology view sizes the fan tiles against. Success once every fan plateaus (or the time cap is hit). A fan reading 0 RPM or an IPMI status of cr/nr under full load fails. If the host cannot be loaded at all, or exposes no fan sensors, the task is cancelled as "not applicable" rather than failed — the platform does not support forcing fans, so this is the closest safe equivalent.`,
`<code>stressapptest</code> / <code>stress-ng</code> + <code>bee-gpu-burn</code> / <code>rvs gst</code>; <code>ipmitool sdr type Fan</code>`,
`~28 min depending on how fast the fan curve settles (hard cap 15 min).`,
))
}
+1
View File
@@ -272,6 +272,7 @@ func NewHandler(opts HandlerOptions) http.Handler {
mux.HandleFunc("POST /api/sat/memory-stress/run", h.handleAPISATRun("memory-stress"))
mux.HandleFunc("POST /api/sat/sat-stress/run", h.handleAPISATRun("sat-stress"))
mux.HandleFunc("POST /api/sat/platform-stress/run", h.handleAPISATRun("platform-stress"))
mux.HandleFunc("POST /api/sat/fan/run", h.handleAPISATRun("fan"))
mux.HandleFunc("POST /api/sat/run-all", h.handleAPISATRunAll)
mux.HandleFunc("GET /api/sat/stream", h.handleAPISATStream)
mux.HandleFunc("POST /api/sat/abort", h.handleAPISATAbort)
+22 -4
View File
@@ -3,6 +3,7 @@ package webui
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
@@ -79,9 +80,11 @@ func finalizeTaskForResult(t *Task, errMsg string, cancelled bool) {
now := time.Now()
t.DoneAt = &now
switch {
case cancelled:
case cancelled || t.Status == TaskCancelled:
t.Status = TaskCancelled
t.ErrMsg = "aborted"
if strings.TrimSpace(t.ErrMsg) == "" {
t.ErrMsg = "aborted"
}
case strings.TrimSpace(errMsg) != "":
t.Status = TaskFailed
t.ErrMsg = errMsg
@@ -377,6 +380,12 @@ func executeTaskWithOptions(opts *HandlerOptions, t *Task, j *jobState, ctx cont
runOpts := resolvePlatformStressPreset(t.params.BurnProfile)
runOpts.Components = t.params.PlatformComponents
archive, err = a.RunPlatformStress(ctx, "", runOpts, j.append)
case "fan":
if a == nil {
err = fmt.Errorf("app not configured")
break
}
archive, err = runFanCheckPackCtx(a, ctx, "", platform.FanCheckOptions{GPUIndices: t.params.GPUIndices}, j.append)
case "audit":
if a == nil {
err = fmt.Errorf("app not configured")
@@ -476,10 +485,19 @@ func executeTaskWithOptions(opts *HandlerOptions, t *Task, j *jobState, ctx cont
}
if err != nil {
if ctx.Err() != nil {
switch {
case ctx.Err() != nil:
j.append("Aborted.")
j.finish("aborted")
} else {
case errors.Is(err, platform.ErrTestNotApplicable):
// The host offered no way to run this test — not a hardware
// fault. Land the task as cancelled ("not applicable") with a
// detailed log, so an engineer never sees a false failure.
j.append("NOT APPLICABLE: " + err.Error())
t.Status = TaskCancelled
t.ErrMsg = "not applicable — " + err.Error()
j.finish("")
default:
j.append("ERROR: " + err.Error())
j.finish(err.Error())
}
+5 -1
View File
@@ -53,6 +53,7 @@ var taskNames = map[string]string{
"memory-stress": "Memory Burn-in",
"sat-stress": "SAT Stress (stressapptest)",
"platform-stress": "Platform Thermal Cycling",
"fan": "Fan Ceiling Check (CPU+GPU load)",
"audit": "Audit",
"support-bundle": "Support Bundle",
"install": "Install to Disk",
@@ -196,7 +197,7 @@ func taskMayLeaveOrphanWorkers(target string) bool {
switch strings.TrimSpace(strings.ToLower(target)) {
case "nvidia", "nvidia-targeted-stress", "nvidia-targeted-power", "nvidia-pulse",
"nvidia-bandwidth", "nvidia-stress", "nvidia-compute", "nvidia-bench-perf",
"memory", "memory-stress", "cpu", "sat-stress", "platform-stress":
"memory", "memory-stress", "cpu", "sat-stress", "platform-stress", "fan":
return true
default:
return false
@@ -343,6 +344,9 @@ var (
runSATStressPackCtx = func(a *app.App, ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
return a.RunSATStressPackCtx(ctx, baseDir, durationSec, logFunc)
}
runFanCheckPackCtx = func(a *app.App, ctx context.Context, baseDir string, opts platform.FanCheckOptions, logFunc func(string)) (string, error) {
return a.RunFanCheckCtx(ctx, baseDir, opts, logFunc)
}
buildSupportBundle = app.BuildSupportBundle
installCommand = func(ctx context.Context, device string, logPath string) *exec.Cmd {
return exec.CommandContext(ctx, "bee-install", device, logPath)
+29
View File
@@ -3,6 +3,7 @@ package webui
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
@@ -700,6 +701,34 @@ func TestRunTaskHonorsCancel(t *testing.T) {
}
}
func TestRunTaskFanNotApplicableIsCancelledNotFailed(t *testing.T) {
q := &taskQueue{opts: &HandlerOptions{App: &app.App{}}}
tk := &Task{
ID: "fan-1", Name: "Fan Ceiling Check", Target: "fan",
Status: TaskRunning, CreatedAt: time.Now(),
}
j := &jobState{}
tk.job = j
orig := runFanCheckPackCtx
runFanCheckPackCtx = func(_ *app.App, _ context.Context, _ string, _ platform.FanCheckOptions, _ func(string)) (string, error) {
return "", fmt.Errorf("no fan sensors readable: %w", platform.ErrTestNotApplicable)
}
defer func() { runFanCheckPackCtx = orig }()
q.runTask(tk, j, context.Background())
if tk.Status != TaskCancelled {
t.Fatalf("status=%q want %q", tk.Status, TaskCancelled)
}
if j.err != "" {
t.Fatalf("job err should be empty for a not-applicable task, got %q", j.err)
}
if !strings.Contains(tk.ErrMsg, "not applicable") {
t.Fatalf("ErrMsg should explain not-applicable, got %q", tk.ErrMsg)
}
}
func TestRunTaskUsesBurnProfileDurationForCPU(t *testing.T) {
var gotDuration int
q := &taskQueue{