Merge feat/fan-ceiling-check: fan ceiling check + topology fan/PSU tiles

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-09-04 14:34:22 +03:00
co-authored by Claude Sonnet 5
31 changed files with 1673 additions and 593 deletions
+1 -1
View File
@@ -173,7 +173,7 @@ type satRunner interface {
RunAMDStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error)
RunMemoryStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error)
RunSATStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error)
RunFanStressTest(ctx context.Context, baseDir string, opts platform.FanStressOptions) (string, error)
RunFanCheck(ctx context.Context, baseDir string, opts platform.FanCheckOptions, logFunc func(string)) (string, error)
RunPlatformStress(ctx context.Context, baseDir string, opts platform.PlatformStressOptions, logFunc func(string)) (string, error)
RunNCCLTests(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error)
RunScenario(ctx context.Context, baseDir string, spec platform.ScenarioSpec, logFunc func(string)) (string, error)
+7
View File
@@ -298,6 +298,13 @@ func (a *App) RunPlatformStress(ctx context.Context, baseDir string, opts platfo
return a.sat.RunPlatformStress(ctx, baseDir, opts, logFunc)
}
func (a *App) RunFanCheckCtx(ctx context.Context, baseDir string, opts platform.FanCheckOptions, logFunc func(string)) (string, error) {
if strings.TrimSpace(baseDir) == "" {
baseDir = DefaultSATBaseDir
}
return a.sat.RunFanCheck(ctx, baseDir, opts, logFunc)
}
func (a *App) RunNCCLTests(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) {
if strings.TrimSpace(baseDir) == "" {
baseDir = DefaultSATBaseDir
+1 -1
View File
@@ -367,7 +367,7 @@ func (f fakeSAT) RunSATStressPack(_ context.Context, _ string, _ int, _ func(str
return "", nil
}
func (f fakeSAT) RunFanStressTest(_ context.Context, _ string, _ platform.FanStressOptions) (string, error) {
func (f fakeSAT) RunFanCheck(_ context.Context, _ string, _ platform.FanCheckOptions, _ func(string)) (string, error) {
return "", nil
}
+17
View File
@@ -318,6 +318,23 @@ func ApplySATResultToDB(db *ComponentStatusDB, target, archivePath string) {
db.Record("memory:all", source, dbStatus, detail)
case "cpu", "platform-stress":
db.Record("cpu:all", source, dbStatus, detail)
case "fan":
// Per-fan keys: summary emits "fan_<name>_status=OK|FAILED (...)".
for key, val := range kv {
name, ok := strings.CutPrefix(key, "fan_")
if !ok {
continue
}
name, ok = strings.CutSuffix(name, "_status")
if !ok || name == "" {
continue
}
upper := strings.ToUpper(strings.TrimSpace(val))
if i := strings.IndexByte(upper, ' '); i > 0 {
upper = upper[:i] // drop the "(reason)" suffix
}
db.Record("fan:"+name, source, satStatusToDBStatus(upper), target+" SAT: "+strings.TrimSpace(val))
}
case "storage":
// Try to record per-device if available in summary.
recordedAny := false
+4
View File
@@ -393,5 +393,9 @@ func samplePSUPower() []PSUReading {
if len(psus) == 0 {
return nil
}
// Feed the observed-capacity store (the "autotune" for PSU load scaling on
// BMCs that report only instantaneous power) — every load run that samples
// PSU power, including the 5 s metrics collector, refines it.
updatePSUObservation(psus, time.Now())
return psus
}
+183
View File
@@ -0,0 +1,183 @@
package platform
import (
"encoding/json"
"math"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
)
// observedPeakStore is the "autotune" primitive for components that expose no
// host-readable nameplate maximum: fan top RPM, PSU capacity. It records the
// highest value seen per key while the box is under load, persists it to a
// JSON file, and hands it back so live readings can be scaled against a real
// maximum. A new peak only sticks after it has been held for at least
// minHold, which rejects transient spikes.
type observedPeakStore struct {
path string // JSON file
jsonKey string // top-level object key, e.g. "max_rpm"
roundUp float64 // round a new peak up to this multiple; 0 = keep raw
minHold time.Duration // a candidate peak must persist this long to stick
mu sync.Mutex
loaded bool
peaks map[string]float64
candidates map[string]peakCandidate
}
type peakCandidate struct {
firstSeen time.Time
val float64
}
// persistedPeaks reads the file fresh (no lock, no cache mutation) and returns
// its sanitized {key -> peak} map. Empty map when the file is missing or
// unparsable.
func (s *observedPeakStore) persistedPeaks() map[string]float64 {
out := map[string]float64{}
raw, err := os.ReadFile(s.path)
if err != nil || len(raw) == 0 {
return out
}
var doc map[string]map[string]float64
if json.Unmarshal(raw, &doc) != nil {
return out
}
for k, v := range doc[s.jsonKey] {
k = strings.TrimSpace(k)
if k == "" || v <= 0 {
continue
}
out[k] = v
}
return out
}
func (s *observedPeakStore) loadLocked() {
if s.loaded {
return
}
s.loaded = true
s.peaks = s.persistedPeaks()
if s.candidates == nil {
s.candidates = map[string]peakCandidate{}
}
}
func (s *observedPeakStore) saveLocked() {
if len(s.peaks) == 0 {
return
}
dir := filepath.Dir(s.path)
if dir == "" || dir == "." {
return
}
if err := os.MkdirAll(dir, 0755); err != nil {
return
}
raw, err := json.MarshalIndent(map[string]map[string]float64{s.jsonKey: s.peaks}, "", " ")
if err != nil {
return
}
_ = os.WriteFile(s.path, raw, 0644)
}
func (s *observedPeakStore) round(v float64) float64 {
if v <= 0 {
return 0
}
if s.roundUp <= 0 {
return v
}
return math.Ceil(v/s.roundUp) * s.roundUp
}
// observe feeds one telemetry sample (key -> current value). Non-positive
// values and blank keys are ignored.
func (s *observedPeakStore) observe(samples map[string]float64, now time.Time) {
if len(samples) == 0 {
return
}
s.mu.Lock()
defer s.mu.Unlock()
s.loadLocked()
changed := false
for key, val := range samples {
key = strings.TrimSpace(key)
if key == "" || val <= 0 {
continue
}
cur := s.peaks[key]
if val <= cur {
delete(s.candidates, key)
continue
}
if cand, ok := s.candidates[key]; ok {
if now.Sub(cand.firstSeen) >= s.minHold {
nv := math.Max(cand.val, val)
if nv > cur {
s.peaks[key] = s.round(nv)
changed = true
}
delete(s.candidates, key)
continue
}
if val > cand.val {
s.candidates[key] = peakCandidate{firstSeen: cand.firstSeen, val: val}
}
continue
}
s.candidates[key] = peakCandidate{firstSeen: now, val: val}
}
if changed {
s.saveLocked()
}
}
// snapshot returns the persisted peaks (fresh from disk), for read-only
// consumers such as the /topo web view.
func (s *observedPeakStore) snapshot() map[string]float64 {
return s.persistedPeaks()
}
// ── PSU capacity ────────────────────────────────────────────────────────────
var psuPeaks = &observedPeakStore{
path: "/var/log/bee-sat/psu-observation.json",
jsonKey: "max_w",
roundUp: 50,
minHold: time.Second,
}
// updatePSUObservation feeds the current per-PSU draw (keyed by ordinal, in
// the order the caller lists them) into the observed-capacity store. On a BMC
// that reports only instantaneous input power this is the only way to know
// what "100% load" looks like for each supply: observe the peak draw during
// any full-load run (the Fan Ceiling Check, a burn, thermal cycling — the 5 s
// metrics collector samples PSUs throughout) and remember it.
func updatePSUObservation(psus []PSUReading, now time.Time) {
if len(psus) == 0 {
return
}
m := make(map[string]float64, len(psus))
for i, p := range psus {
if p.PowerW > 0 {
m[strconv.Itoa(i)] = p.PowerW
}
}
psuPeaks.observe(m, now)
}
// ObservedPSUMaxW returns the persisted per-PSU observed peak draw, keyed by
// ordinal ("0", "1", …), or nil if none recorded yet.
func ObservedPSUMaxW() map[string]float64 {
p := psuPeaks.snapshot()
if len(p) == 0 {
return nil
}
return p
}
+10
View File
@@ -0,0 +1,10 @@
package platform
import "errors"
// ErrTestNotApplicable is returned by a SAT routine when the host provides no
// way to run that test at all (a required tool or capability is absent), as
// opposed to the test running and finding a fault. The task layer maps it to a
// cancelled ("not applicable") task rather than a failure, so an engineer does
// not see a false red.
var ErrTestNotApplicable = errors.New("test not applicable on this platform")
File diff suppressed because it is too large Load Diff
+109 -19
View File
@@ -1,11 +1,116 @@
package platform
import (
"os"
"path/filepath"
"reflect"
"testing"
"time"
)
// resetPeakStore points a store at a fresh temp file and clears its cache for
// the duration of the test.
func resetPeakStore(t *testing.T, s *observedPeakStore) {
t.Helper()
old := *s
s.path = filepath.Join(t.TempDir(), "peaks.json")
s.loaded = false
s.peaks = nil
s.candidates = nil
t.Cleanup(func() { *s = old })
}
func TestResolveFanMaxRPM(t *testing.T) {
resetPeakStore(t, fanPeaks)
// No persisted file yet: unknown fans fall back to their own current RPM.
got := ResolveFanMaxRPM(map[string]float64{"A": 4000, "B": 9000})
if !reflect.DeepEqual(got, map[string]float64{"A": 4000, "B": 9000}) {
t.Fatalf("no-persist fallback: got %v", got)
}
if err := os.WriteFile(fanPeaks.path, []byte(`{"max_rpm":{"A":17000}}`), 0644); err != nil {
t.Fatal(err)
}
got = ResolveFanMaxRPM(map[string]float64{"A": 4000, "B": 9000, "C": 5000})
// A: persisted peak. B/C: no own entry -> largest peer peak (A's 17000).
if want := map[string]float64{"A": 17000, "B": 17000, "C": 17000}; !reflect.DeepEqual(got, want) {
t.Fatalf("peer fallback: got %v want %v", got, want)
}
}
func TestObservedPeakStore(t *testing.T) {
s := &observedPeakStore{
path: filepath.Join(t.TempDir(), "peaks.json"),
jsonKey: "max_w",
roundUp: 50,
minHold: time.Second,
}
t0 := time.Unix(0, 0)
// A single spike does not stick.
s.observe(map[string]float64{"0": 900}, t0)
if len(s.snapshot()) != 0 {
t.Fatalf("transient spike should not persist: %v", s.snapshot())
}
// Held past minHold → sticks, rounded up to the next 50.
s.observe(map[string]float64{"0": 920}, t0.Add(1200*time.Millisecond))
if got := s.snapshot()["0"]; got != 950 {
t.Fatalf("held peak: got %v want 950", got)
}
// A lower reading never lowers the peak.
s.observe(map[string]float64{"0": 400}, t0.Add(5*time.Second))
if got := s.snapshot()["0"]; got != 950 {
t.Fatalf("peak must not drop: got %v", got)
}
// Fresh store reloads from disk.
s2 := &observedPeakStore{path: s.path, jsonKey: "max_w"}
if got := s2.snapshot()["0"]; got != 950 {
t.Fatalf("reload from disk: got %v want 950", got)
}
}
func TestUpdatePSUObservationKeyedByOrdinal(t *testing.T) {
resetPeakStore(t, psuPeaks)
now := time.Unix(0, 0)
psus := []PSUReading{{Name: "PSU1", PowerW: 1200}, {Name: "PSU2", PowerW: 1400}}
updatePSUObservation(psus, now)
updatePSUObservation(psus, now.Add(1200*time.Millisecond))
got := ObservedPSUMaxW()
if got["0"] != 1200 || got["1"] != 1400 {
t.Fatalf("keyed-by-ordinal peaks: got %v", got)
}
}
func TestApplyFanCheckDefaults(t *testing.T) {
var o FanCheckOptions
applyFanCheckDefaults(&o)
if o.PlateauHoldSec != 60 || o.PlateauDeltaRPM != 50 || o.MinLoadSec != 90 || o.MaxLoadSec != 900 || o.RampConfirmRPM != 150 {
t.Fatalf("unexpected defaults: %+v", o)
}
o = FanCheckOptions{PlateauHoldSec: 120, MinLoadSec: 30, MaxLoadSec: 40}
applyFanCheckDefaults(&o)
if o.MinLoadSec < o.PlateauHoldSec {
t.Fatalf("MinLoadSec must be >= PlateauHoldSec, got %d", o.MinLoadSec)
}
if o.MaxLoadSec <= o.MinLoadSec {
t.Fatalf("MaxLoadSec must exceed MinLoadSec, got %d", o.MaxLoadSec)
}
}
func TestSanitizeSummaryKey(t *testing.T) {
for in, want := range map[string]string{
"F2U-1": "F2U-1",
"aspeed / fan1": "aspeed___fan1",
"CPU0_DIMM": "CPU0_DIMM",
"weird=key here": "weird_key_here",
} {
if got := sanitizeSummaryKey(in); got != want {
t.Errorf("sanitizeSummaryKey(%q)=%q want %q", in, got, want)
}
}
}
func TestParseFanSpeeds(t *testing.T) {
raw := "FAN1 | 2400.000 | RPM | ok\nFAN2 | 1800 RPM | ok | ok\nFAN3 | na | RPM | ns\n"
got := parseFanSpeeds(raw)
@@ -52,22 +157,7 @@ func TestParseFanDutyCyclePctSensorsJSON(t *testing.T) {
}
func TestEstimateFanDutyCyclePctFromObservation(t *testing.T) {
t.Parallel()
oldPath := fanObservationStatePath
oldState := fanObservation
oldInit := fanObservationInit
oldCandidates := fanPeakCandidates
fanObservationStatePath = filepath.Join(t.TempDir(), "fan-observation.json")
fanObservation = fanObservationState{}
fanObservationInit = false
fanPeakCandidates = make(map[string]fanPeakCandidate)
t.Cleanup(func() {
fanObservationStatePath = oldPath
fanObservation = oldState
fanObservationInit = oldInit
fanPeakCandidates = oldCandidates
})
resetPeakStore(t, fanPeaks)
start := time.Unix(100, 0)
updateFanObservation([]FanReading{{Name: "FAN1", RPM: 5000}}, start)
@@ -86,9 +176,9 @@ func TestEstimateFanDutyCyclePctFromObservation(t *testing.T) {
t.Fatalf("got=%v want ~43.3", got)
}
fanObservation = fanObservationState{}
fanObservationInit = false
fanPeakCandidates = make(map[string]fanPeakCandidate)
fanPeaks.loaded = false
fanPeaks.peaks = nil
fanPeaks.candidates = nil
got, ok = estimateFanDutyCyclePctFromObservation([]FanReading{{Name: "FAN1", RPM: 2600}})
if !ok {
t.Fatalf("expected persisted observed max to be reloaded from disk")
+12 -3
View File
@@ -14,6 +14,7 @@ type NvidiaGPU struct {
Index int `json:"index"`
Name string `json:"name"`
MemoryMB int `json:"memory_mb"`
Serial string `json:"serial,omitempty"`
}
type NvidiaGPUStatus struct {
@@ -226,7 +227,7 @@ func amdStressJobs(seconds int, cfgFile string) []satJob {
// ListNvidiaGPUs returns GPUs visible to nvidia-smi.
func (s *System) ListNvidiaGPUs() ([]NvidiaGPU, error) {
out, err := exec.Command("nvidia-smi",
"--query-gpu=index,name,memory.total",
"--query-gpu=index,name,memory.total,serial",
"--format=csv,noheader,nounits").Output()
if err != nil {
return nil, fmt.Errorf("nvidia-smi: %w", err)
@@ -237,8 +238,8 @@ func (s *System) ListNvidiaGPUs() ([]NvidiaGPU, error) {
if line == "" {
continue
}
parts := strings.SplitN(line, ", ", 3)
if len(parts) != 3 {
parts := strings.SplitN(line, ", ", 4)
if len(parts) < 3 {
continue
}
idx, err := strconv.Atoi(strings.TrimSpace(parts[0]))
@@ -246,10 +247,18 @@ func (s *System) ListNvidiaGPUs() ([]NvidiaGPU, error) {
continue
}
memMB, _ := strconv.Atoi(strings.TrimSpace(parts[2]))
serial := ""
if len(parts) == 4 {
serial = strings.TrimSpace(parts[3])
if strings.EqualFold(serial, "N/A") || strings.EqualFold(serial, "[N/A]") {
serial = ""
}
}
gpus = append(gpus, NvidiaGPU{
Index: idx,
Name: strings.TrimSpace(parts[1]),
MemoryMB: memMB,
Serial: serial,
})
}
sort.Slice(gpus, func(i, j int) bool {
+1 -1
View File
@@ -128,7 +128,7 @@ func defaultTaskPriority(target string, params taskParams) int {
return taskPriorityAudit
case "nvidia-bench-perf", "nvidia-bench-power", "nvidia-bench-autotune":
return taskPriorityBenchmark
case "nvidia-stress", "amd-stress", "memory-stress", "sat-stress", "platform-stress", "nvidia-compute", "scenario":
case "nvidia-stress", "amd-stress", "memory-stress", "sat-stress", "platform-stress", "fan", "nvidia-compute", "scenario":
return taskPriorityBurn
case "nvidia", "nvidia-targeted-stress", "nvidia-targeted-power", "nvidia-pulse",
"nvidia-interconnect", "nvidia-bandwidth", "memory", "storage", "cpu",
+4 -1
View File
@@ -289,7 +289,7 @@ func (h *handler) handleAPIHardwareSummary(w http.ResponseWriter, _ *http.Reques
}
// handleAPIComponentDetail returns an HTML fragment describing the current and
// historical status for one component type (cpu, memory, storage, gpu, psu).
// historical status for one component type (cpu, memory, storage, gpu, psu, fan).
func (h *handler) handleAPIComponentDetail(w http.ResponseWriter, r *http.Request) {
compType := r.PathValue("type")
var exact, prefixes []string
@@ -315,6 +315,9 @@ func (h *handler) handleAPIComponentDetail(w http.ResponseWriter, r *http.Reques
case "psu":
title = "PSU"
prefixes = []string{"psu:"}
case "fan":
title = "Fans"
prefixes = []string{"fan:"}
case "raid":
title = "RAID"
prefixes = []string{"pcie:raid:"}
+5
View File
@@ -109,6 +109,11 @@ func (h *handler) planSATRunAll(ctx context.Context, req satRunAllRequest) ([]sa
} else {
skip("TPM: no TPM device on this host; check skipped")
}
} else {
// Fan ceiling check runs on the Load tier only. It self-cancels as
// "not applicable" on a host with no fan sensors or no way to load
// the CPU/GPU, so it is safe to queue unconditionally here.
specs = append(specs, satRunAllSpec{target: "fan", params: taskParams{StressMode: true}})
}
gp := h.opts.App.DetectGPUPresence()
+1 -1
View File
@@ -98,7 +98,7 @@ func TestPlanSATRunAllLoadOmitsReadOnlyTPMCheck(t *testing.T) {
for _, s := range specs {
targets = append(targets, s.target)
}
if want := []string{"cpu", "memory", "storage"}; !reflect.DeepEqual(targets, want) {
if want := []string{"cpu", "memory", "storage", "fan"}; !reflect.DeepEqual(targets, want) {
t.Fatalf("targets=%v want %v", targets, want)
}
for _, note := range notes {
+71
View File
@@ -0,0 +1,71 @@
package webui
// Shared NVIDIA GPU selection picker.
//
// Every page that lets the operator pick GPUs (Load, Burn, Benchmark) renders
// the same row markup: "GPU N — <model> · <memory> MiB · sn: <serial>". That
// markup lives here once. Pages keep their own selection-note text, multi-GPU
// mode toggles and CSS-class prefixes, but call beeGpuPicker.render({...}) from
// their *RenderGPUList wrapper instead of hand-building each <label>.
//
// The serial number is rendered monospace, with the digits that differ across
// the listed GPUs (usually a run at the end, sometimes the middle or start)
// emphasised so operators can tell cards apart at a glance.
//
// gpuPickerCSS is injected once by layoutHead; gpuPickerJS once by renderPage.
const gpuPickerCSS = `.bee-gpu-row{display:flex;align-items:flex-start;gap:8px;padding:6px 0;cursor:pointer;font-size:13px}
.bee-gpu-row input[type=checkbox]{width:16px;height:16px;margin-top:2px;flex-shrink:0}
.bee-gpu-sn{font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;font-size:12px}
.bee-gpu-sn b{font-weight:700;color:var(--accent)}`
const gpuPickerJS = `window.beeGpuPicker={
_esc:function(s){return String(s).replace(/[&<>"]/g,function(c){return {'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;'}[c];});},
_commonPrefix:function(arr){
arr=arr.filter(Boolean);
if(arr.length<2)return 0;
var n=0,a=arr[0];
while(n<a.length&&arr.every(function(s){return s.charAt(n)===a.charAt(n);}))n++;
return n;
},
_commonSuffix:function(arr,capLeft){
arr=arr.filter(Boolean);
if(arr.length<2)return 0;
var minLen=Math.min.apply(null,arr.map(function(s){return s.length;}));
var n=0,a=arr[0];
while(n<a.length&&capLeft+n<minLen&&arr.every(function(s){return s.charAt(s.length-1-n)===a.charAt(a.length-1-n);}))n++;
return n;
},
_fmtSerial:function(s,p,q){
s=String(s);
if(p+q>=s.length){p=0;q=0;}
if(p===0&&q===0)return this._esc(s); // nothing distinguishing to highlight
return this._esc(s.slice(0,p))+'<b>'+this._esc(s.slice(p,s.length-q))+'</b>'+this._esc(s.slice(s.length-q));
},
row:function(gpu,checkboxClass,onToggle,prefixLen,suffixLen){
var mem=gpu.memory_mb>0?' · '+gpu.memory_mb+' MiB':'';
var name=this._esc(gpu.name||('GPU '+gpu.index));
var sn=gpu.serial?' · <span class="bee-gpu-sn">sn: '+this._fmtSerial(gpu.serial,prefixLen||0,suffixLen||0)+'</span>':'';
return '<label class="bee-gpu-row">'
+'<input class="'+checkboxClass+'" type="checkbox" value="'+gpu.index+'" checked'
+(onToggle?' onchange="'+onToggle+'"':'')+'>'
+'<span><strong>GPU '+gpu.index+'</strong> '+name+mem+sn+'</span>'
+'</label>';
},
render:function(opts){
var root=document.getElementById(opts.rootId);
if(!root)return;
var gpus=opts.gpus||[];
if(!gpus.length){
root.innerHTML=opts.emptyHTML||'<p style="color:var(--muted);font-size:13px">No NVIDIA GPUs detected.</p>';
if(opts.after)opts.after(0);
return;
}
var serials=gpus.map(function(g){return g.serial||'';});
var p=this._commonPrefix(serials);
var q=this._commonSuffix(serials,p);
var self=this;
root.innerHTML=gpus.map(function(g){return self.row(g,opts.checkboxClass,opts.onToggle,p,q);}).join('');
if(opts.after)opts.after(gpus.length);
}
};`
+1 -1
View File
@@ -296,7 +296,7 @@ func isSATTarget(target string) bool {
case "nvidia", "nvidia-targeted-stress", "nvidia-bench-perf", "nvidia-bench-power", "nvidia-compute", "nvidia-targeted-power", "nvidia-pulse",
"nvidia-interconnect", "nvidia-bandwidth", "nvidia-stress", "memory", "memory-stress", "storage",
"cpu", "sat-stress", "amd", "amd-mem", "amd-bandwidth", "amd-stress",
"platform-stress":
"platform-stress", "fan":
return true
}
return false
+1
View File
@@ -91,6 +91,7 @@ tbody tr:hover td{background:rgba(0,0,0,.03)}
.alert-info{background:#dff0ff;border:1px solid #a9d4f5;color:#1e3a5f}
.alert-warn{background:var(--warn-bg);border:1px solid #c9ba9b;color:var(--warn-fg)}
.alert-crit{background:var(--crit-bg);border:1px solid var(--crit-border);color:var(--crit-fg);font-weight:700}
` + gpuPickerCSS + `
</style>
</head>
<body>
+5 -17
View File
@@ -97,8 +97,6 @@ func renderBenchmark(opts HandlerOptions) string {
<style>
.benchmark-cb-row { display:flex; align-items:flex-start; gap:8px; cursor:pointer; font-size:13px; }
.benchmark-cb-row input[type=checkbox] { width:16px; height:16px; margin-top:2px; flex-shrink:0; }
.benchmark-gpu-row { display:flex; align-items:flex-start; gap:8px; padding:6px 0; cursor:pointer; font-size:13px; }
.benchmark-gpu-row input[type=checkbox] { width:16px; height:16px; margin-top:2px; flex-shrink:0; }
</style>
<script>
@@ -142,21 +140,11 @@ function benchmarkUpdateSelectionNote() {
}
}
function benchmarkRenderGPUList(gpus) {
const root = document.getElementById('benchmark-gpu-list');
if (!gpus || !gpus.length) {
root.innerHTML = '<p style="color:var(--muted);font-size:13px">No NVIDIA GPUs detected.</p>';
benchmarkUpdateSelectionNote();
return;
}
root.innerHTML = gpus.map(function(gpu) {
const mem = gpu.memory_mb > 0 ? ' · ' + gpu.memory_mb + ' MiB' : '';
return '<label class="benchmark-gpu-row">'
+ '<input class="benchmark-gpu-checkbox" type="checkbox" value="' + gpu.index + '" checked onchange="benchmarkUpdateSelectionNote()">'
+ '<span><strong>GPU ' + gpu.index + '</strong> ' + gpu.name + mem + '</span>'
+ '</label>';
}).join('');
benchmarkApplyMultiGPUState(gpus.length);
benchmarkUpdateSelectionNote();
beeGpuPicker.render({
rootId: 'benchmark-gpu-list', gpus: gpus,
checkboxClass: 'benchmark-gpu-checkbox', onToggle: 'benchmarkUpdateSelectionNote()',
after: function(n) { benchmarkApplyMultiGPUState(n); benchmarkUpdateSelectionNote(); },
});
}
function benchmarkApplyMultiGPUState(gpuCount) {
var multiValues = ['parallel', 'ramp-up'];
+5 -17
View File
@@ -92,8 +92,6 @@ func renderBurn() string {
.cb-row input[type=checkbox]:disabled { opacity:0.4; cursor:not-allowed; }
.cb-row input[type=checkbox]:disabled ~ span { opacity:0.45; cursor:not-allowed; }
.cb-note { font-size:11px; color:var(--muted); font-style:italic; }
.burn-gpu-row { display:flex; align-items:flex-start; gap:8px; padding:6px 0; cursor:pointer; font-size:13px; }
.burn-gpu-row input[type=checkbox] { width:16px; height:16px; margin-top:2px; flex-shrink:0; }
.burn-profile-body { display:grid; grid-template-columns:1fr 1fr 1fr; gap:24px; align-items:stretch; }
.burn-profile-col { min-width:0; }
.burn-profile-action { display:flex; flex-direction:column; align-items:center; justify-content:flex-start; gap:8px; }
@@ -159,21 +157,11 @@ function burnUpdateSelectionNote() {
note.textContent = 'Selected NVIDIA GPUs: ' + selected.join(', ') + '. Official and custom NVIDIA tasks will use only these GPUs.';
}
function burnRenderGPUList(gpus) {
const root = document.getElementById('burn-gpu-list');
if (!gpus || !gpus.length) {
root.innerHTML = '<p style="color:var(--muted);font-size:13px">No NVIDIA GPUs detected.</p>';
burnUpdateSelectionNote();
return;
}
root.innerHTML = gpus.map(function(gpu) {
const mem = gpu.memory_mb > 0 ? ' · ' + gpu.memory_mb + ' MiB' : '';
return '<label class="burn-gpu-row">'
+ '<input class="burn-gpu-checkbox" type="checkbox" value="' + gpu.index + '" checked onchange="burnUpdateSelectionNote()">'
+ '<span><strong>GPU ' + gpu.index + '</strong> ' + gpu.name + mem + '</span>'
+ '</label>';
}).join('');
burnApplyMultiGPUState(gpus.length);
burnUpdateSelectionNote();
beeGpuPicker.render({
rootId: 'burn-gpu-list', gpus: gpus,
checkboxClass: 'burn-gpu-checkbox', onToggle: 'burnUpdateSelectionNote()',
after: function(n) { burnApplyMultiGPUState(n); burnUpdateSelectionNote(); },
});
}
function burnSelectAll() {
document.querySelectorAll('.burn-gpu-checkbox').forEach(function(el) { el.checked = true; });
+339 -19
View File
@@ -592,31 +592,343 @@ func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string
}
b.WriteString(renderTopoFlexRow("Firmware", firmwareItems))
if len(hw.PowerSupplies) > 0 {
var tally topoStatusTally
watt := 0
for _, psu := range hw.PowerSupplies {
tally.add(classifyTopoSeverity(psu.Status))
if psu.WattageW != nil {
watt = *psu.WattageW
hasPSU := len(hw.PowerSupplies) > 0
if hasPSU {
b.WriteString(renderTopoPSURow(hw.PowerSupplies, platform.ObservedPSUMaxW()))
}
// Cooling fans — one small clickable square per fan. Square SIZE encodes
// the fan's ceiling RPM (its class); the coloured FILL rising from the
// bottom encodes live duty cycle (current / ceiling).
fans := dedupeFansByName(hw.Sensors)
if len(fans) > 0 {
current := map[string]float64{}
for _, f := range fans {
if f.RPM != nil {
current[strings.TrimSpace(f.Name)] = float64(*f.RPM)
}
}
fill, stroke, text := topoSeverityColors(tally.worst())
sublabel := ""
if watt > 0 {
sublabel = fmt.Sprintf("%dW each", watt)
b.WriteString(renderTopoFanRow(fans, platform.ResolveFanMaxRPM(current), platform.ObservedFanMaxRPM()))
}
b.WriteString(renderTopoFlexRow("Power Supplies", []topoCardInfo{{
label: "Power Supplies", sublabel: sublabel, count: len(hw.PowerSupplies),
statusLine: tally.line(),
fillVar: fill, strokeVar: stroke, textVar: text,
detailType: "psu",
}}))
if hasPSU || len(fans) > 0 {
b.WriteString(topoLiveScript())
}
return topoCard("Topology", b.String())
}
// renderTopoPSURow renders the POWER SUPPLIES row: one card per PSU, coloured
// by that PSU's own status (a failed unit goes red on its own). The card shows
// input voltage and draw, and a load fill rising from the bottom — same idea
// as the fan duty-cycle fill. The load scale is the nameplate rating when the
// BMC reports it; otherwise it is the observed peak draw (observedMaxW, keyed
// by ordinal — the "autotune" recorded during any full-load run), and the
// figure is marked as an estimate.
func renderTopoPSURow(psus []schema.HardwarePowerSupply, observedMaxW map[string]float64) string {
var b strings.Builder
b.WriteString(topoRowHeading("Power Supplies"))
b.WriteString(`<div style="display:flex;flex-wrap:wrap;gap:8px">`)
for i, p := range psus {
sev := classifyTopoSeverity(p.Status)
_, stroke, text := topoSeverityColors(sev)
label := fmt.Sprintf("PSU %d", i)
if p.Slot != nil && strings.TrimSpace(*p.Slot) != "" {
label = strings.TrimSpace(*p.Slot)
}
draw, haveDraw := psuDrawW(p)
rating := 0
if p.WattageW != nil && *p.WattageW > 0 {
rating = *p.WattageW
}
// Load scale: true rating if known, else the observed peak draw — but
// only once that peak sits meaningfully above the current draw. Until
// a real full-load run has bumped it, the "peak" is just idle draw and
// a load % off it would be nonsense, so fall back to plain watts.
scaleMax := float64(rating)
scaleEstimate := false
if scaleMax <= 0 {
if m := observedMaxW[strconv.Itoa(i)]; m > 0 && (!haveDraw || m >= draw*1.25) {
scaleMax = m
scaleEstimate = true
}
}
var parts []string
if p.InputVoltage != nil && *p.InputVoltage > 0 {
parts = append(parts, fmt.Sprintf("%.0f V", *p.InputVoltage))
}
switch {
case haveDraw && scaleMax > 0 && scaleEstimate:
parts = append(parts, fmt.Sprintf("%.0f / ~%.0f W · ~%.0f%% load", draw, scaleMax, draw/scaleMax*100))
case haveDraw && scaleMax > 0:
parts = append(parts, fmt.Sprintf("%.0f / %d W · %.0f%% load", draw, rating, draw/scaleMax*100))
case haveDraw:
parts = append(parts, fmt.Sprintf("%.0f W", draw))
case rating > 0:
parts = append(parts, fmt.Sprintf("%d W rated", rating))
}
detail := strings.Join(parts, " · ")
statusWord := ""
if sev >= 2 {
statusWord = topoSeverityStatus(p.Status)
}
fillH := 0.0
if haveDraw && scaleMax > 0 {
fillH = draw / scaleMax * 100
if fillH < 0 {
fillH = 0
}
if fillH > 100 {
fillH = 100
}
}
voltAttr := ""
if p.InputVoltage != nil && *p.InputVoltage > 0 {
voltAttr = fmt.Sprintf("%.0f", *p.InputVoltage)
}
maxSrc := "rated"
if scaleEstimate {
maxSrc = "observed"
}
fmt.Fprintf(&b, `<div class="topo-psu-tile" data-psu="%d" data-psu-max="%.0f" data-psu-max-src="%s" data-psu-v="%s" onclick="openComponentDetail('psu')" `+
`style="position:relative;overflow:hidden;cursor:pointer;min-width:104px;padding:8px 11px;border-radius:6px;background:var(--surface-2);border:1px solid %s;color:%s">`,
i, scaleMax, maxSrc, voltAttr, stroke, text)
fmt.Fprintf(&b, `<div class="topo-psu-fill" style="position:absolute;left:0;right:0;bottom:0;height:%.0f%%;background:%s;opacity:.5;transition:height .8s linear"></div>`, fillH, stroke)
fmt.Fprintf(&b, `<div style="position:relative"><div style="font-size:13px;font-weight:700">%s</div>`, html.EscapeString(label))
fmt.Fprintf(&b, `<div class="topo-psu-detail" style="font-size:11px;opacity:.9;margin-top:2px">%s</div>`, html.EscapeString(detail))
if statusWord != "" {
fmt.Fprintf(&b, `<div style="font-size:10px;font-weight:700;margin-top:3px">%s</div>`, html.EscapeString(strings.ToUpper(statusWord)))
}
b.WriteString(`</div></div>`)
}
b.WriteString(`</div>`)
return b.String()
}
// psuDrawW returns the PSU's current power draw (measured output preferred,
// else measured input).
func psuDrawW(p schema.HardwarePowerSupply) (float64, bool) {
switch {
case p.OutputPowerW != nil && *p.OutputPowerW > 0:
return *p.OutputPowerW, true
case p.InputPowerW != nil && *p.InputPowerW > 0:
return *p.InputPowerW, true
default:
return 0, false
}
}
// renderTopoFanRow renders the COOLING row. ceilByName (from
// platform.ResolveFanMaxRPM) has a value for every fan and drives tile size.
// observedByName (from platform.ObservedFanMaxRPM) holds only ceilings that
// were actually measured under load — a fan present there gets a duty-cycle
// fill; one that isn't shows no fill (ceiling not measured yet).
func renderTopoFanRow(fans []schema.HardwareFanSensor, ceilByName, observedByName map[string]float64) string {
const (
fanTileMin = 34 // px, the smallest-ceiling fan
fanTileMax = 60 // px, the largest-ceiling fan
)
ceilMax := 0.0
for _, v := range ceilByName {
if v > ceilMax {
ceilMax = v
}
}
var b strings.Builder
b.WriteString(topoRowHeading("Cooling"))
b.WriteString(topoFanSpinStyle())
b.WriteString(`<div style="display:flex;flex-wrap:wrap;gap:6px;align-items:flex-end">`)
for _, f := range fans {
name := strings.TrimSpace(f.Name)
_, stroke, text := topoSeverityColors(classifyTopoSeverity(f.Status))
ceil := ceilByName[name]
sizeRatio := 1.0
if ceilMax > 0 && ceil > 0 {
sizeRatio = ceil / ceilMax
}
side := fanTileMin + int(float64(fanTileMax-fanTileMin)*sizeRatio+0.5)
glyphSz := side * 7 / 16
// Duty cycle: only when the ceiling was actually measured under load.
duty := -1.0
if _, measured := observedByName[name]; measured && ceil > 0 && f.RPM != nil {
duty = float64(*f.RPM) / ceil * 100
if duty < 0 {
duty = 0
}
if duty > 100 {
duty = 100
}
}
title := name
switch {
case f.RPM == nil:
title = name + " · no reading"
case duty >= 0:
title = fmt.Sprintf("%s · %d RPM · %.0f%% duty (ceiling %d)", name, *f.RPM, duty, int(ceil))
default:
title = fmt.Sprintf("%s · %d RPM · ceiling not measured — run Fan Ceiling Check", name, *f.RPM)
}
glyph := fmt.Sprintf(`<svg width="%d" height="%d" viewBox="0 0 24 24" fill="currentColor" style="opacity:.35" aria-hidden="true">`, glyphSz, glyphSz) + topoFanGlyphPaths() + `</svg>`
if f.RPM != nil && *f.RPM > 0 {
period := fanSpinPeriodSec(float64(*f.RPM))
glyph = fmt.Sprintf(`<svg class="topo-fan-spin" style="animation-duration:%.2fs" width="%d" height="%d" viewBox="0 0 24 24" fill="currentColor" aria-hidden="true">`,
period, glyphSz, glyphSz) + topoFanGlyphPaths() + `</svg>`
}
measured := 0
fillH := 0.0
if duty >= 0 {
measured = 1
fillH = duty
}
fillBar := fmt.Sprintf(`<div class="topo-fan-fill" style="position:absolute;left:0;right:0;bottom:0;height:%.0f%%;background:%s;opacity:.55;transition:height .8s linear"></div>`, fillH, stroke)
fmt.Fprintf(&b, `<div class="topo-fan-tile" data-fan="%s" data-ceil="%d" data-measured="%d" title="%s" onclick="openComponentDetail('fan')" `+
`style="position:relative;overflow:hidden;width:%dpx;height:%dpx;display:flex;align-items:center;justify-content:center;`+
`border-radius:5px;background:var(--surface-2);border:1px solid %s;color:%s;cursor:pointer">`+
`%s<span style="position:relative;display:flex">%s</span></div>`,
html.EscapeString(name), int(ceil), measured, html.EscapeString(title),
side, side, stroke, text, fillBar, glyph)
}
b.WriteString(`</div>`)
return b.String()
}
// topoLiveScript polls the already-collected live-metrics snapshot
// (/api/metrics/latest — served from memory, no BMC call) every 5s and
// refreshes the fan tiles (spin rate, duty fill, tooltip) and PSU tiles
// (wattage) in place. 5s is the metrics collector's own sampling period, so
// polling faster only re-reads identical numbers; the endpoint is a mutex
// read + small JSON, so this stays cheap with many viewers.
func topoLiveScript() string {
return `<script>(function(){
var fans=document.querySelectorAll('.topo-fan-tile');
var psus=document.querySelectorAll('.topo-psu-tile');
if(!fans.length&&!psus.length)return;
function period(rpm){var lo=1000,hi=13000,slow=2.2,fast=0.35;
if(rpm<=lo)return slow;if(rpm>=hi)return fast;
return slow+(rpm-lo)/(hi-lo)*(fast-slow);}
function tick(){
fetch('/api/metrics/latest',{cache:'no-store'}).then(function(r){return r.json();}).then(function(m){
if(!m)return;
if(m.fans){
var by={};m.fans.forEach(function(f){by[f.name]=f.rpm;});
fans.forEach(function(t){
var rpm=by[t.dataset.fan];if(rpm==null)return;
var ceil=parseFloat(t.dataset.ceil)||0;
var svg=t.querySelector('.topo-fan-spin');
if(svg&&rpm>0)svg.style.animationDuration=period(rpm).toFixed(2)+'s';
var meas=t.dataset.measured==='1'&&ceil>0;
var duty=meas?Math.max(0,Math.min(100,rpm/ceil*100)):-1;
if(meas){var fill=t.querySelector('.topo-fan-fill');if(fill)fill.style.height=duty.toFixed(0)+'%';}
t.title=t.dataset.fan+' · '+Math.round(rpm)+' RPM'+(meas?' · '+Math.round(duty)+'% duty (ceiling '+ceil+')':' · ceiling not measured run Fan Ceiling Check');
});
}
if(m.psus){
psus.forEach(function(t){
var p=m.psus[parseInt(t.dataset.psu,10)];if(!p)return;
var w=p.power_w||0,max=parseFloat(t.dataset.psuMax)||0,v=t.dataset.psuV;
var est=t.dataset.psuMaxSrc==='observed';
var parts=[];
if(v)parts.push(v+' V');
if(w>0&&max>0){
var pct=Math.round(w/max*100);
parts.push(est?Math.round(w)+' / ~'+Math.round(max)+' W · ~'+pct+'% load'
:Math.round(w)+' / '+Math.round(max)+' W · '+pct+'% load');
}else if(w>0)parts.push(Math.round(w)+' W');
else if(max>0)parts.push(Math.round(max)+' W rated');
var d=t.querySelector('.topo-psu-detail');if(d&&parts.length)d.textContent=parts.join(' · ');
if(w>0&&max>0){var f=t.querySelector('.topo-psu-fill');if(f)f.style.height=Math.max(0,Math.min(100,w/max*100)).toFixed(0)+'%';}
});
}
}).catch(function(){});
}
setInterval(tick,5000);tick();
})();</script>`
}
// fanSpinPeriodSec maps an absolute fan RPM to a CSS animation period (one
// full turn of the glyph, in seconds). The real period would be 60/RPM — a
// blur at any real fan speed — so it is compressed into a band the eye can
// actually read: at/below fanSpinRPMLo the glyph turns at its slowest still
// clearly-moving rate, at/above fanSpinRPMHi at the fastest rate past which
// faster is indistinguishable (and starts to stutter), linear in between.
func fanSpinPeriodSec(rpm float64) float64 {
const (
fanSpinRPMLo = 1000.0
fanSpinRPMHi = 13000.0
fanSpinSlowSec = 2.2
fanSpinFastSec = 0.35
)
switch {
case rpm <= fanSpinRPMLo:
return fanSpinSlowSec
case rpm >= fanSpinRPMHi:
return fanSpinFastSec
default:
t := (rpm - fanSpinRPMLo) / (fanSpinRPMHi - fanSpinRPMLo)
return fanSpinSlowSec + t*(fanSpinFastSec-fanSpinSlowSec)
}
}
// topoFanSpinStyle emits the keyframes + base class for the spinning fan
// glyph once per row. A repeated identical <style> is harmless.
func topoFanSpinStyle() string {
return `<style>@keyframes topoFanSpin{to{transform:rotate(360deg)}}` +
`.topo-fan-spin{transform-box:fill-box;transform-origin:center;` +
`animation-name:topoFanSpin;animation-timing-function:linear;animation-iteration-count:infinite}` +
`@media (prefers-reduced-motion:reduce){.topo-fan-spin{animation:none}}</style>`
}
// topoFanGlyphPaths is the fan-blade drawing shared by every fan square,
// designed on a 24×24 viewBox.
func topoFanGlyphPaths() string {
return `<ellipse cx="12" cy="6.5" rx="3.1" ry="5.2"/>` +
`<ellipse cx="12" cy="6.5" rx="3.1" ry="5.2" transform="rotate(120 12 12)"/>` +
`<ellipse cx="12" cy="6.5" rx="3.1" ry="5.2" transform="rotate(240 12 12)"/>` +
`<circle cx="12" cy="12" r="2.3"/>`
}
// dedupeFansByName returns the fan sensors from a snapshot with duplicate
// names collapsed to their first occurrence, matching the ingest contract's
// "(sensor_type, name) — first wins" rule and skipping unnamed sensors.
func dedupeFansByName(sensors *schema.HardwareSensors) []schema.HardwareFanSensor {
if sensors == nil {
return nil
}
seen := map[string]bool{}
var out []schema.HardwareFanSensor
for _, f := range sensors.Fans {
name := strings.TrimSpace(f.Name)
if name == "" || seen[name] {
continue
}
seen[name] = true
out = append(out, f)
}
return out
}
// topoRowHeading renders the small uppercase section label shared by the
// flex rows below the SVG diagram (Firmware / Power Supplies / Cooling / ...).
func topoRowHeading(title string) string {
return fmt.Sprintf(`<div style="font-size:11px;color:var(--muted);text-transform:uppercase;letter-spacing:.05em;margin:16px 0 6px">%s</div>`,
html.EscapeString(title))
}
// renderTopoFlexRow renders a labeled, wrapping row of component cards.
// Returns "" if items is empty (e.g. no PSU data in this audit).
func renderTopoFlexRow(title string, items []topoCardInfo) string {
@@ -624,8 +936,7 @@ func renderTopoFlexRow(title string, items []topoCardInfo) string {
return ""
}
var b strings.Builder
fmt.Fprintf(&b, `<div style="font-size:11px;color:var(--muted);text-transform:uppercase;letter-spacing:.05em;margin:16px 0 6px">%s</div>`,
html.EscapeString(title))
b.WriteString(topoRowHeading(title))
b.WriteString(`<div style="display:flex;flex-wrap:wrap;gap:10px">`)
for _, item := range items {
onclick := ""
@@ -1064,6 +1375,15 @@ func inventoryFallbackRecords(compType string, opts HandlerOptions) []app.Compon
}
records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(p.Status)})
}
case "fan":
for i, f := range dedupeFansByName(hw.Sensors) {
name := strings.TrimSpace(f.Name)
key := fmt.Sprintf("fan:%d", i)
if name != "" {
key = "fan:" + name
}
records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(f.Status)})
}
case "gpu", "nic", "raid":
for i, dev := range hw.PCIeDevices {
if pcieDeviceKind(dev) != compType {
+139 -8
View File
@@ -90,15 +90,25 @@ func TestTopoPageRendersArbitraryPSUAndFirmwareCountsAsFlexRows(t *testing.T) {
path := filepath.Join(dir, "audit.json")
okStatus := "OK"
failStatus := "Critical"
watt := 3000
volt := 230.0
draw := 1500.0
var psus []schema.HardwarePowerSupply
for i := 0; i < 6; i++ {
slot := strconv.Itoa(i)
st := okStatus
if i == 3 {
st = failStatus
}
d := draw
psus = append(psus, schema.HardwarePowerSupply{
HardwareComponentStatus: schema.HardwareComponentStatus{Status: &okStatus},
HardwareComponentStatus: schema.HardwareComponentStatus{Status: &st},
Slot: &slot,
WattageW: &watt,
InputVoltage: &volt,
InputPowerW: &d,
})
}
@@ -132,14 +142,24 @@ func TestTopoPageRendersArbitraryPSUAndFirmwareCountsAsFlexRows(t *testing.T) {
if !strings.Contains(body, "BIOS") || !strings.Contains(body, "BMC") {
t.Fatalf("topo page missing BIOS/BMC firmware boxes: %s", body)
}
// All 6 PSUs must be represented, grouped into one stacked card with a
// count rather than 6 separate boxes.
if !strings.Contains(body, "Power Supplies ×6") {
t.Fatalf("topo page missing grouped Power Supplies x6 card: %s", body)
// One card per PSU (6), each clickable, each showing voltage + power.
if n := strings.Count(body, `class="topo-psu-tile"`); n != 6 {
t.Fatalf("expected 6 per-PSU cards, got %d", n)
}
if strings.Count(body, `onclick="openComponentDetail(&#39;psu&#39;)"`) != 1 &&
strings.Count(body, `onclick="openComponentDetail('psu')"`) != 1 {
t.Fatalf("expected exactly one clickable PSU group card, not one per PSU: %s", body)
if n := strings.Count(body, `onclick="openComponentDetail('psu')"`) +
strings.Count(body, `onclick="openComponentDetail(&#39;psu&#39;)"`); n != 6 {
t.Fatalf("expected one clickable card per PSU (6), got %d", n)
}
if !strings.Contains(body, "230 V · 1500 / 3000 W · 50% load") {
t.Fatalf("PSU card missing voltage / draw / load line: %s", body)
}
// Load fill (draw/rating) is drawn, same idea as the fan duty fill.
if !strings.Contains(body, `class="topo-psu-fill"`) || !strings.Contains(body, "height:50%") {
t.Fatalf("PSU card missing load fill: %s", body)
}
// The one failed PSU is coloured red on its own (crit token) and labelled.
if !strings.Contains(body, "var(--crit-bg)") || !strings.Contains(body, "CRITICAL") {
t.Fatalf("failed PSU should render individually as critical: %s", body)
}
// Firmware/PSU rows must be flex-wrap HTML (arbitrary count, no overlap),
// not absolutely-positioned SVG rects sharing fixed x/y coordinates.
@@ -152,6 +172,117 @@ func TestTopoPageRendersArbitraryPSUAndFirmwareCountsAsFlexRows(t *testing.T) {
}
}
func TestTopoPageRendersCoolingFansAsFlexRow(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "audit.json")
ok, warn := "OK", "Warning"
rpm := func(v int) *int { return &v }
ingest := schema.HardwareIngestRequest{
CollectedAt: "2026-03-15T00:00:00Z",
Hardware: schema.HardwareSnapshot{
Sensors: &schema.HardwareSensors{
Fans: []schema.HardwareFanSensor{
{Name: "FAN1", RPM: rpm(4200), Status: &ok},
{Name: "FAN2", RPM: rpm(15000), Status: &warn},
{Name: "FAN2", RPM: rpm(15000), Status: &warn}, // dup name, first wins
},
},
},
}
data, err := json.Marshal(ingest)
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, data, 0644); err != nil {
t.Fatal(err)
}
handler := NewHandler(HandlerOptions{AuditPath: path})
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/topo", nil))
if rec.Code != http.StatusOK {
t.Fatalf("status=%d", rec.Code)
}
body := rec.Body.String()
// One clickable square per fan (2 after dedup by name), each with a
// spinning glyph, under a COOLING heading.
if !strings.Contains(body, "Cooling") {
t.Fatalf("topo page missing Cooling heading: %s", body)
}
if n := strings.Count(body, `onclick="openComponentDetail('fan')"`) +
strings.Count(body, `onclick="openComponentDetail(&#39;fan&#39;)"`); n != 2 {
t.Fatalf("expected one clickable square per fan (2), got %d: %s", n, body)
}
if n := strings.Count(body, `class="topo-fan-spin"`); n != 2 {
t.Fatalf("expected 2 spinning fan glyphs, got %d: %s", n, body)
}
if !strings.Contains(body, "FAN1 · 4200 RPM") || !strings.Contains(body, "FAN2 · 15000 RPM") {
t.Fatalf("topo page missing per-fan RPM tooltips: %s", body)
}
// No observed ceiling in this test → tiles show no duty fill and say so.
if !strings.Contains(body, "ceiling not measured") {
t.Fatalf("topo fan tooltip should note the ceiling is unmeasured: %s", body)
}
// Live-update: each tile is addressable and the poll script is present.
if n := strings.Count(body, `class="topo-fan-tile"`); n != 2 {
t.Fatalf("expected 2 addressable fan tiles, got %d", n)
}
if !strings.Contains(body, `data-fan="FAN1"`) || !strings.Contains(body, "/api/metrics/latest") {
t.Fatalf("topo fan row missing live-update wiring: %s", body)
}
// Component-detail fallback endpoint must resolve the "fan" type.
rec2 := httptest.NewRecorder()
handler.ServeHTTP(rec2, httptest.NewRequest(http.MethodGet, "/api/components/fan", nil))
if rec2.Code != http.StatusOK {
t.Fatalf("/api/components/fan status=%d", rec2.Code)
}
if b := rec2.Body.String(); !strings.Contains(b, "FAN1") || !strings.Contains(b, "FAN2") {
t.Fatalf("fan component detail missing fan names: %s", b)
}
}
func TestFanSpinPeriodSec(t *testing.T) {
// Clamped at both ends; monotonically faster (smaller period) with RPM.
if got := fanSpinPeriodSec(200); got != 2.2 {
t.Fatalf("low RPM: got %v want 2.2 (slowest visible)", got)
}
if got := fanSpinPeriodSec(25000); got != 0.35 {
t.Fatalf("high RPM: got %v want 0.35 (fastest visible)", got)
}
mid := fanSpinPeriodSec(7000)
if mid <= 0.35 || mid >= 2.2 {
t.Fatalf("mid RPM period %v out of band", mid)
}
if fanSpinPeriodSec(10000) >= fanSpinPeriodSec(3000) {
t.Fatalf("higher RPM must spin faster (shorter period)")
}
}
func TestRenderTopoPSURowUsesObservedMaxWhenNoRating(t *testing.T) {
ok := "OK"
draw := 340.0
psus := []schema.HardwarePowerSupply{
{HardwareComponentStatus: schema.HardwareComponentStatus{Status: &ok}, InputPowerW: &draw}, // no WattageW
}
// Observed peak (keyed by ordinal) stands in for the missing nameplate.
html := renderTopoPSURow(psus, map[string]float64{"0": 2400})
if !strings.Contains(html, `data-psu-max-src="observed"`) {
t.Fatalf("expected observed-scale marker: %s", html)
}
if !strings.Contains(html, "340 / ~2400 W · ~14% load") {
t.Fatalf("expected estimated load line: %s", html)
}
// Without an observed peak either, just the raw watts, no fill.
html = renderTopoPSURow(psus, nil)
if strings.Contains(html, "% load") {
t.Fatalf("no rating and no observed peak → no load figure: %s", html)
}
}
func TestTopoPageRendersStorageDisksGroupedByType(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "audit.json")
+30 -30
View File
@@ -117,6 +117,16 @@ func renderValidateMode(opts HandlerOptions, stressDefault bool) string {
))
}
// Fan Ceiling Check is a platform cooling test (CPU/memory always, GPU only
// as extra heat when present), so it sits with CPU/Memory/Storage, not in
// the NVIDIA section. Load page only.
fanCard := renderLoadOnlySATCard(stressDefault, renderSATCard("fan", "Fan Ceiling Check", "runSAT('fan')", "", renderValidateCardBody(
"All system fans reported over IPMI / lm-sensors.",
`Drives CPU (+memory) and, when a GPU is present, the hottest sustained GPU load (dcgmproftester targeted-power — the Power/Thermal Fit engine) at the same time, and watches every fan until none has climbed for ~1 min. The peak RPM reached is recorded as each fan's ceiling and is what the Topology view sizes the fan tiles and duty-cycle fill against. Success once every fan plateaus (or the time cap is hit). A fan reading 0 RPM or an IPMI status of cr/nr under full load fails. IPMI polling backs off automatically if the BMC gets slow under load. If the host cannot be loaded at all, or exposes no fan sensors, the task is cancelled as "not applicable" rather than failed — the platform does not support forcing fans, so this is the closest safe equivalent.`,
`<code>stressapptest</code> / <code>stress-ng</code> + <code>dcgmproftester -t 1004</code> / <code>rvs gst</code>; <code>ipmitool sdr type Fan</code>`,
`~310 min depending on how fast the fan curve settles (hard cap 15 min).`,
)))
satStressModeJS := "function satStressMode() { return false; }"
if stressDefault {
satStressModeJS = "function satStressMode() { return true; }"
@@ -155,6 +165,7 @@ func renderValidateMode(opts HandlerOptions, stressDefault bool) string {
`<code>tpm2_getcap properties-fixed</code>, <code>tpm2_getcap pcrs</code>, <code>tpm2_pcrread</code>, <code>tpm2_gettestresult</code>`,
`Seconds - read-only queries; no ownership, NV, PCR, or key changes.`,
))) +
fanCard +
`</div>
<div style="height:1px;background:var(--border);margin:16px 0"></div>
<div class="card" style="margin-bottom:16px">
@@ -212,8 +223,6 @@ func renderValidateMode(opts HandlerOptions, stressDefault bool) string {
.validate-card-body { padding:0; }
.validate-card-section { padding:12px 16px 0; }
.validate-card-section:last-child { padding-bottom:16px; }
.sat-gpu-row { display:flex; align-items:flex-start; gap:8px; padding:6px 0; cursor:pointer; font-size:13px; }
.sat-gpu-row input[type=checkbox] { width:16px; height:16px; margin-top:2px; flex-shrink:0; }
</style>
<script>
let satES = null;
@@ -251,21 +260,11 @@ function satUpdateGPUSelectionNote() {
note.textContent = 'Selected GPUs: ' + selected.join(', ') + '. Multi-GPU tests will use all selected GPUs.';
}
function satRenderGPUList(gpus) {
const root = document.getElementById('sat-gpu-list');
if (!root) return;
if (!gpus || !gpus.length) {
root.innerHTML = '<p style="color:var(--muted);font-size:13px">No NVIDIA GPUs detected.</p>';
satUpdateGPUSelectionNote();
return;
}
root.innerHTML = gpus.map(function(gpu) {
const mem = gpu.memory_mb > 0 ? ' · ' + gpu.memory_mb + ' MiB' : '';
return '<label class="sat-gpu-row">'
+ '<input class="sat-nvidia-checkbox" type="checkbox" value="' + gpu.index + '" checked onchange="satUpdateGPUSelectionNote()">'
+ '<span><strong>GPU ' + gpu.index + '</strong> ' + gpu.name + mem + '</span>'
+ '</label>';
}).join('');
satUpdateGPUSelectionNote();
beeGpuPicker.render({
rootId: 'sat-gpu-list', gpus: gpus,
checkboxClass: 'sat-nvidia-checkbox', onToggle: 'satUpdateGPUSelectionNote()',
after: function() { satUpdateGPUSelectionNote(); },
});
}
function satSelectAllGPUs() {
document.querySelectorAll('.sat-nvidia-checkbox').forEach(function(el) { el.checked = true; });
@@ -627,6 +626,15 @@ func renderCheckOnlySATCard(stressMode bool, card string) string {
return card
}
// renderLoadOnlySATCard is the inverse of renderCheckOnlySATCard: the card
// shows only on the Load page (a sustained-load test, not a Check-tier one).
func renderLoadOnlySATCard(stressMode bool, card string) string {
if !stressMode {
return ""
}
return card
}
// renderCheck renders the non-destructive Check page (step 2).
// Shows validate-mode tests only: CPU, Memory, Storage, NVIDIA L2, NCCL, NVBandwidth, AMD.
// Stress-mode tests (targeted-stress, targeted-power, pulse) are on the Load page.
@@ -729,8 +737,6 @@ func renderCheck(opts HandlerOptions) string {
.validate-card-body { padding:0; }
.validate-card-section { padding:12px 16px 0; }
.validate-card-section:last-child { padding-bottom:16px; }
.sat-gpu-row { display:flex; align-items:flex-start; gap:8px; padding:6px 0; cursor:pointer; font-size:13px; }
.sat-gpu-row input[type=checkbox] { width:16px; height:16px; margin-top:2px; flex-shrink:0; }
.cb-row { display:flex; align-items:flex-start; gap:8px; padding:4px 0; cursor:pointer; font-size:13px; }
.cb-row input[type=checkbox] { width:16px; height:16px; margin-top:2px; flex-shrink:0; }
</style>
@@ -765,17 +771,11 @@ function satUpdateGPUSelectionNote() {
: 'Select at least one NVIDIA GPU to enable NVIDIA check tasks.';
}
function satRenderGPUList(gpus) {
const root = document.getElementById('sat-gpu-list');
if (!root) return;
if (!gpus || !gpus.length) {
root.innerHTML = '<p style="color:var(--muted);font-size:13px">No NVIDIA GPUs detected.</p>';
satUpdateGPUSelectionNote(); return;
}
root.innerHTML = gpus.map(gpu => {
const mem = gpu.memory_mb > 0 ? ' · ' + gpu.memory_mb + ' MiB' : '';
return '<label class="sat-gpu-row"><input class="sat-nvidia-checkbox" type="checkbox" value="' + gpu.index + '" checked onchange="satUpdateGPUSelectionNote()"><span><strong>GPU ' + gpu.index + '</strong> ' + gpu.name + mem + '</span></label>';
}).join('');
satUpdateGPUSelectionNote();
beeGpuPicker.render({
rootId: 'sat-gpu-list', gpus: gpus,
checkboxClass: 'sat-nvidia-checkbox', onToggle: 'satUpdateGPUSelectionNote()',
after: function() { satUpdateGPUSelectionNote(); },
});
}
function satSelectAllGPUs() { document.querySelectorAll('.sat-nvidia-checkbox').forEach(el => { el.checked = true; }); satUpdateGPUSelectionNote(); }
function satSelectNoGPUs() { document.querySelectorAll('.sat-nvidia-checkbox').forEach(el => { el.checked = false; }); satUpdateGPUSelectionNote(); }
+1
View File
@@ -126,6 +126,7 @@ function openComponentDetail(type) {
body.innerHTML = '<div style="padding:20px;color:var(--crit-fg)">Error loading details.</div>';
});
}
` + gpuPickerJS + `
</script>` +
`</body></html>`
}
+1
View File
@@ -272,6 +272,7 @@ func NewHandler(opts HandlerOptions) http.Handler {
mux.HandleFunc("POST /api/sat/memory-stress/run", h.handleAPISATRun("memory-stress"))
mux.HandleFunc("POST /api/sat/sat-stress/run", h.handleAPISATRun("sat-stress"))
mux.HandleFunc("POST /api/sat/platform-stress/run", h.handleAPISATRun("platform-stress"))
mux.HandleFunc("POST /api/sat/fan/run", h.handleAPISATRun("fan"))
mux.HandleFunc("POST /api/sat/run-all", h.handleAPISATRunAll)
mux.HandleFunc("GET /api/sat/stream", h.handleAPISATStream)
mux.HandleFunc("POST /api/sat/abort", h.handleAPISATAbort)
+21 -3
View File
@@ -3,6 +3,7 @@ package webui
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
@@ -79,9 +80,11 @@ func finalizeTaskForResult(t *Task, errMsg string, cancelled bool) {
now := time.Now()
t.DoneAt = &now
switch {
case cancelled:
case cancelled || t.Status == TaskCancelled:
t.Status = TaskCancelled
if strings.TrimSpace(t.ErrMsg) == "" {
t.ErrMsg = "aborted"
}
case strings.TrimSpace(errMsg) != "":
t.Status = TaskFailed
t.ErrMsg = errMsg
@@ -377,6 +380,12 @@ func executeTaskWithOptions(opts *HandlerOptions, t *Task, j *jobState, ctx cont
runOpts := resolvePlatformStressPreset(t.params.BurnProfile)
runOpts.Components = t.params.PlatformComponents
archive, err = a.RunPlatformStress(ctx, "", runOpts, j.append)
case "fan":
if a == nil {
err = fmt.Errorf("app not configured")
break
}
archive, err = runFanCheckPackCtx(a, ctx, "", platform.FanCheckOptions{GPUIndices: t.params.GPUIndices}, j.append)
case "audit":
if a == nil {
err = fmt.Errorf("app not configured")
@@ -476,10 +485,19 @@ func executeTaskWithOptions(opts *HandlerOptions, t *Task, j *jobState, ctx cont
}
if err != nil {
if ctx.Err() != nil {
switch {
case ctx.Err() != nil:
j.append("Aborted.")
j.finish("aborted")
} else {
case errors.Is(err, platform.ErrTestNotApplicable):
// The host offered no way to run this test — not a hardware
// fault. Land the task as cancelled ("not applicable") with a
// detailed log, so an engineer never sees a false failure.
j.append("NOT APPLICABLE: " + err.Error())
t.Status = TaskCancelled
t.ErrMsg = "not applicable — " + err.Error()
j.finish("")
default:
j.append("ERROR: " + err.Error())
j.finish(err.Error())
}
+5 -1
View File
@@ -53,6 +53,7 @@ var taskNames = map[string]string{
"memory-stress": "Memory Burn-in",
"sat-stress": "SAT Stress (stressapptest)",
"platform-stress": "Platform Thermal Cycling",
"fan": "Fan Ceiling Check (CPU+GPU load)",
"audit": "Audit",
"support-bundle": "Support Bundle",
"install": "Install to Disk",
@@ -196,7 +197,7 @@ func taskMayLeaveOrphanWorkers(target string) bool {
switch strings.TrimSpace(strings.ToLower(target)) {
case "nvidia", "nvidia-targeted-stress", "nvidia-targeted-power", "nvidia-pulse",
"nvidia-bandwidth", "nvidia-stress", "nvidia-compute", "nvidia-bench-perf",
"memory", "memory-stress", "cpu", "sat-stress", "platform-stress":
"memory", "memory-stress", "cpu", "sat-stress", "platform-stress", "fan":
return true
default:
return false
@@ -343,6 +344,9 @@ var (
runSATStressPackCtx = func(a *app.App, ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
return a.RunSATStressPackCtx(ctx, baseDir, durationSec, logFunc)
}
runFanCheckPackCtx = func(a *app.App, ctx context.Context, baseDir string, opts platform.FanCheckOptions, logFunc func(string)) (string, error) {
return a.RunFanCheckCtx(ctx, baseDir, opts, logFunc)
}
buildSupportBundle = app.BuildSupportBundle
installCommand = func(ctx context.Context, device string, logPath string) *exec.Cmd {
return exec.CommandContext(ctx, "bee-install", device, logPath)
+29
View File
@@ -3,6 +3,7 @@ package webui
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
@@ -700,6 +701,34 @@ func TestRunTaskHonorsCancel(t *testing.T) {
}
}
func TestRunTaskFanNotApplicableIsCancelledNotFailed(t *testing.T) {
q := &taskQueue{opts: &HandlerOptions{App: &app.App{}}}
tk := &Task{
ID: "fan-1", Name: "Fan Ceiling Check", Target: "fan",
Status: TaskRunning, CreatedAt: time.Now(),
}
j := &jobState{}
tk.job = j
orig := runFanCheckPackCtx
runFanCheckPackCtx = func(_ *app.App, _ context.Context, _ string, _ platform.FanCheckOptions, _ func(string)) (string, error) {
return "", fmt.Errorf("no fan sensors readable: %w", platform.ErrTestNotApplicable)
}
defer func() { runFanCheckPackCtx = orig }()
q.runTask(tk, j, context.Background())
if tk.Status != TaskCancelled {
t.Fatalf("status=%q want %q", tk.Status, TaskCancelled)
}
if j.err != "" {
t.Fatalf("job err should be empty for a not-applicable task, got %q", j.err)
}
if !strings.Contains(tk.ErrMsg, "not applicable") {
t.Fatalf("ErrMsg should explain not-applicable, got %q", tk.ErrMsg)
}
}
func TestRunTaskUsesBurnProfileDurationForCPU(t *testing.T) {
var gotDuration int
q := &taskQueue{
+6
View File
@@ -9,6 +9,7 @@ Generic engineering rules live in `bible/rules/patterns/`.
|---|---|
| `architecture/system-overview.md` | What bee does, scope, tech stack |
| `architecture/runtime-flows.md` | Boot sequence, audit flow, service order |
| `architecture/squashfs-layers.md` | Semantic SquashFS layer model, ownership, order, verification |
| `docs/customer-gpu-test-methodology.md` | Customer-facing GPU PCIe Validate / Validate -> Stress test list |
| `docs/hardware-ingest-contract.md` | Current Reanimator hardware ingest JSON contract |
| `docs/validate-vs-burn.md` | Validate and Validate -> Stress hardware test policy |
@@ -74,3 +75,8 @@ Generic engineering rules live in `bible/rules/patterns/`.
- `all_reduce_perf`
- GPU bandwidth check
- `dcgmi diag -r nvbandwidth` (per CPU socket, then all selected GPUs, on multi-socket systems -- see `decisions/2026-07-27-nvbandwidth-per-socket-split.md`)
- Fan ceiling check (Load tier / `3. Load` only)
- `stressapptest` (or `stress-ng`) + the hottest GPU load — `dcgmproftester -t 1004` / `targeted_power` (Power/Thermal Fit engine), **not** `bee-gpu-burn` — run simultaneously
- `ipmitool sdr type Fan` on an adaptive interval (1 s floor, backs off to 30 s when the BMC gets slow, every read time-boxed) until every fan plateaus (~1 min flat, only trusted while telemetry is healthy) or the 15 min cap
- records each fan's observed peak RPM to the fan-observation store (used by the Topology fan tiles: size ∝ ceiling, fill ∝ duty cycle); FAIL only on a fan at 0 RPM / IPMI cr-nr under load; **cancelled ("not applicable")**, never failed, if the host cannot be loaded or has no fan sensors
- see `decisions/2026-09-04-fan-ceiling-check.md`
+1
View File
@@ -53,6 +53,7 @@ All SAT run endpoints enqueue an async task. Response: `{"task_id": "..."}`.
| POST | `/api/sat/memory-stress/run` | Memory stress |
| POST | `/api/sat/sat-stress/run` | Combined storage+memory stress |
| POST | `/api/sat/platform-stress/run` | Fan + thermal stress |
| POST | `/api/sat/fan/run` | Fan ceiling check (CPU+GPU load until fans plateau; Load tier). Not-applicable → task cancelled, not failed. |
| POST | `/api/sat/run-all` | Plan + enqueue the whole validate/check set server-side. Body: `{stress_mode, amd_targets[], nvidia_gpu_indices[]}` (operator intent only). Response: `{task_ids[], task_count, notes[]}`. Hardware presence/readiness and which tasks to run are decided by `handler.planSATRunAll`, not the page. |
| GET | `/api/sat/stream` | SSE: live SAT log stream |
| POST | `/api/sat/abort` | Abort the running SAT task |
@@ -0,0 +1,107 @@
# Decision: Fan check discovers the RPM ceiling by CPU+GPU load, not by forcing fans over the BMC
**Date:** 2026-09-04
**Status:** active
## Context
We want a SAT test that pushes every system fan to its top speed, records that
speed as the fan's ceiling, and flags a fan that will not spin. The recorded
ceiling is what the Topology view sizes each fan tile against
(`platform.ResolveFanMaxRPM`).
The obvious approach — force the fans to 100% PWM over IPMI/Redfish and read the
resulting RPM — does not work on our current platform:
- Stand: MSI G4201 / MS-S3831, AMI MegaRAC BMC (fw 1.08).
- Every documented host-side fan-control OEM command returns
`rsp=0xc1 Invalid command`: Supermicro `0x30 0x45`, ASRock/AMI reference
`0x3a 0x01 / 0xd0 0x12 / 0xd0 0x0f / 0xd6 / 0xd7 / 0xda`, `0x30 0x30..0x32`.
BIOS `KCS Access Control Policy = Allow All`, so this is not KCS filtering —
MSI simply does not implement them.
- The MSI G4201 Redfish API guide documents Thermal as **GET only**; no
fan-mode / fan-PWM PATCH endpoint exists.
- The only fan knob in BIOS is `Fan PWM Offset` (0100, additive to the auto
curve, reboot-gated) — not a runtime 100% force.
## Decision
The `fan` SAT test (`platform.RunFanCheck`, formerly the unwired
`RunFanStressTest`) drives load, not the BMC:
1. Start `stressapptest` (CPU + memory) and, when a GPU is present, the
**hottest sustained GPU load**`dcgmproftester -t 1004` /
`targeted_power`, via `resolveBenchmarkPowerLoadCommand`, the same engine
Power/Thermal Fit uses — **simultaneously**, each in its own goroutine.
NOT `bee-gpu-burn`: that is a compute-throughput burn that tops out around
88% of TDP (measured ~525 W of 600 W on RTX PRO 6000 Blackwell) and never
makes the fans demand their true ceiling. A first run on the MSI stand with
`bee-gpu-burn` peaked F2U fans at 24 400 RPM vs a historical 26 000.
A missing GPU is not an error — CPU/memory load alone exercises the cooling
loop. Load sources reach full power at different times, so the plateau clock
only starts once every launched source reports its process running (plus a
fixed GPU ramp grace).
2. Sample every fan on an adaptive interval (floor 1 s). Each read is
time-boxed (`readFansBounded` — a goroutine abandoned on timeout, so a
KCS-wedged `ipmitool` can never block the loop); when reads are slow the
interval backs off geometrically to 30 s and tightens again when they
recover. Under 8-GPU + CPU load on the MSI stand, `ipmitool sdr type Fan`
took ~14 s/call and at one point wedged for minutes — without this the
"1 Hz" sampler silently degraded to one sample per 14 s and the plateau
timer ran on stale data. A plateau is only declared while telemetry is
healthy (interval near the floor, ≥5 recent samples); a degraded run just
rides out to `MaxLoadSec` and records the peak it saw. Per fan, track the
peak RPM and the last time it climbed by more than `PlateauDeltaRPM`
(default 50).
3. When no fan has climbed for `PlateauHoldSec` (default 60 s) and at least one
fan rose meaningfully above baseline, declare the ceiling found and stop —
**success**. `MaxLoadSec` (default 900 s) is a hard cap; hitting it is also
success (the highest RPM seen is still recorded).
4. Persist each peak through `updateFanObservation`
(`/var/log/bee-sat/fan-observation.json`), which is what `ResolveFanMaxRPM`
reads.
## The observed-peak store (autotune primitive)
`platform.observedPeakStore` (`observed_peaks.go`) is the shared mechanism:
observe the max value per key while the box is under load, require a candidate
to hold ≥ `minHold` before it sticks (rejects spikes), round up, persist to a
`{"<jsonKey>": {key: peak}}` JSON file. Two instances:
- `fanPeaks``fan-observation.json` `max_rpm`, keyed by fan name, round-up 1000.
- `psuPeaks``psu-observation.json` `max_w`, keyed by **PSU ordinal**, round-up 50.
Both are fed from the ordinary telemetry paths — `sampleFanSpeeds` and
`samplePSUPower` call `update*Observation` — so **any** full-load run refines
them: the Fan Ceiling Check itself, a burn, thermal cycling, power
calibration, and the 5 s web metrics collector while any of those run. There
is no separate "PSU autotune" test: the fan check's max-CPU+GPU load is
already the right moment to observe peak PSU draw, and it samples PSU power at
a slow cadence off its own loop (`psu_<i>_peak_w` in the summary).
This exists because BMCs like the MSI stand's report only instantaneous PSU
input power, no nameplate rating. `/topo` PSU cards scale the load fill by the
real `wattage_w` when present, otherwise by `ObservedPSUMaxW()` — marked as an
estimate (`~N% load`).
## Verdict mapping
- A fan reading **0 RPM**, or IPMI status **cr/nr**, while under full load →
`FAILED` (dead / stuck rotor).
- No load source available (no `stressapptest`/`stress-ng` and no GPU burn
tool), or no fan sensors at all → `platform.ErrTestNotApplicable`, which the
task layer lands as **cancelled ("not applicable")**, never failed, with a
detailed log. An engineer must not see a red for "this platform can't run the
test".
## Tier
Load only (`/load`, "3. Load"). It is a sustained full-load test, so it does
not belong on the read-only Check page. Added to the stress-mode `Run All`.
## Consequences
- The ceiling is *observed*, not a spec value — only as high as the load drove
the fans. Good enough for tile sizing and stuck-fan detection.
- If a future platform does expose a safe host-side fan force, revisit: a real
100% force is a stronger test than load-driven ramp.
+7 -1
View File
@@ -81,7 +81,13 @@ This happens for:
**File:** `audit/internal/webui/pages.go`
- Source: `GET /api/gpus``api.go``ListNvidiaGPUs()` → live nvidia-smi
- Render: `'GPU ' + gpu.index + ' — ' + gpu.name + ' · ' + mem`
- Render: `'GPU N — <model> · <mem> MiB · sn: <serial>'`
(serial from `nvidia-smi --query-gpu=...,serial`; omitted when N/A)
- **Single source:** `audit/internal/webui/gpu_picker.go``beeGpuPicker.render({...})` builds every
row. All three pages (Load/SAT, Burn, Benchmark) call it from their `*RenderGPUList` wrapper;
`gpuPickerCSS`/`gpuPickerJS` are injected once by `layoutHead`/`renderPage`.
- Serial is rendered monospace; the digits that differ across the listed GPUs (common
prefix/suffix stripped) are bolded in accent colour.
- Fallback: `gpu.name || 'GPU ' + idx` (JS, line ~1432)
This always shows the correct model because it queries nvidia-smi live. It is **not** connected to benchmark result data.