diff --git a/audit/internal/platform/live_metrics.go b/audit/internal/platform/live_metrics.go index 0fe5e6a..53a7214 100644 --- a/audit/internal/platform/live_metrics.go +++ b/audit/internal/platform/live_metrics.go @@ -393,5 +393,9 @@ func samplePSUPower() []PSUReading { if len(psus) == 0 { return nil } + // Feed the observed-capacity store (the "autotune" for PSU load scaling on + // BMCs that report only instantaneous power) — every load run that samples + // PSU power, including the 5 s metrics collector, refines it. + updatePSUObservation(psus, time.Now()) return psus } diff --git a/audit/internal/platform/observed_peaks.go b/audit/internal/platform/observed_peaks.go new file mode 100644 index 0000000..3a0540e --- /dev/null +++ b/audit/internal/platform/observed_peaks.go @@ -0,0 +1,183 @@ +package platform + +import ( + "encoding/json" + "math" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "time" +) + +// observedPeakStore is the "autotune" primitive for components that expose no +// host-readable nameplate maximum: fan top RPM, PSU capacity. It records the +// highest value seen per key while the box is under load, persists it to a +// JSON file, and hands it back so live readings can be scaled against a real +// maximum. A new peak only sticks after it has been held for at least +// minHold, which rejects transient spikes. +type observedPeakStore struct { + path string // JSON file + jsonKey string // top-level object key, e.g. "max_rpm" + roundUp float64 // round a new peak up to this multiple; 0 = keep raw + minHold time.Duration // a candidate peak must persist this long to stick + + mu sync.Mutex + loaded bool + peaks map[string]float64 + candidates map[string]peakCandidate +} + +type peakCandidate struct { + firstSeen time.Time + val float64 +} + +// persistedPeaks reads the file fresh (no lock, no cache mutation) and returns +// its sanitized {key -> peak} map. Empty map when the file is missing or +// unparsable. +func (s *observedPeakStore) persistedPeaks() map[string]float64 { + out := map[string]float64{} + raw, err := os.ReadFile(s.path) + if err != nil || len(raw) == 0 { + return out + } + var doc map[string]map[string]float64 + if json.Unmarshal(raw, &doc) != nil { + return out + } + for k, v := range doc[s.jsonKey] { + k = strings.TrimSpace(k) + if k == "" || v <= 0 { + continue + } + out[k] = v + } + return out +} + +func (s *observedPeakStore) loadLocked() { + if s.loaded { + return + } + s.loaded = true + s.peaks = s.persistedPeaks() + if s.candidates == nil { + s.candidates = map[string]peakCandidate{} + } +} + +func (s *observedPeakStore) saveLocked() { + if len(s.peaks) == 0 { + return + } + dir := filepath.Dir(s.path) + if dir == "" || dir == "." { + return + } + if err := os.MkdirAll(dir, 0755); err != nil { + return + } + raw, err := json.MarshalIndent(map[string]map[string]float64{s.jsonKey: s.peaks}, "", " ") + if err != nil { + return + } + _ = os.WriteFile(s.path, raw, 0644) +} + +func (s *observedPeakStore) round(v float64) float64 { + if v <= 0 { + return 0 + } + if s.roundUp <= 0 { + return v + } + return math.Ceil(v/s.roundUp) * s.roundUp +} + +// observe feeds one telemetry sample (key -> current value). Non-positive +// values and blank keys are ignored. +func (s *observedPeakStore) observe(samples map[string]float64, now time.Time) { + if len(samples) == 0 { + return + } + s.mu.Lock() + defer s.mu.Unlock() + s.loadLocked() + changed := false + for key, val := range samples { + key = strings.TrimSpace(key) + if key == "" || val <= 0 { + continue + } + cur := s.peaks[key] + if val <= cur { + delete(s.candidates, key) + continue + } + if cand, ok := s.candidates[key]; ok { + if now.Sub(cand.firstSeen) >= s.minHold { + nv := math.Max(cand.val, val) + if nv > cur { + s.peaks[key] = s.round(nv) + changed = true + } + delete(s.candidates, key) + continue + } + if val > cand.val { + s.candidates[key] = peakCandidate{firstSeen: cand.firstSeen, val: val} + } + continue + } + s.candidates[key] = peakCandidate{firstSeen: now, val: val} + } + if changed { + s.saveLocked() + } +} + +// snapshot returns the persisted peaks (fresh from disk), for read-only +// consumers such as the /topo web view. +func (s *observedPeakStore) snapshot() map[string]float64 { + return s.persistedPeaks() +} + +// ── PSU capacity ──────────────────────────────────────────────────────────── + +var psuPeaks = &observedPeakStore{ + path: "/var/log/bee-sat/psu-observation.json", + jsonKey: "max_w", + roundUp: 50, + minHold: time.Second, +} + +// updatePSUObservation feeds the current per-PSU draw (keyed by ordinal, in +// the order the caller lists them) into the observed-capacity store. On a BMC +// that reports only instantaneous input power this is the only way to know +// what "100% load" looks like for each supply: observe the peak draw during +// any full-load run (the Fan Ceiling Check, a burn, thermal cycling — the 5 s +// metrics collector samples PSUs throughout) and remember it. +func updatePSUObservation(psus []PSUReading, now time.Time) { + if len(psus) == 0 { + return + } + m := make(map[string]float64, len(psus)) + for i, p := range psus { + if p.PowerW > 0 { + m[strconv.Itoa(i)] = p.PowerW + } + } + psuPeaks.observe(m, now) +} + +// ObservedPSUMaxW returns the persisted per-PSU observed peak draw, keyed by +// ordinal ("0", "1", …), or nil if none recorded yet. +func ObservedPSUMaxW() map[string]float64 { + p := psuPeaks.snapshot() + if len(p) == 0 { + return nil + } + return p +} diff --git a/audit/internal/platform/sat_fan_stress.go b/audit/internal/platform/sat_fan_stress.go index 5639b30..d13d173 100644 --- a/audit/internal/platform/sat_fan_stress.go +++ b/audit/internal/platform/sat_fan_stress.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "math" "os" "os/exec" "path/filepath" @@ -44,33 +43,16 @@ type cachedPowerReading struct { UpdatedAt time.Time } -type fanObservationState struct { - MaxRPM map[string]float64 `json:"max_rpm"` -} - -type fanPeakCandidate struct { - FirstSeen time.Time - RPM float64 -} - -var ( - fanObservationMu sync.Mutex - fanObservation fanObservationState - fanObservationInit bool - fanPeakCandidates = make(map[string]fanPeakCandidate) -) - const systemPowerHoldTTL = 15 * time.Second -var fanObservationStatePath = "/var/log/bee-sat/fan-observation.json" - -const fanObservationMinPeakHold = time.Second - -func normalizeObservedFanMaxRPM(rpm float64) float64 { - if rpm <= 0 { - return 0 - } - return math.Ceil(rpm/1000.0) * 1000.0 +// fanPeaks is the observed top-RPM store — the "autotune" for fan ceilings. +// Any full-load run (Fan Ceiling Check, burn, thermal cycling) feeds it via +// updateFanObservation; ResolveFanMaxRPM / ObservedFanMaxRPM read it back. +var fanPeaks = &observedPeakStore{ + path: "/var/log/bee-sat/fan-observation.json", + jsonKey: "max_rpm", + roundUp: 1000, + minHold: time.Second, } // RunFanCheck drives CPU (+memory) and, when a GPU is present, GPU load to @@ -240,6 +222,12 @@ func (s *System) RunFanCheck(ctx context.Context, baseDir string, opts FanCheckO goodSamples := 0 poll := fanPollFloor + // PSU peak draw, sampled at a slow cadence off the same loop — the fan + // check already drives the box to full power, so it is also the right run + // to observe what each PSU tops out at (updatePSUObservation persists it). + psuPeakW := map[int]float64{} + lastPSUSec := -1e9 + csvPath := filepath.Join(runDir, "fan-sensors.csv") _ = os.WriteFile(csvPath, []byte("elapsed_sec,fan_name,rpm\n"), 0644) csvFile, _ := os.OpenFile(csvPath, os.O_APPEND|os.O_WRONLY, 0644) @@ -281,6 +269,17 @@ loop: } goodSamples++ + if elapsed-lastPSUSec >= 15 { + lastPSUSec = elapsed + if ps, ok := readPSUPowerBounded(fanReadTMO); ok { + for i, p := range ps { + if p.PowerW > psuPeakW[i] { + psuPeakW[i] = p.PowerW + } + } + } + } + for _, f := range fans { if csvFile != nil { fmt.Fprintf(csvFile, "%.0f,%s,%.0f\n", elapsed, f.Name, f.RPM) @@ -347,6 +346,16 @@ loop: if t := boundedGPUMaxTemp(opts.GPUIndices); t > 0 { fmt.Fprintf(&summary, "gpu_temp_c=%.0f\n", t) } + if len(psuPeakW) > 0 { + idx := make([]int, 0, len(psuPeakW)) + for i := range psuPeakW { + idx = append(idx, i) + } + sort.Ints(idx) + for _, i := range idx { + fmt.Fprintf(&summary, "psu_%d_peak_w=%.0f\n", i, psuPeakW[i]) + } + } stats := satStats{} names := make([]string, 0, len(baselineRPM)) @@ -441,6 +450,21 @@ func readFansBounded(timeout time.Duration) ([]FanReading, bool) { } } +// readPSUPowerBounded is readFansBounded's PSU sibling: a time-boxed, +// abandonable "ipmitool sdr" read of per-PSU power (samplePSUPower also feeds +// the observed-capacity store). Used at a slow cadence during the fan check so +// the same max-load run that finds fan ceilings also records peak PSU draw. +func readPSUPowerBounded(timeout time.Duration) ([]PSUReading, bool) { + ch := make(chan []PSUReading, 1) + go func() { ch <- samplePSUPower() }() + select { + case ps := <-ch: + return ps, len(ps) > 0 + case <-time.After(timeout): + return nil, false + } +} + // boundedGPUMaxTemp returns the hottest GPU temperature via a single // time-boxed nvidia-smi call, or 0 if unavailable. func boundedGPUMaxTemp(gpuIndices []int) float64 { @@ -569,8 +593,8 @@ func readFanStatuses() map[string]string { // ResolveFanMaxRPM returns, for every fan name in current (name -> current // RPM), the RPM to treat as that fan's 100% reference. Preference order: -// 1. the persisted observed peak (fanObservationStatePath), written by -// RunFanCheck and by live-metrics sampling under load; +// 1. the persisted observed peak, written by RunFanCheck and by live-metrics +// sampling under load; // 2. the largest peak observed on any peer fan (keeps a group visually // consistent when only some fans have a recorded peak); // 3. the fan's own current RPM (so a tile is never sized against zero). @@ -578,7 +602,7 @@ func readFanStatuses() map[string]string { // The fallback lives here, not in the view, so every consumer of a fan // maximum applies the same rule. func ResolveFanMaxRPM(current map[string]float64) map[string]float64 { - persisted := readPersistedFanMaxRPM() + persisted := fanPeaks.snapshot() peerMax := 0.0 for _, v := range persisted { @@ -621,122 +645,41 @@ func sampleFanSpeeds() ([]FanReading, error) { return nil, sensorsErr } -// readPersistedFanMaxRPM reads fanObservationStatePath and returns its -// sanitized {fan name -> observed peak RPM} map (empty names / non-positive -// values dropped). Returns an empty map when the file is missing or unparsable. -func readPersistedFanMaxRPM() map[string]float64 { - out := map[string]float64{} - raw, err := os.ReadFile(fanObservationStatePath) - if err != nil || len(raw) == 0 { - return out - } - var persisted fanObservationState - if json.Unmarshal(raw, &persisted) != nil { - return out - } - for name, rpm := range persisted.MaxRPM { - name = strings.TrimSpace(name) - if name == "" || rpm <= 0 { - continue - } - out[name] = rpm - } - return out -} - // ObservedFanMaxRPM returns the per-fan observed peak RPM map persisted by -// fan-stress SAT runs, or nil if none is recorded yet. It reads the file -// directly without touching the in-process observation cache or its lock, so -// read-only consumers (the /topo web view) can call it without perturbing a -// concurrent SAT run's peak tracking. +// full-load runs, or nil if none is recorded yet. func ObservedFanMaxRPM() map[string]float64 { - out := readPersistedFanMaxRPM() + out := fanPeaks.snapshot() if len(out) == 0 { return nil } return out } -func loadFanObservationLocked() { - if fanObservationInit { - return - } - fanObservationInit = true - fanObservation.MaxRPM = readPersistedFanMaxRPM() -} - -func saveFanObservationLocked() { - if len(fanObservation.MaxRPM) == 0 { - return - } - dir := filepath.Dir(fanObservationStatePath) - if dir == "" || dir == "." { - dir = "/var/log/bee-sat" - } - if err := os.MkdirAll(dir, 0755); err != nil { - return - } - raw, err := json.MarshalIndent(fanObservation, "", " ") - if err != nil { - return - } - _ = os.WriteFile(fanObservationStatePath, raw, 0644) -} - func updateFanObservation(fans []FanReading, now time.Time) { if len(fans) == 0 { return } - fanObservationMu.Lock() - defer fanObservationMu.Unlock() - loadFanObservationLocked() - changed := false + m := make(map[string]float64, len(fans)) for _, fan := range fans { - name := strings.TrimSpace(fan.Name) - if name == "" || fan.RPM <= 0 { - continue + if n := strings.TrimSpace(fan.Name); n != "" && fan.RPM > 0 { + m[n] = fan.RPM } - currentMax := fanObservation.MaxRPM[name] - if fan.RPM <= currentMax { - delete(fanPeakCandidates, name) - continue - } - if cand, ok := fanPeakCandidates[name]; ok { - if now.Sub(cand.FirstSeen) >= fanObservationMinPeakHold { - newMax := math.Max(cand.RPM, fan.RPM) - if newMax > currentMax { - fanObservation.MaxRPM[name] = normalizeObservedFanMaxRPM(newMax) - changed = true - } - delete(fanPeakCandidates, name) - continue - } - if fan.RPM > cand.RPM { - fanPeakCandidates[name] = fanPeakCandidate{FirstSeen: cand.FirstSeen, RPM: fan.RPM} - } - continue - } - fanPeakCandidates[name] = fanPeakCandidate{FirstSeen: now, RPM: fan.RPM} - } - if changed { - saveFanObservationLocked() } + fanPeaks.observe(m, now) } func estimateFanDutyCyclePctFromObservation(fans []FanReading) (float64, bool) { if len(fans) == 0 { return 0, false } - fanObservationMu.Lock() - defer fanObservationMu.Unlock() - loadFanObservationLocked() + maxByName := fanPeaks.snapshot() var samples []float64 for _, fan := range fans { name := strings.TrimSpace(fan.Name) if name == "" || fan.RPM <= 0 { continue } - maxRPM := fanObservation.MaxRPM[name] + maxRPM := maxByName[name] if maxRPM <= 0 { continue } diff --git a/audit/internal/platform/sat_fan_stress_test.go b/audit/internal/platform/sat_fan_stress_test.go index 3d79787..817fccf 100644 --- a/audit/internal/platform/sat_fan_stress_test.go +++ b/audit/internal/platform/sat_fan_stress_test.go @@ -8,15 +8,20 @@ import ( "time" ) +// resetPeakStore points a store at a fresh temp file and clears its cache for +// the duration of the test. +func resetPeakStore(t *testing.T, s *observedPeakStore) { + t.Helper() + old := *s + s.path = filepath.Join(t.TempDir(), "peaks.json") + s.loaded = false + s.peaks = nil + s.candidates = nil + t.Cleanup(func() { *s = old }) +} + func TestResolveFanMaxRPM(t *testing.T) { - oldPath := fanObservationStatePath - oldInit := fanObservationInit - fanObservationStatePath = filepath.Join(t.TempDir(), "fan-observation.json") - fanObservationInit = false - t.Cleanup(func() { - fanObservationStatePath = oldPath - fanObservationInit = oldInit - }) + resetPeakStore(t, fanPeaks) // No persisted file yet: unknown fans fall back to their own current RPM. got := ResolveFanMaxRPM(map[string]float64{"A": 4000, "B": 9000}) @@ -24,7 +29,7 @@ func TestResolveFanMaxRPM(t *testing.T) { t.Fatalf("no-persist fallback: got %v", got) } - if err := os.WriteFile(fanObservationStatePath, []byte(`{"max_rpm":{"A":17000}}`), 0644); err != nil { + if err := os.WriteFile(fanPeaks.path, []byte(`{"max_rpm":{"A":17000}}`), 0644); err != nil { t.Fatal(err) } got = ResolveFanMaxRPM(map[string]float64{"A": 4000, "B": 9000, "C": 5000}) @@ -34,6 +39,49 @@ func TestResolveFanMaxRPM(t *testing.T) { } } +func TestObservedPeakStore(t *testing.T) { + s := &observedPeakStore{ + path: filepath.Join(t.TempDir(), "peaks.json"), + jsonKey: "max_w", + roundUp: 50, + minHold: time.Second, + } + t0 := time.Unix(0, 0) + + // A single spike does not stick. + s.observe(map[string]float64{"0": 900}, t0) + if len(s.snapshot()) != 0 { + t.Fatalf("transient spike should not persist: %v", s.snapshot()) + } + // Held past minHold → sticks, rounded up to the next 50. + s.observe(map[string]float64{"0": 920}, t0.Add(1200*time.Millisecond)) + if got := s.snapshot()["0"]; got != 950 { + t.Fatalf("held peak: got %v want 950", got) + } + // A lower reading never lowers the peak. + s.observe(map[string]float64{"0": 400}, t0.Add(5*time.Second)) + if got := s.snapshot()["0"]; got != 950 { + t.Fatalf("peak must not drop: got %v", got) + } + // Fresh store reloads from disk. + s2 := &observedPeakStore{path: s.path, jsonKey: "max_w"} + if got := s2.snapshot()["0"]; got != 950 { + t.Fatalf("reload from disk: got %v want 950", got) + } +} + +func TestUpdatePSUObservationKeyedByOrdinal(t *testing.T) { + resetPeakStore(t, psuPeaks) + now := time.Unix(0, 0) + psus := []PSUReading{{Name: "PSU1", PowerW: 1200}, {Name: "PSU2", PowerW: 1400}} + updatePSUObservation(psus, now) + updatePSUObservation(psus, now.Add(1200*time.Millisecond)) + got := ObservedPSUMaxW() + if got["0"] != 1200 || got["1"] != 1400 { + t.Fatalf("keyed-by-ordinal peaks: got %v", got) + } +} + func TestApplyFanCheckDefaults(t *testing.T) { var o FanCheckOptions applyFanCheckDefaults(&o) @@ -109,22 +157,7 @@ func TestParseFanDutyCyclePctSensorsJSON(t *testing.T) { } func TestEstimateFanDutyCyclePctFromObservation(t *testing.T) { - t.Parallel() - - oldPath := fanObservationStatePath - oldState := fanObservation - oldInit := fanObservationInit - oldCandidates := fanPeakCandidates - fanObservationStatePath = filepath.Join(t.TempDir(), "fan-observation.json") - fanObservation = fanObservationState{} - fanObservationInit = false - fanPeakCandidates = make(map[string]fanPeakCandidate) - t.Cleanup(func() { - fanObservationStatePath = oldPath - fanObservation = oldState - fanObservationInit = oldInit - fanPeakCandidates = oldCandidates - }) + resetPeakStore(t, fanPeaks) start := time.Unix(100, 0) updateFanObservation([]FanReading{{Name: "FAN1", RPM: 5000}}, start) @@ -143,9 +176,9 @@ func TestEstimateFanDutyCyclePctFromObservation(t *testing.T) { t.Fatalf("got=%v want ~43.3", got) } - fanObservation = fanObservationState{} - fanObservationInit = false - fanPeakCandidates = make(map[string]fanPeakCandidate) + fanPeaks.loaded = false + fanPeaks.peaks = nil + fanPeaks.candidates = nil got, ok = estimateFanDutyCyclePctFromObservation([]FanReading{{Name: "FAN1", RPM: 2600}}) if !ok { t.Fatalf("expected persisted observed max to be reloaded from disk") diff --git a/audit/internal/webui/page_topo_diagram.go b/audit/internal/webui/page_topo_diagram.go index 048cdcd..3cd956b 100644 --- a/audit/internal/webui/page_topo_diagram.go +++ b/audit/internal/webui/page_topo_diagram.go @@ -594,7 +594,7 @@ func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string hasPSU := len(hw.PowerSupplies) > 0 if hasPSU { - b.WriteString(renderTopoPSURow(hw.PowerSupplies)) + b.WriteString(renderTopoPSURow(hw.PowerSupplies, platform.ObservedPSUMaxW())) } // Cooling fans — one small clickable square per fan. Square SIZE encodes @@ -619,10 +619,13 @@ func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string } // renderTopoPSURow renders the POWER SUPPLIES row: one card per PSU, coloured -// by that PSU's own status (a failed unit goes red on its own), showing input -// voltage and draw. Cards click through to the shared PSU detail modal and -// carry data-psu so topoLiveScript can refresh the wattage in place. -func renderTopoPSURow(psus []schema.HardwarePowerSupply) string { +// by that PSU's own status (a failed unit goes red on its own). The card shows +// input voltage and draw, and a load fill rising from the bottom — same idea +// as the fan duty-cycle fill. The load scale is the nameplate rating when the +// BMC reports it; otherwise it is the observed peak draw (observedMaxW, keyed +// by ordinal — the "autotune" recorded during any full-load run), and the +// figure is marked as an estimate. +func renderTopoPSURow(psus []schema.HardwarePowerSupply, observedMaxW map[string]float64) string { var b strings.Builder b.WriteString(topoRowHeading("Power Supplies")) b.WriteString(`
`) @@ -640,14 +643,25 @@ func renderTopoPSURow(psus []schema.HardwarePowerSupply) string { if p.WattageW != nil && *p.WattageW > 0 { rating = *p.WattageW } + // Load scale: true rating if known, else the observed peak draw. + scaleMax := float64(rating) + scaleEstimate := false + if scaleMax <= 0 { + if m := observedMaxW[strconv.Itoa(i)]; m > 0 { + scaleMax = m + scaleEstimate = true + } + } var parts []string if p.InputVoltage != nil && *p.InputVoltage > 0 { parts = append(parts, fmt.Sprintf("%.0f V", *p.InputVoltage)) } switch { - case haveDraw && rating > 0: - parts = append(parts, fmt.Sprintf("%.0f / %d W · %.0f%% load", draw, rating, draw/float64(rating)*100)) + case haveDraw && scaleMax > 0 && scaleEstimate: + parts = append(parts, fmt.Sprintf("%.0f / ~%.0f W · ~%.0f%% load", draw, scaleMax, draw/scaleMax*100)) + case haveDraw && scaleMax > 0: + parts = append(parts, fmt.Sprintf("%.0f / %d W · %.0f%% load", draw, rating, draw/scaleMax*100)) case haveDraw: parts = append(parts, fmt.Sprintf("%.0f W", draw)) case rating > 0: @@ -660,33 +674,29 @@ func renderTopoPSURow(psus []schema.HardwarePowerSupply) string { statusWord = topoSeverityStatus(p.Status) } - // Load fill (draw / rating) rises from the bottom of the card, the - // same idea as the fan duty-cycle fill. Only drawn when the nameplate - // rating is known — a raw wattage with nothing to scale it against is - // not a load figure. - loadPct := -1.0 - if haveDraw && rating > 0 { - loadPct = draw / float64(rating) * 100 - if loadPct < 0 { - loadPct = 0 - } - if loadPct > 100 { - loadPct = 100 - } - } fillH := 0.0 - if loadPct >= 0 { - fillH = loadPct + if haveDraw && scaleMax > 0 { + fillH = draw / scaleMax * 100 + if fillH < 0 { + fillH = 0 + } + if fillH > 100 { + fillH = 100 + } } voltAttr := "" if p.InputVoltage != nil && *p.InputVoltage > 0 { voltAttr = fmt.Sprintf("%.0f", *p.InputVoltage) } + maxSrc := "rated" + if scaleEstimate { + maxSrc = "observed" + } - fmt.Fprintf(&b, `
`, - i, rating, voltAttr, stroke, text) + i, scaleMax, maxSrc, voltAttr, stroke, text) fmt.Fprintf(&b, `
`, fillH, stroke) fmt.Fprintf(&b, `
%s
`, html.EscapeString(label)) fmt.Fprintf(&b, `
%s
`, html.EscapeString(detail)) @@ -828,11 +838,15 @@ func topoLiveScript() string { psus.forEach(function(t){ var p=m.psus[parseInt(t.dataset.psu,10)];if(!p)return; var w=p.power_w||0,max=parseFloat(t.dataset.psuMax)||0,v=t.dataset.psuV; + var est=t.dataset.psuMaxSrc==='observed'; var parts=[]; if(v)parts.push(v+' V'); - if(w>0&&max>0)parts.push(Math.round(w)+' / '+max+' W · '+Math.round(w/max*100)+'% load'); - else if(w>0)parts.push(Math.round(w)+' W'); - else if(max>0)parts.push(max+' W rated'); + if(w>0&&max>0){ + var pct=Math.round(w/max*100); + parts.push(est?Math.round(w)+' / ~'+Math.round(max)+' W · ~'+pct+'% load' + :Math.round(w)+' / '+Math.round(max)+' W · '+pct+'% load'); + }else if(w>0)parts.push(Math.round(w)+' W'); + else if(max>0)parts.push(Math.round(max)+' W rated'); var d=t.querySelector('.topo-psu-detail');if(d&&parts.length)d.textContent=parts.join(' · '); if(w>0&&max>0){var f=t.querySelector('.topo-psu-fill');if(f)f.style.height=Math.max(0,Math.min(100,w/max*100)).toFixed(0)+'%';} }); diff --git a/audit/internal/webui/page_topo_test.go b/audit/internal/webui/page_topo_test.go index 185d5bf..b0ba5f3 100644 --- a/audit/internal/webui/page_topo_test.go +++ b/audit/internal/webui/page_topo_test.go @@ -262,6 +262,27 @@ func TestFanSpinPeriodSec(t *testing.T) { } } +func TestRenderTopoPSURowUsesObservedMaxWhenNoRating(t *testing.T) { + ok := "OK" + draw := 340.0 + psus := []schema.HardwarePowerSupply{ + {HardwareComponentStatus: schema.HardwareComponentStatus{Status: &ok}, InputPowerW: &draw}, // no WattageW + } + // Observed peak (keyed by ordinal) stands in for the missing nameplate. + html := renderTopoPSURow(psus, map[string]float64{"0": 2400}) + if !strings.Contains(html, `data-psu-max-src="observed"`) { + t.Fatalf("expected observed-scale marker: %s", html) + } + if !strings.Contains(html, "340 / ~2400 W · ~14% load") { + t.Fatalf("expected estimated load line: %s", html) + } + // Without an observed peak either, just the raw watts, no fill. + html = renderTopoPSURow(psus, nil) + if strings.Contains(html, "% load") { + t.Fatalf("no rating and no observed peak → no load figure: %s", html) + } +} + func TestTopoPageRendersStorageDisksGroupedByType(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "audit.json") diff --git a/bible-local/decisions/2026-09-04-fan-ceiling-check.md b/bible-local/decisions/2026-09-04-fan-ceiling-check.md index 5dc0619..2baf14d 100644 --- a/bible-local/decisions/2026-09-04-fan-ceiling-check.md +++ b/bible-local/decisions/2026-09-04-fan-ceiling-check.md @@ -57,10 +57,33 @@ The `fan` SAT test (`platform.RunFanCheck`, formerly the unwired fan rose meaningfully above baseline, declare the ceiling found and stop — **success**. `MaxLoadSec` (default 900 s) is a hard cap; hitting it is also success (the highest RPM seen is still recorded). -4. Persist each peak through the existing `updateFanObservation` path +4. Persist each peak through `updateFanObservation` (`/var/log/bee-sat/fan-observation.json`), which is what `ResolveFanMaxRPM` reads. +## The observed-peak store (autotune primitive) + +`platform.observedPeakStore` (`observed_peaks.go`) is the shared mechanism: +observe the max value per key while the box is under load, require a candidate +to hold ≥ `minHold` before it sticks (rejects spikes), round up, persist to a +`{"": {key: peak}}` JSON file. Two instances: + +- `fanPeaks` → `fan-observation.json` `max_rpm`, keyed by fan name, round-up 1000. +- `psuPeaks` → `psu-observation.json` `max_w`, keyed by **PSU ordinal**, round-up 50. + +Both are fed from the ordinary telemetry paths — `sampleFanSpeeds` and +`samplePSUPower` call `update*Observation` — so **any** full-load run refines +them: the Fan Ceiling Check itself, a burn, thermal cycling, power +calibration, and the 5 s web metrics collector while any of those run. There +is no separate "PSU autotune" test: the fan check's max-CPU+GPU load is +already the right moment to observe peak PSU draw, and it samples PSU power at +a slow cadence off its own loop (`psu__peak_w` in the summary). + +This exists because BMCs like the MSI stand's report only instantaneous PSU +input power, no nameplate rating. `/topo` PSU cards scale the load fill by the +real `wattage_w` when present, otherwise by `ObservedPSUMaxW()` — marked as an +estimate (`~N% load`). + ## Verdict mapping - A fan reading **0 RPM**, or IPMI status **cr/nr**, while under full load →