feat: autotune PSU capacity from the same full-load run as fan ceilings

Extract the fan peak-tracking into observedPeakStore (observe max under
load, hold >= minHold to reject spikes, round, persist JSON) and add a
second instance for PSU draw (psu-observation.json, keyed by PSU
ordinal). Fed from samplePSUPower like fans are from sampleFanSpeeds, so
any full-load run refines it — the Fan Ceiling Check (which also samples
PSU power at a slow cadence off its loop and writes psu_<i>_peak_w), a
burn, thermal cycling, and the 5s web metrics collector.

/topo PSU cards now scale the load fill by wattage_w when the BMC
reports it, else by the observed peak draw — marked "~N% load". This
MSI stand's BMC gives only instantaneous input power, so the observed
peak is the only capacity figure available.

Fan behaviour is unchanged (tests exercise updateFanObservation /
estimateFanDutyCyclePctFromObservation / ResolveFanMaxRPM through the
new store).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019VHG21rgTUiR1G3qFHTVmN

audit/internal/platform/live_metrics.go

@@ -393,5 +393,9 @@ func samplePSUPower() []PSUReading {
 	if len(psus) == 0 {
 		return nil
 	}
+	// Feed the observed-capacity store (the "autotune" for PSU load scaling on
+	// BMCs that report only instantaneous power) — every load run that samples
+	// PSU power, including the 5 s metrics collector, refines it.
+	updatePSUObservation(psus, time.Now())
 	return psus
 }

audit/internal/platform/observed_peaks.go

@@ -0,0 +1,183 @@
+package platform
+
+import (
+	"encoding/json"
+	"math"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+)
+
+// observedPeakStore is the "autotune" primitive for components that expose no
+// host-readable nameplate maximum: fan top RPM, PSU capacity. It records the
+// highest value seen per key while the box is under load, persists it to a
+// JSON file, and hands it back so live readings can be scaled against a real
+// maximum. A new peak only sticks after it has been held for at least
+// minHold, which rejects transient spikes.
+type observedPeakStore struct {
+	path    string        // JSON file
+	jsonKey string        // top-level object key, e.g. "max_rpm"
+	roundUp float64       // round a new peak up to this multiple; 0 = keep raw
+	minHold time.Duration // a candidate peak must persist this long to stick
+
+	mu         sync.Mutex
+	loaded     bool
+	peaks      map[string]float64
+	candidates map[string]peakCandidate
+}
+
+type peakCandidate struct {
+	firstSeen time.Time
+	val       float64
+}
+
+// persistedPeaks reads the file fresh (no lock, no cache mutation) and returns
+// its sanitized {key -> peak} map. Empty map when the file is missing or
+// unparsable.
+func (s *observedPeakStore) persistedPeaks() map[string]float64 {
+	out := map[string]float64{}
+	raw, err := os.ReadFile(s.path)
+	if err != nil || len(raw) == 0 {
+		return out
+	}
+	var doc map[string]map[string]float64
+	if json.Unmarshal(raw, &doc) != nil {
+		return out
+	}
+	for k, v := range doc[s.jsonKey] {
+		k = strings.TrimSpace(k)
+		if k == "" || v <= 0 {
+			continue
+		}
+		out[k] = v
+	}
+	return out
+}
+
+func (s *observedPeakStore) loadLocked() {
+	if s.loaded {
+		return
+	}
+	s.loaded = true
+	s.peaks = s.persistedPeaks()
+	if s.candidates == nil {
+		s.candidates = map[string]peakCandidate{}
+	}
+}
+
+func (s *observedPeakStore) saveLocked() {
+	if len(s.peaks) == 0 {
+		return
+	}
+	dir := filepath.Dir(s.path)
+	if dir == "" || dir == "." {
+		return
+	}
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		return
+	}
+	raw, err := json.MarshalIndent(map[string]map[string]float64{s.jsonKey: s.peaks}, "", "  ")
+	if err != nil {
+		return
+	}
+	_ = os.WriteFile(s.path, raw, 0644)
+}
+
+func (s *observedPeakStore) round(v float64) float64 {
+	if v <= 0 {
+		return 0
+	}
+	if s.roundUp <= 0 {
+		return v
+	}
+	return math.Ceil(v/s.roundUp) * s.roundUp
+}
+
+// observe feeds one telemetry sample (key -> current value). Non-positive
+// values and blank keys are ignored.
+func (s *observedPeakStore) observe(samples map[string]float64, now time.Time) {
+	if len(samples) == 0 {
+		return
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.loadLocked()
+	changed := false
+	for key, val := range samples {
+		key = strings.TrimSpace(key)
+		if key == "" || val <= 0 {
+			continue
+		}
+		cur := s.peaks[key]
+		if val <= cur {
+			delete(s.candidates, key)
+			continue
+		}
+		if cand, ok := s.candidates[key]; ok {
+			if now.Sub(cand.firstSeen) >= s.minHold {
+				nv := math.Max(cand.val, val)
+				if nv > cur {
+					s.peaks[key] = s.round(nv)
+					changed = true
+				}
+				delete(s.candidates, key)
+				continue
+			}
+			if val > cand.val {
+				s.candidates[key] = peakCandidate{firstSeen: cand.firstSeen, val: val}
+			}
+			continue
+		}
+		s.candidates[key] = peakCandidate{firstSeen: now, val: val}
+	}
+	if changed {
+		s.saveLocked()
+	}
+}
+
+// snapshot returns the persisted peaks (fresh from disk), for read-only
+// consumers such as the /topo web view.
+func (s *observedPeakStore) snapshot() map[string]float64 {
+	return s.persistedPeaks()
+}
+
+// ── PSU capacity ────────────────────────────────────────────────────────────
+
+var psuPeaks = &observedPeakStore{
+	path:    "/var/log/bee-sat/psu-observation.json",
+	jsonKey: "max_w",
+	roundUp: 50,
+	minHold: time.Second,
+}
+
+// updatePSUObservation feeds the current per-PSU draw (keyed by ordinal, in
+// the order the caller lists them) into the observed-capacity store. On a BMC
+// that reports only instantaneous input power this is the only way to know
+// what "100% load" looks like for each supply: observe the peak draw during
+// any full-load run (the Fan Ceiling Check, a burn, thermal cycling — the 5 s
+// metrics collector samples PSUs throughout) and remember it.
+func updatePSUObservation(psus []PSUReading, now time.Time) {
+	if len(psus) == 0 {
+		return
+	}
+	m := make(map[string]float64, len(psus))
+	for i, p := range psus {
+		if p.PowerW > 0 {
+			m[strconv.Itoa(i)] = p.PowerW
+		}
+	}
+	psuPeaks.observe(m, now)
+}
+
+// ObservedPSUMaxW returns the persisted per-PSU observed peak draw, keyed by
+// ordinal ("0", "1", …), or nil if none recorded yet.
+func ObservedPSUMaxW() map[string]float64 {
+	p := psuPeaks.snapshot()
+	if len(p) == 0 {
+		return nil
+	}
+	return p
+}

audit/internal/platform/sat_fan_stress.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"math"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -44,33 +43,16 @@ type cachedPowerReading struct {
 	UpdatedAt time.Time
 }
 
-type fanObservationState struct {
-	MaxRPM map[string]float64 `json:"max_rpm"`
-}
-
-type fanPeakCandidate struct {
-	FirstSeen time.Time
-	RPM       float64
-}
-
-var (
-	fanObservationMu   sync.Mutex
-	fanObservation     fanObservationState
-	fanObservationInit bool
-	fanPeakCandidates  = make(map[string]fanPeakCandidate)
-)
-
 const systemPowerHoldTTL = 15 * time.Second
 
-var fanObservationStatePath = "/var/log/bee-sat/fan-observation.json"
-
-const fanObservationMinPeakHold = time.Second
-
-func normalizeObservedFanMaxRPM(rpm float64) float64 {
-	if rpm <= 0 {
-		return 0
-	}
-	return math.Ceil(rpm/1000.0) * 1000.0
+// fanPeaks is the observed top-RPM store — the "autotune" for fan ceilings.
+// Any full-load run (Fan Ceiling Check, burn, thermal cycling) feeds it via
+// updateFanObservation; ResolveFanMaxRPM / ObservedFanMaxRPM read it back.
+var fanPeaks = &observedPeakStore{
+	path:    "/var/log/bee-sat/fan-observation.json",
+	jsonKey: "max_rpm",
+	roundUp: 1000,
+	minHold: time.Second,
 }
 
 // RunFanCheck drives CPU (+memory) and, when a GPU is present, GPU load to
@@ -240,6 +222,12 @@ func (s *System) RunFanCheck(ctx context.Context, baseDir string, opts FanCheckO
 	goodSamples := 0
 	poll := fanPollFloor
 
+	// PSU peak draw, sampled at a slow cadence off the same loop — the fan
+	// check already drives the box to full power, so it is also the right run
+	// to observe what each PSU tops out at (updatePSUObservation persists it).
+	psuPeakW := map[int]float64{}
+	lastPSUSec := -1e9
+
 	csvPath := filepath.Join(runDir, "fan-sensors.csv")
 	_ = os.WriteFile(csvPath, []byte("elapsed_sec,fan_name,rpm\n"), 0644)
 	csvFile, _ := os.OpenFile(csvPath, os.O_APPEND|os.O_WRONLY, 0644)
@@ -281,6 +269,17 @@ loop:
 		}
 		goodSamples++
 
+		if elapsed-lastPSUSec >= 15 {
+			lastPSUSec = elapsed
+			if ps, ok := readPSUPowerBounded(fanReadTMO); ok {
+				for i, p := range ps {
+					if p.PowerW > psuPeakW[i] {
+						psuPeakW[i] = p.PowerW
+					}
+				}
+			}
+		}
+
 		for _, f := range fans {
 			if csvFile != nil {
 				fmt.Fprintf(csvFile, "%.0f,%s,%.0f\n", elapsed, f.Name, f.RPM)
@@ -347,6 +346,16 @@ loop:
 	if t := boundedGPUMaxTemp(opts.GPUIndices); t > 0 {
 		fmt.Fprintf(&summary, "gpu_temp_c=%.0f\n", t)
 	}
+	if len(psuPeakW) > 0 {
+		idx := make([]int, 0, len(psuPeakW))
+		for i := range psuPeakW {
+			idx = append(idx, i)
+		}
+		sort.Ints(idx)
+		for _, i := range idx {
+			fmt.Fprintf(&summary, "psu_%d_peak_w=%.0f\n", i, psuPeakW[i])
+		}
+	}
 
 	stats := satStats{}
 	names := make([]string, 0, len(baselineRPM))
@@ -441,6 +450,21 @@ func readFansBounded(timeout time.Duration) ([]FanReading, bool) {
 	}
 }
 
+// readPSUPowerBounded is readFansBounded's PSU sibling: a time-boxed,
+// abandonable "ipmitool sdr" read of per-PSU power (samplePSUPower also feeds
+// the observed-capacity store). Used at a slow cadence during the fan check so
+// the same max-load run that finds fan ceilings also records peak PSU draw.
+func readPSUPowerBounded(timeout time.Duration) ([]PSUReading, bool) {
+	ch := make(chan []PSUReading, 1)
+	go func() { ch <- samplePSUPower() }()
+	select {
+	case ps := <-ch:
+		return ps, len(ps) > 0
+	case <-time.After(timeout):
+		return nil, false
+	}
+}
+
 // boundedGPUMaxTemp returns the hottest GPU temperature via a single
 // time-boxed nvidia-smi call, or 0 if unavailable.
 func boundedGPUMaxTemp(gpuIndices []int) float64 {
@@ -569,8 +593,8 @@ func readFanStatuses() map[string]string {
 
 // ResolveFanMaxRPM returns, for every fan name in current (name -> current
 // RPM), the RPM to treat as that fan's 100% reference. Preference order:
-//  1. the persisted observed peak (fanObservationStatePath), written by
-//     RunFanCheck and by live-metrics sampling under load;
+//  1. the persisted observed peak, written by RunFanCheck and by live-metrics
+//     sampling under load;
 //  2. the largest peak observed on any peer fan (keeps a group visually
 //     consistent when only some fans have a recorded peak);
 //  3. the fan's own current RPM (so a tile is never sized against zero).
@@ -578,7 +602,7 @@ func readFanStatuses() map[string]string {
 // The fallback lives here, not in the view, so every consumer of a fan
 // maximum applies the same rule.
 func ResolveFanMaxRPM(current map[string]float64) map[string]float64 {
-	persisted := readPersistedFanMaxRPM()
+	persisted := fanPeaks.snapshot()
 
 	peerMax := 0.0
 	for _, v := range persisted {
@@ -621,122 +645,41 @@ func sampleFanSpeeds() ([]FanReading, error) {
 	return nil, sensorsErr
 }
 
-// readPersistedFanMaxRPM reads fanObservationStatePath and returns its
-// sanitized {fan name -> observed peak RPM} map (empty names / non-positive
-// values dropped). Returns an empty map when the file is missing or unparsable.
-func readPersistedFanMaxRPM() map[string]float64 {
-	out := map[string]float64{}
-	raw, err := os.ReadFile(fanObservationStatePath)
-	if err != nil || len(raw) == 0 {
-		return out
-	}
-	var persisted fanObservationState
-	if json.Unmarshal(raw, &persisted) != nil {
-		return out
-	}
-	for name, rpm := range persisted.MaxRPM {
-		name = strings.TrimSpace(name)
-		if name == "" || rpm <= 0 {
-			continue
-		}
-		out[name] = rpm
-	}
-	return out
-}
-
 // ObservedFanMaxRPM returns the per-fan observed peak RPM map persisted by
-// fan-stress SAT runs, or nil if none is recorded yet. It reads the file
-// directly without touching the in-process observation cache or its lock, so
-// read-only consumers (the /topo web view) can call it without perturbing a
-// concurrent SAT run's peak tracking.
+// full-load runs, or nil if none is recorded yet.
 func ObservedFanMaxRPM() map[string]float64 {
-	out := readPersistedFanMaxRPM()
+	out := fanPeaks.snapshot()
 	if len(out) == 0 {
 		return nil
 	}
 	return out
 }
 
-func loadFanObservationLocked() {
-	if fanObservationInit {
-		return
-	}
-	fanObservationInit = true
-	fanObservation.MaxRPM = readPersistedFanMaxRPM()
-}
-
-func saveFanObservationLocked() {
-	if len(fanObservation.MaxRPM) == 0 {
-		return
-	}
-	dir := filepath.Dir(fanObservationStatePath)
-	if dir == "" || dir == "." {
-		dir = "/var/log/bee-sat"
-	}
-	if err := os.MkdirAll(dir, 0755); err != nil {
-		return
-	}
-	raw, err := json.MarshalIndent(fanObservation, "", "  ")
-	if err != nil {
-		return
-	}
-	_ = os.WriteFile(fanObservationStatePath, raw, 0644)
-}
-
 func updateFanObservation(fans []FanReading, now time.Time) {
 	if len(fans) == 0 {
 		return
 	}
-	fanObservationMu.Lock()
-	defer fanObservationMu.Unlock()
-	loadFanObservationLocked()
-	changed := false
+	m := make(map[string]float64, len(fans))
 	for _, fan := range fans {
-		name := strings.TrimSpace(fan.Name)
-		if name == "" || fan.RPM <= 0 {
-			continue
+		if n := strings.TrimSpace(fan.Name); n != "" && fan.RPM > 0 {
+			m[n] = fan.RPM
 		}
-		currentMax := fanObservation.MaxRPM[name]
-		if fan.RPM <= currentMax {
-			delete(fanPeakCandidates, name)
-			continue
-		}
-		if cand, ok := fanPeakCandidates[name]; ok {
-			if now.Sub(cand.FirstSeen) >= fanObservationMinPeakHold {
-				newMax := math.Max(cand.RPM, fan.RPM)
-				if newMax > currentMax {
-					fanObservation.MaxRPM[name] = normalizeObservedFanMaxRPM(newMax)
-					changed = true
-				}
-				delete(fanPeakCandidates, name)
-				continue
-			}
-			if fan.RPM > cand.RPM {
-				fanPeakCandidates[name] = fanPeakCandidate{FirstSeen: cand.FirstSeen, RPM: fan.RPM}
-			}
-			continue
-		}
-		fanPeakCandidates[name] = fanPeakCandidate{FirstSeen: now, RPM: fan.RPM}
-	}
-	if changed {
-		saveFanObservationLocked()
 	}
+	fanPeaks.observe(m, now)
 }
 
 func estimateFanDutyCyclePctFromObservation(fans []FanReading) (float64, bool) {
 	if len(fans) == 0 {
 		return 0, false
 	}
-	fanObservationMu.Lock()
-	defer fanObservationMu.Unlock()
-	loadFanObservationLocked()
+	maxByName := fanPeaks.snapshot()
 	var samples []float64
 	for _, fan := range fans {
 		name := strings.TrimSpace(fan.Name)
 		if name == "" || fan.RPM <= 0 {
 			continue
 		}
-		maxRPM := fanObservation.MaxRPM[name]
+		maxRPM := maxByName[name]
 		if maxRPM <= 0 {
 			continue
 		}

audit/internal/platform/sat_fan_stress_test.go

@@ -8,15 +8,20 @@ import (
 	"time"
 )
 
+// resetPeakStore points a store at a fresh temp file and clears its cache for
+// the duration of the test.
+func resetPeakStore(t *testing.T, s *observedPeakStore) {
+	t.Helper()
+	old := *s
+	s.path = filepath.Join(t.TempDir(), "peaks.json")
+	s.loaded = false
+	s.peaks = nil
+	s.candidates = nil
+	t.Cleanup(func() { *s = old })
+}
+
 func TestResolveFanMaxRPM(t *testing.T) {
-	oldPath := fanObservationStatePath
-	oldInit := fanObservationInit
-	fanObservationStatePath = filepath.Join(t.TempDir(), "fan-observation.json")
-	fanObservationInit = false
-	t.Cleanup(func() {
-		fanObservationStatePath = oldPath
-		fanObservationInit = oldInit
-	})
+	resetPeakStore(t, fanPeaks)
 
 	// No persisted file yet: unknown fans fall back to their own current RPM.
 	got := ResolveFanMaxRPM(map[string]float64{"A": 4000, "B": 9000})
@@ -24,7 +29,7 @@ func TestResolveFanMaxRPM(t *testing.T) {
 		t.Fatalf("no-persist fallback: got %v", got)
 	}
 
-	if err := os.WriteFile(fanObservationStatePath, []byte(`{"max_rpm":{"A":17000}}`), 0644); err != nil {
+	if err := os.WriteFile(fanPeaks.path, []byte(`{"max_rpm":{"A":17000}}`), 0644); err != nil {
 		t.Fatal(err)
 	}
 	got = ResolveFanMaxRPM(map[string]float64{"A": 4000, "B": 9000, "C": 5000})
@@ -34,6 +39,49 @@ func TestResolveFanMaxRPM(t *testing.T) {
 	}
 }
 
+func TestObservedPeakStore(t *testing.T) {
+	s := &observedPeakStore{
+		path:    filepath.Join(t.TempDir(), "peaks.json"),
+		jsonKey: "max_w",
+		roundUp: 50,
+		minHold: time.Second,
+	}
+	t0 := time.Unix(0, 0)
+
+	// A single spike does not stick.
+	s.observe(map[string]float64{"0": 900}, t0)
+	if len(s.snapshot()) != 0 {
+		t.Fatalf("transient spike should not persist: %v", s.snapshot())
+	}
+	// Held past minHold → sticks, rounded up to the next 50.
+	s.observe(map[string]float64{"0": 920}, t0.Add(1200*time.Millisecond))
+	if got := s.snapshot()["0"]; got != 950 {
+		t.Fatalf("held peak: got %v want 950", got)
+	}
+	// A lower reading never lowers the peak.
+	s.observe(map[string]float64{"0": 400}, t0.Add(5*time.Second))
+	if got := s.snapshot()["0"]; got != 950 {
+		t.Fatalf("peak must not drop: got %v", got)
+	}
+	// Fresh store reloads from disk.
+	s2 := &observedPeakStore{path: s.path, jsonKey: "max_w"}
+	if got := s2.snapshot()["0"]; got != 950 {
+		t.Fatalf("reload from disk: got %v want 950", got)
+	}
+}
+
+func TestUpdatePSUObservationKeyedByOrdinal(t *testing.T) {
+	resetPeakStore(t, psuPeaks)
+	now := time.Unix(0, 0)
+	psus := []PSUReading{{Name: "PSU1", PowerW: 1200}, {Name: "PSU2", PowerW: 1400}}
+	updatePSUObservation(psus, now)
+	updatePSUObservation(psus, now.Add(1200*time.Millisecond))
+	got := ObservedPSUMaxW()
+	if got["0"] != 1200 || got["1"] != 1400 {
+		t.Fatalf("keyed-by-ordinal peaks: got %v", got)
+	}
+}
+
 func TestApplyFanCheckDefaults(t *testing.T) {
 	var o FanCheckOptions
 	applyFanCheckDefaults(&o)
@@ -109,22 +157,7 @@ func TestParseFanDutyCyclePctSensorsJSON(t *testing.T) {
 }
 
 func TestEstimateFanDutyCyclePctFromObservation(t *testing.T) {
-	t.Parallel()
-
-	oldPath := fanObservationStatePath
-	oldState := fanObservation
-	oldInit := fanObservationInit
-	oldCandidates := fanPeakCandidates
-	fanObservationStatePath = filepath.Join(t.TempDir(), "fan-observation.json")
-	fanObservation = fanObservationState{}
-	fanObservationInit = false
-	fanPeakCandidates = make(map[string]fanPeakCandidate)
-	t.Cleanup(func() {
-		fanObservationStatePath = oldPath
-		fanObservation = oldState
-		fanObservationInit = oldInit
-		fanPeakCandidates = oldCandidates
-	})
+	resetPeakStore(t, fanPeaks)
 
 	start := time.Unix(100, 0)
 	updateFanObservation([]FanReading{{Name: "FAN1", RPM: 5000}}, start)
@@ -143,9 +176,9 @@ func TestEstimateFanDutyCyclePctFromObservation(t *testing.T) {
 		t.Fatalf("got=%v want ~43.3", got)
 	}
 
-	fanObservation = fanObservationState{}
-	fanObservationInit = false
-	fanPeakCandidates = make(map[string]fanPeakCandidate)
+	fanPeaks.loaded = false
+	fanPeaks.peaks = nil
+	fanPeaks.candidates = nil
 	got, ok = estimateFanDutyCyclePctFromObservation([]FanReading{{Name: "FAN1", RPM: 2600}})
 	if !ok {
 		t.Fatalf("expected persisted observed max to be reloaded from disk")