feat: autotune PSU capacity from the same full-load run as fan ceilings
Extract the fan peak-tracking into observedPeakStore (observe max under load, hold >= minHold to reject spikes, round, persist JSON) and add a second instance for PSU draw (psu-observation.json, keyed by PSU ordinal). Fed from samplePSUPower like fans are from sampleFanSpeeds, so any full-load run refines it — the Fan Ceiling Check (which also samples PSU power at a slow cadence off its loop and writes psu_<i>_peak_w), a burn, thermal cycling, and the 5s web metrics collector. /topo PSU cards now scale the load fill by wattage_w when the BMC reports it, else by the observed peak draw — marked "~N% load". This MSI stand's BMC gives only instantaneous input power, so the observed peak is the only capacity figure available. Fan behaviour is unchanged (tests exercise updateFanObservation / estimateFanDutyCyclePctFromObservation / ResolveFanMaxRPM through the new store). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019VHG21rgTUiR1G3qFHTVmN
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
e713504fc9
commit
7ed652c5b1
@@ -393,5 +393,9 @@ func samplePSUPower() []PSUReading {
|
||||
if len(psus) == 0 {
|
||||
return nil
|
||||
}
|
||||
// Feed the observed-capacity store (the "autotune" for PSU load scaling on
|
||||
// BMCs that report only instantaneous power) — every load run that samples
|
||||
// PSU power, including the 5 s metrics collector, refines it.
|
||||
updatePSUObservation(psus, time.Now())
|
||||
return psus
|
||||
}
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// observedPeakStore is the "autotune" primitive for components that expose no
|
||||
// host-readable nameplate maximum: fan top RPM, PSU capacity. It records the
|
||||
// highest value seen per key while the box is under load, persists it to a
|
||||
// JSON file, and hands it back so live readings can be scaled against a real
|
||||
// maximum. A new peak only sticks after it has been held for at least
|
||||
// minHold, which rejects transient spikes.
|
||||
type observedPeakStore struct {
|
||||
path string // JSON file
|
||||
jsonKey string // top-level object key, e.g. "max_rpm"
|
||||
roundUp float64 // round a new peak up to this multiple; 0 = keep raw
|
||||
minHold time.Duration // a candidate peak must persist this long to stick
|
||||
|
||||
mu sync.Mutex
|
||||
loaded bool
|
||||
peaks map[string]float64
|
||||
candidates map[string]peakCandidate
|
||||
}
|
||||
|
||||
type peakCandidate struct {
|
||||
firstSeen time.Time
|
||||
val float64
|
||||
}
|
||||
|
||||
// persistedPeaks reads the file fresh (no lock, no cache mutation) and returns
|
||||
// its sanitized {key -> peak} map. Empty map when the file is missing or
|
||||
// unparsable.
|
||||
func (s *observedPeakStore) persistedPeaks() map[string]float64 {
|
||||
out := map[string]float64{}
|
||||
raw, err := os.ReadFile(s.path)
|
||||
if err != nil || len(raw) == 0 {
|
||||
return out
|
||||
}
|
||||
var doc map[string]map[string]float64
|
||||
if json.Unmarshal(raw, &doc) != nil {
|
||||
return out
|
||||
}
|
||||
for k, v := range doc[s.jsonKey] {
|
||||
k = strings.TrimSpace(k)
|
||||
if k == "" || v <= 0 {
|
||||
continue
|
||||
}
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (s *observedPeakStore) loadLocked() {
|
||||
if s.loaded {
|
||||
return
|
||||
}
|
||||
s.loaded = true
|
||||
s.peaks = s.persistedPeaks()
|
||||
if s.candidates == nil {
|
||||
s.candidates = map[string]peakCandidate{}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *observedPeakStore) saveLocked() {
|
||||
if len(s.peaks) == 0 {
|
||||
return
|
||||
}
|
||||
dir := filepath.Dir(s.path)
|
||||
if dir == "" || dir == "." {
|
||||
return
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return
|
||||
}
|
||||
raw, err := json.MarshalIndent(map[string]map[string]float64{s.jsonKey: s.peaks}, "", " ")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_ = os.WriteFile(s.path, raw, 0644)
|
||||
}
|
||||
|
||||
func (s *observedPeakStore) round(v float64) float64 {
|
||||
if v <= 0 {
|
||||
return 0
|
||||
}
|
||||
if s.roundUp <= 0 {
|
||||
return v
|
||||
}
|
||||
return math.Ceil(v/s.roundUp) * s.roundUp
|
||||
}
|
||||
|
||||
// observe feeds one telemetry sample (key -> current value). Non-positive
|
||||
// values and blank keys are ignored.
|
||||
func (s *observedPeakStore) observe(samples map[string]float64, now time.Time) {
|
||||
if len(samples) == 0 {
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.loadLocked()
|
||||
changed := false
|
||||
for key, val := range samples {
|
||||
key = strings.TrimSpace(key)
|
||||
if key == "" || val <= 0 {
|
||||
continue
|
||||
}
|
||||
cur := s.peaks[key]
|
||||
if val <= cur {
|
||||
delete(s.candidates, key)
|
||||
continue
|
||||
}
|
||||
if cand, ok := s.candidates[key]; ok {
|
||||
if now.Sub(cand.firstSeen) >= s.minHold {
|
||||
nv := math.Max(cand.val, val)
|
||||
if nv > cur {
|
||||
s.peaks[key] = s.round(nv)
|
||||
changed = true
|
||||
}
|
||||
delete(s.candidates, key)
|
||||
continue
|
||||
}
|
||||
if val > cand.val {
|
||||
s.candidates[key] = peakCandidate{firstSeen: cand.firstSeen, val: val}
|
||||
}
|
||||
continue
|
||||
}
|
||||
s.candidates[key] = peakCandidate{firstSeen: now, val: val}
|
||||
}
|
||||
if changed {
|
||||
s.saveLocked()
|
||||
}
|
||||
}
|
||||
|
||||
// snapshot returns the persisted peaks (fresh from disk), for read-only
|
||||
// consumers such as the /topo web view.
|
||||
func (s *observedPeakStore) snapshot() map[string]float64 {
|
||||
return s.persistedPeaks()
|
||||
}
|
||||
|
||||
// ── PSU capacity ────────────────────────────────────────────────────────────
|
||||
|
||||
var psuPeaks = &observedPeakStore{
|
||||
path: "/var/log/bee-sat/psu-observation.json",
|
||||
jsonKey: "max_w",
|
||||
roundUp: 50,
|
||||
minHold: time.Second,
|
||||
}
|
||||
|
||||
// updatePSUObservation feeds the current per-PSU draw (keyed by ordinal, in
|
||||
// the order the caller lists them) into the observed-capacity store. On a BMC
|
||||
// that reports only instantaneous input power this is the only way to know
|
||||
// what "100% load" looks like for each supply: observe the peak draw during
|
||||
// any full-load run (the Fan Ceiling Check, a burn, thermal cycling — the 5 s
|
||||
// metrics collector samples PSUs throughout) and remember it.
|
||||
func updatePSUObservation(psus []PSUReading, now time.Time) {
|
||||
if len(psus) == 0 {
|
||||
return
|
||||
}
|
||||
m := make(map[string]float64, len(psus))
|
||||
for i, p := range psus {
|
||||
if p.PowerW > 0 {
|
||||
m[strconv.Itoa(i)] = p.PowerW
|
||||
}
|
||||
}
|
||||
psuPeaks.observe(m, now)
|
||||
}
|
||||
|
||||
// ObservedPSUMaxW returns the persisted per-PSU observed peak draw, keyed by
|
||||
// ordinal ("0", "1", …), or nil if none recorded yet.
|
||||
func ObservedPSUMaxW() map[string]float64 {
|
||||
p := psuPeaks.snapshot()
|
||||
if len(p) == 0 {
|
||||
return nil
|
||||
}
|
||||
return p
|
||||
}
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
@@ -44,33 +43,16 @@ type cachedPowerReading struct {
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
type fanObservationState struct {
|
||||
MaxRPM map[string]float64 `json:"max_rpm"`
|
||||
}
|
||||
|
||||
type fanPeakCandidate struct {
|
||||
FirstSeen time.Time
|
||||
RPM float64
|
||||
}
|
||||
|
||||
var (
|
||||
fanObservationMu sync.Mutex
|
||||
fanObservation fanObservationState
|
||||
fanObservationInit bool
|
||||
fanPeakCandidates = make(map[string]fanPeakCandidate)
|
||||
)
|
||||
|
||||
const systemPowerHoldTTL = 15 * time.Second
|
||||
|
||||
var fanObservationStatePath = "/var/log/bee-sat/fan-observation.json"
|
||||
|
||||
const fanObservationMinPeakHold = time.Second
|
||||
|
||||
func normalizeObservedFanMaxRPM(rpm float64) float64 {
|
||||
if rpm <= 0 {
|
||||
return 0
|
||||
}
|
||||
return math.Ceil(rpm/1000.0) * 1000.0
|
||||
// fanPeaks is the observed top-RPM store — the "autotune" for fan ceilings.
|
||||
// Any full-load run (Fan Ceiling Check, burn, thermal cycling) feeds it via
|
||||
// updateFanObservation; ResolveFanMaxRPM / ObservedFanMaxRPM read it back.
|
||||
var fanPeaks = &observedPeakStore{
|
||||
path: "/var/log/bee-sat/fan-observation.json",
|
||||
jsonKey: "max_rpm",
|
||||
roundUp: 1000,
|
||||
minHold: time.Second,
|
||||
}
|
||||
|
||||
// RunFanCheck drives CPU (+memory) and, when a GPU is present, GPU load to
|
||||
@@ -240,6 +222,12 @@ func (s *System) RunFanCheck(ctx context.Context, baseDir string, opts FanCheckO
|
||||
goodSamples := 0
|
||||
poll := fanPollFloor
|
||||
|
||||
// PSU peak draw, sampled at a slow cadence off the same loop — the fan
|
||||
// check already drives the box to full power, so it is also the right run
|
||||
// to observe what each PSU tops out at (updatePSUObservation persists it).
|
||||
psuPeakW := map[int]float64{}
|
||||
lastPSUSec := -1e9
|
||||
|
||||
csvPath := filepath.Join(runDir, "fan-sensors.csv")
|
||||
_ = os.WriteFile(csvPath, []byte("elapsed_sec,fan_name,rpm\n"), 0644)
|
||||
csvFile, _ := os.OpenFile(csvPath, os.O_APPEND|os.O_WRONLY, 0644)
|
||||
@@ -281,6 +269,17 @@ loop:
|
||||
}
|
||||
goodSamples++
|
||||
|
||||
if elapsed-lastPSUSec >= 15 {
|
||||
lastPSUSec = elapsed
|
||||
if ps, ok := readPSUPowerBounded(fanReadTMO); ok {
|
||||
for i, p := range ps {
|
||||
if p.PowerW > psuPeakW[i] {
|
||||
psuPeakW[i] = p.PowerW
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, f := range fans {
|
||||
if csvFile != nil {
|
||||
fmt.Fprintf(csvFile, "%.0f,%s,%.0f\n", elapsed, f.Name, f.RPM)
|
||||
@@ -347,6 +346,16 @@ loop:
|
||||
if t := boundedGPUMaxTemp(opts.GPUIndices); t > 0 {
|
||||
fmt.Fprintf(&summary, "gpu_temp_c=%.0f\n", t)
|
||||
}
|
||||
if len(psuPeakW) > 0 {
|
||||
idx := make([]int, 0, len(psuPeakW))
|
||||
for i := range psuPeakW {
|
||||
idx = append(idx, i)
|
||||
}
|
||||
sort.Ints(idx)
|
||||
for _, i := range idx {
|
||||
fmt.Fprintf(&summary, "psu_%d_peak_w=%.0f\n", i, psuPeakW[i])
|
||||
}
|
||||
}
|
||||
|
||||
stats := satStats{}
|
||||
names := make([]string, 0, len(baselineRPM))
|
||||
@@ -441,6 +450,21 @@ func readFansBounded(timeout time.Duration) ([]FanReading, bool) {
|
||||
}
|
||||
}
|
||||
|
||||
// readPSUPowerBounded is readFansBounded's PSU sibling: a time-boxed,
|
||||
// abandonable "ipmitool sdr" read of per-PSU power (samplePSUPower also feeds
|
||||
// the observed-capacity store). Used at a slow cadence during the fan check so
|
||||
// the same max-load run that finds fan ceilings also records peak PSU draw.
|
||||
func readPSUPowerBounded(timeout time.Duration) ([]PSUReading, bool) {
|
||||
ch := make(chan []PSUReading, 1)
|
||||
go func() { ch <- samplePSUPower() }()
|
||||
select {
|
||||
case ps := <-ch:
|
||||
return ps, len(ps) > 0
|
||||
case <-time.After(timeout):
|
||||
return nil, false
|
||||
}
|
||||
}
|
||||
|
||||
// boundedGPUMaxTemp returns the hottest GPU temperature via a single
|
||||
// time-boxed nvidia-smi call, or 0 if unavailable.
|
||||
func boundedGPUMaxTemp(gpuIndices []int) float64 {
|
||||
@@ -569,8 +593,8 @@ func readFanStatuses() map[string]string {
|
||||
|
||||
// ResolveFanMaxRPM returns, for every fan name in current (name -> current
|
||||
// RPM), the RPM to treat as that fan's 100% reference. Preference order:
|
||||
// 1. the persisted observed peak (fanObservationStatePath), written by
|
||||
// RunFanCheck and by live-metrics sampling under load;
|
||||
// 1. the persisted observed peak, written by RunFanCheck and by live-metrics
|
||||
// sampling under load;
|
||||
// 2. the largest peak observed on any peer fan (keeps a group visually
|
||||
// consistent when only some fans have a recorded peak);
|
||||
// 3. the fan's own current RPM (so a tile is never sized against zero).
|
||||
@@ -578,7 +602,7 @@ func readFanStatuses() map[string]string {
|
||||
// The fallback lives here, not in the view, so every consumer of a fan
|
||||
// maximum applies the same rule.
|
||||
func ResolveFanMaxRPM(current map[string]float64) map[string]float64 {
|
||||
persisted := readPersistedFanMaxRPM()
|
||||
persisted := fanPeaks.snapshot()
|
||||
|
||||
peerMax := 0.0
|
||||
for _, v := range persisted {
|
||||
@@ -621,122 +645,41 @@ func sampleFanSpeeds() ([]FanReading, error) {
|
||||
return nil, sensorsErr
|
||||
}
|
||||
|
||||
// readPersistedFanMaxRPM reads fanObservationStatePath and returns its
|
||||
// sanitized {fan name -> observed peak RPM} map (empty names / non-positive
|
||||
// values dropped). Returns an empty map when the file is missing or unparsable.
|
||||
func readPersistedFanMaxRPM() map[string]float64 {
|
||||
out := map[string]float64{}
|
||||
raw, err := os.ReadFile(fanObservationStatePath)
|
||||
if err != nil || len(raw) == 0 {
|
||||
return out
|
||||
}
|
||||
var persisted fanObservationState
|
||||
if json.Unmarshal(raw, &persisted) != nil {
|
||||
return out
|
||||
}
|
||||
for name, rpm := range persisted.MaxRPM {
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" || rpm <= 0 {
|
||||
continue
|
||||
}
|
||||
out[name] = rpm
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ObservedFanMaxRPM returns the per-fan observed peak RPM map persisted by
|
||||
// fan-stress SAT runs, or nil if none is recorded yet. It reads the file
|
||||
// directly without touching the in-process observation cache or its lock, so
|
||||
// read-only consumers (the /topo web view) can call it without perturbing a
|
||||
// concurrent SAT run's peak tracking.
|
||||
// full-load runs, or nil if none is recorded yet.
|
||||
func ObservedFanMaxRPM() map[string]float64 {
|
||||
out := readPersistedFanMaxRPM()
|
||||
out := fanPeaks.snapshot()
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func loadFanObservationLocked() {
|
||||
if fanObservationInit {
|
||||
return
|
||||
}
|
||||
fanObservationInit = true
|
||||
fanObservation.MaxRPM = readPersistedFanMaxRPM()
|
||||
}
|
||||
|
||||
func saveFanObservationLocked() {
|
||||
if len(fanObservation.MaxRPM) == 0 {
|
||||
return
|
||||
}
|
||||
dir := filepath.Dir(fanObservationStatePath)
|
||||
if dir == "" || dir == "." {
|
||||
dir = "/var/log/bee-sat"
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return
|
||||
}
|
||||
raw, err := json.MarshalIndent(fanObservation, "", " ")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_ = os.WriteFile(fanObservationStatePath, raw, 0644)
|
||||
}
|
||||
|
||||
func updateFanObservation(fans []FanReading, now time.Time) {
|
||||
if len(fans) == 0 {
|
||||
return
|
||||
}
|
||||
fanObservationMu.Lock()
|
||||
defer fanObservationMu.Unlock()
|
||||
loadFanObservationLocked()
|
||||
changed := false
|
||||
m := make(map[string]float64, len(fans))
|
||||
for _, fan := range fans {
|
||||
name := strings.TrimSpace(fan.Name)
|
||||
if name == "" || fan.RPM <= 0 {
|
||||
continue
|
||||
if n := strings.TrimSpace(fan.Name); n != "" && fan.RPM > 0 {
|
||||
m[n] = fan.RPM
|
||||
}
|
||||
currentMax := fanObservation.MaxRPM[name]
|
||||
if fan.RPM <= currentMax {
|
||||
delete(fanPeakCandidates, name)
|
||||
continue
|
||||
}
|
||||
if cand, ok := fanPeakCandidates[name]; ok {
|
||||
if now.Sub(cand.FirstSeen) >= fanObservationMinPeakHold {
|
||||
newMax := math.Max(cand.RPM, fan.RPM)
|
||||
if newMax > currentMax {
|
||||
fanObservation.MaxRPM[name] = normalizeObservedFanMaxRPM(newMax)
|
||||
changed = true
|
||||
}
|
||||
delete(fanPeakCandidates, name)
|
||||
continue
|
||||
}
|
||||
if fan.RPM > cand.RPM {
|
||||
fanPeakCandidates[name] = fanPeakCandidate{FirstSeen: cand.FirstSeen, RPM: fan.RPM}
|
||||
}
|
||||
continue
|
||||
}
|
||||
fanPeakCandidates[name] = fanPeakCandidate{FirstSeen: now, RPM: fan.RPM}
|
||||
}
|
||||
if changed {
|
||||
saveFanObservationLocked()
|
||||
}
|
||||
fanPeaks.observe(m, now)
|
||||
}
|
||||
|
||||
func estimateFanDutyCyclePctFromObservation(fans []FanReading) (float64, bool) {
|
||||
if len(fans) == 0 {
|
||||
return 0, false
|
||||
}
|
||||
fanObservationMu.Lock()
|
||||
defer fanObservationMu.Unlock()
|
||||
loadFanObservationLocked()
|
||||
maxByName := fanPeaks.snapshot()
|
||||
var samples []float64
|
||||
for _, fan := range fans {
|
||||
name := strings.TrimSpace(fan.Name)
|
||||
if name == "" || fan.RPM <= 0 {
|
||||
continue
|
||||
}
|
||||
maxRPM := fanObservation.MaxRPM[name]
|
||||
maxRPM := maxByName[name]
|
||||
if maxRPM <= 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -8,15 +8,20 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// resetPeakStore points a store at a fresh temp file and clears its cache for
|
||||
// the duration of the test.
|
||||
func resetPeakStore(t *testing.T, s *observedPeakStore) {
|
||||
t.Helper()
|
||||
old := *s
|
||||
s.path = filepath.Join(t.TempDir(), "peaks.json")
|
||||
s.loaded = false
|
||||
s.peaks = nil
|
||||
s.candidates = nil
|
||||
t.Cleanup(func() { *s = old })
|
||||
}
|
||||
|
||||
func TestResolveFanMaxRPM(t *testing.T) {
|
||||
oldPath := fanObservationStatePath
|
||||
oldInit := fanObservationInit
|
||||
fanObservationStatePath = filepath.Join(t.TempDir(), "fan-observation.json")
|
||||
fanObservationInit = false
|
||||
t.Cleanup(func() {
|
||||
fanObservationStatePath = oldPath
|
||||
fanObservationInit = oldInit
|
||||
})
|
||||
resetPeakStore(t, fanPeaks)
|
||||
|
||||
// No persisted file yet: unknown fans fall back to their own current RPM.
|
||||
got := ResolveFanMaxRPM(map[string]float64{"A": 4000, "B": 9000})
|
||||
@@ -24,7 +29,7 @@ func TestResolveFanMaxRPM(t *testing.T) {
|
||||
t.Fatalf("no-persist fallback: got %v", got)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(fanObservationStatePath, []byte(`{"max_rpm":{"A":17000}}`), 0644); err != nil {
|
||||
if err := os.WriteFile(fanPeaks.path, []byte(`{"max_rpm":{"A":17000}}`), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got = ResolveFanMaxRPM(map[string]float64{"A": 4000, "B": 9000, "C": 5000})
|
||||
@@ -34,6 +39,49 @@ func TestResolveFanMaxRPM(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestObservedPeakStore(t *testing.T) {
|
||||
s := &observedPeakStore{
|
||||
path: filepath.Join(t.TempDir(), "peaks.json"),
|
||||
jsonKey: "max_w",
|
||||
roundUp: 50,
|
||||
minHold: time.Second,
|
||||
}
|
||||
t0 := time.Unix(0, 0)
|
||||
|
||||
// A single spike does not stick.
|
||||
s.observe(map[string]float64{"0": 900}, t0)
|
||||
if len(s.snapshot()) != 0 {
|
||||
t.Fatalf("transient spike should not persist: %v", s.snapshot())
|
||||
}
|
||||
// Held past minHold → sticks, rounded up to the next 50.
|
||||
s.observe(map[string]float64{"0": 920}, t0.Add(1200*time.Millisecond))
|
||||
if got := s.snapshot()["0"]; got != 950 {
|
||||
t.Fatalf("held peak: got %v want 950", got)
|
||||
}
|
||||
// A lower reading never lowers the peak.
|
||||
s.observe(map[string]float64{"0": 400}, t0.Add(5*time.Second))
|
||||
if got := s.snapshot()["0"]; got != 950 {
|
||||
t.Fatalf("peak must not drop: got %v", got)
|
||||
}
|
||||
// Fresh store reloads from disk.
|
||||
s2 := &observedPeakStore{path: s.path, jsonKey: "max_w"}
|
||||
if got := s2.snapshot()["0"]; got != 950 {
|
||||
t.Fatalf("reload from disk: got %v want 950", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdatePSUObservationKeyedByOrdinal(t *testing.T) {
|
||||
resetPeakStore(t, psuPeaks)
|
||||
now := time.Unix(0, 0)
|
||||
psus := []PSUReading{{Name: "PSU1", PowerW: 1200}, {Name: "PSU2", PowerW: 1400}}
|
||||
updatePSUObservation(psus, now)
|
||||
updatePSUObservation(psus, now.Add(1200*time.Millisecond))
|
||||
got := ObservedPSUMaxW()
|
||||
if got["0"] != 1200 || got["1"] != 1400 {
|
||||
t.Fatalf("keyed-by-ordinal peaks: got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyFanCheckDefaults(t *testing.T) {
|
||||
var o FanCheckOptions
|
||||
applyFanCheckDefaults(&o)
|
||||
@@ -109,22 +157,7 @@ func TestParseFanDutyCyclePctSensorsJSON(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestEstimateFanDutyCyclePctFromObservation(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
oldPath := fanObservationStatePath
|
||||
oldState := fanObservation
|
||||
oldInit := fanObservationInit
|
||||
oldCandidates := fanPeakCandidates
|
||||
fanObservationStatePath = filepath.Join(t.TempDir(), "fan-observation.json")
|
||||
fanObservation = fanObservationState{}
|
||||
fanObservationInit = false
|
||||
fanPeakCandidates = make(map[string]fanPeakCandidate)
|
||||
t.Cleanup(func() {
|
||||
fanObservationStatePath = oldPath
|
||||
fanObservation = oldState
|
||||
fanObservationInit = oldInit
|
||||
fanPeakCandidates = oldCandidates
|
||||
})
|
||||
resetPeakStore(t, fanPeaks)
|
||||
|
||||
start := time.Unix(100, 0)
|
||||
updateFanObservation([]FanReading{{Name: "FAN1", RPM: 5000}}, start)
|
||||
@@ -143,9 +176,9 @@ func TestEstimateFanDutyCyclePctFromObservation(t *testing.T) {
|
||||
t.Fatalf("got=%v want ~43.3", got)
|
||||
}
|
||||
|
||||
fanObservation = fanObservationState{}
|
||||
fanObservationInit = false
|
||||
fanPeakCandidates = make(map[string]fanPeakCandidate)
|
||||
fanPeaks.loaded = false
|
||||
fanPeaks.peaks = nil
|
||||
fanPeaks.candidates = nil
|
||||
got, ok = estimateFanDutyCyclePctFromObservation([]FanReading{{Name: "FAN1", RPM: 2600}})
|
||||
if !ok {
|
||||
t.Fatalf("expected persisted observed max to be reloaded from disk")
|
||||
|
||||
@@ -594,7 +594,7 @@ func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string
|
||||
|
||||
hasPSU := len(hw.PowerSupplies) > 0
|
||||
if hasPSU {
|
||||
b.WriteString(renderTopoPSURow(hw.PowerSupplies))
|
||||
b.WriteString(renderTopoPSURow(hw.PowerSupplies, platform.ObservedPSUMaxW()))
|
||||
}
|
||||
|
||||
// Cooling fans — one small clickable square per fan. Square SIZE encodes
|
||||
@@ -619,10 +619,13 @@ func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string
|
||||
}
|
||||
|
||||
// renderTopoPSURow renders the POWER SUPPLIES row: one card per PSU, coloured
|
||||
// by that PSU's own status (a failed unit goes red on its own), showing input
|
||||
// voltage and draw. Cards click through to the shared PSU detail modal and
|
||||
// carry data-psu so topoLiveScript can refresh the wattage in place.
|
||||
func renderTopoPSURow(psus []schema.HardwarePowerSupply) string {
|
||||
// by that PSU's own status (a failed unit goes red on its own). The card shows
|
||||
// input voltage and draw, and a load fill rising from the bottom — same idea
|
||||
// as the fan duty-cycle fill. The load scale is the nameplate rating when the
|
||||
// BMC reports it; otherwise it is the observed peak draw (observedMaxW, keyed
|
||||
// by ordinal — the "autotune" recorded during any full-load run), and the
|
||||
// figure is marked as an estimate.
|
||||
func renderTopoPSURow(psus []schema.HardwarePowerSupply, observedMaxW map[string]float64) string {
|
||||
var b strings.Builder
|
||||
b.WriteString(topoRowHeading("Power Supplies"))
|
||||
b.WriteString(`<div style="display:flex;flex-wrap:wrap;gap:8px">`)
|
||||
@@ -640,14 +643,25 @@ func renderTopoPSURow(psus []schema.HardwarePowerSupply) string {
|
||||
if p.WattageW != nil && *p.WattageW > 0 {
|
||||
rating = *p.WattageW
|
||||
}
|
||||
// Load scale: true rating if known, else the observed peak draw.
|
||||
scaleMax := float64(rating)
|
||||
scaleEstimate := false
|
||||
if scaleMax <= 0 {
|
||||
if m := observedMaxW[strconv.Itoa(i)]; m > 0 {
|
||||
scaleMax = m
|
||||
scaleEstimate = true
|
||||
}
|
||||
}
|
||||
|
||||
var parts []string
|
||||
if p.InputVoltage != nil && *p.InputVoltage > 0 {
|
||||
parts = append(parts, fmt.Sprintf("%.0f V", *p.InputVoltage))
|
||||
}
|
||||
switch {
|
||||
case haveDraw && rating > 0:
|
||||
parts = append(parts, fmt.Sprintf("%.0f / %d W · %.0f%% load", draw, rating, draw/float64(rating)*100))
|
||||
case haveDraw && scaleMax > 0 && scaleEstimate:
|
||||
parts = append(parts, fmt.Sprintf("%.0f / ~%.0f W · ~%.0f%% load", draw, scaleMax, draw/scaleMax*100))
|
||||
case haveDraw && scaleMax > 0:
|
||||
parts = append(parts, fmt.Sprintf("%.0f / %d W · %.0f%% load", draw, rating, draw/scaleMax*100))
|
||||
case haveDraw:
|
||||
parts = append(parts, fmt.Sprintf("%.0f W", draw))
|
||||
case rating > 0:
|
||||
@@ -660,33 +674,29 @@ func renderTopoPSURow(psus []schema.HardwarePowerSupply) string {
|
||||
statusWord = topoSeverityStatus(p.Status)
|
||||
}
|
||||
|
||||
// Load fill (draw / rating) rises from the bottom of the card, the
|
||||
// same idea as the fan duty-cycle fill. Only drawn when the nameplate
|
||||
// rating is known — a raw wattage with nothing to scale it against is
|
||||
// not a load figure.
|
||||
loadPct := -1.0
|
||||
if haveDraw && rating > 0 {
|
||||
loadPct = draw / float64(rating) * 100
|
||||
if loadPct < 0 {
|
||||
loadPct = 0
|
||||
}
|
||||
if loadPct > 100 {
|
||||
loadPct = 100
|
||||
}
|
||||
}
|
||||
fillH := 0.0
|
||||
if loadPct >= 0 {
|
||||
fillH = loadPct
|
||||
if haveDraw && scaleMax > 0 {
|
||||
fillH = draw / scaleMax * 100
|
||||
if fillH < 0 {
|
||||
fillH = 0
|
||||
}
|
||||
if fillH > 100 {
|
||||
fillH = 100
|
||||
}
|
||||
}
|
||||
|
||||
voltAttr := ""
|
||||
if p.InputVoltage != nil && *p.InputVoltage > 0 {
|
||||
voltAttr = fmt.Sprintf("%.0f", *p.InputVoltage)
|
||||
}
|
||||
maxSrc := "rated"
|
||||
if scaleEstimate {
|
||||
maxSrc = "observed"
|
||||
}
|
||||
|
||||
fmt.Fprintf(&b, `<div class="topo-psu-tile" data-psu="%d" data-psu-max="%d" data-psu-v="%s" onclick="openComponentDetail('psu')" `+
|
||||
fmt.Fprintf(&b, `<div class="topo-psu-tile" data-psu="%d" data-psu-max="%.0f" data-psu-max-src="%s" data-psu-v="%s" onclick="openComponentDetail('psu')" `+
|
||||
`style="position:relative;overflow:hidden;cursor:pointer;min-width:104px;padding:8px 11px;border-radius:6px;background:var(--surface-2);border:1px solid %s;color:%s">`,
|
||||
i, rating, voltAttr, stroke, text)
|
||||
i, scaleMax, maxSrc, voltAttr, stroke, text)
|
||||
fmt.Fprintf(&b, `<div class="topo-psu-fill" style="position:absolute;left:0;right:0;bottom:0;height:%.0f%%;background:%s;opacity:.5;transition:height .8s linear"></div>`, fillH, stroke)
|
||||
fmt.Fprintf(&b, `<div style="position:relative"><div style="font-size:13px;font-weight:700">%s</div>`, html.EscapeString(label))
|
||||
fmt.Fprintf(&b, `<div class="topo-psu-detail" style="font-size:11px;opacity:.9;margin-top:2px">%s</div>`, html.EscapeString(detail))
|
||||
@@ -828,11 +838,15 @@ func topoLiveScript() string {
|
||||
psus.forEach(function(t){
|
||||
var p=m.psus[parseInt(t.dataset.psu,10)];if(!p)return;
|
||||
var w=p.power_w||0,max=parseFloat(t.dataset.psuMax)||0,v=t.dataset.psuV;
|
||||
var est=t.dataset.psuMaxSrc==='observed';
|
||||
var parts=[];
|
||||
if(v)parts.push(v+' V');
|
||||
if(w>0&&max>0)parts.push(Math.round(w)+' / '+max+' W · '+Math.round(w/max*100)+'% load');
|
||||
else if(w>0)parts.push(Math.round(w)+' W');
|
||||
else if(max>0)parts.push(max+' W rated');
|
||||
if(w>0&&max>0){
|
||||
var pct=Math.round(w/max*100);
|
||||
parts.push(est?Math.round(w)+' / ~'+Math.round(max)+' W · ~'+pct+'% load'
|
||||
:Math.round(w)+' / '+Math.round(max)+' W · '+pct+'% load');
|
||||
}else if(w>0)parts.push(Math.round(w)+' W');
|
||||
else if(max>0)parts.push(Math.round(max)+' W rated');
|
||||
var d=t.querySelector('.topo-psu-detail');if(d&&parts.length)d.textContent=parts.join(' · ');
|
||||
if(w>0&&max>0){var f=t.querySelector('.topo-psu-fill');if(f)f.style.height=Math.max(0,Math.min(100,w/max*100)).toFixed(0)+'%';}
|
||||
});
|
||||
|
||||
@@ -262,6 +262,27 @@ func TestFanSpinPeriodSec(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderTopoPSURowUsesObservedMaxWhenNoRating(t *testing.T) {
|
||||
ok := "OK"
|
||||
draw := 340.0
|
||||
psus := []schema.HardwarePowerSupply{
|
||||
{HardwareComponentStatus: schema.HardwareComponentStatus{Status: &ok}, InputPowerW: &draw}, // no WattageW
|
||||
}
|
||||
// Observed peak (keyed by ordinal) stands in for the missing nameplate.
|
||||
html := renderTopoPSURow(psus, map[string]float64{"0": 2400})
|
||||
if !strings.Contains(html, `data-psu-max-src="observed"`) {
|
||||
t.Fatalf("expected observed-scale marker: %s", html)
|
||||
}
|
||||
if !strings.Contains(html, "340 / ~2400 W · ~14% load") {
|
||||
t.Fatalf("expected estimated load line: %s", html)
|
||||
}
|
||||
// Without an observed peak either, just the raw watts, no fill.
|
||||
html = renderTopoPSURow(psus, nil)
|
||||
if strings.Contains(html, "% load") {
|
||||
t.Fatalf("no rating and no observed peak → no load figure: %s", html)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTopoPageRendersStorageDisksGroupedByType(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "audit.json")
|
||||
|
||||
@@ -57,10 +57,33 @@ The `fan` SAT test (`platform.RunFanCheck`, formerly the unwired
|
||||
fan rose meaningfully above baseline, declare the ceiling found and stop —
|
||||
**success**. `MaxLoadSec` (default 900 s) is a hard cap; hitting it is also
|
||||
success (the highest RPM seen is still recorded).
|
||||
4. Persist each peak through the existing `updateFanObservation` path
|
||||
4. Persist each peak through `updateFanObservation`
|
||||
(`/var/log/bee-sat/fan-observation.json`), which is what `ResolveFanMaxRPM`
|
||||
reads.
|
||||
|
||||
## The observed-peak store (autotune primitive)
|
||||
|
||||
`platform.observedPeakStore` (`observed_peaks.go`) is the shared mechanism:
|
||||
observe the max value per key while the box is under load, require a candidate
|
||||
to hold ≥ `minHold` before it sticks (rejects spikes), round up, persist to a
|
||||
`{"<jsonKey>": {key: peak}}` JSON file. Two instances:
|
||||
|
||||
- `fanPeaks` → `fan-observation.json` `max_rpm`, keyed by fan name, round-up 1000.
|
||||
- `psuPeaks` → `psu-observation.json` `max_w`, keyed by **PSU ordinal**, round-up 50.
|
||||
|
||||
Both are fed from the ordinary telemetry paths — `sampleFanSpeeds` and
|
||||
`samplePSUPower` call `update*Observation` — so **any** full-load run refines
|
||||
them: the Fan Ceiling Check itself, a burn, thermal cycling, power
|
||||
calibration, and the 5 s web metrics collector while any of those run. There
|
||||
is no separate "PSU autotune" test: the fan check's max-CPU+GPU load is
|
||||
already the right moment to observe peak PSU draw, and it samples PSU power at
|
||||
a slow cadence off its own loop (`psu_<i>_peak_w` in the summary).
|
||||
|
||||
This exists because BMCs like the MSI stand's report only instantaneous PSU
|
||||
input power, no nameplate rating. `/topo` PSU cards scale the load fill by the
|
||||
real `wattage_w` when present, otherwise by `ObservedPSUMaxW()` — marked as an
|
||||
estimate (`~N% load`).
|
||||
|
||||
## Verdict mapping
|
||||
|
||||
- A fan reading **0 RPM**, or IPMI status **cr/nr**, while under full load →
|
||||
|
||||
Reference in New Issue
Block a user