reanimator/bee
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: autotune PSU capacity from the same full-load run as fan ceilings

Browse Source 
Extract the fan peak-tracking into observedPeakStore (observe max under
load, hold >= minHold to reject spikes, round, persist JSON) and add a
second instance for PSU draw (psu-observation.json, keyed by PSU
ordinal). Fed from samplePSUPower like fans are from sampleFanSpeeds, so
any full-load run refines it — the Fan Ceiling Check (which also samples
PSU power at a slow cadence off its loop and writes psu_<i>_peak_w), a
burn, thermal cycling, and the 5s web metrics collector.

/topo PSU cards now scale the load fill by wattage_w when the BMC
reports it, else by the observed peak draw — marked "~N% load". This
MSI stand's BMC gives only instantaneous input power, so the observed
peak is the only capacity figure available.

Fan behaviour is unchanged (tests exercise updateFanObservation /
estimateFanDutyCyclePctFromObservation / ResolveFanMaxRPM through the
new store).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019VHG21rgTUiR1G3qFHTVmN
This commit is contained in:
Mikhail Chusavitin
2026-09-04 12:53:13 +03:00
co-authored by Claude Sonnet 5
parent e713504fc9
commit 7ed652c5b1
7 changed files with 396 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files
audit/internal/platform/sat_fan_stress.go
+61 -118
View File
@@ -4,7 +4,6 @@ import (
"context"
"encoding/json"
"fmt"
"math"
"os"
"os/exec"
"path/filepath"
@@ -44,33 +43,16 @@ type cachedPowerReading struct {
UpdatedAt time.Time
}
type fanObservationState struct {
MaxRPM map[string]float64 `json:"max_rpm"`
}
type fanPeakCandidate struct {
FirstSeen time.Time
RPM float64
}
var (
fanObservationMu sync.Mutex
fanObservation fanObservationState
fanObservationInit bool
fanPeakCandidates = make(map[string]fanPeakCandidate)
)
const systemPowerHoldTTL = 15 * time.Second
var fanObservationStatePath = "/var/log/bee-sat/fan-observation.json"
const fanObservationMinPeakHold = time.Second
func normalizeObservedFanMaxRPM(rpm float64) float64 {
if rpm <= 0 {
return 0
}
return math.Ceil(rpm/1000.0) * 1000.0
// fanPeaks is the observed top-RPM store — the "autotune" for fan ceilings.
// Any full-load run (Fan Ceiling Check, burn, thermal cycling) feeds it via
// updateFanObservation; ResolveFanMaxRPM / ObservedFanMaxRPM read it back.
var fanPeaks = &observedPeakStore{
path: "/var/log/bee-sat/fan-observation.json",
jsonKey: "max_rpm",
roundUp: 1000,
minHold: time.Second,
}
// RunFanCheck drives CPU (+memory) and, when a GPU is present, GPU load to
@@ -240,6 +222,12 @@ func (s *System) RunFanCheck(ctx context.Context, baseDir string, opts FanCheckO
goodSamples := 0
poll := fanPollFloor
// PSU peak draw, sampled at a slow cadence off the same loop — the fan
// check already drives the box to full power, so it is also the right run
// to observe what each PSU tops out at (updatePSUObservation persists it).
psuPeakW := map[int]float64{}
lastPSUSec := -1e9
csvPath := filepath.Join(runDir, "fan-sensors.csv")
_ = os.WriteFile(csvPath, []byte("elapsed_sec,fan_name,rpm\n"), 0644)
csvFile, _ := os.OpenFile(csvPath, os.O_APPEND|os.O_WRONLY, 0644)
@@ -281,6 +269,17 @@ loop:
}
goodSamples++
if elapsed-lastPSUSec >= 15 {
lastPSUSec = elapsed
if ps, ok := readPSUPowerBounded(fanReadTMO); ok {
for i, p := range ps {
if p.PowerW > psuPeakW[i] {
psuPeakW[i] = p.PowerW
}
}
}
}
for _, f := range fans {
if csvFile != nil {
fmt.Fprintf(csvFile, "%.0f,%s,%.0f\n", elapsed, f.Name, f.RPM)
@@ -347,6 +346,16 @@ loop:
if t := boundedGPUMaxTemp(opts.GPUIndices); t > 0 {
fmt.Fprintf(&summary, "gpu_temp_c=%.0f\n", t)
}
if len(psuPeakW) > 0 {
idx := make([]int, 0, len(psuPeakW))
for i := range psuPeakW {
idx = append(idx, i)
}
sort.Ints(idx)
for _, i := range idx {
fmt.Fprintf(&summary, "psu_%d_peak_w=%.0f\n", i, psuPeakW[i])
}
}
stats := satStats{}
names := make([]string, 0, len(baselineRPM))
@@ -441,6 +450,21 @@ func readFansBounded(timeout time.Duration) ([]FanReading, bool) {
}
}
// readPSUPowerBounded is readFansBounded's PSU sibling: a time-boxed,
// abandonable "ipmitool sdr" read of per-PSU power (samplePSUPower also feeds
// the observed-capacity store). Used at a slow cadence during the fan check so
// the same max-load run that finds fan ceilings also records peak PSU draw.
func readPSUPowerBounded(timeout time.Duration) ([]PSUReading, bool) {
ch := make(chan []PSUReading, 1)
go func() { ch <- samplePSUPower() }()
select {
case ps := <-ch:
return ps, len(ps) > 0
case <-time.After(timeout):
return nil, false
}
}
// boundedGPUMaxTemp returns the hottest GPU temperature via a single
// time-boxed nvidia-smi call, or 0 if unavailable.
func boundedGPUMaxTemp(gpuIndices []int) float64 {
@@ -569,8 +593,8 @@ func readFanStatuses() map[string]string {
// ResolveFanMaxRPM returns, for every fan name in current (name -> current
// RPM), the RPM to treat as that fan's 100% reference. Preference order:
// 1. the persisted observed peak (fanObservationStatePath), written by
// RunFanCheck and by live-metrics sampling under load;
// 1. the persisted observed peak, written by RunFanCheck and by live-metrics
// sampling under load;
// 2. the largest peak observed on any peer fan (keeps a group visually
// consistent when only some fans have a recorded peak);
// 3. the fan's own current RPM (so a tile is never sized against zero).
@@ -578,7 +602,7 @@ func readFanStatuses() map[string]string {
// The fallback lives here, not in the view, so every consumer of a fan
// maximum applies the same rule.
func ResolveFanMaxRPM(current map[string]float64) map[string]float64 {
persisted := readPersistedFanMaxRPM()
persisted := fanPeaks.snapshot()
peerMax := 0.0
for _, v := range persisted {
@@ -621,122 +645,41 @@ func sampleFanSpeeds() ([]FanReading, error) {
return nil, sensorsErr
}
// readPersistedFanMaxRPM reads fanObservationStatePath and returns its
// sanitized {fan name -> observed peak RPM} map (empty names / non-positive
// values dropped). Returns an empty map when the file is missing or unparsable.
func readPersistedFanMaxRPM() map[string]float64 {
out := map[string]float64{}
raw, err := os.ReadFile(fanObservationStatePath)
if err != nil || len(raw) == 0 {
return out
}
var persisted fanObservationState
if json.Unmarshal(raw, &persisted) != nil {
return out
}
for name, rpm := range persisted.MaxRPM {
name = strings.TrimSpace(name)
if name == "" || rpm <= 0 {
continue
}
out[name] = rpm
}
return out
}
// ObservedFanMaxRPM returns the per-fan observed peak RPM map persisted by
// fan-stress SAT runs, or nil if none is recorded yet. It reads the file
// directly without touching the in-process observation cache or its lock, so
// read-only consumers (the /topo web view) can call it without perturbing a
// concurrent SAT run's peak tracking.
// full-load runs, or nil if none is recorded yet.
func ObservedFanMaxRPM() map[string]float64 {
out := readPersistedFanMaxRPM()
out := fanPeaks.snapshot()
if len(out) == 0 {
return nil
}
return out
}
func loadFanObservationLocked() {
if fanObservationInit {
return
}
fanObservationInit = true
fanObservation.MaxRPM = readPersistedFanMaxRPM()
}
func saveFanObservationLocked() {
if len(fanObservation.MaxRPM) == 0 {
return
}
dir := filepath.Dir(fanObservationStatePath)
if dir == "" || dir == "." {
dir = "/var/log/bee-sat"
}
if err := os.MkdirAll(dir, 0755); err != nil {
return
}
raw, err := json.MarshalIndent(fanObservation, "", " ")
if err != nil {
return
}
_ = os.WriteFile(fanObservationStatePath, raw, 0644)
}
func updateFanObservation(fans []FanReading, now time.Time) {
if len(fans) == 0 {
return
}
fanObservationMu.Lock()
defer fanObservationMu.Unlock()
loadFanObservationLocked()
changed := false
m := make(map[string]float64, len(fans))
for _, fan := range fans {
name := strings.TrimSpace(fan.Name)
if name == "" || fan.RPM <= 0 {
continue
if n := strings.TrimSpace(fan.Name); n != "" && fan.RPM > 0 {
m[n] = fan.RPM
}
currentMax := fanObservation.MaxRPM[name]
if fan.RPM <= currentMax {
delete(fanPeakCandidates, name)
continue
}
if cand, ok := fanPeakCandidates[name]; ok {
if now.Sub(cand.FirstSeen) >= fanObservationMinPeakHold {
newMax := math.Max(cand.RPM, fan.RPM)
if newMax > currentMax {
fanObservation.MaxRPM[name] = normalizeObservedFanMaxRPM(newMax)
changed = true
}
delete(fanPeakCandidates, name)
continue
}
if fan.RPM > cand.RPM {
fanPeakCandidates[name] = fanPeakCandidate{FirstSeen: cand.FirstSeen, RPM: fan.RPM}
}
continue
}
fanPeakCandidates[name] = fanPeakCandidate{FirstSeen: now, RPM: fan.RPM}
}
if changed {
saveFanObservationLocked()
}
fanPeaks.observe(m, now)
}
func estimateFanDutyCyclePctFromObservation(fans []FanReading) (float64, bool) {
if len(fans) == 0 {
return 0, false
}
fanObservationMu.Lock()
defer fanObservationMu.Unlock()
loadFanObservationLocked()
maxByName := fanPeaks.snapshot()
var samples []float64
for _, fan := range fans {
name := strings.TrimSpace(fan.Name)
if name == "" || fan.RPM <= 0 {
continue
}
maxRPM := fanObservation.MaxRPM[name]
maxRPM := maxByName[name]
if maxRPM <= 0 {
continue
}