reanimator/bee
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix(sat): serialize recurring IPMI polling behind one shared telemetry pipeline

Browse Source 
The fan-ceiling check, the webui metrics collector (every 5s) and the PSU
health poller each shelled out to ipmitool independently. The BMC's KCS
interface serializes those calls anyway, so under load the concurrent
`ipmitool sdr`/`dcmi power reading` invocations just queued behind each
other — that's what produced "IPMI slow" backoff during a fan-ceiling run in
a blackbox dump, while the dashboard looked fine only because it was reading
its own, separately-stale data from a different ipmitool call.

hw_telemetry.go is now the sole recurring poller (fan RPM, temperature, PSU
power/status, DCMI system power), with the adaptive 1s-30s backoff that used
to be duplicated inside the fan check. Every hot-path consumer reads the
shared cache (hwSnapshot / platform.HardwareSDRSnapshot) instead of calling
ipmitool itself.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-09-15 18:55:50 +03:00
co-authored by Claude Sonnet 5
parent 9c5c29239c
commit b09e94d02a
5 changed files with 297 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files
audit/internal/platform/sat_fan_stress.go
+59 -106
View File
@@ -196,20 +196,19 @@ func (s *System) RunFanCheck(ctx context.Context, baseDir string, opts FanCheckO
appendSATVerboseLog(verboseLog, fmt.Sprintf("%d load source(s) active; plateau clock effective from +%.0fs",
activeLoads, readyAt.Sub(start).Seconds()))
// ── Sample loop with IPMI-hang protection.
// ── Sample loop, reading the shared hardware telemetry cache.
//
// Under full load ipmitool over KCS can take tens of seconds per call or
// wedge outright. So: every fan read is time-boxed (readFansBounded runs
// it in a goroutine we abandon on timeout — a wedged KCS read can never
// block this loop), and the polling interval backs off geometrically when
// reads are slow and tightens again when they recover. A plateau is only
// declared while telemetry is healthy (interval near the floor); a
// degraded run just rides out to MaxLoadSec and records the peak it saw.
const (
fanPollFloor = 1 * time.Second
fanPollCeil = 30 * time.Second
fanReadTMO = 8 * time.Second
)
// Fan RPM, PSU power and IPMI-hang protection all live in the shared
// hwTelemetry poller (hw_telemetry.go) now: it is the only thing that
// ever shells out to ipmitool on a recurring cadence, backs its own
// polling interval off geometrically when reads are slow (or wedge
// outright) and tightens again when they recover. This loop just ticks
// at the cache's floor interval and picks up whatever the shared poller
// last saw — it never spawns an ipmitool call of its own, so it can't
// add to the exact contention that made ipmitool slow in the first
// place. A plateau is only declared while telemetry is healthy (shared
// poller interval near the floor); a degraded run just rides out to
// MaxLoadSec and records the peak it saw.
type fanState struct {
peak float64
lastRiseSec float64
@@ -220,7 +219,8 @@ func (s *System) RunFanCheck(ctx context.Context, baseDir string, opts FanCheckO
aborted := false
degraded := false
goodSamples := 0
poll := fanPollFloor
lastSampleAt := time.Time{}
lastLoggedInterval := hwPollFloor
// PSU peak draw, sampled at a slow cadence off the same loop — the fan
// check already drives the box to full power, so it is also the right run
@@ -243,39 +243,34 @@ loop:
break loop
case <-loadCtx.Done():
break loop // MaxLoadSec reached
case <-time.After(poll):
case <-time.After(hwPollFloor):
}
elapsed := time.Since(start).Seconds()
readStart := time.Now()
fans, ok := readFansBounded(fanReadTMO)
readDur := time.Since(readStart)
// Adapt the interval to how ipmitool is behaving.
switch {
case !ok || readDur > fanReadTMO*3/4:
if poll < fanPollCeil {
poll = minDuration(poll*2, fanPollCeil)
curInterval := hwCurrentInterval()
if curInterval != lastLoggedInterval {
if curInterval > lastLoggedInterval {
degraded = true
appendSATVerboseLog(verboseLog, fmt.Sprintf("[%.0fs] ipmitool slow (%.1fs, ok=%v) — polling backed off to %s",
elapsed, readDur.Seconds(), ok, poll))
logFunc(fmt.Sprintf("IPMI slow — fan polling backed off to %s", poll))
appendSATVerboseLog(verboseLog, fmt.Sprintf("[%.0fs] ipmitool slow — shared polling backed off to %s",
elapsed, curInterval))
logFunc(fmt.Sprintf("IPMI slow — fan polling backed off to %s", curInterval))
}
case poll > fanPollFloor && readDur < fanPollFloor:
poll = maxDuration(poll*2/3, fanPollFloor)
lastLoggedInterval = curInterval
}
if !ok {
continue
sample := hwSnapshot()
if sample.At.IsZero() || !sample.At.After(lastSampleAt) {
continue // no fresh reading from the shared poller yet
}
lastSampleAt = sample.At
goodSamples++
fans := sample.Fans
if elapsed-lastPSUSec >= 15 {
lastPSUSec = elapsed
if ps, ok := readPSUPowerBounded(fanReadTMO); ok {
for i, p := range ps {
if p.PowerW > psuPeakW[i] {
psuPeakW[i] = p.PowerW
}
for i, p := range sample.PSUs {
if p.PowerW > psuPeakW[i] {
psuPeakW[i] = p.PowerW
}
}
}
@@ -302,7 +297,7 @@ loop:
// Only trust a plateau while telemetry is healthy and we have enough
// recent samples to have actually seen a flat window.
healthy := poll <= 2*fanPollFloor && goodSamples >= 5
healthy := curInterval <= 2*hwPollFloor && goodSamples >= 5
if healthy && rampConfirmed && elapsed >= float64(opts.MinLoadSec) &&
time.Since(readyAt) >= time.Duration(opts.PlateauHoldSec)*time.Second && len(fanBy) > 0 {
allFlat := true
@@ -415,56 +410,6 @@ func maxDuration(a, b time.Duration) time.Duration {
return b
}
// readFansBounded runs "ipmitool sdr type Fan" but never blocks the caller
// longer than timeout: the read happens in a goroutine that is abandoned if it
// does not return in time (a KCS read wedged in uninterruptible I/O cannot be
// killed, so we leave it and move on). ok=false means "no usable sample this
// tick" — the caller must treat that as missing data, not as a flat fan.
func readFansBounded(timeout time.Duration) ([]FanReading, bool) {
type result struct {
fans []FanReading
ok bool
}
ch := make(chan result, 1)
go func() {
out, err := exec.Command("ipmitool", "sdr", "type", "Fan").Output()
if err != nil {
ch <- result{}
return
}
fans := parseFanSpeeds(string(out))
if len(fans) == 0 {
ch <- result{}
return
}
ch <- result{fans, true}
}()
select {
case r := <-ch:
if r.ok {
updateFanObservation(r.fans, time.Now())
}
return r.fans, r.ok
case <-time.After(timeout):
return nil, false
}
}
// readPSUPowerBounded is readFansBounded's PSU sibling: a time-boxed,
// abandonable "ipmitool sdr" read of per-PSU power (samplePSUPower also feeds
// the observed-capacity store). Used at a slow cadence during the fan check so
// the same max-load run that finds fan ceilings also records peak PSU draw.
func readPSUPowerBounded(timeout time.Duration) ([]PSUReading, bool) {
ch := make(chan []PSUReading, 1)
go func() { ch <- samplePSUPower() }()
select {
case ps := <-ch:
return ps, len(ps) > 0
case <-time.After(timeout):
return nil, false
}
}
// boundedGPUMaxTemp returns the hottest GPU temperature via a single
// time-boxed nvidia-smi call, or 0 if unavailable.
func boundedGPUMaxTemp(gpuIndices []int) float64 {
@@ -570,14 +515,23 @@ func sanitizeSummaryKey(name string) string {
}
// readFanStatuses returns the per-fan IPMI status word ("ok", "cr", "nr", ...)
// from "ipmitool sdr type Fan". Empty map when ipmitool is unavailable.
// from the shared hardware telemetry cache (hwSnapshot) instead of shelling
// out to ipmitool itself — see hw_telemetry.go for why.
func readFanStatuses() map[string]string {
out, err := exec.Command("ipmitool", "sdr", "type", "Fan").Output()
if err != nil {
return hwSnapshot().FanStatus
}
// parseFanStatuses parses the per-fan IPMI status word out of `ipmitool sdr`
// text (the full dump or a "type Fan"-filtered one — both use the same
// per-line format). Non-fan lines end up in the map too (keyed by whatever
// sensor name they have) but that's harmless: callers only look up known fan
// names.
func parseFanStatuses(raw string) map[string]string {
if raw == "" {
return nil
}
m := map[string]string{}
for _, line := range strings.Split(string(out), "\n") {
for _, line := range strings.Split(raw, "\n") {
parts := strings.Split(line, "|")
if len(parts) < 3 {
continue
@@ -625,23 +579,19 @@ func ResolveFanMaxRPM(current map[string]float64) map[string]float64 {
return out
}
// sampleFanSpeeds reads fan RPM values from ipmitool sdr.
// sampleFanSpeeds reads fan RPM values from the shared hardware telemetry
// cache (hwSnapshot) instead of shelling out to ipmitool itself — see
// hw_telemetry.go for why. Falls back to lm-sensors when IPMI has nothing
// (unavailable BMC, or a platform without IPMI fan sensors at all).
func sampleFanSpeeds() ([]FanReading, error) {
out, err := exec.Command("ipmitool", "sdr", "type", "Fan").Output()
if err == nil {
if fans := parseFanSpeeds(string(out)); len(fans) > 0 {
updateFanObservation(fans, time.Now())
return fans, nil
}
if fans := hwSnapshot().Fans; len(fans) > 0 {
return fans, nil
}
fans, sensorsErr := sampleFanSpeedsViaSensorsJSON()
if len(fans) > 0 {
updateFanObservation(fans, time.Now())
return fans, nil
}
if err != nil {
return nil, err
}
return nil, sensorsErr
}
@@ -900,13 +850,16 @@ func firstFanInputValue(feature map[string]any) (float64, bool) {
return firstSensorInputValue(feature, "fan")
}
// sampleCPUMaxTemp returns the highest CPU/inlet temperature from ipmitool or sensors.
// sampleCPUMaxTemp returns the highest CPU/inlet temperature, from the
// shared hardware telemetry cache (hwSnapshot) when IPMI has it, else
// lm-sensors directly.
func sampleCPUMaxTemp() float64 {
out, err := exec.Command("ipmitool", "sdr", "type", "Temperature").Output()
if err != nil {
return sampleCPUTempViaSensors()
if raw := hwSnapshot().Raw; raw != "" {
if t := parseIPMIMaxTemp(raw); t > 0 {
return t
}
}
return parseIPMIMaxTemp(string(out))
return sampleCPUTempViaSensors()
}
// parseIPMIMaxTemp extracts the maximum temperature from "ipmitool sdr type Temperature".