package platform import ( "os/exec" "sync" "time" ) // hwSample is one full snapshot from the shared hardware telemetry pipeline. type hwSample struct { At time.Time // Raw is the full `ipmitool sdr` text, for consumers that need to // re-derive something the pre-parsed fields below don't cover. Raw string Fans []FanReading Temps []TempReading PSUs []PSUReading FanStatus map[string]string // fan name -> ipmi status word (ok/cr/nr/...) DCMIPowerW float64 ReadDur time.Duration } const ( hwPollFloor = 1 * time.Second hwPollCeil = 30 * time.Second hwReadTMO = 8 * time.Second ) // hwTelemetry is the single owner of every recurring, hot-path IPMI poll for // fan RPM, temperature and PSU power/status (plus DCMI system power). Before // this existed, the webui metrics collector (every 5s), the PSU health // poller and the fan-ceiling check each shelled out to ipmitool on their own // schedule. The BMC's KCS interface serializes those calls anyway, so // running several concurrently only produced queueing — that's what shows up // as "ipmitool slow" backoff in blackbox dumps taken during a fan-ceiling // run, even while the dashboard looked fine (it was quietly reading its own, // separately-stale data from a different ipmitool invocation). // // Now exactly one `ipmitool sdr` (+ one `dcmi power reading`) round trip is // ever in flight, on a single adaptively-throttled background goroutine; // every consumer below reads the shared cache via hwSnapshot instead of // shelling out itself. One-off, non-recurring ipmitool calls (audit bundle // export, pre/post-test PSU fault diffing) are intentionally left alone — // they don't run on a sustained cadence, so they aren't the source of // contention this pipeline exists to remove. type hwTelemetry struct { mu sync.Mutex sample hwSample interval time.Duration degraded bool started bool } var hw = &hwTelemetry{interval: hwPollFloor} // hwSnapshot returns the most recent shared telemetry sample, starting the // background poller (and blocking for its first read) on first use. func hwSnapshot() hwSample { hw.ensureStarted() hw.mu.Lock() defer hw.mu.Unlock() return hw.sample } // hwCurrentInterval returns the poller's current adaptive polling interval — // above hwPollFloor means IPMI has been observed slow recently. func hwCurrentInterval() time.Duration { hw.mu.Lock() defer hw.mu.Unlock() return hw.interval } // HardwareSDRSnapshot exposes the shared `ipmitool sdr` cache to other // packages (e.g. webui's PSU health poller) so they don't have to run their // own independent ipmitool poll — see hw_telemetry.go for why that matters. // Returns the raw SDR text and when it was captured; raw is empty if no // successful read has happened yet. func HardwareSDRSnapshot() (raw string, at time.Time) { s := hwSnapshot() return s.Raw, s.At } func (p *hwTelemetry) ensureStarted() { p.mu.Lock() if p.started { p.mu.Unlock() return } p.started = true p.mu.Unlock() p.pollOnce() // synchronous warm-up: the first caller gets a real reading, not an empty cache go p.loop() } func (p *hwTelemetry) loop() { for { wait := p.pollOnce() time.Sleep(wait) } } // pollOnce runs exactly one sequential BMC round trip (sdr, then dcmi), // updates the cache and the observed-peak stores, adapts the polling // interval to how slow the read was, and returns the interval to sleep // before the next poll. func (p *hwTelemetry) pollOnce() time.Duration { start := time.Now() raw, ok := runIPMIBounded(hwReadTMO, "sdr") readDur := time.Since(start) var next time.Duration p.mu.Lock() next = p.interval switch { case !ok || readDur > hwReadTMO*3/4: if next < hwPollCeil { next = minDuration(next*2, hwPollCeil) } p.degraded = true case next > hwPollFloor && readDur < hwPollFloor: next = maxDuration(next*2/3, hwPollFloor) } p.interval = next p.mu.Unlock() if !ok { return next } fans := parseFanSpeeds(raw) temps := parseIPMITemps(raw) psus := parsePSUReadings(raw) fanStatus := parseFanStatuses(raw) var dcmiW float64 if dcmiOut, dcmiOK := runIPMIBounded(hwReadTMO, "dcmi", "power", "reading"); dcmiOK { dcmiW = parseDCMIPowerReading(dcmiOut) } now := time.Now() p.mu.Lock() p.sample = hwSample{ At: now, Raw: raw, Fans: fans, Temps: temps, PSUs: psus, FanStatus: fanStatus, DCMIPowerW: dcmiW, ReadDur: readDur, } p.mu.Unlock() if len(fans) > 0 { updateFanObservation(fans, now) } if len(psus) > 0 { updatePSUObservation(psus, now) } return next } // runIPMIBounded runs an ipmitool subcommand but never blocks the caller // longer than timeout: the read happens in a goroutine that is abandoned if // it does not return in time (a KCS read wedged in uninterruptible I/O // cannot be killed, so we leave it and move on). ok=false means "no usable // output this round" — the caller must treat that as missing data. func runIPMIBounded(timeout time.Duration, args ...string) (string, bool) { type result struct { out string ok bool } ch := make(chan result, 1) go func() { out, err := exec.Command("ipmitool", args...).Output() if err != nil || len(out) == 0 { ch <- result{} return } ch <- result{string(out), true} }() select { case r := <-ch: return r.out, r.ok case <-time.After(timeout): return "", false } }