bee/audit/internal/platform/hw_telemetry.go

package platform

import (
	"os/exec"
	"sync"
	"time"
)

// hwSample is one full snapshot from the shared hardware telemetry pipeline.
type hwSample struct {
	At time.Time
	// Raw is the full `ipmitool sdr` text, for consumers that need to
	// re-derive something the pre-parsed fields below don't cover.
	Raw        string
	Fans       []FanReading
	Temps      []TempReading
	PSUs       []PSUReading
	FanStatus  map[string]string // fan name -> ipmi status word (ok/cr/nr/...)
	DCMIPowerW float64
	ReadDur    time.Duration
}

const (
	hwPollFloor = 1 * time.Second
	hwPollCeil  = 30 * time.Second
	hwReadTMO   = 8 * time.Second
)

// hwTelemetry is the single owner of every recurring, hot-path IPMI poll for
// fan RPM, temperature and PSU power/status (plus DCMI system power). Before
// this existed, the webui metrics collector (every 5s), the PSU health
// poller and the fan-ceiling check each shelled out to ipmitool on their own
// schedule. The BMC's KCS interface serializes those calls anyway, so
// running several concurrently only produced queueing — that's what shows up
// as "ipmitool slow" backoff in blackbox dumps taken during a fan-ceiling
// run, even while the dashboard looked fine (it was quietly reading its own,
// separately-stale data from a different ipmitool invocation).
//
// Now exactly one `ipmitool sdr` (+ one `dcmi power reading`) round trip is
// ever in flight, on a single adaptively-throttled background goroutine;
// every consumer below reads the shared cache via hwSnapshot instead of
// shelling out itself. One-off, non-recurring ipmitool calls (audit bundle
// export, pre/post-test PSU fault diffing) are intentionally left alone —
// they don't run on a sustained cadence, so they aren't the source of
// contention this pipeline exists to remove.
type hwTelemetry struct {
	mu       sync.Mutex
	sample   hwSample
	interval time.Duration
	degraded bool
	started  bool
}

var hw = &hwTelemetry{interval: hwPollFloor}

// hwSnapshot returns the most recent shared telemetry sample, starting the
// background poller (and blocking for its first read) on first use.
func hwSnapshot() hwSample {
	hw.ensureStarted()
	hw.mu.Lock()
	defer hw.mu.Unlock()
	return hw.sample
}

// hwCurrentInterval returns the poller's current adaptive polling interval —
// above hwPollFloor means IPMI has been observed slow recently.
func hwCurrentInterval() time.Duration {
	hw.mu.Lock()
	defer hw.mu.Unlock()
	return hw.interval
}

// HardwareSDRSnapshot exposes the shared `ipmitool sdr` cache to other
// packages (e.g. webui's PSU health poller) so they don't have to run their
// own independent ipmitool poll — see hw_telemetry.go for why that matters.
// Returns the raw SDR text and when it was captured; raw is empty if no
// successful read has happened yet.
func HardwareSDRSnapshot() (raw string, at time.Time) {
	s := hwSnapshot()
	return s.Raw, s.At
}

func (p *hwTelemetry) ensureStarted() {
	p.mu.Lock()
	if p.started {
		p.mu.Unlock()
		return
	}
	p.started = true
	p.mu.Unlock()
	p.pollOnce() // synchronous warm-up: the first caller gets a real reading, not an empty cache
	go p.loop()
}

func (p *hwTelemetry) loop() {
	for {
		wait := p.pollOnce()
		time.Sleep(wait)
	}
}

// pollOnce runs exactly one sequential BMC round trip (sdr, then dcmi),
// updates the cache and the observed-peak stores, adapts the polling
// interval to how slow the read was, and returns the interval to sleep
// before the next poll.
func (p *hwTelemetry) pollOnce() time.Duration {
	start := time.Now()
	raw, ok := runIPMIBounded(hwReadTMO, "sdr")
	readDur := time.Since(start)

	var next time.Duration
	p.mu.Lock()
	next = p.interval
	switch {
	case !ok || readDur > hwReadTMO*3/4:
		if next < hwPollCeil {
			next = minDuration(next*2, hwPollCeil)
		}
		p.degraded = true
	case next > hwPollFloor && readDur < hwPollFloor:
		next = maxDuration(next*2/3, hwPollFloor)
	}
	p.interval = next
	p.mu.Unlock()

	if !ok {
		return next
	}

	fans := parseFanSpeeds(raw)
	temps := parseIPMITemps(raw)
	psus := parsePSUReadings(raw)
	fanStatus := parseFanStatuses(raw)

	var dcmiW float64
	if dcmiOut, dcmiOK := runIPMIBounded(hwReadTMO, "dcmi", "power", "reading"); dcmiOK {
		dcmiW = parseDCMIPowerReading(dcmiOut)
	}

	now := time.Now()
	p.mu.Lock()
	p.sample = hwSample{
		At:         now,
		Raw:        raw,
		Fans:       fans,
		Temps:      temps,
		PSUs:       psus,
		FanStatus:  fanStatus,
		DCMIPowerW: dcmiW,
		ReadDur:    readDur,
	}
	p.mu.Unlock()

	if len(fans) > 0 {
		updateFanObservation(fans, now)
	}
	if len(psus) > 0 {
		updatePSUObservation(psus, now)
	}
	return next
}

// runIPMIBounded runs an ipmitool subcommand but never blocks the caller
// longer than timeout: the read happens in a goroutine that is abandoned if
// it does not return in time (a KCS read wedged in uninterruptible I/O
// cannot be killed, so we leave it and move on). ok=false means "no usable
// output this round" — the caller must treat that as missing data.
func runIPMIBounded(timeout time.Duration, args ...string) (string, bool) {
	type result struct {
		out string
		ok  bool
	}
	ch := make(chan result, 1)
	go func() {
		out, err := exec.Command("ipmitool", args...).Output()
		if err != nil || len(out) == 0 {
			ch <- result{}
			return
		}
		ch <- result{string(out), true}
	}()
	select {
	case r := <-ch:
		return r.out, r.ok
	case <-time.After(timeout):
		return "", false
	}
}