fix(sat): serialize recurring IPMI polling behind one shared telemetry pipeline
The fan-ceiling check, the webui metrics collector (every 5s) and the PSU health poller each shelled out to ipmitool independently. The BMC's KCS interface serializes those calls anyway, so under load the concurrent `ipmitool sdr`/`dcmi power reading` invocations just queued behind each other — that's what produced "IPMI slow" backoff during a fan-ceiling run in a blackbox dump, while the dashboard looked fine only because it was reading its own, separately-stale data from a different ipmitool call. hw_telemetry.go is now the sole recurring poller (fan RPM, temperature, PSU power/status, DCMI system power), with the adaptive 1s-30s backoff that used to be duplicated inside the fan check. Every hot-path consumer reads the shared cache (hwSnapshot / platform.HardwareSDRSnapshot) instead of calling ipmitool itself. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
9c5c29239c
commit
b09e94d02a
@@ -1,14 +1,12 @@
|
||||
package webui
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"log/slog"
|
||||
"os/exec"
|
||||
"time"
|
||||
|
||||
"bee/audit/internal/app"
|
||||
"bee/audit/internal/collector"
|
||||
"bee/audit/internal/platform"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -18,7 +16,6 @@ const (
|
||||
// is still caught within a bounded window even after a long quiet
|
||||
// stretch.
|
||||
healthPollIntervalMax = 30 * time.Minute
|
||||
psuIPMITimeout = 15 * time.Second
|
||||
)
|
||||
|
||||
// healthPoller runs periodic health checks for hardware components that do not
|
||||
@@ -66,26 +63,26 @@ func nextHealthPollInterval(current time.Duration, changed bool) time.Duration {
|
||||
return next
|
||||
}
|
||||
|
||||
// pollPSU polls PSU status via ipmitool and records it to statusDB. Returns
|
||||
// true if any PSU's status differs from what statusDB currently has for it,
|
||||
// which run uses to reset the backoff.
|
||||
// pollPSU reads PSU status from the shared hardware telemetry cache
|
||||
// (platform.HardwareSDRSnapshot) and records it to statusDB, instead of
|
||||
// shelling out to ipmitool itself — see platform/hw_telemetry.go for why:
|
||||
// this poller used to run its own independent `ipmitool sdr` on a 60s+
|
||||
// cadence, competing with the webui metrics collector and any running SAT
|
||||
// test for the same serialized BMC interface. Returns true if any PSU's
|
||||
// status differs from what statusDB currently has for it, which run uses to
|
||||
// reset the backoff.
|
||||
func (p *healthPoller) pollPSU() bool {
|
||||
if p.statusDB == nil {
|
||||
return false
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), psuIPMITimeout)
|
||||
defer cancel()
|
||||
|
||||
cmd := exec.CommandContext(ctx, "ipmitool", "sdr")
|
||||
var out bytes.Buffer
|
||||
cmd.Stdout = &out
|
||||
if err := cmd.Run(); err != nil {
|
||||
// IPMI not available or not a server — skip silently.
|
||||
slog.Debug("health poller: ipmitool sdr unavailable", "err", err)
|
||||
raw, at := platform.HardwareSDRSnapshot()
|
||||
if raw == "" || at.IsZero() {
|
||||
// IPMI not available, or the shared poller hasn't completed a read yet.
|
||||
slog.Debug("health poller: no shared ipmitool sdr sample available")
|
||||
return false
|
||||
}
|
||||
|
||||
slots := collector.PSUSlotsFromSDR(out.String())
|
||||
slots := collector.PSUSlotsFromSDR(raw)
|
||||
if len(slots) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user