fix(sat): serialize recurring IPMI polling behind one shared telemetry pipeline
The fan-ceiling check, the webui metrics collector (every 5s) and the PSU health poller each shelled out to ipmitool independently. The BMC's KCS interface serializes those calls anyway, so under load the concurrent `ipmitool sdr`/`dcmi power reading` invocations just queued behind each other — that's what produced "IPMI slow" backoff during a fan-ceiling run in a blackbox dump, while the dashboard looked fine only because it was reading its own, separately-stale data from a different ipmitool call. hw_telemetry.go is now the sole recurring poller (fan RPM, temperature, PSU power/status, DCMI system power), with the adaptive 1s-30s backoff that used to be duplicated inside the fan check. Every hot-path consumer reads the shared cache (hwSnapshot / platform.HardwareSDRSnapshot) instead of calling ipmitool itself. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
9c5c29239c
commit
b09e94d02a
@@ -197,11 +197,12 @@ func filterSensorGroup(sensors []sdrSensor, maxPerSensorW float64) (valid []sdrS
|
||||
// GPU slot sensors (GPU_POWER_SLOTx, GPU1 Power, …) are classified separately
|
||||
// since the audit collector does not track GPU PCIe slot power.
|
||||
func sampleIPMISDRPowerSensors() sdrPowerSnapshot {
|
||||
raw, err := exec.Command("ipmitool", "sdr").Output()
|
||||
if err != nil || len(raw) == 0 {
|
||||
// Reads from the shared hardware telemetry cache (hwSnapshot) rather than
|
||||
// shelling out to `ipmitool sdr` itself — see hw_telemetry.go for why.
|
||||
sdrStr := hwSnapshot().Raw
|
||||
if sdrStr == "" {
|
||||
return sdrPowerSnapshot{}
|
||||
}
|
||||
sdrStr := string(raw)
|
||||
var snap sdrPowerSnapshot
|
||||
|
||||
// ── PSU data via audit collector ─────────────────────────────────────────
|
||||
@@ -390,20 +391,16 @@ func summarizeSDRPowerSeries(samples []sdrPowerSnapshot) benchmarkSDRSeriesSumma
|
||||
return summary
|
||||
}
|
||||
|
||||
// queryIPMIServerPowerW reads the current server power draw via ipmitool dcmi.
|
||||
// Returns 0 and an error if IPMI is unavailable or the output cannot be parsed.
|
||||
// queryIPMIServerPowerW reads the current server power draw from the shared
|
||||
// hardware telemetry cache (hwSnapshot), which runs `ipmitool dcmi power
|
||||
// reading` itself on its own cadence — see hw_telemetry.go for why this
|
||||
// doesn't shell out directly. Returns 0 and an error if IPMI dcmi power is
|
||||
// unavailable on this BMC.
|
||||
func queryIPMIServerPowerW() (float64, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
cmd := exec.CommandContext(ctx, "ipmitool", "dcmi", "power", "reading")
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("ipmitool dcmi power reading: %w", err)
|
||||
}
|
||||
if w := parseDCMIPowerReading(string(out)); w > 0 {
|
||||
if w := hwSnapshot().DCMIPowerW; w > 0 {
|
||||
return w, nil
|
||||
}
|
||||
return 0, fmt.Errorf("could not parse ipmitool dcmi power reading output")
|
||||
return 0, fmt.Errorf("ipmitool dcmi power reading unavailable")
|
||||
}
|
||||
|
||||
// characterizeServerPower computes BenchmarkServerPower from idle and loaded
|
||||
|
||||
Reference in New Issue
Block a user