fix(sat): serialize recurring IPMI polling behind one shared telemetry pipeline
The fan-ceiling check, the webui metrics collector (every 5s) and the PSU health poller each shelled out to ipmitool independently. The BMC's KCS interface serializes those calls anyway, so under load the concurrent `ipmitool sdr`/`dcmi power reading` invocations just queued behind each other — that's what produced "IPMI slow" backoff during a fan-ceiling run in a blackbox dump, while the dashboard looked fine only because it was reading its own, separately-stale data from a different ipmitool call. hw_telemetry.go is now the sole recurring poller (fan RPM, temperature, PSU power/status, DCMI system power), with the adaptive 1s-30s backoff that used to be duplicated inside the fan check. Every hot-path consumer reads the shared cache (hwSnapshot / platform.HardwareSDRSnapshot) instead of calling ipmitool itself. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
9c5c29239c
commit
b09e94d02a
@@ -246,14 +246,23 @@ func sampleLiveTempsViaSensorsJSON() []TempReading {
|
||||
return temps
|
||||
}
|
||||
|
||||
// sampleLiveTempsViaIPMI reads temperatures from the shared hardware
|
||||
// telemetry cache (hwSnapshot) instead of shelling out to ipmitool itself —
|
||||
// see hw_telemetry.go for why.
|
||||
func sampleLiveTempsViaIPMI() []TempReading {
|
||||
out, err := exec.Command("ipmitool", "sdr", "type", "Temperature").Output()
|
||||
if err != nil || len(out) == 0 {
|
||||
return hwSnapshot().Temps
|
||||
}
|
||||
|
||||
// parseIPMITemps parses temperature entries out of `ipmitool sdr` text
|
||||
// (the full dump or a "type Temperature"-filtered one — both use the same
|
||||
// per-line format).
|
||||
func parseIPMITemps(raw string) []TempReading {
|
||||
if raw == "" {
|
||||
return nil
|
||||
}
|
||||
var temps []TempReading
|
||||
seen := map[string]struct{}{}
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||||
for _, line := range strings.Split(strings.TrimSpace(raw), "\n") {
|
||||
parts := strings.Split(line, "|")
|
||||
if len(parts) < 3 {
|
||||
continue
|
||||
@@ -353,16 +362,23 @@ func compactAmbientTempName(chip, name string) string {
|
||||
return chip + " / " + name
|
||||
}
|
||||
|
||||
// samplePSUPower reads per-PSU input power via IPMI SDR.
|
||||
// samplePSUPower reads per-PSU power from the shared hardware telemetry
|
||||
// cache (hwSnapshot) instead of shelling out to ipmitool itself — see
|
||||
// hw_telemetry.go for why. The observed-capacity store is fed once, inside
|
||||
// the shared poller, rather than by every caller.
|
||||
func samplePSUPower() []PSUReading {
|
||||
return hwSnapshot().PSUs
|
||||
}
|
||||
|
||||
// parsePSUReadings parses per-PSU power out of `ipmitool sdr` text.
|
||||
// Uses collector.PSUSlotsFromSDR (name-based matching) which works across
|
||||
// vendors where PSU sensors may not carry entity ID "10.N".
|
||||
// Returns nil when IPMI is unavailable or no PSU Watt sensors exist.
|
||||
func samplePSUPower() []PSUReading {
|
||||
out, err := exec.Command("ipmitool", "sdr").Output()
|
||||
if err != nil || len(out) == 0 {
|
||||
// Returns nil when no PSU Watt sensors exist.
|
||||
func parsePSUReadings(raw string) []PSUReading {
|
||||
if raw == "" {
|
||||
return nil
|
||||
}
|
||||
slots := collector.PSUSlotsFromSDR(string(out))
|
||||
slots := collector.PSUSlotsFromSDR(raw)
|
||||
if len(slots) == 0 {
|
||||
return nil
|
||||
}
|
||||
@@ -393,9 +409,5 @@ func samplePSUPower() []PSUReading {
|
||||
if len(psus) == 0 {
|
||||
return nil
|
||||
}
|
||||
// Feed the observed-capacity store (the "autotune" for PSU load scaling on
|
||||
// BMCs that report only instantaneous power) — every load run that samples
|
||||
// PSU power, including the 5 s metrics collector, refines it.
|
||||
updatePSUObservation(psus, time.Now())
|
||||
return psus
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user