Mikhail ChusavitinandClaude Sonnet 5
cc3997f7b1
The long-lived bee-web process (writing PSU/kmsg watchdog records ~every
60s) and each short-lived "bee bee-worker" SAT-task subprocess each held
an independent in-memory copy of component-status.json. Whichever saved
last won outright, silently erasing whatever the other had just written —
e.g. a GPU SAT task's pcie:gpu:nvidia result vanishing the next time the
PSU watchdog ticked. ComponentStatusDB.Record now reloads on-disk state
(keyed by newer LastCheckedAt) before merging its own update.
Also stops re-logging identical repeat observations to History: the
ingest contract defines status_history as a transition log ("История
переходов статусов"), not a per-poll journal, but Record appended one
entry per call regardless — a continuously-polled PSU grew an unbounded
run of identical "still OK" entries. Now only appends when a source's
last recorded status for a key actually changes.
The PSU watchdog itself now backs off (60s -> doubling, capped at 30min)
while steady and resets to 60s the moment any PSU's status changes, cutting
ipmitool shellouts and file writes for a fleet that's been stable for a
while.
Also adds the missing "raid" case to the component-detail API (was
returning 404 for any RAID card's detail click on /topo).
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
115 lines
3.2 KiB
Go
115 lines
3.2 KiB
Go
package webui
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"log/slog"
|
|
"os/exec"
|
|
"time"
|
|
|
|
"bee/audit/internal/app"
|
|
"bee/audit/internal/collector"
|
|
)
|
|
|
|
const (
|
|
healthPollIntervalMin = 60 * time.Second
|
|
// healthPollIntervalMax caps the backoff below: a PSU that has been
|
|
// steady for a while is polled at most this rarely, so a real failure
|
|
// is still caught within a bounded window even after a long quiet
|
|
// stretch.
|
|
healthPollIntervalMax = 30 * time.Minute
|
|
psuIPMITimeout = 15 * time.Second
|
|
)
|
|
|
|
// healthPoller runs periodic health checks for hardware components that do not
|
|
// emit kernel log events (e.g. PSU). Results are written to ComponentStatusDB.
|
|
//
|
|
// The poll interval backs off (doubling, capped at healthPollIntervalMax)
|
|
// each tick where no PSU's status changed, and resets to
|
|
// healthPollIntervalMin the moment any PSU does change — a steady-state PSU
|
|
// bank doesn't need re-checking every 60s forever (each poll is a
|
|
// component-status.json write and an ipmitool shellout), while a PSU that
|
|
// just started flapping gets caught quickly again.
|
|
type healthPoller struct {
|
|
statusDB *app.ComponentStatusDB
|
|
interval time.Duration
|
|
}
|
|
|
|
func newHealthPoller(statusDB *app.ComponentStatusDB) *healthPoller {
|
|
return &healthPoller{statusDB: statusDB, interval: healthPollIntervalMin}
|
|
}
|
|
|
|
func (p *healthPoller) start() {
|
|
goRecoverLoop("health poller", 5*time.Second, p.run)
|
|
}
|
|
|
|
func (p *healthPoller) run() {
|
|
timer := time.NewTimer(p.interval)
|
|
defer timer.Stop()
|
|
for range timer.C {
|
|
p.interval = nextHealthPollInterval(p.interval, p.pollPSU())
|
|
timer.Reset(p.interval)
|
|
}
|
|
}
|
|
|
|
// nextHealthPollInterval computes the next poll interval given whether the
|
|
// last poll observed any status change: reset to the fast floor on change,
|
|
// otherwise double (capped) toward the slow ceiling.
|
|
func nextHealthPollInterval(current time.Duration, changed bool) time.Duration {
|
|
if changed {
|
|
return healthPollIntervalMin
|
|
}
|
|
next := current * 2
|
|
if next > healthPollIntervalMax {
|
|
next = healthPollIntervalMax
|
|
}
|
|
return next
|
|
}
|
|
|
|
// pollPSU polls PSU status via ipmitool and records it to statusDB. Returns
|
|
// true if any PSU's status differs from what statusDB currently has for it,
|
|
// which run uses to reset the backoff.
|
|
func (p *healthPoller) pollPSU() bool {
|
|
if p.statusDB == nil {
|
|
return false
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), psuIPMITimeout)
|
|
defer cancel()
|
|
|
|
cmd := exec.CommandContext(ctx, "ipmitool", "sdr")
|
|
var out bytes.Buffer
|
|
cmd.Stdout = &out
|
|
if err := cmd.Run(); err != nil {
|
|
// IPMI not available or not a server — skip silently.
|
|
slog.Debug("health poller: ipmitool sdr unavailable", "err", err)
|
|
return false
|
|
}
|
|
|
|
slots := collector.PSUSlotsFromSDR(out.String())
|
|
if len(slots) == 0 {
|
|
return false
|
|
}
|
|
|
|
const source = "watchdog:psu"
|
|
changed := false
|
|
for slot, psu := range slots {
|
|
key := "psu:" + slot
|
|
status := psu.Status
|
|
if status == "" {
|
|
status = "Unknown"
|
|
}
|
|
if prev, ok := p.statusDB.Get(key); !ok || prev.Status != status {
|
|
changed = true
|
|
}
|
|
detail := ""
|
|
switch status {
|
|
case "Critical":
|
|
detail = "PSU sensor reported non-OK state"
|
|
case "Warning":
|
|
detail = "PSU sensor in warning state"
|
|
}
|
|
p.statusDB.Record(key, source, status, detail)
|
|
}
|
|
return changed
|
|
}
|