Files
bee/audit/internal/platform/sat_fan_stress_test.go
T
Mikhail ChusavitinandClaude Sonnet 5 bb2a501a28 feat(sat): fan ceiling check + topology fan tiles
Repurpose the previously-unwired RunFanStressTest into RunFanCheck, a
Load-tier SAT test that drives stressapptest (CPU+memory) and, when a GPU
is present, a GPU burn to 100% simultaneously, then watches every fan
until none has climbed for ~60s. The observed peak RPM per fan is the
"ceiling"; it is persisted through the existing fan-observation store.

MSI G4201 / AMI MegaRAC exposes no host-side fan force (every OEM IPMI
command returns 0xc1; Redfish Thermal is GET-only), so load-driven ramp
is the closest safe equivalent. See
bible-local/decisions/2026-09-04-fan-ceiling-check.md.

- platform.ResolveFanMaxRPM: per-fan max with fallback (persisted peak ->
  peer peak -> current RPM), resolved in platform, not the view.
- platform.ErrTestNotApplicable: no load source or no fan sensors ->
  task lands as cancelled ("not applicable"), never failed, so an
  engineer never sees a false red. executeTaskWithOptions maps the
  sentinel; finalizeTaskForResult honours a pre-set TaskCancelled.
- Verdict FAIL only for a fan at 0 RPM / IPMI cr-nr under load.
- /topo: one small spinning square per fan, sized by RPM / resolved max,
  clickable through to a new "fan" component-detail type; per-fan status
  recorded to the component-status DB from the fan SAT summary.
- Wiring: /api/sat/fan/run route, "fan" task target, Load-page card,
  stress-mode Run All.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019VHG21rgTUiR1G3qFHTVmN
2026-09-04 10:35:34 +03:00

194 lines
5.7 KiB
Go

package platform
import (
"os"
"path/filepath"
"reflect"
"testing"
"time"
)
func TestResolveFanMaxRPM(t *testing.T) {
oldPath := fanObservationStatePath
oldInit := fanObservationInit
fanObservationStatePath = filepath.Join(t.TempDir(), "fan-observation.json")
fanObservationInit = false
t.Cleanup(func() {
fanObservationStatePath = oldPath
fanObservationInit = oldInit
})
// No persisted file yet: unknown fans fall back to their own current RPM.
got := ResolveFanMaxRPM(map[string]float64{"A": 4000, "B": 9000})
if !reflect.DeepEqual(got, map[string]float64{"A": 4000, "B": 9000}) {
t.Fatalf("no-persist fallback: got %v", got)
}
if err := os.WriteFile(fanObservationStatePath, []byte(`{"max_rpm":{"A":17000}}`), 0644); err != nil {
t.Fatal(err)
}
got = ResolveFanMaxRPM(map[string]float64{"A": 4000, "B": 9000, "C": 5000})
// A: persisted peak. B/C: no own entry -> largest peer peak (A's 17000).
if want := map[string]float64{"A": 17000, "B": 17000, "C": 17000}; !reflect.DeepEqual(got, want) {
t.Fatalf("peer fallback: got %v want %v", got, want)
}
}
func TestApplyFanCheckDefaults(t *testing.T) {
var o FanCheckOptions
applyFanCheckDefaults(&o)
if o.PlateauHoldSec != 60 || o.PlateauDeltaRPM != 50 || o.MinLoadSec != 90 || o.MaxLoadSec != 900 || o.RampConfirmRPM != 150 {
t.Fatalf("unexpected defaults: %+v", o)
}
o = FanCheckOptions{PlateauHoldSec: 120, MinLoadSec: 30, MaxLoadSec: 40}
applyFanCheckDefaults(&o)
if o.MinLoadSec < o.PlateauHoldSec {
t.Fatalf("MinLoadSec must be >= PlateauHoldSec, got %d", o.MinLoadSec)
}
if o.MaxLoadSec <= o.MinLoadSec {
t.Fatalf("MaxLoadSec must exceed MinLoadSec, got %d", o.MaxLoadSec)
}
}
func TestSanitizeSummaryKey(t *testing.T) {
for in, want := range map[string]string{
"F2U-1": "F2U-1",
"aspeed / fan1": "aspeed___fan1",
"CPU0_DIMM": "CPU0_DIMM",
"weird=key here": "weird_key_here",
} {
if got := sanitizeSummaryKey(in); got != want {
t.Errorf("sanitizeSummaryKey(%q)=%q want %q", in, got, want)
}
}
}
func TestParseFanSpeeds(t *testing.T) {
raw := "FAN1 | 2400.000 | RPM | ok\nFAN2 | 1800 RPM | ok | ok\nFAN3 | na | RPM | ns\n"
got := parseFanSpeeds(raw)
if len(got) != 2 {
t.Fatalf("fans=%d want 2 (%v)", len(got), got)
}
if got[0].Name != "FAN1" || got[0].RPM != 2400 {
t.Fatalf("fan0=%+v", got[0])
}
if got[1].Name != "FAN2" || got[1].RPM != 1800 {
t.Fatalf("fan1=%+v", got[1])
}
}
func TestFirstFanInputValue(t *testing.T) {
feature := map[string]any{
"fan1_input": 9200.0,
}
got, ok := firstFanInputValue(feature)
if !ok || got != 9200 {
t.Fatalf("got=%v ok=%v", got, ok)
}
}
func TestParseFanDutyCyclePctSensorsJSON(t *testing.T) {
raw := []byte(`{
"chip0": {
"fan1": {"input": 9000},
"pwm1": {"input": 128},
"pwm1_enable": {"input": 1}
},
"chip1": {
"pwm2": {"input": 64}
}
}`)
got, ok := parseFanDutyCyclePctSensorsJSON(raw)
if !ok {
t.Fatalf("expected duty cycle telemetry to be parsed")
}
if got < 57 || got > 58 {
t.Fatalf("got=%v want ~57.1", got)
}
}
func TestEstimateFanDutyCyclePctFromObservation(t *testing.T) {
t.Parallel()
oldPath := fanObservationStatePath
oldState := fanObservation
oldInit := fanObservationInit
oldCandidates := fanPeakCandidates
fanObservationStatePath = filepath.Join(t.TempDir(), "fan-observation.json")
fanObservation = fanObservationState{}
fanObservationInit = false
fanPeakCandidates = make(map[string]fanPeakCandidate)
t.Cleanup(func() {
fanObservationStatePath = oldPath
fanObservation = oldState
fanObservationInit = oldInit
fanPeakCandidates = oldCandidates
})
start := time.Unix(100, 0)
updateFanObservation([]FanReading{{Name: "FAN1", RPM: 5000}}, start)
if _, ok := estimateFanDutyCyclePctFromObservation([]FanReading{{Name: "FAN1", RPM: 2500}}); ok {
t.Fatalf("single-sample spike should not establish observed max")
}
updateFanObservation([]FanReading{{Name: "FAN1", RPM: 5200}}, start.Add(500*time.Millisecond))
updateFanObservation([]FanReading{{Name: "FAN1", RPM: 5100}}, start.Add(1500*time.Millisecond))
got, ok := estimateFanDutyCyclePctFromObservation([]FanReading{{Name: "FAN1", RPM: 2600}})
if !ok {
t.Fatalf("expected estimated duty cycle from persisted observed max")
}
if got < 43 || got > 44 {
t.Fatalf("got=%v want ~43.3", got)
}
fanObservation = fanObservationState{}
fanObservationInit = false
fanPeakCandidates = make(map[string]fanPeakCandidate)
got, ok = estimateFanDutyCyclePctFromObservation([]FanReading{{Name: "FAN1", RPM: 2600}})
if !ok {
t.Fatalf("expected persisted observed max to be reloaded from disk")
}
if got < 43 || got > 44 {
t.Fatalf("reloaded got=%v want ~43.3", got)
}
}
func TestParseDCMIPowerReading(t *testing.T) {
raw := `
Instantaneous power reading: 512 Watts
Minimum during sampling period: 498 Watts
`
if got := parseDCMIPowerReading(raw); got != 512 {
t.Fatalf("parseDCMIPowerReading()=%v want 512", got)
}
}
func TestEffectiveSystemPowerReading(t *testing.T) {
now := time.Now()
cache := cachedPowerReading{Value: 480, UpdatedAt: now.Add(-5 * time.Second)}
got, updated := effectiveSystemPowerReading(cache, 0, "", "", "", now)
if got != 480 {
t.Fatalf("got=%v want cached 480", got)
}
if updated.Value != 480 {
t.Fatalf("updated=%+v", updated)
}
got, updated = effectiveSystemPowerReading(cache, 530, "dcmi", "fallback", "test", now)
if got != 530 {
t.Fatalf("got=%v want 530", got)
}
if updated.Value != 530 {
t.Fatalf("updated=%+v", updated)
}
expired := cachedPowerReading{Value: 480, UpdatedAt: now.Add(-systemPowerHoldTTL - time.Second)}
got, _ = effectiveSystemPowerReading(expired, 0, "", "", "", now)
if got != 0 {
t.Fatalf("expired cache returned %v want 0", got)
}
}