feat(sat): fan ceiling check + topology fan tiles

Repurpose the previously-unwired RunFanStressTest into RunFanCheck, a
Load-tier SAT test that drives stressapptest (CPU+memory) and, when a GPU
is present, a GPU burn to 100% simultaneously, then watches every fan
until none has climbed for ~60s. The observed peak RPM per fan is the
"ceiling"; it is persisted through the existing fan-observation store.

MSI G4201 / AMI MegaRAC exposes no host-side fan force (every OEM IPMI
command returns 0xc1; Redfish Thermal is GET-only), so load-driven ramp
is the closest safe equivalent. See
bible-local/decisions/2026-09-04-fan-ceiling-check.md.

- platform.ResolveFanMaxRPM: per-fan max with fallback (persisted peak ->
  peer peak -> current RPM), resolved in platform, not the view.
- platform.ErrTestNotApplicable: no load source or no fan sensors ->
  task lands as cancelled ("not applicable"), never failed, so an
  engineer never sees a false red. executeTaskWithOptions maps the
  sentinel; finalizeTaskForResult honours a pre-set TaskCancelled.
- Verdict FAIL only for a fan at 0 RPM / IPMI cr-nr under load.
- /topo: one small spinning square per fan, sized by RPM / resolved max,
  clickable through to a new "fan" component-detail type; per-fan status
  recorded to the component-status DB from the fan SAT summary.
- Wiring: /api/sat/fan/run route, "fan" task target, Load-page card,
  stress-mode Run All.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019VHG21rgTUiR1G3qFHTVmN
This commit is contained in:
Mikhail Chusavitin
2026-09-04 10:35:34 +03:00
co-authored by Claude Sonnet 5
parent e4f7519ef3
commit bb2a501a28
22 changed files with 798 additions and 222 deletions
+1 -1
View File
@@ -173,7 +173,7 @@ type satRunner interface {
RunAMDStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error)
RunMemoryStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error)
RunSATStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error)
RunFanStressTest(ctx context.Context, baseDir string, opts platform.FanStressOptions) (string, error)
RunFanCheck(ctx context.Context, baseDir string, opts platform.FanCheckOptions, logFunc func(string)) (string, error)
RunPlatformStress(ctx context.Context, baseDir string, opts platform.PlatformStressOptions, logFunc func(string)) (string, error)
RunNCCLTests(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error)
RunScenario(ctx context.Context, baseDir string, spec platform.ScenarioSpec, logFunc func(string)) (string, error)
+7
View File
@@ -298,6 +298,13 @@ func (a *App) RunPlatformStress(ctx context.Context, baseDir string, opts platfo
return a.sat.RunPlatformStress(ctx, baseDir, opts, logFunc)
}
func (a *App) RunFanCheckCtx(ctx context.Context, baseDir string, opts platform.FanCheckOptions, logFunc func(string)) (string, error) {
if strings.TrimSpace(baseDir) == "" {
baseDir = DefaultSATBaseDir
}
return a.sat.RunFanCheck(ctx, baseDir, opts, logFunc)
}
func (a *App) RunNCCLTests(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) {
if strings.TrimSpace(baseDir) == "" {
baseDir = DefaultSATBaseDir
+1 -1
View File
@@ -367,7 +367,7 @@ func (f fakeSAT) RunSATStressPack(_ context.Context, _ string, _ int, _ func(str
return "", nil
}
func (f fakeSAT) RunFanStressTest(_ context.Context, _ string, _ platform.FanStressOptions) (string, error) {
func (f fakeSAT) RunFanCheck(_ context.Context, _ string, _ platform.FanCheckOptions, _ func(string)) (string, error) {
return "", nil
}
+17
View File
@@ -318,6 +318,23 @@ func ApplySATResultToDB(db *ComponentStatusDB, target, archivePath string) {
db.Record("memory:all", source, dbStatus, detail)
case "cpu", "platform-stress":
db.Record("cpu:all", source, dbStatus, detail)
case "fan":
// Per-fan keys: summary emits "fan_<name>_status=OK|FAILED (...)".
for key, val := range kv {
name, ok := strings.CutPrefix(key, "fan_")
if !ok {
continue
}
name, ok = strings.CutSuffix(name, "_status")
if !ok || name == "" {
continue
}
upper := strings.ToUpper(strings.TrimSpace(val))
if i := strings.IndexByte(upper, ' '); i > 0 {
upper = upper[:i] // drop the "(reason)" suffix
}
db.Record("fan:"+name, source, satStatusToDBStatus(upper), target+" SAT: "+strings.TrimSpace(val))
}
case "storage":
// Try to record per-device if available in summary.
recordedAny := false
+10
View File
@@ -0,0 +1,10 @@
package platform
import "errors"
// ErrTestNotApplicable is returned by a SAT routine when the host provides no
// way to run that test at all (a required tool or capability is absent), as
// opposed to the test running and finding a fault. The task layer maps it to a
// cancelled ("not applicable") task rather than a failure, so an engineer does
// not see a false red.
var ErrTestNotApplicable = errors.New("test not applicable on this platform")
+351 -209
View File
@@ -15,14 +15,18 @@ import (
"time"
)
// FanStressOptions configures the fan-stress / thermal cycling test.
type FanStressOptions struct {
BaselineSec int // idle monitoring before and after load (default 30)
Phase1DurSec int // first load phase duration in seconds (default 300)
PauseSec int // pause between the two load phases (default 60)
Phase2DurSec int // second load phase duration in seconds (default 300)
SizeMB int // GPU memory to allocate per GPU during stress (0 = auto: 95% of VRAM)
GPUIndices []int // which GPU indices to stress (empty = all detected)
// FanCheckOptions configures the fan-ceiling check: it drives CPU (+memory)
// and, when present, GPU load to 100% simultaneously, then watches every fan
// until none has climbed for PlateauHoldSec — at which point each fan is
// considered to be at its physical ceiling and the observed peak is recorded.
type FanCheckOptions struct {
PlateauHoldSec int // a fan must not rise > PlateauDeltaRPM for this long to count as plateaued (default 60)
PlateauDeltaRPM int // RPM increase that still counts as "climbing" (default 50)
MinLoadSec int // never declare a plateau before this many seconds of load (default 90)
MaxLoadSec int // hard cap on the load phase; finish (success) even if not every fan plateaued (default 900)
RampConfirmRPM int // at least one fan must exceed baseline by this before a plateau is "real" (default 150)
SizeMB int // GPU memory to allocate per GPU (0 = auto)
GPUIndices []int // which GPU indices to load (empty = all detected)
}
// FanReading holds one fan sensor reading.
@@ -93,128 +97,217 @@ func normalizeObservedFanMaxRPM(rpm float64) float64 {
return math.Ceil(rpm/1000.0) * 1000.0
}
// RunFanStressTest runs a two-phase GPU stress test while monitoring fan speeds,
// temperatures, and power draw every second. Exports metrics.csv and fan-sensors.csv.
// Designed to reproduce case-04 fan-speed lag and detect GPU thermal throttling.
func (s *System) RunFanStressTest(ctx context.Context, baseDir string, opts FanStressOptions) (string, error) {
// RunFanCheck drives CPU (+memory) and, when a GPU is present, GPU load to
// 100% simultaneously and watches every fan until none has climbed for
// PlateauHoldSec. At that point each fan is taken to be at its physical
// ceiling; the observed peak RPM is persisted (fanObservationStatePath, the
// same store ObservedFanMaxRPM reads) so the topology view can size each fan
// tile against a real maximum.
//
// Outcome:
// - success ("ceiling found") once every fan plateaus, or when MaxLoadSec is
// hit — a run that simply ran out of time still recorded the highest RPM
// seen and is not a failure.
// - a fan reading 0 RPM, or an IPMI status of cr/nr, while under full load is
// a real defect → FAILED.
// - if there is no way to load this box (no stressapptest/stress-ng and no
// GPU burn tool) or no fan sensors are readable, the test cannot say
// anything about the hardware and returns ErrTestNotApplicable so the task
// is cancelled, not failed.
//
// No GPU is not an error: CPU/memory load alone is enough to exercise the
// cooling loop on most platforms.
func (s *System) RunFanCheck(ctx context.Context, baseDir string, opts FanCheckOptions, logFunc func(string)) (string, error) {
if logFunc == nil {
logFunc = func(string) {}
}
if baseDir == "" {
baseDir = "/var/log/bee-sat"
}
applyFanStressDefaults(&opts)
applyFanCheckDefaults(&opts)
baseFans, fanErr := sampleFanSpeeds()
if len(baseFans) == 0 {
return "", fmt.Errorf("no fan sensors readable via ipmitool or lm-sensors (%v): %w", fanErr, ErrTestNotApplicable)
}
baselineRPM := make(map[string]float64, len(baseFans))
for _, f := range baseFans {
baselineRPM[f.Name] = f.RPM
}
vendor := s.DetectGPUVendor()
haveGPU := vendor == "nvidia" || vendor == "amd"
_, cpuPathErr := satLookPath("stressapptest")
if cpuPathErr != nil {
_, cpuPathErr = satLookPath("stress-ng")
}
haveCPU := cpuPathErr == nil
if !haveCPU && !haveGPU {
return "", fmt.Errorf("no load source: stressapptest/stress-ng missing and no NVIDIA/AMD GPU stress tool available: %w", ErrTestNotApplicable)
}
ts := time.Now().UTC().Format("20060102-150405")
runDir := filepath.Join(baseDir, "fan-stress-"+ts)
runDir := filepath.Join(baseDir, "fan-check-"+ts)
if err := os.MkdirAll(runDir, 0755); err != nil {
return "", err
}
verboseLog := filepath.Join(runDir, "verbose.log")
appendSATVerboseLog(verboseLog, fmt.Sprintf("[%s] fan check start: %d fans, gpu=%s cpu=%v",
time.Now().UTC().Format(time.RFC3339), len(baseFans), vendor, haveCPU))
logFunc(fmt.Sprintf("Fan check: %d fans; load = CPU/mem:%v + GPU:%s", len(baseFans), haveCPU, orNone(haveGPU, vendor)))
// Phase name shared between sampler goroutine and main goroutine.
var phaseMu sync.Mutex
currentPhase := "init"
setPhase := func(name string) {
phaseMu.Lock()
currentPhase = name
phaseMu.Unlock()
// ── Load: every source runs at the same time, each in its own goroutine.
// Sources can take different amounts of time to actually reach full load
// (stressapptest is near-instant; a GPU burn kernel needs to compile and
// ramp), so the plateau clock does not start until every launched source
// has reported its process running, plus a fixed GPU ramp grace.
loadCtx, loadCancel := context.WithTimeout(ctx, time.Duration(opts.MaxLoadSec)*time.Second)
defer loadCancel()
var loadWG sync.WaitGroup
started := make(chan bool, 2) // true = source is running, false = failed to launch
launched := 0
if haveCPU {
launched++
loadWG.Add(1)
go func() {
defer loadWG.Done()
cmd, err := buildCPUStressCmd(loadCtx)
if err != nil {
logFunc("CPU/memory load failed to start: " + err.Error())
appendSATVerboseLog(verboseLog, "cpu load start error: "+err.Error())
started <- false
return
}
logFunc("CPU/memory load running (stressapptest)")
started <- true
_ = cmd.Wait()
}()
}
getPhase := func() string {
phaseMu.Lock()
defer phaseMu.Unlock()
return currentPhase
if haveGPU {
launched++
loadWG.Add(1)
go func() {
defer loadWG.Done()
cmd := buildGPUStressCmd(loadCtx, vendor, opts.MaxLoadSec)
if cmd == nil {
logFunc("GPU load unavailable (no burn tool for " + vendor + ")")
appendSATVerboseLog(verboseLog, "gpu load: no burn tool")
started <- false
return
}
logFunc("GPU load running (" + vendor + ")")
started <- true
_ = cmd.Wait()
}()
}
start := time.Now()
var rowsMu sync.Mutex
var allRows []FanStressRow
// Start background sampler (every second).
stopCh := make(chan struct{})
doneCh := make(chan struct{})
go func() {
defer close(doneCh)
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
for {
select {
case <-stopCh:
return
case <-ticker.C:
row := sampleFanStressRow(opts.GPUIndices, getPhase(), time.Since(start).Seconds())
rowsMu.Lock()
allRows = append(allRows, row)
rowsMu.Unlock()
activeLoads := 0
for i := 0; i < launched; i++ {
select {
case ok := <-started:
if ok {
activeLoads++
}
case <-ctx.Done():
}
}()
}
if activeLoads == 0 {
loadCancel()
loadWG.Wait()
return "", fmt.Errorf("every load source failed to start: %w", ErrTestNotApplicable)
}
readyAt := time.Now()
if haveGPU {
readyAt = readyAt.Add(20 * time.Second) // GPU kernel ramp grace
}
appendSATVerboseLog(verboseLog, fmt.Sprintf("%d load source(s) active; plateau clock effective from +%.0fs",
activeLoads, readyAt.Sub(start).Seconds()))
var summary strings.Builder
fmt.Fprintf(&summary, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339))
// ── Sample loop: one row/second, per-fan plateau tracking.
type fanState struct {
peak float64
lastRiseSec float64
}
fanBy := map[string]*fanState{}
var rows []FanStressRow
rampConfirmed := false
plateauReached := false
aborted := false
stats := satStats{}
// idlePhase sleeps for durSec while the sampler stamps phaseName on each row.
idlePhase := func(phaseName, stepName string, durSec int) {
if ctx.Err() != nil {
return
}
setPhase(phaseName)
appendSATVerboseLog(verboseLog,
fmt.Sprintf("[%s] start %s (idle %ds)", time.Now().UTC().Format(time.RFC3339), stepName, durSec),
)
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
loop:
for {
select {
case <-ctx.Done():
case <-time.After(time.Duration(durSec) * time.Second):
aborted = true
break loop
case <-loadCtx.Done():
break loop // MaxLoadSec reached
case <-ticker.C:
}
appendSATVerboseLog(verboseLog,
fmt.Sprintf("[%s] finish %s", time.Now().UTC().Format(time.RFC3339), stepName),
)
fmt.Fprintf(&summary, "%s_status=OK\n", stepName)
stats.OK++
}
elapsed := time.Since(start).Seconds()
row := sampleFanStressRow(opts.GPUIndices, "load", elapsed)
rows = append(rows, row)
// loadPhase runs bee-gpu-burn for durSec; sampler stamps phaseName on each row.
loadPhase := func(phaseName, stepName string, durSec int) {
if ctx.Err() != nil {
return
for _, f := range row.Fans {
st := fanBy[f.Name]
if st == nil {
fanBy[f.Name] = &fanState{peak: f.RPM, lastRiseSec: elapsed}
continue
}
if f.RPM > st.peak {
if f.RPM-st.peak > float64(opts.PlateauDeltaRPM) {
st.lastRiseSec = elapsed
}
st.peak = f.RPM
}
if f.RPM >= baselineRPM[f.Name]+float64(opts.RampConfirmRPM) {
rampConfirmed = true
}
}
setPhase(phaseName)
cmd := []string{
"bee-gpu-burn",
"--seconds", strconv.Itoa(durSec),
"--size-mb", strconv.Itoa(opts.SizeMB),
}
if len(opts.GPUIndices) > 0 {
cmd = append(cmd, "--devices", joinIndexList(dedupeSortedIndices(opts.GPUIndices)))
}
out, err := runSATCommandCtx(ctx, verboseLog, stepName, cmd, nil, nil)
_ = os.WriteFile(filepath.Join(runDir, stepName+".log"), out, 0644)
if err != nil && err != context.Canceled && err.Error() != "signal: killed" {
fmt.Fprintf(&summary, "%s_status=FAILED\n", stepName)
stats.Failed++
} else {
fmt.Fprintf(&summary, "%s_status=OK\n", stepName)
stats.OK++
if elapsed >= float64(opts.MinLoadSec) && time.Since(readyAt) >= time.Duration(opts.PlateauHoldSec)*time.Second && len(fanBy) > 0 {
allFlat := true
for _, st := range fanBy {
if elapsed-st.lastRiseSec < float64(opts.PlateauHoldSec) {
allFlat = false
break
}
}
if allFlat && rampConfirmed {
plateauReached = true
logFunc(fmt.Sprintf("All %d fans plateaued at %.0fs of load", len(fanBy), elapsed))
break loop
}
}
}
// Execute test phases.
idlePhase("baseline", "01-baseline", opts.BaselineSec)
loadPhase("load1", "02-load1", opts.Phase1DurSec)
idlePhase("pause", "03-pause", opts.PauseSec)
loadPhase("load2", "04-load2", opts.Phase2DurSec)
idlePhase("cooldown", "05-cooldown", opts.BaselineSec)
loadCancel()
loadWG.Wait()
// Stop sampler and collect rows.
close(stopCh)
<-doneCh
if aborted && ctx.Err() != nil {
_ = os.WriteFile(filepath.Join(runDir, "summary.txt"),
[]byte("run_at_utc="+time.Now().UTC().Format(time.RFC3339)+"\noverall_status=UNKNOWN\naborted=true\n"), 0644)
return runDir, ctx.Err()
}
rowsMu.Lock()
rows := allRows
rowsMu.Unlock()
// Analysis.
throttled := analyzeThrottling(rows)
maxGPUTemp := analyzeMaxTemp(rows, func(r FanStressRow) float64 {
// ── Verdict.
statuses := readFanStatuses()
var summary strings.Builder
fmt.Fprintf(&summary, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339))
fmt.Fprintf(&summary, "fans_total=%d\n", len(baseFans))
fmt.Fprintf(&summary, "active_load_sources=%d\n", activeLoads)
fmt.Fprintf(&summary, "gpu_vendor=%s\n", orNone(haveGPU, vendor))
fmt.Fprintf(&summary, "plateau_reached=%v\n", plateauReached)
fmt.Fprintf(&summary, "ramp_confirmed=%v\n", rampConfirmed)
if len(rows) > 0 {
fmt.Fprintf(&summary, "load_duration_sec=%.0f\n", rows[len(rows)-1].ElapsedSec)
}
fmt.Fprintf(&summary, "max_gpu_temp_c=%.1f\n", analyzeMaxTemp(rows, func(r FanStressRow) float64 {
var m float64
for _, g := range r.GPUs {
if g.TempC > m {
@@ -222,55 +315,148 @@ func (s *System) RunFanStressTest(ctx context.Context, baseDir string, opts FanS
}
}
return m
})
maxCPUTemp := analyzeMaxTemp(rows, func(r FanStressRow) float64 {
return r.CPUMaxTempC
})
fanResponseSec := analyzeFanResponse(rows)
}))
fmt.Fprintf(&summary, "max_cpu_temp_c=%.1f\n", analyzeMaxTemp(rows, func(r FanStressRow) float64 { return r.CPUMaxTempC }))
fmt.Fprintf(&summary, "throttling_detected=%v\n", throttled)
fmt.Fprintf(&summary, "max_gpu_temp_c=%.1f\n", maxGPUTemp)
fmt.Fprintf(&summary, "max_cpu_temp_c=%.1f\n", maxCPUTemp)
if fanResponseSec >= 0 {
fmt.Fprintf(&summary, "fan_response_sec=%.1f\n", fanResponseSec)
} else {
fmt.Fprintf(&summary, "fan_response_sec=N/A\n")
stats := satStats{}
names := make([]string, 0, len(baselineRPM))
for n := range baselineRPM {
names = append(names, n)
}
// Throttling failure counts against overall result.
if throttled {
stats.Failed++
sort.Strings(names)
for _, name := range names {
peak := baselineRPM[name]
if st := fanBy[name]; st != nil {
peak = st.peak
}
st := strings.ToLower(strings.TrimSpace(statuses[name]))
bad := peak <= 0 || st == "cr" || st == "nr"
key := sanitizeSummaryKey(name)
fmt.Fprintf(&summary, "fan_%s_baseline_rpm=%.0f\n", key, baselineRPM[name])
fmt.Fprintf(&summary, "fan_%s_max_rpm=%.0f\n", key, peak)
if bad {
reason := "0 RPM under load"
if st == "cr" || st == "nr" {
reason = "IPMI status " + st
}
fmt.Fprintf(&summary, "fan_%s_status=FAILED (%s)\n", key, reason)
logFunc(fmt.Sprintf("FAIL %s: %s", name, reason))
stats.Failed++
} else {
fmt.Fprintf(&summary, "fan_%s_status=OK\n", key)
stats.OK++
}
}
writeSATStats(&summary, stats)
// Write CSV outputs.
if err := WriteFanStressCSV(filepath.Join(runDir, "metrics.csv"), rows, opts.GPUIndices); err != nil {
return "", err
}
_ = WriteFanStressCSV(filepath.Join(runDir, "metrics.csv"), rows, opts.GPUIndices)
_ = WriteFanSensorsCSV(filepath.Join(runDir, "fan-sensors.csv"), rows)
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary.String()), 0644); err != nil {
return "", err
}
return runDir, nil
}
func applyFanStressDefaults(opts *FanStressOptions) {
if opts.BaselineSec <= 0 {
opts.BaselineSec = 30
func applyFanCheckDefaults(o *FanCheckOptions) {
if o.PlateauHoldSec <= 0 {
o.PlateauHoldSec = 60
}
if opts.Phase1DurSec <= 0 {
opts.Phase1DurSec = 300
if o.PlateauDeltaRPM <= 0 {
o.PlateauDeltaRPM = 50
}
if opts.PauseSec <= 0 {
opts.PauseSec = 60
if o.MinLoadSec <= 0 {
o.MinLoadSec = 90
}
if opts.Phase2DurSec <= 0 {
opts.Phase2DurSec = 300
if o.MaxLoadSec <= 0 {
o.MaxLoadSec = 900
}
// SizeMB == 0 means "auto" (worker picks 95% of GPU VRAM for maximum power draw).
// Leave at 0 to avoid passing a too-small size that starves the tensor-core path.
if o.RampConfirmRPM <= 0 {
o.RampConfirmRPM = 150
}
if o.MinLoadSec < o.PlateauHoldSec {
o.MinLoadSec = o.PlateauHoldSec
}
if o.MaxLoadSec <= o.MinLoadSec {
o.MaxLoadSec = o.MinLoadSec + o.PlateauHoldSec
}
}
func orNone(present bool, v string) string {
if present && v != "" {
return v
}
return "none"
}
// sanitizeSummaryKey makes a fan sensor name safe as a summary.txt key
// fragment (keys are parsed by splitting on '=' and whitespace).
func sanitizeSummaryKey(name string) string {
var b strings.Builder
for _, r := range name {
switch {
case r >= 'A' && r <= 'Z', r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '-', r == '_', r == '.':
b.WriteRune(r)
default:
b.WriteByte('_')
}
}
return b.String()
}
// readFanStatuses returns the per-fan IPMI status word ("ok", "cr", "nr", ...)
// from "ipmitool sdr type Fan". Empty map when ipmitool is unavailable.
func readFanStatuses() map[string]string {
out, err := exec.Command("ipmitool", "sdr", "type", "Fan").Output()
if err != nil {
return nil
}
m := map[string]string{}
for _, line := range strings.Split(string(out), "\n") {
parts := strings.Split(line, "|")
if len(parts) < 3 {
continue
}
name := strings.TrimSpace(parts[0])
if name == "" {
continue
}
m[name] = strings.ToLower(strings.TrimSpace(parts[2]))
}
return m
}
// ResolveFanMaxRPM returns, for every fan name in current (name -> current
// RPM), the RPM to treat as that fan's 100% reference. Preference order:
// 1. the persisted observed peak (fanObservationStatePath), written by
// RunFanCheck and by live-metrics sampling under load;
// 2. the largest peak observed on any peer fan (keeps a group visually
// consistent when only some fans have a recorded peak);
// 3. the fan's own current RPM (so a tile is never sized against zero).
//
// The fallback lives here, not in the view, so every consumer of a fan
// maximum applies the same rule.
func ResolveFanMaxRPM(current map[string]float64) map[string]float64 {
persisted := readPersistedFanMaxRPM()
peerMax := 0.0
for _, v := range persisted {
if v > peerMax {
peerMax = v
}
}
out := make(map[string]float64, len(current))
for name, rpm := range current {
switch {
case persisted[name] > 0:
out[name] = persisted[name]
case peerMax > 0:
out[name] = peerMax
default:
out[name] = rpm
}
}
return out
}
// sampleFanStressRow collects all metrics for one telemetry sample.
@@ -355,27 +541,48 @@ func sampleFanSpeeds() ([]FanReading, error) {
return nil, sensorsErr
}
func loadFanObservationLocked() {
if fanObservationInit {
return
}
fanObservationInit = true
fanObservation.MaxRPM = make(map[string]float64)
// readPersistedFanMaxRPM reads fanObservationStatePath and returns its
// sanitized {fan name -> observed peak RPM} map (empty names / non-positive
// values dropped). Returns an empty map when the file is missing or unparsable.
func readPersistedFanMaxRPM() map[string]float64 {
out := map[string]float64{}
raw, err := os.ReadFile(fanObservationStatePath)
if err != nil || len(raw) == 0 {
return
return out
}
var persisted fanObservationState
if json.Unmarshal(raw, &persisted) != nil {
return
return out
}
for name, rpm := range persisted.MaxRPM {
name = strings.TrimSpace(name)
if name == "" || rpm <= 0 {
continue
}
fanObservation.MaxRPM[name] = rpm
out[name] = rpm
}
return out
}
// ObservedFanMaxRPM returns the per-fan observed peak RPM map persisted by
// fan-stress SAT runs, or nil if none is recorded yet. It reads the file
// directly without touching the in-process observation cache or its lock, so
// read-only consumers (the /topo web view) can call it without perturbing a
// concurrent SAT run's peak tracking.
func ObservedFanMaxRPM() map[string]float64 {
out := readPersistedFanMaxRPM()
if len(out) == 0 {
return nil
}
return out
}
func loadFanObservationLocked() {
if fanObservationInit {
return
}
fanObservationInit = true
fanObservation.MaxRPM = readPersistedFanMaxRPM()
}
func saveFanObservationLocked() {
@@ -779,22 +986,6 @@ func effectiveSystemPowerReading(cache cachedPowerReading, current float64, sour
return 0, cache
}
// analyzeThrottling returns true if any GPU reported an active throttle reason
// during either load phase.
func analyzeThrottling(rows []FanStressRow) bool {
for _, row := range rows {
if row.Phase != "load1" && row.Phase != "load2" {
continue
}
for _, gpu := range row.GPUs {
if gpu.Throttled {
return true
}
}
}
return false
}
// analyzeMaxTemp returns the maximum value of the given extractor across all rows.
func analyzeMaxTemp(rows []FanStressRow, extract func(FanStressRow) float64) float64 {
var max float64
@@ -806,55 +997,6 @@ func analyzeMaxTemp(rows []FanStressRow, extract func(FanStressRow) float64) flo
return max
}
// analyzeFanResponse returns the seconds from load1 start until fan RPM first
// increased by more than 5% above the baseline average. Returns -1 if undetermined.
func analyzeFanResponse(rows []FanStressRow) float64 {
// Compute baseline average fan RPM.
var baseTotal, baseCount float64
for _, row := range rows {
if row.Phase != "baseline" {
continue
}
for _, f := range row.Fans {
baseTotal += f.RPM
baseCount++
}
}
if baseCount == 0 || baseTotal == 0 {
return -1
}
baseAvg := baseTotal / baseCount
threshold := baseAvg * 1.05 // 5% increase signals fan ramp-up
// Find elapsed time when load1 started.
var load1Start float64 = -1
for _, row := range rows {
if row.Phase == "load1" {
load1Start = row.ElapsedSec
break
}
}
if load1Start < 0 {
return -1
}
// Find first load1 row where average RPM crosses the threshold.
for _, row := range rows {
if row.Phase != "load1" {
continue
}
var total, count float64
for _, f := range row.Fans {
total += f.RPM
count++
}
if count > 0 && total/count >= threshold {
return row.ElapsedSec - load1Start
}
}
return -1
}
// WriteFanStressCSV writes the wide-format metrics CSV with one row per second.
// GPU columns are generated per index in gpuIndices order.
func WriteFanStressCSV(path string, rows []FanStressRow, gpuIndices []int) error {
@@ -1,11 +1,68 @@
package platform
import (
"os"
"path/filepath"
"reflect"
"testing"
"time"
)
func TestResolveFanMaxRPM(t *testing.T) {
oldPath := fanObservationStatePath
oldInit := fanObservationInit
fanObservationStatePath = filepath.Join(t.TempDir(), "fan-observation.json")
fanObservationInit = false
t.Cleanup(func() {
fanObservationStatePath = oldPath
fanObservationInit = oldInit
})
// No persisted file yet: unknown fans fall back to their own current RPM.
got := ResolveFanMaxRPM(map[string]float64{"A": 4000, "B": 9000})
if !reflect.DeepEqual(got, map[string]float64{"A": 4000, "B": 9000}) {
t.Fatalf("no-persist fallback: got %v", got)
}
if err := os.WriteFile(fanObservationStatePath, []byte(`{"max_rpm":{"A":17000}}`), 0644); err != nil {
t.Fatal(err)
}
got = ResolveFanMaxRPM(map[string]float64{"A": 4000, "B": 9000, "C": 5000})
// A: persisted peak. B/C: no own entry -> largest peer peak (A's 17000).
if want := map[string]float64{"A": 17000, "B": 17000, "C": 17000}; !reflect.DeepEqual(got, want) {
t.Fatalf("peer fallback: got %v want %v", got, want)
}
}
func TestApplyFanCheckDefaults(t *testing.T) {
var o FanCheckOptions
applyFanCheckDefaults(&o)
if o.PlateauHoldSec != 60 || o.PlateauDeltaRPM != 50 || o.MinLoadSec != 90 || o.MaxLoadSec != 900 || o.RampConfirmRPM != 150 {
t.Fatalf("unexpected defaults: %+v", o)
}
o = FanCheckOptions{PlateauHoldSec: 120, MinLoadSec: 30, MaxLoadSec: 40}
applyFanCheckDefaults(&o)
if o.MinLoadSec < o.PlateauHoldSec {
t.Fatalf("MinLoadSec must be >= PlateauHoldSec, got %d", o.MinLoadSec)
}
if o.MaxLoadSec <= o.MinLoadSec {
t.Fatalf("MaxLoadSec must exceed MinLoadSec, got %d", o.MaxLoadSec)
}
}
func TestSanitizeSummaryKey(t *testing.T) {
for in, want := range map[string]string{
"F2U-1": "F2U-1",
"aspeed / fan1": "aspeed___fan1",
"CPU0_DIMM": "CPU0_DIMM",
"weird=key here": "weird_key_here",
} {
if got := sanitizeSummaryKey(in); got != want {
t.Errorf("sanitizeSummaryKey(%q)=%q want %q", in, got, want)
}
}
}
func TestParseFanSpeeds(t *testing.T) {
raw := "FAN1 | 2400.000 | RPM | ok\nFAN2 | 1800 RPM | ok | ok\nFAN3 | na | RPM | ns\n"
got := parseFanSpeeds(raw)
+1 -1
View File
@@ -128,7 +128,7 @@ func defaultTaskPriority(target string, params taskParams) int {
return taskPriorityAudit
case "nvidia-bench-perf", "nvidia-bench-power", "nvidia-bench-autotune":
return taskPriorityBenchmark
case "nvidia-stress", "amd-stress", "memory-stress", "sat-stress", "platform-stress", "nvidia-compute", "scenario":
case "nvidia-stress", "amd-stress", "memory-stress", "sat-stress", "platform-stress", "fan", "nvidia-compute", "scenario":
return taskPriorityBurn
case "nvidia", "nvidia-targeted-stress", "nvidia-targeted-power", "nvidia-pulse",
"nvidia-interconnect", "nvidia-bandwidth", "memory", "storage", "cpu",
+4 -1
View File
@@ -289,7 +289,7 @@ func (h *handler) handleAPIHardwareSummary(w http.ResponseWriter, _ *http.Reques
}
// handleAPIComponentDetail returns an HTML fragment describing the current and
// historical status for one component type (cpu, memory, storage, gpu, psu).
// historical status for one component type (cpu, memory, storage, gpu, psu, fan).
func (h *handler) handleAPIComponentDetail(w http.ResponseWriter, r *http.Request) {
compType := r.PathValue("type")
var exact, prefixes []string
@@ -315,6 +315,9 @@ func (h *handler) handleAPIComponentDetail(w http.ResponseWriter, r *http.Reques
case "psu":
title = "PSU"
prefixes = []string{"psu:"}
case "fan":
title = "Fans"
prefixes = []string{"fan:"}
case "raid":
title = "RAID"
prefixes = []string{"pcie:raid:"}
+5
View File
@@ -109,6 +109,11 @@ func (h *handler) planSATRunAll(ctx context.Context, req satRunAllRequest) ([]sa
} else {
skip("TPM: no TPM device on this host; check skipped")
}
} else {
// Fan ceiling check runs on the Load tier only. It self-cancels as
// "not applicable" on a host with no fan sensors or no way to load
// the CPU/GPU, so it is safe to queue unconditionally here.
specs = append(specs, satRunAllSpec{target: "fan", params: taskParams{StressMode: true}})
}
gp := h.opts.App.DetectGPUPresence()
+1 -1
View File
@@ -98,7 +98,7 @@ func TestPlanSATRunAllLoadOmitsReadOnlyTPMCheck(t *testing.T) {
for _, s := range specs {
targets = append(targets, s.target)
}
if want := []string{"cpu", "memory", "storage"}; !reflect.DeepEqual(targets, want) {
if want := []string{"cpu", "memory", "storage", "fan"}; !reflect.DeepEqual(targets, want) {
t.Fatalf("targets=%v want %v", targets, want)
}
for _, note := range notes {
+1 -1
View File
@@ -296,7 +296,7 @@ func isSATTarget(target string) bool {
case "nvidia", "nvidia-targeted-stress", "nvidia-bench-perf", "nvidia-bench-power", "nvidia-compute", "nvidia-targeted-power", "nvidia-pulse",
"nvidia-interconnect", "nvidia-bandwidth", "nvidia-stress", "memory", "memory-stress", "storage",
"cpu", "sat-stress", "amd", "amd-mem", "amd-bandwidth", "amd-stress",
"platform-stress":
"platform-stress", "fan":
return true
}
return false
+139 -2
View File
@@ -614,9 +614,138 @@ func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string
}}))
}
// Cooling fans — one small square per fan (no PCIe/CPU affinity, arbitrary
// count, so a wrapping flex row like PSUs rather than SVG boxes). Each
// square is sized by rpm / observed-max-rpm and carries a fan glyph that
// spins via CSS — faster when the fan is spinning faster.
if fans := dedupeFansByName(hw.Sensors); len(fans) > 0 {
current := map[string]float64{}
for _, f := range fans {
if f.RPM != nil {
current[strings.TrimSpace(f.Name)] = float64(*f.RPM)
}
}
b.WriteString(renderTopoFanRow(fans, platform.ResolveFanMaxRPM(current)))
}
return topoCard("Topology", b.String())
}
// renderTopoFanRow renders the COOLING row: one clickable square per fan,
// side length scaled by rpm/maxRPM and a fan glyph whose spin rate tracks the
// same ratio. maxByName comes from platform.ResolveFanMaxRPM — it already has
// an entry for every fan (persisted peak, else peer peak, else current RPM),
// so no fallback logic lives here.
func renderTopoFanRow(fans []schema.HardwareFanSensor, maxByName map[string]float64) string {
const (
fanTileMin = 30 // px, a stalled / slowest fan
fanTileMax = 58 // px, a fan at its ceiling
)
var tally topoStatusTally
for _, f := range fans {
tally.add(classifyTopoSeverity(f.Status))
}
var b strings.Builder
b.WriteString(topoRowHeading("Cooling"))
b.WriteString(topoFanSpinStyle())
b.WriteString(`<div style="display:flex;flex-wrap:wrap;gap:6px;align-items:flex-end">`)
for _, f := range fans {
name := strings.TrimSpace(f.Name)
fill, stroke, text := topoSeverityColors(classifyTopoSeverity(f.Status))
denom := maxByName[name]
ratio := 0.0
title := name
if f.RPM != nil {
if denom > 0 {
ratio = float64(*f.RPM) / denom
}
if ratio < 0 {
ratio = 0
}
if ratio > 1 {
ratio = 1
}
if denom > float64(*f.RPM) {
title = fmt.Sprintf("%s · %d RPM (max %d)", name, *f.RPM, int(denom))
} else {
title = fmt.Sprintf("%s · %d RPM", name, *f.RPM)
}
} else {
title = name + " · no reading"
}
side := fanTileMin + int(float64(fanTileMax-fanTileMin)*ratio+0.5)
// Spin period: 2.6s at rest down to 0.5s at the observed peak. A fan
// with no reading doesn't spin.
spin := ""
if f.RPM != nil && *f.RPM > 0 {
period := 2.6 - 2.1*ratio
spin = fmt.Sprintf(`<svg class="topo-fan-spin" style="animation-duration:%.2fs" width="%d" height="%d" viewBox="0 0 24 24" fill="currentColor" aria-hidden="true">`,
period, side*7/16, side*7/16) + topoFanGlyphPaths() + `</svg>`
} else {
spin = fmt.Sprintf(`<svg width="%d" height="%d" viewBox="0 0 24 24" fill="currentColor" style="opacity:.4" aria-hidden="true">`,
side*7/16, side*7/16) + topoFanGlyphPaths() + `</svg>`
}
fmt.Fprintf(&b, `<div title="%s" onclick="openComponentDetail('fan')" `+
`style="width:%dpx;height:%dpx;display:flex;align-items:center;justify-content:center;`+
`border-radius:5px;background:%s;border:1px solid %s;color:%s;cursor:pointer">%s</div>`,
html.EscapeString(title), side, side, fill, stroke, text, spin)
}
b.WriteString(`</div>`)
fmt.Fprintf(&b, `<div style="font-size:11px;color:var(--muted);margin-top:6px">%d fans · %s · tile size ∝ RPM / observed max</div>`,
len(fans), html.EscapeString(tally.line()))
return b.String()
}
// topoFanSpinStyle emits the keyframes + base class for the spinning fan
// glyph once per row. A repeated identical <style> is harmless.
func topoFanSpinStyle() string {
return `<style>@keyframes topoFanSpin{to{transform:rotate(360deg)}}` +
`.topo-fan-spin{transform-box:fill-box;transform-origin:center;` +
`animation-name:topoFanSpin;animation-timing-function:linear;animation-iteration-count:infinite}` +
`@media (prefers-reduced-motion:reduce){.topo-fan-spin{animation:none}}</style>`
}
// topoFanGlyphPaths is the fan-blade drawing shared by every fan square,
// designed on a 24×24 viewBox.
func topoFanGlyphPaths() string {
return `<ellipse cx="12" cy="6.5" rx="3.1" ry="5.2"/>` +
`<ellipse cx="12" cy="6.5" rx="3.1" ry="5.2" transform="rotate(120 12 12)"/>` +
`<ellipse cx="12" cy="6.5" rx="3.1" ry="5.2" transform="rotate(240 12 12)"/>` +
`<circle cx="12" cy="12" r="2.3"/>`
}
// dedupeFansByName returns the fan sensors from a snapshot with duplicate
// names collapsed to their first occurrence, matching the ingest contract's
// "(sensor_type, name) — first wins" rule and skipping unnamed sensors.
func dedupeFansByName(sensors *schema.HardwareSensors) []schema.HardwareFanSensor {
if sensors == nil {
return nil
}
seen := map[string]bool{}
var out []schema.HardwareFanSensor
for _, f := range sensors.Fans {
name := strings.TrimSpace(f.Name)
if name == "" || seen[name] {
continue
}
seen[name] = true
out = append(out, f)
}
return out
}
// topoRowHeading renders the small uppercase section label shared by the
// flex rows below the SVG diagram (Firmware / Power Supplies / Cooling / ...).
func topoRowHeading(title string) string {
return fmt.Sprintf(`<div style="font-size:11px;color:var(--muted);text-transform:uppercase;letter-spacing:.05em;margin:16px 0 6px">%s</div>`,
html.EscapeString(title))
}
// renderTopoFlexRow renders a labeled, wrapping row of component cards.
// Returns "" if items is empty (e.g. no PSU data in this audit).
func renderTopoFlexRow(title string, items []topoCardInfo) string {
@@ -624,8 +753,7 @@ func renderTopoFlexRow(title string, items []topoCardInfo) string {
return ""
}
var b strings.Builder
fmt.Fprintf(&b, `<div style="font-size:11px;color:var(--muted);text-transform:uppercase;letter-spacing:.05em;margin:16px 0 6px">%s</div>`,
html.EscapeString(title))
b.WriteString(topoRowHeading(title))
b.WriteString(`<div style="display:flex;flex-wrap:wrap;gap:10px">`)
for _, item := range items {
onclick := ""
@@ -1064,6 +1192,15 @@ func inventoryFallbackRecords(compType string, opts HandlerOptions) []app.Compon
}
records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(p.Status)})
}
case "fan":
for i, f := range dedupeFansByName(hw.Sensors) {
name := strings.TrimSpace(f.Name)
key := fmt.Sprintf("fan:%d", i)
if name != "" {
key = "fan:" + name
}
records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(f.Status)})
}
case "gpu", "nic", "raid":
for i, dev := range hw.PCIeDevices {
if pcieDeviceKind(dev) != compType {
+65
View File
@@ -152,6 +152,71 @@ func TestTopoPageRendersArbitraryPSUAndFirmwareCountsAsFlexRows(t *testing.T) {
}
}
func TestTopoPageRendersCoolingFansAsFlexRow(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "audit.json")
ok, warn := "OK", "Warning"
rpm := func(v int) *int { return &v }
ingest := schema.HardwareIngestRequest{
CollectedAt: "2026-03-15T00:00:00Z",
Hardware: schema.HardwareSnapshot{
Sensors: &schema.HardwareSensors{
Fans: []schema.HardwareFanSensor{
{Name: "FAN1", RPM: rpm(4200), Status: &ok},
{Name: "FAN2", RPM: rpm(15000), Status: &warn},
{Name: "FAN2", RPM: rpm(15000), Status: &warn}, // dup name, first wins
},
},
},
}
data, err := json.Marshal(ingest)
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, data, 0644); err != nil {
t.Fatal(err)
}
handler := NewHandler(HandlerOptions{AuditPath: path})
rec := httptest.NewRecorder()
handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/topo", nil))
if rec.Code != http.StatusOK {
t.Fatalf("status=%d", rec.Code)
}
body := rec.Body.String()
// One clickable square per fan (2 after dedup by name), each with a
// spinning glyph, under a COOLING heading.
if !strings.Contains(body, "Cooling") {
t.Fatalf("topo page missing Cooling heading: %s", body)
}
if n := strings.Count(body, `onclick="openComponentDetail('fan')"`) +
strings.Count(body, `onclick="openComponentDetail(&#39;fan&#39;)"`); n != 2 {
t.Fatalf("expected one clickable square per fan (2), got %d: %s", n, body)
}
if n := strings.Count(body, `class="topo-fan-spin"`); n != 2 {
t.Fatalf("expected 2 spinning fan glyphs, got %d: %s", n, body)
}
if !strings.Contains(body, "FAN1 · 4200 RPM") || !strings.Contains(body, "FAN2 · 15000 RPM") {
t.Fatalf("topo page missing per-fan RPM tooltips: %s", body)
}
if !strings.Contains(body, "2 fans") {
t.Fatalf("topo page missing fan count caption: %s", body)
}
// Component-detail fallback endpoint must resolve the "fan" type.
rec2 := httptest.NewRecorder()
handler.ServeHTTP(rec2, httptest.NewRequest(http.MethodGet, "/api/components/fan", nil))
if rec2.Code != http.StatusOK {
t.Fatalf("/api/components/fan status=%d", rec2.Code)
}
if b := rec2.Body.String(); !strings.Contains(b, "FAN1") || !strings.Contains(b, "FAN2") {
t.Fatalf("fan component detail missing fan names: %s", b)
}
}
func TestTopoPageRendersStorageDisksGroupedByType(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "audit.json")
+6
View File
@@ -114,6 +114,12 @@ func renderValidateMode(opts HandlerOptions, stressDefault bool) string {
`Tests power supply transient response by pulsing all GPUs simultaneously between idle and full load. Synchronous pulses across all GPUs create worst-case PSU load spikes — running per-GPU would miss PSU-level failures.`,
`<code>dcgmi diag pulse_test</code>`,
validateFmtDur(platform.SATEstimatedNvidiaPulseTestSec)+` (all GPUs simultaneously; measured on 8-GPU system).`,
)) +
renderSATCard("fan", "Fan Ceiling Check", "runSAT('fan')", "", renderValidateCardBody(
"All system fans reported over IPMI / lm-sensors.",
`Drives CPU (+memory) and, when a GPU is present, GPU load to 100% at the same time and watches every fan until none has climbed for ~1 min. The peak RPM reached is recorded as each fan's ceiling and is what the Topology view sizes the fan tiles against. Success once every fan plateaus (or the time cap is hit). A fan reading 0 RPM or an IPMI status of cr/nr under full load fails. If the host cannot be loaded at all, or exposes no fan sensors, the task is cancelled as "not applicable" rather than failed — the platform does not support forcing fans, so this is the closest safe equivalent.`,
`<code>stressapptest</code> / <code>stress-ng</code> + <code>bee-gpu-burn</code> / <code>rvs gst</code>; <code>ipmitool sdr type Fan</code>`,
`~28 min depending on how fast the fan curve settles (hard cap 15 min).`,
))
}
+1
View File
@@ -272,6 +272,7 @@ func NewHandler(opts HandlerOptions) http.Handler {
mux.HandleFunc("POST /api/sat/memory-stress/run", h.handleAPISATRun("memory-stress"))
mux.HandleFunc("POST /api/sat/sat-stress/run", h.handleAPISATRun("sat-stress"))
mux.HandleFunc("POST /api/sat/platform-stress/run", h.handleAPISATRun("platform-stress"))
mux.HandleFunc("POST /api/sat/fan/run", h.handleAPISATRun("fan"))
mux.HandleFunc("POST /api/sat/run-all", h.handleAPISATRunAll)
mux.HandleFunc("GET /api/sat/stream", h.handleAPISATStream)
mux.HandleFunc("POST /api/sat/abort", h.handleAPISATAbort)
+22 -4
View File
@@ -3,6 +3,7 @@ package webui
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
@@ -79,9 +80,11 @@ func finalizeTaskForResult(t *Task, errMsg string, cancelled bool) {
now := time.Now()
t.DoneAt = &now
switch {
case cancelled:
case cancelled || t.Status == TaskCancelled:
t.Status = TaskCancelled
t.ErrMsg = "aborted"
if strings.TrimSpace(t.ErrMsg) == "" {
t.ErrMsg = "aborted"
}
case strings.TrimSpace(errMsg) != "":
t.Status = TaskFailed
t.ErrMsg = errMsg
@@ -377,6 +380,12 @@ func executeTaskWithOptions(opts *HandlerOptions, t *Task, j *jobState, ctx cont
runOpts := resolvePlatformStressPreset(t.params.BurnProfile)
runOpts.Components = t.params.PlatformComponents
archive, err = a.RunPlatformStress(ctx, "", runOpts, j.append)
case "fan":
if a == nil {
err = fmt.Errorf("app not configured")
break
}
archive, err = runFanCheckPackCtx(a, ctx, "", platform.FanCheckOptions{GPUIndices: t.params.GPUIndices}, j.append)
case "audit":
if a == nil {
err = fmt.Errorf("app not configured")
@@ -476,10 +485,19 @@ func executeTaskWithOptions(opts *HandlerOptions, t *Task, j *jobState, ctx cont
}
if err != nil {
if ctx.Err() != nil {
switch {
case ctx.Err() != nil:
j.append("Aborted.")
j.finish("aborted")
} else {
case errors.Is(err, platform.ErrTestNotApplicable):
// The host offered no way to run this test — not a hardware
// fault. Land the task as cancelled ("not applicable") with a
// detailed log, so an engineer never sees a false failure.
j.append("NOT APPLICABLE: " + err.Error())
t.Status = TaskCancelled
t.ErrMsg = "not applicable — " + err.Error()
j.finish("")
default:
j.append("ERROR: " + err.Error())
j.finish(err.Error())
}
+5 -1
View File
@@ -53,6 +53,7 @@ var taskNames = map[string]string{
"memory-stress": "Memory Burn-in",
"sat-stress": "SAT Stress (stressapptest)",
"platform-stress": "Platform Thermal Cycling",
"fan": "Fan Ceiling Check (CPU+GPU load)",
"audit": "Audit",
"support-bundle": "Support Bundle",
"install": "Install to Disk",
@@ -196,7 +197,7 @@ func taskMayLeaveOrphanWorkers(target string) bool {
switch strings.TrimSpace(strings.ToLower(target)) {
case "nvidia", "nvidia-targeted-stress", "nvidia-targeted-power", "nvidia-pulse",
"nvidia-bandwidth", "nvidia-stress", "nvidia-compute", "nvidia-bench-perf",
"memory", "memory-stress", "cpu", "sat-stress", "platform-stress":
"memory", "memory-stress", "cpu", "sat-stress", "platform-stress", "fan":
return true
default:
return false
@@ -343,6 +344,9 @@ var (
runSATStressPackCtx = func(a *app.App, ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
return a.RunSATStressPackCtx(ctx, baseDir, durationSec, logFunc)
}
runFanCheckPackCtx = func(a *app.App, ctx context.Context, baseDir string, opts platform.FanCheckOptions, logFunc func(string)) (string, error) {
return a.RunFanCheckCtx(ctx, baseDir, opts, logFunc)
}
buildSupportBundle = app.BuildSupportBundle
installCommand = func(ctx context.Context, device string, logPath string) *exec.Cmd {
return exec.CommandContext(ctx, "bee-install", device, logPath)
+29
View File
@@ -3,6 +3,7 @@ package webui
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
@@ -700,6 +701,34 @@ func TestRunTaskHonorsCancel(t *testing.T) {
}
}
func TestRunTaskFanNotApplicableIsCancelledNotFailed(t *testing.T) {
q := &taskQueue{opts: &HandlerOptions{App: &app.App{}}}
tk := &Task{
ID: "fan-1", Name: "Fan Ceiling Check", Target: "fan",
Status: TaskRunning, CreatedAt: time.Now(),
}
j := &jobState{}
tk.job = j
orig := runFanCheckPackCtx
runFanCheckPackCtx = func(_ *app.App, _ context.Context, _ string, _ platform.FanCheckOptions, _ func(string)) (string, error) {
return "", fmt.Errorf("no fan sensors readable: %w", platform.ErrTestNotApplicable)
}
defer func() { runFanCheckPackCtx = orig }()
q.runTask(tk, j, context.Background())
if tk.Status != TaskCancelled {
t.Fatalf("status=%q want %q", tk.Status, TaskCancelled)
}
if j.err != "" {
t.Fatalf("job err should be empty for a not-applicable task, got %q", j.err)
}
if !strings.Contains(tk.ErrMsg, "not applicable") {
t.Fatalf("ErrMsg should explain not-applicable, got %q", tk.ErrMsg)
}
}
func TestRunTaskUsesBurnProfileDurationForCPU(t *testing.T) {
var gotDuration int
q := &taskQueue{