diff --git a/audit/internal/app/app.go b/audit/internal/app/app.go index 2d4d7ab..90ac626 100644 --- a/audit/internal/app/app.go +++ b/audit/internal/app/app.go @@ -173,7 +173,7 @@ type satRunner interface { RunAMDStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) RunMemoryStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) RunSATStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) - RunFanStressTest(ctx context.Context, baseDir string, opts platform.FanStressOptions) (string, error) + RunFanCheck(ctx context.Context, baseDir string, opts platform.FanCheckOptions, logFunc func(string)) (string, error) RunPlatformStress(ctx context.Context, baseDir string, opts platform.PlatformStressOptions, logFunc func(string)) (string, error) RunNCCLTests(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) RunScenario(ctx context.Context, baseDir string, spec platform.ScenarioSpec, logFunc func(string)) (string, error) diff --git a/audit/internal/app/app_packs.go b/audit/internal/app/app_packs.go index 4f88355..b57ab26 100644 --- a/audit/internal/app/app_packs.go +++ b/audit/internal/app/app_packs.go @@ -298,6 +298,13 @@ func (a *App) RunPlatformStress(ctx context.Context, baseDir string, opts platfo return a.sat.RunPlatformStress(ctx, baseDir, opts, logFunc) } +func (a *App) RunFanCheckCtx(ctx context.Context, baseDir string, opts platform.FanCheckOptions, logFunc func(string)) (string, error) { + if strings.TrimSpace(baseDir) == "" { + baseDir = DefaultSATBaseDir + } + return a.sat.RunFanCheck(ctx, baseDir, opts, logFunc) +} + func (a *App) RunNCCLTests(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) { if strings.TrimSpace(baseDir) == "" { baseDir = DefaultSATBaseDir diff --git a/audit/internal/app/app_test.go b/audit/internal/app/app_test.go index a1df766..b7b6cd0 100644 --- a/audit/internal/app/app_test.go +++ b/audit/internal/app/app_test.go @@ -367,7 +367,7 @@ func (f fakeSAT) RunSATStressPack(_ context.Context, _ string, _ int, _ func(str return "", nil } -func (f fakeSAT) RunFanStressTest(_ context.Context, _ string, _ platform.FanStressOptions) (string, error) { +func (f fakeSAT) RunFanCheck(_ context.Context, _ string, _ platform.FanCheckOptions, _ func(string)) (string, error) { return "", nil } diff --git a/audit/internal/app/component_status_db.go b/audit/internal/app/component_status_db.go index 45654de..ae2c994 100644 --- a/audit/internal/app/component_status_db.go +++ b/audit/internal/app/component_status_db.go @@ -318,6 +318,23 @@ func ApplySATResultToDB(db *ComponentStatusDB, target, archivePath string) { db.Record("memory:all", source, dbStatus, detail) case "cpu", "platform-stress": db.Record("cpu:all", source, dbStatus, detail) + case "fan": + // Per-fan keys: summary emits "fan__status=OK|FAILED (...)". + for key, val := range kv { + name, ok := strings.CutPrefix(key, "fan_") + if !ok { + continue + } + name, ok = strings.CutSuffix(name, "_status") + if !ok || name == "" { + continue + } + upper := strings.ToUpper(strings.TrimSpace(val)) + if i := strings.IndexByte(upper, ' '); i > 0 { + upper = upper[:i] // drop the "(reason)" suffix + } + db.Record("fan:"+name, source, satStatusToDBStatus(upper), target+" SAT: "+strings.TrimSpace(val)) + } case "storage": // Try to record per-device if available in summary. recordedAny := false diff --git a/audit/internal/platform/sat_errors.go b/audit/internal/platform/sat_errors.go new file mode 100644 index 0000000..bf7f465 --- /dev/null +++ b/audit/internal/platform/sat_errors.go @@ -0,0 +1,10 @@ +package platform + +import "errors" + +// ErrTestNotApplicable is returned by a SAT routine when the host provides no +// way to run that test at all (a required tool or capability is absent), as +// opposed to the test running and finding a fault. The task layer maps it to a +// cancelled ("not applicable") task rather than a failure, so an engineer does +// not see a false red. +var ErrTestNotApplicable = errors.New("test not applicable on this platform") diff --git a/audit/internal/platform/sat_fan_stress.go b/audit/internal/platform/sat_fan_stress.go index 5d22f8f..4648b25 100644 --- a/audit/internal/platform/sat_fan_stress.go +++ b/audit/internal/platform/sat_fan_stress.go @@ -15,14 +15,18 @@ import ( "time" ) -// FanStressOptions configures the fan-stress / thermal cycling test. -type FanStressOptions struct { - BaselineSec int // idle monitoring before and after load (default 30) - Phase1DurSec int // first load phase duration in seconds (default 300) - PauseSec int // pause between the two load phases (default 60) - Phase2DurSec int // second load phase duration in seconds (default 300) - SizeMB int // GPU memory to allocate per GPU during stress (0 = auto: 95% of VRAM) - GPUIndices []int // which GPU indices to stress (empty = all detected) +// FanCheckOptions configures the fan-ceiling check: it drives CPU (+memory) +// and, when present, GPU load to 100% simultaneously, then watches every fan +// until none has climbed for PlateauHoldSec — at which point each fan is +// considered to be at its physical ceiling and the observed peak is recorded. +type FanCheckOptions struct { + PlateauHoldSec int // a fan must not rise > PlateauDeltaRPM for this long to count as plateaued (default 60) + PlateauDeltaRPM int // RPM increase that still counts as "climbing" (default 50) + MinLoadSec int // never declare a plateau before this many seconds of load (default 90) + MaxLoadSec int // hard cap on the load phase; finish (success) even if not every fan plateaued (default 900) + RampConfirmRPM int // at least one fan must exceed baseline by this before a plateau is "real" (default 150) + SizeMB int // GPU memory to allocate per GPU (0 = auto) + GPUIndices []int // which GPU indices to load (empty = all detected) } // FanReading holds one fan sensor reading. @@ -93,128 +97,217 @@ func normalizeObservedFanMaxRPM(rpm float64) float64 { return math.Ceil(rpm/1000.0) * 1000.0 } -// RunFanStressTest runs a two-phase GPU stress test while monitoring fan speeds, -// temperatures, and power draw every second. Exports metrics.csv and fan-sensors.csv. -// Designed to reproduce case-04 fan-speed lag and detect GPU thermal throttling. -func (s *System) RunFanStressTest(ctx context.Context, baseDir string, opts FanStressOptions) (string, error) { +// RunFanCheck drives CPU (+memory) and, when a GPU is present, GPU load to +// 100% simultaneously and watches every fan until none has climbed for +// PlateauHoldSec. At that point each fan is taken to be at its physical +// ceiling; the observed peak RPM is persisted (fanObservationStatePath, the +// same store ObservedFanMaxRPM reads) so the topology view can size each fan +// tile against a real maximum. +// +// Outcome: +// - success ("ceiling found") once every fan plateaus, or when MaxLoadSec is +// hit — a run that simply ran out of time still recorded the highest RPM +// seen and is not a failure. +// - a fan reading 0 RPM, or an IPMI status of cr/nr, while under full load is +// a real defect → FAILED. +// - if there is no way to load this box (no stressapptest/stress-ng and no +// GPU burn tool) or no fan sensors are readable, the test cannot say +// anything about the hardware and returns ErrTestNotApplicable so the task +// is cancelled, not failed. +// +// No GPU is not an error: CPU/memory load alone is enough to exercise the +// cooling loop on most platforms. +func (s *System) RunFanCheck(ctx context.Context, baseDir string, opts FanCheckOptions, logFunc func(string)) (string, error) { + if logFunc == nil { + logFunc = func(string) {} + } if baseDir == "" { baseDir = "/var/log/bee-sat" } - applyFanStressDefaults(&opts) + applyFanCheckDefaults(&opts) + + baseFans, fanErr := sampleFanSpeeds() + if len(baseFans) == 0 { + return "", fmt.Errorf("no fan sensors readable via ipmitool or lm-sensors (%v): %w", fanErr, ErrTestNotApplicable) + } + baselineRPM := make(map[string]float64, len(baseFans)) + for _, f := range baseFans { + baselineRPM[f.Name] = f.RPM + } + + vendor := s.DetectGPUVendor() + haveGPU := vendor == "nvidia" || vendor == "amd" + _, cpuPathErr := satLookPath("stressapptest") + if cpuPathErr != nil { + _, cpuPathErr = satLookPath("stress-ng") + } + haveCPU := cpuPathErr == nil + if !haveCPU && !haveGPU { + return "", fmt.Errorf("no load source: stressapptest/stress-ng missing and no NVIDIA/AMD GPU stress tool available: %w", ErrTestNotApplicable) + } ts := time.Now().UTC().Format("20060102-150405") - runDir := filepath.Join(baseDir, "fan-stress-"+ts) + runDir := filepath.Join(baseDir, "fan-check-"+ts) if err := os.MkdirAll(runDir, 0755); err != nil { return "", err } verboseLog := filepath.Join(runDir, "verbose.log") + appendSATVerboseLog(verboseLog, fmt.Sprintf("[%s] fan check start: %d fans, gpu=%s cpu=%v", + time.Now().UTC().Format(time.RFC3339), len(baseFans), vendor, haveCPU)) + logFunc(fmt.Sprintf("Fan check: %d fans; load = CPU/mem:%v + GPU:%s", len(baseFans), haveCPU, orNone(haveGPU, vendor))) - // Phase name shared between sampler goroutine and main goroutine. - var phaseMu sync.Mutex - currentPhase := "init" - setPhase := func(name string) { - phaseMu.Lock() - currentPhase = name - phaseMu.Unlock() + // ── Load: every source runs at the same time, each in its own goroutine. + // Sources can take different amounts of time to actually reach full load + // (stressapptest is near-instant; a GPU burn kernel needs to compile and + // ramp), so the plateau clock does not start until every launched source + // has reported its process running, plus a fixed GPU ramp grace. + loadCtx, loadCancel := context.WithTimeout(ctx, time.Duration(opts.MaxLoadSec)*time.Second) + defer loadCancel() + + var loadWG sync.WaitGroup + started := make(chan bool, 2) // true = source is running, false = failed to launch + launched := 0 + + if haveCPU { + launched++ + loadWG.Add(1) + go func() { + defer loadWG.Done() + cmd, err := buildCPUStressCmd(loadCtx) + if err != nil { + logFunc("CPU/memory load failed to start: " + err.Error()) + appendSATVerboseLog(verboseLog, "cpu load start error: "+err.Error()) + started <- false + return + } + logFunc("CPU/memory load running (stressapptest)") + started <- true + _ = cmd.Wait() + }() } - getPhase := func() string { - phaseMu.Lock() - defer phaseMu.Unlock() - return currentPhase + if haveGPU { + launched++ + loadWG.Add(1) + go func() { + defer loadWG.Done() + cmd := buildGPUStressCmd(loadCtx, vendor, opts.MaxLoadSec) + if cmd == nil { + logFunc("GPU load unavailable (no burn tool for " + vendor + ")") + appendSATVerboseLog(verboseLog, "gpu load: no burn tool") + started <- false + return + } + logFunc("GPU load running (" + vendor + ")") + started <- true + _ = cmd.Wait() + }() } start := time.Now() - var rowsMu sync.Mutex - var allRows []FanStressRow - - // Start background sampler (every second). - stopCh := make(chan struct{}) - doneCh := make(chan struct{}) - go func() { - defer close(doneCh) - ticker := time.NewTicker(time.Second) - defer ticker.Stop() - for { - select { - case <-stopCh: - return - case <-ticker.C: - row := sampleFanStressRow(opts.GPUIndices, getPhase(), time.Since(start).Seconds()) - rowsMu.Lock() - allRows = append(allRows, row) - rowsMu.Unlock() + activeLoads := 0 + for i := 0; i < launched; i++ { + select { + case ok := <-started: + if ok { + activeLoads++ } + case <-ctx.Done(): } - }() + } + if activeLoads == 0 { + loadCancel() + loadWG.Wait() + return "", fmt.Errorf("every load source failed to start: %w", ErrTestNotApplicable) + } + readyAt := time.Now() + if haveGPU { + readyAt = readyAt.Add(20 * time.Second) // GPU kernel ramp grace + } + appendSATVerboseLog(verboseLog, fmt.Sprintf("%d load source(s) active; plateau clock effective from +%.0fs", + activeLoads, readyAt.Sub(start).Seconds())) - var summary strings.Builder - fmt.Fprintf(&summary, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339)) + // ── Sample loop: one row/second, per-fan plateau tracking. + type fanState struct { + peak float64 + lastRiseSec float64 + } + fanBy := map[string]*fanState{} + var rows []FanStressRow + rampConfirmed := false + plateauReached := false + aborted := false - stats := satStats{} - - // idlePhase sleeps for durSec while the sampler stamps phaseName on each row. - idlePhase := func(phaseName, stepName string, durSec int) { - if ctx.Err() != nil { - return - } - setPhase(phaseName) - appendSATVerboseLog(verboseLog, - fmt.Sprintf("[%s] start %s (idle %ds)", time.Now().UTC().Format(time.RFC3339), stepName, durSec), - ) + ticker := time.NewTicker(time.Second) + defer ticker.Stop() +loop: + for { select { case <-ctx.Done(): - case <-time.After(time.Duration(durSec) * time.Second): + aborted = true + break loop + case <-loadCtx.Done(): + break loop // MaxLoadSec reached + case <-ticker.C: } - appendSATVerboseLog(verboseLog, - fmt.Sprintf("[%s] finish %s", time.Now().UTC().Format(time.RFC3339), stepName), - ) - fmt.Fprintf(&summary, "%s_status=OK\n", stepName) - stats.OK++ - } + elapsed := time.Since(start).Seconds() + row := sampleFanStressRow(opts.GPUIndices, "load", elapsed) + rows = append(rows, row) - // loadPhase runs bee-gpu-burn for durSec; sampler stamps phaseName on each row. - loadPhase := func(phaseName, stepName string, durSec int) { - if ctx.Err() != nil { - return + for _, f := range row.Fans { + st := fanBy[f.Name] + if st == nil { + fanBy[f.Name] = &fanState{peak: f.RPM, lastRiseSec: elapsed} + continue + } + if f.RPM > st.peak { + if f.RPM-st.peak > float64(opts.PlateauDeltaRPM) { + st.lastRiseSec = elapsed + } + st.peak = f.RPM + } + if f.RPM >= baselineRPM[f.Name]+float64(opts.RampConfirmRPM) { + rampConfirmed = true + } } - setPhase(phaseName) - cmd := []string{ - "bee-gpu-burn", - "--seconds", strconv.Itoa(durSec), - "--size-mb", strconv.Itoa(opts.SizeMB), - } - if len(opts.GPUIndices) > 0 { - cmd = append(cmd, "--devices", joinIndexList(dedupeSortedIndices(opts.GPUIndices))) - } - out, err := runSATCommandCtx(ctx, verboseLog, stepName, cmd, nil, nil) - _ = os.WriteFile(filepath.Join(runDir, stepName+".log"), out, 0644) - if err != nil && err != context.Canceled && err.Error() != "signal: killed" { - fmt.Fprintf(&summary, "%s_status=FAILED\n", stepName) - stats.Failed++ - } else { - fmt.Fprintf(&summary, "%s_status=OK\n", stepName) - stats.OK++ + + if elapsed >= float64(opts.MinLoadSec) && time.Since(readyAt) >= time.Duration(opts.PlateauHoldSec)*time.Second && len(fanBy) > 0 { + allFlat := true + for _, st := range fanBy { + if elapsed-st.lastRiseSec < float64(opts.PlateauHoldSec) { + allFlat = false + break + } + } + if allFlat && rampConfirmed { + plateauReached = true + logFunc(fmt.Sprintf("All %d fans plateaued at %.0fs of load", len(fanBy), elapsed)) + break loop + } } } - // Execute test phases. - idlePhase("baseline", "01-baseline", opts.BaselineSec) - loadPhase("load1", "02-load1", opts.Phase1DurSec) - idlePhase("pause", "03-pause", opts.PauseSec) - loadPhase("load2", "04-load2", opts.Phase2DurSec) - idlePhase("cooldown", "05-cooldown", opts.BaselineSec) + loadCancel() + loadWG.Wait() - // Stop sampler and collect rows. - close(stopCh) - <-doneCh + if aborted && ctx.Err() != nil { + _ = os.WriteFile(filepath.Join(runDir, "summary.txt"), + []byte("run_at_utc="+time.Now().UTC().Format(time.RFC3339)+"\noverall_status=UNKNOWN\naborted=true\n"), 0644) + return runDir, ctx.Err() + } - rowsMu.Lock() - rows := allRows - rowsMu.Unlock() - - // Analysis. - throttled := analyzeThrottling(rows) - maxGPUTemp := analyzeMaxTemp(rows, func(r FanStressRow) float64 { + // ── Verdict. + statuses := readFanStatuses() + var summary strings.Builder + fmt.Fprintf(&summary, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339)) + fmt.Fprintf(&summary, "fans_total=%d\n", len(baseFans)) + fmt.Fprintf(&summary, "active_load_sources=%d\n", activeLoads) + fmt.Fprintf(&summary, "gpu_vendor=%s\n", orNone(haveGPU, vendor)) + fmt.Fprintf(&summary, "plateau_reached=%v\n", plateauReached) + fmt.Fprintf(&summary, "ramp_confirmed=%v\n", rampConfirmed) + if len(rows) > 0 { + fmt.Fprintf(&summary, "load_duration_sec=%.0f\n", rows[len(rows)-1].ElapsedSec) + } + fmt.Fprintf(&summary, "max_gpu_temp_c=%.1f\n", analyzeMaxTemp(rows, func(r FanStressRow) float64 { var m float64 for _, g := range r.GPUs { if g.TempC > m { @@ -222,55 +315,148 @@ func (s *System) RunFanStressTest(ctx context.Context, baseDir string, opts FanS } } return m - }) - maxCPUTemp := analyzeMaxTemp(rows, func(r FanStressRow) float64 { - return r.CPUMaxTempC - }) - fanResponseSec := analyzeFanResponse(rows) + })) + fmt.Fprintf(&summary, "max_cpu_temp_c=%.1f\n", analyzeMaxTemp(rows, func(r FanStressRow) float64 { return r.CPUMaxTempC })) - fmt.Fprintf(&summary, "throttling_detected=%v\n", throttled) - fmt.Fprintf(&summary, "max_gpu_temp_c=%.1f\n", maxGPUTemp) - fmt.Fprintf(&summary, "max_cpu_temp_c=%.1f\n", maxCPUTemp) - if fanResponseSec >= 0 { - fmt.Fprintf(&summary, "fan_response_sec=%.1f\n", fanResponseSec) - } else { - fmt.Fprintf(&summary, "fan_response_sec=N/A\n") + stats := satStats{} + names := make([]string, 0, len(baselineRPM)) + for n := range baselineRPM { + names = append(names, n) } - - // Throttling failure counts against overall result. - if throttled { - stats.Failed++ + sort.Strings(names) + for _, name := range names { + peak := baselineRPM[name] + if st := fanBy[name]; st != nil { + peak = st.peak + } + st := strings.ToLower(strings.TrimSpace(statuses[name])) + bad := peak <= 0 || st == "cr" || st == "nr" + key := sanitizeSummaryKey(name) + fmt.Fprintf(&summary, "fan_%s_baseline_rpm=%.0f\n", key, baselineRPM[name]) + fmt.Fprintf(&summary, "fan_%s_max_rpm=%.0f\n", key, peak) + if bad { + reason := "0 RPM under load" + if st == "cr" || st == "nr" { + reason = "IPMI status " + st + } + fmt.Fprintf(&summary, "fan_%s_status=FAILED (%s)\n", key, reason) + logFunc(fmt.Sprintf("FAIL %s: %s", name, reason)) + stats.Failed++ + } else { + fmt.Fprintf(&summary, "fan_%s_status=OK\n", key) + stats.OK++ + } } writeSATStats(&summary, stats) - // Write CSV outputs. - if err := WriteFanStressCSV(filepath.Join(runDir, "metrics.csv"), rows, opts.GPUIndices); err != nil { - return "", err - } + _ = WriteFanStressCSV(filepath.Join(runDir, "metrics.csv"), rows, opts.GPUIndices) _ = WriteFanSensorsCSV(filepath.Join(runDir, "fan-sensors.csv"), rows) - if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary.String()), 0644); err != nil { return "", err } - return runDir, nil } -func applyFanStressDefaults(opts *FanStressOptions) { - if opts.BaselineSec <= 0 { - opts.BaselineSec = 30 +func applyFanCheckDefaults(o *FanCheckOptions) { + if o.PlateauHoldSec <= 0 { + o.PlateauHoldSec = 60 } - if opts.Phase1DurSec <= 0 { - opts.Phase1DurSec = 300 + if o.PlateauDeltaRPM <= 0 { + o.PlateauDeltaRPM = 50 } - if opts.PauseSec <= 0 { - opts.PauseSec = 60 + if o.MinLoadSec <= 0 { + o.MinLoadSec = 90 } - if opts.Phase2DurSec <= 0 { - opts.Phase2DurSec = 300 + if o.MaxLoadSec <= 0 { + o.MaxLoadSec = 900 } - // SizeMB == 0 means "auto" (worker picks 95% of GPU VRAM for maximum power draw). - // Leave at 0 to avoid passing a too-small size that starves the tensor-core path. + if o.RampConfirmRPM <= 0 { + o.RampConfirmRPM = 150 + } + if o.MinLoadSec < o.PlateauHoldSec { + o.MinLoadSec = o.PlateauHoldSec + } + if o.MaxLoadSec <= o.MinLoadSec { + o.MaxLoadSec = o.MinLoadSec + o.PlateauHoldSec + } +} + +func orNone(present bool, v string) string { + if present && v != "" { + return v + } + return "none" +} + +// sanitizeSummaryKey makes a fan sensor name safe as a summary.txt key +// fragment (keys are parsed by splitting on '=' and whitespace). +func sanitizeSummaryKey(name string) string { + var b strings.Builder + for _, r := range name { + switch { + case r >= 'A' && r <= 'Z', r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '-', r == '_', r == '.': + b.WriteRune(r) + default: + b.WriteByte('_') + } + } + return b.String() +} + +// readFanStatuses returns the per-fan IPMI status word ("ok", "cr", "nr", ...) +// from "ipmitool sdr type Fan". Empty map when ipmitool is unavailable. +func readFanStatuses() map[string]string { + out, err := exec.Command("ipmitool", "sdr", "type", "Fan").Output() + if err != nil { + return nil + } + m := map[string]string{} + for _, line := range strings.Split(string(out), "\n") { + parts := strings.Split(line, "|") + if len(parts) < 3 { + continue + } + name := strings.TrimSpace(parts[0]) + if name == "" { + continue + } + m[name] = strings.ToLower(strings.TrimSpace(parts[2])) + } + return m +} + +// ResolveFanMaxRPM returns, for every fan name in current (name -> current +// RPM), the RPM to treat as that fan's 100% reference. Preference order: +// 1. the persisted observed peak (fanObservationStatePath), written by +// RunFanCheck and by live-metrics sampling under load; +// 2. the largest peak observed on any peer fan (keeps a group visually +// consistent when only some fans have a recorded peak); +// 3. the fan's own current RPM (so a tile is never sized against zero). +// +// The fallback lives here, not in the view, so every consumer of a fan +// maximum applies the same rule. +func ResolveFanMaxRPM(current map[string]float64) map[string]float64 { + persisted := readPersistedFanMaxRPM() + + peerMax := 0.0 + for _, v := range persisted { + if v > peerMax { + peerMax = v + } + } + + out := make(map[string]float64, len(current)) + for name, rpm := range current { + switch { + case persisted[name] > 0: + out[name] = persisted[name] + case peerMax > 0: + out[name] = peerMax + default: + out[name] = rpm + } + } + return out } // sampleFanStressRow collects all metrics for one telemetry sample. @@ -355,27 +541,48 @@ func sampleFanSpeeds() ([]FanReading, error) { return nil, sensorsErr } -func loadFanObservationLocked() { - if fanObservationInit { - return - } - fanObservationInit = true - fanObservation.MaxRPM = make(map[string]float64) +// readPersistedFanMaxRPM reads fanObservationStatePath and returns its +// sanitized {fan name -> observed peak RPM} map (empty names / non-positive +// values dropped). Returns an empty map when the file is missing or unparsable. +func readPersistedFanMaxRPM() map[string]float64 { + out := map[string]float64{} raw, err := os.ReadFile(fanObservationStatePath) if err != nil || len(raw) == 0 { - return + return out } var persisted fanObservationState if json.Unmarshal(raw, &persisted) != nil { - return + return out } for name, rpm := range persisted.MaxRPM { name = strings.TrimSpace(name) if name == "" || rpm <= 0 { continue } - fanObservation.MaxRPM[name] = rpm + out[name] = rpm } + return out +} + +// ObservedFanMaxRPM returns the per-fan observed peak RPM map persisted by +// fan-stress SAT runs, or nil if none is recorded yet. It reads the file +// directly without touching the in-process observation cache or its lock, so +// read-only consumers (the /topo web view) can call it without perturbing a +// concurrent SAT run's peak tracking. +func ObservedFanMaxRPM() map[string]float64 { + out := readPersistedFanMaxRPM() + if len(out) == 0 { + return nil + } + return out +} + +func loadFanObservationLocked() { + if fanObservationInit { + return + } + fanObservationInit = true + fanObservation.MaxRPM = readPersistedFanMaxRPM() } func saveFanObservationLocked() { @@ -779,22 +986,6 @@ func effectiveSystemPowerReading(cache cachedPowerReading, current float64, sour return 0, cache } -// analyzeThrottling returns true if any GPU reported an active throttle reason -// during either load phase. -func analyzeThrottling(rows []FanStressRow) bool { - for _, row := range rows { - if row.Phase != "load1" && row.Phase != "load2" { - continue - } - for _, gpu := range row.GPUs { - if gpu.Throttled { - return true - } - } - } - return false -} - // analyzeMaxTemp returns the maximum value of the given extractor across all rows. func analyzeMaxTemp(rows []FanStressRow, extract func(FanStressRow) float64) float64 { var max float64 @@ -806,55 +997,6 @@ func analyzeMaxTemp(rows []FanStressRow, extract func(FanStressRow) float64) flo return max } -// analyzeFanResponse returns the seconds from load1 start until fan RPM first -// increased by more than 5% above the baseline average. Returns -1 if undetermined. -func analyzeFanResponse(rows []FanStressRow) float64 { - // Compute baseline average fan RPM. - var baseTotal, baseCount float64 - for _, row := range rows { - if row.Phase != "baseline" { - continue - } - for _, f := range row.Fans { - baseTotal += f.RPM - baseCount++ - } - } - if baseCount == 0 || baseTotal == 0 { - return -1 - } - baseAvg := baseTotal / baseCount - threshold := baseAvg * 1.05 // 5% increase signals fan ramp-up - - // Find elapsed time when load1 started. - var load1Start float64 = -1 - for _, row := range rows { - if row.Phase == "load1" { - load1Start = row.ElapsedSec - break - } - } - if load1Start < 0 { - return -1 - } - - // Find first load1 row where average RPM crosses the threshold. - for _, row := range rows { - if row.Phase != "load1" { - continue - } - var total, count float64 - for _, f := range row.Fans { - total += f.RPM - count++ - } - if count > 0 && total/count >= threshold { - return row.ElapsedSec - load1Start - } - } - return -1 -} - // WriteFanStressCSV writes the wide-format metrics CSV with one row per second. // GPU columns are generated per index in gpuIndices order. func WriteFanStressCSV(path string, rows []FanStressRow, gpuIndices []int) error { diff --git a/audit/internal/platform/sat_fan_stress_test.go b/audit/internal/platform/sat_fan_stress_test.go index 20ac394..3d79787 100644 --- a/audit/internal/platform/sat_fan_stress_test.go +++ b/audit/internal/platform/sat_fan_stress_test.go @@ -1,11 +1,68 @@ package platform import ( + "os" "path/filepath" + "reflect" "testing" "time" ) +func TestResolveFanMaxRPM(t *testing.T) { + oldPath := fanObservationStatePath + oldInit := fanObservationInit + fanObservationStatePath = filepath.Join(t.TempDir(), "fan-observation.json") + fanObservationInit = false + t.Cleanup(func() { + fanObservationStatePath = oldPath + fanObservationInit = oldInit + }) + + // No persisted file yet: unknown fans fall back to their own current RPM. + got := ResolveFanMaxRPM(map[string]float64{"A": 4000, "B": 9000}) + if !reflect.DeepEqual(got, map[string]float64{"A": 4000, "B": 9000}) { + t.Fatalf("no-persist fallback: got %v", got) + } + + if err := os.WriteFile(fanObservationStatePath, []byte(`{"max_rpm":{"A":17000}}`), 0644); err != nil { + t.Fatal(err) + } + got = ResolveFanMaxRPM(map[string]float64{"A": 4000, "B": 9000, "C": 5000}) + // A: persisted peak. B/C: no own entry -> largest peer peak (A's 17000). + if want := map[string]float64{"A": 17000, "B": 17000, "C": 17000}; !reflect.DeepEqual(got, want) { + t.Fatalf("peer fallback: got %v want %v", got, want) + } +} + +func TestApplyFanCheckDefaults(t *testing.T) { + var o FanCheckOptions + applyFanCheckDefaults(&o) + if o.PlateauHoldSec != 60 || o.PlateauDeltaRPM != 50 || o.MinLoadSec != 90 || o.MaxLoadSec != 900 || o.RampConfirmRPM != 150 { + t.Fatalf("unexpected defaults: %+v", o) + } + o = FanCheckOptions{PlateauHoldSec: 120, MinLoadSec: 30, MaxLoadSec: 40} + applyFanCheckDefaults(&o) + if o.MinLoadSec < o.PlateauHoldSec { + t.Fatalf("MinLoadSec must be >= PlateauHoldSec, got %d", o.MinLoadSec) + } + if o.MaxLoadSec <= o.MinLoadSec { + t.Fatalf("MaxLoadSec must exceed MinLoadSec, got %d", o.MaxLoadSec) + } +} + +func TestSanitizeSummaryKey(t *testing.T) { + for in, want := range map[string]string{ + "F2U-1": "F2U-1", + "aspeed / fan1": "aspeed___fan1", + "CPU0_DIMM": "CPU0_DIMM", + "weird=key here": "weird_key_here", + } { + if got := sanitizeSummaryKey(in); got != want { + t.Errorf("sanitizeSummaryKey(%q)=%q want %q", in, got, want) + } + } +} + func TestParseFanSpeeds(t *testing.T) { raw := "FAN1 | 2400.000 | RPM | ok\nFAN2 | 1800 RPM | ok | ok\nFAN3 | na | RPM | ns\n" got := parseFanSpeeds(raw) diff --git a/audit/internal/webui/api.go b/audit/internal/webui/api.go index f88fed4..89a98df 100644 --- a/audit/internal/webui/api.go +++ b/audit/internal/webui/api.go @@ -128,7 +128,7 @@ func defaultTaskPriority(target string, params taskParams) int { return taskPriorityAudit case "nvidia-bench-perf", "nvidia-bench-power", "nvidia-bench-autotune": return taskPriorityBenchmark - case "nvidia-stress", "amd-stress", "memory-stress", "sat-stress", "platform-stress", "nvidia-compute", "scenario": + case "nvidia-stress", "amd-stress", "memory-stress", "sat-stress", "platform-stress", "fan", "nvidia-compute", "scenario": return taskPriorityBurn case "nvidia", "nvidia-targeted-stress", "nvidia-targeted-power", "nvidia-pulse", "nvidia-interconnect", "nvidia-bandwidth", "memory", "storage", "cpu", diff --git a/audit/internal/webui/api_metrics_network.go b/audit/internal/webui/api_metrics_network.go index 3ae78cc..a4cb43b 100644 --- a/audit/internal/webui/api_metrics_network.go +++ b/audit/internal/webui/api_metrics_network.go @@ -289,7 +289,7 @@ func (h *handler) handleAPIHardwareSummary(w http.ResponseWriter, _ *http.Reques } // handleAPIComponentDetail returns an HTML fragment describing the current and -// historical status for one component type (cpu, memory, storage, gpu, psu). +// historical status for one component type (cpu, memory, storage, gpu, psu, fan). func (h *handler) handleAPIComponentDetail(w http.ResponseWriter, r *http.Request) { compType := r.PathValue("type") var exact, prefixes []string @@ -315,6 +315,9 @@ func (h *handler) handleAPIComponentDetail(w http.ResponseWriter, r *http.Reques case "psu": title = "PSU" prefixes = []string{"psu:"} + case "fan": + title = "Fans" + prefixes = []string{"fan:"} case "raid": title = "RAID" prefixes = []string{"pcie:raid:"} diff --git a/audit/internal/webui/api_sat_runall.go b/audit/internal/webui/api_sat_runall.go index 21ce3c9..4fc03af 100644 --- a/audit/internal/webui/api_sat_runall.go +++ b/audit/internal/webui/api_sat_runall.go @@ -109,6 +109,11 @@ func (h *handler) planSATRunAll(ctx context.Context, req satRunAllRequest) ([]sa } else { skip("TPM: no TPM device on this host; check skipped") } + } else { + // Fan ceiling check runs on the Load tier only. It self-cancels as + // "not applicable" on a host with no fan sensors or no way to load + // the CPU/GPU, so it is safe to queue unconditionally here. + specs = append(specs, satRunAllSpec{target: "fan", params: taskParams{StressMode: true}}) } gp := h.opts.App.DetectGPUPresence() diff --git a/audit/internal/webui/api_sat_runall_test.go b/audit/internal/webui/api_sat_runall_test.go index 8fcdd2b..b0b0400 100644 --- a/audit/internal/webui/api_sat_runall_test.go +++ b/audit/internal/webui/api_sat_runall_test.go @@ -98,7 +98,7 @@ func TestPlanSATRunAllLoadOmitsReadOnlyTPMCheck(t *testing.T) { for _, s := range specs { targets = append(targets, s.target) } - if want := []string{"cpu", "memory", "storage"}; !reflect.DeepEqual(targets, want) { + if want := []string{"cpu", "memory", "storage", "fan"}; !reflect.DeepEqual(targets, want) { t.Fatalf("targets=%v want %v", targets, want) } for _, note := range notes { diff --git a/audit/internal/webui/kmsg_watcher.go b/audit/internal/webui/kmsg_watcher.go index 214b1f2..47165d5 100644 --- a/audit/internal/webui/kmsg_watcher.go +++ b/audit/internal/webui/kmsg_watcher.go @@ -296,7 +296,7 @@ func isSATTarget(target string) bool { case "nvidia", "nvidia-targeted-stress", "nvidia-bench-perf", "nvidia-bench-power", "nvidia-compute", "nvidia-targeted-power", "nvidia-pulse", "nvidia-interconnect", "nvidia-bandwidth", "nvidia-stress", "memory", "memory-stress", "storage", "cpu", "sat-stress", "amd", "amd-mem", "amd-bandwidth", "amd-stress", - "platform-stress": + "platform-stress", "fan": return true } return false diff --git a/audit/internal/webui/page_topo_diagram.go b/audit/internal/webui/page_topo_diagram.go index 62e224d..9a0af7c 100644 --- a/audit/internal/webui/page_topo_diagram.go +++ b/audit/internal/webui/page_topo_diagram.go @@ -614,9 +614,138 @@ func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string }})) } + // Cooling fans — one small square per fan (no PCIe/CPU affinity, arbitrary + // count, so a wrapping flex row like PSUs rather than SVG boxes). Each + // square is sized by rpm / observed-max-rpm and carries a fan glyph that + // spins via CSS — faster when the fan is spinning faster. + if fans := dedupeFansByName(hw.Sensors); len(fans) > 0 { + current := map[string]float64{} + for _, f := range fans { + if f.RPM != nil { + current[strings.TrimSpace(f.Name)] = float64(*f.RPM) + } + } + b.WriteString(renderTopoFanRow(fans, platform.ResolveFanMaxRPM(current))) + } + return topoCard("Topology", b.String()) } +// renderTopoFanRow renders the COOLING row: one clickable square per fan, +// side length scaled by rpm/maxRPM and a fan glyph whose spin rate tracks the +// same ratio. maxByName comes from platform.ResolveFanMaxRPM — it already has +// an entry for every fan (persisted peak, else peer peak, else current RPM), +// so no fallback logic lives here. +func renderTopoFanRow(fans []schema.HardwareFanSensor, maxByName map[string]float64) string { + const ( + fanTileMin = 30 // px, a stalled / slowest fan + fanTileMax = 58 // px, a fan at its ceiling + ) + + var tally topoStatusTally + for _, f := range fans { + tally.add(classifyTopoSeverity(f.Status)) + } + + var b strings.Builder + b.WriteString(topoRowHeading("Cooling")) + b.WriteString(topoFanSpinStyle()) + b.WriteString(`
`) + for _, f := range fans { + name := strings.TrimSpace(f.Name) + fill, stroke, text := topoSeverityColors(classifyTopoSeverity(f.Status)) + + denom := maxByName[name] + ratio := 0.0 + title := name + if f.RPM != nil { + if denom > 0 { + ratio = float64(*f.RPM) / denom + } + if ratio < 0 { + ratio = 0 + } + if ratio > 1 { + ratio = 1 + } + if denom > float64(*f.RPM) { + title = fmt.Sprintf("%s · %d RPM (max %d)", name, *f.RPM, int(denom)) + } else { + title = fmt.Sprintf("%s · %d RPM", name, *f.RPM) + } + } else { + title = name + " · no reading" + } + + side := fanTileMin + int(float64(fanTileMax-fanTileMin)*ratio+0.5) + // Spin period: 2.6s at rest down to 0.5s at the observed peak. A fan + // with no reading doesn't spin. + spin := "" + if f.RPM != nil && *f.RPM > 0 { + period := 2.6 - 2.1*ratio + spin = fmt.Sprintf(`` + } else { + spin = fmt.Sprintf(`` + } + + fmt.Fprintf(&b, `
%s
`, + html.EscapeString(title), side, side, fill, stroke, text, spin) + } + b.WriteString(`
`) + fmt.Fprintf(&b, `
%d fans · %s · tile size ∝ RPM / observed max
`, + len(fans), html.EscapeString(tally.line())) + return b.String() +} + +// topoFanSpinStyle emits the keyframes + base class for the spinning fan +// glyph once per row. A repeated identical ` +} + +// topoFanGlyphPaths is the fan-blade drawing shared by every fan square, +// designed on a 24×24 viewBox. +func topoFanGlyphPaths() string { + return `` + + `` + + `` + + `` +} + +// dedupeFansByName returns the fan sensors from a snapshot with duplicate +// names collapsed to their first occurrence, matching the ingest contract's +// "(sensor_type, name) — first wins" rule and skipping unnamed sensors. +func dedupeFansByName(sensors *schema.HardwareSensors) []schema.HardwareFanSensor { + if sensors == nil { + return nil + } + seen := map[string]bool{} + var out []schema.HardwareFanSensor + for _, f := range sensors.Fans { + name := strings.TrimSpace(f.Name) + if name == "" || seen[name] { + continue + } + seen[name] = true + out = append(out, f) + } + return out +} + +// topoRowHeading renders the small uppercase section label shared by the +// flex rows below the SVG diagram (Firmware / Power Supplies / Cooling / ...). +func topoRowHeading(title string) string { + return fmt.Sprintf(`
%s
`, + html.EscapeString(title)) +} + // renderTopoFlexRow renders a labeled, wrapping row of component cards. // Returns "" if items is empty (e.g. no PSU data in this audit). func renderTopoFlexRow(title string, items []topoCardInfo) string { @@ -624,8 +753,7 @@ func renderTopoFlexRow(title string, items []topoCardInfo) string { return "" } var b strings.Builder - fmt.Fprintf(&b, `
%s
`, - html.EscapeString(title)) + b.WriteString(topoRowHeading(title)) b.WriteString(`
`) for _, item := range items { onclick := "" @@ -1064,6 +1192,15 @@ func inventoryFallbackRecords(compType string, opts HandlerOptions) []app.Compon } records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(p.Status)}) } + case "fan": + for i, f := range dedupeFansByName(hw.Sensors) { + name := strings.TrimSpace(f.Name) + key := fmt.Sprintf("fan:%d", i) + if name != "" { + key = "fan:" + name + } + records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(f.Status)}) + } case "gpu", "nic", "raid": for i, dev := range hw.PCIeDevices { if pcieDeviceKind(dev) != compType { diff --git a/audit/internal/webui/page_topo_test.go b/audit/internal/webui/page_topo_test.go index 36ed1b2..7d011b9 100644 --- a/audit/internal/webui/page_topo_test.go +++ b/audit/internal/webui/page_topo_test.go @@ -152,6 +152,71 @@ func TestTopoPageRendersArbitraryPSUAndFirmwareCountsAsFlexRows(t *testing.T) { } } +func TestTopoPageRendersCoolingFansAsFlexRow(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "audit.json") + + ok, warn := "OK", "Warning" + rpm := func(v int) *int { return &v } + + ingest := schema.HardwareIngestRequest{ + CollectedAt: "2026-03-15T00:00:00Z", + Hardware: schema.HardwareSnapshot{ + Sensors: &schema.HardwareSensors{ + Fans: []schema.HardwareFanSensor{ + {Name: "FAN1", RPM: rpm(4200), Status: &ok}, + {Name: "FAN2", RPM: rpm(15000), Status: &warn}, + {Name: "FAN2", RPM: rpm(15000), Status: &warn}, // dup name, first wins + }, + }, + }, + } + data, err := json.Marshal(ingest) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0644); err != nil { + t.Fatal(err) + } + + handler := NewHandler(HandlerOptions{AuditPath: path}) + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/topo", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("status=%d", rec.Code) + } + body := rec.Body.String() + + // One clickable square per fan (2 after dedup by name), each with a + // spinning glyph, under a COOLING heading. + if !strings.Contains(body, "Cooling") { + t.Fatalf("topo page missing Cooling heading: %s", body) + } + if n := strings.Count(body, `onclick="openComponentDetail('fan')"`) + + strings.Count(body, `onclick="openComponentDetail('fan')"`); n != 2 { + t.Fatalf("expected one clickable square per fan (2), got %d: %s", n, body) + } + if n := strings.Count(body, `class="topo-fan-spin"`); n != 2 { + t.Fatalf("expected 2 spinning fan glyphs, got %d: %s", n, body) + } + if !strings.Contains(body, "FAN1 · 4200 RPM") || !strings.Contains(body, "FAN2 · 15000 RPM") { + t.Fatalf("topo page missing per-fan RPM tooltips: %s", body) + } + if !strings.Contains(body, "2 fans") { + t.Fatalf("topo page missing fan count caption: %s", body) + } + + // Component-detail fallback endpoint must resolve the "fan" type. + rec2 := httptest.NewRecorder() + handler.ServeHTTP(rec2, httptest.NewRequest(http.MethodGet, "/api/components/fan", nil)) + if rec2.Code != http.StatusOK { + t.Fatalf("/api/components/fan status=%d", rec2.Code) + } + if b := rec2.Body.String(); !strings.Contains(b, "FAN1") || !strings.Contains(b, "FAN2") { + t.Fatalf("fan component detail missing fan names: %s", b) + } +} + func TestTopoPageRendersStorageDisksGroupedByType(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "audit.json") diff --git a/audit/internal/webui/page_validate.go b/audit/internal/webui/page_validate.go index 53b184d..45f802c 100644 --- a/audit/internal/webui/page_validate.go +++ b/audit/internal/webui/page_validate.go @@ -114,6 +114,12 @@ func renderValidateMode(opts HandlerOptions, stressDefault bool) string { `Tests power supply transient response by pulsing all GPUs simultaneously between idle and full load. Synchronous pulses across all GPUs create worst-case PSU load spikes — running per-GPU would miss PSU-level failures.`, `dcgmi diag pulse_test`, validateFmtDur(platform.SATEstimatedNvidiaPulseTestSec)+` (all GPUs simultaneously; measured on 8-GPU system).`, + )) + + renderSATCard("fan", "Fan Ceiling Check", "runSAT('fan')", "", renderValidateCardBody( + "All system fans reported over IPMI / lm-sensors.", + `Drives CPU (+memory) and, when a GPU is present, GPU load to 100% at the same time and watches every fan until none has climbed for ~1 min. The peak RPM reached is recorded as each fan's ceiling and is what the Topology view sizes the fan tiles against. Success once every fan plateaus (or the time cap is hit). A fan reading 0 RPM or an IPMI status of cr/nr under full load fails. If the host cannot be loaded at all, or exposes no fan sensors, the task is cancelled as "not applicable" rather than failed — the platform does not support forcing fans, so this is the closest safe equivalent.`, + `stressapptest / stress-ng + bee-gpu-burn / rvs gst; ipmitool sdr type Fan`, + `~2–8 min depending on how fast the fan curve settles (hard cap 15 min).`, )) } diff --git a/audit/internal/webui/server.go b/audit/internal/webui/server.go index 2ffb7fe..b35f514 100644 --- a/audit/internal/webui/server.go +++ b/audit/internal/webui/server.go @@ -272,6 +272,7 @@ func NewHandler(opts HandlerOptions) http.Handler { mux.HandleFunc("POST /api/sat/memory-stress/run", h.handleAPISATRun("memory-stress")) mux.HandleFunc("POST /api/sat/sat-stress/run", h.handleAPISATRun("sat-stress")) mux.HandleFunc("POST /api/sat/platform-stress/run", h.handleAPISATRun("platform-stress")) + mux.HandleFunc("POST /api/sat/fan/run", h.handleAPISATRun("fan")) mux.HandleFunc("POST /api/sat/run-all", h.handleAPISATRunAll) mux.HandleFunc("GET /api/sat/stream", h.handleAPISATStream) mux.HandleFunc("POST /api/sat/abort", h.handleAPISATAbort) diff --git a/audit/internal/webui/task_runner.go b/audit/internal/webui/task_runner.go index 9715664..2841760 100644 --- a/audit/internal/webui/task_runner.go +++ b/audit/internal/webui/task_runner.go @@ -3,6 +3,7 @@ package webui import ( "context" "encoding/json" + "errors" "fmt" "io" "log/slog" @@ -79,9 +80,11 @@ func finalizeTaskForResult(t *Task, errMsg string, cancelled bool) { now := time.Now() t.DoneAt = &now switch { - case cancelled: + case cancelled || t.Status == TaskCancelled: t.Status = TaskCancelled - t.ErrMsg = "aborted" + if strings.TrimSpace(t.ErrMsg) == "" { + t.ErrMsg = "aborted" + } case strings.TrimSpace(errMsg) != "": t.Status = TaskFailed t.ErrMsg = errMsg @@ -377,6 +380,12 @@ func executeTaskWithOptions(opts *HandlerOptions, t *Task, j *jobState, ctx cont runOpts := resolvePlatformStressPreset(t.params.BurnProfile) runOpts.Components = t.params.PlatformComponents archive, err = a.RunPlatformStress(ctx, "", runOpts, j.append) + case "fan": + if a == nil { + err = fmt.Errorf("app not configured") + break + } + archive, err = runFanCheckPackCtx(a, ctx, "", platform.FanCheckOptions{GPUIndices: t.params.GPUIndices}, j.append) case "audit": if a == nil { err = fmt.Errorf("app not configured") @@ -476,10 +485,19 @@ func executeTaskWithOptions(opts *HandlerOptions, t *Task, j *jobState, ctx cont } if err != nil { - if ctx.Err() != nil { + switch { + case ctx.Err() != nil: j.append("Aborted.") j.finish("aborted") - } else { + case errors.Is(err, platform.ErrTestNotApplicable): + // The host offered no way to run this test — not a hardware + // fault. Land the task as cancelled ("not applicable") with a + // detailed log, so an engineer never sees a false failure. + j.append("NOT APPLICABLE: " + err.Error()) + t.Status = TaskCancelled + t.ErrMsg = "not applicable — " + err.Error() + j.finish("") + default: j.append("ERROR: " + err.Error()) j.finish(err.Error()) } diff --git a/audit/internal/webui/tasks.go b/audit/internal/webui/tasks.go index 4b1cfe8..a0ba100 100644 --- a/audit/internal/webui/tasks.go +++ b/audit/internal/webui/tasks.go @@ -53,6 +53,7 @@ var taskNames = map[string]string{ "memory-stress": "Memory Burn-in", "sat-stress": "SAT Stress (stressapptest)", "platform-stress": "Platform Thermal Cycling", + "fan": "Fan Ceiling Check (CPU+GPU load)", "audit": "Audit", "support-bundle": "Support Bundle", "install": "Install to Disk", @@ -196,7 +197,7 @@ func taskMayLeaveOrphanWorkers(target string) bool { switch strings.TrimSpace(strings.ToLower(target)) { case "nvidia", "nvidia-targeted-stress", "nvidia-targeted-power", "nvidia-pulse", "nvidia-bandwidth", "nvidia-stress", "nvidia-compute", "nvidia-bench-perf", - "memory", "memory-stress", "cpu", "sat-stress", "platform-stress": + "memory", "memory-stress", "cpu", "sat-stress", "platform-stress", "fan": return true default: return false @@ -343,6 +344,9 @@ var ( runSATStressPackCtx = func(a *app.App, ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) { return a.RunSATStressPackCtx(ctx, baseDir, durationSec, logFunc) } + runFanCheckPackCtx = func(a *app.App, ctx context.Context, baseDir string, opts platform.FanCheckOptions, logFunc func(string)) (string, error) { + return a.RunFanCheckCtx(ctx, baseDir, opts, logFunc) + } buildSupportBundle = app.BuildSupportBundle installCommand = func(ctx context.Context, device string, logPath string) *exec.Cmd { return exec.CommandContext(ctx, "bee-install", device, logPath) diff --git a/audit/internal/webui/tasks_test.go b/audit/internal/webui/tasks_test.go index 5743bf4..2c567b9 100644 --- a/audit/internal/webui/tasks_test.go +++ b/audit/internal/webui/tasks_test.go @@ -3,6 +3,7 @@ package webui import ( "context" "encoding/json" + "fmt" "net/http" "net/http/httptest" "os" @@ -700,6 +701,34 @@ func TestRunTaskHonorsCancel(t *testing.T) { } } +func TestRunTaskFanNotApplicableIsCancelledNotFailed(t *testing.T) { + q := &taskQueue{opts: &HandlerOptions{App: &app.App{}}} + tk := &Task{ + ID: "fan-1", Name: "Fan Ceiling Check", Target: "fan", + Status: TaskRunning, CreatedAt: time.Now(), + } + j := &jobState{} + tk.job = j + + orig := runFanCheckPackCtx + runFanCheckPackCtx = func(_ *app.App, _ context.Context, _ string, _ platform.FanCheckOptions, _ func(string)) (string, error) { + return "", fmt.Errorf("no fan sensors readable: %w", platform.ErrTestNotApplicable) + } + defer func() { runFanCheckPackCtx = orig }() + + q.runTask(tk, j, context.Background()) + + if tk.Status != TaskCancelled { + t.Fatalf("status=%q want %q", tk.Status, TaskCancelled) + } + if j.err != "" { + t.Fatalf("job err should be empty for a not-applicable task, got %q", j.err) + } + if !strings.Contains(tk.ErrMsg, "not applicable") { + t.Fatalf("ErrMsg should explain not-applicable, got %q", tk.ErrMsg) + } +} + func TestRunTaskUsesBurnProfileDurationForCPU(t *testing.T) { var gotDuration int q := &taskQueue{ diff --git a/bible-local/README.md b/bible-local/README.md index fa0b0a4..d0f1d4f 100644 --- a/bible-local/README.md +++ b/bible-local/README.md @@ -9,6 +9,7 @@ Generic engineering rules live in `bible/rules/patterns/`. |---|---| | `architecture/system-overview.md` | What bee does, scope, tech stack | | `architecture/runtime-flows.md` | Boot sequence, audit flow, service order | +| `architecture/squashfs-layers.md` | Semantic SquashFS layer model, ownership, order, verification | | `docs/customer-gpu-test-methodology.md` | Customer-facing GPU PCIe Validate / Validate -> Stress test list | | `docs/hardware-ingest-contract.md` | Current Reanimator hardware ingest JSON contract | | `docs/validate-vs-burn.md` | Validate and Validate -> Stress hardware test policy | @@ -74,3 +75,8 @@ Generic engineering rules live in `bible/rules/patterns/`. - `all_reduce_perf` - GPU bandwidth check - `dcgmi diag -r nvbandwidth` (per CPU socket, then all selected GPUs, on multi-socket systems -- see `decisions/2026-07-27-nvbandwidth-per-socket-split.md`) +- Fan ceiling check (Load tier / `3. Load` only) + - `stressapptest` (or `stress-ng`) + `bee-gpu-burn` / `rvs gst` at 100%, run simultaneously + - `ipmitool sdr type Fan` sampled 1 Hz until every fan plateaus (~1 min flat) or the 15 min cap + - records each fan's observed peak RPM to `/var/log/bee-sat/fan-observation.json` (used by the Topology fan tiles); FAIL only on a fan at 0 RPM / IPMI cr-nr under load; **cancelled ("not applicable")**, never failed, if the host cannot be loaded or has no fan sensors + - see `decisions/2026-09-04-fan-ceiling-check.md` diff --git a/bible-local/architecture/api-surface.md b/bible-local/architecture/api-surface.md index bbae0fa..948e306 100644 --- a/bible-local/architecture/api-surface.md +++ b/bible-local/architecture/api-surface.md @@ -53,6 +53,7 @@ All SAT run endpoints enqueue an async task. Response: `{"task_id": "..."}`. | POST | `/api/sat/memory-stress/run` | Memory stress | | POST | `/api/sat/sat-stress/run` | Combined storage+memory stress | | POST | `/api/sat/platform-stress/run` | Fan + thermal stress | +| POST | `/api/sat/fan/run` | Fan ceiling check (CPU+GPU load until fans plateau; Load tier). Not-applicable → task cancelled, not failed. | | POST | `/api/sat/run-all` | Plan + enqueue the whole validate/check set server-side. Body: `{stress_mode, amd_targets[], nvidia_gpu_indices[]}` (operator intent only). Response: `{task_ids[], task_count, notes[]}`. Hardware presence/readiness and which tasks to run are decided by `handler.planSATRunAll`, not the page. | | GET | `/api/sat/stream` | SSE: live SAT log stream | | POST | `/api/sat/abort` | Abort the running SAT task | diff --git a/bible-local/decisions/2026-09-04-fan-ceiling-check.md b/bible-local/decisions/2026-09-04-fan-ceiling-check.md new file mode 100644 index 0000000..9bbdca5 --- /dev/null +++ b/bible-local/decisions/2026-09-04-fan-ceiling-check.md @@ -0,0 +1,68 @@ +# Decision: Fan check discovers the RPM ceiling by CPU+GPU load, not by forcing fans over the BMC + +**Date:** 2026-09-04 +**Status:** active + +## Context + +We want a SAT test that pushes every system fan to its top speed, records that +speed as the fan's ceiling, and flags a fan that will not spin. The recorded +ceiling is what the Topology view sizes each fan tile against +(`platform.ResolveFanMaxRPM`). + +The obvious approach — force the fans to 100% PWM over IPMI/Redfish and read the +resulting RPM — does not work on our current platform: + +- Stand: MSI G4201 / MS-S3831, AMI MegaRAC BMC (fw 1.08). +- Every documented host-side fan-control OEM command returns + `rsp=0xc1 Invalid command`: Supermicro `0x30 0x45`, ASRock/AMI reference + `0x3a 0x01 / 0xd0 0x12 / 0xd0 0x0f / 0xd6 / 0xd7 / 0xda`, `0x30 0x30..0x32`. + BIOS `KCS Access Control Policy = Allow All`, so this is not KCS filtering — + MSI simply does not implement them. +- The MSI G4201 Redfish API guide documents Thermal as **GET only**; no + fan-mode / fan-PWM PATCH endpoint exists. +- The only fan knob in BIOS is `Fan PWM Offset` (0–100, additive to the auto + curve, reboot-gated) — not a runtime 100% force. + +## Decision + +The `fan` SAT test (`platform.RunFanCheck`, formerly the unwired +`RunFanStressTest`) drives load, not the BMC: + +1. Start `stressapptest` (CPU + memory) and, when a GPU is present, a GPU burn + (`bee-gpu-burn` / `rvs gst`) **simultaneously**, each in its own goroutine. + A missing GPU is not an error — CPU/memory load alone exercises the cooling + loop. Load sources reach full power at different times, so the plateau clock + only starts once every launched source reports its process running (plus a + fixed GPU ramp grace). +2. Sample every fan once a second. Per fan, track the peak RPM and the last time + it climbed by more than `PlateauDeltaRPM` (default 50). +3. When no fan has climbed for `PlateauHoldSec` (default 60 s) and at least one + fan rose meaningfully above baseline, declare the ceiling found and stop — + **success**. `MaxLoadSec` (default 900 s) is a hard cap; hitting it is also + success (the highest RPM seen is still recorded). +4. Persist each peak through the existing `updateFanObservation` path + (`/var/log/bee-sat/fan-observation.json`), which is what `ResolveFanMaxRPM` + reads. + +## Verdict mapping + +- A fan reading **0 RPM**, or IPMI status **cr/nr**, while under full load → + `FAILED` (dead / stuck rotor). +- No load source available (no `stressapptest`/`stress-ng` and no GPU burn + tool), or no fan sensors at all → `platform.ErrTestNotApplicable`, which the + task layer lands as **cancelled ("not applicable")**, never failed, with a + detailed log. An engineer must not see a red for "this platform can't run the + test". + +## Tier + +Load only (`/load`, "3. Load"). It is a sustained full-load test, so it does +not belong on the read-only Check page. Added to the stress-mode `Run All`. + +## Consequences + +- The ceiling is *observed*, not a spec value — only as high as the load drove + the fans. Good enough for tile sizing and stuck-fan detection. +- If a future platform does expose a safe host-side fan force, revisit: a real + 100% force is a stronger test than load-driven ramp.