diff --git a/audit/internal/app/app.go b/audit/internal/app/app.go index 2d4d7ab..90ac626 100644 --- a/audit/internal/app/app.go +++ b/audit/internal/app/app.go @@ -173,7 +173,7 @@ type satRunner interface { RunAMDStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) RunMemoryStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) RunSATStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) - RunFanStressTest(ctx context.Context, baseDir string, opts platform.FanStressOptions) (string, error) + RunFanCheck(ctx context.Context, baseDir string, opts platform.FanCheckOptions, logFunc func(string)) (string, error) RunPlatformStress(ctx context.Context, baseDir string, opts platform.PlatformStressOptions, logFunc func(string)) (string, error) RunNCCLTests(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) RunScenario(ctx context.Context, baseDir string, spec platform.ScenarioSpec, logFunc func(string)) (string, error) diff --git a/audit/internal/app/app_packs.go b/audit/internal/app/app_packs.go index 4f88355..b57ab26 100644 --- a/audit/internal/app/app_packs.go +++ b/audit/internal/app/app_packs.go @@ -298,6 +298,13 @@ func (a *App) RunPlatformStress(ctx context.Context, baseDir string, opts platfo return a.sat.RunPlatformStress(ctx, baseDir, opts, logFunc) } +func (a *App) RunFanCheckCtx(ctx context.Context, baseDir string, opts platform.FanCheckOptions, logFunc func(string)) (string, error) { + if strings.TrimSpace(baseDir) == "" { + baseDir = DefaultSATBaseDir + } + return a.sat.RunFanCheck(ctx, baseDir, opts, logFunc) +} + func (a *App) RunNCCLTests(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) { if strings.TrimSpace(baseDir) == "" { baseDir = DefaultSATBaseDir diff --git a/audit/internal/app/app_test.go b/audit/internal/app/app_test.go index a1df766..b7b6cd0 100644 --- a/audit/internal/app/app_test.go +++ b/audit/internal/app/app_test.go @@ -367,7 +367,7 @@ func (f fakeSAT) RunSATStressPack(_ context.Context, _ string, _ int, _ func(str return "", nil } -func (f fakeSAT) RunFanStressTest(_ context.Context, _ string, _ platform.FanStressOptions) (string, error) { +func (f fakeSAT) RunFanCheck(_ context.Context, _ string, _ platform.FanCheckOptions, _ func(string)) (string, error) { return "", nil } diff --git a/audit/internal/app/component_status_db.go b/audit/internal/app/component_status_db.go index 45654de..ae2c994 100644 --- a/audit/internal/app/component_status_db.go +++ b/audit/internal/app/component_status_db.go @@ -318,6 +318,23 @@ func ApplySATResultToDB(db *ComponentStatusDB, target, archivePath string) { db.Record("memory:all", source, dbStatus, detail) case "cpu", "platform-stress": db.Record("cpu:all", source, dbStatus, detail) + case "fan": + // Per-fan keys: summary emits "fan__status=OK|FAILED (...)". + for key, val := range kv { + name, ok := strings.CutPrefix(key, "fan_") + if !ok { + continue + } + name, ok = strings.CutSuffix(name, "_status") + if !ok || name == "" { + continue + } + upper := strings.ToUpper(strings.TrimSpace(val)) + if i := strings.IndexByte(upper, ' '); i > 0 { + upper = upper[:i] // drop the "(reason)" suffix + } + db.Record("fan:"+name, source, satStatusToDBStatus(upper), target+" SAT: "+strings.TrimSpace(val)) + } case "storage": // Try to record per-device if available in summary. recordedAny := false diff --git a/audit/internal/platform/live_metrics.go b/audit/internal/platform/live_metrics.go index 0fe5e6a..53a7214 100644 --- a/audit/internal/platform/live_metrics.go +++ b/audit/internal/platform/live_metrics.go @@ -393,5 +393,9 @@ func samplePSUPower() []PSUReading { if len(psus) == 0 { return nil } + // Feed the observed-capacity store (the "autotune" for PSU load scaling on + // BMCs that report only instantaneous power) — every load run that samples + // PSU power, including the 5 s metrics collector, refines it. + updatePSUObservation(psus, time.Now()) return psus } diff --git a/audit/internal/platform/observed_peaks.go b/audit/internal/platform/observed_peaks.go new file mode 100644 index 0000000..3a0540e --- /dev/null +++ b/audit/internal/platform/observed_peaks.go @@ -0,0 +1,183 @@ +package platform + +import ( + "encoding/json" + "math" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "time" +) + +// observedPeakStore is the "autotune" primitive for components that expose no +// host-readable nameplate maximum: fan top RPM, PSU capacity. It records the +// highest value seen per key while the box is under load, persists it to a +// JSON file, and hands it back so live readings can be scaled against a real +// maximum. A new peak only sticks after it has been held for at least +// minHold, which rejects transient spikes. +type observedPeakStore struct { + path string // JSON file + jsonKey string // top-level object key, e.g. "max_rpm" + roundUp float64 // round a new peak up to this multiple; 0 = keep raw + minHold time.Duration // a candidate peak must persist this long to stick + + mu sync.Mutex + loaded bool + peaks map[string]float64 + candidates map[string]peakCandidate +} + +type peakCandidate struct { + firstSeen time.Time + val float64 +} + +// persistedPeaks reads the file fresh (no lock, no cache mutation) and returns +// its sanitized {key -> peak} map. Empty map when the file is missing or +// unparsable. +func (s *observedPeakStore) persistedPeaks() map[string]float64 { + out := map[string]float64{} + raw, err := os.ReadFile(s.path) + if err != nil || len(raw) == 0 { + return out + } + var doc map[string]map[string]float64 + if json.Unmarshal(raw, &doc) != nil { + return out + } + for k, v := range doc[s.jsonKey] { + k = strings.TrimSpace(k) + if k == "" || v <= 0 { + continue + } + out[k] = v + } + return out +} + +func (s *observedPeakStore) loadLocked() { + if s.loaded { + return + } + s.loaded = true + s.peaks = s.persistedPeaks() + if s.candidates == nil { + s.candidates = map[string]peakCandidate{} + } +} + +func (s *observedPeakStore) saveLocked() { + if len(s.peaks) == 0 { + return + } + dir := filepath.Dir(s.path) + if dir == "" || dir == "." { + return + } + if err := os.MkdirAll(dir, 0755); err != nil { + return + } + raw, err := json.MarshalIndent(map[string]map[string]float64{s.jsonKey: s.peaks}, "", " ") + if err != nil { + return + } + _ = os.WriteFile(s.path, raw, 0644) +} + +func (s *observedPeakStore) round(v float64) float64 { + if v <= 0 { + return 0 + } + if s.roundUp <= 0 { + return v + } + return math.Ceil(v/s.roundUp) * s.roundUp +} + +// observe feeds one telemetry sample (key -> current value). Non-positive +// values and blank keys are ignored. +func (s *observedPeakStore) observe(samples map[string]float64, now time.Time) { + if len(samples) == 0 { + return + } + s.mu.Lock() + defer s.mu.Unlock() + s.loadLocked() + changed := false + for key, val := range samples { + key = strings.TrimSpace(key) + if key == "" || val <= 0 { + continue + } + cur := s.peaks[key] + if val <= cur { + delete(s.candidates, key) + continue + } + if cand, ok := s.candidates[key]; ok { + if now.Sub(cand.firstSeen) >= s.minHold { + nv := math.Max(cand.val, val) + if nv > cur { + s.peaks[key] = s.round(nv) + changed = true + } + delete(s.candidates, key) + continue + } + if val > cand.val { + s.candidates[key] = peakCandidate{firstSeen: cand.firstSeen, val: val} + } + continue + } + s.candidates[key] = peakCandidate{firstSeen: now, val: val} + } + if changed { + s.saveLocked() + } +} + +// snapshot returns the persisted peaks (fresh from disk), for read-only +// consumers such as the /topo web view. +func (s *observedPeakStore) snapshot() map[string]float64 { + return s.persistedPeaks() +} + +// ── PSU capacity ──────────────────────────────────────────────────────────── + +var psuPeaks = &observedPeakStore{ + path: "/var/log/bee-sat/psu-observation.json", + jsonKey: "max_w", + roundUp: 50, + minHold: time.Second, +} + +// updatePSUObservation feeds the current per-PSU draw (keyed by ordinal, in +// the order the caller lists them) into the observed-capacity store. On a BMC +// that reports only instantaneous input power this is the only way to know +// what "100% load" looks like for each supply: observe the peak draw during +// any full-load run (the Fan Ceiling Check, a burn, thermal cycling — the 5 s +// metrics collector samples PSUs throughout) and remember it. +func updatePSUObservation(psus []PSUReading, now time.Time) { + if len(psus) == 0 { + return + } + m := make(map[string]float64, len(psus)) + for i, p := range psus { + if p.PowerW > 0 { + m[strconv.Itoa(i)] = p.PowerW + } + } + psuPeaks.observe(m, now) +} + +// ObservedPSUMaxW returns the persisted per-PSU observed peak draw, keyed by +// ordinal ("0", "1", …), or nil if none recorded yet. +func ObservedPSUMaxW() map[string]float64 { + p := psuPeaks.snapshot() + if len(p) == 0 { + return nil + } + return p +} diff --git a/audit/internal/platform/sat_errors.go b/audit/internal/platform/sat_errors.go new file mode 100644 index 0000000..bf7f465 --- /dev/null +++ b/audit/internal/platform/sat_errors.go @@ -0,0 +1,10 @@ +package platform + +import "errors" + +// ErrTestNotApplicable is returned by a SAT routine when the host provides no +// way to run that test at all (a required tool or capability is absent), as +// opposed to the test running and finding a fault. The task layer maps it to a +// cancelled ("not applicable") task rather than a failure, so an engineer does +// not see a false red. +var ErrTestNotApplicable = errors.New("test not applicable on this platform") diff --git a/audit/internal/platform/sat_fan_stress.go b/audit/internal/platform/sat_fan_stress.go index 5d22f8f..d13d173 100644 --- a/audit/internal/platform/sat_fan_stress.go +++ b/audit/internal/platform/sat_fan_stress.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "math" "os" "os/exec" "path/filepath" @@ -12,46 +11,28 @@ import ( "strconv" "strings" "sync" + "syscall" "time" ) -// FanStressOptions configures the fan-stress / thermal cycling test. -type FanStressOptions struct { - BaselineSec int // idle monitoring before and after load (default 30) - Phase1DurSec int // first load phase duration in seconds (default 300) - PauseSec int // pause between the two load phases (default 60) - Phase2DurSec int // second load phase duration in seconds (default 300) - SizeMB int // GPU memory to allocate per GPU during stress (0 = auto: 95% of VRAM) - GPUIndices []int // which GPU indices to stress (empty = all detected) +// FanCheckOptions configures the fan-ceiling check: it drives CPU (+memory) +// and, when present, GPU load to 100% simultaneously, then watches every fan +// until none has climbed for PlateauHoldSec — at which point each fan is +// considered to be at its physical ceiling and the observed peak is recorded. +type FanCheckOptions struct { + PlateauHoldSec int // a fan must not rise > PlateauDeltaRPM for this long to count as plateaued (default 60) + PlateauDeltaRPM int // RPM increase that still counts as "climbing" (default 50) + MinLoadSec int // never declare a plateau before this many seconds of load (default 90) + MaxLoadSec int // hard cap on the load phase; finish (success) even if not every fan plateaued (default 900) + RampConfirmRPM int // at least one fan must exceed baseline by this before a plateau is "real" (default 150) + SizeMB int // GPU memory to allocate per GPU (0 = auto) + GPUIndices []int // which GPU indices to load (empty = all detected) } // FanReading holds one fan sensor reading. type FanReading struct { - Name string - RPM float64 -} - -// GPUStressMetric holds per-GPU metrics during the stress test. -type GPUStressMetric struct { - Index int - TempC float64 - UsagePct float64 - PowerW float64 - ClockMHz float64 - Throttled bool // true if any throttle reason is active -} - -// FanStressRow is one second-interval telemetry sample covering all monitored dimensions. -type FanStressRow struct { - TimestampUTC string - ElapsedSec float64 - Phase string // "baseline", "load1", "pause", "load2", "cooldown" - GPUs []GPUStressMetric - Fans []FanReading - CPUMaxTempC float64 // highest CPU temperature from ipmitool / sensors - SysPowerW float64 - SysPowerSource string - SysPowerMode string + Name string `json:"name"` + RPM float64 `json:"rpm"` } type cachedPowerReading struct { @@ -62,238 +43,434 @@ type cachedPowerReading struct { UpdatedAt time.Time } -type fanObservationState struct { - MaxRPM map[string]float64 `json:"max_rpm"` -} - -type fanPeakCandidate struct { - FirstSeen time.Time - RPM float64 -} - -var ( - systemPowerCacheMu sync.Mutex - systemPowerCache cachedPowerReading - fanObservationMu sync.Mutex - fanObservation fanObservationState - fanObservationInit bool - fanPeakCandidates = make(map[string]fanPeakCandidate) -) - const systemPowerHoldTTL = 15 * time.Second -var fanObservationStatePath = "/var/log/bee-sat/fan-observation.json" - -const fanObservationMinPeakHold = time.Second - -func normalizeObservedFanMaxRPM(rpm float64) float64 { - if rpm <= 0 { - return 0 - } - return math.Ceil(rpm/1000.0) * 1000.0 +// fanPeaks is the observed top-RPM store — the "autotune" for fan ceilings. +// Any full-load run (Fan Ceiling Check, burn, thermal cycling) feeds it via +// updateFanObservation; ResolveFanMaxRPM / ObservedFanMaxRPM read it back. +var fanPeaks = &observedPeakStore{ + path: "/var/log/bee-sat/fan-observation.json", + jsonKey: "max_rpm", + roundUp: 1000, + minHold: time.Second, } -// RunFanStressTest runs a two-phase GPU stress test while monitoring fan speeds, -// temperatures, and power draw every second. Exports metrics.csv and fan-sensors.csv. -// Designed to reproduce case-04 fan-speed lag and detect GPU thermal throttling. -func (s *System) RunFanStressTest(ctx context.Context, baseDir string, opts FanStressOptions) (string, error) { +// RunFanCheck drives CPU (+memory) and, when a GPU is present, GPU load to +// 100% simultaneously and watches every fan until none has climbed for +// PlateauHoldSec. At that point each fan is taken to be at its physical +// ceiling; the observed peak RPM is persisted (fanObservationStatePath, the +// same store ObservedFanMaxRPM reads) so the topology view can size each fan +// tile against a real maximum. +// +// Outcome: +// - success ("ceiling found") once every fan plateaus, or when MaxLoadSec is +// hit — a run that simply ran out of time still recorded the highest RPM +// seen and is not a failure. +// - a fan reading 0 RPM, or an IPMI status of cr/nr, while under full load is +// a real defect → FAILED. +// - if there is no way to load this box (no stressapptest/stress-ng and no +// GPU burn tool) or no fan sensors are readable, the test cannot say +// anything about the hardware and returns ErrTestNotApplicable so the task +// is cancelled, not failed. +// +// No GPU is not an error: CPU/memory load alone is enough to exercise the +// cooling loop on most platforms. +func (s *System) RunFanCheck(ctx context.Context, baseDir string, opts FanCheckOptions, logFunc func(string)) (string, error) { + if logFunc == nil { + logFunc = func(string) {} + } if baseDir == "" { baseDir = "/var/log/bee-sat" } - applyFanStressDefaults(&opts) + applyFanCheckDefaults(&opts) + + baseFans, fanErr := sampleFanSpeeds() + if len(baseFans) == 0 { + return "", fmt.Errorf("no fan sensors readable via ipmitool or lm-sensors (%v): %w", fanErr, ErrTestNotApplicable) + } + baselineRPM := make(map[string]float64, len(baseFans)) + for _, f := range baseFans { + baselineRPM[f.Name] = f.RPM + } + + vendor := s.DetectGPUVendor() + haveGPU := vendor == "nvidia" || vendor == "amd" + _, cpuPathErr := satLookPath("stressapptest") + if cpuPathErr != nil { + _, cpuPathErr = satLookPath("stress-ng") + } + haveCPU := cpuPathErr == nil + if !haveCPU && !haveGPU { + return "", fmt.Errorf("no load source: stressapptest/stress-ng missing and no NVIDIA/AMD GPU stress tool available: %w", ErrTestNotApplicable) + } ts := time.Now().UTC().Format("20060102-150405") - runDir := filepath.Join(baseDir, "fan-stress-"+ts) + runDir := filepath.Join(baseDir, "fan-check-"+ts) if err := os.MkdirAll(runDir, 0755); err != nil { return "", err } verboseLog := filepath.Join(runDir, "verbose.log") + appendSATVerboseLog(verboseLog, fmt.Sprintf("[%s] fan check start: %d fans, gpu=%s cpu=%v", + time.Now().UTC().Format(time.RFC3339), len(baseFans), vendor, haveCPU)) + logFunc(fmt.Sprintf("Fan check: %d fans; load = CPU/mem:%v + GPU:%s", len(baseFans), haveCPU, orNone(haveGPU, vendor))) - // Phase name shared between sampler goroutine and main goroutine. - var phaseMu sync.Mutex - currentPhase := "init" - setPhase := func(name string) { - phaseMu.Lock() - currentPhase = name - phaseMu.Unlock() + // ── Load: every source runs at the same time, each in its own goroutine. + // GPU load is the hottest sustained NVIDIA load we have — dcgmproftester + // -t 1004 / targeted_power, the same engine the Power/Thermal Fit + // benchmark uses (resolveBenchmarkPowerLoadCommand) — not the + // compute-throughput bee-gpu-burn, which tops out well below TDP and so + // never demands the fans' true ceiling. + // + // Sources reach full load at different times (stressapptest is instant; a + // dcgmproftester kernel compiles and ramps), so the plateau clock does not + // start until every launched source reports its process running, plus a + // fixed GPU ramp grace. + loadCtx, loadCancel := context.WithTimeout(ctx, time.Duration(opts.MaxLoadSec)*time.Second) + defer loadCancel() + + var loadWG sync.WaitGroup + started := make(chan bool, 2) // true = source is running, false = failed to launch + launched := 0 + + if haveCPU { + launched++ + loadWG.Add(1) + go func() { + defer loadWG.Done() + cmd, err := buildCPUStressCmd(loadCtx) + if err != nil { + logFunc("CPU/memory load failed to start: " + err.Error()) + appendSATVerboseLog(verboseLog, "cpu load start error: "+err.Error()) + started <- false + return + } + logFunc("CPU/memory load running (stressapptest)") + started <- true + _ = cmd.Wait() + }() } - getPhase := func() string { - phaseMu.Lock() - defer phaseMu.Unlock() - return currentPhase + if haveGPU { + launched++ + loadWG.Add(1) + go func() { + defer loadWG.Done() + cmd, label, err := buildFanCheckGPULoadCmd(loadCtx, vendor, opts.MaxLoadSec, opts.GPUIndices) + if err != nil || cmd == nil { + logFunc("GPU load unavailable: " + errString(err)) + appendSATVerboseLog(verboseLog, "gpu load unavailable: "+errString(err)) + started <- false + return + } + if err := cmd.Start(); err != nil { + logFunc("GPU load failed to start: " + err.Error()) + appendSATVerboseLog(verboseLog, "gpu load start error: "+err.Error()) + started <- false + return + } + logFunc("GPU load running (" + label + ")") + started <- true + _ = cmd.Wait() + }() } start := time.Now() - var rowsMu sync.Mutex - var allRows []FanStressRow - - // Start background sampler (every second). - stopCh := make(chan struct{}) - doneCh := make(chan struct{}) - go func() { - defer close(doneCh) - ticker := time.NewTicker(time.Second) - defer ticker.Stop() - for { - select { - case <-stopCh: - return - case <-ticker.C: - row := sampleFanStressRow(opts.GPUIndices, getPhase(), time.Since(start).Seconds()) - rowsMu.Lock() - allRows = append(allRows, row) - rowsMu.Unlock() - } - } - }() - - var summary strings.Builder - fmt.Fprintf(&summary, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339)) - - stats := satStats{} - - // idlePhase sleeps for durSec while the sampler stamps phaseName on each row. - idlePhase := func(phaseName, stepName string, durSec int) { - if ctx.Err() != nil { - return - } - setPhase(phaseName) - appendSATVerboseLog(verboseLog, - fmt.Sprintf("[%s] start %s (idle %ds)", time.Now().UTC().Format(time.RFC3339), stepName, durSec), - ) + activeLoads := 0 + for i := 0; i < launched; i++ { select { + case ok := <-started: + if ok { + activeLoads++ + } case <-ctx.Done(): - case <-time.After(time.Duration(durSec) * time.Second): } - appendSATVerboseLog(verboseLog, - fmt.Sprintf("[%s] finish %s", time.Now().UTC().Format(time.RFC3339), stepName), - ) - fmt.Fprintf(&summary, "%s_status=OK\n", stepName) - stats.OK++ + } + if activeLoads == 0 { + loadCancel() + loadWG.Wait() + return "", fmt.Errorf("every load source failed to start: %w", ErrTestNotApplicable) + } + readyAt := time.Now() + if haveGPU { + readyAt = readyAt.Add(20 * time.Second) // GPU kernel ramp grace + } + appendSATVerboseLog(verboseLog, fmt.Sprintf("%d load source(s) active; plateau clock effective from +%.0fs", + activeLoads, readyAt.Sub(start).Seconds())) + + // ── Sample loop with IPMI-hang protection. + // + // Under full load ipmitool over KCS can take tens of seconds per call or + // wedge outright. So: every fan read is time-boxed (readFansBounded runs + // it in a goroutine we abandon on timeout — a wedged KCS read can never + // block this loop), and the polling interval backs off geometrically when + // reads are slow and tightens again when they recover. A plateau is only + // declared while telemetry is healthy (interval near the floor); a + // degraded run just rides out to MaxLoadSec and records the peak it saw. + const ( + fanPollFloor = 1 * time.Second + fanPollCeil = 30 * time.Second + fanReadTMO = 8 * time.Second + ) + type fanState struct { + peak float64 + lastRiseSec float64 + } + fanBy := map[string]*fanState{} + rampConfirmed := false + plateauReached := false + aborted := false + degraded := false + goodSamples := 0 + poll := fanPollFloor + + // PSU peak draw, sampled at a slow cadence off the same loop — the fan + // check already drives the box to full power, so it is also the right run + // to observe what each PSU tops out at (updatePSUObservation persists it). + psuPeakW := map[int]float64{} + lastPSUSec := -1e9 + + csvPath := filepath.Join(runDir, "fan-sensors.csv") + _ = os.WriteFile(csvPath, []byte("elapsed_sec,fan_name,rpm\n"), 0644) + csvFile, _ := os.OpenFile(csvPath, os.O_APPEND|os.O_WRONLY, 0644) + if csvFile != nil { + defer csvFile.Close() } - // loadPhase runs bee-gpu-burn for durSec; sampler stamps phaseName on each row. - loadPhase := func(phaseName, stepName string, durSec int) { - if ctx.Err() != nil { - return +loop: + for { + select { + case <-ctx.Done(): + aborted = true + break loop + case <-loadCtx.Done(): + break loop // MaxLoadSec reached + case <-time.After(poll): } - setPhase(phaseName) - cmd := []string{ - "bee-gpu-burn", - "--seconds", strconv.Itoa(durSec), - "--size-mb", strconv.Itoa(opts.SizeMB), + elapsed := time.Since(start).Seconds() + + readStart := time.Now() + fans, ok := readFansBounded(fanReadTMO) + readDur := time.Since(readStart) + + // Adapt the interval to how ipmitool is behaving. + switch { + case !ok || readDur > fanReadTMO*3/4: + if poll < fanPollCeil { + poll = minDuration(poll*2, fanPollCeil) + degraded = true + appendSATVerboseLog(verboseLog, fmt.Sprintf("[%.0fs] ipmitool slow (%.1fs, ok=%v) — polling backed off to %s", + elapsed, readDur.Seconds(), ok, poll)) + logFunc(fmt.Sprintf("IPMI slow — fan polling backed off to %s", poll)) + } + case poll > fanPollFloor && readDur < fanPollFloor: + poll = maxDuration(poll*2/3, fanPollFloor) } - if len(opts.GPUIndices) > 0 { - cmd = append(cmd, "--devices", joinIndexList(dedupeSortedIndices(opts.GPUIndices))) + if !ok { + continue } - out, err := runSATCommandCtx(ctx, verboseLog, stepName, cmd, nil, nil) - _ = os.WriteFile(filepath.Join(runDir, stepName+".log"), out, 0644) - if err != nil && err != context.Canceled && err.Error() != "signal: killed" { - fmt.Fprintf(&summary, "%s_status=FAILED\n", stepName) + goodSamples++ + + if elapsed-lastPSUSec >= 15 { + lastPSUSec = elapsed + if ps, ok := readPSUPowerBounded(fanReadTMO); ok { + for i, p := range ps { + if p.PowerW > psuPeakW[i] { + psuPeakW[i] = p.PowerW + } + } + } + } + + for _, f := range fans { + if csvFile != nil { + fmt.Fprintf(csvFile, "%.0f,%s,%.0f\n", elapsed, f.Name, f.RPM) + } + st := fanBy[f.Name] + if st == nil { + fanBy[f.Name] = &fanState{peak: f.RPM, lastRiseSec: elapsed} + continue + } + if f.RPM > st.peak { + if f.RPM-st.peak > float64(opts.PlateauDeltaRPM) { + st.lastRiseSec = elapsed + } + st.peak = f.RPM + } + if f.RPM >= baselineRPM[f.Name]+float64(opts.RampConfirmRPM) { + rampConfirmed = true + } + } + + // Only trust a plateau while telemetry is healthy and we have enough + // recent samples to have actually seen a flat window. + healthy := poll <= 2*fanPollFloor && goodSamples >= 5 + if healthy && rampConfirmed && elapsed >= float64(opts.MinLoadSec) && + time.Since(readyAt) >= time.Duration(opts.PlateauHoldSec)*time.Second && len(fanBy) > 0 { + allFlat := true + for _, st := range fanBy { + if elapsed-st.lastRiseSec < float64(opts.PlateauHoldSec) { + allFlat = false + break + } + } + if allFlat { + plateauReached = true + logFunc(fmt.Sprintf("All %d fans plateaued at %.0fs of load", len(fanBy), elapsed)) + break loop + } + } + } + + loadCancel() + loadWG.Wait() + + if aborted && ctx.Err() != nil { + _ = os.WriteFile(filepath.Join(runDir, "summary.txt"), + []byte("run_at_utc="+time.Now().UTC().Format(time.RFC3339)+"\noverall_status=UNKNOWN\naborted=true\n"), 0644) + return runDir, ctx.Err() + } + + loadDur := time.Since(start).Seconds() + + // ── Verdict. + statuses := readFanStatuses() + var summary strings.Builder + fmt.Fprintf(&summary, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339)) + fmt.Fprintf(&summary, "fans_total=%d\n", len(baseFans)) + fmt.Fprintf(&summary, "active_load_sources=%d\n", activeLoads) + fmt.Fprintf(&summary, "gpu_vendor=%s\n", orNone(haveGPU, vendor)) + fmt.Fprintf(&summary, "plateau_reached=%v\n", plateauReached) + fmt.Fprintf(&summary, "ramp_confirmed=%v\n", rampConfirmed) + fmt.Fprintf(&summary, "load_duration_sec=%.0f\n", loadDur) + fmt.Fprintf(&summary, "fan_samples=%d\n", goodSamples) + fmt.Fprintf(&summary, "telemetry_degraded=%v\n", degraded) + if t := boundedGPUMaxTemp(opts.GPUIndices); t > 0 { + fmt.Fprintf(&summary, "gpu_temp_c=%.0f\n", t) + } + if len(psuPeakW) > 0 { + idx := make([]int, 0, len(psuPeakW)) + for i := range psuPeakW { + idx = append(idx, i) + } + sort.Ints(idx) + for _, i := range idx { + fmt.Fprintf(&summary, "psu_%d_peak_w=%.0f\n", i, psuPeakW[i]) + } + } + + stats := satStats{} + names := make([]string, 0, len(baselineRPM)) + for n := range baselineRPM { + names = append(names, n) + } + sort.Strings(names) + for _, name := range names { + peak := baselineRPM[name] + if st := fanBy[name]; st != nil { + peak = st.peak + } + st := strings.ToLower(strings.TrimSpace(statuses[name])) + bad := peak <= 0 || st == "cr" || st == "nr" + key := sanitizeSummaryKey(name) + fmt.Fprintf(&summary, "fan_%s_baseline_rpm=%.0f\n", key, baselineRPM[name]) + fmt.Fprintf(&summary, "fan_%s_max_rpm=%.0f\n", key, peak) + if bad { + reason := "0 RPM under load" + if st == "cr" || st == "nr" { + reason = "IPMI status " + st + } + fmt.Fprintf(&summary, "fan_%s_status=FAILED (%s)\n", key, reason) + logFunc(fmt.Sprintf("FAIL %s: %s", name, reason)) stats.Failed++ } else { - fmt.Fprintf(&summary, "%s_status=OK\n", stepName) + fmt.Fprintf(&summary, "fan_%s_status=OK\n", key) stats.OK++ } } - - // Execute test phases. - idlePhase("baseline", "01-baseline", opts.BaselineSec) - loadPhase("load1", "02-load1", opts.Phase1DurSec) - idlePhase("pause", "03-pause", opts.PauseSec) - loadPhase("load2", "04-load2", opts.Phase2DurSec) - idlePhase("cooldown", "05-cooldown", opts.BaselineSec) - - // Stop sampler and collect rows. - close(stopCh) - <-doneCh - - rowsMu.Lock() - rows := allRows - rowsMu.Unlock() - - // Analysis. - throttled := analyzeThrottling(rows) - maxGPUTemp := analyzeMaxTemp(rows, func(r FanStressRow) float64 { - var m float64 - for _, g := range r.GPUs { - if g.TempC > m { - m = g.TempC - } - } - return m - }) - maxCPUTemp := analyzeMaxTemp(rows, func(r FanStressRow) float64 { - return r.CPUMaxTempC - }) - fanResponseSec := analyzeFanResponse(rows) - - fmt.Fprintf(&summary, "throttling_detected=%v\n", throttled) - fmt.Fprintf(&summary, "max_gpu_temp_c=%.1f\n", maxGPUTemp) - fmt.Fprintf(&summary, "max_cpu_temp_c=%.1f\n", maxCPUTemp) - if fanResponseSec >= 0 { - fmt.Fprintf(&summary, "fan_response_sec=%.1f\n", fanResponseSec) - } else { - fmt.Fprintf(&summary, "fan_response_sec=N/A\n") - } - - // Throttling failure counts against overall result. - if throttled { - stats.Failed++ - } writeSATStats(&summary, stats) - // Write CSV outputs. - if err := WriteFanStressCSV(filepath.Join(runDir, "metrics.csv"), rows, opts.GPUIndices); err != nil { - return "", err - } - _ = WriteFanSensorsCSV(filepath.Join(runDir, "fan-sensors.csv"), rows) - if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary.String()), 0644); err != nil { return "", err } - return runDir, nil } -func applyFanStressDefaults(opts *FanStressOptions) { - if opts.BaselineSec <= 0 { - opts.BaselineSec = 30 +func errString(err error) string { + if err == nil { + return "no GPU stress tool" } - if opts.Phase1DurSec <= 0 { - opts.Phase1DurSec = 300 - } - if opts.PauseSec <= 0 { - opts.PauseSec = 60 - } - if opts.Phase2DurSec <= 0 { - opts.Phase2DurSec = 300 - } - // SizeMB == 0 means "auto" (worker picks 95% of GPU VRAM for maximum power draw). - // Leave at 0 to avoid passing a too-small size that starves the tensor-core path. + return err.Error() } -// sampleFanStressRow collects all metrics for one telemetry sample. -func sampleFanStressRow(gpuIndices []int, phase string, elapsed float64) FanStressRow { - row := FanStressRow{ - TimestampUTC: time.Now().UTC().Format(time.RFC3339), - ElapsedSec: elapsed, - Phase: phase, +func minDuration(a, b time.Duration) time.Duration { + if a < b { + return a } - row.GPUs = sampleGPUStressMetrics(gpuIndices) - row.Fans, _ = sampleFanSpeeds() - row.CPUMaxTempC = sampleCPUMaxTemp() - row.SysPowerW, row.SysPowerSource, row.SysPowerMode = sampleSystemPowerResolved() - return row + return b } -// sampleGPUStressMetrics queries nvidia-smi for temperature, utilization, power, -// clock frequency, and active throttle reasons for each GPU. -func sampleGPUStressMetrics(gpuIndices []int) []GPUStressMetric { - args := []string{ - "--query-gpu=index,temperature.gpu,utilization.gpu,power.draw,clocks.current.graphics,clocks_throttle_reasons.active", - "--format=csv,noheader,nounits", +func maxDuration(a, b time.Duration) time.Duration { + if a > b { + return a } + return b +} + +// readFansBounded runs "ipmitool sdr type Fan" but never blocks the caller +// longer than timeout: the read happens in a goroutine that is abandoned if it +// does not return in time (a KCS read wedged in uninterruptible I/O cannot be +// killed, so we leave it and move on). ok=false means "no usable sample this +// tick" — the caller must treat that as missing data, not as a flat fan. +func readFansBounded(timeout time.Duration) ([]FanReading, bool) { + type result struct { + fans []FanReading + ok bool + } + ch := make(chan result, 1) + go func() { + out, err := exec.Command("ipmitool", "sdr", "type", "Fan").Output() + if err != nil { + ch <- result{} + return + } + fans := parseFanSpeeds(string(out)) + if len(fans) == 0 { + ch <- result{} + return + } + ch <- result{fans, true} + }() + select { + case r := <-ch: + if r.ok { + updateFanObservation(r.fans, time.Now()) + } + return r.fans, r.ok + case <-time.After(timeout): + return nil, false + } +} + +// readPSUPowerBounded is readFansBounded's PSU sibling: a time-boxed, +// abandonable "ipmitool sdr" read of per-PSU power (samplePSUPower also feeds +// the observed-capacity store). Used at a slow cadence during the fan check so +// the same max-load run that finds fan ceilings also records peak PSU draw. +func readPSUPowerBounded(timeout time.Duration) ([]PSUReading, bool) { + ch := make(chan []PSUReading, 1) + go func() { ch <- samplePSUPower() }() + select { + case ps := <-ch: + return ps, len(ps) > 0 + case <-time.After(timeout): + return nil, false + } +} + +// boundedGPUMaxTemp returns the hottest GPU temperature via a single +// time-boxed nvidia-smi call, or 0 if unavailable. +func boundedGPUMaxTemp(gpuIndices []int) float64 { + ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second) + defer cancel() + args := []string{"--query-gpu=temperature.gpu", "--format=csv,noheader,nounits"} if len(gpuIndices) > 0 { ids := make([]string, len(gpuIndices)) for i, idx := range gpuIndices { @@ -301,38 +478,151 @@ func sampleGPUStressMetrics(gpuIndices []int) []GPUStressMetric { } args = append([]string{"--id=" + strings.Join(ids, ",")}, args...) } - out, err := exec.Command("nvidia-smi", args...).Output() + out, err := exec.CommandContext(ctx, "nvidia-smi", args...).Output() + if err != nil { + return 0 + } + var max float64 + for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { + if v, err := strconv.ParseFloat(strings.TrimSpace(line), 64); err == nil && v > max { + max = v + } + } + return max +} + +// buildFanCheckGPULoadCmd builds the hottest sustained GPU load for the fan +// check. NVIDIA uses the Power/Thermal Fit engine (dcgmproftester -t 1004 / +// targeted_power); AMD uses the RVS gst stressor. +func buildFanCheckGPULoadCmd(ctx context.Context, vendor string, durSec int, gpuIndices []int) (*exec.Cmd, string, error) { + switch strings.ToLower(vendor) { + case "nvidia": + argv, env, err := resolveBenchmarkPowerLoadCommand(durSec, gpuIndices) + if err != nil { + return nil, "", err + } + cmd := exec.CommandContext(ctx, argv[0], argv[1:]...) + if len(env) > 0 { + cmd.Env = append(os.Environ(), env...) + } + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + cmd.Cancel = func() error { + if cmd.Process != nil { + _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + } + return nil + } + return cmd, "dcgmproftester targeted_power", nil + case "amd": + cmd := buildAMDGPUStressCmd(ctx, durSec) + if cmd == nil { + return nil, "", nil + } + return cmd, "rvs gst", nil + } + return nil, "", nil +} + +func applyFanCheckDefaults(o *FanCheckOptions) { + if o.PlateauHoldSec <= 0 { + o.PlateauHoldSec = 60 + } + if o.PlateauDeltaRPM <= 0 { + o.PlateauDeltaRPM = 50 + } + if o.MinLoadSec <= 0 { + o.MinLoadSec = 90 + } + if o.MaxLoadSec <= 0 { + o.MaxLoadSec = 900 + } + if o.RampConfirmRPM <= 0 { + o.RampConfirmRPM = 150 + } + if o.MinLoadSec < o.PlateauHoldSec { + o.MinLoadSec = o.PlateauHoldSec + } + if o.MaxLoadSec <= o.MinLoadSec { + o.MaxLoadSec = o.MinLoadSec + o.PlateauHoldSec + } +} + +func orNone(present bool, v string) string { + if present && v != "" { + return v + } + return "none" +} + +// sanitizeSummaryKey makes a fan sensor name safe as a summary.txt key +// fragment (keys are parsed by splitting on '=' and whitespace). +func sanitizeSummaryKey(name string) string { + var b strings.Builder + for _, r := range name { + switch { + case r >= 'A' && r <= 'Z', r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '-', r == '_', r == '.': + b.WriteRune(r) + default: + b.WriteByte('_') + } + } + return b.String() +} + +// readFanStatuses returns the per-fan IPMI status word ("ok", "cr", "nr", ...) +// from "ipmitool sdr type Fan". Empty map when ipmitool is unavailable. +func readFanStatuses() map[string]string { + out, err := exec.Command("ipmitool", "sdr", "type", "Fan").Output() if err != nil { return nil } - var metrics []GPUStressMetric - for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { - line = strings.TrimSpace(line) - if line == "" { + m := map[string]string{} + for _, line := range strings.Split(string(out), "\n") { + parts := strings.Split(line, "|") + if len(parts) < 3 { continue } - parts := strings.Split(line, ", ") - if len(parts) < 6 { + name := strings.TrimSpace(parts[0]) + if name == "" { continue } - idx, _ := strconv.Atoi(strings.TrimSpace(parts[0])) - throttleVal := strings.TrimSpace(parts[5]) - // Throttled if active reasons bitmask is non-zero. - throttled := throttleVal != "0x0000000000000000" && - throttleVal != "0x0" && - throttleVal != "0" && - throttleVal != "" && - throttleVal != "N/A" - metrics = append(metrics, GPUStressMetric{ - Index: idx, - TempC: parseGPUFloat(parts[1]), - UsagePct: parseGPUFloat(parts[2]), - PowerW: parseGPUFloat(parts[3]), - ClockMHz: parseGPUFloat(parts[4]), - Throttled: throttled, - }) + m[name] = strings.ToLower(strings.TrimSpace(parts[2])) } - return metrics + return m +} + +// ResolveFanMaxRPM returns, for every fan name in current (name -> current +// RPM), the RPM to treat as that fan's 100% reference. Preference order: +// 1. the persisted observed peak, written by RunFanCheck and by live-metrics +// sampling under load; +// 2. the largest peak observed on any peer fan (keeps a group visually +// consistent when only some fans have a recorded peak); +// 3. the fan's own current RPM (so a tile is never sized against zero). +// +// The fallback lives here, not in the view, so every consumer of a fan +// maximum applies the same rule. +func ResolveFanMaxRPM(current map[string]float64) map[string]float64 { + persisted := fanPeaks.snapshot() + + peerMax := 0.0 + for _, v := range persisted { + if v > peerMax { + peerMax = v + } + } + + out := make(map[string]float64, len(current)) + for name, rpm := range current { + switch { + case persisted[name] > 0: + out[name] = persisted[name] + case peerMax > 0: + out[name] = peerMax + default: + out[name] = rpm + } + } + return out } // sampleFanSpeeds reads fan RPM values from ipmitool sdr. @@ -355,101 +645,41 @@ func sampleFanSpeeds() ([]FanReading, error) { return nil, sensorsErr } -func loadFanObservationLocked() { - if fanObservationInit { - return +// ObservedFanMaxRPM returns the per-fan observed peak RPM map persisted by +// full-load runs, or nil if none is recorded yet. +func ObservedFanMaxRPM() map[string]float64 { + out := fanPeaks.snapshot() + if len(out) == 0 { + return nil } - fanObservationInit = true - fanObservation.MaxRPM = make(map[string]float64) - raw, err := os.ReadFile(fanObservationStatePath) - if err != nil || len(raw) == 0 { - return - } - var persisted fanObservationState - if json.Unmarshal(raw, &persisted) != nil { - return - } - for name, rpm := range persisted.MaxRPM { - name = strings.TrimSpace(name) - if name == "" || rpm <= 0 { - continue - } - fanObservation.MaxRPM[name] = rpm - } -} - -func saveFanObservationLocked() { - if len(fanObservation.MaxRPM) == 0 { - return - } - dir := filepath.Dir(fanObservationStatePath) - if dir == "" || dir == "." { - dir = "/var/log/bee-sat" - } - if err := os.MkdirAll(dir, 0755); err != nil { - return - } - raw, err := json.MarshalIndent(fanObservation, "", " ") - if err != nil { - return - } - _ = os.WriteFile(fanObservationStatePath, raw, 0644) + return out } func updateFanObservation(fans []FanReading, now time.Time) { if len(fans) == 0 { return } - fanObservationMu.Lock() - defer fanObservationMu.Unlock() - loadFanObservationLocked() - changed := false + m := make(map[string]float64, len(fans)) for _, fan := range fans { - name := strings.TrimSpace(fan.Name) - if name == "" || fan.RPM <= 0 { - continue + if n := strings.TrimSpace(fan.Name); n != "" && fan.RPM > 0 { + m[n] = fan.RPM } - currentMax := fanObservation.MaxRPM[name] - if fan.RPM <= currentMax { - delete(fanPeakCandidates, name) - continue - } - if cand, ok := fanPeakCandidates[name]; ok { - if now.Sub(cand.FirstSeen) >= fanObservationMinPeakHold { - newMax := math.Max(cand.RPM, fan.RPM) - if newMax > currentMax { - fanObservation.MaxRPM[name] = normalizeObservedFanMaxRPM(newMax) - changed = true - } - delete(fanPeakCandidates, name) - continue - } - if fan.RPM > cand.RPM { - fanPeakCandidates[name] = fanPeakCandidate{FirstSeen: cand.FirstSeen, RPM: fan.RPM} - } - continue - } - fanPeakCandidates[name] = fanPeakCandidate{FirstSeen: now, RPM: fan.RPM} - } - if changed { - saveFanObservationLocked() } + fanPeaks.observe(m, now) } func estimateFanDutyCyclePctFromObservation(fans []FanReading) (float64, bool) { if len(fans) == 0 { return 0, false } - fanObservationMu.Lock() - defer fanObservationMu.Unlock() - loadFanObservationLocked() + maxByName := fanPeaks.snapshot() var samples []float64 for _, fan := range fans { name := strings.TrimSpace(fan.Name) if name == "" || fan.RPM <= 0 { continue } - maxRPM := fanObservation.MaxRPM[name] + maxRPM := maxByName[name] if maxRPM <= 0 { continue } @@ -733,21 +963,6 @@ func sampleCPUTempViaSensors() float64 { return max } -// sampleSystemPowerResolved reads system power via the global autotune source, -// falling back to the historical heuristic before autotune or when degraded. -func sampleSystemPowerResolved() (float64, string, string) { - now := time.Now() - current, decision, err := SampleSystemPowerResolved("") - systemPowerCacheMu.Lock() - defer systemPowerCacheMu.Unlock() - if err != nil { - current = 0 - } - value, updated := effectiveSystemPowerReading(systemPowerCache, current, decision.EffectiveSource, decision.Mode, decision.Reason, now) - systemPowerCache = updated - return value, updated.Source, updated.Mode -} - // parseDCMIPowerReading extracts the instantaneous power reading from ipmitool dcmi output. // Sample: " Instantaneous power reading: 500 Watts" func parseDCMIPowerReading(raw string) float64 { @@ -779,141 +994,6 @@ func effectiveSystemPowerReading(cache cachedPowerReading, current float64, sour return 0, cache } -// analyzeThrottling returns true if any GPU reported an active throttle reason -// during either load phase. -func analyzeThrottling(rows []FanStressRow) bool { - for _, row := range rows { - if row.Phase != "load1" && row.Phase != "load2" { - continue - } - for _, gpu := range row.GPUs { - if gpu.Throttled { - return true - } - } - } - return false -} - -// analyzeMaxTemp returns the maximum value of the given extractor across all rows. -func analyzeMaxTemp(rows []FanStressRow, extract func(FanStressRow) float64) float64 { - var max float64 - for _, row := range rows { - if v := extract(row); v > max { - max = v - } - } - return max -} - -// analyzeFanResponse returns the seconds from load1 start until fan RPM first -// increased by more than 5% above the baseline average. Returns -1 if undetermined. -func analyzeFanResponse(rows []FanStressRow) float64 { - // Compute baseline average fan RPM. - var baseTotal, baseCount float64 - for _, row := range rows { - if row.Phase != "baseline" { - continue - } - for _, f := range row.Fans { - baseTotal += f.RPM - baseCount++ - } - } - if baseCount == 0 || baseTotal == 0 { - return -1 - } - baseAvg := baseTotal / baseCount - threshold := baseAvg * 1.05 // 5% increase signals fan ramp-up - - // Find elapsed time when load1 started. - var load1Start float64 = -1 - for _, row := range rows { - if row.Phase == "load1" { - load1Start = row.ElapsedSec - break - } - } - if load1Start < 0 { - return -1 - } - - // Find first load1 row where average RPM crosses the threshold. - for _, row := range rows { - if row.Phase != "load1" { - continue - } - var total, count float64 - for _, f := range row.Fans { - total += f.RPM - count++ - } - if count > 0 && total/count >= threshold { - return row.ElapsedSec - load1Start - } - } - return -1 -} - -// WriteFanStressCSV writes the wide-format metrics CSV with one row per second. -// GPU columns are generated per index in gpuIndices order. -func WriteFanStressCSV(path string, rows []FanStressRow, gpuIndices []int) error { - if len(rows) == 0 { - return os.WriteFile(path, []byte("no data\n"), 0644) - } - - var b strings.Builder - - // Header: fixed system columns + per-GPU columns. - b.WriteString("timestamp_utc,elapsed_sec,phase,fan_avg_rpm,fan_min_rpm,fan_max_rpm,cpu_max_temp_c,sys_power_w") - for _, idx := range gpuIndices { - fmt.Fprintf(&b, ",gpu%d_temp_c,gpu%d_usage_pct,gpu%d_power_w,gpu%d_clock_mhz,gpu%d_throttled", - idx, idx, idx, idx, idx) - } - b.WriteRune('\n') - - for _, row := range rows { - favg, fmin, fmax := fanRPMStats(row.Fans) - fmt.Fprintf(&b, "%s,%.1f,%s,%.0f,%.0f,%.0f,%.1f,%.1f", - row.TimestampUTC, - row.ElapsedSec, - row.Phase, - favg, fmin, fmax, - row.CPUMaxTempC, - row.SysPowerW, - ) - gpuByIdx := make(map[int]GPUStressMetric, len(row.GPUs)) - for _, g := range row.GPUs { - gpuByIdx[g.Index] = g - } - for _, idx := range gpuIndices { - g := gpuByIdx[idx] - throttled := 0 - if g.Throttled { - throttled = 1 - } - fmt.Fprintf(&b, ",%.1f,%.1f,%.1f,%.0f,%d", - g.TempC, g.UsagePct, g.PowerW, g.ClockMHz, throttled) - } - b.WriteRune('\n') - } - - return os.WriteFile(path, []byte(b.String()), 0644) -} - -// WriteFanSensorsCSV writes individual fan sensor readings in long (tidy) format. -func WriteFanSensorsCSV(path string, rows []FanStressRow) error { - var b strings.Builder - b.WriteString("timestamp_utc,elapsed_sec,phase,fan_name,rpm\n") - for _, row := range rows { - for _, f := range row.Fans { - fmt.Fprintf(&b, "%s,%.1f,%s,%s,%.0f\n", - row.TimestampUTC, row.ElapsedSec, row.Phase, f.Name, f.RPM) - } - } - return os.WriteFile(path, []byte(b.String()), 0644) -} - // fanRPMStats computes average, min, max RPM across all fans in a sample row. func fanRPMStats(fans []FanReading) (avg, min, max float64) { if len(fans) == 0 { diff --git a/audit/internal/platform/sat_fan_stress_test.go b/audit/internal/platform/sat_fan_stress_test.go index 20ac394..817fccf 100644 --- a/audit/internal/platform/sat_fan_stress_test.go +++ b/audit/internal/platform/sat_fan_stress_test.go @@ -1,11 +1,116 @@ package platform import ( + "os" "path/filepath" + "reflect" "testing" "time" ) +// resetPeakStore points a store at a fresh temp file and clears its cache for +// the duration of the test. +func resetPeakStore(t *testing.T, s *observedPeakStore) { + t.Helper() + old := *s + s.path = filepath.Join(t.TempDir(), "peaks.json") + s.loaded = false + s.peaks = nil + s.candidates = nil + t.Cleanup(func() { *s = old }) +} + +func TestResolveFanMaxRPM(t *testing.T) { + resetPeakStore(t, fanPeaks) + + // No persisted file yet: unknown fans fall back to their own current RPM. + got := ResolveFanMaxRPM(map[string]float64{"A": 4000, "B": 9000}) + if !reflect.DeepEqual(got, map[string]float64{"A": 4000, "B": 9000}) { + t.Fatalf("no-persist fallback: got %v", got) + } + + if err := os.WriteFile(fanPeaks.path, []byte(`{"max_rpm":{"A":17000}}`), 0644); err != nil { + t.Fatal(err) + } + got = ResolveFanMaxRPM(map[string]float64{"A": 4000, "B": 9000, "C": 5000}) + // A: persisted peak. B/C: no own entry -> largest peer peak (A's 17000). + if want := map[string]float64{"A": 17000, "B": 17000, "C": 17000}; !reflect.DeepEqual(got, want) { + t.Fatalf("peer fallback: got %v want %v", got, want) + } +} + +func TestObservedPeakStore(t *testing.T) { + s := &observedPeakStore{ + path: filepath.Join(t.TempDir(), "peaks.json"), + jsonKey: "max_w", + roundUp: 50, + minHold: time.Second, + } + t0 := time.Unix(0, 0) + + // A single spike does not stick. + s.observe(map[string]float64{"0": 900}, t0) + if len(s.snapshot()) != 0 { + t.Fatalf("transient spike should not persist: %v", s.snapshot()) + } + // Held past minHold → sticks, rounded up to the next 50. + s.observe(map[string]float64{"0": 920}, t0.Add(1200*time.Millisecond)) + if got := s.snapshot()["0"]; got != 950 { + t.Fatalf("held peak: got %v want 950", got) + } + // A lower reading never lowers the peak. + s.observe(map[string]float64{"0": 400}, t0.Add(5*time.Second)) + if got := s.snapshot()["0"]; got != 950 { + t.Fatalf("peak must not drop: got %v", got) + } + // Fresh store reloads from disk. + s2 := &observedPeakStore{path: s.path, jsonKey: "max_w"} + if got := s2.snapshot()["0"]; got != 950 { + t.Fatalf("reload from disk: got %v want 950", got) + } +} + +func TestUpdatePSUObservationKeyedByOrdinal(t *testing.T) { + resetPeakStore(t, psuPeaks) + now := time.Unix(0, 0) + psus := []PSUReading{{Name: "PSU1", PowerW: 1200}, {Name: "PSU2", PowerW: 1400}} + updatePSUObservation(psus, now) + updatePSUObservation(psus, now.Add(1200*time.Millisecond)) + got := ObservedPSUMaxW() + if got["0"] != 1200 || got["1"] != 1400 { + t.Fatalf("keyed-by-ordinal peaks: got %v", got) + } +} + +func TestApplyFanCheckDefaults(t *testing.T) { + var o FanCheckOptions + applyFanCheckDefaults(&o) + if o.PlateauHoldSec != 60 || o.PlateauDeltaRPM != 50 || o.MinLoadSec != 90 || o.MaxLoadSec != 900 || o.RampConfirmRPM != 150 { + t.Fatalf("unexpected defaults: %+v", o) + } + o = FanCheckOptions{PlateauHoldSec: 120, MinLoadSec: 30, MaxLoadSec: 40} + applyFanCheckDefaults(&o) + if o.MinLoadSec < o.PlateauHoldSec { + t.Fatalf("MinLoadSec must be >= PlateauHoldSec, got %d", o.MinLoadSec) + } + if o.MaxLoadSec <= o.MinLoadSec { + t.Fatalf("MaxLoadSec must exceed MinLoadSec, got %d", o.MaxLoadSec) + } +} + +func TestSanitizeSummaryKey(t *testing.T) { + for in, want := range map[string]string{ + "F2U-1": "F2U-1", + "aspeed / fan1": "aspeed___fan1", + "CPU0_DIMM": "CPU0_DIMM", + "weird=key here": "weird_key_here", + } { + if got := sanitizeSummaryKey(in); got != want { + t.Errorf("sanitizeSummaryKey(%q)=%q want %q", in, got, want) + } + } +} + func TestParseFanSpeeds(t *testing.T) { raw := "FAN1 | 2400.000 | RPM | ok\nFAN2 | 1800 RPM | ok | ok\nFAN3 | na | RPM | ns\n" got := parseFanSpeeds(raw) @@ -52,22 +157,7 @@ func TestParseFanDutyCyclePctSensorsJSON(t *testing.T) { } func TestEstimateFanDutyCyclePctFromObservation(t *testing.T) { - t.Parallel() - - oldPath := fanObservationStatePath - oldState := fanObservation - oldInit := fanObservationInit - oldCandidates := fanPeakCandidates - fanObservationStatePath = filepath.Join(t.TempDir(), "fan-observation.json") - fanObservation = fanObservationState{} - fanObservationInit = false - fanPeakCandidates = make(map[string]fanPeakCandidate) - t.Cleanup(func() { - fanObservationStatePath = oldPath - fanObservation = oldState - fanObservationInit = oldInit - fanPeakCandidates = oldCandidates - }) + resetPeakStore(t, fanPeaks) start := time.Unix(100, 0) updateFanObservation([]FanReading{{Name: "FAN1", RPM: 5000}}, start) @@ -86,9 +176,9 @@ func TestEstimateFanDutyCyclePctFromObservation(t *testing.T) { t.Fatalf("got=%v want ~43.3", got) } - fanObservation = fanObservationState{} - fanObservationInit = false - fanPeakCandidates = make(map[string]fanPeakCandidate) + fanPeaks.loaded = false + fanPeaks.peaks = nil + fanPeaks.candidates = nil got, ok = estimateFanDutyCyclePctFromObservation([]FanReading{{Name: "FAN1", RPM: 2600}}) if !ok { t.Fatalf("expected persisted observed max to be reloaded from disk") diff --git a/audit/internal/platform/sat_gpu_packs.go b/audit/internal/platform/sat_gpu_packs.go index cf9e64f..ef0acae 100644 --- a/audit/internal/platform/sat_gpu_packs.go +++ b/audit/internal/platform/sat_gpu_packs.go @@ -14,6 +14,7 @@ type NvidiaGPU struct { Index int `json:"index"` Name string `json:"name"` MemoryMB int `json:"memory_mb"` + Serial string `json:"serial,omitempty"` } type NvidiaGPUStatus struct { @@ -226,7 +227,7 @@ func amdStressJobs(seconds int, cfgFile string) []satJob { // ListNvidiaGPUs returns GPUs visible to nvidia-smi. func (s *System) ListNvidiaGPUs() ([]NvidiaGPU, error) { out, err := exec.Command("nvidia-smi", - "--query-gpu=index,name,memory.total", + "--query-gpu=index,name,memory.total,serial", "--format=csv,noheader,nounits").Output() if err != nil { return nil, fmt.Errorf("nvidia-smi: %w", err) @@ -237,8 +238,8 @@ func (s *System) ListNvidiaGPUs() ([]NvidiaGPU, error) { if line == "" { continue } - parts := strings.SplitN(line, ", ", 3) - if len(parts) != 3 { + parts := strings.SplitN(line, ", ", 4) + if len(parts) < 3 { continue } idx, err := strconv.Atoi(strings.TrimSpace(parts[0])) @@ -246,10 +247,18 @@ func (s *System) ListNvidiaGPUs() ([]NvidiaGPU, error) { continue } memMB, _ := strconv.Atoi(strings.TrimSpace(parts[2])) + serial := "" + if len(parts) == 4 { + serial = strings.TrimSpace(parts[3]) + if strings.EqualFold(serial, "N/A") || strings.EqualFold(serial, "[N/A]") { + serial = "" + } + } gpus = append(gpus, NvidiaGPU{ Index: idx, Name: strings.TrimSpace(parts[1]), MemoryMB: memMB, + Serial: serial, }) } sort.Slice(gpus, func(i, j int) bool { diff --git a/audit/internal/webui/api.go b/audit/internal/webui/api.go index f88fed4..89a98df 100644 --- a/audit/internal/webui/api.go +++ b/audit/internal/webui/api.go @@ -128,7 +128,7 @@ func defaultTaskPriority(target string, params taskParams) int { return taskPriorityAudit case "nvidia-bench-perf", "nvidia-bench-power", "nvidia-bench-autotune": return taskPriorityBenchmark - case "nvidia-stress", "amd-stress", "memory-stress", "sat-stress", "platform-stress", "nvidia-compute", "scenario": + case "nvidia-stress", "amd-stress", "memory-stress", "sat-stress", "platform-stress", "fan", "nvidia-compute", "scenario": return taskPriorityBurn case "nvidia", "nvidia-targeted-stress", "nvidia-targeted-power", "nvidia-pulse", "nvidia-interconnect", "nvidia-bandwidth", "memory", "storage", "cpu", diff --git a/audit/internal/webui/api_metrics_network.go b/audit/internal/webui/api_metrics_network.go index 3ae78cc..a4cb43b 100644 --- a/audit/internal/webui/api_metrics_network.go +++ b/audit/internal/webui/api_metrics_network.go @@ -289,7 +289,7 @@ func (h *handler) handleAPIHardwareSummary(w http.ResponseWriter, _ *http.Reques } // handleAPIComponentDetail returns an HTML fragment describing the current and -// historical status for one component type (cpu, memory, storage, gpu, psu). +// historical status for one component type (cpu, memory, storage, gpu, psu, fan). func (h *handler) handleAPIComponentDetail(w http.ResponseWriter, r *http.Request) { compType := r.PathValue("type") var exact, prefixes []string @@ -315,6 +315,9 @@ func (h *handler) handleAPIComponentDetail(w http.ResponseWriter, r *http.Reques case "psu": title = "PSU" prefixes = []string{"psu:"} + case "fan": + title = "Fans" + prefixes = []string{"fan:"} case "raid": title = "RAID" prefixes = []string{"pcie:raid:"} diff --git a/audit/internal/webui/api_sat_runall.go b/audit/internal/webui/api_sat_runall.go index 21ce3c9..4fc03af 100644 --- a/audit/internal/webui/api_sat_runall.go +++ b/audit/internal/webui/api_sat_runall.go @@ -109,6 +109,11 @@ func (h *handler) planSATRunAll(ctx context.Context, req satRunAllRequest) ([]sa } else { skip("TPM: no TPM device on this host; check skipped") } + } else { + // Fan ceiling check runs on the Load tier only. It self-cancels as + // "not applicable" on a host with no fan sensors or no way to load + // the CPU/GPU, so it is safe to queue unconditionally here. + specs = append(specs, satRunAllSpec{target: "fan", params: taskParams{StressMode: true}}) } gp := h.opts.App.DetectGPUPresence() diff --git a/audit/internal/webui/api_sat_runall_test.go b/audit/internal/webui/api_sat_runall_test.go index 8fcdd2b..b0b0400 100644 --- a/audit/internal/webui/api_sat_runall_test.go +++ b/audit/internal/webui/api_sat_runall_test.go @@ -98,7 +98,7 @@ func TestPlanSATRunAllLoadOmitsReadOnlyTPMCheck(t *testing.T) { for _, s := range specs { targets = append(targets, s.target) } - if want := []string{"cpu", "memory", "storage"}; !reflect.DeepEqual(targets, want) { + if want := []string{"cpu", "memory", "storage", "fan"}; !reflect.DeepEqual(targets, want) { t.Fatalf("targets=%v want %v", targets, want) } for _, note := range notes { diff --git a/audit/internal/webui/gpu_picker.go b/audit/internal/webui/gpu_picker.go new file mode 100644 index 0000000..7f382ce --- /dev/null +++ b/audit/internal/webui/gpu_picker.go @@ -0,0 +1,71 @@ +package webui + +// Shared NVIDIA GPU selection picker. +// +// Every page that lets the operator pick GPUs (Load, Burn, Benchmark) renders +// the same row markup: "GPU N — · MiB · sn: ". That +// markup lives here once. Pages keep their own selection-note text, multi-GPU +// mode toggles and CSS-class prefixes, but call beeGpuPicker.render({...}) from +// their *RenderGPUList wrapper instead of hand-building each