package platform import ( "context" "encoding/json" "fmt" "math" "os" "os/exec" "path/filepath" "sort" "strconv" "strings" "sync" "time" ) // FanCheckOptions configures the fan-ceiling check: it drives CPU (+memory) // and, when present, GPU load to 100% simultaneously, then watches every fan // until none has climbed for PlateauHoldSec — at which point each fan is // considered to be at its physical ceiling and the observed peak is recorded. type FanCheckOptions struct { PlateauHoldSec int // a fan must not rise > PlateauDeltaRPM for this long to count as plateaued (default 60) PlateauDeltaRPM int // RPM increase that still counts as "climbing" (default 50) MinLoadSec int // never declare a plateau before this many seconds of load (default 90) MaxLoadSec int // hard cap on the load phase; finish (success) even if not every fan plateaued (default 900) RampConfirmRPM int // at least one fan must exceed baseline by this before a plateau is "real" (default 150) SizeMB int // GPU memory to allocate per GPU (0 = auto) GPUIndices []int // which GPU indices to load (empty = all detected) } // FanReading holds one fan sensor reading. type FanReading struct { Name string RPM float64 } // GPUStressMetric holds per-GPU metrics during the stress test. type GPUStressMetric struct { Index int TempC float64 UsagePct float64 PowerW float64 ClockMHz float64 Throttled bool // true if any throttle reason is active } // FanStressRow is one second-interval telemetry sample covering all monitored dimensions. type FanStressRow struct { TimestampUTC string ElapsedSec float64 Phase string // "baseline", "load1", "pause", "load2", "cooldown" GPUs []GPUStressMetric Fans []FanReading CPUMaxTempC float64 // highest CPU temperature from ipmitool / sensors SysPowerW float64 SysPowerSource string SysPowerMode string } type cachedPowerReading struct { Value float64 Source string Mode string Reason string UpdatedAt time.Time } type fanObservationState struct { MaxRPM map[string]float64 `json:"max_rpm"` } type fanPeakCandidate struct { FirstSeen time.Time RPM float64 } var ( systemPowerCacheMu sync.Mutex systemPowerCache cachedPowerReading fanObservationMu sync.Mutex fanObservation fanObservationState fanObservationInit bool fanPeakCandidates = make(map[string]fanPeakCandidate) ) const systemPowerHoldTTL = 15 * time.Second var fanObservationStatePath = "/var/log/bee-sat/fan-observation.json" const fanObservationMinPeakHold = time.Second func normalizeObservedFanMaxRPM(rpm float64) float64 { if rpm <= 0 { return 0 } return math.Ceil(rpm/1000.0) * 1000.0 } // RunFanCheck drives CPU (+memory) and, when a GPU is present, GPU load to // 100% simultaneously and watches every fan until none has climbed for // PlateauHoldSec. At that point each fan is taken to be at its physical // ceiling; the observed peak RPM is persisted (fanObservationStatePath, the // same store ObservedFanMaxRPM reads) so the topology view can size each fan // tile against a real maximum. // // Outcome: // - success ("ceiling found") once every fan plateaus, or when MaxLoadSec is // hit — a run that simply ran out of time still recorded the highest RPM // seen and is not a failure. // - a fan reading 0 RPM, or an IPMI status of cr/nr, while under full load is // a real defect → FAILED. // - if there is no way to load this box (no stressapptest/stress-ng and no // GPU burn tool) or no fan sensors are readable, the test cannot say // anything about the hardware and returns ErrTestNotApplicable so the task // is cancelled, not failed. // // No GPU is not an error: CPU/memory load alone is enough to exercise the // cooling loop on most platforms. func (s *System) RunFanCheck(ctx context.Context, baseDir string, opts FanCheckOptions, logFunc func(string)) (string, error) { if logFunc == nil { logFunc = func(string) {} } if baseDir == "" { baseDir = "/var/log/bee-sat" } applyFanCheckDefaults(&opts) baseFans, fanErr := sampleFanSpeeds() if len(baseFans) == 0 { return "", fmt.Errorf("no fan sensors readable via ipmitool or lm-sensors (%v): %w", fanErr, ErrTestNotApplicable) } baselineRPM := make(map[string]float64, len(baseFans)) for _, f := range baseFans { baselineRPM[f.Name] = f.RPM } vendor := s.DetectGPUVendor() haveGPU := vendor == "nvidia" || vendor == "amd" _, cpuPathErr := satLookPath("stressapptest") if cpuPathErr != nil { _, cpuPathErr = satLookPath("stress-ng") } haveCPU := cpuPathErr == nil if !haveCPU && !haveGPU { return "", fmt.Errorf("no load source: stressapptest/stress-ng missing and no NVIDIA/AMD GPU stress tool available: %w", ErrTestNotApplicable) } ts := time.Now().UTC().Format("20060102-150405") runDir := filepath.Join(baseDir, "fan-check-"+ts) if err := os.MkdirAll(runDir, 0755); err != nil { return "", err } verboseLog := filepath.Join(runDir, "verbose.log") appendSATVerboseLog(verboseLog, fmt.Sprintf("[%s] fan check start: %d fans, gpu=%s cpu=%v", time.Now().UTC().Format(time.RFC3339), len(baseFans), vendor, haveCPU)) logFunc(fmt.Sprintf("Fan check: %d fans; load = CPU/mem:%v + GPU:%s", len(baseFans), haveCPU, orNone(haveGPU, vendor))) // ── Load: every source runs at the same time, each in its own goroutine. // Sources can take different amounts of time to actually reach full load // (stressapptest is near-instant; a GPU burn kernel needs to compile and // ramp), so the plateau clock does not start until every launched source // has reported its process running, plus a fixed GPU ramp grace. loadCtx, loadCancel := context.WithTimeout(ctx, time.Duration(opts.MaxLoadSec)*time.Second) defer loadCancel() var loadWG sync.WaitGroup started := make(chan bool, 2) // true = source is running, false = failed to launch launched := 0 if haveCPU { launched++ loadWG.Add(1) go func() { defer loadWG.Done() cmd, err := buildCPUStressCmd(loadCtx) if err != nil { logFunc("CPU/memory load failed to start: " + err.Error()) appendSATVerboseLog(verboseLog, "cpu load start error: "+err.Error()) started <- false return } logFunc("CPU/memory load running (stressapptest)") started <- true _ = cmd.Wait() }() } if haveGPU { launched++ loadWG.Add(1) go func() { defer loadWG.Done() cmd := buildGPUStressCmd(loadCtx, vendor, opts.MaxLoadSec) if cmd == nil { logFunc("GPU load unavailable (no burn tool for " + vendor + ")") appendSATVerboseLog(verboseLog, "gpu load: no burn tool") started <- false return } logFunc("GPU load running (" + vendor + ")") started <- true _ = cmd.Wait() }() } start := time.Now() activeLoads := 0 for i := 0; i < launched; i++ { select { case ok := <-started: if ok { activeLoads++ } case <-ctx.Done(): } } if activeLoads == 0 { loadCancel() loadWG.Wait() return "", fmt.Errorf("every load source failed to start: %w", ErrTestNotApplicable) } readyAt := time.Now() if haveGPU { readyAt = readyAt.Add(20 * time.Second) // GPU kernel ramp grace } appendSATVerboseLog(verboseLog, fmt.Sprintf("%d load source(s) active; plateau clock effective from +%.0fs", activeLoads, readyAt.Sub(start).Seconds())) // ── Sample loop: one row/second, per-fan plateau tracking. type fanState struct { peak float64 lastRiseSec float64 } fanBy := map[string]*fanState{} var rows []FanStressRow rampConfirmed := false plateauReached := false aborted := false ticker := time.NewTicker(time.Second) defer ticker.Stop() loop: for { select { case <-ctx.Done(): aborted = true break loop case <-loadCtx.Done(): break loop // MaxLoadSec reached case <-ticker.C: } elapsed := time.Since(start).Seconds() row := sampleFanStressRow(opts.GPUIndices, "load", elapsed) rows = append(rows, row) for _, f := range row.Fans { st := fanBy[f.Name] if st == nil { fanBy[f.Name] = &fanState{peak: f.RPM, lastRiseSec: elapsed} continue } if f.RPM > st.peak { if f.RPM-st.peak > float64(opts.PlateauDeltaRPM) { st.lastRiseSec = elapsed } st.peak = f.RPM } if f.RPM >= baselineRPM[f.Name]+float64(opts.RampConfirmRPM) { rampConfirmed = true } } if elapsed >= float64(opts.MinLoadSec) && time.Since(readyAt) >= time.Duration(opts.PlateauHoldSec)*time.Second && len(fanBy) > 0 { allFlat := true for _, st := range fanBy { if elapsed-st.lastRiseSec < float64(opts.PlateauHoldSec) { allFlat = false break } } if allFlat && rampConfirmed { plateauReached = true logFunc(fmt.Sprintf("All %d fans plateaued at %.0fs of load", len(fanBy), elapsed)) break loop } } } loadCancel() loadWG.Wait() if aborted && ctx.Err() != nil { _ = os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte("run_at_utc="+time.Now().UTC().Format(time.RFC3339)+"\noverall_status=UNKNOWN\naborted=true\n"), 0644) return runDir, ctx.Err() } // ── Verdict. statuses := readFanStatuses() var summary strings.Builder fmt.Fprintf(&summary, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339)) fmt.Fprintf(&summary, "fans_total=%d\n", len(baseFans)) fmt.Fprintf(&summary, "active_load_sources=%d\n", activeLoads) fmt.Fprintf(&summary, "gpu_vendor=%s\n", orNone(haveGPU, vendor)) fmt.Fprintf(&summary, "plateau_reached=%v\n", plateauReached) fmt.Fprintf(&summary, "ramp_confirmed=%v\n", rampConfirmed) if len(rows) > 0 { fmt.Fprintf(&summary, "load_duration_sec=%.0f\n", rows[len(rows)-1].ElapsedSec) } fmt.Fprintf(&summary, "max_gpu_temp_c=%.1f\n", analyzeMaxTemp(rows, func(r FanStressRow) float64 { var m float64 for _, g := range r.GPUs { if g.TempC > m { m = g.TempC } } return m })) fmt.Fprintf(&summary, "max_cpu_temp_c=%.1f\n", analyzeMaxTemp(rows, func(r FanStressRow) float64 { return r.CPUMaxTempC })) stats := satStats{} names := make([]string, 0, len(baselineRPM)) for n := range baselineRPM { names = append(names, n) } sort.Strings(names) for _, name := range names { peak := baselineRPM[name] if st := fanBy[name]; st != nil { peak = st.peak } st := strings.ToLower(strings.TrimSpace(statuses[name])) bad := peak <= 0 || st == "cr" || st == "nr" key := sanitizeSummaryKey(name) fmt.Fprintf(&summary, "fan_%s_baseline_rpm=%.0f\n", key, baselineRPM[name]) fmt.Fprintf(&summary, "fan_%s_max_rpm=%.0f\n", key, peak) if bad { reason := "0 RPM under load" if st == "cr" || st == "nr" { reason = "IPMI status " + st } fmt.Fprintf(&summary, "fan_%s_status=FAILED (%s)\n", key, reason) logFunc(fmt.Sprintf("FAIL %s: %s", name, reason)) stats.Failed++ } else { fmt.Fprintf(&summary, "fan_%s_status=OK\n", key) stats.OK++ } } writeSATStats(&summary, stats) _ = WriteFanStressCSV(filepath.Join(runDir, "metrics.csv"), rows, opts.GPUIndices) _ = WriteFanSensorsCSV(filepath.Join(runDir, "fan-sensors.csv"), rows) if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary.String()), 0644); err != nil { return "", err } return runDir, nil } func applyFanCheckDefaults(o *FanCheckOptions) { if o.PlateauHoldSec <= 0 { o.PlateauHoldSec = 60 } if o.PlateauDeltaRPM <= 0 { o.PlateauDeltaRPM = 50 } if o.MinLoadSec <= 0 { o.MinLoadSec = 90 } if o.MaxLoadSec <= 0 { o.MaxLoadSec = 900 } if o.RampConfirmRPM <= 0 { o.RampConfirmRPM = 150 } if o.MinLoadSec < o.PlateauHoldSec { o.MinLoadSec = o.PlateauHoldSec } if o.MaxLoadSec <= o.MinLoadSec { o.MaxLoadSec = o.MinLoadSec + o.PlateauHoldSec } } func orNone(present bool, v string) string { if present && v != "" { return v } return "none" } // sanitizeSummaryKey makes a fan sensor name safe as a summary.txt key // fragment (keys are parsed by splitting on '=' and whitespace). func sanitizeSummaryKey(name string) string { var b strings.Builder for _, r := range name { switch { case r >= 'A' && r <= 'Z', r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '-', r == '_', r == '.': b.WriteRune(r) default: b.WriteByte('_') } } return b.String() } // readFanStatuses returns the per-fan IPMI status word ("ok", "cr", "nr", ...) // from "ipmitool sdr type Fan". Empty map when ipmitool is unavailable. func readFanStatuses() map[string]string { out, err := exec.Command("ipmitool", "sdr", "type", "Fan").Output() if err != nil { return nil } m := map[string]string{} for _, line := range strings.Split(string(out), "\n") { parts := strings.Split(line, "|") if len(parts) < 3 { continue } name := strings.TrimSpace(parts[0]) if name == "" { continue } m[name] = strings.ToLower(strings.TrimSpace(parts[2])) } return m } // ResolveFanMaxRPM returns, for every fan name in current (name -> current // RPM), the RPM to treat as that fan's 100% reference. Preference order: // 1. the persisted observed peak (fanObservationStatePath), written by // RunFanCheck and by live-metrics sampling under load; // 2. the largest peak observed on any peer fan (keeps a group visually // consistent when only some fans have a recorded peak); // 3. the fan's own current RPM (so a tile is never sized against zero). // // The fallback lives here, not in the view, so every consumer of a fan // maximum applies the same rule. func ResolveFanMaxRPM(current map[string]float64) map[string]float64 { persisted := readPersistedFanMaxRPM() peerMax := 0.0 for _, v := range persisted { if v > peerMax { peerMax = v } } out := make(map[string]float64, len(current)) for name, rpm := range current { switch { case persisted[name] > 0: out[name] = persisted[name] case peerMax > 0: out[name] = peerMax default: out[name] = rpm } } return out } // sampleFanStressRow collects all metrics for one telemetry sample. func sampleFanStressRow(gpuIndices []int, phase string, elapsed float64) FanStressRow { row := FanStressRow{ TimestampUTC: time.Now().UTC().Format(time.RFC3339), ElapsedSec: elapsed, Phase: phase, } row.GPUs = sampleGPUStressMetrics(gpuIndices) row.Fans, _ = sampleFanSpeeds() row.CPUMaxTempC = sampleCPUMaxTemp() row.SysPowerW, row.SysPowerSource, row.SysPowerMode = sampleSystemPowerResolved() return row } // sampleGPUStressMetrics queries nvidia-smi for temperature, utilization, power, // clock frequency, and active throttle reasons for each GPU. func sampleGPUStressMetrics(gpuIndices []int) []GPUStressMetric { args := []string{ "--query-gpu=index,temperature.gpu,utilization.gpu,power.draw,clocks.current.graphics,clocks_throttle_reasons.active", "--format=csv,noheader,nounits", } if len(gpuIndices) > 0 { ids := make([]string, len(gpuIndices)) for i, idx := range gpuIndices { ids[i] = strconv.Itoa(idx) } args = append([]string{"--id=" + strings.Join(ids, ",")}, args...) } out, err := exec.Command("nvidia-smi", args...).Output() if err != nil { return nil } var metrics []GPUStressMetric for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { line = strings.TrimSpace(line) if line == "" { continue } parts := strings.Split(line, ", ") if len(parts) < 6 { continue } idx, _ := strconv.Atoi(strings.TrimSpace(parts[0])) throttleVal := strings.TrimSpace(parts[5]) // Throttled if active reasons bitmask is non-zero. throttled := throttleVal != "0x0000000000000000" && throttleVal != "0x0" && throttleVal != "0" && throttleVal != "" && throttleVal != "N/A" metrics = append(metrics, GPUStressMetric{ Index: idx, TempC: parseGPUFloat(parts[1]), UsagePct: parseGPUFloat(parts[2]), PowerW: parseGPUFloat(parts[3]), ClockMHz: parseGPUFloat(parts[4]), Throttled: throttled, }) } return metrics } // sampleFanSpeeds reads fan RPM values from ipmitool sdr. func sampleFanSpeeds() ([]FanReading, error) { out, err := exec.Command("ipmitool", "sdr", "type", "Fan").Output() if err == nil { if fans := parseFanSpeeds(string(out)); len(fans) > 0 { updateFanObservation(fans, time.Now()) return fans, nil } } fans, sensorsErr := sampleFanSpeedsViaSensorsJSON() if len(fans) > 0 { updateFanObservation(fans, time.Now()) return fans, nil } if err != nil { return nil, err } return nil, sensorsErr } // readPersistedFanMaxRPM reads fanObservationStatePath and returns its // sanitized {fan name -> observed peak RPM} map (empty names / non-positive // values dropped). Returns an empty map when the file is missing or unparsable. func readPersistedFanMaxRPM() map[string]float64 { out := map[string]float64{} raw, err := os.ReadFile(fanObservationStatePath) if err != nil || len(raw) == 0 { return out } var persisted fanObservationState if json.Unmarshal(raw, &persisted) != nil { return out } for name, rpm := range persisted.MaxRPM { name = strings.TrimSpace(name) if name == "" || rpm <= 0 { continue } out[name] = rpm } return out } // ObservedFanMaxRPM returns the per-fan observed peak RPM map persisted by // fan-stress SAT runs, or nil if none is recorded yet. It reads the file // directly without touching the in-process observation cache or its lock, so // read-only consumers (the /topo web view) can call it without perturbing a // concurrent SAT run's peak tracking. func ObservedFanMaxRPM() map[string]float64 { out := readPersistedFanMaxRPM() if len(out) == 0 { return nil } return out } func loadFanObservationLocked() { if fanObservationInit { return } fanObservationInit = true fanObservation.MaxRPM = readPersistedFanMaxRPM() } func saveFanObservationLocked() { if len(fanObservation.MaxRPM) == 0 { return } dir := filepath.Dir(fanObservationStatePath) if dir == "" || dir == "." { dir = "/var/log/bee-sat" } if err := os.MkdirAll(dir, 0755); err != nil { return } raw, err := json.MarshalIndent(fanObservation, "", " ") if err != nil { return } _ = os.WriteFile(fanObservationStatePath, raw, 0644) } func updateFanObservation(fans []FanReading, now time.Time) { if len(fans) == 0 { return } fanObservationMu.Lock() defer fanObservationMu.Unlock() loadFanObservationLocked() changed := false for _, fan := range fans { name := strings.TrimSpace(fan.Name) if name == "" || fan.RPM <= 0 { continue } currentMax := fanObservation.MaxRPM[name] if fan.RPM <= currentMax { delete(fanPeakCandidates, name) continue } if cand, ok := fanPeakCandidates[name]; ok { if now.Sub(cand.FirstSeen) >= fanObservationMinPeakHold { newMax := math.Max(cand.RPM, fan.RPM) if newMax > currentMax { fanObservation.MaxRPM[name] = normalizeObservedFanMaxRPM(newMax) changed = true } delete(fanPeakCandidates, name) continue } if fan.RPM > cand.RPM { fanPeakCandidates[name] = fanPeakCandidate{FirstSeen: cand.FirstSeen, RPM: fan.RPM} } continue } fanPeakCandidates[name] = fanPeakCandidate{FirstSeen: now, RPM: fan.RPM} } if changed { saveFanObservationLocked() } } func estimateFanDutyCyclePctFromObservation(fans []FanReading) (float64, bool) { if len(fans) == 0 { return 0, false } fanObservationMu.Lock() defer fanObservationMu.Unlock() loadFanObservationLocked() var samples []float64 for _, fan := range fans { name := strings.TrimSpace(fan.Name) if name == "" || fan.RPM <= 0 { continue } maxRPM := fanObservation.MaxRPM[name] if maxRPM <= 0 { continue } pct := fan.RPM / maxRPM * 100.0 if pct > 100 { pct = 100 } if pct < 0 { pct = 0 } samples = append(samples, pct) } if len(samples) == 0 { return 0, false } return benchmarkMean(samples), true } // parseFanSpeeds parses "ipmitool sdr type Fan" output. // Handles two formats: // // Old: "FAN1 | 2400.000 | RPM | ok" (value in col[1], unit in col[2]) // New: "FAN1 | 41h | ok | 29.1 | 4340 RPM" (value+unit combined in last col) func parseFanSpeeds(raw string) []FanReading { var fans []FanReading for _, line := range strings.Split(strings.TrimSpace(raw), "\n") { parts := strings.Split(line, "|") if len(parts) < 2 { continue } name := strings.TrimSpace(parts[0]) // Find the first field that contains "RPM" (either as a standalone unit or inline) rpmVal := 0.0 found := false for _, p := range parts[1:] { p = strings.TrimSpace(p) if !strings.Contains(strings.ToUpper(p), "RPM") { continue } if strings.EqualFold(p, "RPM") { continue // unit-only column in old format; value is in previous field } val, err := parseFanRPMValue(p) if err == nil { rpmVal = val found = true break } } // Old format: unit "RPM" is in col[2], value is in col[1] if !found && len(parts) >= 3 && strings.EqualFold(strings.TrimSpace(parts[2]), "RPM") { valStr := strings.TrimSpace(parts[1]) if !strings.EqualFold(valStr, "na") && !strings.EqualFold(valStr, "disabled") && valStr != "" { if val, err := parseFanRPMValue(valStr); err == nil { rpmVal = val found = true } } } if !found { continue } fans = append(fans, FanReading{Name: name, RPM: rpmVal}) } return fans } func parseFanRPMValue(raw string) (float64, error) { fields := strings.Fields(strings.TrimSpace(strings.ReplaceAll(raw, ",", ""))) if len(fields) == 0 { return 0, strconv.ErrSyntax } return strconv.ParseFloat(fields[0], 64) } func sampleFanSpeedsViaSensorsJSON() ([]FanReading, error) { out, err := exec.Command("sensors", "-j").Output() if err != nil || len(out) == 0 { return nil, err } var doc map[string]map[string]any if err := json.Unmarshal(out, &doc); err != nil { return nil, err } chips := make([]string, 0, len(doc)) for chip := range doc { chips = append(chips, chip) } sort.Strings(chips) var fans []FanReading seen := map[string]struct{}{} for _, chip := range chips { features := doc[chip] names := make([]string, 0, len(features)) for name := range features { names = append(names, name) } sort.Strings(names) for _, name := range names { feature, ok := features[name].(map[string]any) if !ok { continue } rpm, ok := firstFanInputValue(feature) if !ok || rpm <= 0 { continue } label := strings.TrimSpace(name) if chip != "" && !strings.Contains(strings.ToLower(label), strings.ToLower(chip)) { label = chip + " / " + label } if _, ok := seen[label]; ok { continue } seen[label] = struct{}{} fans = append(fans, FanReading{Name: label, RPM: rpm}) } } return fans, nil } func sampleFanDutyCyclePctFromFans(fans []FanReading) (float64, bool, bool) { if len(fans) == 0 { return 0, false, false } if pct, ok := estimateFanDutyCyclePctFromObservation(fans); ok { return pct, true, true } return 0, false, false } func parseFanDutyCyclePctSensorsJSON(raw []byte) (float64, bool) { var doc map[string]map[string]any if err := json.Unmarshal(raw, &doc); err != nil { return 0, false } var samples []float64 for _, features := range doc { for name, feature := range features { if strings.EqualFold(name, "Adapter") { continue } featureMap, ok := feature.(map[string]any) if !ok { continue } if duty, ok := firstFanDutyValue(name, featureMap); ok { samples = append(samples, duty) } } } if len(samples) == 0 { return 0, false } return benchmarkMean(samples), true } func firstFanDutyValue(featureName string, feature map[string]any) (float64, bool) { featureName = strings.ToLower(strings.TrimSpace(featureName)) if strings.Contains(featureName, "enable") || strings.Contains(featureName, "mode") || strings.Contains(featureName, "alarm") { return 0, false } if strings.Contains(featureName, "pwm") { for _, key := range []string{"input", "value", "current"} { if value, ok := feature[key]; ok { if duty, parsed := parseFanDutyValue(value); parsed { return duty, true } } } } keys := make([]string, 0, len(feature)) for key := range feature { keys = append(keys, key) } sort.Strings(keys) for _, key := range keys { lower := strings.ToLower(key) if !strings.Contains(lower, "pwm") { continue } if strings.Contains(lower, "enable") || strings.Contains(lower, "mode") || strings.Contains(lower, "alarm") { continue } if duty, parsed := parseFanDutyValue(feature[key]); parsed { return duty, true } } return 0, false } func parseFanDutyValue(value any) (float64, bool) { switch v := value.(type) { case float64: return normalizePWMAsDutyPct(v) case string: if f, err := strconv.ParseFloat(strings.TrimSpace(v), 64); err == nil { return normalizePWMAsDutyPct(f) } } return 0, false } func normalizePWMAsDutyPct(raw float64) (float64, bool) { if raw < 0 { return 0, false } if raw <= 100 { return raw, true } if raw <= 255 { return raw / 255.0 * 100.0, true } return 0, false } func firstFanInputValue(feature map[string]any) (float64, bool) { return firstSensorInputValue(feature, "fan") } // sampleCPUMaxTemp returns the highest CPU/inlet temperature from ipmitool or sensors. func sampleCPUMaxTemp() float64 { out, err := exec.Command("ipmitool", "sdr", "type", "Temperature").Output() if err != nil { return sampleCPUTempViaSensors() } return parseIPMIMaxTemp(string(out)) } // parseIPMIMaxTemp extracts the maximum temperature from "ipmitool sdr type Temperature". func parseIPMIMaxTemp(raw string) float64 { var max float64 for _, line := range strings.Split(strings.TrimSpace(raw), "\n") { parts := strings.Split(line, "|") if len(parts) < 3 { continue } unit := strings.TrimSpace(parts[2]) if !strings.Contains(strings.ToLower(unit), "degrees") { continue } valStr := strings.TrimSpace(parts[1]) if strings.EqualFold(valStr, "na") || valStr == "" { continue } val, err := strconv.ParseFloat(valStr, 64) if err != nil { continue } if val > max { max = val } } return max } // sampleCPUTempViaSensors falls back to lm-sensors when ipmitool is unavailable. func sampleCPUTempViaSensors() float64 { out, err := exec.Command("sensors", "-u").Output() if err != nil { return 0 } var max float64 for _, line := range strings.Split(string(out), "\n") { line = strings.TrimSpace(line) fields := strings.Fields(line) if len(fields) < 2 { continue } if !strings.HasSuffix(fields[0], "_input:") { continue } val, err := strconv.ParseFloat(fields[1], 64) if err != nil { continue } if val > 0 && val < 150 && val > max { max = val } } return max } // sampleSystemPowerResolved reads system power via the global autotune source, // falling back to the historical heuristic before autotune or when degraded. func sampleSystemPowerResolved() (float64, string, string) { now := time.Now() current, decision, err := SampleSystemPowerResolved("") systemPowerCacheMu.Lock() defer systemPowerCacheMu.Unlock() if err != nil { current = 0 } value, updated := effectiveSystemPowerReading(systemPowerCache, current, decision.EffectiveSource, decision.Mode, decision.Reason, now) systemPowerCache = updated return value, updated.Source, updated.Mode } // parseDCMIPowerReading extracts the instantaneous power reading from ipmitool dcmi output. // Sample: " Instantaneous power reading: 500 Watts" func parseDCMIPowerReading(raw string) float64 { for _, line := range strings.Split(raw, "\n") { if !strings.Contains(strings.ToLower(line), "instantaneous") { continue } parts := strings.Fields(line) for i, p := range parts { if strings.EqualFold(p, "Watts") && i > 0 { val, err := strconv.ParseFloat(parts[i-1], 64) if err == nil { return val } } } } return 0 } func effectiveSystemPowerReading(cache cachedPowerReading, current float64, source, mode, reason string, now time.Time) (float64, cachedPowerReading) { if current > 0 { cache = cachedPowerReading{Value: current, Source: source, Mode: mode, Reason: reason, UpdatedAt: now} return current, cache } if cache.Value > 0 && !cache.UpdatedAt.IsZero() && now.Sub(cache.UpdatedAt) <= systemPowerHoldTTL { return cache.Value, cache } return 0, cache } // analyzeMaxTemp returns the maximum value of the given extractor across all rows. func analyzeMaxTemp(rows []FanStressRow, extract func(FanStressRow) float64) float64 { var max float64 for _, row := range rows { if v := extract(row); v > max { max = v } } return max } // WriteFanStressCSV writes the wide-format metrics CSV with one row per second. // GPU columns are generated per index in gpuIndices order. func WriteFanStressCSV(path string, rows []FanStressRow, gpuIndices []int) error { if len(rows) == 0 { return os.WriteFile(path, []byte("no data\n"), 0644) } var b strings.Builder // Header: fixed system columns + per-GPU columns. b.WriteString("timestamp_utc,elapsed_sec,phase,fan_avg_rpm,fan_min_rpm,fan_max_rpm,cpu_max_temp_c,sys_power_w") for _, idx := range gpuIndices { fmt.Fprintf(&b, ",gpu%d_temp_c,gpu%d_usage_pct,gpu%d_power_w,gpu%d_clock_mhz,gpu%d_throttled", idx, idx, idx, idx, idx) } b.WriteRune('\n') for _, row := range rows { favg, fmin, fmax := fanRPMStats(row.Fans) fmt.Fprintf(&b, "%s,%.1f,%s,%.0f,%.0f,%.0f,%.1f,%.1f", row.TimestampUTC, row.ElapsedSec, row.Phase, favg, fmin, fmax, row.CPUMaxTempC, row.SysPowerW, ) gpuByIdx := make(map[int]GPUStressMetric, len(row.GPUs)) for _, g := range row.GPUs { gpuByIdx[g.Index] = g } for _, idx := range gpuIndices { g := gpuByIdx[idx] throttled := 0 if g.Throttled { throttled = 1 } fmt.Fprintf(&b, ",%.1f,%.1f,%.1f,%.0f,%d", g.TempC, g.UsagePct, g.PowerW, g.ClockMHz, throttled) } b.WriteRune('\n') } return os.WriteFile(path, []byte(b.String()), 0644) } // WriteFanSensorsCSV writes individual fan sensor readings in long (tidy) format. func WriteFanSensorsCSV(path string, rows []FanStressRow) error { var b strings.Builder b.WriteString("timestamp_utc,elapsed_sec,phase,fan_name,rpm\n") for _, row := range rows { for _, f := range row.Fans { fmt.Fprintf(&b, "%s,%.1f,%s,%s,%.0f\n", row.TimestampUTC, row.ElapsedSec, row.Phase, f.Name, f.RPM) } } return os.WriteFile(path, []byte(b.String()), 0644) } // fanRPMStats computes average, min, max RPM across all fans in a sample row. func fanRPMStats(fans []FanReading) (avg, min, max float64) { if len(fans) == 0 { return 0, 0, 0 } min = fans[0].RPM max = fans[0].RPM var total float64 for _, f := range fans { total += f.RPM if f.RPM < min { min = f.RPM } if f.RPM > max { max = f.RPM } } return total / float64(len(fans)), min, max }