package platform import ( "context" "encoding/json" "fmt" "os" "os/exec" "path/filepath" "sort" "strconv" "strings" "sync" "syscall" "time" ) // FanCheckOptions configures the fan-ceiling check: it drives CPU (+memory) // and, when present, GPU load to 100% simultaneously, then watches every fan // until none has climbed for PlateauHoldSec — at which point each fan is // considered to be at its physical ceiling and the observed peak is recorded. type FanCheckOptions struct { PlateauHoldSec int // a fan must not rise > PlateauDeltaRPM for this long to count as plateaued (default 60) PlateauDeltaRPM int // RPM increase that still counts as "climbing" (default 50) MinLoadSec int // never declare a plateau before this many seconds of load (default 90) MaxLoadSec int // hard cap on the load phase; finish (success) even if not every fan plateaued (default 900) RampConfirmRPM int // at least one fan must exceed baseline by this before a plateau is "real" (default 150) SizeMB int // GPU memory to allocate per GPU (0 = auto) GPUIndices []int // which GPU indices to load (empty = all detected) } // FanReading holds one fan sensor reading. type FanReading struct { Name string `json:"name"` RPM float64 `json:"rpm"` } type cachedPowerReading struct { Value float64 Source string Mode string Reason string UpdatedAt time.Time } const systemPowerHoldTTL = 15 * time.Second // fanPeaks is the observed top-RPM store — the "autotune" for fan ceilings. // Any full-load run (Fan Ceiling Check, burn, thermal cycling) feeds it via // updateFanObservation; ResolveFanMaxRPM / ObservedFanMaxRPM read it back. var fanPeaks = &observedPeakStore{ path: "/var/log/bee-sat/fan-observation.json", jsonKey: "max_rpm", roundUp: 1000, minHold: time.Second, } // RunFanCheck drives CPU (+memory) and, when a GPU is present, GPU load to // 100% simultaneously and watches every fan until none has climbed for // PlateauHoldSec. At that point each fan is taken to be at its physical // ceiling; the observed peak RPM is persisted (fanObservationStatePath, the // same store ObservedFanMaxRPM reads) so the topology view can size each fan // tile against a real maximum. // // Outcome: // - success ("ceiling found") once every fan plateaus, or when MaxLoadSec is // hit — a run that simply ran out of time still recorded the highest RPM // seen and is not a failure. // - a fan reading 0 RPM, or an IPMI status of cr/nr, while under full load is // a real defect → FAILED. // - if there is no way to load this box (no stressapptest/stress-ng and no // GPU burn tool) or no fan sensors are readable, the test cannot say // anything about the hardware and returns ErrTestNotApplicable so the task // is cancelled, not failed. // // No GPU is not an error: CPU/memory load alone is enough to exercise the // cooling loop on most platforms. func (s *System) RunFanCheck(ctx context.Context, baseDir string, opts FanCheckOptions, logFunc func(string)) (string, error) { if logFunc == nil { logFunc = func(string) {} } if baseDir == "" { baseDir = "/var/log/bee-sat" } applyFanCheckDefaults(&opts) baseFans, fanErr := sampleFanSpeeds() if len(baseFans) == 0 { return "", fmt.Errorf("no fan sensors readable via ipmitool or lm-sensors (%v): %w", fanErr, ErrTestNotApplicable) } baselineRPM := make(map[string]float64, len(baseFans)) for _, f := range baseFans { baselineRPM[f.Name] = f.RPM } vendor := s.DetectGPUVendor() haveGPU := vendor == "nvidia" || vendor == "amd" _, cpuPathErr := satLookPath("stressapptest") if cpuPathErr != nil { _, cpuPathErr = satLookPath("stress-ng") } haveCPU := cpuPathErr == nil if !haveCPU && !haveGPU { return "", fmt.Errorf("no load source: stressapptest/stress-ng missing and no NVIDIA/AMD GPU stress tool available: %w", ErrTestNotApplicable) } ts := time.Now().UTC().Format("20060102-150405") runDir := filepath.Join(baseDir, "fan-check-"+ts) if err := os.MkdirAll(runDir, 0755); err != nil { return "", err } verboseLog := filepath.Join(runDir, "verbose.log") appendSATVerboseLog(verboseLog, fmt.Sprintf("[%s] fan check start: %d fans, gpu=%s cpu=%v", time.Now().UTC().Format(time.RFC3339), len(baseFans), vendor, haveCPU)) logFunc(fmt.Sprintf("Fan check: %d fans; load = CPU/mem:%v + GPU:%s", len(baseFans), haveCPU, orNone(haveGPU, vendor))) // ── Load: every source runs at the same time, each in its own goroutine. // GPU load is the hottest sustained NVIDIA load we have — dcgmproftester // -t 1004 / targeted_power, the same engine the Power/Thermal Fit // benchmark uses (resolveBenchmarkPowerLoadCommand) — not the // compute-throughput bee-gpu-burn, which tops out well below TDP and so // never demands the fans' true ceiling. // // Sources reach full load at different times (stressapptest is instant; a // dcgmproftester kernel compiles and ramps), so the plateau clock does not // start until every launched source reports its process running, plus a // fixed GPU ramp grace. loadCtx, loadCancel := context.WithTimeout(ctx, time.Duration(opts.MaxLoadSec)*time.Second) defer loadCancel() var loadWG sync.WaitGroup started := make(chan bool, 2) // true = source is running, false = failed to launch launched := 0 if haveCPU { launched++ loadWG.Add(1) go func() { defer loadWG.Done() cmd, err := buildCPUStressCmd(loadCtx) if err != nil { logFunc("CPU/memory load failed to start: " + err.Error()) appendSATVerboseLog(verboseLog, "cpu load start error: "+err.Error()) started <- false return } logFunc("CPU/memory load running (stressapptest)") started <- true _ = cmd.Wait() }() } if haveGPU { launched++ loadWG.Add(1) go func() { defer loadWG.Done() cmd, label, err := buildFanCheckGPULoadCmd(loadCtx, vendor, opts.MaxLoadSec, opts.GPUIndices) if err != nil || cmd == nil { logFunc("GPU load unavailable: " + errString(err)) appendSATVerboseLog(verboseLog, "gpu load unavailable: "+errString(err)) started <- false return } if err := cmd.Start(); err != nil { logFunc("GPU load failed to start: " + err.Error()) appendSATVerboseLog(verboseLog, "gpu load start error: "+err.Error()) started <- false return } logFunc("GPU load running (" + label + ")") started <- true _ = cmd.Wait() }() } start := time.Now() activeLoads := 0 for i := 0; i < launched; i++ { select { case ok := <-started: if ok { activeLoads++ } case <-ctx.Done(): } } if activeLoads == 0 { loadCancel() loadWG.Wait() return "", fmt.Errorf("every load source failed to start: %w", ErrTestNotApplicable) } readyAt := time.Now() if haveGPU { readyAt = readyAt.Add(20 * time.Second) // GPU kernel ramp grace } appendSATVerboseLog(verboseLog, fmt.Sprintf("%d load source(s) active; plateau clock effective from +%.0fs", activeLoads, readyAt.Sub(start).Seconds())) // ── Sample loop with IPMI-hang protection. // // Under full load ipmitool over KCS can take tens of seconds per call or // wedge outright. So: every fan read is time-boxed (readFansBounded runs // it in a goroutine we abandon on timeout — a wedged KCS read can never // block this loop), and the polling interval backs off geometrically when // reads are slow and tightens again when they recover. A plateau is only // declared while telemetry is healthy (interval near the floor); a // degraded run just rides out to MaxLoadSec and records the peak it saw. const ( fanPollFloor = 1 * time.Second fanPollCeil = 30 * time.Second fanReadTMO = 8 * time.Second ) type fanState struct { peak float64 lastRiseSec float64 } fanBy := map[string]*fanState{} rampConfirmed := false plateauReached := false aborted := false degraded := false goodSamples := 0 poll := fanPollFloor // PSU peak draw, sampled at a slow cadence off the same loop — the fan // check already drives the box to full power, so it is also the right run // to observe what each PSU tops out at (updatePSUObservation persists it). psuPeakW := map[int]float64{} lastPSUSec := -1e9 csvPath := filepath.Join(runDir, "fan-sensors.csv") _ = os.WriteFile(csvPath, []byte("elapsed_sec,fan_name,rpm\n"), 0644) csvFile, _ := os.OpenFile(csvPath, os.O_APPEND|os.O_WRONLY, 0644) if csvFile != nil { defer csvFile.Close() } loop: for { select { case <-ctx.Done(): aborted = true break loop case <-loadCtx.Done(): break loop // MaxLoadSec reached case <-time.After(poll): } elapsed := time.Since(start).Seconds() readStart := time.Now() fans, ok := readFansBounded(fanReadTMO) readDur := time.Since(readStart) // Adapt the interval to how ipmitool is behaving. switch { case !ok || readDur > fanReadTMO*3/4: if poll < fanPollCeil { poll = minDuration(poll*2, fanPollCeil) degraded = true appendSATVerboseLog(verboseLog, fmt.Sprintf("[%.0fs] ipmitool slow (%.1fs, ok=%v) — polling backed off to %s", elapsed, readDur.Seconds(), ok, poll)) logFunc(fmt.Sprintf("IPMI slow — fan polling backed off to %s", poll)) } case poll > fanPollFloor && readDur < fanPollFloor: poll = maxDuration(poll*2/3, fanPollFloor) } if !ok { continue } goodSamples++ if elapsed-lastPSUSec >= 15 { lastPSUSec = elapsed if ps, ok := readPSUPowerBounded(fanReadTMO); ok { for i, p := range ps { if p.PowerW > psuPeakW[i] { psuPeakW[i] = p.PowerW } } } } for _, f := range fans { if csvFile != nil { fmt.Fprintf(csvFile, "%.0f,%s,%.0f\n", elapsed, f.Name, f.RPM) } st := fanBy[f.Name] if st == nil { fanBy[f.Name] = &fanState{peak: f.RPM, lastRiseSec: elapsed} continue } if f.RPM > st.peak { if f.RPM-st.peak > float64(opts.PlateauDeltaRPM) { st.lastRiseSec = elapsed } st.peak = f.RPM } if f.RPM >= baselineRPM[f.Name]+float64(opts.RampConfirmRPM) { rampConfirmed = true } } // Only trust a plateau while telemetry is healthy and we have enough // recent samples to have actually seen a flat window. healthy := poll <= 2*fanPollFloor && goodSamples >= 5 if healthy && rampConfirmed && elapsed >= float64(opts.MinLoadSec) && time.Since(readyAt) >= time.Duration(opts.PlateauHoldSec)*time.Second && len(fanBy) > 0 { allFlat := true for _, st := range fanBy { if elapsed-st.lastRiseSec < float64(opts.PlateauHoldSec) { allFlat = false break } } if allFlat { plateauReached = true logFunc(fmt.Sprintf("All %d fans plateaued at %.0fs of load", len(fanBy), elapsed)) break loop } } } loadCancel() loadWG.Wait() if aborted && ctx.Err() != nil { _ = os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte("run_at_utc="+time.Now().UTC().Format(time.RFC3339)+"\noverall_status=UNKNOWN\naborted=true\n"), 0644) return runDir, ctx.Err() } loadDur := time.Since(start).Seconds() // ── Verdict. statuses := readFanStatuses() var summary strings.Builder fmt.Fprintf(&summary, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339)) fmt.Fprintf(&summary, "fans_total=%d\n", len(baseFans)) fmt.Fprintf(&summary, "active_load_sources=%d\n", activeLoads) fmt.Fprintf(&summary, "gpu_vendor=%s\n", orNone(haveGPU, vendor)) fmt.Fprintf(&summary, "plateau_reached=%v\n", plateauReached) fmt.Fprintf(&summary, "ramp_confirmed=%v\n", rampConfirmed) fmt.Fprintf(&summary, "load_duration_sec=%.0f\n", loadDur) fmt.Fprintf(&summary, "fan_samples=%d\n", goodSamples) fmt.Fprintf(&summary, "telemetry_degraded=%v\n", degraded) if t := boundedGPUMaxTemp(opts.GPUIndices); t > 0 { fmt.Fprintf(&summary, "gpu_temp_c=%.0f\n", t) } if len(psuPeakW) > 0 { idx := make([]int, 0, len(psuPeakW)) for i := range psuPeakW { idx = append(idx, i) } sort.Ints(idx) for _, i := range idx { fmt.Fprintf(&summary, "psu_%d_peak_w=%.0f\n", i, psuPeakW[i]) } } stats := satStats{} names := make([]string, 0, len(baselineRPM)) for n := range baselineRPM { names = append(names, n) } sort.Strings(names) for _, name := range names { peak := baselineRPM[name] if st := fanBy[name]; st != nil { peak = st.peak } st := strings.ToLower(strings.TrimSpace(statuses[name])) bad := peak <= 0 || st == "cr" || st == "nr" key := sanitizeSummaryKey(name) fmt.Fprintf(&summary, "fan_%s_baseline_rpm=%.0f\n", key, baselineRPM[name]) fmt.Fprintf(&summary, "fan_%s_max_rpm=%.0f\n", key, peak) if bad { reason := "0 RPM under load" if st == "cr" || st == "nr" { reason = "IPMI status " + st } fmt.Fprintf(&summary, "fan_%s_status=FAILED (%s)\n", key, reason) logFunc(fmt.Sprintf("FAIL %s: %s", name, reason)) stats.Failed++ } else { fmt.Fprintf(&summary, "fan_%s_status=OK\n", key) stats.OK++ } } writeSATStats(&summary, stats) if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary.String()), 0644); err != nil { return "", err } return runDir, nil } func errString(err error) string { if err == nil { return "no GPU stress tool" } return err.Error() } func minDuration(a, b time.Duration) time.Duration { if a < b { return a } return b } func maxDuration(a, b time.Duration) time.Duration { if a > b { return a } return b } // readFansBounded runs "ipmitool sdr type Fan" but never blocks the caller // longer than timeout: the read happens in a goroutine that is abandoned if it // does not return in time (a KCS read wedged in uninterruptible I/O cannot be // killed, so we leave it and move on). ok=false means "no usable sample this // tick" — the caller must treat that as missing data, not as a flat fan. func readFansBounded(timeout time.Duration) ([]FanReading, bool) { type result struct { fans []FanReading ok bool } ch := make(chan result, 1) go func() { out, err := exec.Command("ipmitool", "sdr", "type", "Fan").Output() if err != nil { ch <- result{} return } fans := parseFanSpeeds(string(out)) if len(fans) == 0 { ch <- result{} return } ch <- result{fans, true} }() select { case r := <-ch: if r.ok { updateFanObservation(r.fans, time.Now()) } return r.fans, r.ok case <-time.After(timeout): return nil, false } } // readPSUPowerBounded is readFansBounded's PSU sibling: a time-boxed, // abandonable "ipmitool sdr" read of per-PSU power (samplePSUPower also feeds // the observed-capacity store). Used at a slow cadence during the fan check so // the same max-load run that finds fan ceilings also records peak PSU draw. func readPSUPowerBounded(timeout time.Duration) ([]PSUReading, bool) { ch := make(chan []PSUReading, 1) go func() { ch <- samplePSUPower() }() select { case ps := <-ch: return ps, len(ps) > 0 case <-time.After(timeout): return nil, false } } // boundedGPUMaxTemp returns the hottest GPU temperature via a single // time-boxed nvidia-smi call, or 0 if unavailable. func boundedGPUMaxTemp(gpuIndices []int) float64 { ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second) defer cancel() args := []string{"--query-gpu=temperature.gpu", "--format=csv,noheader,nounits"} if len(gpuIndices) > 0 { ids := make([]string, len(gpuIndices)) for i, idx := range gpuIndices { ids[i] = strconv.Itoa(idx) } args = append([]string{"--id=" + strings.Join(ids, ",")}, args...) } out, err := exec.CommandContext(ctx, "nvidia-smi", args...).Output() if err != nil { return 0 } var max float64 for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { if v, err := strconv.ParseFloat(strings.TrimSpace(line), 64); err == nil && v > max { max = v } } return max } // buildFanCheckGPULoadCmd builds the hottest sustained GPU load for the fan // check. NVIDIA uses the Power/Thermal Fit engine (dcgmproftester -t 1004 / // targeted_power); AMD uses the RVS gst stressor. func buildFanCheckGPULoadCmd(ctx context.Context, vendor string, durSec int, gpuIndices []int) (*exec.Cmd, string, error) { switch strings.ToLower(vendor) { case "nvidia": argv, env, err := resolveBenchmarkPowerLoadCommand(durSec, gpuIndices) if err != nil { return nil, "", err } cmd := exec.CommandContext(ctx, argv[0], argv[1:]...) if len(env) > 0 { cmd.Env = append(os.Environ(), env...) } cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} cmd.Cancel = func() error { if cmd.Process != nil { _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) } return nil } return cmd, "dcgmproftester targeted_power", nil case "amd": cmd := buildAMDGPUStressCmd(ctx, durSec) if cmd == nil { return nil, "", nil } return cmd, "rvs gst", nil } return nil, "", nil } func applyFanCheckDefaults(o *FanCheckOptions) { if o.PlateauHoldSec <= 0 { o.PlateauHoldSec = 60 } if o.PlateauDeltaRPM <= 0 { o.PlateauDeltaRPM = 50 } if o.MinLoadSec <= 0 { o.MinLoadSec = 90 } if o.MaxLoadSec <= 0 { o.MaxLoadSec = 900 } if o.RampConfirmRPM <= 0 { o.RampConfirmRPM = 150 } if o.MinLoadSec < o.PlateauHoldSec { o.MinLoadSec = o.PlateauHoldSec } if o.MaxLoadSec <= o.MinLoadSec { o.MaxLoadSec = o.MinLoadSec + o.PlateauHoldSec } } func orNone(present bool, v string) string { if present && v != "" { return v } return "none" } // sanitizeSummaryKey makes a fan sensor name safe as a summary.txt key // fragment (keys are parsed by splitting on '=' and whitespace). func sanitizeSummaryKey(name string) string { var b strings.Builder for _, r := range name { switch { case r >= 'A' && r <= 'Z', r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '-', r == '_', r == '.': b.WriteRune(r) default: b.WriteByte('_') } } return b.String() } // readFanStatuses returns the per-fan IPMI status word ("ok", "cr", "nr", ...) // from "ipmitool sdr type Fan". Empty map when ipmitool is unavailable. func readFanStatuses() map[string]string { out, err := exec.Command("ipmitool", "sdr", "type", "Fan").Output() if err != nil { return nil } m := map[string]string{} for _, line := range strings.Split(string(out), "\n") { parts := strings.Split(line, "|") if len(parts) < 3 { continue } name := strings.TrimSpace(parts[0]) if name == "" { continue } m[name] = strings.ToLower(strings.TrimSpace(parts[2])) } return m } // ResolveFanMaxRPM returns, for every fan name in current (name -> current // RPM), the RPM to treat as that fan's 100% reference. Preference order: // 1. the persisted observed peak, written by RunFanCheck and by live-metrics // sampling under load; // 2. the largest peak observed on any peer fan (keeps a group visually // consistent when only some fans have a recorded peak); // 3. the fan's own current RPM (so a tile is never sized against zero). // // The fallback lives here, not in the view, so every consumer of a fan // maximum applies the same rule. func ResolveFanMaxRPM(current map[string]float64) map[string]float64 { persisted := fanPeaks.snapshot() peerMax := 0.0 for _, v := range persisted { if v > peerMax { peerMax = v } } out := make(map[string]float64, len(current)) for name, rpm := range current { switch { case persisted[name] > 0: out[name] = persisted[name] case peerMax > 0: out[name] = peerMax default: out[name] = rpm } } return out } // sampleFanSpeeds reads fan RPM values from ipmitool sdr. func sampleFanSpeeds() ([]FanReading, error) { out, err := exec.Command("ipmitool", "sdr", "type", "Fan").Output() if err == nil { if fans := parseFanSpeeds(string(out)); len(fans) > 0 { updateFanObservation(fans, time.Now()) return fans, nil } } fans, sensorsErr := sampleFanSpeedsViaSensorsJSON() if len(fans) > 0 { updateFanObservation(fans, time.Now()) return fans, nil } if err != nil { return nil, err } return nil, sensorsErr } // ObservedFanMaxRPM returns the per-fan observed peak RPM map persisted by // full-load runs, or nil if none is recorded yet. func ObservedFanMaxRPM() map[string]float64 { out := fanPeaks.snapshot() if len(out) == 0 { return nil } return out } func updateFanObservation(fans []FanReading, now time.Time) { if len(fans) == 0 { return } m := make(map[string]float64, len(fans)) for _, fan := range fans { if n := strings.TrimSpace(fan.Name); n != "" && fan.RPM > 0 { m[n] = fan.RPM } } fanPeaks.observe(m, now) } func estimateFanDutyCyclePctFromObservation(fans []FanReading) (float64, bool) { if len(fans) == 0 { return 0, false } maxByName := fanPeaks.snapshot() var samples []float64 for _, fan := range fans { name := strings.TrimSpace(fan.Name) if name == "" || fan.RPM <= 0 { continue } maxRPM := maxByName[name] if maxRPM <= 0 { continue } pct := fan.RPM / maxRPM * 100.0 if pct > 100 { pct = 100 } if pct < 0 { pct = 0 } samples = append(samples, pct) } if len(samples) == 0 { return 0, false } return benchmarkMean(samples), true } // parseFanSpeeds parses "ipmitool sdr type Fan" output. // Handles two formats: // // Old: "FAN1 | 2400.000 | RPM | ok" (value in col[1], unit in col[2]) // New: "FAN1 | 41h | ok | 29.1 | 4340 RPM" (value+unit combined in last col) func parseFanSpeeds(raw string) []FanReading { var fans []FanReading for _, line := range strings.Split(strings.TrimSpace(raw), "\n") { parts := strings.Split(line, "|") if len(parts) < 2 { continue } name := strings.TrimSpace(parts[0]) // Find the first field that contains "RPM" (either as a standalone unit or inline) rpmVal := 0.0 found := false for _, p := range parts[1:] { p = strings.TrimSpace(p) if !strings.Contains(strings.ToUpper(p), "RPM") { continue } if strings.EqualFold(p, "RPM") { continue // unit-only column in old format; value is in previous field } val, err := parseFanRPMValue(p) if err == nil { rpmVal = val found = true break } } // Old format: unit "RPM" is in col[2], value is in col[1] if !found && len(parts) >= 3 && strings.EqualFold(strings.TrimSpace(parts[2]), "RPM") { valStr := strings.TrimSpace(parts[1]) if !strings.EqualFold(valStr, "na") && !strings.EqualFold(valStr, "disabled") && valStr != "" { if val, err := parseFanRPMValue(valStr); err == nil { rpmVal = val found = true } } } if !found { continue } fans = append(fans, FanReading{Name: name, RPM: rpmVal}) } return fans } func parseFanRPMValue(raw string) (float64, error) { fields := strings.Fields(strings.TrimSpace(strings.ReplaceAll(raw, ",", ""))) if len(fields) == 0 { return 0, strconv.ErrSyntax } return strconv.ParseFloat(fields[0], 64) } func sampleFanSpeedsViaSensorsJSON() ([]FanReading, error) { out, err := exec.Command("sensors", "-j").Output() if err != nil || len(out) == 0 { return nil, err } var doc map[string]map[string]any if err := json.Unmarshal(out, &doc); err != nil { return nil, err } chips := make([]string, 0, len(doc)) for chip := range doc { chips = append(chips, chip) } sort.Strings(chips) var fans []FanReading seen := map[string]struct{}{} for _, chip := range chips { features := doc[chip] names := make([]string, 0, len(features)) for name := range features { names = append(names, name) } sort.Strings(names) for _, name := range names { feature, ok := features[name].(map[string]any) if !ok { continue } rpm, ok := firstFanInputValue(feature) if !ok || rpm <= 0 { continue } label := strings.TrimSpace(name) if chip != "" && !strings.Contains(strings.ToLower(label), strings.ToLower(chip)) { label = chip + " / " + label } if _, ok := seen[label]; ok { continue } seen[label] = struct{}{} fans = append(fans, FanReading{Name: label, RPM: rpm}) } } return fans, nil } func sampleFanDutyCyclePctFromFans(fans []FanReading) (float64, bool, bool) { if len(fans) == 0 { return 0, false, false } if pct, ok := estimateFanDutyCyclePctFromObservation(fans); ok { return pct, true, true } return 0, false, false } func parseFanDutyCyclePctSensorsJSON(raw []byte) (float64, bool) { var doc map[string]map[string]any if err := json.Unmarshal(raw, &doc); err != nil { return 0, false } var samples []float64 for _, features := range doc { for name, feature := range features { if strings.EqualFold(name, "Adapter") { continue } featureMap, ok := feature.(map[string]any) if !ok { continue } if duty, ok := firstFanDutyValue(name, featureMap); ok { samples = append(samples, duty) } } } if len(samples) == 0 { return 0, false } return benchmarkMean(samples), true } func firstFanDutyValue(featureName string, feature map[string]any) (float64, bool) { featureName = strings.ToLower(strings.TrimSpace(featureName)) if strings.Contains(featureName, "enable") || strings.Contains(featureName, "mode") || strings.Contains(featureName, "alarm") { return 0, false } if strings.Contains(featureName, "pwm") { for _, key := range []string{"input", "value", "current"} { if value, ok := feature[key]; ok { if duty, parsed := parseFanDutyValue(value); parsed { return duty, true } } } } keys := make([]string, 0, len(feature)) for key := range feature { keys = append(keys, key) } sort.Strings(keys) for _, key := range keys { lower := strings.ToLower(key) if !strings.Contains(lower, "pwm") { continue } if strings.Contains(lower, "enable") || strings.Contains(lower, "mode") || strings.Contains(lower, "alarm") { continue } if duty, parsed := parseFanDutyValue(feature[key]); parsed { return duty, true } } return 0, false } func parseFanDutyValue(value any) (float64, bool) { switch v := value.(type) { case float64: return normalizePWMAsDutyPct(v) case string: if f, err := strconv.ParseFloat(strings.TrimSpace(v), 64); err == nil { return normalizePWMAsDutyPct(f) } } return 0, false } func normalizePWMAsDutyPct(raw float64) (float64, bool) { if raw < 0 { return 0, false } if raw <= 100 { return raw, true } if raw <= 255 { return raw / 255.0 * 100.0, true } return 0, false } func firstFanInputValue(feature map[string]any) (float64, bool) { return firstSensorInputValue(feature, "fan") } // sampleCPUMaxTemp returns the highest CPU/inlet temperature from ipmitool or sensors. func sampleCPUMaxTemp() float64 { out, err := exec.Command("ipmitool", "sdr", "type", "Temperature").Output() if err != nil { return sampleCPUTempViaSensors() } return parseIPMIMaxTemp(string(out)) } // parseIPMIMaxTemp extracts the maximum temperature from "ipmitool sdr type Temperature". func parseIPMIMaxTemp(raw string) float64 { var max float64 for _, line := range strings.Split(strings.TrimSpace(raw), "\n") { parts := strings.Split(line, "|") if len(parts) < 3 { continue } unit := strings.TrimSpace(parts[2]) if !strings.Contains(strings.ToLower(unit), "degrees") { continue } valStr := strings.TrimSpace(parts[1]) if strings.EqualFold(valStr, "na") || valStr == "" { continue } val, err := strconv.ParseFloat(valStr, 64) if err != nil { continue } if val > max { max = val } } return max } // sampleCPUTempViaSensors falls back to lm-sensors when ipmitool is unavailable. func sampleCPUTempViaSensors() float64 { out, err := exec.Command("sensors", "-u").Output() if err != nil { return 0 } var max float64 for _, line := range strings.Split(string(out), "\n") { line = strings.TrimSpace(line) fields := strings.Fields(line) if len(fields) < 2 { continue } if !strings.HasSuffix(fields[0], "_input:") { continue } val, err := strconv.ParseFloat(fields[1], 64) if err != nil { continue } if val > 0 && val < 150 && val > max { max = val } } return max } // parseDCMIPowerReading extracts the instantaneous power reading from ipmitool dcmi output. // Sample: " Instantaneous power reading: 500 Watts" func parseDCMIPowerReading(raw string) float64 { for _, line := range strings.Split(raw, "\n") { if !strings.Contains(strings.ToLower(line), "instantaneous") { continue } parts := strings.Fields(line) for i, p := range parts { if strings.EqualFold(p, "Watts") && i > 0 { val, err := strconv.ParseFloat(parts[i-1], 64) if err == nil { return val } } } } return 0 } func effectiveSystemPowerReading(cache cachedPowerReading, current float64, source, mode, reason string, now time.Time) (float64, cachedPowerReading) { if current > 0 { cache = cachedPowerReading{Value: current, Source: source, Mode: mode, Reason: reason, UpdatedAt: now} return current, cache } if cache.Value > 0 && !cache.UpdatedAt.IsZero() && now.Sub(cache.UpdatedAt) <= systemPowerHoldTTL { return cache.Value, cache } return 0, cache } // fanRPMStats computes average, min, max RPM across all fans in a sample row. func fanRPMStats(fans []FanReading) (avg, min, max float64) { if len(fans) == 0 { return 0, 0, 0 } min = fans[0].RPM max = fans[0].RPM var total float64 for _, f := range fans { total += f.RPM if f.RPM < min { min = f.RPM } if f.RPM > max { max = f.RPM } } return total / float64(len(fans)), min, max }