bee/audit/internal/platform/sat_fan_stress.go

package platform

import (
	"context"
	"encoding/json"
	"fmt"
	"math"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"
)

// FanCheckOptions configures the fan-ceiling check: it drives CPU (+memory)
// and, when present, GPU load to 100% simultaneously, then watches every fan
// until none has climbed for PlateauHoldSec — at which point each fan is
// considered to be at its physical ceiling and the observed peak is recorded.
type FanCheckOptions struct {
	PlateauHoldSec  int   // a fan must not rise > PlateauDeltaRPM for this long to count as plateaued (default 60)
	PlateauDeltaRPM int   // RPM increase that still counts as "climbing" (default 50)
	MinLoadSec      int   // never declare a plateau before this many seconds of load (default 90)
	MaxLoadSec      int   // hard cap on the load phase; finish (success) even if not every fan plateaued (default 900)
	RampConfirmRPM  int   // at least one fan must exceed baseline by this before a plateau is "real" (default 150)
	SizeMB          int   // GPU memory to allocate per GPU (0 = auto)
	GPUIndices      []int // which GPU indices to load (empty = all detected)
}

// FanReading holds one fan sensor reading.
type FanReading struct {
	Name string
	RPM  float64
}

type cachedPowerReading struct {
	Value     float64
	Source    string
	Mode      string
	Reason    string
	UpdatedAt time.Time
}

type fanObservationState struct {
	MaxRPM map[string]float64 `json:"max_rpm"`
}

type fanPeakCandidate struct {
	FirstSeen time.Time
	RPM       float64
}

var (
	fanObservationMu   sync.Mutex
	fanObservation     fanObservationState
	fanObservationInit bool
	fanPeakCandidates  = make(map[string]fanPeakCandidate)
)

const systemPowerHoldTTL = 15 * time.Second

var fanObservationStatePath = "/var/log/bee-sat/fan-observation.json"

const fanObservationMinPeakHold = time.Second

func normalizeObservedFanMaxRPM(rpm float64) float64 {
	if rpm <= 0 {
		return 0
	}
	return math.Ceil(rpm/1000.0) * 1000.0
}

// RunFanCheck drives CPU (+memory) and, when a GPU is present, GPU load to
// 100% simultaneously and watches every fan until none has climbed for
// PlateauHoldSec. At that point each fan is taken to be at its physical
// ceiling; the observed peak RPM is persisted (fanObservationStatePath, the
// same store ObservedFanMaxRPM reads) so the topology view can size each fan
// tile against a real maximum.
//
// Outcome:
//   - success ("ceiling found") once every fan plateaus, or when MaxLoadSec is
//     hit — a run that simply ran out of time still recorded the highest RPM
//     seen and is not a failure.
//   - a fan reading 0 RPM, or an IPMI status of cr/nr, while under full load is
//     a real defect → FAILED.
//   - if there is no way to load this box (no stressapptest/stress-ng and no
//     GPU burn tool) or no fan sensors are readable, the test cannot say
//     anything about the hardware and returns ErrTestNotApplicable so the task
//     is cancelled, not failed.
//
// No GPU is not an error: CPU/memory load alone is enough to exercise the
// cooling loop on most platforms.
func (s *System) RunFanCheck(ctx context.Context, baseDir string, opts FanCheckOptions, logFunc func(string)) (string, error) {
	if logFunc == nil {
		logFunc = func(string) {}
	}
	if baseDir == "" {
		baseDir = "/var/log/bee-sat"
	}
	applyFanCheckDefaults(&opts)

	baseFans, fanErr := sampleFanSpeeds()
	if len(baseFans) == 0 {
		return "", fmt.Errorf("no fan sensors readable via ipmitool or lm-sensors (%v): %w", fanErr, ErrTestNotApplicable)
	}
	baselineRPM := make(map[string]float64, len(baseFans))
	for _, f := range baseFans {
		baselineRPM[f.Name] = f.RPM
	}

	vendor := s.DetectGPUVendor()
	haveGPU := vendor == "nvidia" || vendor == "amd"
	_, cpuPathErr := satLookPath("stressapptest")
	if cpuPathErr != nil {
		_, cpuPathErr = satLookPath("stress-ng")
	}
	haveCPU := cpuPathErr == nil
	if !haveCPU && !haveGPU {
		return "", fmt.Errorf("no load source: stressapptest/stress-ng missing and no NVIDIA/AMD GPU stress tool available: %w", ErrTestNotApplicable)
	}

	ts := time.Now().UTC().Format("20060102-150405")
	runDir := filepath.Join(baseDir, "fan-check-"+ts)
	if err := os.MkdirAll(runDir, 0755); err != nil {
		return "", err
	}
	verboseLog := filepath.Join(runDir, "verbose.log")
	appendSATVerboseLog(verboseLog, fmt.Sprintf("[%s] fan check start: %d fans, gpu=%s cpu=%v",
		time.Now().UTC().Format(time.RFC3339), len(baseFans), vendor, haveCPU))
	logFunc(fmt.Sprintf("Fan check: %d fans; load = CPU/mem:%v + GPU:%s", len(baseFans), haveCPU, orNone(haveGPU, vendor)))

	// ── Load: every source runs at the same time, each in its own goroutine.
	// GPU load is the hottest sustained NVIDIA load we have — dcgmproftester
	// -t 1004 / targeted_power, the same engine the Power/Thermal Fit
	// benchmark uses (resolveBenchmarkPowerLoadCommand) — not the
	// compute-throughput bee-gpu-burn, which tops out well below TDP and so
	// never demands the fans' true ceiling.
	//
	// Sources reach full load at different times (stressapptest is instant; a
	// dcgmproftester kernel compiles and ramps), so the plateau clock does not
	// start until every launched source reports its process running, plus a
	// fixed GPU ramp grace.
	loadCtx, loadCancel := context.WithTimeout(ctx, time.Duration(opts.MaxLoadSec)*time.Second)
	defer loadCancel()

	var loadWG sync.WaitGroup
	started := make(chan bool, 2) // true = source is running, false = failed to launch
	launched := 0

	if haveCPU {
		launched++
		loadWG.Add(1)
		go func() {
			defer loadWG.Done()
			cmd, err := buildCPUStressCmd(loadCtx)
			if err != nil {
				logFunc("CPU/memory load failed to start: " + err.Error())
				appendSATVerboseLog(verboseLog, "cpu load start error: "+err.Error())
				started <- false
				return
			}
			logFunc("CPU/memory load running (stressapptest)")
			started <- true
			_ = cmd.Wait()
		}()
	}
	if haveGPU {
		launched++
		loadWG.Add(1)
		go func() {
			defer loadWG.Done()
			cmd, label, err := buildFanCheckGPULoadCmd(loadCtx, vendor, opts.MaxLoadSec, opts.GPUIndices)
			if err != nil || cmd == nil {
				logFunc("GPU load unavailable: " + errString(err))
				appendSATVerboseLog(verboseLog, "gpu load unavailable: "+errString(err))
				started <- false
				return
			}
			if err := cmd.Start(); err != nil {
				logFunc("GPU load failed to start: " + err.Error())
				appendSATVerboseLog(verboseLog, "gpu load start error: "+err.Error())
				started <- false
				return
			}
			logFunc("GPU load running (" + label + ")")
			started <- true
			_ = cmd.Wait()
		}()
	}

	start := time.Now()
	activeLoads := 0
	for i := 0; i < launched; i++ {
		select {
		case ok := <-started:
			if ok {
				activeLoads++
			}
		case <-ctx.Done():
		}
	}
	if activeLoads == 0 {
		loadCancel()
		loadWG.Wait()
		return "", fmt.Errorf("every load source failed to start: %w", ErrTestNotApplicable)
	}
	readyAt := time.Now()
	if haveGPU {
		readyAt = readyAt.Add(20 * time.Second) // GPU kernel ramp grace
	}
	appendSATVerboseLog(verboseLog, fmt.Sprintf("%d load source(s) active; plateau clock effective from +%.0fs",
		activeLoads, readyAt.Sub(start).Seconds()))

	// ── Sample loop with IPMI-hang protection.
	//
	// Under full load ipmitool over KCS can take tens of seconds per call or
	// wedge outright. So: every fan read is time-boxed (readFansBounded runs
	// it in a goroutine we abandon on timeout — a wedged KCS read can never
	// block this loop), and the polling interval backs off geometrically when
	// reads are slow and tightens again when they recover. A plateau is only
	// declared while telemetry is healthy (interval near the floor); a
	// degraded run just rides out to MaxLoadSec and records the peak it saw.
	const (
		fanPollFloor = 1 * time.Second
		fanPollCeil  = 30 * time.Second
		fanReadTMO   = 8 * time.Second
	)
	type fanState struct {
		peak        float64
		lastRiseSec float64
	}
	fanBy := map[string]*fanState{}
	rampConfirmed := false
	plateauReached := false
	aborted := false
	degraded := false
	goodSamples := 0
	poll := fanPollFloor

	csvPath := filepath.Join(runDir, "fan-sensors.csv")
	_ = os.WriteFile(csvPath, []byte("elapsed_sec,fan_name,rpm\n"), 0644)
	csvFile, _ := os.OpenFile(csvPath, os.O_APPEND|os.O_WRONLY, 0644)
	if csvFile != nil {
		defer csvFile.Close()
	}

loop:
	for {
		select {
		case <-ctx.Done():
			aborted = true
			break loop
		case <-loadCtx.Done():
			break loop // MaxLoadSec reached
		case <-time.After(poll):
		}
		elapsed := time.Since(start).Seconds()

		readStart := time.Now()
		fans, ok := readFansBounded(fanReadTMO)
		readDur := time.Since(readStart)

		// Adapt the interval to how ipmitool is behaving.
		switch {
		case !ok || readDur > fanReadTMO*3/4:
			if poll < fanPollCeil {
				poll = minDuration(poll*2, fanPollCeil)
				degraded = true
				appendSATVerboseLog(verboseLog, fmt.Sprintf("[%.0fs] ipmitool slow (%.1fs, ok=%v) — polling backed off to %s",
					elapsed, readDur.Seconds(), ok, poll))
				logFunc(fmt.Sprintf("IPMI slow — fan polling backed off to %s", poll))
			}
		case poll > fanPollFloor && readDur < fanPollFloor:
			poll = maxDuration(poll*2/3, fanPollFloor)
		}
		if !ok {
			continue
		}
		goodSamples++

		for _, f := range fans {
			if csvFile != nil {
				fmt.Fprintf(csvFile, "%.0f,%s,%.0f\n", elapsed, f.Name, f.RPM)
			}
			st := fanBy[f.Name]
			if st == nil {
				fanBy[f.Name] = &fanState{peak: f.RPM, lastRiseSec: elapsed}
				continue
			}
			if f.RPM > st.peak {
				if f.RPM-st.peak > float64(opts.PlateauDeltaRPM) {
					st.lastRiseSec = elapsed
				}
				st.peak = f.RPM
			}
			if f.RPM >= baselineRPM[f.Name]+float64(opts.RampConfirmRPM) {
				rampConfirmed = true
			}
		}

		// Only trust a plateau while telemetry is healthy and we have enough
		// recent samples to have actually seen a flat window.
		healthy := poll <= 2*fanPollFloor && goodSamples >= 5
		if healthy && rampConfirmed && elapsed >= float64(opts.MinLoadSec) &&
			time.Since(readyAt) >= time.Duration(opts.PlateauHoldSec)*time.Second && len(fanBy) > 0 {
			allFlat := true
			for _, st := range fanBy {
				if elapsed-st.lastRiseSec < float64(opts.PlateauHoldSec) {
					allFlat = false
					break
				}
			}
			if allFlat {
				plateauReached = true
				logFunc(fmt.Sprintf("All %d fans plateaued at %.0fs of load", len(fanBy), elapsed))
				break loop
			}
		}
	}

	loadCancel()
	loadWG.Wait()

	if aborted && ctx.Err() != nil {
		_ = os.WriteFile(filepath.Join(runDir, "summary.txt"),
			[]byte("run_at_utc="+time.Now().UTC().Format(time.RFC3339)+"\noverall_status=UNKNOWN\naborted=true\n"), 0644)
		return runDir, ctx.Err()
	}

	loadDur := time.Since(start).Seconds()

	// ── Verdict.
	statuses := readFanStatuses()
	var summary strings.Builder
	fmt.Fprintf(&summary, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339))
	fmt.Fprintf(&summary, "fans_total=%d\n", len(baseFans))
	fmt.Fprintf(&summary, "active_load_sources=%d\n", activeLoads)
	fmt.Fprintf(&summary, "gpu_vendor=%s\n", orNone(haveGPU, vendor))
	fmt.Fprintf(&summary, "plateau_reached=%v\n", plateauReached)
	fmt.Fprintf(&summary, "ramp_confirmed=%v\n", rampConfirmed)
	fmt.Fprintf(&summary, "load_duration_sec=%.0f\n", loadDur)
	fmt.Fprintf(&summary, "fan_samples=%d\n", goodSamples)
	fmt.Fprintf(&summary, "telemetry_degraded=%v\n", degraded)
	if t := boundedGPUMaxTemp(opts.GPUIndices); t > 0 {
		fmt.Fprintf(&summary, "gpu_temp_c=%.0f\n", t)
	}

	stats := satStats{}
	names := make([]string, 0, len(baselineRPM))
	for n := range baselineRPM {
		names = append(names, n)
	}
	sort.Strings(names)
	for _, name := range names {
		peak := baselineRPM[name]
		if st := fanBy[name]; st != nil {
			peak = st.peak
		}
		st := strings.ToLower(strings.TrimSpace(statuses[name]))
		bad := peak <= 0 || st == "cr" || st == "nr"
		key := sanitizeSummaryKey(name)
		fmt.Fprintf(&summary, "fan_%s_baseline_rpm=%.0f\n", key, baselineRPM[name])
		fmt.Fprintf(&summary, "fan_%s_max_rpm=%.0f\n", key, peak)
		if bad {
			reason := "0 RPM under load"
			if st == "cr" || st == "nr" {
				reason = "IPMI status " + st
			}
			fmt.Fprintf(&summary, "fan_%s_status=FAILED (%s)\n", key, reason)
			logFunc(fmt.Sprintf("FAIL %s: %s", name, reason))
			stats.Failed++
		} else {
			fmt.Fprintf(&summary, "fan_%s_status=OK\n", key)
			stats.OK++
		}
	}
	writeSATStats(&summary, stats)

	if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary.String()), 0644); err != nil {
		return "", err
	}
	return runDir, nil
}

func errString(err error) string {
	if err == nil {
		return "no GPU stress tool"
	}
	return err.Error()
}

func minDuration(a, b time.Duration) time.Duration {
	if a < b {
		return a
	}
	return b
}

func maxDuration(a, b time.Duration) time.Duration {
	if a > b {
		return a
	}
	return b
}

// readFansBounded runs "ipmitool sdr type Fan" but never blocks the caller
// longer than timeout: the read happens in a goroutine that is abandoned if it
// does not return in time (a KCS read wedged in uninterruptible I/O cannot be
// killed, so we leave it and move on). ok=false means "no usable sample this
// tick" — the caller must treat that as missing data, not as a flat fan.
func readFansBounded(timeout time.Duration) ([]FanReading, bool) {
	type result struct {
		fans []FanReading
		ok   bool
	}
	ch := make(chan result, 1)
	go func() {
		out, err := exec.Command("ipmitool", "sdr", "type", "Fan").Output()
		if err != nil {
			ch <- result{}
			return
		}
		fans := parseFanSpeeds(string(out))
		if len(fans) == 0 {
			ch <- result{}
			return
		}
		ch <- result{fans, true}
	}()
	select {
	case r := <-ch:
		if r.ok {
			updateFanObservation(r.fans, time.Now())
		}
		return r.fans, r.ok
	case <-time.After(timeout):
		return nil, false
	}
}

// boundedGPUMaxTemp returns the hottest GPU temperature via a single
// time-boxed nvidia-smi call, or 0 if unavailable.
func boundedGPUMaxTemp(gpuIndices []int) float64 {
	ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second)
	defer cancel()
	args := []string{"--query-gpu=temperature.gpu", "--format=csv,noheader,nounits"}
	if len(gpuIndices) > 0 {
		ids := make([]string, len(gpuIndices))
		for i, idx := range gpuIndices {
			ids[i] = strconv.Itoa(idx)
		}
		args = append([]string{"--id=" + strings.Join(ids, ",")}, args...)
	}
	out, err := exec.CommandContext(ctx, "nvidia-smi", args...).Output()
	if err != nil {
		return 0
	}
	var max float64
	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
		if v, err := strconv.ParseFloat(strings.TrimSpace(line), 64); err == nil && v > max {
			max = v
		}
	}
	return max
}

// buildFanCheckGPULoadCmd builds the hottest sustained GPU load for the fan
// check. NVIDIA uses the Power/Thermal Fit engine (dcgmproftester -t 1004 /
// targeted_power); AMD uses the RVS gst stressor.
func buildFanCheckGPULoadCmd(ctx context.Context, vendor string, durSec int, gpuIndices []int) (*exec.Cmd, string, error) {
	switch strings.ToLower(vendor) {
	case "nvidia":
		argv, env, err := resolveBenchmarkPowerLoadCommand(durSec, gpuIndices)
		if err != nil {
			return nil, "", err
		}
		cmd := exec.CommandContext(ctx, argv[0], argv[1:]...)
		if len(env) > 0 {
			cmd.Env = append(os.Environ(), env...)
		}
		cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
		cmd.Cancel = func() error {
			if cmd.Process != nil {
				_ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
			}
			return nil
		}
		return cmd, "dcgmproftester targeted_power", nil
	case "amd":
		cmd := buildAMDGPUStressCmd(ctx, durSec)
		if cmd == nil {
			return nil, "", nil
		}
		return cmd, "rvs gst", nil
	}
	return nil, "", nil
}

func applyFanCheckDefaults(o *FanCheckOptions) {
	if o.PlateauHoldSec <= 0 {
		o.PlateauHoldSec = 60
	}
	if o.PlateauDeltaRPM <= 0 {
		o.PlateauDeltaRPM = 50
	}
	if o.MinLoadSec <= 0 {
		o.MinLoadSec = 90
	}
	if o.MaxLoadSec <= 0 {
		o.MaxLoadSec = 900
	}
	if o.RampConfirmRPM <= 0 {
		o.RampConfirmRPM = 150
	}
	if o.MinLoadSec < o.PlateauHoldSec {
		o.MinLoadSec = o.PlateauHoldSec
	}
	if o.MaxLoadSec <= o.MinLoadSec {
		o.MaxLoadSec = o.MinLoadSec + o.PlateauHoldSec
	}
}

func orNone(present bool, v string) string {
	if present && v != "" {
		return v
	}
	return "none"
}

// sanitizeSummaryKey makes a fan sensor name safe as a summary.txt key
// fragment (keys are parsed by splitting on '=' and whitespace).
func sanitizeSummaryKey(name string) string {
	var b strings.Builder
	for _, r := range name {
		switch {
		case r >= 'A' && r <= 'Z', r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '-', r == '_', r == '.':
			b.WriteRune(r)
		default:
			b.WriteByte('_')
		}
	}
	return b.String()
}

// readFanStatuses returns the per-fan IPMI status word ("ok", "cr", "nr", ...)
// from "ipmitool sdr type Fan". Empty map when ipmitool is unavailable.
func readFanStatuses() map[string]string {
	out, err := exec.Command("ipmitool", "sdr", "type", "Fan").Output()
	if err != nil {
		return nil
	}
	m := map[string]string{}
	for _, line := range strings.Split(string(out), "\n") {
		parts := strings.Split(line, "|")
		if len(parts) < 3 {
			continue
		}
		name := strings.TrimSpace(parts[0])
		if name == "" {
			continue
		}
		m[name] = strings.ToLower(strings.TrimSpace(parts[2]))
	}
	return m
}

// ResolveFanMaxRPM returns, for every fan name in current (name -> current
// RPM), the RPM to treat as that fan's 100% reference. Preference order:
//  1. the persisted observed peak (fanObservationStatePath), written by
//     RunFanCheck and by live-metrics sampling under load;
//  2. the largest peak observed on any peer fan (keeps a group visually
//     consistent when only some fans have a recorded peak);
//  3. the fan's own current RPM (so a tile is never sized against zero).
//
// The fallback lives here, not in the view, so every consumer of a fan
// maximum applies the same rule.
func ResolveFanMaxRPM(current map[string]float64) map[string]float64 {
	persisted := readPersistedFanMaxRPM()

	peerMax := 0.0
	for _, v := range persisted {
		if v > peerMax {
			peerMax = v
		}
	}

	out := make(map[string]float64, len(current))
	for name, rpm := range current {
		switch {
		case persisted[name] > 0:
			out[name] = persisted[name]
		case peerMax > 0:
			out[name] = peerMax
		default:
			out[name] = rpm
		}
	}
	return out
}

// sampleFanSpeeds reads fan RPM values from ipmitool sdr.
func sampleFanSpeeds() ([]FanReading, error) {
	out, err := exec.Command("ipmitool", "sdr", "type", "Fan").Output()
	if err == nil {
		if fans := parseFanSpeeds(string(out)); len(fans) > 0 {
			updateFanObservation(fans, time.Now())
			return fans, nil
		}
	}
	fans, sensorsErr := sampleFanSpeedsViaSensorsJSON()
	if len(fans) > 0 {
		updateFanObservation(fans, time.Now())
		return fans, nil
	}
	if err != nil {
		return nil, err
	}
	return nil, sensorsErr
}

// readPersistedFanMaxRPM reads fanObservationStatePath and returns its
// sanitized {fan name -> observed peak RPM} map (empty names / non-positive
// values dropped). Returns an empty map when the file is missing or unparsable.
func readPersistedFanMaxRPM() map[string]float64 {
	out := map[string]float64{}
	raw, err := os.ReadFile(fanObservationStatePath)
	if err != nil || len(raw) == 0 {
		return out
	}
	var persisted fanObservationState
	if json.Unmarshal(raw, &persisted) != nil {
		return out
	}
	for name, rpm := range persisted.MaxRPM {
		name = strings.TrimSpace(name)
		if name == "" || rpm <= 0 {
			continue
		}
		out[name] = rpm
	}
	return out
}

// ObservedFanMaxRPM returns the per-fan observed peak RPM map persisted by
// fan-stress SAT runs, or nil if none is recorded yet. It reads the file
// directly without touching the in-process observation cache or its lock, so
// read-only consumers (the /topo web view) can call it without perturbing a
// concurrent SAT run's peak tracking.
func ObservedFanMaxRPM() map[string]float64 {
	out := readPersistedFanMaxRPM()
	if len(out) == 0 {
		return nil
	}
	return out
}

func loadFanObservationLocked() {
	if fanObservationInit {
		return
	}
	fanObservationInit = true
	fanObservation.MaxRPM = readPersistedFanMaxRPM()
}

func saveFanObservationLocked() {
	if len(fanObservation.MaxRPM) == 0 {
		return
	}
	dir := filepath.Dir(fanObservationStatePath)
	if dir == "" || dir == "." {
		dir = "/var/log/bee-sat"
	}
	if err := os.MkdirAll(dir, 0755); err != nil {
		return
	}
	raw, err := json.MarshalIndent(fanObservation, "", "  ")
	if err != nil {
		return
	}
	_ = os.WriteFile(fanObservationStatePath, raw, 0644)
}

func updateFanObservation(fans []FanReading, now time.Time) {
	if len(fans) == 0 {
		return
	}
	fanObservationMu.Lock()
	defer fanObservationMu.Unlock()
	loadFanObservationLocked()
	changed := false
	for _, fan := range fans {
		name := strings.TrimSpace(fan.Name)
		if name == "" || fan.RPM <= 0 {
			continue
		}
		currentMax := fanObservation.MaxRPM[name]
		if fan.RPM <= currentMax {
			delete(fanPeakCandidates, name)
			continue
		}
		if cand, ok := fanPeakCandidates[name]; ok {
			if now.Sub(cand.FirstSeen) >= fanObservationMinPeakHold {
				newMax := math.Max(cand.RPM, fan.RPM)
				if newMax > currentMax {
					fanObservation.MaxRPM[name] = normalizeObservedFanMaxRPM(newMax)
					changed = true
				}
				delete(fanPeakCandidates, name)
				continue
			}
			if fan.RPM > cand.RPM {
				fanPeakCandidates[name] = fanPeakCandidate{FirstSeen: cand.FirstSeen, RPM: fan.RPM}
			}
			continue
		}
		fanPeakCandidates[name] = fanPeakCandidate{FirstSeen: now, RPM: fan.RPM}
	}
	if changed {
		saveFanObservationLocked()
	}
}

func estimateFanDutyCyclePctFromObservation(fans []FanReading) (float64, bool) {
	if len(fans) == 0 {
		return 0, false
	}
	fanObservationMu.Lock()
	defer fanObservationMu.Unlock()
	loadFanObservationLocked()
	var samples []float64
	for _, fan := range fans {
		name := strings.TrimSpace(fan.Name)
		if name == "" || fan.RPM <= 0 {
			continue
		}
		maxRPM := fanObservation.MaxRPM[name]
		if maxRPM <= 0 {
			continue
		}
		pct := fan.RPM / maxRPM * 100.0
		if pct > 100 {
			pct = 100
		}
		if pct < 0 {
			pct = 0
		}
		samples = append(samples, pct)
	}
	if len(samples) == 0 {
		return 0, false
	}
	return benchmarkMean(samples), true
}

// parseFanSpeeds parses "ipmitool sdr type Fan" output.
// Handles two formats:
//
//	Old: "FAN1 | 2400.000 | RPM | ok"           (value in col[1], unit in col[2])
//	New: "FAN1 | 41h | ok | 29.1 | 4340 RPM"   (value+unit combined in last col)
func parseFanSpeeds(raw string) []FanReading {
	var fans []FanReading
	for _, line := range strings.Split(strings.TrimSpace(raw), "\n") {
		parts := strings.Split(line, "|")
		if len(parts) < 2 {
			continue
		}
		name := strings.TrimSpace(parts[0])
		// Find the first field that contains "RPM" (either as a standalone unit or inline)
		rpmVal := 0.0
		found := false
		for _, p := range parts[1:] {
			p = strings.TrimSpace(p)
			if !strings.Contains(strings.ToUpper(p), "RPM") {
				continue
			}
			if strings.EqualFold(p, "RPM") {
				continue // unit-only column in old format; value is in previous field
			}
			val, err := parseFanRPMValue(p)
			if err == nil {
				rpmVal = val
				found = true
				break
			}
		}
		// Old format: unit "RPM" is in col[2], value is in col[1]
		if !found && len(parts) >= 3 && strings.EqualFold(strings.TrimSpace(parts[2]), "RPM") {
			valStr := strings.TrimSpace(parts[1])
			if !strings.EqualFold(valStr, "na") && !strings.EqualFold(valStr, "disabled") && valStr != "" {
				if val, err := parseFanRPMValue(valStr); err == nil {
					rpmVal = val
					found = true
				}
			}
		}
		if !found {
			continue
		}
		fans = append(fans, FanReading{Name: name, RPM: rpmVal})
	}
	return fans
}

func parseFanRPMValue(raw string) (float64, error) {
	fields := strings.Fields(strings.TrimSpace(strings.ReplaceAll(raw, ",", "")))
	if len(fields) == 0 {
		return 0, strconv.ErrSyntax
	}
	return strconv.ParseFloat(fields[0], 64)
}

func sampleFanSpeedsViaSensorsJSON() ([]FanReading, error) {
	out, err := exec.Command("sensors", "-j").Output()
	if err != nil || len(out) == 0 {
		return nil, err
	}
	var doc map[string]map[string]any
	if err := json.Unmarshal(out, &doc); err != nil {
		return nil, err
	}
	chips := make([]string, 0, len(doc))
	for chip := range doc {
		chips = append(chips, chip)
	}
	sort.Strings(chips)
	var fans []FanReading
	seen := map[string]struct{}{}
	for _, chip := range chips {
		features := doc[chip]
		names := make([]string, 0, len(features))
		for name := range features {
			names = append(names, name)
		}
		sort.Strings(names)
		for _, name := range names {
			feature, ok := features[name].(map[string]any)
			if !ok {
				continue
			}
			rpm, ok := firstFanInputValue(feature)
			if !ok || rpm <= 0 {
				continue
			}
			label := strings.TrimSpace(name)
			if chip != "" && !strings.Contains(strings.ToLower(label), strings.ToLower(chip)) {
				label = chip + " / " + label
			}
			if _, ok := seen[label]; ok {
				continue
			}
			seen[label] = struct{}{}
			fans = append(fans, FanReading{Name: label, RPM: rpm})
		}
	}
	return fans, nil
}

func sampleFanDutyCyclePctFromFans(fans []FanReading) (float64, bool, bool) {
	if len(fans) == 0 {
		return 0, false, false
	}
	if pct, ok := estimateFanDutyCyclePctFromObservation(fans); ok {
		return pct, true, true
	}
	return 0, false, false
}

func parseFanDutyCyclePctSensorsJSON(raw []byte) (float64, bool) {
	var doc map[string]map[string]any
	if err := json.Unmarshal(raw, &doc); err != nil {
		return 0, false
	}
	var samples []float64
	for _, features := range doc {
		for name, feature := range features {
			if strings.EqualFold(name, "Adapter") {
				continue
			}
			featureMap, ok := feature.(map[string]any)
			if !ok {
				continue
			}
			if duty, ok := firstFanDutyValue(name, featureMap); ok {
				samples = append(samples, duty)
			}
		}
	}
	if len(samples) == 0 {
		return 0, false
	}
	return benchmarkMean(samples), true
}

func firstFanDutyValue(featureName string, feature map[string]any) (float64, bool) {
	featureName = strings.ToLower(strings.TrimSpace(featureName))
	if strings.Contains(featureName, "enable") || strings.Contains(featureName, "mode") || strings.Contains(featureName, "alarm") {
		return 0, false
	}
	if strings.Contains(featureName, "pwm") {
		for _, key := range []string{"input", "value", "current"} {
			if value, ok := feature[key]; ok {
				if duty, parsed := parseFanDutyValue(value); parsed {
					return duty, true
				}
			}
		}
	}
	keys := make([]string, 0, len(feature))
	for key := range feature {
		keys = append(keys, key)
	}
	sort.Strings(keys)
	for _, key := range keys {
		lower := strings.ToLower(key)
		if !strings.Contains(lower, "pwm") {
			continue
		}
		if strings.Contains(lower, "enable") || strings.Contains(lower, "mode") || strings.Contains(lower, "alarm") {
			continue
		}
		if duty, parsed := parseFanDutyValue(feature[key]); parsed {
			return duty, true
		}
	}
	return 0, false
}

func parseFanDutyValue(value any) (float64, bool) {
	switch v := value.(type) {
	case float64:
		return normalizePWMAsDutyPct(v)
	case string:
		if f, err := strconv.ParseFloat(strings.TrimSpace(v), 64); err == nil {
			return normalizePWMAsDutyPct(f)
		}
	}
	return 0, false
}

func normalizePWMAsDutyPct(raw float64) (float64, bool) {
	if raw < 0 {
		return 0, false
	}
	if raw <= 100 {
		return raw, true
	}
	if raw <= 255 {
		return raw / 255.0 * 100.0, true
	}
	return 0, false
}

func firstFanInputValue(feature map[string]any) (float64, bool) {
	return firstSensorInputValue(feature, "fan")
}

// sampleCPUMaxTemp returns the highest CPU/inlet temperature from ipmitool or sensors.
func sampleCPUMaxTemp() float64 {
	out, err := exec.Command("ipmitool", "sdr", "type", "Temperature").Output()
	if err != nil {
		return sampleCPUTempViaSensors()
	}
	return parseIPMIMaxTemp(string(out))
}

// parseIPMIMaxTemp extracts the maximum temperature from "ipmitool sdr type Temperature".
func parseIPMIMaxTemp(raw string) float64 {
	var max float64
	for _, line := range strings.Split(strings.TrimSpace(raw), "\n") {
		parts := strings.Split(line, "|")
		if len(parts) < 3 {
			continue
		}
		unit := strings.TrimSpace(parts[2])
		if !strings.Contains(strings.ToLower(unit), "degrees") {
			continue
		}
		valStr := strings.TrimSpace(parts[1])
		if strings.EqualFold(valStr, "na") || valStr == "" {
			continue
		}
		val, err := strconv.ParseFloat(valStr, 64)
		if err != nil {
			continue
		}
		if val > max {
			max = val
		}
	}
	return max
}

// sampleCPUTempViaSensors falls back to lm-sensors when ipmitool is unavailable.
func sampleCPUTempViaSensors() float64 {
	out, err := exec.Command("sensors", "-u").Output()
	if err != nil {
		return 0
	}
	var max float64
	for _, line := range strings.Split(string(out), "\n") {
		line = strings.TrimSpace(line)
		fields := strings.Fields(line)
		if len(fields) < 2 {
			continue
		}
		if !strings.HasSuffix(fields[0], "_input:") {
			continue
		}
		val, err := strconv.ParseFloat(fields[1], 64)
		if err != nil {
			continue
		}
		if val > 0 && val < 150 && val > max {
			max = val
		}
	}
	return max
}

// parseDCMIPowerReading extracts the instantaneous power reading from ipmitool dcmi output.
// Sample: "    Instantaneous power reading:                   500 Watts"
func parseDCMIPowerReading(raw string) float64 {
	for _, line := range strings.Split(raw, "\n") {
		if !strings.Contains(strings.ToLower(line), "instantaneous") {
			continue
		}
		parts := strings.Fields(line)
		for i, p := range parts {
			if strings.EqualFold(p, "Watts") && i > 0 {
				val, err := strconv.ParseFloat(parts[i-1], 64)
				if err == nil {
					return val
				}
			}
		}
	}
	return 0
}

func effectiveSystemPowerReading(cache cachedPowerReading, current float64, source, mode, reason string, now time.Time) (float64, cachedPowerReading) {
	if current > 0 {
		cache = cachedPowerReading{Value: current, Source: source, Mode: mode, Reason: reason, UpdatedAt: now}
		return current, cache
	}
	if cache.Value > 0 && !cache.UpdatedAt.IsZero() && now.Sub(cache.UpdatedAt) <= systemPowerHoldTTL {
		return cache.Value, cache
	}
	return 0, cache
}

// fanRPMStats computes average, min, max RPM across all fans in a sample row.
func fanRPMStats(fans []FanReading) (avg, min, max float64) {
	if len(fans) == 0 {
		return 0, 0, 0
	}
	min = fans[0].RPM
	max = fans[0].RPM
	var total float64
	for _, f := range fans {
		total += f.RPM
		if f.RPM < min {
			min = f.RPM
		}
		if f.RPM > max {
			max = f.RPM
		}
	}
	return total / float64(len(fans)), min, max
}