feat(sat): fan ceiling check + topology fan tiles

Repurpose the previously-unwired RunFanStressTest into RunFanCheck, a Load-tier SAT test that drives stressapptest (CPU+memory) and, when a GPU is present, a GPU burn to 100% simultaneously, then watches every fan until none has climbed for ~60s. The observed peak RPM per fan is the "ceiling"; it is persisted through the existing fan-observation store. MSI G4201 / AMI MegaRAC exposes no host-side fan force (every OEM IPMI command returns 0xc1; Redfish Thermal is GET-only), so load-driven ramp is the closest safe equivalent. See bible-local/decisions/2026-09-04-fan-ceiling-check.md. - platform.ResolveFanMaxRPM: per-fan max with fallback (persisted peak -> peer peak -> current RPM), resolved in platform, not the view. - platform.ErrTestNotApplicable: no load source or no fan sensors -> task lands as cancelled ("not applicable"), never failed, so an engineer never sees a false red. executeTaskWithOptions maps the sentinel; finalizeTaskForResult honours a pre-set TaskCancelled. - Verdict FAIL only for a fan at 0 RPM / IPMI cr-nr under load. - /topo: one small spinning square per fan, sized by RPM / resolved max, clickable through to a new "fan" component-detail type; per-fan status recorded to the component-status DB from the fan SAT summary. - Wiring: /api/sat/fan/run route, "fan" task target, Load-page card, stress-mode Run All. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019VHG21rgTUiR1G3qFHTVmN
2026-09-04 10:35:34 +03:00
parent e4f7519ef3
commit bb2a501a28
22 changed files with 798 additions and 222 deletions
@@ -0,0 +1,10 @@
+package platform
+
+import "errors"
+
+// ErrTestNotApplicable is returned by a SAT routine when the host provides no
+// way to run that test at all (a required tool or capability is absent), as
+// opposed to the test running and finding a fault. The task layer maps it to a
+// cancelled ("not applicable") task rather than a failure, so an engineer does
+// not see a false red.
+var ErrTestNotApplicable = errors.New("test not applicable on this platform")
@@ -15,14 +15,18 @@ import (
 	"time"
 )

-// FanStressOptions configures the fan-stress / thermal cycling test.
-type FanStressOptions struct {
-	BaselineSec  int   // idle monitoring before and after load (default 30)
-	Phase1DurSec int   // first load phase duration in seconds (default 300)
-	PauseSec     int   // pause between the two load phases (default 60)
-	Phase2DurSec int   // second load phase duration in seconds (default 300)
-	SizeMB       int   // GPU memory to allocate per GPU during stress (0 = auto: 95% of VRAM)
-	GPUIndices   []int // which GPU indices to stress (empty = all detected)
+// FanCheckOptions configures the fan-ceiling check: it drives CPU (+memory)
+// and, when present, GPU load to 100% simultaneously, then watches every fan
+// until none has climbed for PlateauHoldSec — at which point each fan is
+// considered to be at its physical ceiling and the observed peak is recorded.
+type FanCheckOptions struct {
+	PlateauHoldSec  int   // a fan must not rise > PlateauDeltaRPM for this long to count as plateaued (default 60)
+	PlateauDeltaRPM int   // RPM increase that still counts as "climbing" (default 50)
+	MinLoadSec      int   // never declare a plateau before this many seconds of load (default 90)
+	MaxLoadSec      int   // hard cap on the load phase; finish (success) even if not every fan plateaued (default 900)
+	RampConfirmRPM  int   // at least one fan must exceed baseline by this before a plateau is "real" (default 150)
+	SizeMB          int   // GPU memory to allocate per GPU (0 = auto)
+	GPUIndices      []int // which GPU indices to load (empty = all detected)
 }

 // FanReading holds one fan sensor reading.
@@ -93,128 +97,217 @@ func normalizeObservedFanMaxRPM(rpm float64) float64 {
 	return math.Ceil(rpm/1000.0) * 1000.0
 }

-// RunFanStressTest runs a two-phase GPU stress test while monitoring fan speeds,
-// temperatures, and power draw every second. Exports metrics.csv and fan-sensors.csv.
-// Designed to reproduce case-04 fan-speed lag and detect GPU thermal throttling.
-func (s *System) RunFanStressTest(ctx context.Context, baseDir string, opts FanStressOptions) (string, error) {
+// RunFanCheck drives CPU (+memory) and, when a GPU is present, GPU load to
+// 100% simultaneously and watches every fan until none has climbed for
+// PlateauHoldSec. At that point each fan is taken to be at its physical
+// ceiling; the observed peak RPM is persisted (fanObservationStatePath, the
+// same store ObservedFanMaxRPM reads) so the topology view can size each fan
+// tile against a real maximum.
+//
+// Outcome:
+//   - success ("ceiling found") once every fan plateaus, or when MaxLoadSec is
+//     hit — a run that simply ran out of time still recorded the highest RPM
+//     seen and is not a failure.
+//   - a fan reading 0 RPM, or an IPMI status of cr/nr, while under full load is
+//     a real defect → FAILED.
+//   - if there is no way to load this box (no stressapptest/stress-ng and no
+//     GPU burn tool) or no fan sensors are readable, the test cannot say
+//     anything about the hardware and returns ErrTestNotApplicable so the task
+//     is cancelled, not failed.
+//
+// No GPU is not an error: CPU/memory load alone is enough to exercise the
+// cooling loop on most platforms.
+func (s *System) RunFanCheck(ctx context.Context, baseDir string, opts FanCheckOptions, logFunc func(string)) (string, error) {
+	if logFunc == nil {
+		logFunc = func(string) {}
+	}
 	if baseDir == "" {
 		baseDir = "/var/log/bee-sat"
 	}
-	applyFanStressDefaults(&opts)
+	applyFanCheckDefaults(&opts)
+
+	baseFans, fanErr := sampleFanSpeeds()
+	if len(baseFans) == 0 {
+		return "", fmt.Errorf("no fan sensors readable via ipmitool or lm-sensors (%v): %w", fanErr, ErrTestNotApplicable)
+	}
+	baselineRPM := make(map[string]float64, len(baseFans))
+	for _, f := range baseFans {
+		baselineRPM[f.Name] = f.RPM
+	}
+
+	vendor := s.DetectGPUVendor()
+	haveGPU := vendor == "nvidia" || vendor == "amd"
+	_, cpuPathErr := satLookPath("stressapptest")
+	if cpuPathErr != nil {
+		_, cpuPathErr = satLookPath("stress-ng")
+	}
+	haveCPU := cpuPathErr == nil
+	if !haveCPU && !haveGPU {
+		return "", fmt.Errorf("no load source: stressapptest/stress-ng missing and no NVIDIA/AMD GPU stress tool available: %w", ErrTestNotApplicable)
+	}

 	ts := time.Now().UTC().Format("20060102-150405")
-	runDir := filepath.Join(baseDir, "fan-stress-"+ts)
+	runDir := filepath.Join(baseDir, "fan-check-"+ts)
 	if err := os.MkdirAll(runDir, 0755); err != nil {
 		return "", err
 	}
 	verboseLog := filepath.Join(runDir, "verbose.log")
+	appendSATVerboseLog(verboseLog, fmt.Sprintf("[%s] fan check start: %d fans, gpu=%s cpu=%v",
+		time.Now().UTC().Format(time.RFC3339), len(baseFans), vendor, haveCPU))
+	logFunc(fmt.Sprintf("Fan check: %d fans; load = CPU/mem:%v + GPU:%s", len(baseFans), haveCPU, orNone(haveGPU, vendor)))

-	// Phase name shared between sampler goroutine and main goroutine.
-	var phaseMu sync.Mutex
-	currentPhase := "init"
-	setPhase := func(name string) {
-		phaseMu.Lock()
-		currentPhase = name
-		phaseMu.Unlock()
+	// ── Load: every source runs at the same time, each in its own goroutine.
+	// Sources can take different amounts of time to actually reach full load
+	// (stressapptest is near-instant; a GPU burn kernel needs to compile and
+	// ramp), so the plateau clock does not start until every launched source
+	// has reported its process running, plus a fixed GPU ramp grace.
+	loadCtx, loadCancel := context.WithTimeout(ctx, time.Duration(opts.MaxLoadSec)*time.Second)
+	defer loadCancel()
+
+	var loadWG sync.WaitGroup
+	started := make(chan bool, 2) // true = source is running, false = failed to launch
+	launched := 0
+
+	if haveCPU {
+		launched++
+		loadWG.Add(1)
+		go func() {
+			defer loadWG.Done()
+			cmd, err := buildCPUStressCmd(loadCtx)
+			if err != nil {
+				logFunc("CPU/memory load failed to start: " + err.Error())
+				appendSATVerboseLog(verboseLog, "cpu load start error: "+err.Error())
+				started <- false
+				return
+			}
+			logFunc("CPU/memory load running (stressapptest)")
+			started <- true
+			_ = cmd.Wait()
+		}()
 	}
-	getPhase := func() string {
-		phaseMu.Lock()
-		defer phaseMu.Unlock()
-		return currentPhase
+	if haveGPU {
+		launched++
+		loadWG.Add(1)
+		go func() {
+			defer loadWG.Done()
+			cmd := buildGPUStressCmd(loadCtx, vendor, opts.MaxLoadSec)
+			if cmd == nil {
+				logFunc("GPU load unavailable (no burn tool for " + vendor + ")")
+				appendSATVerboseLog(verboseLog, "gpu load: no burn tool")
+				started <- false
+				return
+			}
+			logFunc("GPU load running (" + vendor + ")")
+			started <- true
+			_ = cmd.Wait()
+		}()
 	}

 	start := time.Now()
-	var rowsMu sync.Mutex
-	var allRows []FanStressRow
-
-	// Start background sampler (every second).
-	stopCh := make(chan struct{})
-	doneCh := make(chan struct{})
-	go func() {
-		defer close(doneCh)
-		ticker := time.NewTicker(time.Second)
-		defer ticker.Stop()
-		for {
-			select {
-			case <-stopCh:
-				return
-			case <-ticker.C:
-				row := sampleFanStressRow(opts.GPUIndices, getPhase(), time.Since(start).Seconds())
-				rowsMu.Lock()
-				allRows = append(allRows, row)
-				rowsMu.Unlock()
+	activeLoads := 0
+	for i := 0; i < launched; i++ {
+		select {
+		case ok := <-started:
+			if ok {
+				activeLoads++
 			}
+		case <-ctx.Done():
 		}
-	}()
+	}
+	if activeLoads == 0 {
+		loadCancel()
+		loadWG.Wait()
+		return "", fmt.Errorf("every load source failed to start: %w", ErrTestNotApplicable)
+	}
+	readyAt := time.Now()
+	if haveGPU {
+		readyAt = readyAt.Add(20 * time.Second) // GPU kernel ramp grace
+	}
+	appendSATVerboseLog(verboseLog, fmt.Sprintf("%d load source(s) active; plateau clock effective from +%.0fs",
+		activeLoads, readyAt.Sub(start).Seconds()))

-	var summary strings.Builder
-	fmt.Fprintf(&summary, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339))
+	// ── Sample loop: one row/second, per-fan plateau tracking.
+	type fanState struct {
+		peak        float64
+		lastRiseSec float64
+	}
+	fanBy := map[string]*fanState{}
+	var rows []FanStressRow
+	rampConfirmed := false
+	plateauReached := false
+	aborted := false

-	stats := satStats{}
-
-	// idlePhase sleeps for durSec while the sampler stamps phaseName on each row.
-	idlePhase := func(phaseName, stepName string, durSec int) {
-		if ctx.Err() != nil {
-			return
-		}
-		setPhase(phaseName)
-		appendSATVerboseLog(verboseLog,
-			fmt.Sprintf("[%s] start %s (idle %ds)", time.Now().UTC().Format(time.RFC3339), stepName, durSec),
-		)
+	ticker := time.NewTicker(time.Second)
+	defer ticker.Stop()
+loop:
+	for {
 		select {
 		case <-ctx.Done():
-		case <-time.After(time.Duration(durSec) * time.Second):
+			aborted = true
+			break loop
+		case <-loadCtx.Done():
+			break loop // MaxLoadSec reached
+		case <-ticker.C:
 		}
-		appendSATVerboseLog(verboseLog,
-			fmt.Sprintf("[%s] finish %s", time.Now().UTC().Format(time.RFC3339), stepName),
-		)
-		fmt.Fprintf(&summary, "%s_status=OK\n", stepName)
-		stats.OK++
-	}
+		elapsed := time.Since(start).Seconds()
+		row := sampleFanStressRow(opts.GPUIndices, "load", elapsed)
+		rows = append(rows, row)

-	// loadPhase runs bee-gpu-burn for durSec; sampler stamps phaseName on each row.
-	loadPhase := func(phaseName, stepName string, durSec int) {
-		if ctx.Err() != nil {
-			return
+		for _, f := range row.Fans {
+			st := fanBy[f.Name]
+			if st == nil {
+				fanBy[f.Name] = &fanState{peak: f.RPM, lastRiseSec: elapsed}
+				continue
+			}
+			if f.RPM > st.peak {
+				if f.RPM-st.peak > float64(opts.PlateauDeltaRPM) {
+					st.lastRiseSec = elapsed
+				}
+				st.peak = f.RPM
+			}
+			if f.RPM >= baselineRPM[f.Name]+float64(opts.RampConfirmRPM) {
+				rampConfirmed = true
+			}
 		}
-		setPhase(phaseName)
-		cmd := []string{
-			"bee-gpu-burn",
-			"--seconds", strconv.Itoa(durSec),
-			"--size-mb", strconv.Itoa(opts.SizeMB),
-		}
-		if len(opts.GPUIndices) > 0 {
-			cmd = append(cmd, "--devices", joinIndexList(dedupeSortedIndices(opts.GPUIndices)))
-		}
-		out, err := runSATCommandCtx(ctx, verboseLog, stepName, cmd, nil, nil)
-		_ = os.WriteFile(filepath.Join(runDir, stepName+".log"), out, 0644)
-		if err != nil && err != context.Canceled && err.Error() != "signal: killed" {
-			fmt.Fprintf(&summary, "%s_status=FAILED\n", stepName)
-			stats.Failed++
-		} else {
-			fmt.Fprintf(&summary, "%s_status=OK\n", stepName)
-			stats.OK++
+
+		if elapsed >= float64(opts.MinLoadSec) && time.Since(readyAt) >= time.Duration(opts.PlateauHoldSec)*time.Second && len(fanBy) > 0 {
+			allFlat := true
+			for _, st := range fanBy {
+				if elapsed-st.lastRiseSec < float64(opts.PlateauHoldSec) {
+					allFlat = false
+					break
+				}
+			}
+			if allFlat && rampConfirmed {
+				plateauReached = true
+				logFunc(fmt.Sprintf("All %d fans plateaued at %.0fs of load", len(fanBy), elapsed))
+				break loop
+			}
 		}
 	}

-	// Execute test phases.
-	idlePhase("baseline", "01-baseline", opts.BaselineSec)
-	loadPhase("load1", "02-load1", opts.Phase1DurSec)
-	idlePhase("pause", "03-pause", opts.PauseSec)
-	loadPhase("load2", "04-load2", opts.Phase2DurSec)
-	idlePhase("cooldown", "05-cooldown", opts.BaselineSec)
+	loadCancel()
+	loadWG.Wait()

-	// Stop sampler and collect rows.
-	close(stopCh)
-	<-doneCh
+	if aborted && ctx.Err() != nil {
+		_ = os.WriteFile(filepath.Join(runDir, "summary.txt"),
+			[]byte("run_at_utc="+time.Now().UTC().Format(time.RFC3339)+"\noverall_status=UNKNOWN\naborted=true\n"), 0644)
+		return runDir, ctx.Err()
+	}

-	rowsMu.Lock()
-	rows := allRows
-	rowsMu.Unlock()
-
-	// Analysis.
-	throttled := analyzeThrottling(rows)
-	maxGPUTemp := analyzeMaxTemp(rows, func(r FanStressRow) float64 {
+	// ── Verdict.
+	statuses := readFanStatuses()
+	var summary strings.Builder
+	fmt.Fprintf(&summary, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339))
+	fmt.Fprintf(&summary, "fans_total=%d\n", len(baseFans))
+	fmt.Fprintf(&summary, "active_load_sources=%d\n", activeLoads)
+	fmt.Fprintf(&summary, "gpu_vendor=%s\n", orNone(haveGPU, vendor))
+	fmt.Fprintf(&summary, "plateau_reached=%v\n", plateauReached)
+	fmt.Fprintf(&summary, "ramp_confirmed=%v\n", rampConfirmed)
+	if len(rows) > 0 {
+		fmt.Fprintf(&summary, "load_duration_sec=%.0f\n", rows[len(rows)-1].ElapsedSec)
+	}
+	fmt.Fprintf(&summary, "max_gpu_temp_c=%.1f\n", analyzeMaxTemp(rows, func(r FanStressRow) float64 {
 		var m float64
 		for _, g := range r.GPUs {
 			if g.TempC > m {
@@ -222,55 +315,148 @@ func (s *System) RunFanStressTest(ctx context.Context, baseDir string, opts FanS
 			}
 		}
 		return m
-	})
-	maxCPUTemp := analyzeMaxTemp(rows, func(r FanStressRow) float64 {
-		return r.CPUMaxTempC
-	})
-	fanResponseSec := analyzeFanResponse(rows)
+	}))
+	fmt.Fprintf(&summary, "max_cpu_temp_c=%.1f\n", analyzeMaxTemp(rows, func(r FanStressRow) float64 { return r.CPUMaxTempC }))

-	fmt.Fprintf(&summary, "throttling_detected=%v\n", throttled)
-	fmt.Fprintf(&summary, "max_gpu_temp_c=%.1f\n", maxGPUTemp)
-	fmt.Fprintf(&summary, "max_cpu_temp_c=%.1f\n", maxCPUTemp)
-	if fanResponseSec >= 0 {
-		fmt.Fprintf(&summary, "fan_response_sec=%.1f\n", fanResponseSec)
-	} else {
-		fmt.Fprintf(&summary, "fan_response_sec=N/A\n")
+	stats := satStats{}
+	names := make([]string, 0, len(baselineRPM))
+	for n := range baselineRPM {
+		names = append(names, n)
 	}
-
-	// Throttling failure counts against overall result.
-	if throttled {
-		stats.Failed++
+	sort.Strings(names)
+	for _, name := range names {
+		peak := baselineRPM[name]
+		if st := fanBy[name]; st != nil {
+			peak = st.peak
+		}
+		st := strings.ToLower(strings.TrimSpace(statuses[name]))
+		bad := peak <= 0 || st == "cr" || st == "nr"
+		key := sanitizeSummaryKey(name)
+		fmt.Fprintf(&summary, "fan_%s_baseline_rpm=%.0f\n", key, baselineRPM[name])
+		fmt.Fprintf(&summary, "fan_%s_max_rpm=%.0f\n", key, peak)
+		if bad {
+			reason := "0 RPM under load"
+			if st == "cr" || st == "nr" {
+				reason = "IPMI status " + st
+			}
+			fmt.Fprintf(&summary, "fan_%s_status=FAILED (%s)\n", key, reason)
+			logFunc(fmt.Sprintf("FAIL %s: %s", name, reason))
+			stats.Failed++
+		} else {
+			fmt.Fprintf(&summary, "fan_%s_status=OK\n", key)
+			stats.OK++
+		}
 	}
 	writeSATStats(&summary, stats)

-	// Write CSV outputs.
-	if err := WriteFanStressCSV(filepath.Join(runDir, "metrics.csv"), rows, opts.GPUIndices); err != nil {
-		return "", err
-	}
+	_ = WriteFanStressCSV(filepath.Join(runDir, "metrics.csv"), rows, opts.GPUIndices)
 	_ = WriteFanSensorsCSV(filepath.Join(runDir, "fan-sensors.csv"), rows)
-
 	if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary.String()), 0644); err != nil {
 		return "", err
 	}
-
 	return runDir, nil
 }

-func applyFanStressDefaults(opts *FanStressOptions) {
-	if opts.BaselineSec <= 0 {
-		opts.BaselineSec = 30
+func applyFanCheckDefaults(o *FanCheckOptions) {
+	if o.PlateauHoldSec <= 0 {
+		o.PlateauHoldSec = 60
 	}
-	if opts.Phase1DurSec <= 0 {
-		opts.Phase1DurSec = 300
+	if o.PlateauDeltaRPM <= 0 {
+		o.PlateauDeltaRPM = 50
 	}
-	if opts.PauseSec <= 0 {
-		opts.PauseSec = 60
+	if o.MinLoadSec <= 0 {
+		o.MinLoadSec = 90
 	}
-	if opts.Phase2DurSec <= 0 {
-		opts.Phase2DurSec = 300
+	if o.MaxLoadSec <= 0 {
+		o.MaxLoadSec = 900
 	}
-	// SizeMB == 0 means "auto" (worker picks 95% of GPU VRAM for maximum power draw).
-	// Leave at 0 to avoid passing a too-small size that starves the tensor-core path.
+	if o.RampConfirmRPM <= 0 {
+		o.RampConfirmRPM = 150
+	}
+	if o.MinLoadSec < o.PlateauHoldSec {
+		o.MinLoadSec = o.PlateauHoldSec
+	}
+	if o.MaxLoadSec <= o.MinLoadSec {
+		o.MaxLoadSec = o.MinLoadSec + o.PlateauHoldSec
+	}
+}
+
+func orNone(present bool, v string) string {
+	if present && v != "" {
+		return v
+	}
+	return "none"
+}
+
+// sanitizeSummaryKey makes a fan sensor name safe as a summary.txt key
+// fragment (keys are parsed by splitting on '=' and whitespace).
+func sanitizeSummaryKey(name string) string {
+	var b strings.Builder
+	for _, r := range name {
+		switch {
+		case r >= 'A' && r <= 'Z', r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '-', r == '_', r == '.':
+			b.WriteRune(r)
+		default:
+			b.WriteByte('_')
+		}
+	}
+	return b.String()
+}
+
+// readFanStatuses returns the per-fan IPMI status word ("ok", "cr", "nr", ...)
+// from "ipmitool sdr type Fan". Empty map when ipmitool is unavailable.
+func readFanStatuses() map[string]string {
+	out, err := exec.Command("ipmitool", "sdr", "type", "Fan").Output()
+	if err != nil {
+		return nil
+	}
+	m := map[string]string{}
+	for _, line := range strings.Split(string(out), "\n") {
+		parts := strings.Split(line, "|")
+		if len(parts) < 3 {
+			continue
+		}
+		name := strings.TrimSpace(parts[0])
+		if name == "" {
+			continue
+		}
+		m[name] = strings.ToLower(strings.TrimSpace(parts[2]))
+	}
+	return m
+}
+
+// ResolveFanMaxRPM returns, for every fan name in current (name -> current
+// RPM), the RPM to treat as that fan's 100% reference. Preference order:
+//  1. the persisted observed peak (fanObservationStatePath), written by
+//     RunFanCheck and by live-metrics sampling under load;
+//  2. the largest peak observed on any peer fan (keeps a group visually
+//     consistent when only some fans have a recorded peak);
+//  3. the fan's own current RPM (so a tile is never sized against zero).
+//
+// The fallback lives here, not in the view, so every consumer of a fan
+// maximum applies the same rule.
+func ResolveFanMaxRPM(current map[string]float64) map[string]float64 {
+	persisted := readPersistedFanMaxRPM()
+
+	peerMax := 0.0
+	for _, v := range persisted {
+		if v > peerMax {
+			peerMax = v
+		}
+	}
+
+	out := make(map[string]float64, len(current))
+	for name, rpm := range current {
+		switch {
+		case persisted[name] > 0:
+			out[name] = persisted[name]
+		case peerMax > 0:
+			out[name] = peerMax
+		default:
+			out[name] = rpm
+		}
+	}
+	return out
 }

 // sampleFanStressRow collects all metrics for one telemetry sample.
@@ -355,27 +541,48 @@ func sampleFanSpeeds() ([]FanReading, error) {
 	return nil, sensorsErr
 }

-func loadFanObservationLocked() {
-	if fanObservationInit {
-		return
-	}
-	fanObservationInit = true
-	fanObservation.MaxRPM = make(map[string]float64)
+// readPersistedFanMaxRPM reads fanObservationStatePath and returns its
+// sanitized {fan name -> observed peak RPM} map (empty names / non-positive
+// values dropped). Returns an empty map when the file is missing or unparsable.
+func readPersistedFanMaxRPM() map[string]float64 {
+	out := map[string]float64{}
 	raw, err := os.ReadFile(fanObservationStatePath)
 	if err != nil || len(raw) == 0 {
-		return
+		return out
 	}
 	var persisted fanObservationState
 	if json.Unmarshal(raw, &persisted) != nil {
-		return
+		return out
 	}
 	for name, rpm := range persisted.MaxRPM {
 		name = strings.TrimSpace(name)
 		if name == "" || rpm <= 0 {
 			continue
 		}
-		fanObservation.MaxRPM[name] = rpm
+		out[name] = rpm
 	}
+	return out
+}
+
+// ObservedFanMaxRPM returns the per-fan observed peak RPM map persisted by
+// fan-stress SAT runs, or nil if none is recorded yet. It reads the file
+// directly without touching the in-process observation cache or its lock, so
+// read-only consumers (the /topo web view) can call it without perturbing a
+// concurrent SAT run's peak tracking.
+func ObservedFanMaxRPM() map[string]float64 {
+	out := readPersistedFanMaxRPM()
+	if len(out) == 0 {
+		return nil
+	}
+	return out
+}
+
+func loadFanObservationLocked() {
+	if fanObservationInit {
+		return
+	}
+	fanObservationInit = true
+	fanObservation.MaxRPM = readPersistedFanMaxRPM()
 }

 func saveFanObservationLocked() {
@@ -779,22 +986,6 @@ func effectiveSystemPowerReading(cache cachedPowerReading, current float64, sour
 	return 0, cache
 }

-// analyzeThrottling returns true if any GPU reported an active throttle reason
-// during either load phase.
-func analyzeThrottling(rows []FanStressRow) bool {
-	for _, row := range rows {
-		if row.Phase != "load1" && row.Phase != "load2" {
-			continue
-		}
-		for _, gpu := range row.GPUs {
-			if gpu.Throttled {
-				return true
-			}
-		}
-	}
-	return false
-}
-
 // analyzeMaxTemp returns the maximum value of the given extractor across all rows.
 func analyzeMaxTemp(rows []FanStressRow, extract func(FanStressRow) float64) float64 {
 	var max float64
@@ -806,55 +997,6 @@ func analyzeMaxTemp(rows []FanStressRow, extract func(FanStressRow) float64) flo
 	return max
 }

-// analyzeFanResponse returns the seconds from load1 start until fan RPM first
-// increased by more than 5% above the baseline average. Returns -1 if undetermined.
-func analyzeFanResponse(rows []FanStressRow) float64 {
-	// Compute baseline average fan RPM.
-	var baseTotal, baseCount float64
-	for _, row := range rows {
-		if row.Phase != "baseline" {
-			continue
-		}
-		for _, f := range row.Fans {
-			baseTotal += f.RPM
-			baseCount++
-		}
-	}
-	if baseCount == 0 || baseTotal == 0 {
-		return -1
-	}
-	baseAvg := baseTotal / baseCount
-	threshold := baseAvg * 1.05 // 5% increase signals fan ramp-up
-
-	// Find elapsed time when load1 started.
-	var load1Start float64 = -1
-	for _, row := range rows {
-		if row.Phase == "load1" {
-			load1Start = row.ElapsedSec
-			break
-		}
-	}
-	if load1Start < 0 {
-		return -1
-	}
-
-	// Find first load1 row where average RPM crosses the threshold.
-	for _, row := range rows {
-		if row.Phase != "load1" {
-			continue
-		}
-		var total, count float64
-		for _, f := range row.Fans {
-			total += f.RPM
-			count++
-		}
-		if count > 0 && total/count >= threshold {
-			return row.ElapsedSec - load1Start
-		}
-	}
-	return -1
-}
-
 // WriteFanStressCSV writes the wide-format metrics CSV with one row per second.
 // GPU columns are generated per index in gpuIndices order.
 func WriteFanStressCSV(path string, rows []FanStressRow, gpuIndices []int) error {
@@ -1,11 +1,68 @@
 package platform

 import (
+	"os"
 	"path/filepath"
+	"reflect"
 	"testing"
 	"time"
 )

+func TestResolveFanMaxRPM(t *testing.T) {
+	oldPath := fanObservationStatePath
+	oldInit := fanObservationInit
+	fanObservationStatePath = filepath.Join(t.TempDir(), "fan-observation.json")
+	fanObservationInit = false
+	t.Cleanup(func() {
+		fanObservationStatePath = oldPath
+		fanObservationInit = oldInit
+	})
+
+	// No persisted file yet: unknown fans fall back to their own current RPM.
+	got := ResolveFanMaxRPM(map[string]float64{"A": 4000, "B": 9000})
+	if !reflect.DeepEqual(got, map[string]float64{"A": 4000, "B": 9000}) {
+		t.Fatalf("no-persist fallback: got %v", got)
+	}
+
+	if err := os.WriteFile(fanObservationStatePath, []byte(`{"max_rpm":{"A":17000}}`), 0644); err != nil {
+		t.Fatal(err)
+	}
+	got = ResolveFanMaxRPM(map[string]float64{"A": 4000, "B": 9000, "C": 5000})
+	// A: persisted peak. B/C: no own entry -> largest peer peak (A's 17000).
+	if want := map[string]float64{"A": 17000, "B": 17000, "C": 17000}; !reflect.DeepEqual(got, want) {
+		t.Fatalf("peer fallback: got %v want %v", got, want)
+	}
+}
+
+func TestApplyFanCheckDefaults(t *testing.T) {
+	var o FanCheckOptions
+	applyFanCheckDefaults(&o)
+	if o.PlateauHoldSec != 60 || o.PlateauDeltaRPM != 50 || o.MinLoadSec != 90 || o.MaxLoadSec != 900 || o.RampConfirmRPM != 150 {
+		t.Fatalf("unexpected defaults: %+v", o)
+	}
+	o = FanCheckOptions{PlateauHoldSec: 120, MinLoadSec: 30, MaxLoadSec: 40}
+	applyFanCheckDefaults(&o)
+	if o.MinLoadSec < o.PlateauHoldSec {
+		t.Fatalf("MinLoadSec must be >= PlateauHoldSec, got %d", o.MinLoadSec)
+	}
+	if o.MaxLoadSec <= o.MinLoadSec {
+		t.Fatalf("MaxLoadSec must exceed MinLoadSec, got %d", o.MaxLoadSec)
+	}
+}
+
+func TestSanitizeSummaryKey(t *testing.T) {
+	for in, want := range map[string]string{
+		"F2U-1":          "F2U-1",
+		"aspeed / fan1":  "aspeed___fan1",
+		"CPU0_DIMM":      "CPU0_DIMM",
+		"weird=key here": "weird_key_here",
+	} {
+		if got := sanitizeSummaryKey(in); got != want {
+			t.Errorf("sanitizeSummaryKey(%q)=%q want %q", in, got, want)
+		}
+	}
+}
+
 func TestParseFanSpeeds(t *testing.T) {
 	raw := "FAN1 | 2400.000 | RPM | ok\nFAN2 | 1800 RPM | ok | ok\nFAN3 | na | RPM | ns\n"
 	got := parseFanSpeeds(raw)