feat(sat): fan ceiling check + topology fan tiles
Repurpose the previously-unwired RunFanStressTest into RunFanCheck, a
Load-tier SAT test that drives stressapptest (CPU+memory) and, when a GPU
is present, a GPU burn to 100% simultaneously, then watches every fan
until none has climbed for ~60s. The observed peak RPM per fan is the
"ceiling"; it is persisted through the existing fan-observation store.
MSI G4201 / AMI MegaRAC exposes no host-side fan force (every OEM IPMI
command returns 0xc1; Redfish Thermal is GET-only), so load-driven ramp
is the closest safe equivalent. See
bible-local/decisions/2026-09-04-fan-ceiling-check.md.
- platform.ResolveFanMaxRPM: per-fan max with fallback (persisted peak ->
peer peak -> current RPM), resolved in platform, not the view.
- platform.ErrTestNotApplicable: no load source or no fan sensors ->
task lands as cancelled ("not applicable"), never failed, so an
engineer never sees a false red. executeTaskWithOptions maps the
sentinel; finalizeTaskForResult honours a pre-set TaskCancelled.
- Verdict FAIL only for a fan at 0 RPM / IPMI cr-nr under load.
- /topo: one small spinning square per fan, sized by RPM / resolved max,
clickable through to a new "fan" component-detail type; per-fan status
recorded to the component-status DB from the fan SAT summary.
- Wiring: /api/sat/fan/run route, "fan" task target, Load-page card,
stress-mode Run All.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019VHG21rgTUiR1G3qFHTVmN
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
e4f7519ef3
commit
bb2a501a28
@@ -15,14 +15,18 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// FanStressOptions configures the fan-stress / thermal cycling test.
|
||||
type FanStressOptions struct {
|
||||
BaselineSec int // idle monitoring before and after load (default 30)
|
||||
Phase1DurSec int // first load phase duration in seconds (default 300)
|
||||
PauseSec int // pause between the two load phases (default 60)
|
||||
Phase2DurSec int // second load phase duration in seconds (default 300)
|
||||
SizeMB int // GPU memory to allocate per GPU during stress (0 = auto: 95% of VRAM)
|
||||
GPUIndices []int // which GPU indices to stress (empty = all detected)
|
||||
// FanCheckOptions configures the fan-ceiling check: it drives CPU (+memory)
|
||||
// and, when present, GPU load to 100% simultaneously, then watches every fan
|
||||
// until none has climbed for PlateauHoldSec — at which point each fan is
|
||||
// considered to be at its physical ceiling and the observed peak is recorded.
|
||||
type FanCheckOptions struct {
|
||||
PlateauHoldSec int // a fan must not rise > PlateauDeltaRPM for this long to count as plateaued (default 60)
|
||||
PlateauDeltaRPM int // RPM increase that still counts as "climbing" (default 50)
|
||||
MinLoadSec int // never declare a plateau before this many seconds of load (default 90)
|
||||
MaxLoadSec int // hard cap on the load phase; finish (success) even if not every fan plateaued (default 900)
|
||||
RampConfirmRPM int // at least one fan must exceed baseline by this before a plateau is "real" (default 150)
|
||||
SizeMB int // GPU memory to allocate per GPU (0 = auto)
|
||||
GPUIndices []int // which GPU indices to load (empty = all detected)
|
||||
}
|
||||
|
||||
// FanReading holds one fan sensor reading.
|
||||
@@ -93,128 +97,217 @@ func normalizeObservedFanMaxRPM(rpm float64) float64 {
|
||||
return math.Ceil(rpm/1000.0) * 1000.0
|
||||
}
|
||||
|
||||
// RunFanStressTest runs a two-phase GPU stress test while monitoring fan speeds,
|
||||
// temperatures, and power draw every second. Exports metrics.csv and fan-sensors.csv.
|
||||
// Designed to reproduce case-04 fan-speed lag and detect GPU thermal throttling.
|
||||
func (s *System) RunFanStressTest(ctx context.Context, baseDir string, opts FanStressOptions) (string, error) {
|
||||
// RunFanCheck drives CPU (+memory) and, when a GPU is present, GPU load to
|
||||
// 100% simultaneously and watches every fan until none has climbed for
|
||||
// PlateauHoldSec. At that point each fan is taken to be at its physical
|
||||
// ceiling; the observed peak RPM is persisted (fanObservationStatePath, the
|
||||
// same store ObservedFanMaxRPM reads) so the topology view can size each fan
|
||||
// tile against a real maximum.
|
||||
//
|
||||
// Outcome:
|
||||
// - success ("ceiling found") once every fan plateaus, or when MaxLoadSec is
|
||||
// hit — a run that simply ran out of time still recorded the highest RPM
|
||||
// seen and is not a failure.
|
||||
// - a fan reading 0 RPM, or an IPMI status of cr/nr, while under full load is
|
||||
// a real defect → FAILED.
|
||||
// - if there is no way to load this box (no stressapptest/stress-ng and no
|
||||
// GPU burn tool) or no fan sensors are readable, the test cannot say
|
||||
// anything about the hardware and returns ErrTestNotApplicable so the task
|
||||
// is cancelled, not failed.
|
||||
//
|
||||
// No GPU is not an error: CPU/memory load alone is enough to exercise the
|
||||
// cooling loop on most platforms.
|
||||
func (s *System) RunFanCheck(ctx context.Context, baseDir string, opts FanCheckOptions, logFunc func(string)) (string, error) {
|
||||
if logFunc == nil {
|
||||
logFunc = func(string) {}
|
||||
}
|
||||
if baseDir == "" {
|
||||
baseDir = "/var/log/bee-sat"
|
||||
}
|
||||
applyFanStressDefaults(&opts)
|
||||
applyFanCheckDefaults(&opts)
|
||||
|
||||
baseFans, fanErr := sampleFanSpeeds()
|
||||
if len(baseFans) == 0 {
|
||||
return "", fmt.Errorf("no fan sensors readable via ipmitool or lm-sensors (%v): %w", fanErr, ErrTestNotApplicable)
|
||||
}
|
||||
baselineRPM := make(map[string]float64, len(baseFans))
|
||||
for _, f := range baseFans {
|
||||
baselineRPM[f.Name] = f.RPM
|
||||
}
|
||||
|
||||
vendor := s.DetectGPUVendor()
|
||||
haveGPU := vendor == "nvidia" || vendor == "amd"
|
||||
_, cpuPathErr := satLookPath("stressapptest")
|
||||
if cpuPathErr != nil {
|
||||
_, cpuPathErr = satLookPath("stress-ng")
|
||||
}
|
||||
haveCPU := cpuPathErr == nil
|
||||
if !haveCPU && !haveGPU {
|
||||
return "", fmt.Errorf("no load source: stressapptest/stress-ng missing and no NVIDIA/AMD GPU stress tool available: %w", ErrTestNotApplicable)
|
||||
}
|
||||
|
||||
ts := time.Now().UTC().Format("20060102-150405")
|
||||
runDir := filepath.Join(baseDir, "fan-stress-"+ts)
|
||||
runDir := filepath.Join(baseDir, "fan-check-"+ts)
|
||||
if err := os.MkdirAll(runDir, 0755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
verboseLog := filepath.Join(runDir, "verbose.log")
|
||||
appendSATVerboseLog(verboseLog, fmt.Sprintf("[%s] fan check start: %d fans, gpu=%s cpu=%v",
|
||||
time.Now().UTC().Format(time.RFC3339), len(baseFans), vendor, haveCPU))
|
||||
logFunc(fmt.Sprintf("Fan check: %d fans; load = CPU/mem:%v + GPU:%s", len(baseFans), haveCPU, orNone(haveGPU, vendor)))
|
||||
|
||||
// Phase name shared between sampler goroutine and main goroutine.
|
||||
var phaseMu sync.Mutex
|
||||
currentPhase := "init"
|
||||
setPhase := func(name string) {
|
||||
phaseMu.Lock()
|
||||
currentPhase = name
|
||||
phaseMu.Unlock()
|
||||
// ── Load: every source runs at the same time, each in its own goroutine.
|
||||
// Sources can take different amounts of time to actually reach full load
|
||||
// (stressapptest is near-instant; a GPU burn kernel needs to compile and
|
||||
// ramp), so the plateau clock does not start until every launched source
|
||||
// has reported its process running, plus a fixed GPU ramp grace.
|
||||
loadCtx, loadCancel := context.WithTimeout(ctx, time.Duration(opts.MaxLoadSec)*time.Second)
|
||||
defer loadCancel()
|
||||
|
||||
var loadWG sync.WaitGroup
|
||||
started := make(chan bool, 2) // true = source is running, false = failed to launch
|
||||
launched := 0
|
||||
|
||||
if haveCPU {
|
||||
launched++
|
||||
loadWG.Add(1)
|
||||
go func() {
|
||||
defer loadWG.Done()
|
||||
cmd, err := buildCPUStressCmd(loadCtx)
|
||||
if err != nil {
|
||||
logFunc("CPU/memory load failed to start: " + err.Error())
|
||||
appendSATVerboseLog(verboseLog, "cpu load start error: "+err.Error())
|
||||
started <- false
|
||||
return
|
||||
}
|
||||
logFunc("CPU/memory load running (stressapptest)")
|
||||
started <- true
|
||||
_ = cmd.Wait()
|
||||
}()
|
||||
}
|
||||
getPhase := func() string {
|
||||
phaseMu.Lock()
|
||||
defer phaseMu.Unlock()
|
||||
return currentPhase
|
||||
if haveGPU {
|
||||
launched++
|
||||
loadWG.Add(1)
|
||||
go func() {
|
||||
defer loadWG.Done()
|
||||
cmd := buildGPUStressCmd(loadCtx, vendor, opts.MaxLoadSec)
|
||||
if cmd == nil {
|
||||
logFunc("GPU load unavailable (no burn tool for " + vendor + ")")
|
||||
appendSATVerboseLog(verboseLog, "gpu load: no burn tool")
|
||||
started <- false
|
||||
return
|
||||
}
|
||||
logFunc("GPU load running (" + vendor + ")")
|
||||
started <- true
|
||||
_ = cmd.Wait()
|
||||
}()
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
var rowsMu sync.Mutex
|
||||
var allRows []FanStressRow
|
||||
|
||||
// Start background sampler (every second).
|
||||
stopCh := make(chan struct{})
|
||||
doneCh := make(chan struct{})
|
||||
go func() {
|
||||
defer close(doneCh)
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-stopCh:
|
||||
return
|
||||
case <-ticker.C:
|
||||
row := sampleFanStressRow(opts.GPUIndices, getPhase(), time.Since(start).Seconds())
|
||||
rowsMu.Lock()
|
||||
allRows = append(allRows, row)
|
||||
rowsMu.Unlock()
|
||||
activeLoads := 0
|
||||
for i := 0; i < launched; i++ {
|
||||
select {
|
||||
case ok := <-started:
|
||||
if ok {
|
||||
activeLoads++
|
||||
}
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}()
|
||||
}
|
||||
if activeLoads == 0 {
|
||||
loadCancel()
|
||||
loadWG.Wait()
|
||||
return "", fmt.Errorf("every load source failed to start: %w", ErrTestNotApplicable)
|
||||
}
|
||||
readyAt := time.Now()
|
||||
if haveGPU {
|
||||
readyAt = readyAt.Add(20 * time.Second) // GPU kernel ramp grace
|
||||
}
|
||||
appendSATVerboseLog(verboseLog, fmt.Sprintf("%d load source(s) active; plateau clock effective from +%.0fs",
|
||||
activeLoads, readyAt.Sub(start).Seconds()))
|
||||
|
||||
var summary strings.Builder
|
||||
fmt.Fprintf(&summary, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339))
|
||||
// ── Sample loop: one row/second, per-fan plateau tracking.
|
||||
type fanState struct {
|
||||
peak float64
|
||||
lastRiseSec float64
|
||||
}
|
||||
fanBy := map[string]*fanState{}
|
||||
var rows []FanStressRow
|
||||
rampConfirmed := false
|
||||
plateauReached := false
|
||||
aborted := false
|
||||
|
||||
stats := satStats{}
|
||||
|
||||
// idlePhase sleeps for durSec while the sampler stamps phaseName on each row.
|
||||
idlePhase := func(phaseName, stepName string, durSec int) {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
setPhase(phaseName)
|
||||
appendSATVerboseLog(verboseLog,
|
||||
fmt.Sprintf("[%s] start %s (idle %ds)", time.Now().UTC().Format(time.RFC3339), stepName, durSec),
|
||||
)
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
loop:
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(time.Duration(durSec) * time.Second):
|
||||
aborted = true
|
||||
break loop
|
||||
case <-loadCtx.Done():
|
||||
break loop // MaxLoadSec reached
|
||||
case <-ticker.C:
|
||||
}
|
||||
appendSATVerboseLog(verboseLog,
|
||||
fmt.Sprintf("[%s] finish %s", time.Now().UTC().Format(time.RFC3339), stepName),
|
||||
)
|
||||
fmt.Fprintf(&summary, "%s_status=OK\n", stepName)
|
||||
stats.OK++
|
||||
}
|
||||
elapsed := time.Since(start).Seconds()
|
||||
row := sampleFanStressRow(opts.GPUIndices, "load", elapsed)
|
||||
rows = append(rows, row)
|
||||
|
||||
// loadPhase runs bee-gpu-burn for durSec; sampler stamps phaseName on each row.
|
||||
loadPhase := func(phaseName, stepName string, durSec int) {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
for _, f := range row.Fans {
|
||||
st := fanBy[f.Name]
|
||||
if st == nil {
|
||||
fanBy[f.Name] = &fanState{peak: f.RPM, lastRiseSec: elapsed}
|
||||
continue
|
||||
}
|
||||
if f.RPM > st.peak {
|
||||
if f.RPM-st.peak > float64(opts.PlateauDeltaRPM) {
|
||||
st.lastRiseSec = elapsed
|
||||
}
|
||||
st.peak = f.RPM
|
||||
}
|
||||
if f.RPM >= baselineRPM[f.Name]+float64(opts.RampConfirmRPM) {
|
||||
rampConfirmed = true
|
||||
}
|
||||
}
|
||||
setPhase(phaseName)
|
||||
cmd := []string{
|
||||
"bee-gpu-burn",
|
||||
"--seconds", strconv.Itoa(durSec),
|
||||
"--size-mb", strconv.Itoa(opts.SizeMB),
|
||||
}
|
||||
if len(opts.GPUIndices) > 0 {
|
||||
cmd = append(cmd, "--devices", joinIndexList(dedupeSortedIndices(opts.GPUIndices)))
|
||||
}
|
||||
out, err := runSATCommandCtx(ctx, verboseLog, stepName, cmd, nil, nil)
|
||||
_ = os.WriteFile(filepath.Join(runDir, stepName+".log"), out, 0644)
|
||||
if err != nil && err != context.Canceled && err.Error() != "signal: killed" {
|
||||
fmt.Fprintf(&summary, "%s_status=FAILED\n", stepName)
|
||||
stats.Failed++
|
||||
} else {
|
||||
fmt.Fprintf(&summary, "%s_status=OK\n", stepName)
|
||||
stats.OK++
|
||||
|
||||
if elapsed >= float64(opts.MinLoadSec) && time.Since(readyAt) >= time.Duration(opts.PlateauHoldSec)*time.Second && len(fanBy) > 0 {
|
||||
allFlat := true
|
||||
for _, st := range fanBy {
|
||||
if elapsed-st.lastRiseSec < float64(opts.PlateauHoldSec) {
|
||||
allFlat = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if allFlat && rampConfirmed {
|
||||
plateauReached = true
|
||||
logFunc(fmt.Sprintf("All %d fans plateaued at %.0fs of load", len(fanBy), elapsed))
|
||||
break loop
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Execute test phases.
|
||||
idlePhase("baseline", "01-baseline", opts.BaselineSec)
|
||||
loadPhase("load1", "02-load1", opts.Phase1DurSec)
|
||||
idlePhase("pause", "03-pause", opts.PauseSec)
|
||||
loadPhase("load2", "04-load2", opts.Phase2DurSec)
|
||||
idlePhase("cooldown", "05-cooldown", opts.BaselineSec)
|
||||
loadCancel()
|
||||
loadWG.Wait()
|
||||
|
||||
// Stop sampler and collect rows.
|
||||
close(stopCh)
|
||||
<-doneCh
|
||||
if aborted && ctx.Err() != nil {
|
||||
_ = os.WriteFile(filepath.Join(runDir, "summary.txt"),
|
||||
[]byte("run_at_utc="+time.Now().UTC().Format(time.RFC3339)+"\noverall_status=UNKNOWN\naborted=true\n"), 0644)
|
||||
return runDir, ctx.Err()
|
||||
}
|
||||
|
||||
rowsMu.Lock()
|
||||
rows := allRows
|
||||
rowsMu.Unlock()
|
||||
|
||||
// Analysis.
|
||||
throttled := analyzeThrottling(rows)
|
||||
maxGPUTemp := analyzeMaxTemp(rows, func(r FanStressRow) float64 {
|
||||
// ── Verdict.
|
||||
statuses := readFanStatuses()
|
||||
var summary strings.Builder
|
||||
fmt.Fprintf(&summary, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339))
|
||||
fmt.Fprintf(&summary, "fans_total=%d\n", len(baseFans))
|
||||
fmt.Fprintf(&summary, "active_load_sources=%d\n", activeLoads)
|
||||
fmt.Fprintf(&summary, "gpu_vendor=%s\n", orNone(haveGPU, vendor))
|
||||
fmt.Fprintf(&summary, "plateau_reached=%v\n", plateauReached)
|
||||
fmt.Fprintf(&summary, "ramp_confirmed=%v\n", rampConfirmed)
|
||||
if len(rows) > 0 {
|
||||
fmt.Fprintf(&summary, "load_duration_sec=%.0f\n", rows[len(rows)-1].ElapsedSec)
|
||||
}
|
||||
fmt.Fprintf(&summary, "max_gpu_temp_c=%.1f\n", analyzeMaxTemp(rows, func(r FanStressRow) float64 {
|
||||
var m float64
|
||||
for _, g := range r.GPUs {
|
||||
if g.TempC > m {
|
||||
@@ -222,55 +315,148 @@ func (s *System) RunFanStressTest(ctx context.Context, baseDir string, opts FanS
|
||||
}
|
||||
}
|
||||
return m
|
||||
})
|
||||
maxCPUTemp := analyzeMaxTemp(rows, func(r FanStressRow) float64 {
|
||||
return r.CPUMaxTempC
|
||||
})
|
||||
fanResponseSec := analyzeFanResponse(rows)
|
||||
}))
|
||||
fmt.Fprintf(&summary, "max_cpu_temp_c=%.1f\n", analyzeMaxTemp(rows, func(r FanStressRow) float64 { return r.CPUMaxTempC }))
|
||||
|
||||
fmt.Fprintf(&summary, "throttling_detected=%v\n", throttled)
|
||||
fmt.Fprintf(&summary, "max_gpu_temp_c=%.1f\n", maxGPUTemp)
|
||||
fmt.Fprintf(&summary, "max_cpu_temp_c=%.1f\n", maxCPUTemp)
|
||||
if fanResponseSec >= 0 {
|
||||
fmt.Fprintf(&summary, "fan_response_sec=%.1f\n", fanResponseSec)
|
||||
} else {
|
||||
fmt.Fprintf(&summary, "fan_response_sec=N/A\n")
|
||||
stats := satStats{}
|
||||
names := make([]string, 0, len(baselineRPM))
|
||||
for n := range baselineRPM {
|
||||
names = append(names, n)
|
||||
}
|
||||
|
||||
// Throttling failure counts against overall result.
|
||||
if throttled {
|
||||
stats.Failed++
|
||||
sort.Strings(names)
|
||||
for _, name := range names {
|
||||
peak := baselineRPM[name]
|
||||
if st := fanBy[name]; st != nil {
|
||||
peak = st.peak
|
||||
}
|
||||
st := strings.ToLower(strings.TrimSpace(statuses[name]))
|
||||
bad := peak <= 0 || st == "cr" || st == "nr"
|
||||
key := sanitizeSummaryKey(name)
|
||||
fmt.Fprintf(&summary, "fan_%s_baseline_rpm=%.0f\n", key, baselineRPM[name])
|
||||
fmt.Fprintf(&summary, "fan_%s_max_rpm=%.0f\n", key, peak)
|
||||
if bad {
|
||||
reason := "0 RPM under load"
|
||||
if st == "cr" || st == "nr" {
|
||||
reason = "IPMI status " + st
|
||||
}
|
||||
fmt.Fprintf(&summary, "fan_%s_status=FAILED (%s)\n", key, reason)
|
||||
logFunc(fmt.Sprintf("FAIL %s: %s", name, reason))
|
||||
stats.Failed++
|
||||
} else {
|
||||
fmt.Fprintf(&summary, "fan_%s_status=OK\n", key)
|
||||
stats.OK++
|
||||
}
|
||||
}
|
||||
writeSATStats(&summary, stats)
|
||||
|
||||
// Write CSV outputs.
|
||||
if err := WriteFanStressCSV(filepath.Join(runDir, "metrics.csv"), rows, opts.GPUIndices); err != nil {
|
||||
return "", err
|
||||
}
|
||||
_ = WriteFanStressCSV(filepath.Join(runDir, "metrics.csv"), rows, opts.GPUIndices)
|
||||
_ = WriteFanSensorsCSV(filepath.Join(runDir, "fan-sensors.csv"), rows)
|
||||
|
||||
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary.String()), 0644); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return runDir, nil
|
||||
}
|
||||
|
||||
func applyFanStressDefaults(opts *FanStressOptions) {
|
||||
if opts.BaselineSec <= 0 {
|
||||
opts.BaselineSec = 30
|
||||
func applyFanCheckDefaults(o *FanCheckOptions) {
|
||||
if o.PlateauHoldSec <= 0 {
|
||||
o.PlateauHoldSec = 60
|
||||
}
|
||||
if opts.Phase1DurSec <= 0 {
|
||||
opts.Phase1DurSec = 300
|
||||
if o.PlateauDeltaRPM <= 0 {
|
||||
o.PlateauDeltaRPM = 50
|
||||
}
|
||||
if opts.PauseSec <= 0 {
|
||||
opts.PauseSec = 60
|
||||
if o.MinLoadSec <= 0 {
|
||||
o.MinLoadSec = 90
|
||||
}
|
||||
if opts.Phase2DurSec <= 0 {
|
||||
opts.Phase2DurSec = 300
|
||||
if o.MaxLoadSec <= 0 {
|
||||
o.MaxLoadSec = 900
|
||||
}
|
||||
// SizeMB == 0 means "auto" (worker picks 95% of GPU VRAM for maximum power draw).
|
||||
// Leave at 0 to avoid passing a too-small size that starves the tensor-core path.
|
||||
if o.RampConfirmRPM <= 0 {
|
||||
o.RampConfirmRPM = 150
|
||||
}
|
||||
if o.MinLoadSec < o.PlateauHoldSec {
|
||||
o.MinLoadSec = o.PlateauHoldSec
|
||||
}
|
||||
if o.MaxLoadSec <= o.MinLoadSec {
|
||||
o.MaxLoadSec = o.MinLoadSec + o.PlateauHoldSec
|
||||
}
|
||||
}
|
||||
|
||||
func orNone(present bool, v string) string {
|
||||
if present && v != "" {
|
||||
return v
|
||||
}
|
||||
return "none"
|
||||
}
|
||||
|
||||
// sanitizeSummaryKey makes a fan sensor name safe as a summary.txt key
|
||||
// fragment (keys are parsed by splitting on '=' and whitespace).
|
||||
func sanitizeSummaryKey(name string) string {
|
||||
var b strings.Builder
|
||||
for _, r := range name {
|
||||
switch {
|
||||
case r >= 'A' && r <= 'Z', r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '-', r == '_', r == '.':
|
||||
b.WriteRune(r)
|
||||
default:
|
||||
b.WriteByte('_')
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// readFanStatuses returns the per-fan IPMI status word ("ok", "cr", "nr", ...)
|
||||
// from "ipmitool sdr type Fan". Empty map when ipmitool is unavailable.
|
||||
func readFanStatuses() map[string]string {
|
||||
out, err := exec.Command("ipmitool", "sdr", "type", "Fan").Output()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
m := map[string]string{}
|
||||
for _, line := range strings.Split(string(out), "\n") {
|
||||
parts := strings.Split(line, "|")
|
||||
if len(parts) < 3 {
|
||||
continue
|
||||
}
|
||||
name := strings.TrimSpace(parts[0])
|
||||
if name == "" {
|
||||
continue
|
||||
}
|
||||
m[name] = strings.ToLower(strings.TrimSpace(parts[2]))
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// ResolveFanMaxRPM returns, for every fan name in current (name -> current
|
||||
// RPM), the RPM to treat as that fan's 100% reference. Preference order:
|
||||
// 1. the persisted observed peak (fanObservationStatePath), written by
|
||||
// RunFanCheck and by live-metrics sampling under load;
|
||||
// 2. the largest peak observed on any peer fan (keeps a group visually
|
||||
// consistent when only some fans have a recorded peak);
|
||||
// 3. the fan's own current RPM (so a tile is never sized against zero).
|
||||
//
|
||||
// The fallback lives here, not in the view, so every consumer of a fan
|
||||
// maximum applies the same rule.
|
||||
func ResolveFanMaxRPM(current map[string]float64) map[string]float64 {
|
||||
persisted := readPersistedFanMaxRPM()
|
||||
|
||||
peerMax := 0.0
|
||||
for _, v := range persisted {
|
||||
if v > peerMax {
|
||||
peerMax = v
|
||||
}
|
||||
}
|
||||
|
||||
out := make(map[string]float64, len(current))
|
||||
for name, rpm := range current {
|
||||
switch {
|
||||
case persisted[name] > 0:
|
||||
out[name] = persisted[name]
|
||||
case peerMax > 0:
|
||||
out[name] = peerMax
|
||||
default:
|
||||
out[name] = rpm
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sampleFanStressRow collects all metrics for one telemetry sample.
|
||||
@@ -355,27 +541,48 @@ func sampleFanSpeeds() ([]FanReading, error) {
|
||||
return nil, sensorsErr
|
||||
}
|
||||
|
||||
func loadFanObservationLocked() {
|
||||
if fanObservationInit {
|
||||
return
|
||||
}
|
||||
fanObservationInit = true
|
||||
fanObservation.MaxRPM = make(map[string]float64)
|
||||
// readPersistedFanMaxRPM reads fanObservationStatePath and returns its
|
||||
// sanitized {fan name -> observed peak RPM} map (empty names / non-positive
|
||||
// values dropped). Returns an empty map when the file is missing or unparsable.
|
||||
func readPersistedFanMaxRPM() map[string]float64 {
|
||||
out := map[string]float64{}
|
||||
raw, err := os.ReadFile(fanObservationStatePath)
|
||||
if err != nil || len(raw) == 0 {
|
||||
return
|
||||
return out
|
||||
}
|
||||
var persisted fanObservationState
|
||||
if json.Unmarshal(raw, &persisted) != nil {
|
||||
return
|
||||
return out
|
||||
}
|
||||
for name, rpm := range persisted.MaxRPM {
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" || rpm <= 0 {
|
||||
continue
|
||||
}
|
||||
fanObservation.MaxRPM[name] = rpm
|
||||
out[name] = rpm
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ObservedFanMaxRPM returns the per-fan observed peak RPM map persisted by
|
||||
// fan-stress SAT runs, or nil if none is recorded yet. It reads the file
|
||||
// directly without touching the in-process observation cache or its lock, so
|
||||
// read-only consumers (the /topo web view) can call it without perturbing a
|
||||
// concurrent SAT run's peak tracking.
|
||||
func ObservedFanMaxRPM() map[string]float64 {
|
||||
out := readPersistedFanMaxRPM()
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func loadFanObservationLocked() {
|
||||
if fanObservationInit {
|
||||
return
|
||||
}
|
||||
fanObservationInit = true
|
||||
fanObservation.MaxRPM = readPersistedFanMaxRPM()
|
||||
}
|
||||
|
||||
func saveFanObservationLocked() {
|
||||
@@ -779,22 +986,6 @@ func effectiveSystemPowerReading(cache cachedPowerReading, current float64, sour
|
||||
return 0, cache
|
||||
}
|
||||
|
||||
// analyzeThrottling returns true if any GPU reported an active throttle reason
|
||||
// during either load phase.
|
||||
func analyzeThrottling(rows []FanStressRow) bool {
|
||||
for _, row := range rows {
|
||||
if row.Phase != "load1" && row.Phase != "load2" {
|
||||
continue
|
||||
}
|
||||
for _, gpu := range row.GPUs {
|
||||
if gpu.Throttled {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// analyzeMaxTemp returns the maximum value of the given extractor across all rows.
|
||||
func analyzeMaxTemp(rows []FanStressRow, extract func(FanStressRow) float64) float64 {
|
||||
var max float64
|
||||
@@ -806,55 +997,6 @@ func analyzeMaxTemp(rows []FanStressRow, extract func(FanStressRow) float64) flo
|
||||
return max
|
||||
}
|
||||
|
||||
// analyzeFanResponse returns the seconds from load1 start until fan RPM first
|
||||
// increased by more than 5% above the baseline average. Returns -1 if undetermined.
|
||||
func analyzeFanResponse(rows []FanStressRow) float64 {
|
||||
// Compute baseline average fan RPM.
|
||||
var baseTotal, baseCount float64
|
||||
for _, row := range rows {
|
||||
if row.Phase != "baseline" {
|
||||
continue
|
||||
}
|
||||
for _, f := range row.Fans {
|
||||
baseTotal += f.RPM
|
||||
baseCount++
|
||||
}
|
||||
}
|
||||
if baseCount == 0 || baseTotal == 0 {
|
||||
return -1
|
||||
}
|
||||
baseAvg := baseTotal / baseCount
|
||||
threshold := baseAvg * 1.05 // 5% increase signals fan ramp-up
|
||||
|
||||
// Find elapsed time when load1 started.
|
||||
var load1Start float64 = -1
|
||||
for _, row := range rows {
|
||||
if row.Phase == "load1" {
|
||||
load1Start = row.ElapsedSec
|
||||
break
|
||||
}
|
||||
}
|
||||
if load1Start < 0 {
|
||||
return -1
|
||||
}
|
||||
|
||||
// Find first load1 row where average RPM crosses the threshold.
|
||||
for _, row := range rows {
|
||||
if row.Phase != "load1" {
|
||||
continue
|
||||
}
|
||||
var total, count float64
|
||||
for _, f := range row.Fans {
|
||||
total += f.RPM
|
||||
count++
|
||||
}
|
||||
if count > 0 && total/count >= threshold {
|
||||
return row.ElapsedSec - load1Start
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// WriteFanStressCSV writes the wide-format metrics CSV with one row per second.
|
||||
// GPU columns are generated per index in gpuIndices order.
|
||||
func WriteFanStressCSV(path string, rows []FanStressRow, gpuIndices []int) error {
|
||||
|
||||
Reference in New Issue
Block a user