fix(sat): fan check uses hottest GPU load + IPMI-hang-proof polling
- GPU load switches from bee-gpu-burn (compute burn, ~88% TDP) to dcgmproftester -t 1004 / targeted_power via resolveBenchmarkPowerLoadCommand — the same engine Power/Thermal Fit uses and the hottest sustained NVIDIA load we have, so fans are actually pushed toward their ceiling. - Sample loop is now IPMI-hang-proof: every ipmitool read is time-boxed in an abandonable goroutine, and the poll interval backs off geometrically (1s→30s) when reads are slow, tightening again on recovery. A plateau is only trusted while telemetry is healthy; degraded runs ride out to MaxLoadSec. Summary gains fan_samples / telemetry_degraded. Drops the per-second nvidia-smi+power+cpu-temp sampling from the hot loop. - Dead code removed: FanStressRow, GPUStressMetric, sampleFanStressRow, sampleGPUStressMetrics, WriteFanStressCSV/WriteFanSensorsCSV, analyzeMaxTemp, sampleSystemPowerResolved. Topology fan tiles: - size encodes the fan's ceiling RPM (its class), not current speed; coloured fill rising from the bottom encodes live duty cycle (current / ceiling), shown only when the ceiling was measured. - glyph spin rate now maps absolute RPM into a human-perceptible band (fanSpinPeriodSec: 2.2s/turn at <=1000 RPM, 0.35s at >=13000). - the "N fans · N OK · tile size ∝ …" caption line is gone. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019VHG21rgTUiR1G3qFHTVmN
This commit is contained in:
Mikhail Chusavitin
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
bb2a501a28
commit
8cb250f3f4
@@ -12,6 +12,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -35,29 +36,6 @@ type FanReading struct {
|
||||
RPM float64
|
||||
}
|
||||
|
||||
// GPUStressMetric holds per-GPU metrics during the stress test.
|
||||
type GPUStressMetric struct {
|
||||
Index int
|
||||
TempC float64
|
||||
UsagePct float64
|
||||
PowerW float64
|
||||
ClockMHz float64
|
||||
Throttled bool // true if any throttle reason is active
|
||||
}
|
||||
|
||||
// FanStressRow is one second-interval telemetry sample covering all monitored dimensions.
|
||||
type FanStressRow struct {
|
||||
TimestampUTC string
|
||||
ElapsedSec float64
|
||||
Phase string // "baseline", "load1", "pause", "load2", "cooldown"
|
||||
GPUs []GPUStressMetric
|
||||
Fans []FanReading
|
||||
CPUMaxTempC float64 // highest CPU temperature from ipmitool / sensors
|
||||
SysPowerW float64
|
||||
SysPowerSource string
|
||||
SysPowerMode string
|
||||
}
|
||||
|
||||
type cachedPowerReading struct {
|
||||
Value float64
|
||||
Source string
|
||||
@@ -76,8 +54,6 @@ type fanPeakCandidate struct {
|
||||
}
|
||||
|
||||
var (
|
||||
systemPowerCacheMu sync.Mutex
|
||||
systemPowerCache cachedPowerReading
|
||||
fanObservationMu sync.Mutex
|
||||
fanObservation fanObservationState
|
||||
fanObservationInit bool
|
||||
@@ -157,10 +133,16 @@ func (s *System) RunFanCheck(ctx context.Context, baseDir string, opts FanCheckO
|
||||
logFunc(fmt.Sprintf("Fan check: %d fans; load = CPU/mem:%v + GPU:%s", len(baseFans), haveCPU, orNone(haveGPU, vendor)))
|
||||
|
||||
// ── Load: every source runs at the same time, each in its own goroutine.
|
||||
// Sources can take different amounts of time to actually reach full load
|
||||
// (stressapptest is near-instant; a GPU burn kernel needs to compile and
|
||||
// ramp), so the plateau clock does not start until every launched source
|
||||
// has reported its process running, plus a fixed GPU ramp grace.
|
||||
// GPU load is the hottest sustained NVIDIA load we have — dcgmproftester
|
||||
// -t 1004 / targeted_power, the same engine the Power/Thermal Fit
|
||||
// benchmark uses (resolveBenchmarkPowerLoadCommand) — not the
|
||||
// compute-throughput bee-gpu-burn, which tops out well below TDP and so
|
||||
// never demands the fans' true ceiling.
|
||||
//
|
||||
// Sources reach full load at different times (stressapptest is instant; a
|
||||
// dcgmproftester kernel compiles and ramps), so the plateau clock does not
|
||||
// start until every launched source reports its process running, plus a
|
||||
// fixed GPU ramp grace.
|
||||
loadCtx, loadCancel := context.WithTimeout(ctx, time.Duration(opts.MaxLoadSec)*time.Second)
|
||||
defer loadCancel()
|
||||
|
||||
@@ -190,14 +172,20 @@ func (s *System) RunFanCheck(ctx context.Context, baseDir string, opts FanCheckO
|
||||
loadWG.Add(1)
|
||||
go func() {
|
||||
defer loadWG.Done()
|
||||
cmd := buildGPUStressCmd(loadCtx, vendor, opts.MaxLoadSec)
|
||||
if cmd == nil {
|
||||
logFunc("GPU load unavailable (no burn tool for " + vendor + ")")
|
||||
appendSATVerboseLog(verboseLog, "gpu load: no burn tool")
|
||||
cmd, label, err := buildFanCheckGPULoadCmd(loadCtx, vendor, opts.MaxLoadSec, opts.GPUIndices)
|
||||
if err != nil || cmd == nil {
|
||||
logFunc("GPU load unavailable: " + errString(err))
|
||||
appendSATVerboseLog(verboseLog, "gpu load unavailable: "+errString(err))
|
||||
started <- false
|
||||
return
|
||||
}
|
||||
logFunc("GPU load running (" + vendor + ")")
|
||||
if err := cmd.Start(); err != nil {
|
||||
logFunc("GPU load failed to start: " + err.Error())
|
||||
appendSATVerboseLog(verboseLog, "gpu load start error: "+err.Error())
|
||||
started <- false
|
||||
return
|
||||
}
|
||||
logFunc("GPU load running (" + label + ")")
|
||||
started <- true
|
||||
_ = cmd.Wait()
|
||||
}()
|
||||
@@ -226,19 +214,39 @@ func (s *System) RunFanCheck(ctx context.Context, baseDir string, opts FanCheckO
|
||||
appendSATVerboseLog(verboseLog, fmt.Sprintf("%d load source(s) active; plateau clock effective from +%.0fs",
|
||||
activeLoads, readyAt.Sub(start).Seconds()))
|
||||
|
||||
// ── Sample loop: one row/second, per-fan plateau tracking.
|
||||
// ── Sample loop with IPMI-hang protection.
|
||||
//
|
||||
// Under full load ipmitool over KCS can take tens of seconds per call or
|
||||
// wedge outright. So: every fan read is time-boxed (readFansBounded runs
|
||||
// it in a goroutine we abandon on timeout — a wedged KCS read can never
|
||||
// block this loop), and the polling interval backs off geometrically when
|
||||
// reads are slow and tightens again when they recover. A plateau is only
|
||||
// declared while telemetry is healthy (interval near the floor); a
|
||||
// degraded run just rides out to MaxLoadSec and records the peak it saw.
|
||||
const (
|
||||
fanPollFloor = 1 * time.Second
|
||||
fanPollCeil = 30 * time.Second
|
||||
fanReadTMO = 8 * time.Second
|
||||
)
|
||||
type fanState struct {
|
||||
peak float64
|
||||
lastRiseSec float64
|
||||
}
|
||||
fanBy := map[string]*fanState{}
|
||||
var rows []FanStressRow
|
||||
rampConfirmed := false
|
||||
plateauReached := false
|
||||
aborted := false
|
||||
degraded := false
|
||||
goodSamples := 0
|
||||
poll := fanPollFloor
|
||||
|
||||
csvPath := filepath.Join(runDir, "fan-sensors.csv")
|
||||
_ = os.WriteFile(csvPath, []byte("elapsed_sec,fan_name,rpm\n"), 0644)
|
||||
csvFile, _ := os.OpenFile(csvPath, os.O_APPEND|os.O_WRONLY, 0644)
|
||||
if csvFile != nil {
|
||||
defer csvFile.Close()
|
||||
}
|
||||
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
loop:
|
||||
for {
|
||||
select {
|
||||
@@ -247,13 +255,36 @@ loop:
|
||||
break loop
|
||||
case <-loadCtx.Done():
|
||||
break loop // MaxLoadSec reached
|
||||
case <-ticker.C:
|
||||
case <-time.After(poll):
|
||||
}
|
||||
elapsed := time.Since(start).Seconds()
|
||||
row := sampleFanStressRow(opts.GPUIndices, "load", elapsed)
|
||||
rows = append(rows, row)
|
||||
|
||||
for _, f := range row.Fans {
|
||||
readStart := time.Now()
|
||||
fans, ok := readFansBounded(fanReadTMO)
|
||||
readDur := time.Since(readStart)
|
||||
|
||||
// Adapt the interval to how ipmitool is behaving.
|
||||
switch {
|
||||
case !ok || readDur > fanReadTMO*3/4:
|
||||
if poll < fanPollCeil {
|
||||
poll = minDuration(poll*2, fanPollCeil)
|
||||
degraded = true
|
||||
appendSATVerboseLog(verboseLog, fmt.Sprintf("[%.0fs] ipmitool slow (%.1fs, ok=%v) — polling backed off to %s",
|
||||
elapsed, readDur.Seconds(), ok, poll))
|
||||
logFunc(fmt.Sprintf("IPMI slow — fan polling backed off to %s", poll))
|
||||
}
|
||||
case poll > fanPollFloor && readDur < fanPollFloor:
|
||||
poll = maxDuration(poll*2/3, fanPollFloor)
|
||||
}
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
goodSamples++
|
||||
|
||||
for _, f := range fans {
|
||||
if csvFile != nil {
|
||||
fmt.Fprintf(csvFile, "%.0f,%s,%.0f\n", elapsed, f.Name, f.RPM)
|
||||
}
|
||||
st := fanBy[f.Name]
|
||||
if st == nil {
|
||||
fanBy[f.Name] = &fanState{peak: f.RPM, lastRiseSec: elapsed}
|
||||
@@ -270,7 +301,11 @@ loop:
|
||||
}
|
||||
}
|
||||
|
||||
if elapsed >= float64(opts.MinLoadSec) && time.Since(readyAt) >= time.Duration(opts.PlateauHoldSec)*time.Second && len(fanBy) > 0 {
|
||||
// Only trust a plateau while telemetry is healthy and we have enough
|
||||
// recent samples to have actually seen a flat window.
|
||||
healthy := poll <= 2*fanPollFloor && goodSamples >= 5
|
||||
if healthy && rampConfirmed && elapsed >= float64(opts.MinLoadSec) &&
|
||||
time.Since(readyAt) >= time.Duration(opts.PlateauHoldSec)*time.Second && len(fanBy) > 0 {
|
||||
allFlat := true
|
||||
for _, st := range fanBy {
|
||||
if elapsed-st.lastRiseSec < float64(opts.PlateauHoldSec) {
|
||||
@@ -278,7 +313,7 @@ loop:
|
||||
break
|
||||
}
|
||||
}
|
||||
if allFlat && rampConfirmed {
|
||||
if allFlat {
|
||||
plateauReached = true
|
||||
logFunc(fmt.Sprintf("All %d fans plateaued at %.0fs of load", len(fanBy), elapsed))
|
||||
break loop
|
||||
@@ -295,6 +330,8 @@ loop:
|
||||
return runDir, ctx.Err()
|
||||
}
|
||||
|
||||
loadDur := time.Since(start).Seconds()
|
||||
|
||||
// ── Verdict.
|
||||
statuses := readFanStatuses()
|
||||
var summary strings.Builder
|
||||
@@ -304,19 +341,12 @@ loop:
|
||||
fmt.Fprintf(&summary, "gpu_vendor=%s\n", orNone(haveGPU, vendor))
|
||||
fmt.Fprintf(&summary, "plateau_reached=%v\n", plateauReached)
|
||||
fmt.Fprintf(&summary, "ramp_confirmed=%v\n", rampConfirmed)
|
||||
if len(rows) > 0 {
|
||||
fmt.Fprintf(&summary, "load_duration_sec=%.0f\n", rows[len(rows)-1].ElapsedSec)
|
||||
fmt.Fprintf(&summary, "load_duration_sec=%.0f\n", loadDur)
|
||||
fmt.Fprintf(&summary, "fan_samples=%d\n", goodSamples)
|
||||
fmt.Fprintf(&summary, "telemetry_degraded=%v\n", degraded)
|
||||
if t := boundedGPUMaxTemp(opts.GPUIndices); t > 0 {
|
||||
fmt.Fprintf(&summary, "gpu_temp_c=%.0f\n", t)
|
||||
}
|
||||
fmt.Fprintf(&summary, "max_gpu_temp_c=%.1f\n", analyzeMaxTemp(rows, func(r FanStressRow) float64 {
|
||||
var m float64
|
||||
for _, g := range r.GPUs {
|
||||
if g.TempC > m {
|
||||
m = g.TempC
|
||||
}
|
||||
}
|
||||
return m
|
||||
}))
|
||||
fmt.Fprintf(&summary, "max_cpu_temp_c=%.1f\n", analyzeMaxTemp(rows, func(r FanStressRow) float64 { return r.CPUMaxTempC }))
|
||||
|
||||
stats := satStats{}
|
||||
names := make([]string, 0, len(baselineRPM))
|
||||
@@ -349,14 +379,126 @@ loop:
|
||||
}
|
||||
writeSATStats(&summary, stats)
|
||||
|
||||
_ = WriteFanStressCSV(filepath.Join(runDir, "metrics.csv"), rows, opts.GPUIndices)
|
||||
_ = WriteFanSensorsCSV(filepath.Join(runDir, "fan-sensors.csv"), rows)
|
||||
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary.String()), 0644); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return runDir, nil
|
||||
}
|
||||
|
||||
func errString(err error) string {
|
||||
if err == nil {
|
||||
return "no GPU stress tool"
|
||||
}
|
||||
return err.Error()
|
||||
}
|
||||
|
||||
func minDuration(a, b time.Duration) time.Duration {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func maxDuration(a, b time.Duration) time.Duration {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// readFansBounded runs "ipmitool sdr type Fan" but never blocks the caller
|
||||
// longer than timeout: the read happens in a goroutine that is abandoned if it
|
||||
// does not return in time (a KCS read wedged in uninterruptible I/O cannot be
|
||||
// killed, so we leave it and move on). ok=false means "no usable sample this
|
||||
// tick" — the caller must treat that as missing data, not as a flat fan.
|
||||
func readFansBounded(timeout time.Duration) ([]FanReading, bool) {
|
||||
type result struct {
|
||||
fans []FanReading
|
||||
ok bool
|
||||
}
|
||||
ch := make(chan result, 1)
|
||||
go func() {
|
||||
out, err := exec.Command("ipmitool", "sdr", "type", "Fan").Output()
|
||||
if err != nil {
|
||||
ch <- result{}
|
||||
return
|
||||
}
|
||||
fans := parseFanSpeeds(string(out))
|
||||
if len(fans) == 0 {
|
||||
ch <- result{}
|
||||
return
|
||||
}
|
||||
ch <- result{fans, true}
|
||||
}()
|
||||
select {
|
||||
case r := <-ch:
|
||||
if r.ok {
|
||||
updateFanObservation(r.fans, time.Now())
|
||||
}
|
||||
return r.fans, r.ok
|
||||
case <-time.After(timeout):
|
||||
return nil, false
|
||||
}
|
||||
}
|
||||
|
||||
// boundedGPUMaxTemp returns the hottest GPU temperature via a single
|
||||
// time-boxed nvidia-smi call, or 0 if unavailable.
|
||||
func boundedGPUMaxTemp(gpuIndices []int) float64 {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second)
|
||||
defer cancel()
|
||||
args := []string{"--query-gpu=temperature.gpu", "--format=csv,noheader,nounits"}
|
||||
if len(gpuIndices) > 0 {
|
||||
ids := make([]string, len(gpuIndices))
|
||||
for i, idx := range gpuIndices {
|
||||
ids[i] = strconv.Itoa(idx)
|
||||
}
|
||||
args = append([]string{"--id=" + strings.Join(ids, ",")}, args...)
|
||||
}
|
||||
out, err := exec.CommandContext(ctx, "nvidia-smi", args...).Output()
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
var max float64
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||||
if v, err := strconv.ParseFloat(strings.TrimSpace(line), 64); err == nil && v > max {
|
||||
max = v
|
||||
}
|
||||
}
|
||||
return max
|
||||
}
|
||||
|
||||
// buildFanCheckGPULoadCmd builds the hottest sustained GPU load for the fan
|
||||
// check. NVIDIA uses the Power/Thermal Fit engine (dcgmproftester -t 1004 /
|
||||
// targeted_power); AMD uses the RVS gst stressor.
|
||||
func buildFanCheckGPULoadCmd(ctx context.Context, vendor string, durSec int, gpuIndices []int) (*exec.Cmd, string, error) {
|
||||
switch strings.ToLower(vendor) {
|
||||
case "nvidia":
|
||||
argv, env, err := resolveBenchmarkPowerLoadCommand(durSec, gpuIndices)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, argv[0], argv[1:]...)
|
||||
if len(env) > 0 {
|
||||
cmd.Env = append(os.Environ(), env...)
|
||||
}
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||
cmd.Cancel = func() error {
|
||||
if cmd.Process != nil {
|
||||
_ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return cmd, "dcgmproftester targeted_power", nil
|
||||
case "amd":
|
||||
cmd := buildAMDGPUStressCmd(ctx, durSec)
|
||||
if cmd == nil {
|
||||
return nil, "", nil
|
||||
}
|
||||
return cmd, "rvs gst", nil
|
||||
}
|
||||
return nil, "", nil
|
||||
}
|
||||
|
||||
func applyFanCheckDefaults(o *FanCheckOptions) {
|
||||
if o.PlateauHoldSec <= 0 {
|
||||
o.PlateauHoldSec = 60
|
||||
@@ -459,68 +601,6 @@ func ResolveFanMaxRPM(current map[string]float64) map[string]float64 {
|
||||
return out
|
||||
}
|
||||
|
||||
// sampleFanStressRow collects all metrics for one telemetry sample.
|
||||
func sampleFanStressRow(gpuIndices []int, phase string, elapsed float64) FanStressRow {
|
||||
row := FanStressRow{
|
||||
TimestampUTC: time.Now().UTC().Format(time.RFC3339),
|
||||
ElapsedSec: elapsed,
|
||||
Phase: phase,
|
||||
}
|
||||
row.GPUs = sampleGPUStressMetrics(gpuIndices)
|
||||
row.Fans, _ = sampleFanSpeeds()
|
||||
row.CPUMaxTempC = sampleCPUMaxTemp()
|
||||
row.SysPowerW, row.SysPowerSource, row.SysPowerMode = sampleSystemPowerResolved()
|
||||
return row
|
||||
}
|
||||
|
||||
// sampleGPUStressMetrics queries nvidia-smi for temperature, utilization, power,
|
||||
// clock frequency, and active throttle reasons for each GPU.
|
||||
func sampleGPUStressMetrics(gpuIndices []int) []GPUStressMetric {
|
||||
args := []string{
|
||||
"--query-gpu=index,temperature.gpu,utilization.gpu,power.draw,clocks.current.graphics,clocks_throttle_reasons.active",
|
||||
"--format=csv,noheader,nounits",
|
||||
}
|
||||
if len(gpuIndices) > 0 {
|
||||
ids := make([]string, len(gpuIndices))
|
||||
for i, idx := range gpuIndices {
|
||||
ids[i] = strconv.Itoa(idx)
|
||||
}
|
||||
args = append([]string{"--id=" + strings.Join(ids, ",")}, args...)
|
||||
}
|
||||
out, err := exec.Command("nvidia-smi", args...).Output()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var metrics []GPUStressMetric
|
||||
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
parts := strings.Split(line, ", ")
|
||||
if len(parts) < 6 {
|
||||
continue
|
||||
}
|
||||
idx, _ := strconv.Atoi(strings.TrimSpace(parts[0]))
|
||||
throttleVal := strings.TrimSpace(parts[5])
|
||||
// Throttled if active reasons bitmask is non-zero.
|
||||
throttled := throttleVal != "0x0000000000000000" &&
|
||||
throttleVal != "0x0" &&
|
||||
throttleVal != "0" &&
|
||||
throttleVal != "" &&
|
||||
throttleVal != "N/A"
|
||||
metrics = append(metrics, GPUStressMetric{
|
||||
Index: idx,
|
||||
TempC: parseGPUFloat(parts[1]),
|
||||
UsagePct: parseGPUFloat(parts[2]),
|
||||
PowerW: parseGPUFloat(parts[3]),
|
||||
ClockMHz: parseGPUFloat(parts[4]),
|
||||
Throttled: throttled,
|
||||
})
|
||||
}
|
||||
return metrics
|
||||
}
|
||||
|
||||
// sampleFanSpeeds reads fan RPM values from ipmitool sdr.
|
||||
func sampleFanSpeeds() ([]FanReading, error) {
|
||||
out, err := exec.Command("ipmitool", "sdr", "type", "Fan").Output()
|
||||
@@ -940,21 +1020,6 @@ func sampleCPUTempViaSensors() float64 {
|
||||
return max
|
||||
}
|
||||
|
||||
// sampleSystemPowerResolved reads system power via the global autotune source,
|
||||
// falling back to the historical heuristic before autotune or when degraded.
|
||||
func sampleSystemPowerResolved() (float64, string, string) {
|
||||
now := time.Now()
|
||||
current, decision, err := SampleSystemPowerResolved("")
|
||||
systemPowerCacheMu.Lock()
|
||||
defer systemPowerCacheMu.Unlock()
|
||||
if err != nil {
|
||||
current = 0
|
||||
}
|
||||
value, updated := effectiveSystemPowerReading(systemPowerCache, current, decision.EffectiveSource, decision.Mode, decision.Reason, now)
|
||||
systemPowerCache = updated
|
||||
return value, updated.Source, updated.Mode
|
||||
}
|
||||
|
||||
// parseDCMIPowerReading extracts the instantaneous power reading from ipmitool dcmi output.
|
||||
// Sample: " Instantaneous power reading: 500 Watts"
|
||||
func parseDCMIPowerReading(raw string) float64 {
|
||||
@@ -986,76 +1051,6 @@ func effectiveSystemPowerReading(cache cachedPowerReading, current float64, sour
|
||||
return 0, cache
|
||||
}
|
||||
|
||||
// analyzeMaxTemp returns the maximum value of the given extractor across all rows.
|
||||
func analyzeMaxTemp(rows []FanStressRow, extract func(FanStressRow) float64) float64 {
|
||||
var max float64
|
||||
for _, row := range rows {
|
||||
if v := extract(row); v > max {
|
||||
max = v
|
||||
}
|
||||
}
|
||||
return max
|
||||
}
|
||||
|
||||
// WriteFanStressCSV writes the wide-format metrics CSV with one row per second.
|
||||
// GPU columns are generated per index in gpuIndices order.
|
||||
func WriteFanStressCSV(path string, rows []FanStressRow, gpuIndices []int) error {
|
||||
if len(rows) == 0 {
|
||||
return os.WriteFile(path, []byte("no data\n"), 0644)
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
|
||||
// Header: fixed system columns + per-GPU columns.
|
||||
b.WriteString("timestamp_utc,elapsed_sec,phase,fan_avg_rpm,fan_min_rpm,fan_max_rpm,cpu_max_temp_c,sys_power_w")
|
||||
for _, idx := range gpuIndices {
|
||||
fmt.Fprintf(&b, ",gpu%d_temp_c,gpu%d_usage_pct,gpu%d_power_w,gpu%d_clock_mhz,gpu%d_throttled",
|
||||
idx, idx, idx, idx, idx)
|
||||
}
|
||||
b.WriteRune('\n')
|
||||
|
||||
for _, row := range rows {
|
||||
favg, fmin, fmax := fanRPMStats(row.Fans)
|
||||
fmt.Fprintf(&b, "%s,%.1f,%s,%.0f,%.0f,%.0f,%.1f,%.1f",
|
||||
row.TimestampUTC,
|
||||
row.ElapsedSec,
|
||||
row.Phase,
|
||||
favg, fmin, fmax,
|
||||
row.CPUMaxTempC,
|
||||
row.SysPowerW,
|
||||
)
|
||||
gpuByIdx := make(map[int]GPUStressMetric, len(row.GPUs))
|
||||
for _, g := range row.GPUs {
|
||||
gpuByIdx[g.Index] = g
|
||||
}
|
||||
for _, idx := range gpuIndices {
|
||||
g := gpuByIdx[idx]
|
||||
throttled := 0
|
||||
if g.Throttled {
|
||||
throttled = 1
|
||||
}
|
||||
fmt.Fprintf(&b, ",%.1f,%.1f,%.1f,%.0f,%d",
|
||||
g.TempC, g.UsagePct, g.PowerW, g.ClockMHz, throttled)
|
||||
}
|
||||
b.WriteRune('\n')
|
||||
}
|
||||
|
||||
return os.WriteFile(path, []byte(b.String()), 0644)
|
||||
}
|
||||
|
||||
// WriteFanSensorsCSV writes individual fan sensor readings in long (tidy) format.
|
||||
func WriteFanSensorsCSV(path string, rows []FanStressRow) error {
|
||||
var b strings.Builder
|
||||
b.WriteString("timestamp_utc,elapsed_sec,phase,fan_name,rpm\n")
|
||||
for _, row := range rows {
|
||||
for _, f := range row.Fans {
|
||||
fmt.Fprintf(&b, "%s,%.1f,%s,%s,%.0f\n",
|
||||
row.TimestampUTC, row.ElapsedSec, row.Phase, f.Name, f.RPM)
|
||||
}
|
||||
}
|
||||
return os.WriteFile(path, []byte(b.String()), 0644)
|
||||
}
|
||||
|
||||
// fanRPMStats computes average, min, max RPM across all fans in a sample row.
|
||||
func fanRPMStats(fans []FanReading) (avg, min, max float64) {
|
||||
if len(fans) == 0 {
|
||||
|
||||
@@ -614,10 +614,10 @@ func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string
|
||||
}}))
|
||||
}
|
||||
|
||||
// Cooling fans — one small square per fan (no PCIe/CPU affinity, arbitrary
|
||||
// count, so a wrapping flex row like PSUs rather than SVG boxes). Each
|
||||
// square is sized by rpm / observed-max-rpm and carries a fan glyph that
|
||||
// spins via CSS — faster when the fan is spinning faster.
|
||||
// Cooling fans — one small clickable square per fan (no PCIe/CPU affinity,
|
||||
// arbitrary count, so a wrapping flex row like PSUs rather than SVG boxes).
|
||||
// Square SIZE encodes the fan's ceiling RPM (its class); the coloured FILL
|
||||
// rising from the bottom encodes live duty cycle (current / ceiling).
|
||||
if fans := dedupeFansByName(hw.Sensors); len(fans) > 0 {
|
||||
current := map[string]float64{}
|
||||
for _, f := range fans {
|
||||
@@ -625,26 +625,28 @@ func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string
|
||||
current[strings.TrimSpace(f.Name)] = float64(*f.RPM)
|
||||
}
|
||||
}
|
||||
b.WriteString(renderTopoFanRow(fans, platform.ResolveFanMaxRPM(current)))
|
||||
b.WriteString(renderTopoFanRow(fans, platform.ResolveFanMaxRPM(current), platform.ObservedFanMaxRPM()))
|
||||
}
|
||||
|
||||
return topoCard("Topology", b.String())
|
||||
}
|
||||
|
||||
// renderTopoFanRow renders the COOLING row: one clickable square per fan,
|
||||
// side length scaled by rpm/maxRPM and a fan glyph whose spin rate tracks the
|
||||
// same ratio. maxByName comes from platform.ResolveFanMaxRPM — it already has
|
||||
// an entry for every fan (persisted peak, else peer peak, else current RPM),
|
||||
// so no fallback logic lives here.
|
||||
func renderTopoFanRow(fans []schema.HardwareFanSensor, maxByName map[string]float64) string {
|
||||
// renderTopoFanRow renders the COOLING row. ceilByName (from
|
||||
// platform.ResolveFanMaxRPM) has a value for every fan and drives tile size.
|
||||
// observedByName (from platform.ObservedFanMaxRPM) holds only ceilings that
|
||||
// were actually measured under load — a fan present there gets a duty-cycle
|
||||
// fill; one that isn't shows no fill (ceiling not measured yet).
|
||||
func renderTopoFanRow(fans []schema.HardwareFanSensor, ceilByName, observedByName map[string]float64) string {
|
||||
const (
|
||||
fanTileMin = 30 // px, a stalled / slowest fan
|
||||
fanTileMax = 58 // px, a fan at its ceiling
|
||||
fanTileMin = 34 // px, the smallest-ceiling fan
|
||||
fanTileMax = 60 // px, the largest-ceiling fan
|
||||
)
|
||||
|
||||
var tally topoStatusTally
|
||||
for _, f := range fans {
|
||||
tally.add(classifyTopoSeverity(f.Status))
|
||||
ceilMax := 0.0
|
||||
for _, v := range ceilByName {
|
||||
if v > ceilMax {
|
||||
ceilMax = v
|
||||
}
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
@@ -653,54 +655,84 @@ func renderTopoFanRow(fans []schema.HardwareFanSensor, maxByName map[string]floa
|
||||
b.WriteString(`<div style="display:flex;flex-wrap:wrap;gap:6px;align-items:flex-end">`)
|
||||
for _, f := range fans {
|
||||
name := strings.TrimSpace(f.Name)
|
||||
fill, stroke, text := topoSeverityColors(classifyTopoSeverity(f.Status))
|
||||
_, stroke, text := topoSeverityColors(classifyTopoSeverity(f.Status))
|
||||
ceil := ceilByName[name]
|
||||
|
||||
denom := maxByName[name]
|
||||
ratio := 0.0
|
||||
title := name
|
||||
if f.RPM != nil {
|
||||
if denom > 0 {
|
||||
ratio = float64(*f.RPM) / denom
|
||||
sizeRatio := 1.0
|
||||
if ceilMax > 0 && ceil > 0 {
|
||||
sizeRatio = ceil / ceilMax
|
||||
}
|
||||
side := fanTileMin + int(float64(fanTileMax-fanTileMin)*sizeRatio+0.5)
|
||||
glyphSz := side * 7 / 16
|
||||
|
||||
// Duty cycle: only when the ceiling was actually measured under load.
|
||||
duty := -1.0
|
||||
if _, measured := observedByName[name]; measured && ceil > 0 && f.RPM != nil {
|
||||
duty = float64(*f.RPM) / ceil * 100
|
||||
if duty < 0 {
|
||||
duty = 0
|
||||
}
|
||||
if ratio < 0 {
|
||||
ratio = 0
|
||||
if duty > 100 {
|
||||
duty = 100
|
||||
}
|
||||
if ratio > 1 {
|
||||
ratio = 1
|
||||
}
|
||||
if denom > float64(*f.RPM) {
|
||||
title = fmt.Sprintf("%s · %d RPM (max %d)", name, *f.RPM, int(denom))
|
||||
} else {
|
||||
title = fmt.Sprintf("%s · %d RPM", name, *f.RPM)
|
||||
}
|
||||
} else {
|
||||
title = name + " · no reading"
|
||||
}
|
||||
|
||||
side := fanTileMin + int(float64(fanTileMax-fanTileMin)*ratio+0.5)
|
||||
// Spin period: 2.6s at rest down to 0.5s at the observed peak. A fan
|
||||
// with no reading doesn't spin.
|
||||
spin := ""
|
||||
title := name
|
||||
switch {
|
||||
case f.RPM == nil:
|
||||
title = name + " · no reading"
|
||||
case duty >= 0:
|
||||
title = fmt.Sprintf("%s · %d RPM · %.0f%% duty (ceiling %d)", name, *f.RPM, duty, int(ceil))
|
||||
default:
|
||||
title = fmt.Sprintf("%s · %d RPM · ceiling not measured — run Fan Ceiling Check", name, *f.RPM)
|
||||
}
|
||||
|
||||
glyph := fmt.Sprintf(`<svg width="%d" height="%d" viewBox="0 0 24 24" fill="currentColor" style="opacity:.35" aria-hidden="true">`, glyphSz, glyphSz) + topoFanGlyphPaths() + `</svg>`
|
||||
if f.RPM != nil && *f.RPM > 0 {
|
||||
period := 2.6 - 2.1*ratio
|
||||
spin = fmt.Sprintf(`<svg class="topo-fan-spin" style="animation-duration:%.2fs" width="%d" height="%d" viewBox="0 0 24 24" fill="currentColor" aria-hidden="true">`,
|
||||
period, side*7/16, side*7/16) + topoFanGlyphPaths() + `</svg>`
|
||||
} else {
|
||||
spin = fmt.Sprintf(`<svg width="%d" height="%d" viewBox="0 0 24 24" fill="currentColor" style="opacity:.4" aria-hidden="true">`,
|
||||
side*7/16, side*7/16) + topoFanGlyphPaths() + `</svg>`
|
||||
period := fanSpinPeriodSec(float64(*f.RPM))
|
||||
glyph = fmt.Sprintf(`<svg class="topo-fan-spin" style="animation-duration:%.2fs" width="%d" height="%d" viewBox="0 0 24 24" fill="currentColor" aria-hidden="true">`,
|
||||
period, glyphSz, glyphSz) + topoFanGlyphPaths() + `</svg>`
|
||||
}
|
||||
|
||||
fillBar := ""
|
||||
if duty >= 0 {
|
||||
fillBar = fmt.Sprintf(`<div style="position:absolute;left:0;right:0;bottom:0;height:%.0f%%;background:%s;opacity:.55"></div>`, duty, stroke)
|
||||
}
|
||||
|
||||
fmt.Fprintf(&b, `<div title="%s" onclick="openComponentDetail('fan')" `+
|
||||
`style="width:%dpx;height:%dpx;display:flex;align-items:center;justify-content:center;`+
|
||||
`border-radius:5px;background:%s;border:1px solid %s;color:%s;cursor:pointer">%s</div>`,
|
||||
html.EscapeString(title), side, side, fill, stroke, text, spin)
|
||||
`style="position:relative;overflow:hidden;width:%dpx;height:%dpx;display:flex;align-items:center;justify-content:center;`+
|
||||
`border-radius:5px;background:var(--surface-2);border:1px solid %s;color:%s;cursor:pointer">`+
|
||||
`%s<span style="position:relative;display:flex">%s</span></div>`,
|
||||
html.EscapeString(title), side, side, stroke, text, fillBar, glyph)
|
||||
}
|
||||
b.WriteString(`</div>`)
|
||||
fmt.Fprintf(&b, `<div style="font-size:11px;color:var(--muted);margin-top:6px">%d fans · %s · tile size ∝ RPM / observed max</div>`,
|
||||
len(fans), html.EscapeString(tally.line()))
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// fanSpinPeriodSec maps an absolute fan RPM to a CSS animation period (one
|
||||
// full turn of the glyph, in seconds). The real period would be 60/RPM — a
|
||||
// blur at any real fan speed — so it is compressed into a band the eye can
|
||||
// actually read: at/below fanSpinRPMLo the glyph turns at its slowest still
|
||||
// clearly-moving rate, at/above fanSpinRPMHi at the fastest rate past which
|
||||
// faster is indistinguishable (and starts to stutter), linear in between.
|
||||
func fanSpinPeriodSec(rpm float64) float64 {
|
||||
const (
|
||||
fanSpinRPMLo = 1000.0
|
||||
fanSpinRPMHi = 13000.0
|
||||
fanSpinSlowSec = 2.2
|
||||
fanSpinFastSec = 0.35
|
||||
)
|
||||
switch {
|
||||
case rpm <= fanSpinRPMLo:
|
||||
return fanSpinSlowSec
|
||||
case rpm >= fanSpinRPMHi:
|
||||
return fanSpinFastSec
|
||||
default:
|
||||
t := (rpm - fanSpinRPMLo) / (fanSpinRPMHi - fanSpinRPMLo)
|
||||
return fanSpinSlowSec + t*(fanSpinFastSec-fanSpinSlowSec)
|
||||
}
|
||||
}
|
||||
|
||||
// topoFanSpinStyle emits the keyframes + base class for the spinning fan
|
||||
// glyph once per row. A repeated identical <style> is harmless.
|
||||
func topoFanSpinStyle() string {
|
||||
|
||||
@@ -202,8 +202,9 @@ func TestTopoPageRendersCoolingFansAsFlexRow(t *testing.T) {
|
||||
if !strings.Contains(body, "FAN1 · 4200 RPM") || !strings.Contains(body, "FAN2 · 15000 RPM") {
|
||||
t.Fatalf("topo page missing per-fan RPM tooltips: %s", body)
|
||||
}
|
||||
if !strings.Contains(body, "2 fans") {
|
||||
t.Fatalf("topo page missing fan count caption: %s", body)
|
||||
// No observed ceiling in this test → tiles show no duty fill and say so.
|
||||
if !strings.Contains(body, "ceiling not measured") {
|
||||
t.Fatalf("topo fan tooltip should note the ceiling is unmeasured: %s", body)
|
||||
}
|
||||
|
||||
// Component-detail fallback endpoint must resolve the "fan" type.
|
||||
@@ -217,6 +218,23 @@ func TestTopoPageRendersCoolingFansAsFlexRow(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFanSpinPeriodSec(t *testing.T) {
|
||||
// Clamped at both ends; monotonically faster (smaller period) with RPM.
|
||||
if got := fanSpinPeriodSec(200); got != 2.2 {
|
||||
t.Fatalf("low RPM: got %v want 2.2 (slowest visible)", got)
|
||||
}
|
||||
if got := fanSpinPeriodSec(25000); got != 0.35 {
|
||||
t.Fatalf("high RPM: got %v want 0.35 (fastest visible)", got)
|
||||
}
|
||||
mid := fanSpinPeriodSec(7000)
|
||||
if mid <= 0.35 || mid >= 2.2 {
|
||||
t.Fatalf("mid RPM period %v out of band", mid)
|
||||
}
|
||||
if fanSpinPeriodSec(10000) >= fanSpinPeriodSec(3000) {
|
||||
t.Fatalf("higher RPM must spin faster (shorter period)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTopoPageRendersStorageDisksGroupedByType(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "audit.json")
|
||||
|
||||
@@ -117,9 +117,9 @@ func renderValidateMode(opts HandlerOptions, stressDefault bool) string {
|
||||
)) +
|
||||
renderSATCard("fan", "Fan Ceiling Check", "runSAT('fan')", "", renderValidateCardBody(
|
||||
"All system fans reported over IPMI / lm-sensors.",
|
||||
`Drives CPU (+memory) and, when a GPU is present, GPU load to 100% at the same time and watches every fan until none has climbed for ~1 min. The peak RPM reached is recorded as each fan's ceiling and is what the Topology view sizes the fan tiles against. Success once every fan plateaus (or the time cap is hit). A fan reading 0 RPM or an IPMI status of cr/nr under full load fails. If the host cannot be loaded at all, or exposes no fan sensors, the task is cancelled as "not applicable" rather than failed — the platform does not support forcing fans, so this is the closest safe equivalent.`,
|
||||
`<code>stressapptest</code> / <code>stress-ng</code> + <code>bee-gpu-burn</code> / <code>rvs gst</code>; <code>ipmitool sdr type Fan</code>`,
|
||||
`~2–8 min depending on how fast the fan curve settles (hard cap 15 min).`,
|
||||
`Drives CPU (+memory) and, when a GPU is present, the hottest sustained GPU load (dcgmproftester targeted-power — the Power/Thermal Fit engine) at the same time, and watches every fan until none has climbed for ~1 min. The peak RPM reached is recorded as each fan's ceiling and is what the Topology view sizes the fan tiles and duty-cycle fill against. Success once every fan plateaus (or the time cap is hit). A fan reading 0 RPM or an IPMI status of cr/nr under full load fails. IPMI polling backs off automatically if the BMC gets slow under load. If the host cannot be loaded at all, or exposes no fan sensors, the task is cancelled as "not applicable" rather than failed — the platform does not support forcing fans, so this is the closest safe equivalent.`,
|
||||
`<code>stressapptest</code> / <code>stress-ng</code> + <code>dcgmproftester -t 1004</code> / <code>rvs gst</code>; <code>ipmitool sdr type Fan</code>`,
|
||||
`~3–10 min depending on how fast the fan curve settles (hard cap 15 min).`,
|
||||
))
|
||||
}
|
||||
|
||||
|
||||
@@ -76,7 +76,7 @@ Generic engineering rules live in `bible/rules/patterns/`.
|
||||
- GPU bandwidth check
|
||||
- `dcgmi diag -r nvbandwidth` (per CPU socket, then all selected GPUs, on multi-socket systems -- see `decisions/2026-07-27-nvbandwidth-per-socket-split.md`)
|
||||
- Fan ceiling check (Load tier / `3. Load` only)
|
||||
- `stressapptest` (or `stress-ng`) + `bee-gpu-burn` / `rvs gst` at 100%, run simultaneously
|
||||
- `ipmitool sdr type Fan` sampled 1 Hz until every fan plateaus (~1 min flat) or the 15 min cap
|
||||
- records each fan's observed peak RPM to `/var/log/bee-sat/fan-observation.json` (used by the Topology fan tiles); FAIL only on a fan at 0 RPM / IPMI cr-nr under load; **cancelled ("not applicable")**, never failed, if the host cannot be loaded or has no fan sensors
|
||||
- `stressapptest` (or `stress-ng`) + the hottest GPU load — `dcgmproftester -t 1004` / `targeted_power` (Power/Thermal Fit engine), **not** `bee-gpu-burn` — run simultaneously
|
||||
- `ipmitool sdr type Fan` on an adaptive interval (1 s floor, backs off to 30 s when the BMC gets slow, every read time-boxed) until every fan plateaus (~1 min flat, only trusted while telemetry is healthy) or the 15 min cap
|
||||
- records each fan's observed peak RPM to the fan-observation store (used by the Topology fan tiles: size ∝ ceiling, fill ∝ duty cycle); FAIL only on a fan at 0 RPM / IPMI cr-nr under load; **cancelled ("not applicable")**, never failed, if the host cannot be loaded or has no fan sensors
|
||||
- see `decisions/2026-09-04-fan-ceiling-check.md`
|
||||
|
||||
@@ -29,14 +29,30 @@ resulting RPM — does not work on our current platform:
|
||||
The `fan` SAT test (`platform.RunFanCheck`, formerly the unwired
|
||||
`RunFanStressTest`) drives load, not the BMC:
|
||||
|
||||
1. Start `stressapptest` (CPU + memory) and, when a GPU is present, a GPU burn
|
||||
(`bee-gpu-burn` / `rvs gst`) **simultaneously**, each in its own goroutine.
|
||||
1. Start `stressapptest` (CPU + memory) and, when a GPU is present, the
|
||||
**hottest sustained GPU load** — `dcgmproftester -t 1004` /
|
||||
`targeted_power`, via `resolveBenchmarkPowerLoadCommand`, the same engine
|
||||
Power/Thermal Fit uses — **simultaneously**, each in its own goroutine.
|
||||
NOT `bee-gpu-burn`: that is a compute-throughput burn that tops out around
|
||||
88% of TDP (measured ~525 W of 600 W on RTX PRO 6000 Blackwell) and never
|
||||
makes the fans demand their true ceiling. A first run on the MSI stand with
|
||||
`bee-gpu-burn` peaked F2U fans at 24 400 RPM vs a historical 26 000.
|
||||
A missing GPU is not an error — CPU/memory load alone exercises the cooling
|
||||
loop. Load sources reach full power at different times, so the plateau clock
|
||||
only starts once every launched source reports its process running (plus a
|
||||
fixed GPU ramp grace).
|
||||
2. Sample every fan once a second. Per fan, track the peak RPM and the last time
|
||||
it climbed by more than `PlateauDeltaRPM` (default 50).
|
||||
2. Sample every fan on an adaptive interval (floor 1 s). Each read is
|
||||
time-boxed (`readFansBounded` — a goroutine abandoned on timeout, so a
|
||||
KCS-wedged `ipmitool` can never block the loop); when reads are slow the
|
||||
interval backs off geometrically to 30 s and tightens again when they
|
||||
recover. Under 8-GPU + CPU load on the MSI stand, `ipmitool sdr type Fan`
|
||||
took ~14 s/call and at one point wedged for minutes — without this the
|
||||
"1 Hz" sampler silently degraded to one sample per 14 s and the plateau
|
||||
timer ran on stale data. A plateau is only declared while telemetry is
|
||||
healthy (interval near the floor, ≥5 recent samples); a degraded run just
|
||||
rides out to `MaxLoadSec` and records the peak it saw. Per fan, track the
|
||||
peak RPM and the last time it climbed by more than `PlateauDeltaRPM`
|
||||
(default 50).
|
||||
3. When no fan has climbed for `PlateauHoldSec` (default 60 s) and at least one
|
||||
fan rose meaningfully above baseline, declare the ceiling found and stop —
|
||||
**success**. `MaxLoadSec` (default 900 s) is a hard cap; hitting it is also
|
||||
|
||||
Reference in New Issue
Block a user