fix(sat): fan check uses hottest GPU load + IPMI-hang-proof polling
- GPU load switches from bee-gpu-burn (compute burn, ~88% TDP) to dcgmproftester -t 1004 / targeted_power via resolveBenchmarkPowerLoadCommand — the same engine Power/Thermal Fit uses and the hottest sustained NVIDIA load we have, so fans are actually pushed toward their ceiling. - Sample loop is now IPMI-hang-proof: every ipmitool read is time-boxed in an abandonable goroutine, and the poll interval backs off geometrically (1s→30s) when reads are slow, tightening again on recovery. A plateau is only trusted while telemetry is healthy; degraded runs ride out to MaxLoadSec. Summary gains fan_samples / telemetry_degraded. Drops the per-second nvidia-smi+power+cpu-temp sampling from the hot loop. - Dead code removed: FanStressRow, GPUStressMetric, sampleFanStressRow, sampleGPUStressMetrics, WriteFanStressCSV/WriteFanSensorsCSV, analyzeMaxTemp, sampleSystemPowerResolved. Topology fan tiles: - size encodes the fan's ceiling RPM (its class), not current speed; coloured fill rising from the bottom encodes live duty cycle (current / ceiling), shown only when the ceiling was measured. - glyph spin rate now maps absolute RPM into a human-perceptible band (fanSpinPeriodSec: 2.2s/turn at <=1000 RPM, 0.35s at >=13000). - the "N fans · N OK · tile size ∝ …" caption line is gone. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019VHG21rgTUiR1G3qFHTVmN
This commit is contained in:
Mikhail Chusavitin
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
bb2a501a28
commit
8cb250f3f4
@@ -614,10 +614,10 @@ func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string
|
||||
}}))
|
||||
}
|
||||
|
||||
// Cooling fans — one small square per fan (no PCIe/CPU affinity, arbitrary
|
||||
// count, so a wrapping flex row like PSUs rather than SVG boxes). Each
|
||||
// square is sized by rpm / observed-max-rpm and carries a fan glyph that
|
||||
// spins via CSS — faster when the fan is spinning faster.
|
||||
// Cooling fans — one small clickable square per fan (no PCIe/CPU affinity,
|
||||
// arbitrary count, so a wrapping flex row like PSUs rather than SVG boxes).
|
||||
// Square SIZE encodes the fan's ceiling RPM (its class); the coloured FILL
|
||||
// rising from the bottom encodes live duty cycle (current / ceiling).
|
||||
if fans := dedupeFansByName(hw.Sensors); len(fans) > 0 {
|
||||
current := map[string]float64{}
|
||||
for _, f := range fans {
|
||||
@@ -625,26 +625,28 @@ func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string
|
||||
current[strings.TrimSpace(f.Name)] = float64(*f.RPM)
|
||||
}
|
||||
}
|
||||
b.WriteString(renderTopoFanRow(fans, platform.ResolveFanMaxRPM(current)))
|
||||
b.WriteString(renderTopoFanRow(fans, platform.ResolveFanMaxRPM(current), platform.ObservedFanMaxRPM()))
|
||||
}
|
||||
|
||||
return topoCard("Topology", b.String())
|
||||
}
|
||||
|
||||
// renderTopoFanRow renders the COOLING row: one clickable square per fan,
|
||||
// side length scaled by rpm/maxRPM and a fan glyph whose spin rate tracks the
|
||||
// same ratio. maxByName comes from platform.ResolveFanMaxRPM — it already has
|
||||
// an entry for every fan (persisted peak, else peer peak, else current RPM),
|
||||
// so no fallback logic lives here.
|
||||
func renderTopoFanRow(fans []schema.HardwareFanSensor, maxByName map[string]float64) string {
|
||||
// renderTopoFanRow renders the COOLING row. ceilByName (from
|
||||
// platform.ResolveFanMaxRPM) has a value for every fan and drives tile size.
|
||||
// observedByName (from platform.ObservedFanMaxRPM) holds only ceilings that
|
||||
// were actually measured under load — a fan present there gets a duty-cycle
|
||||
// fill; one that isn't shows no fill (ceiling not measured yet).
|
||||
func renderTopoFanRow(fans []schema.HardwareFanSensor, ceilByName, observedByName map[string]float64) string {
|
||||
const (
|
||||
fanTileMin = 30 // px, a stalled / slowest fan
|
||||
fanTileMax = 58 // px, a fan at its ceiling
|
||||
fanTileMin = 34 // px, the smallest-ceiling fan
|
||||
fanTileMax = 60 // px, the largest-ceiling fan
|
||||
)
|
||||
|
||||
var tally topoStatusTally
|
||||
for _, f := range fans {
|
||||
tally.add(classifyTopoSeverity(f.Status))
|
||||
ceilMax := 0.0
|
||||
for _, v := range ceilByName {
|
||||
if v > ceilMax {
|
||||
ceilMax = v
|
||||
}
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
@@ -653,54 +655,84 @@ func renderTopoFanRow(fans []schema.HardwareFanSensor, maxByName map[string]floa
|
||||
b.WriteString(`<div style="display:flex;flex-wrap:wrap;gap:6px;align-items:flex-end">`)
|
||||
for _, f := range fans {
|
||||
name := strings.TrimSpace(f.Name)
|
||||
fill, stroke, text := topoSeverityColors(classifyTopoSeverity(f.Status))
|
||||
_, stroke, text := topoSeverityColors(classifyTopoSeverity(f.Status))
|
||||
ceil := ceilByName[name]
|
||||
|
||||
denom := maxByName[name]
|
||||
ratio := 0.0
|
||||
title := name
|
||||
if f.RPM != nil {
|
||||
if denom > 0 {
|
||||
ratio = float64(*f.RPM) / denom
|
||||
sizeRatio := 1.0
|
||||
if ceilMax > 0 && ceil > 0 {
|
||||
sizeRatio = ceil / ceilMax
|
||||
}
|
||||
side := fanTileMin + int(float64(fanTileMax-fanTileMin)*sizeRatio+0.5)
|
||||
glyphSz := side * 7 / 16
|
||||
|
||||
// Duty cycle: only when the ceiling was actually measured under load.
|
||||
duty := -1.0
|
||||
if _, measured := observedByName[name]; measured && ceil > 0 && f.RPM != nil {
|
||||
duty = float64(*f.RPM) / ceil * 100
|
||||
if duty < 0 {
|
||||
duty = 0
|
||||
}
|
||||
if ratio < 0 {
|
||||
ratio = 0
|
||||
if duty > 100 {
|
||||
duty = 100
|
||||
}
|
||||
if ratio > 1 {
|
||||
ratio = 1
|
||||
}
|
||||
if denom > float64(*f.RPM) {
|
||||
title = fmt.Sprintf("%s · %d RPM (max %d)", name, *f.RPM, int(denom))
|
||||
} else {
|
||||
title = fmt.Sprintf("%s · %d RPM", name, *f.RPM)
|
||||
}
|
||||
} else {
|
||||
title = name + " · no reading"
|
||||
}
|
||||
|
||||
side := fanTileMin + int(float64(fanTileMax-fanTileMin)*ratio+0.5)
|
||||
// Spin period: 2.6s at rest down to 0.5s at the observed peak. A fan
|
||||
// with no reading doesn't spin.
|
||||
spin := ""
|
||||
title := name
|
||||
switch {
|
||||
case f.RPM == nil:
|
||||
title = name + " · no reading"
|
||||
case duty >= 0:
|
||||
title = fmt.Sprintf("%s · %d RPM · %.0f%% duty (ceiling %d)", name, *f.RPM, duty, int(ceil))
|
||||
default:
|
||||
title = fmt.Sprintf("%s · %d RPM · ceiling not measured — run Fan Ceiling Check", name, *f.RPM)
|
||||
}
|
||||
|
||||
glyph := fmt.Sprintf(`<svg width="%d" height="%d" viewBox="0 0 24 24" fill="currentColor" style="opacity:.35" aria-hidden="true">`, glyphSz, glyphSz) + topoFanGlyphPaths() + `</svg>`
|
||||
if f.RPM != nil && *f.RPM > 0 {
|
||||
period := 2.6 - 2.1*ratio
|
||||
spin = fmt.Sprintf(`<svg class="topo-fan-spin" style="animation-duration:%.2fs" width="%d" height="%d" viewBox="0 0 24 24" fill="currentColor" aria-hidden="true">`,
|
||||
period, side*7/16, side*7/16) + topoFanGlyphPaths() + `</svg>`
|
||||
} else {
|
||||
spin = fmt.Sprintf(`<svg width="%d" height="%d" viewBox="0 0 24 24" fill="currentColor" style="opacity:.4" aria-hidden="true">`,
|
||||
side*7/16, side*7/16) + topoFanGlyphPaths() + `</svg>`
|
||||
period := fanSpinPeriodSec(float64(*f.RPM))
|
||||
glyph = fmt.Sprintf(`<svg class="topo-fan-spin" style="animation-duration:%.2fs" width="%d" height="%d" viewBox="0 0 24 24" fill="currentColor" aria-hidden="true">`,
|
||||
period, glyphSz, glyphSz) + topoFanGlyphPaths() + `</svg>`
|
||||
}
|
||||
|
||||
fillBar := ""
|
||||
if duty >= 0 {
|
||||
fillBar = fmt.Sprintf(`<div style="position:absolute;left:0;right:0;bottom:0;height:%.0f%%;background:%s;opacity:.55"></div>`, duty, stroke)
|
||||
}
|
||||
|
||||
fmt.Fprintf(&b, `<div title="%s" onclick="openComponentDetail('fan')" `+
|
||||
`style="width:%dpx;height:%dpx;display:flex;align-items:center;justify-content:center;`+
|
||||
`border-radius:5px;background:%s;border:1px solid %s;color:%s;cursor:pointer">%s</div>`,
|
||||
html.EscapeString(title), side, side, fill, stroke, text, spin)
|
||||
`style="position:relative;overflow:hidden;width:%dpx;height:%dpx;display:flex;align-items:center;justify-content:center;`+
|
||||
`border-radius:5px;background:var(--surface-2);border:1px solid %s;color:%s;cursor:pointer">`+
|
||||
`%s<span style="position:relative;display:flex">%s</span></div>`,
|
||||
html.EscapeString(title), side, side, stroke, text, fillBar, glyph)
|
||||
}
|
||||
b.WriteString(`</div>`)
|
||||
fmt.Fprintf(&b, `<div style="font-size:11px;color:var(--muted);margin-top:6px">%d fans · %s · tile size ∝ RPM / observed max</div>`,
|
||||
len(fans), html.EscapeString(tally.line()))
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// fanSpinPeriodSec maps an absolute fan RPM to a CSS animation period (one
|
||||
// full turn of the glyph, in seconds). The real period would be 60/RPM — a
|
||||
// blur at any real fan speed — so it is compressed into a band the eye can
|
||||
// actually read: at/below fanSpinRPMLo the glyph turns at its slowest still
|
||||
// clearly-moving rate, at/above fanSpinRPMHi at the fastest rate past which
|
||||
// faster is indistinguishable (and starts to stutter), linear in between.
|
||||
func fanSpinPeriodSec(rpm float64) float64 {
|
||||
const (
|
||||
fanSpinRPMLo = 1000.0
|
||||
fanSpinRPMHi = 13000.0
|
||||
fanSpinSlowSec = 2.2
|
||||
fanSpinFastSec = 0.35
|
||||
)
|
||||
switch {
|
||||
case rpm <= fanSpinRPMLo:
|
||||
return fanSpinSlowSec
|
||||
case rpm >= fanSpinRPMHi:
|
||||
return fanSpinFastSec
|
||||
default:
|
||||
t := (rpm - fanSpinRPMLo) / (fanSpinRPMHi - fanSpinRPMLo)
|
||||
return fanSpinSlowSec + t*(fanSpinFastSec-fanSpinSlowSec)
|
||||
}
|
||||
}
|
||||
|
||||
// topoFanSpinStyle emits the keyframes + base class for the spinning fan
|
||||
// glyph once per row. A repeated identical <style> is harmless.
|
||||
func topoFanSpinStyle() string {
|
||||
|
||||
@@ -202,8 +202,9 @@ func TestTopoPageRendersCoolingFansAsFlexRow(t *testing.T) {
|
||||
if !strings.Contains(body, "FAN1 · 4200 RPM") || !strings.Contains(body, "FAN2 · 15000 RPM") {
|
||||
t.Fatalf("topo page missing per-fan RPM tooltips: %s", body)
|
||||
}
|
||||
if !strings.Contains(body, "2 fans") {
|
||||
t.Fatalf("topo page missing fan count caption: %s", body)
|
||||
// No observed ceiling in this test → tiles show no duty fill and say so.
|
||||
if !strings.Contains(body, "ceiling not measured") {
|
||||
t.Fatalf("topo fan tooltip should note the ceiling is unmeasured: %s", body)
|
||||
}
|
||||
|
||||
// Component-detail fallback endpoint must resolve the "fan" type.
|
||||
@@ -217,6 +218,23 @@ func TestTopoPageRendersCoolingFansAsFlexRow(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFanSpinPeriodSec(t *testing.T) {
|
||||
// Clamped at both ends; monotonically faster (smaller period) with RPM.
|
||||
if got := fanSpinPeriodSec(200); got != 2.2 {
|
||||
t.Fatalf("low RPM: got %v want 2.2 (slowest visible)", got)
|
||||
}
|
||||
if got := fanSpinPeriodSec(25000); got != 0.35 {
|
||||
t.Fatalf("high RPM: got %v want 0.35 (fastest visible)", got)
|
||||
}
|
||||
mid := fanSpinPeriodSec(7000)
|
||||
if mid <= 0.35 || mid >= 2.2 {
|
||||
t.Fatalf("mid RPM period %v out of band", mid)
|
||||
}
|
||||
if fanSpinPeriodSec(10000) >= fanSpinPeriodSec(3000) {
|
||||
t.Fatalf("higher RPM must spin faster (shorter period)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTopoPageRendersStorageDisksGroupedByType(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "audit.json")
|
||||
|
||||
@@ -117,9 +117,9 @@ func renderValidateMode(opts HandlerOptions, stressDefault bool) string {
|
||||
)) +
|
||||
renderSATCard("fan", "Fan Ceiling Check", "runSAT('fan')", "", renderValidateCardBody(
|
||||
"All system fans reported over IPMI / lm-sensors.",
|
||||
`Drives CPU (+memory) and, when a GPU is present, GPU load to 100% at the same time and watches every fan until none has climbed for ~1 min. The peak RPM reached is recorded as each fan's ceiling and is what the Topology view sizes the fan tiles against. Success once every fan plateaus (or the time cap is hit). A fan reading 0 RPM or an IPMI status of cr/nr under full load fails. If the host cannot be loaded at all, or exposes no fan sensors, the task is cancelled as "not applicable" rather than failed — the platform does not support forcing fans, so this is the closest safe equivalent.`,
|
||||
`<code>stressapptest</code> / <code>stress-ng</code> + <code>bee-gpu-burn</code> / <code>rvs gst</code>; <code>ipmitool sdr type Fan</code>`,
|
||||
`~2–8 min depending on how fast the fan curve settles (hard cap 15 min).`,
|
||||
`Drives CPU (+memory) and, when a GPU is present, the hottest sustained GPU load (dcgmproftester targeted-power — the Power/Thermal Fit engine) at the same time, and watches every fan until none has climbed for ~1 min. The peak RPM reached is recorded as each fan's ceiling and is what the Topology view sizes the fan tiles and duty-cycle fill against. Success once every fan plateaus (or the time cap is hit). A fan reading 0 RPM or an IPMI status of cr/nr under full load fails. IPMI polling backs off automatically if the BMC gets slow under load. If the host cannot be loaded at all, or exposes no fan sensors, the task is cancelled as "not applicable" rather than failed — the platform does not support forcing fans, so this is the closest safe equivalent.`,
|
||||
`<code>stressapptest</code> / <code>stress-ng</code> + <code>dcgmproftester -t 1004</code> / <code>rvs gst</code>; <code>ipmitool sdr type Fan</code>`,
|
||||
`~3–10 min depending on how fast the fan curve settles (hard cap 15 min).`,
|
||||
))
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user