fix(sat): fan check uses hottest GPU load + IPMI-hang-proof polling
- GPU load switches from bee-gpu-burn (compute burn, ~88% TDP) to dcgmproftester -t 1004 / targeted_power via resolveBenchmarkPowerLoadCommand — the same engine Power/Thermal Fit uses and the hottest sustained NVIDIA load we have, so fans are actually pushed toward their ceiling. - Sample loop is now IPMI-hang-proof: every ipmitool read is time-boxed in an abandonable goroutine, and the poll interval backs off geometrically (1s→30s) when reads are slow, tightening again on recovery. A plateau is only trusted while telemetry is healthy; degraded runs ride out to MaxLoadSec. Summary gains fan_samples / telemetry_degraded. Drops the per-second nvidia-smi+power+cpu-temp sampling from the hot loop. - Dead code removed: FanStressRow, GPUStressMetric, sampleFanStressRow, sampleGPUStressMetrics, WriteFanStressCSV/WriteFanSensorsCSV, analyzeMaxTemp, sampleSystemPowerResolved. Topology fan tiles: - size encodes the fan's ceiling RPM (its class), not current speed; coloured fill rising from the bottom encodes live duty cycle (current / ceiling), shown only when the ceiling was measured. - glyph spin rate now maps absolute RPM into a human-perceptible band (fanSpinPeriodSec: 2.2s/turn at <=1000 RPM, 0.35s at >=13000). - the "N fans · N OK · tile size ∝ …" caption line is gone. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019VHG21rgTUiR1G3qFHTVmN
This commit is contained in:
Mikhail Chusavitin
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
bb2a501a28
commit
8cb250f3f4
@@ -614,10 +614,10 @@ func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string
|
||||
}}))
|
||||
}
|
||||
|
||||
// Cooling fans — one small square per fan (no PCIe/CPU affinity, arbitrary
|
||||
// count, so a wrapping flex row like PSUs rather than SVG boxes). Each
|
||||
// square is sized by rpm / observed-max-rpm and carries a fan glyph that
|
||||
// spins via CSS — faster when the fan is spinning faster.
|
||||
// Cooling fans — one small clickable square per fan (no PCIe/CPU affinity,
|
||||
// arbitrary count, so a wrapping flex row like PSUs rather than SVG boxes).
|
||||
// Square SIZE encodes the fan's ceiling RPM (its class); the coloured FILL
|
||||
// rising from the bottom encodes live duty cycle (current / ceiling).
|
||||
if fans := dedupeFansByName(hw.Sensors); len(fans) > 0 {
|
||||
current := map[string]float64{}
|
||||
for _, f := range fans {
|
||||
@@ -625,26 +625,28 @@ func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string
|
||||
current[strings.TrimSpace(f.Name)] = float64(*f.RPM)
|
||||
}
|
||||
}
|
||||
b.WriteString(renderTopoFanRow(fans, platform.ResolveFanMaxRPM(current)))
|
||||
b.WriteString(renderTopoFanRow(fans, platform.ResolveFanMaxRPM(current), platform.ObservedFanMaxRPM()))
|
||||
}
|
||||
|
||||
return topoCard("Topology", b.String())
|
||||
}
|
||||
|
||||
// renderTopoFanRow renders the COOLING row: one clickable square per fan,
|
||||
// side length scaled by rpm/maxRPM and a fan glyph whose spin rate tracks the
|
||||
// same ratio. maxByName comes from platform.ResolveFanMaxRPM — it already has
|
||||
// an entry for every fan (persisted peak, else peer peak, else current RPM),
|
||||
// so no fallback logic lives here.
|
||||
func renderTopoFanRow(fans []schema.HardwareFanSensor, maxByName map[string]float64) string {
|
||||
// renderTopoFanRow renders the COOLING row. ceilByName (from
|
||||
// platform.ResolveFanMaxRPM) has a value for every fan and drives tile size.
|
||||
// observedByName (from platform.ObservedFanMaxRPM) holds only ceilings that
|
||||
// were actually measured under load — a fan present there gets a duty-cycle
|
||||
// fill; one that isn't shows no fill (ceiling not measured yet).
|
||||
func renderTopoFanRow(fans []schema.HardwareFanSensor, ceilByName, observedByName map[string]float64) string {
|
||||
const (
|
||||
fanTileMin = 30 // px, a stalled / slowest fan
|
||||
fanTileMax = 58 // px, a fan at its ceiling
|
||||
fanTileMin = 34 // px, the smallest-ceiling fan
|
||||
fanTileMax = 60 // px, the largest-ceiling fan
|
||||
)
|
||||
|
||||
var tally topoStatusTally
|
||||
for _, f := range fans {
|
||||
tally.add(classifyTopoSeverity(f.Status))
|
||||
ceilMax := 0.0
|
||||
for _, v := range ceilByName {
|
||||
if v > ceilMax {
|
||||
ceilMax = v
|
||||
}
|
||||
}
|
||||
|
||||
var b strings.Builder
|
||||
@@ -653,54 +655,84 @@ func renderTopoFanRow(fans []schema.HardwareFanSensor, maxByName map[string]floa
|
||||
b.WriteString(`<div style="display:flex;flex-wrap:wrap;gap:6px;align-items:flex-end">`)
|
||||
for _, f := range fans {
|
||||
name := strings.TrimSpace(f.Name)
|
||||
fill, stroke, text := topoSeverityColors(classifyTopoSeverity(f.Status))
|
||||
_, stroke, text := topoSeverityColors(classifyTopoSeverity(f.Status))
|
||||
ceil := ceilByName[name]
|
||||
|
||||
denom := maxByName[name]
|
||||
ratio := 0.0
|
||||
title := name
|
||||
if f.RPM != nil {
|
||||
if denom > 0 {
|
||||
ratio = float64(*f.RPM) / denom
|
||||
sizeRatio := 1.0
|
||||
if ceilMax > 0 && ceil > 0 {
|
||||
sizeRatio = ceil / ceilMax
|
||||
}
|
||||
side := fanTileMin + int(float64(fanTileMax-fanTileMin)*sizeRatio+0.5)
|
||||
glyphSz := side * 7 / 16
|
||||
|
||||
// Duty cycle: only when the ceiling was actually measured under load.
|
||||
duty := -1.0
|
||||
if _, measured := observedByName[name]; measured && ceil > 0 && f.RPM != nil {
|
||||
duty = float64(*f.RPM) / ceil * 100
|
||||
if duty < 0 {
|
||||
duty = 0
|
||||
}
|
||||
if ratio < 0 {
|
||||
ratio = 0
|
||||
if duty > 100 {
|
||||
duty = 100
|
||||
}
|
||||
if ratio > 1 {
|
||||
ratio = 1
|
||||
}
|
||||
if denom > float64(*f.RPM) {
|
||||
title = fmt.Sprintf("%s · %d RPM (max %d)", name, *f.RPM, int(denom))
|
||||
} else {
|
||||
title = fmt.Sprintf("%s · %d RPM", name, *f.RPM)
|
||||
}
|
||||
} else {
|
||||
title = name + " · no reading"
|
||||
}
|
||||
|
||||
side := fanTileMin + int(float64(fanTileMax-fanTileMin)*ratio+0.5)
|
||||
// Spin period: 2.6s at rest down to 0.5s at the observed peak. A fan
|
||||
// with no reading doesn't spin.
|
||||
spin := ""
|
||||
title := name
|
||||
switch {
|
||||
case f.RPM == nil:
|
||||
title = name + " · no reading"
|
||||
case duty >= 0:
|
||||
title = fmt.Sprintf("%s · %d RPM · %.0f%% duty (ceiling %d)", name, *f.RPM, duty, int(ceil))
|
||||
default:
|
||||
title = fmt.Sprintf("%s · %d RPM · ceiling not measured — run Fan Ceiling Check", name, *f.RPM)
|
||||
}
|
||||
|
||||
glyph := fmt.Sprintf(`<svg width="%d" height="%d" viewBox="0 0 24 24" fill="currentColor" style="opacity:.35" aria-hidden="true">`, glyphSz, glyphSz) + topoFanGlyphPaths() + `</svg>`
|
||||
if f.RPM != nil && *f.RPM > 0 {
|
||||
period := 2.6 - 2.1*ratio
|
||||
spin = fmt.Sprintf(`<svg class="topo-fan-spin" style="animation-duration:%.2fs" width="%d" height="%d" viewBox="0 0 24 24" fill="currentColor" aria-hidden="true">`,
|
||||
period, side*7/16, side*7/16) + topoFanGlyphPaths() + `</svg>`
|
||||
} else {
|
||||
spin = fmt.Sprintf(`<svg width="%d" height="%d" viewBox="0 0 24 24" fill="currentColor" style="opacity:.4" aria-hidden="true">`,
|
||||
side*7/16, side*7/16) + topoFanGlyphPaths() + `</svg>`
|
||||
period := fanSpinPeriodSec(float64(*f.RPM))
|
||||
glyph = fmt.Sprintf(`<svg class="topo-fan-spin" style="animation-duration:%.2fs" width="%d" height="%d" viewBox="0 0 24 24" fill="currentColor" aria-hidden="true">`,
|
||||
period, glyphSz, glyphSz) + topoFanGlyphPaths() + `</svg>`
|
||||
}
|
||||
|
||||
fillBar := ""
|
||||
if duty >= 0 {
|
||||
fillBar = fmt.Sprintf(`<div style="position:absolute;left:0;right:0;bottom:0;height:%.0f%%;background:%s;opacity:.55"></div>`, duty, stroke)
|
||||
}
|
||||
|
||||
fmt.Fprintf(&b, `<div title="%s" onclick="openComponentDetail('fan')" `+
|
||||
`style="width:%dpx;height:%dpx;display:flex;align-items:center;justify-content:center;`+
|
||||
`border-radius:5px;background:%s;border:1px solid %s;color:%s;cursor:pointer">%s</div>`,
|
||||
html.EscapeString(title), side, side, fill, stroke, text, spin)
|
||||
`style="position:relative;overflow:hidden;width:%dpx;height:%dpx;display:flex;align-items:center;justify-content:center;`+
|
||||
`border-radius:5px;background:var(--surface-2);border:1px solid %s;color:%s;cursor:pointer">`+
|
||||
`%s<span style="position:relative;display:flex">%s</span></div>`,
|
||||
html.EscapeString(title), side, side, stroke, text, fillBar, glyph)
|
||||
}
|
||||
b.WriteString(`</div>`)
|
||||
fmt.Fprintf(&b, `<div style="font-size:11px;color:var(--muted);margin-top:6px">%d fans · %s · tile size ∝ RPM / observed max</div>`,
|
||||
len(fans), html.EscapeString(tally.line()))
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// fanSpinPeriodSec maps an absolute fan RPM to a CSS animation period (one
|
||||
// full turn of the glyph, in seconds). The real period would be 60/RPM — a
|
||||
// blur at any real fan speed — so it is compressed into a band the eye can
|
||||
// actually read: at/below fanSpinRPMLo the glyph turns at its slowest still
|
||||
// clearly-moving rate, at/above fanSpinRPMHi at the fastest rate past which
|
||||
// faster is indistinguishable (and starts to stutter), linear in between.
|
||||
func fanSpinPeriodSec(rpm float64) float64 {
|
||||
const (
|
||||
fanSpinRPMLo = 1000.0
|
||||
fanSpinRPMHi = 13000.0
|
||||
fanSpinSlowSec = 2.2
|
||||
fanSpinFastSec = 0.35
|
||||
)
|
||||
switch {
|
||||
case rpm <= fanSpinRPMLo:
|
||||
return fanSpinSlowSec
|
||||
case rpm >= fanSpinRPMHi:
|
||||
return fanSpinFastSec
|
||||
default:
|
||||
t := (rpm - fanSpinRPMLo) / (fanSpinRPMHi - fanSpinRPMLo)
|
||||
return fanSpinSlowSec + t*(fanSpinFastSec-fanSpinSlowSec)
|
||||
}
|
||||
}
|
||||
|
||||
// topoFanSpinStyle emits the keyframes + base class for the spinning fan
|
||||
// glyph once per row. A repeated identical <style> is harmless.
|
||||
func topoFanSpinStyle() string {
|
||||
|
||||
Reference in New Issue
Block a user