feat(sat): fan ceiling check + topology fan tiles
Repurpose the previously-unwired RunFanStressTest into RunFanCheck, a
Load-tier SAT test that drives stressapptest (CPU+memory) and, when a GPU
is present, a GPU burn to 100% simultaneously, then watches every fan
until none has climbed for ~60s. The observed peak RPM per fan is the
"ceiling"; it is persisted through the existing fan-observation store.
MSI G4201 / AMI MegaRAC exposes no host-side fan force (every OEM IPMI
command returns 0xc1; Redfish Thermal is GET-only), so load-driven ramp
is the closest safe equivalent. See
bible-local/decisions/2026-09-04-fan-ceiling-check.md.
- platform.ResolveFanMaxRPM: per-fan max with fallback (persisted peak ->
peer peak -> current RPM), resolved in platform, not the view.
- platform.ErrTestNotApplicable: no load source or no fan sensors ->
task lands as cancelled ("not applicable"), never failed, so an
engineer never sees a false red. executeTaskWithOptions maps the
sentinel; finalizeTaskForResult honours a pre-set TaskCancelled.
- Verdict FAIL only for a fan at 0 RPM / IPMI cr-nr under load.
- /topo: one small spinning square per fan, sized by RPM / resolved max,
clickable through to a new "fan" component-detail type; per-fan status
recorded to the component-status DB from the fan SAT summary.
- Wiring: /api/sat/fan/run route, "fan" task target, Load-page card,
stress-mode Run All.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019VHG21rgTUiR1G3qFHTVmN
This commit is contained in:
co-authored by
Claude Sonnet 5
parent
e4f7519ef3
commit
bb2a501a28
@@ -173,7 +173,7 @@ type satRunner interface {
|
|||||||
RunAMDStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error)
|
RunAMDStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error)
|
||||||
RunMemoryStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error)
|
RunMemoryStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error)
|
||||||
RunSATStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error)
|
RunSATStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error)
|
||||||
RunFanStressTest(ctx context.Context, baseDir string, opts platform.FanStressOptions) (string, error)
|
RunFanCheck(ctx context.Context, baseDir string, opts platform.FanCheckOptions, logFunc func(string)) (string, error)
|
||||||
RunPlatformStress(ctx context.Context, baseDir string, opts platform.PlatformStressOptions, logFunc func(string)) (string, error)
|
RunPlatformStress(ctx context.Context, baseDir string, opts platform.PlatformStressOptions, logFunc func(string)) (string, error)
|
||||||
RunNCCLTests(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error)
|
RunNCCLTests(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error)
|
||||||
RunScenario(ctx context.Context, baseDir string, spec platform.ScenarioSpec, logFunc func(string)) (string, error)
|
RunScenario(ctx context.Context, baseDir string, spec platform.ScenarioSpec, logFunc func(string)) (string, error)
|
||||||
|
|||||||
@@ -298,6 +298,13 @@ func (a *App) RunPlatformStress(ctx context.Context, baseDir string, opts platfo
|
|||||||
return a.sat.RunPlatformStress(ctx, baseDir, opts, logFunc)
|
return a.sat.RunPlatformStress(ctx, baseDir, opts, logFunc)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (a *App) RunFanCheckCtx(ctx context.Context, baseDir string, opts platform.FanCheckOptions, logFunc func(string)) (string, error) {
|
||||||
|
if strings.TrimSpace(baseDir) == "" {
|
||||||
|
baseDir = DefaultSATBaseDir
|
||||||
|
}
|
||||||
|
return a.sat.RunFanCheck(ctx, baseDir, opts, logFunc)
|
||||||
|
}
|
||||||
|
|
||||||
func (a *App) RunNCCLTests(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) {
|
func (a *App) RunNCCLTests(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) {
|
||||||
if strings.TrimSpace(baseDir) == "" {
|
if strings.TrimSpace(baseDir) == "" {
|
||||||
baseDir = DefaultSATBaseDir
|
baseDir = DefaultSATBaseDir
|
||||||
|
|||||||
@@ -367,7 +367,7 @@ func (f fakeSAT) RunSATStressPack(_ context.Context, _ string, _ int, _ func(str
|
|||||||
return "", nil
|
return "", nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f fakeSAT) RunFanStressTest(_ context.Context, _ string, _ platform.FanStressOptions) (string, error) {
|
func (f fakeSAT) RunFanCheck(_ context.Context, _ string, _ platform.FanCheckOptions, _ func(string)) (string, error) {
|
||||||
return "", nil
|
return "", nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -318,6 +318,23 @@ func ApplySATResultToDB(db *ComponentStatusDB, target, archivePath string) {
|
|||||||
db.Record("memory:all", source, dbStatus, detail)
|
db.Record("memory:all", source, dbStatus, detail)
|
||||||
case "cpu", "platform-stress":
|
case "cpu", "platform-stress":
|
||||||
db.Record("cpu:all", source, dbStatus, detail)
|
db.Record("cpu:all", source, dbStatus, detail)
|
||||||
|
case "fan":
|
||||||
|
// Per-fan keys: summary emits "fan_<name>_status=OK|FAILED (...)".
|
||||||
|
for key, val := range kv {
|
||||||
|
name, ok := strings.CutPrefix(key, "fan_")
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
name, ok = strings.CutSuffix(name, "_status")
|
||||||
|
if !ok || name == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
upper := strings.ToUpper(strings.TrimSpace(val))
|
||||||
|
if i := strings.IndexByte(upper, ' '); i > 0 {
|
||||||
|
upper = upper[:i] // drop the "(reason)" suffix
|
||||||
|
}
|
||||||
|
db.Record("fan:"+name, source, satStatusToDBStatus(upper), target+" SAT: "+strings.TrimSpace(val))
|
||||||
|
}
|
||||||
case "storage":
|
case "storage":
|
||||||
// Try to record per-device if available in summary.
|
// Try to record per-device if available in summary.
|
||||||
recordedAny := false
|
recordedAny := false
|
||||||
|
|||||||
@@ -0,0 +1,10 @@
|
|||||||
|
package platform
|
||||||
|
|
||||||
|
import "errors"
|
||||||
|
|
||||||
|
// ErrTestNotApplicable is returned by a SAT routine when the host provides no
|
||||||
|
// way to run that test at all (a required tool or capability is absent), as
|
||||||
|
// opposed to the test running and finding a fault. The task layer maps it to a
|
||||||
|
// cancelled ("not applicable") task rather than a failure, so an engineer does
|
||||||
|
// not see a false red.
|
||||||
|
var ErrTestNotApplicable = errors.New("test not applicable on this platform")
|
||||||
@@ -15,14 +15,18 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
// FanStressOptions configures the fan-stress / thermal cycling test.
|
// FanCheckOptions configures the fan-ceiling check: it drives CPU (+memory)
|
||||||
type FanStressOptions struct {
|
// and, when present, GPU load to 100% simultaneously, then watches every fan
|
||||||
BaselineSec int // idle monitoring before and after load (default 30)
|
// until none has climbed for PlateauHoldSec — at which point each fan is
|
||||||
Phase1DurSec int // first load phase duration in seconds (default 300)
|
// considered to be at its physical ceiling and the observed peak is recorded.
|
||||||
PauseSec int // pause between the two load phases (default 60)
|
type FanCheckOptions struct {
|
||||||
Phase2DurSec int // second load phase duration in seconds (default 300)
|
PlateauHoldSec int // a fan must not rise > PlateauDeltaRPM for this long to count as plateaued (default 60)
|
||||||
SizeMB int // GPU memory to allocate per GPU during stress (0 = auto: 95% of VRAM)
|
PlateauDeltaRPM int // RPM increase that still counts as "climbing" (default 50)
|
||||||
GPUIndices []int // which GPU indices to stress (empty = all detected)
|
MinLoadSec int // never declare a plateau before this many seconds of load (default 90)
|
||||||
|
MaxLoadSec int // hard cap on the load phase; finish (success) even if not every fan plateaued (default 900)
|
||||||
|
RampConfirmRPM int // at least one fan must exceed baseline by this before a plateau is "real" (default 150)
|
||||||
|
SizeMB int // GPU memory to allocate per GPU (0 = auto)
|
||||||
|
GPUIndices []int // which GPU indices to load (empty = all detected)
|
||||||
}
|
}
|
||||||
|
|
||||||
// FanReading holds one fan sensor reading.
|
// FanReading holds one fan sensor reading.
|
||||||
@@ -93,128 +97,217 @@ func normalizeObservedFanMaxRPM(rpm float64) float64 {
|
|||||||
return math.Ceil(rpm/1000.0) * 1000.0
|
return math.Ceil(rpm/1000.0) * 1000.0
|
||||||
}
|
}
|
||||||
|
|
||||||
// RunFanStressTest runs a two-phase GPU stress test while monitoring fan speeds,
|
// RunFanCheck drives CPU (+memory) and, when a GPU is present, GPU load to
|
||||||
// temperatures, and power draw every second. Exports metrics.csv and fan-sensors.csv.
|
// 100% simultaneously and watches every fan until none has climbed for
|
||||||
// Designed to reproduce case-04 fan-speed lag and detect GPU thermal throttling.
|
// PlateauHoldSec. At that point each fan is taken to be at its physical
|
||||||
func (s *System) RunFanStressTest(ctx context.Context, baseDir string, opts FanStressOptions) (string, error) {
|
// ceiling; the observed peak RPM is persisted (fanObservationStatePath, the
|
||||||
|
// same store ObservedFanMaxRPM reads) so the topology view can size each fan
|
||||||
|
// tile against a real maximum.
|
||||||
|
//
|
||||||
|
// Outcome:
|
||||||
|
// - success ("ceiling found") once every fan plateaus, or when MaxLoadSec is
|
||||||
|
// hit — a run that simply ran out of time still recorded the highest RPM
|
||||||
|
// seen and is not a failure.
|
||||||
|
// - a fan reading 0 RPM, or an IPMI status of cr/nr, while under full load is
|
||||||
|
// a real defect → FAILED.
|
||||||
|
// - if there is no way to load this box (no stressapptest/stress-ng and no
|
||||||
|
// GPU burn tool) or no fan sensors are readable, the test cannot say
|
||||||
|
// anything about the hardware and returns ErrTestNotApplicable so the task
|
||||||
|
// is cancelled, not failed.
|
||||||
|
//
|
||||||
|
// No GPU is not an error: CPU/memory load alone is enough to exercise the
|
||||||
|
// cooling loop on most platforms.
|
||||||
|
func (s *System) RunFanCheck(ctx context.Context, baseDir string, opts FanCheckOptions, logFunc func(string)) (string, error) {
|
||||||
|
if logFunc == nil {
|
||||||
|
logFunc = func(string) {}
|
||||||
|
}
|
||||||
if baseDir == "" {
|
if baseDir == "" {
|
||||||
baseDir = "/var/log/bee-sat"
|
baseDir = "/var/log/bee-sat"
|
||||||
}
|
}
|
||||||
applyFanStressDefaults(&opts)
|
applyFanCheckDefaults(&opts)
|
||||||
|
|
||||||
|
baseFans, fanErr := sampleFanSpeeds()
|
||||||
|
if len(baseFans) == 0 {
|
||||||
|
return "", fmt.Errorf("no fan sensors readable via ipmitool or lm-sensors (%v): %w", fanErr, ErrTestNotApplicable)
|
||||||
|
}
|
||||||
|
baselineRPM := make(map[string]float64, len(baseFans))
|
||||||
|
for _, f := range baseFans {
|
||||||
|
baselineRPM[f.Name] = f.RPM
|
||||||
|
}
|
||||||
|
|
||||||
|
vendor := s.DetectGPUVendor()
|
||||||
|
haveGPU := vendor == "nvidia" || vendor == "amd"
|
||||||
|
_, cpuPathErr := satLookPath("stressapptest")
|
||||||
|
if cpuPathErr != nil {
|
||||||
|
_, cpuPathErr = satLookPath("stress-ng")
|
||||||
|
}
|
||||||
|
haveCPU := cpuPathErr == nil
|
||||||
|
if !haveCPU && !haveGPU {
|
||||||
|
return "", fmt.Errorf("no load source: stressapptest/stress-ng missing and no NVIDIA/AMD GPU stress tool available: %w", ErrTestNotApplicable)
|
||||||
|
}
|
||||||
|
|
||||||
ts := time.Now().UTC().Format("20060102-150405")
|
ts := time.Now().UTC().Format("20060102-150405")
|
||||||
runDir := filepath.Join(baseDir, "fan-stress-"+ts)
|
runDir := filepath.Join(baseDir, "fan-check-"+ts)
|
||||||
if err := os.MkdirAll(runDir, 0755); err != nil {
|
if err := os.MkdirAll(runDir, 0755); err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
verboseLog := filepath.Join(runDir, "verbose.log")
|
verboseLog := filepath.Join(runDir, "verbose.log")
|
||||||
|
appendSATVerboseLog(verboseLog, fmt.Sprintf("[%s] fan check start: %d fans, gpu=%s cpu=%v",
|
||||||
|
time.Now().UTC().Format(time.RFC3339), len(baseFans), vendor, haveCPU))
|
||||||
|
logFunc(fmt.Sprintf("Fan check: %d fans; load = CPU/mem:%v + GPU:%s", len(baseFans), haveCPU, orNone(haveGPU, vendor)))
|
||||||
|
|
||||||
// Phase name shared between sampler goroutine and main goroutine.
|
// ── Load: every source runs at the same time, each in its own goroutine.
|
||||||
var phaseMu sync.Mutex
|
// Sources can take different amounts of time to actually reach full load
|
||||||
currentPhase := "init"
|
// (stressapptest is near-instant; a GPU burn kernel needs to compile and
|
||||||
setPhase := func(name string) {
|
// ramp), so the plateau clock does not start until every launched source
|
||||||
phaseMu.Lock()
|
// has reported its process running, plus a fixed GPU ramp grace.
|
||||||
currentPhase = name
|
loadCtx, loadCancel := context.WithTimeout(ctx, time.Duration(opts.MaxLoadSec)*time.Second)
|
||||||
phaseMu.Unlock()
|
defer loadCancel()
|
||||||
|
|
||||||
|
var loadWG sync.WaitGroup
|
||||||
|
started := make(chan bool, 2) // true = source is running, false = failed to launch
|
||||||
|
launched := 0
|
||||||
|
|
||||||
|
if haveCPU {
|
||||||
|
launched++
|
||||||
|
loadWG.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer loadWG.Done()
|
||||||
|
cmd, err := buildCPUStressCmd(loadCtx)
|
||||||
|
if err != nil {
|
||||||
|
logFunc("CPU/memory load failed to start: " + err.Error())
|
||||||
|
appendSATVerboseLog(verboseLog, "cpu load start error: "+err.Error())
|
||||||
|
started <- false
|
||||||
|
return
|
||||||
|
}
|
||||||
|
logFunc("CPU/memory load running (stressapptest)")
|
||||||
|
started <- true
|
||||||
|
_ = cmd.Wait()
|
||||||
|
}()
|
||||||
}
|
}
|
||||||
getPhase := func() string {
|
if haveGPU {
|
||||||
phaseMu.Lock()
|
launched++
|
||||||
defer phaseMu.Unlock()
|
loadWG.Add(1)
|
||||||
return currentPhase
|
go func() {
|
||||||
|
defer loadWG.Done()
|
||||||
|
cmd := buildGPUStressCmd(loadCtx, vendor, opts.MaxLoadSec)
|
||||||
|
if cmd == nil {
|
||||||
|
logFunc("GPU load unavailable (no burn tool for " + vendor + ")")
|
||||||
|
appendSATVerboseLog(verboseLog, "gpu load: no burn tool")
|
||||||
|
started <- false
|
||||||
|
return
|
||||||
|
}
|
||||||
|
logFunc("GPU load running (" + vendor + ")")
|
||||||
|
started <- true
|
||||||
|
_ = cmd.Wait()
|
||||||
|
}()
|
||||||
}
|
}
|
||||||
|
|
||||||
start := time.Now()
|
start := time.Now()
|
||||||
var rowsMu sync.Mutex
|
activeLoads := 0
|
||||||
var allRows []FanStressRow
|
for i := 0; i < launched; i++ {
|
||||||
|
select {
|
||||||
// Start background sampler (every second).
|
case ok := <-started:
|
||||||
stopCh := make(chan struct{})
|
if ok {
|
||||||
doneCh := make(chan struct{})
|
activeLoads++
|
||||||
go func() {
|
|
||||||
defer close(doneCh)
|
|
||||||
ticker := time.NewTicker(time.Second)
|
|
||||||
defer ticker.Stop()
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-stopCh:
|
|
||||||
return
|
|
||||||
case <-ticker.C:
|
|
||||||
row := sampleFanStressRow(opts.GPUIndices, getPhase(), time.Since(start).Seconds())
|
|
||||||
rowsMu.Lock()
|
|
||||||
allRows = append(allRows, row)
|
|
||||||
rowsMu.Unlock()
|
|
||||||
}
|
}
|
||||||
|
case <-ctx.Done():
|
||||||
}
|
}
|
||||||
}()
|
}
|
||||||
|
if activeLoads == 0 {
|
||||||
|
loadCancel()
|
||||||
|
loadWG.Wait()
|
||||||
|
return "", fmt.Errorf("every load source failed to start: %w", ErrTestNotApplicable)
|
||||||
|
}
|
||||||
|
readyAt := time.Now()
|
||||||
|
if haveGPU {
|
||||||
|
readyAt = readyAt.Add(20 * time.Second) // GPU kernel ramp grace
|
||||||
|
}
|
||||||
|
appendSATVerboseLog(verboseLog, fmt.Sprintf("%d load source(s) active; plateau clock effective from +%.0fs",
|
||||||
|
activeLoads, readyAt.Sub(start).Seconds()))
|
||||||
|
|
||||||
var summary strings.Builder
|
// ── Sample loop: one row/second, per-fan plateau tracking.
|
||||||
fmt.Fprintf(&summary, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339))
|
type fanState struct {
|
||||||
|
peak float64
|
||||||
|
lastRiseSec float64
|
||||||
|
}
|
||||||
|
fanBy := map[string]*fanState{}
|
||||||
|
var rows []FanStressRow
|
||||||
|
rampConfirmed := false
|
||||||
|
plateauReached := false
|
||||||
|
aborted := false
|
||||||
|
|
||||||
stats := satStats{}
|
ticker := time.NewTicker(time.Second)
|
||||||
|
defer ticker.Stop()
|
||||||
// idlePhase sleeps for durSec while the sampler stamps phaseName on each row.
|
loop:
|
||||||
idlePhase := func(phaseName, stepName string, durSec int) {
|
for {
|
||||||
if ctx.Err() != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
setPhase(phaseName)
|
|
||||||
appendSATVerboseLog(verboseLog,
|
|
||||||
fmt.Sprintf("[%s] start %s (idle %ds)", time.Now().UTC().Format(time.RFC3339), stepName, durSec),
|
|
||||||
)
|
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
case <-time.After(time.Duration(durSec) * time.Second):
|
aborted = true
|
||||||
|
break loop
|
||||||
|
case <-loadCtx.Done():
|
||||||
|
break loop // MaxLoadSec reached
|
||||||
|
case <-ticker.C:
|
||||||
}
|
}
|
||||||
appendSATVerboseLog(verboseLog,
|
elapsed := time.Since(start).Seconds()
|
||||||
fmt.Sprintf("[%s] finish %s", time.Now().UTC().Format(time.RFC3339), stepName),
|
row := sampleFanStressRow(opts.GPUIndices, "load", elapsed)
|
||||||
)
|
rows = append(rows, row)
|
||||||
fmt.Fprintf(&summary, "%s_status=OK\n", stepName)
|
|
||||||
stats.OK++
|
|
||||||
}
|
|
||||||
|
|
||||||
// loadPhase runs bee-gpu-burn for durSec; sampler stamps phaseName on each row.
|
for _, f := range row.Fans {
|
||||||
loadPhase := func(phaseName, stepName string, durSec int) {
|
st := fanBy[f.Name]
|
||||||
if ctx.Err() != nil {
|
if st == nil {
|
||||||
return
|
fanBy[f.Name] = &fanState{peak: f.RPM, lastRiseSec: elapsed}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if f.RPM > st.peak {
|
||||||
|
if f.RPM-st.peak > float64(opts.PlateauDeltaRPM) {
|
||||||
|
st.lastRiseSec = elapsed
|
||||||
|
}
|
||||||
|
st.peak = f.RPM
|
||||||
|
}
|
||||||
|
if f.RPM >= baselineRPM[f.Name]+float64(opts.RampConfirmRPM) {
|
||||||
|
rampConfirmed = true
|
||||||
|
}
|
||||||
}
|
}
|
||||||
setPhase(phaseName)
|
|
||||||
cmd := []string{
|
if elapsed >= float64(opts.MinLoadSec) && time.Since(readyAt) >= time.Duration(opts.PlateauHoldSec)*time.Second && len(fanBy) > 0 {
|
||||||
"bee-gpu-burn",
|
allFlat := true
|
||||||
"--seconds", strconv.Itoa(durSec),
|
for _, st := range fanBy {
|
||||||
"--size-mb", strconv.Itoa(opts.SizeMB),
|
if elapsed-st.lastRiseSec < float64(opts.PlateauHoldSec) {
|
||||||
}
|
allFlat = false
|
||||||
if len(opts.GPUIndices) > 0 {
|
break
|
||||||
cmd = append(cmd, "--devices", joinIndexList(dedupeSortedIndices(opts.GPUIndices)))
|
}
|
||||||
}
|
}
|
||||||
out, err := runSATCommandCtx(ctx, verboseLog, stepName, cmd, nil, nil)
|
if allFlat && rampConfirmed {
|
||||||
_ = os.WriteFile(filepath.Join(runDir, stepName+".log"), out, 0644)
|
plateauReached = true
|
||||||
if err != nil && err != context.Canceled && err.Error() != "signal: killed" {
|
logFunc(fmt.Sprintf("All %d fans plateaued at %.0fs of load", len(fanBy), elapsed))
|
||||||
fmt.Fprintf(&summary, "%s_status=FAILED\n", stepName)
|
break loop
|
||||||
stats.Failed++
|
}
|
||||||
} else {
|
|
||||||
fmt.Fprintf(&summary, "%s_status=OK\n", stepName)
|
|
||||||
stats.OK++
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Execute test phases.
|
loadCancel()
|
||||||
idlePhase("baseline", "01-baseline", opts.BaselineSec)
|
loadWG.Wait()
|
||||||
loadPhase("load1", "02-load1", opts.Phase1DurSec)
|
|
||||||
idlePhase("pause", "03-pause", opts.PauseSec)
|
|
||||||
loadPhase("load2", "04-load2", opts.Phase2DurSec)
|
|
||||||
idlePhase("cooldown", "05-cooldown", opts.BaselineSec)
|
|
||||||
|
|
||||||
// Stop sampler and collect rows.
|
if aborted && ctx.Err() != nil {
|
||||||
close(stopCh)
|
_ = os.WriteFile(filepath.Join(runDir, "summary.txt"),
|
||||||
<-doneCh
|
[]byte("run_at_utc="+time.Now().UTC().Format(time.RFC3339)+"\noverall_status=UNKNOWN\naborted=true\n"), 0644)
|
||||||
|
return runDir, ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
rowsMu.Lock()
|
// ── Verdict.
|
||||||
rows := allRows
|
statuses := readFanStatuses()
|
||||||
rowsMu.Unlock()
|
var summary strings.Builder
|
||||||
|
fmt.Fprintf(&summary, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339))
|
||||||
// Analysis.
|
fmt.Fprintf(&summary, "fans_total=%d\n", len(baseFans))
|
||||||
throttled := analyzeThrottling(rows)
|
fmt.Fprintf(&summary, "active_load_sources=%d\n", activeLoads)
|
||||||
maxGPUTemp := analyzeMaxTemp(rows, func(r FanStressRow) float64 {
|
fmt.Fprintf(&summary, "gpu_vendor=%s\n", orNone(haveGPU, vendor))
|
||||||
|
fmt.Fprintf(&summary, "plateau_reached=%v\n", plateauReached)
|
||||||
|
fmt.Fprintf(&summary, "ramp_confirmed=%v\n", rampConfirmed)
|
||||||
|
if len(rows) > 0 {
|
||||||
|
fmt.Fprintf(&summary, "load_duration_sec=%.0f\n", rows[len(rows)-1].ElapsedSec)
|
||||||
|
}
|
||||||
|
fmt.Fprintf(&summary, "max_gpu_temp_c=%.1f\n", analyzeMaxTemp(rows, func(r FanStressRow) float64 {
|
||||||
var m float64
|
var m float64
|
||||||
for _, g := range r.GPUs {
|
for _, g := range r.GPUs {
|
||||||
if g.TempC > m {
|
if g.TempC > m {
|
||||||
@@ -222,55 +315,148 @@ func (s *System) RunFanStressTest(ctx context.Context, baseDir string, opts FanS
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
return m
|
return m
|
||||||
})
|
}))
|
||||||
maxCPUTemp := analyzeMaxTemp(rows, func(r FanStressRow) float64 {
|
fmt.Fprintf(&summary, "max_cpu_temp_c=%.1f\n", analyzeMaxTemp(rows, func(r FanStressRow) float64 { return r.CPUMaxTempC }))
|
||||||
return r.CPUMaxTempC
|
|
||||||
})
|
|
||||||
fanResponseSec := analyzeFanResponse(rows)
|
|
||||||
|
|
||||||
fmt.Fprintf(&summary, "throttling_detected=%v\n", throttled)
|
stats := satStats{}
|
||||||
fmt.Fprintf(&summary, "max_gpu_temp_c=%.1f\n", maxGPUTemp)
|
names := make([]string, 0, len(baselineRPM))
|
||||||
fmt.Fprintf(&summary, "max_cpu_temp_c=%.1f\n", maxCPUTemp)
|
for n := range baselineRPM {
|
||||||
if fanResponseSec >= 0 {
|
names = append(names, n)
|
||||||
fmt.Fprintf(&summary, "fan_response_sec=%.1f\n", fanResponseSec)
|
|
||||||
} else {
|
|
||||||
fmt.Fprintf(&summary, "fan_response_sec=N/A\n")
|
|
||||||
}
|
}
|
||||||
|
sort.Strings(names)
|
||||||
// Throttling failure counts against overall result.
|
for _, name := range names {
|
||||||
if throttled {
|
peak := baselineRPM[name]
|
||||||
stats.Failed++
|
if st := fanBy[name]; st != nil {
|
||||||
|
peak = st.peak
|
||||||
|
}
|
||||||
|
st := strings.ToLower(strings.TrimSpace(statuses[name]))
|
||||||
|
bad := peak <= 0 || st == "cr" || st == "nr"
|
||||||
|
key := sanitizeSummaryKey(name)
|
||||||
|
fmt.Fprintf(&summary, "fan_%s_baseline_rpm=%.0f\n", key, baselineRPM[name])
|
||||||
|
fmt.Fprintf(&summary, "fan_%s_max_rpm=%.0f\n", key, peak)
|
||||||
|
if bad {
|
||||||
|
reason := "0 RPM under load"
|
||||||
|
if st == "cr" || st == "nr" {
|
||||||
|
reason = "IPMI status " + st
|
||||||
|
}
|
||||||
|
fmt.Fprintf(&summary, "fan_%s_status=FAILED (%s)\n", key, reason)
|
||||||
|
logFunc(fmt.Sprintf("FAIL %s: %s", name, reason))
|
||||||
|
stats.Failed++
|
||||||
|
} else {
|
||||||
|
fmt.Fprintf(&summary, "fan_%s_status=OK\n", key)
|
||||||
|
stats.OK++
|
||||||
|
}
|
||||||
}
|
}
|
||||||
writeSATStats(&summary, stats)
|
writeSATStats(&summary, stats)
|
||||||
|
|
||||||
// Write CSV outputs.
|
_ = WriteFanStressCSV(filepath.Join(runDir, "metrics.csv"), rows, opts.GPUIndices)
|
||||||
if err := WriteFanStressCSV(filepath.Join(runDir, "metrics.csv"), rows, opts.GPUIndices); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
_ = WriteFanSensorsCSV(filepath.Join(runDir, "fan-sensors.csv"), rows)
|
_ = WriteFanSensorsCSV(filepath.Join(runDir, "fan-sensors.csv"), rows)
|
||||||
|
|
||||||
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary.String()), 0644); err != nil {
|
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary.String()), 0644); err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
return runDir, nil
|
return runDir, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func applyFanStressDefaults(opts *FanStressOptions) {
|
func applyFanCheckDefaults(o *FanCheckOptions) {
|
||||||
if opts.BaselineSec <= 0 {
|
if o.PlateauHoldSec <= 0 {
|
||||||
opts.BaselineSec = 30
|
o.PlateauHoldSec = 60
|
||||||
}
|
}
|
||||||
if opts.Phase1DurSec <= 0 {
|
if o.PlateauDeltaRPM <= 0 {
|
||||||
opts.Phase1DurSec = 300
|
o.PlateauDeltaRPM = 50
|
||||||
}
|
}
|
||||||
if opts.PauseSec <= 0 {
|
if o.MinLoadSec <= 0 {
|
||||||
opts.PauseSec = 60
|
o.MinLoadSec = 90
|
||||||
}
|
}
|
||||||
if opts.Phase2DurSec <= 0 {
|
if o.MaxLoadSec <= 0 {
|
||||||
opts.Phase2DurSec = 300
|
o.MaxLoadSec = 900
|
||||||
}
|
}
|
||||||
// SizeMB == 0 means "auto" (worker picks 95% of GPU VRAM for maximum power draw).
|
if o.RampConfirmRPM <= 0 {
|
||||||
// Leave at 0 to avoid passing a too-small size that starves the tensor-core path.
|
o.RampConfirmRPM = 150
|
||||||
|
}
|
||||||
|
if o.MinLoadSec < o.PlateauHoldSec {
|
||||||
|
o.MinLoadSec = o.PlateauHoldSec
|
||||||
|
}
|
||||||
|
if o.MaxLoadSec <= o.MinLoadSec {
|
||||||
|
o.MaxLoadSec = o.MinLoadSec + o.PlateauHoldSec
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func orNone(present bool, v string) string {
|
||||||
|
if present && v != "" {
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
return "none"
|
||||||
|
}
|
||||||
|
|
||||||
|
// sanitizeSummaryKey makes a fan sensor name safe as a summary.txt key
|
||||||
|
// fragment (keys are parsed by splitting on '=' and whitespace).
|
||||||
|
func sanitizeSummaryKey(name string) string {
|
||||||
|
var b strings.Builder
|
||||||
|
for _, r := range name {
|
||||||
|
switch {
|
||||||
|
case r >= 'A' && r <= 'Z', r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '-', r == '_', r == '.':
|
||||||
|
b.WriteRune(r)
|
||||||
|
default:
|
||||||
|
b.WriteByte('_')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// readFanStatuses returns the per-fan IPMI status word ("ok", "cr", "nr", ...)
|
||||||
|
// from "ipmitool sdr type Fan". Empty map when ipmitool is unavailable.
|
||||||
|
func readFanStatuses() map[string]string {
|
||||||
|
out, err := exec.Command("ipmitool", "sdr", "type", "Fan").Output()
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
m := map[string]string{}
|
||||||
|
for _, line := range strings.Split(string(out), "\n") {
|
||||||
|
parts := strings.Split(line, "|")
|
||||||
|
if len(parts) < 3 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
name := strings.TrimSpace(parts[0])
|
||||||
|
if name == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
m[name] = strings.ToLower(strings.TrimSpace(parts[2]))
|
||||||
|
}
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
// ResolveFanMaxRPM returns, for every fan name in current (name -> current
|
||||||
|
// RPM), the RPM to treat as that fan's 100% reference. Preference order:
|
||||||
|
// 1. the persisted observed peak (fanObservationStatePath), written by
|
||||||
|
// RunFanCheck and by live-metrics sampling under load;
|
||||||
|
// 2. the largest peak observed on any peer fan (keeps a group visually
|
||||||
|
// consistent when only some fans have a recorded peak);
|
||||||
|
// 3. the fan's own current RPM (so a tile is never sized against zero).
|
||||||
|
//
|
||||||
|
// The fallback lives here, not in the view, so every consumer of a fan
|
||||||
|
// maximum applies the same rule.
|
||||||
|
func ResolveFanMaxRPM(current map[string]float64) map[string]float64 {
|
||||||
|
persisted := readPersistedFanMaxRPM()
|
||||||
|
|
||||||
|
peerMax := 0.0
|
||||||
|
for _, v := range persisted {
|
||||||
|
if v > peerMax {
|
||||||
|
peerMax = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
out := make(map[string]float64, len(current))
|
||||||
|
for name, rpm := range current {
|
||||||
|
switch {
|
||||||
|
case persisted[name] > 0:
|
||||||
|
out[name] = persisted[name]
|
||||||
|
case peerMax > 0:
|
||||||
|
out[name] = peerMax
|
||||||
|
default:
|
||||||
|
out[name] = rpm
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// sampleFanStressRow collects all metrics for one telemetry sample.
|
// sampleFanStressRow collects all metrics for one telemetry sample.
|
||||||
@@ -355,27 +541,48 @@ func sampleFanSpeeds() ([]FanReading, error) {
|
|||||||
return nil, sensorsErr
|
return nil, sensorsErr
|
||||||
}
|
}
|
||||||
|
|
||||||
func loadFanObservationLocked() {
|
// readPersistedFanMaxRPM reads fanObservationStatePath and returns its
|
||||||
if fanObservationInit {
|
// sanitized {fan name -> observed peak RPM} map (empty names / non-positive
|
||||||
return
|
// values dropped). Returns an empty map when the file is missing or unparsable.
|
||||||
}
|
func readPersistedFanMaxRPM() map[string]float64 {
|
||||||
fanObservationInit = true
|
out := map[string]float64{}
|
||||||
fanObservation.MaxRPM = make(map[string]float64)
|
|
||||||
raw, err := os.ReadFile(fanObservationStatePath)
|
raw, err := os.ReadFile(fanObservationStatePath)
|
||||||
if err != nil || len(raw) == 0 {
|
if err != nil || len(raw) == 0 {
|
||||||
return
|
return out
|
||||||
}
|
}
|
||||||
var persisted fanObservationState
|
var persisted fanObservationState
|
||||||
if json.Unmarshal(raw, &persisted) != nil {
|
if json.Unmarshal(raw, &persisted) != nil {
|
||||||
return
|
return out
|
||||||
}
|
}
|
||||||
for name, rpm := range persisted.MaxRPM {
|
for name, rpm := range persisted.MaxRPM {
|
||||||
name = strings.TrimSpace(name)
|
name = strings.TrimSpace(name)
|
||||||
if name == "" || rpm <= 0 {
|
if name == "" || rpm <= 0 {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
fanObservation.MaxRPM[name] = rpm
|
out[name] = rpm
|
||||||
}
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// ObservedFanMaxRPM returns the per-fan observed peak RPM map persisted by
|
||||||
|
// fan-stress SAT runs, or nil if none is recorded yet. It reads the file
|
||||||
|
// directly without touching the in-process observation cache or its lock, so
|
||||||
|
// read-only consumers (the /topo web view) can call it without perturbing a
|
||||||
|
// concurrent SAT run's peak tracking.
|
||||||
|
func ObservedFanMaxRPM() map[string]float64 {
|
||||||
|
out := readPersistedFanMaxRPM()
|
||||||
|
if len(out) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadFanObservationLocked() {
|
||||||
|
if fanObservationInit {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fanObservationInit = true
|
||||||
|
fanObservation.MaxRPM = readPersistedFanMaxRPM()
|
||||||
}
|
}
|
||||||
|
|
||||||
func saveFanObservationLocked() {
|
func saveFanObservationLocked() {
|
||||||
@@ -779,22 +986,6 @@ func effectiveSystemPowerReading(cache cachedPowerReading, current float64, sour
|
|||||||
return 0, cache
|
return 0, cache
|
||||||
}
|
}
|
||||||
|
|
||||||
// analyzeThrottling returns true if any GPU reported an active throttle reason
|
|
||||||
// during either load phase.
|
|
||||||
func analyzeThrottling(rows []FanStressRow) bool {
|
|
||||||
for _, row := range rows {
|
|
||||||
if row.Phase != "load1" && row.Phase != "load2" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
for _, gpu := range row.GPUs {
|
|
||||||
if gpu.Throttled {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// analyzeMaxTemp returns the maximum value of the given extractor across all rows.
|
// analyzeMaxTemp returns the maximum value of the given extractor across all rows.
|
||||||
func analyzeMaxTemp(rows []FanStressRow, extract func(FanStressRow) float64) float64 {
|
func analyzeMaxTemp(rows []FanStressRow, extract func(FanStressRow) float64) float64 {
|
||||||
var max float64
|
var max float64
|
||||||
@@ -806,55 +997,6 @@ func analyzeMaxTemp(rows []FanStressRow, extract func(FanStressRow) float64) flo
|
|||||||
return max
|
return max
|
||||||
}
|
}
|
||||||
|
|
||||||
// analyzeFanResponse returns the seconds from load1 start until fan RPM first
|
|
||||||
// increased by more than 5% above the baseline average. Returns -1 if undetermined.
|
|
||||||
func analyzeFanResponse(rows []FanStressRow) float64 {
|
|
||||||
// Compute baseline average fan RPM.
|
|
||||||
var baseTotal, baseCount float64
|
|
||||||
for _, row := range rows {
|
|
||||||
if row.Phase != "baseline" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
for _, f := range row.Fans {
|
|
||||||
baseTotal += f.RPM
|
|
||||||
baseCount++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if baseCount == 0 || baseTotal == 0 {
|
|
||||||
return -1
|
|
||||||
}
|
|
||||||
baseAvg := baseTotal / baseCount
|
|
||||||
threshold := baseAvg * 1.05 // 5% increase signals fan ramp-up
|
|
||||||
|
|
||||||
// Find elapsed time when load1 started.
|
|
||||||
var load1Start float64 = -1
|
|
||||||
for _, row := range rows {
|
|
||||||
if row.Phase == "load1" {
|
|
||||||
load1Start = row.ElapsedSec
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if load1Start < 0 {
|
|
||||||
return -1
|
|
||||||
}
|
|
||||||
|
|
||||||
// Find first load1 row where average RPM crosses the threshold.
|
|
||||||
for _, row := range rows {
|
|
||||||
if row.Phase != "load1" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
var total, count float64
|
|
||||||
for _, f := range row.Fans {
|
|
||||||
total += f.RPM
|
|
||||||
count++
|
|
||||||
}
|
|
||||||
if count > 0 && total/count >= threshold {
|
|
||||||
return row.ElapsedSec - load1Start
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return -1
|
|
||||||
}
|
|
||||||
|
|
||||||
// WriteFanStressCSV writes the wide-format metrics CSV with one row per second.
|
// WriteFanStressCSV writes the wide-format metrics CSV with one row per second.
|
||||||
// GPU columns are generated per index in gpuIndices order.
|
// GPU columns are generated per index in gpuIndices order.
|
||||||
func WriteFanStressCSV(path string, rows []FanStressRow, gpuIndices []int) error {
|
func WriteFanStressCSV(path string, rows []FanStressRow, gpuIndices []int) error {
|
||||||
|
|||||||
@@ -1,11 +1,68 @@
|
|||||||
package platform
|
package platform
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func TestResolveFanMaxRPM(t *testing.T) {
|
||||||
|
oldPath := fanObservationStatePath
|
||||||
|
oldInit := fanObservationInit
|
||||||
|
fanObservationStatePath = filepath.Join(t.TempDir(), "fan-observation.json")
|
||||||
|
fanObservationInit = false
|
||||||
|
t.Cleanup(func() {
|
||||||
|
fanObservationStatePath = oldPath
|
||||||
|
fanObservationInit = oldInit
|
||||||
|
})
|
||||||
|
|
||||||
|
// No persisted file yet: unknown fans fall back to their own current RPM.
|
||||||
|
got := ResolveFanMaxRPM(map[string]float64{"A": 4000, "B": 9000})
|
||||||
|
if !reflect.DeepEqual(got, map[string]float64{"A": 4000, "B": 9000}) {
|
||||||
|
t.Fatalf("no-persist fallback: got %v", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.WriteFile(fanObservationStatePath, []byte(`{"max_rpm":{"A":17000}}`), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got = ResolveFanMaxRPM(map[string]float64{"A": 4000, "B": 9000, "C": 5000})
|
||||||
|
// A: persisted peak. B/C: no own entry -> largest peer peak (A's 17000).
|
||||||
|
if want := map[string]float64{"A": 17000, "B": 17000, "C": 17000}; !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("peer fallback: got %v want %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplyFanCheckDefaults(t *testing.T) {
|
||||||
|
var o FanCheckOptions
|
||||||
|
applyFanCheckDefaults(&o)
|
||||||
|
if o.PlateauHoldSec != 60 || o.PlateauDeltaRPM != 50 || o.MinLoadSec != 90 || o.MaxLoadSec != 900 || o.RampConfirmRPM != 150 {
|
||||||
|
t.Fatalf("unexpected defaults: %+v", o)
|
||||||
|
}
|
||||||
|
o = FanCheckOptions{PlateauHoldSec: 120, MinLoadSec: 30, MaxLoadSec: 40}
|
||||||
|
applyFanCheckDefaults(&o)
|
||||||
|
if o.MinLoadSec < o.PlateauHoldSec {
|
||||||
|
t.Fatalf("MinLoadSec must be >= PlateauHoldSec, got %d", o.MinLoadSec)
|
||||||
|
}
|
||||||
|
if o.MaxLoadSec <= o.MinLoadSec {
|
||||||
|
t.Fatalf("MaxLoadSec must exceed MinLoadSec, got %d", o.MaxLoadSec)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSanitizeSummaryKey(t *testing.T) {
|
||||||
|
for in, want := range map[string]string{
|
||||||
|
"F2U-1": "F2U-1",
|
||||||
|
"aspeed / fan1": "aspeed___fan1",
|
||||||
|
"CPU0_DIMM": "CPU0_DIMM",
|
||||||
|
"weird=key here": "weird_key_here",
|
||||||
|
} {
|
||||||
|
if got := sanitizeSummaryKey(in); got != want {
|
||||||
|
t.Errorf("sanitizeSummaryKey(%q)=%q want %q", in, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestParseFanSpeeds(t *testing.T) {
|
func TestParseFanSpeeds(t *testing.T) {
|
||||||
raw := "FAN1 | 2400.000 | RPM | ok\nFAN2 | 1800 RPM | ok | ok\nFAN3 | na | RPM | ns\n"
|
raw := "FAN1 | 2400.000 | RPM | ok\nFAN2 | 1800 RPM | ok | ok\nFAN3 | na | RPM | ns\n"
|
||||||
got := parseFanSpeeds(raw)
|
got := parseFanSpeeds(raw)
|
||||||
|
|||||||
@@ -128,7 +128,7 @@ func defaultTaskPriority(target string, params taskParams) int {
|
|||||||
return taskPriorityAudit
|
return taskPriorityAudit
|
||||||
case "nvidia-bench-perf", "nvidia-bench-power", "nvidia-bench-autotune":
|
case "nvidia-bench-perf", "nvidia-bench-power", "nvidia-bench-autotune":
|
||||||
return taskPriorityBenchmark
|
return taskPriorityBenchmark
|
||||||
case "nvidia-stress", "amd-stress", "memory-stress", "sat-stress", "platform-stress", "nvidia-compute", "scenario":
|
case "nvidia-stress", "amd-stress", "memory-stress", "sat-stress", "platform-stress", "fan", "nvidia-compute", "scenario":
|
||||||
return taskPriorityBurn
|
return taskPriorityBurn
|
||||||
case "nvidia", "nvidia-targeted-stress", "nvidia-targeted-power", "nvidia-pulse",
|
case "nvidia", "nvidia-targeted-stress", "nvidia-targeted-power", "nvidia-pulse",
|
||||||
"nvidia-interconnect", "nvidia-bandwidth", "memory", "storage", "cpu",
|
"nvidia-interconnect", "nvidia-bandwidth", "memory", "storage", "cpu",
|
||||||
|
|||||||
@@ -289,7 +289,7 @@ func (h *handler) handleAPIHardwareSummary(w http.ResponseWriter, _ *http.Reques
|
|||||||
}
|
}
|
||||||
|
|
||||||
// handleAPIComponentDetail returns an HTML fragment describing the current and
|
// handleAPIComponentDetail returns an HTML fragment describing the current and
|
||||||
// historical status for one component type (cpu, memory, storage, gpu, psu).
|
// historical status for one component type (cpu, memory, storage, gpu, psu, fan).
|
||||||
func (h *handler) handleAPIComponentDetail(w http.ResponseWriter, r *http.Request) {
|
func (h *handler) handleAPIComponentDetail(w http.ResponseWriter, r *http.Request) {
|
||||||
compType := r.PathValue("type")
|
compType := r.PathValue("type")
|
||||||
var exact, prefixes []string
|
var exact, prefixes []string
|
||||||
@@ -315,6 +315,9 @@ func (h *handler) handleAPIComponentDetail(w http.ResponseWriter, r *http.Reques
|
|||||||
case "psu":
|
case "psu":
|
||||||
title = "PSU"
|
title = "PSU"
|
||||||
prefixes = []string{"psu:"}
|
prefixes = []string{"psu:"}
|
||||||
|
case "fan":
|
||||||
|
title = "Fans"
|
||||||
|
prefixes = []string{"fan:"}
|
||||||
case "raid":
|
case "raid":
|
||||||
title = "RAID"
|
title = "RAID"
|
||||||
prefixes = []string{"pcie:raid:"}
|
prefixes = []string{"pcie:raid:"}
|
||||||
|
|||||||
@@ -109,6 +109,11 @@ func (h *handler) planSATRunAll(ctx context.Context, req satRunAllRequest) ([]sa
|
|||||||
} else {
|
} else {
|
||||||
skip("TPM: no TPM device on this host; check skipped")
|
skip("TPM: no TPM device on this host; check skipped")
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
// Fan ceiling check runs on the Load tier only. It self-cancels as
|
||||||
|
// "not applicable" on a host with no fan sensors or no way to load
|
||||||
|
// the CPU/GPU, so it is safe to queue unconditionally here.
|
||||||
|
specs = append(specs, satRunAllSpec{target: "fan", params: taskParams{StressMode: true}})
|
||||||
}
|
}
|
||||||
|
|
||||||
gp := h.opts.App.DetectGPUPresence()
|
gp := h.opts.App.DetectGPUPresence()
|
||||||
|
|||||||
@@ -98,7 +98,7 @@ func TestPlanSATRunAllLoadOmitsReadOnlyTPMCheck(t *testing.T) {
|
|||||||
for _, s := range specs {
|
for _, s := range specs {
|
||||||
targets = append(targets, s.target)
|
targets = append(targets, s.target)
|
||||||
}
|
}
|
||||||
if want := []string{"cpu", "memory", "storage"}; !reflect.DeepEqual(targets, want) {
|
if want := []string{"cpu", "memory", "storage", "fan"}; !reflect.DeepEqual(targets, want) {
|
||||||
t.Fatalf("targets=%v want %v", targets, want)
|
t.Fatalf("targets=%v want %v", targets, want)
|
||||||
}
|
}
|
||||||
for _, note := range notes {
|
for _, note := range notes {
|
||||||
|
|||||||
@@ -296,7 +296,7 @@ func isSATTarget(target string) bool {
|
|||||||
case "nvidia", "nvidia-targeted-stress", "nvidia-bench-perf", "nvidia-bench-power", "nvidia-compute", "nvidia-targeted-power", "nvidia-pulse",
|
case "nvidia", "nvidia-targeted-stress", "nvidia-bench-perf", "nvidia-bench-power", "nvidia-compute", "nvidia-targeted-power", "nvidia-pulse",
|
||||||
"nvidia-interconnect", "nvidia-bandwidth", "nvidia-stress", "memory", "memory-stress", "storage",
|
"nvidia-interconnect", "nvidia-bandwidth", "nvidia-stress", "memory", "memory-stress", "storage",
|
||||||
"cpu", "sat-stress", "amd", "amd-mem", "amd-bandwidth", "amd-stress",
|
"cpu", "sat-stress", "amd", "amd-mem", "amd-bandwidth", "amd-stress",
|
||||||
"platform-stress":
|
"platform-stress", "fan":
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
return false
|
return false
|
||||||
|
|||||||
@@ -614,9 +614,138 @@ func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string
|
|||||||
}}))
|
}}))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Cooling fans — one small square per fan (no PCIe/CPU affinity, arbitrary
|
||||||
|
// count, so a wrapping flex row like PSUs rather than SVG boxes). Each
|
||||||
|
// square is sized by rpm / observed-max-rpm and carries a fan glyph that
|
||||||
|
// spins via CSS — faster when the fan is spinning faster.
|
||||||
|
if fans := dedupeFansByName(hw.Sensors); len(fans) > 0 {
|
||||||
|
current := map[string]float64{}
|
||||||
|
for _, f := range fans {
|
||||||
|
if f.RPM != nil {
|
||||||
|
current[strings.TrimSpace(f.Name)] = float64(*f.RPM)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
b.WriteString(renderTopoFanRow(fans, platform.ResolveFanMaxRPM(current)))
|
||||||
|
}
|
||||||
|
|
||||||
return topoCard("Topology", b.String())
|
return topoCard("Topology", b.String())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// renderTopoFanRow renders the COOLING row: one clickable square per fan,
|
||||||
|
// side length scaled by rpm/maxRPM and a fan glyph whose spin rate tracks the
|
||||||
|
// same ratio. maxByName comes from platform.ResolveFanMaxRPM — it already has
|
||||||
|
// an entry for every fan (persisted peak, else peer peak, else current RPM),
|
||||||
|
// so no fallback logic lives here.
|
||||||
|
func renderTopoFanRow(fans []schema.HardwareFanSensor, maxByName map[string]float64) string {
|
||||||
|
const (
|
||||||
|
fanTileMin = 30 // px, a stalled / slowest fan
|
||||||
|
fanTileMax = 58 // px, a fan at its ceiling
|
||||||
|
)
|
||||||
|
|
||||||
|
var tally topoStatusTally
|
||||||
|
for _, f := range fans {
|
||||||
|
tally.add(classifyTopoSeverity(f.Status))
|
||||||
|
}
|
||||||
|
|
||||||
|
var b strings.Builder
|
||||||
|
b.WriteString(topoRowHeading("Cooling"))
|
||||||
|
b.WriteString(topoFanSpinStyle())
|
||||||
|
b.WriteString(`<div style="display:flex;flex-wrap:wrap;gap:6px;align-items:flex-end">`)
|
||||||
|
for _, f := range fans {
|
||||||
|
name := strings.TrimSpace(f.Name)
|
||||||
|
fill, stroke, text := topoSeverityColors(classifyTopoSeverity(f.Status))
|
||||||
|
|
||||||
|
denom := maxByName[name]
|
||||||
|
ratio := 0.0
|
||||||
|
title := name
|
||||||
|
if f.RPM != nil {
|
||||||
|
if denom > 0 {
|
||||||
|
ratio = float64(*f.RPM) / denom
|
||||||
|
}
|
||||||
|
if ratio < 0 {
|
||||||
|
ratio = 0
|
||||||
|
}
|
||||||
|
if ratio > 1 {
|
||||||
|
ratio = 1
|
||||||
|
}
|
||||||
|
if denom > float64(*f.RPM) {
|
||||||
|
title = fmt.Sprintf("%s · %d RPM (max %d)", name, *f.RPM, int(denom))
|
||||||
|
} else {
|
||||||
|
title = fmt.Sprintf("%s · %d RPM", name, *f.RPM)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
title = name + " · no reading"
|
||||||
|
}
|
||||||
|
|
||||||
|
side := fanTileMin + int(float64(fanTileMax-fanTileMin)*ratio+0.5)
|
||||||
|
// Spin period: 2.6s at rest down to 0.5s at the observed peak. A fan
|
||||||
|
// with no reading doesn't spin.
|
||||||
|
spin := ""
|
||||||
|
if f.RPM != nil && *f.RPM > 0 {
|
||||||
|
period := 2.6 - 2.1*ratio
|
||||||
|
spin = fmt.Sprintf(`<svg class="topo-fan-spin" style="animation-duration:%.2fs" width="%d" height="%d" viewBox="0 0 24 24" fill="currentColor" aria-hidden="true">`,
|
||||||
|
period, side*7/16, side*7/16) + topoFanGlyphPaths() + `</svg>`
|
||||||
|
} else {
|
||||||
|
spin = fmt.Sprintf(`<svg width="%d" height="%d" viewBox="0 0 24 24" fill="currentColor" style="opacity:.4" aria-hidden="true">`,
|
||||||
|
side*7/16, side*7/16) + topoFanGlyphPaths() + `</svg>`
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Fprintf(&b, `<div title="%s" onclick="openComponentDetail('fan')" `+
|
||||||
|
`style="width:%dpx;height:%dpx;display:flex;align-items:center;justify-content:center;`+
|
||||||
|
`border-radius:5px;background:%s;border:1px solid %s;color:%s;cursor:pointer">%s</div>`,
|
||||||
|
html.EscapeString(title), side, side, fill, stroke, text, spin)
|
||||||
|
}
|
||||||
|
b.WriteString(`</div>`)
|
||||||
|
fmt.Fprintf(&b, `<div style="font-size:11px;color:var(--muted);margin-top:6px">%d fans · %s · tile size ∝ RPM / observed max</div>`,
|
||||||
|
len(fans), html.EscapeString(tally.line()))
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// topoFanSpinStyle emits the keyframes + base class for the spinning fan
|
||||||
|
// glyph once per row. A repeated identical <style> is harmless.
|
||||||
|
func topoFanSpinStyle() string {
|
||||||
|
return `<style>@keyframes topoFanSpin{to{transform:rotate(360deg)}}` +
|
||||||
|
`.topo-fan-spin{transform-box:fill-box;transform-origin:center;` +
|
||||||
|
`animation-name:topoFanSpin;animation-timing-function:linear;animation-iteration-count:infinite}` +
|
||||||
|
`@media (prefers-reduced-motion:reduce){.topo-fan-spin{animation:none}}</style>`
|
||||||
|
}
|
||||||
|
|
||||||
|
// topoFanGlyphPaths is the fan-blade drawing shared by every fan square,
|
||||||
|
// designed on a 24×24 viewBox.
|
||||||
|
func topoFanGlyphPaths() string {
|
||||||
|
return `<ellipse cx="12" cy="6.5" rx="3.1" ry="5.2"/>` +
|
||||||
|
`<ellipse cx="12" cy="6.5" rx="3.1" ry="5.2" transform="rotate(120 12 12)"/>` +
|
||||||
|
`<ellipse cx="12" cy="6.5" rx="3.1" ry="5.2" transform="rotate(240 12 12)"/>` +
|
||||||
|
`<circle cx="12" cy="12" r="2.3"/>`
|
||||||
|
}
|
||||||
|
|
||||||
|
// dedupeFansByName returns the fan sensors from a snapshot with duplicate
|
||||||
|
// names collapsed to their first occurrence, matching the ingest contract's
|
||||||
|
// "(sensor_type, name) — first wins" rule and skipping unnamed sensors.
|
||||||
|
func dedupeFansByName(sensors *schema.HardwareSensors) []schema.HardwareFanSensor {
|
||||||
|
if sensors == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
seen := map[string]bool{}
|
||||||
|
var out []schema.HardwareFanSensor
|
||||||
|
for _, f := range sensors.Fans {
|
||||||
|
name := strings.TrimSpace(f.Name)
|
||||||
|
if name == "" || seen[name] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[name] = true
|
||||||
|
out = append(out, f)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// topoRowHeading renders the small uppercase section label shared by the
|
||||||
|
// flex rows below the SVG diagram (Firmware / Power Supplies / Cooling / ...).
|
||||||
|
func topoRowHeading(title string) string {
|
||||||
|
return fmt.Sprintf(`<div style="font-size:11px;color:var(--muted);text-transform:uppercase;letter-spacing:.05em;margin:16px 0 6px">%s</div>`,
|
||||||
|
html.EscapeString(title))
|
||||||
|
}
|
||||||
|
|
||||||
// renderTopoFlexRow renders a labeled, wrapping row of component cards.
|
// renderTopoFlexRow renders a labeled, wrapping row of component cards.
|
||||||
// Returns "" if items is empty (e.g. no PSU data in this audit).
|
// Returns "" if items is empty (e.g. no PSU data in this audit).
|
||||||
func renderTopoFlexRow(title string, items []topoCardInfo) string {
|
func renderTopoFlexRow(title string, items []topoCardInfo) string {
|
||||||
@@ -624,8 +753,7 @@ func renderTopoFlexRow(title string, items []topoCardInfo) string {
|
|||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
var b strings.Builder
|
var b strings.Builder
|
||||||
fmt.Fprintf(&b, `<div style="font-size:11px;color:var(--muted);text-transform:uppercase;letter-spacing:.05em;margin:16px 0 6px">%s</div>`,
|
b.WriteString(topoRowHeading(title))
|
||||||
html.EscapeString(title))
|
|
||||||
b.WriteString(`<div style="display:flex;flex-wrap:wrap;gap:10px">`)
|
b.WriteString(`<div style="display:flex;flex-wrap:wrap;gap:10px">`)
|
||||||
for _, item := range items {
|
for _, item := range items {
|
||||||
onclick := ""
|
onclick := ""
|
||||||
@@ -1064,6 +1192,15 @@ func inventoryFallbackRecords(compType string, opts HandlerOptions) []app.Compon
|
|||||||
}
|
}
|
||||||
records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(p.Status)})
|
records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(p.Status)})
|
||||||
}
|
}
|
||||||
|
case "fan":
|
||||||
|
for i, f := range dedupeFansByName(hw.Sensors) {
|
||||||
|
name := strings.TrimSpace(f.Name)
|
||||||
|
key := fmt.Sprintf("fan:%d", i)
|
||||||
|
if name != "" {
|
||||||
|
key = "fan:" + name
|
||||||
|
}
|
||||||
|
records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(f.Status)})
|
||||||
|
}
|
||||||
case "gpu", "nic", "raid":
|
case "gpu", "nic", "raid":
|
||||||
for i, dev := range hw.PCIeDevices {
|
for i, dev := range hw.PCIeDevices {
|
||||||
if pcieDeviceKind(dev) != compType {
|
if pcieDeviceKind(dev) != compType {
|
||||||
|
|||||||
@@ -152,6 +152,71 @@ func TestTopoPageRendersArbitraryPSUAndFirmwareCountsAsFlexRows(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestTopoPageRendersCoolingFansAsFlexRow(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
path := filepath.Join(dir, "audit.json")
|
||||||
|
|
||||||
|
ok, warn := "OK", "Warning"
|
||||||
|
rpm := func(v int) *int { return &v }
|
||||||
|
|
||||||
|
ingest := schema.HardwareIngestRequest{
|
||||||
|
CollectedAt: "2026-03-15T00:00:00Z",
|
||||||
|
Hardware: schema.HardwareSnapshot{
|
||||||
|
Sensors: &schema.HardwareSensors{
|
||||||
|
Fans: []schema.HardwareFanSensor{
|
||||||
|
{Name: "FAN1", RPM: rpm(4200), Status: &ok},
|
||||||
|
{Name: "FAN2", RPM: rpm(15000), Status: &warn},
|
||||||
|
{Name: "FAN2", RPM: rpm(15000), Status: &warn}, // dup name, first wins
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
data, err := json.Marshal(ingest)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(path, data, 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
handler := NewHandler(HandlerOptions{AuditPath: path})
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/topo", nil))
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("status=%d", rec.Code)
|
||||||
|
}
|
||||||
|
body := rec.Body.String()
|
||||||
|
|
||||||
|
// One clickable square per fan (2 after dedup by name), each with a
|
||||||
|
// spinning glyph, under a COOLING heading.
|
||||||
|
if !strings.Contains(body, "Cooling") {
|
||||||
|
t.Fatalf("topo page missing Cooling heading: %s", body)
|
||||||
|
}
|
||||||
|
if n := strings.Count(body, `onclick="openComponentDetail('fan')"`) +
|
||||||
|
strings.Count(body, `onclick="openComponentDetail('fan')"`); n != 2 {
|
||||||
|
t.Fatalf("expected one clickable square per fan (2), got %d: %s", n, body)
|
||||||
|
}
|
||||||
|
if n := strings.Count(body, `class="topo-fan-spin"`); n != 2 {
|
||||||
|
t.Fatalf("expected 2 spinning fan glyphs, got %d: %s", n, body)
|
||||||
|
}
|
||||||
|
if !strings.Contains(body, "FAN1 · 4200 RPM") || !strings.Contains(body, "FAN2 · 15000 RPM") {
|
||||||
|
t.Fatalf("topo page missing per-fan RPM tooltips: %s", body)
|
||||||
|
}
|
||||||
|
if !strings.Contains(body, "2 fans") {
|
||||||
|
t.Fatalf("topo page missing fan count caption: %s", body)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Component-detail fallback endpoint must resolve the "fan" type.
|
||||||
|
rec2 := httptest.NewRecorder()
|
||||||
|
handler.ServeHTTP(rec2, httptest.NewRequest(http.MethodGet, "/api/components/fan", nil))
|
||||||
|
if rec2.Code != http.StatusOK {
|
||||||
|
t.Fatalf("/api/components/fan status=%d", rec2.Code)
|
||||||
|
}
|
||||||
|
if b := rec2.Body.String(); !strings.Contains(b, "FAN1") || !strings.Contains(b, "FAN2") {
|
||||||
|
t.Fatalf("fan component detail missing fan names: %s", b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestTopoPageRendersStorageDisksGroupedByType(t *testing.T) {
|
func TestTopoPageRendersStorageDisksGroupedByType(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
path := filepath.Join(dir, "audit.json")
|
path := filepath.Join(dir, "audit.json")
|
||||||
|
|||||||
@@ -114,6 +114,12 @@ func renderValidateMode(opts HandlerOptions, stressDefault bool) string {
|
|||||||
`Tests power supply transient response by pulsing all GPUs simultaneously between idle and full load. Synchronous pulses across all GPUs create worst-case PSU load spikes — running per-GPU would miss PSU-level failures.`,
|
`Tests power supply transient response by pulsing all GPUs simultaneously between idle and full load. Synchronous pulses across all GPUs create worst-case PSU load spikes — running per-GPU would miss PSU-level failures.`,
|
||||||
`<code>dcgmi diag pulse_test</code>`,
|
`<code>dcgmi diag pulse_test</code>`,
|
||||||
validateFmtDur(platform.SATEstimatedNvidiaPulseTestSec)+` (all GPUs simultaneously; measured on 8-GPU system).`,
|
validateFmtDur(platform.SATEstimatedNvidiaPulseTestSec)+` (all GPUs simultaneously; measured on 8-GPU system).`,
|
||||||
|
)) +
|
||||||
|
renderSATCard("fan", "Fan Ceiling Check", "runSAT('fan')", "", renderValidateCardBody(
|
||||||
|
"All system fans reported over IPMI / lm-sensors.",
|
||||||
|
`Drives CPU (+memory) and, when a GPU is present, GPU load to 100% at the same time and watches every fan until none has climbed for ~1 min. The peak RPM reached is recorded as each fan's ceiling and is what the Topology view sizes the fan tiles against. Success once every fan plateaus (or the time cap is hit). A fan reading 0 RPM or an IPMI status of cr/nr under full load fails. If the host cannot be loaded at all, or exposes no fan sensors, the task is cancelled as "not applicable" rather than failed — the platform does not support forcing fans, so this is the closest safe equivalent.`,
|
||||||
|
`<code>stressapptest</code> / <code>stress-ng</code> + <code>bee-gpu-burn</code> / <code>rvs gst</code>; <code>ipmitool sdr type Fan</code>`,
|
||||||
|
`~2–8 min depending on how fast the fan curve settles (hard cap 15 min).`,
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -272,6 +272,7 @@ func NewHandler(opts HandlerOptions) http.Handler {
|
|||||||
mux.HandleFunc("POST /api/sat/memory-stress/run", h.handleAPISATRun("memory-stress"))
|
mux.HandleFunc("POST /api/sat/memory-stress/run", h.handleAPISATRun("memory-stress"))
|
||||||
mux.HandleFunc("POST /api/sat/sat-stress/run", h.handleAPISATRun("sat-stress"))
|
mux.HandleFunc("POST /api/sat/sat-stress/run", h.handleAPISATRun("sat-stress"))
|
||||||
mux.HandleFunc("POST /api/sat/platform-stress/run", h.handleAPISATRun("platform-stress"))
|
mux.HandleFunc("POST /api/sat/platform-stress/run", h.handleAPISATRun("platform-stress"))
|
||||||
|
mux.HandleFunc("POST /api/sat/fan/run", h.handleAPISATRun("fan"))
|
||||||
mux.HandleFunc("POST /api/sat/run-all", h.handleAPISATRunAll)
|
mux.HandleFunc("POST /api/sat/run-all", h.handleAPISATRunAll)
|
||||||
mux.HandleFunc("GET /api/sat/stream", h.handleAPISATStream)
|
mux.HandleFunc("GET /api/sat/stream", h.handleAPISATStream)
|
||||||
mux.HandleFunc("POST /api/sat/abort", h.handleAPISATAbort)
|
mux.HandleFunc("POST /api/sat/abort", h.handleAPISATAbort)
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package webui
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
@@ -79,9 +80,11 @@ func finalizeTaskForResult(t *Task, errMsg string, cancelled bool) {
|
|||||||
now := time.Now()
|
now := time.Now()
|
||||||
t.DoneAt = &now
|
t.DoneAt = &now
|
||||||
switch {
|
switch {
|
||||||
case cancelled:
|
case cancelled || t.Status == TaskCancelled:
|
||||||
t.Status = TaskCancelled
|
t.Status = TaskCancelled
|
||||||
t.ErrMsg = "aborted"
|
if strings.TrimSpace(t.ErrMsg) == "" {
|
||||||
|
t.ErrMsg = "aborted"
|
||||||
|
}
|
||||||
case strings.TrimSpace(errMsg) != "":
|
case strings.TrimSpace(errMsg) != "":
|
||||||
t.Status = TaskFailed
|
t.Status = TaskFailed
|
||||||
t.ErrMsg = errMsg
|
t.ErrMsg = errMsg
|
||||||
@@ -377,6 +380,12 @@ func executeTaskWithOptions(opts *HandlerOptions, t *Task, j *jobState, ctx cont
|
|||||||
runOpts := resolvePlatformStressPreset(t.params.BurnProfile)
|
runOpts := resolvePlatformStressPreset(t.params.BurnProfile)
|
||||||
runOpts.Components = t.params.PlatformComponents
|
runOpts.Components = t.params.PlatformComponents
|
||||||
archive, err = a.RunPlatformStress(ctx, "", runOpts, j.append)
|
archive, err = a.RunPlatformStress(ctx, "", runOpts, j.append)
|
||||||
|
case "fan":
|
||||||
|
if a == nil {
|
||||||
|
err = fmt.Errorf("app not configured")
|
||||||
|
break
|
||||||
|
}
|
||||||
|
archive, err = runFanCheckPackCtx(a, ctx, "", platform.FanCheckOptions{GPUIndices: t.params.GPUIndices}, j.append)
|
||||||
case "audit":
|
case "audit":
|
||||||
if a == nil {
|
if a == nil {
|
||||||
err = fmt.Errorf("app not configured")
|
err = fmt.Errorf("app not configured")
|
||||||
@@ -476,10 +485,19 @@ func executeTaskWithOptions(opts *HandlerOptions, t *Task, j *jobState, ctx cont
|
|||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if ctx.Err() != nil {
|
switch {
|
||||||
|
case ctx.Err() != nil:
|
||||||
j.append("Aborted.")
|
j.append("Aborted.")
|
||||||
j.finish("aborted")
|
j.finish("aborted")
|
||||||
} else {
|
case errors.Is(err, platform.ErrTestNotApplicable):
|
||||||
|
// The host offered no way to run this test — not a hardware
|
||||||
|
// fault. Land the task as cancelled ("not applicable") with a
|
||||||
|
// detailed log, so an engineer never sees a false failure.
|
||||||
|
j.append("NOT APPLICABLE: " + err.Error())
|
||||||
|
t.Status = TaskCancelled
|
||||||
|
t.ErrMsg = "not applicable — " + err.Error()
|
||||||
|
j.finish("")
|
||||||
|
default:
|
||||||
j.append("ERROR: " + err.Error())
|
j.append("ERROR: " + err.Error())
|
||||||
j.finish(err.Error())
|
j.finish(err.Error())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -53,6 +53,7 @@ var taskNames = map[string]string{
|
|||||||
"memory-stress": "Memory Burn-in",
|
"memory-stress": "Memory Burn-in",
|
||||||
"sat-stress": "SAT Stress (stressapptest)",
|
"sat-stress": "SAT Stress (stressapptest)",
|
||||||
"platform-stress": "Platform Thermal Cycling",
|
"platform-stress": "Platform Thermal Cycling",
|
||||||
|
"fan": "Fan Ceiling Check (CPU+GPU load)",
|
||||||
"audit": "Audit",
|
"audit": "Audit",
|
||||||
"support-bundle": "Support Bundle",
|
"support-bundle": "Support Bundle",
|
||||||
"install": "Install to Disk",
|
"install": "Install to Disk",
|
||||||
@@ -196,7 +197,7 @@ func taskMayLeaveOrphanWorkers(target string) bool {
|
|||||||
switch strings.TrimSpace(strings.ToLower(target)) {
|
switch strings.TrimSpace(strings.ToLower(target)) {
|
||||||
case "nvidia", "nvidia-targeted-stress", "nvidia-targeted-power", "nvidia-pulse",
|
case "nvidia", "nvidia-targeted-stress", "nvidia-targeted-power", "nvidia-pulse",
|
||||||
"nvidia-bandwidth", "nvidia-stress", "nvidia-compute", "nvidia-bench-perf",
|
"nvidia-bandwidth", "nvidia-stress", "nvidia-compute", "nvidia-bench-perf",
|
||||||
"memory", "memory-stress", "cpu", "sat-stress", "platform-stress":
|
"memory", "memory-stress", "cpu", "sat-stress", "platform-stress", "fan":
|
||||||
return true
|
return true
|
||||||
default:
|
default:
|
||||||
return false
|
return false
|
||||||
@@ -343,6 +344,9 @@ var (
|
|||||||
runSATStressPackCtx = func(a *app.App, ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
|
runSATStressPackCtx = func(a *app.App, ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
|
||||||
return a.RunSATStressPackCtx(ctx, baseDir, durationSec, logFunc)
|
return a.RunSATStressPackCtx(ctx, baseDir, durationSec, logFunc)
|
||||||
}
|
}
|
||||||
|
runFanCheckPackCtx = func(a *app.App, ctx context.Context, baseDir string, opts platform.FanCheckOptions, logFunc func(string)) (string, error) {
|
||||||
|
return a.RunFanCheckCtx(ctx, baseDir, opts, logFunc)
|
||||||
|
}
|
||||||
buildSupportBundle = app.BuildSupportBundle
|
buildSupportBundle = app.BuildSupportBundle
|
||||||
installCommand = func(ctx context.Context, device string, logPath string) *exec.Cmd {
|
installCommand = func(ctx context.Context, device string, logPath string) *exec.Cmd {
|
||||||
return exec.CommandContext(ctx, "bee-install", device, logPath)
|
return exec.CommandContext(ctx, "bee-install", device, logPath)
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package webui
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"os"
|
"os"
|
||||||
@@ -700,6 +701,34 @@ func TestRunTaskHonorsCancel(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunTaskFanNotApplicableIsCancelledNotFailed(t *testing.T) {
|
||||||
|
q := &taskQueue{opts: &HandlerOptions{App: &app.App{}}}
|
||||||
|
tk := &Task{
|
||||||
|
ID: "fan-1", Name: "Fan Ceiling Check", Target: "fan",
|
||||||
|
Status: TaskRunning, CreatedAt: time.Now(),
|
||||||
|
}
|
||||||
|
j := &jobState{}
|
||||||
|
tk.job = j
|
||||||
|
|
||||||
|
orig := runFanCheckPackCtx
|
||||||
|
runFanCheckPackCtx = func(_ *app.App, _ context.Context, _ string, _ platform.FanCheckOptions, _ func(string)) (string, error) {
|
||||||
|
return "", fmt.Errorf("no fan sensors readable: %w", platform.ErrTestNotApplicable)
|
||||||
|
}
|
||||||
|
defer func() { runFanCheckPackCtx = orig }()
|
||||||
|
|
||||||
|
q.runTask(tk, j, context.Background())
|
||||||
|
|
||||||
|
if tk.Status != TaskCancelled {
|
||||||
|
t.Fatalf("status=%q want %q", tk.Status, TaskCancelled)
|
||||||
|
}
|
||||||
|
if j.err != "" {
|
||||||
|
t.Fatalf("job err should be empty for a not-applicable task, got %q", j.err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(tk.ErrMsg, "not applicable") {
|
||||||
|
t.Fatalf("ErrMsg should explain not-applicable, got %q", tk.ErrMsg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunTaskUsesBurnProfileDurationForCPU(t *testing.T) {
|
func TestRunTaskUsesBurnProfileDurationForCPU(t *testing.T) {
|
||||||
var gotDuration int
|
var gotDuration int
|
||||||
q := &taskQueue{
|
q := &taskQueue{
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ Generic engineering rules live in `bible/rules/patterns/`.
|
|||||||
|---|---|
|
|---|---|
|
||||||
| `architecture/system-overview.md` | What bee does, scope, tech stack |
|
| `architecture/system-overview.md` | What bee does, scope, tech stack |
|
||||||
| `architecture/runtime-flows.md` | Boot sequence, audit flow, service order |
|
| `architecture/runtime-flows.md` | Boot sequence, audit flow, service order |
|
||||||
|
| `architecture/squashfs-layers.md` | Semantic SquashFS layer model, ownership, order, verification |
|
||||||
| `docs/customer-gpu-test-methodology.md` | Customer-facing GPU PCIe Validate / Validate -> Stress test list |
|
| `docs/customer-gpu-test-methodology.md` | Customer-facing GPU PCIe Validate / Validate -> Stress test list |
|
||||||
| `docs/hardware-ingest-contract.md` | Current Reanimator hardware ingest JSON contract |
|
| `docs/hardware-ingest-contract.md` | Current Reanimator hardware ingest JSON contract |
|
||||||
| `docs/validate-vs-burn.md` | Validate and Validate -> Stress hardware test policy |
|
| `docs/validate-vs-burn.md` | Validate and Validate -> Stress hardware test policy |
|
||||||
@@ -74,3 +75,8 @@ Generic engineering rules live in `bible/rules/patterns/`.
|
|||||||
- `all_reduce_perf`
|
- `all_reduce_perf`
|
||||||
- GPU bandwidth check
|
- GPU bandwidth check
|
||||||
- `dcgmi diag -r nvbandwidth` (per CPU socket, then all selected GPUs, on multi-socket systems -- see `decisions/2026-07-27-nvbandwidth-per-socket-split.md`)
|
- `dcgmi diag -r nvbandwidth` (per CPU socket, then all selected GPUs, on multi-socket systems -- see `decisions/2026-07-27-nvbandwidth-per-socket-split.md`)
|
||||||
|
- Fan ceiling check (Load tier / `3. Load` only)
|
||||||
|
- `stressapptest` (or `stress-ng`) + `bee-gpu-burn` / `rvs gst` at 100%, run simultaneously
|
||||||
|
- `ipmitool sdr type Fan` sampled 1 Hz until every fan plateaus (~1 min flat) or the 15 min cap
|
||||||
|
- records each fan's observed peak RPM to `/var/log/bee-sat/fan-observation.json` (used by the Topology fan tiles); FAIL only on a fan at 0 RPM / IPMI cr-nr under load; **cancelled ("not applicable")**, never failed, if the host cannot be loaded or has no fan sensors
|
||||||
|
- see `decisions/2026-09-04-fan-ceiling-check.md`
|
||||||
|
|||||||
@@ -53,6 +53,7 @@ All SAT run endpoints enqueue an async task. Response: `{"task_id": "..."}`.
|
|||||||
| POST | `/api/sat/memory-stress/run` | Memory stress |
|
| POST | `/api/sat/memory-stress/run` | Memory stress |
|
||||||
| POST | `/api/sat/sat-stress/run` | Combined storage+memory stress |
|
| POST | `/api/sat/sat-stress/run` | Combined storage+memory stress |
|
||||||
| POST | `/api/sat/platform-stress/run` | Fan + thermal stress |
|
| POST | `/api/sat/platform-stress/run` | Fan + thermal stress |
|
||||||
|
| POST | `/api/sat/fan/run` | Fan ceiling check (CPU+GPU load until fans plateau; Load tier). Not-applicable → task cancelled, not failed. |
|
||||||
| POST | `/api/sat/run-all` | Plan + enqueue the whole validate/check set server-side. Body: `{stress_mode, amd_targets[], nvidia_gpu_indices[]}` (operator intent only). Response: `{task_ids[], task_count, notes[]}`. Hardware presence/readiness and which tasks to run are decided by `handler.planSATRunAll`, not the page. |
|
| POST | `/api/sat/run-all` | Plan + enqueue the whole validate/check set server-side. Body: `{stress_mode, amd_targets[], nvidia_gpu_indices[]}` (operator intent only). Response: `{task_ids[], task_count, notes[]}`. Hardware presence/readiness and which tasks to run are decided by `handler.planSATRunAll`, not the page. |
|
||||||
| GET | `/api/sat/stream` | SSE: live SAT log stream |
|
| GET | `/api/sat/stream` | SSE: live SAT log stream |
|
||||||
| POST | `/api/sat/abort` | Abort the running SAT task |
|
| POST | `/api/sat/abort` | Abort the running SAT task |
|
||||||
|
|||||||
@@ -0,0 +1,68 @@
|
|||||||
|
# Decision: Fan check discovers the RPM ceiling by CPU+GPU load, not by forcing fans over the BMC
|
||||||
|
|
||||||
|
**Date:** 2026-09-04
|
||||||
|
**Status:** active
|
||||||
|
|
||||||
|
## Context
|
||||||
|
|
||||||
|
We want a SAT test that pushes every system fan to its top speed, records that
|
||||||
|
speed as the fan's ceiling, and flags a fan that will not spin. The recorded
|
||||||
|
ceiling is what the Topology view sizes each fan tile against
|
||||||
|
(`platform.ResolveFanMaxRPM`).
|
||||||
|
|
||||||
|
The obvious approach — force the fans to 100% PWM over IPMI/Redfish and read the
|
||||||
|
resulting RPM — does not work on our current platform:
|
||||||
|
|
||||||
|
- Stand: MSI G4201 / MS-S3831, AMI MegaRAC BMC (fw 1.08).
|
||||||
|
- Every documented host-side fan-control OEM command returns
|
||||||
|
`rsp=0xc1 Invalid command`: Supermicro `0x30 0x45`, ASRock/AMI reference
|
||||||
|
`0x3a 0x01 / 0xd0 0x12 / 0xd0 0x0f / 0xd6 / 0xd7 / 0xda`, `0x30 0x30..0x32`.
|
||||||
|
BIOS `KCS Access Control Policy = Allow All`, so this is not KCS filtering —
|
||||||
|
MSI simply does not implement them.
|
||||||
|
- The MSI G4201 Redfish API guide documents Thermal as **GET only**; no
|
||||||
|
fan-mode / fan-PWM PATCH endpoint exists.
|
||||||
|
- The only fan knob in BIOS is `Fan PWM Offset` (0–100, additive to the auto
|
||||||
|
curve, reboot-gated) — not a runtime 100% force.
|
||||||
|
|
||||||
|
## Decision
|
||||||
|
|
||||||
|
The `fan` SAT test (`platform.RunFanCheck`, formerly the unwired
|
||||||
|
`RunFanStressTest`) drives load, not the BMC:
|
||||||
|
|
||||||
|
1. Start `stressapptest` (CPU + memory) and, when a GPU is present, a GPU burn
|
||||||
|
(`bee-gpu-burn` / `rvs gst`) **simultaneously**, each in its own goroutine.
|
||||||
|
A missing GPU is not an error — CPU/memory load alone exercises the cooling
|
||||||
|
loop. Load sources reach full power at different times, so the plateau clock
|
||||||
|
only starts once every launched source reports its process running (plus a
|
||||||
|
fixed GPU ramp grace).
|
||||||
|
2. Sample every fan once a second. Per fan, track the peak RPM and the last time
|
||||||
|
it climbed by more than `PlateauDeltaRPM` (default 50).
|
||||||
|
3. When no fan has climbed for `PlateauHoldSec` (default 60 s) and at least one
|
||||||
|
fan rose meaningfully above baseline, declare the ceiling found and stop —
|
||||||
|
**success**. `MaxLoadSec` (default 900 s) is a hard cap; hitting it is also
|
||||||
|
success (the highest RPM seen is still recorded).
|
||||||
|
4. Persist each peak through the existing `updateFanObservation` path
|
||||||
|
(`/var/log/bee-sat/fan-observation.json`), which is what `ResolveFanMaxRPM`
|
||||||
|
reads.
|
||||||
|
|
||||||
|
## Verdict mapping
|
||||||
|
|
||||||
|
- A fan reading **0 RPM**, or IPMI status **cr/nr**, while under full load →
|
||||||
|
`FAILED` (dead / stuck rotor).
|
||||||
|
- No load source available (no `stressapptest`/`stress-ng` and no GPU burn
|
||||||
|
tool), or no fan sensors at all → `platform.ErrTestNotApplicable`, which the
|
||||||
|
task layer lands as **cancelled ("not applicable")**, never failed, with a
|
||||||
|
detailed log. An engineer must not see a red for "this platform can't run the
|
||||||
|
test".
|
||||||
|
|
||||||
|
## Tier
|
||||||
|
|
||||||
|
Load only (`/load`, "3. Load"). It is a sustained full-load test, so it does
|
||||||
|
not belong on the read-only Check page. Added to the stress-mode `Run All`.
|
||||||
|
|
||||||
|
## Consequences
|
||||||
|
|
||||||
|
- The ceiling is *observed*, not a spec value — only as high as the load drove
|
||||||
|
the fans. Good enough for tile sizing and stuck-fan detection.
|
||||||
|
- If a future platform does expose a safe host-side fan force, revisit: a real
|
||||||
|
100% force is a stronger test than load-driven ramp.
|
||||||
Reference in New Issue
Block a user