Repurpose the previously-unwired RunFanStressTest into RunFanCheck, a
Load-tier SAT test that drives stressapptest (CPU+memory) and, when a GPU
is present, a GPU burn to 100% simultaneously, then watches every fan
until none has climbed for ~60s. The observed peak RPM per fan is the
"ceiling"; it is persisted through the existing fan-observation store.
MSI G4201 / AMI MegaRAC exposes no host-side fan force (every OEM IPMI
command returns 0xc1; Redfish Thermal is GET-only), so load-driven ramp
is the closest safe equivalent. See
bible-local/decisions/2026-09-04-fan-ceiling-check.md.
- platform.ResolveFanMaxRPM: per-fan max with fallback (persisted peak ->
peer peak -> current RPM), resolved in platform, not the view.
- platform.ErrTestNotApplicable: no load source or no fan sensors ->
task lands as cancelled ("not applicable"), never failed, so an
engineer never sees a false red. executeTaskWithOptions maps the
sentinel; finalizeTaskForResult honours a pre-set TaskCancelled.
- Verdict FAIL only for a fan at 0 RPM / IPMI cr-nr under load.
- /topo: one small spinning square per fan, sized by RPM / resolved max,
clickable through to a new "fan" component-detail type; per-fan status
recorded to the component-status DB from the fan SAT summary.
- Wiring: /api/sat/fan/run route, "fan" task target, Load-page card,
stress-mode Run All.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019VHG21rgTUiR1G3qFHTVmN
328 lines
13 KiB
Go
328 lines
13 KiB
Go
package app
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
|
|
"bee/audit/internal/platform"
|
|
)
|
|
|
|
func (a *App) RunNvidiaAcceptancePack(baseDir string, logFunc func(string)) (string, error) {
|
|
if strings.TrimSpace(baseDir) == "" {
|
|
baseDir = DefaultSATBaseDir
|
|
}
|
|
return a.sat.RunNvidiaAcceptancePack(baseDir, logFunc)
|
|
}
|
|
|
|
// RunScenario runs a user-authored ScenarioSpec (see
|
|
// platform.ParseScenarioJSON) — the webui Scenario page and `bee run` CLI
|
|
// command both go through this so a scenario run gets the same SAT base
|
|
// directory default as every other pack.
|
|
func (a *App) RunScenario(ctx context.Context, baseDir string, spec platform.ScenarioSpec, logFunc func(string)) (string, error) {
|
|
if strings.TrimSpace(baseDir) == "" {
|
|
baseDir = DefaultSATBaseDir
|
|
}
|
|
return a.sat.RunScenario(ctx, baseDir, spec, logFunc)
|
|
}
|
|
|
|
func (a *App) RunNvidiaAcceptancePackResult(baseDir string) (ActionResult, error) {
|
|
path, err := a.RunNvidiaAcceptancePack(baseDir, nil)
|
|
body := "Archive written."
|
|
if path != "" {
|
|
body = "Archive written to " + path
|
|
}
|
|
return ActionResult{Title: "NVIDIA SAT", Body: body}, err
|
|
}
|
|
|
|
func (a *App) ListNvidiaGPUs() ([]platform.NvidiaGPU, error) {
|
|
return a.sat.ListNvidiaGPUs()
|
|
}
|
|
|
|
func (a *App) ListNvidiaGPUStatuses() ([]platform.NvidiaGPUStatus, error) {
|
|
return a.sat.ListNvidiaGPUStatuses()
|
|
}
|
|
|
|
func (a *App) ResetNvidiaGPU(index int) (ActionResult, error) {
|
|
out, err := a.sat.ResetNvidiaGPU(index)
|
|
return ActionResult{Title: fmt.Sprintf("Reset NVIDIA GPU %d", index), Body: strings.TrimSpace(out)}, err
|
|
}
|
|
|
|
func (a *App) ListNvidiaGPUSettings() ([]platform.NvidiaGPUSetting, error) {
|
|
return a.sat.ListNvidiaGPUSettings()
|
|
}
|
|
|
|
func (a *App) SetNvidiaGPUECC(index int, enabled bool) (ActionResult, error) {
|
|
out, err := a.sat.SetNvidiaGPUECC(index, enabled)
|
|
title := fmt.Sprintf("Enable ECC on GPU %d", index)
|
|
if !enabled {
|
|
title = fmt.Sprintf("Disable ECC on GPU %d", index)
|
|
}
|
|
return ActionResult{Title: title, Body: strings.TrimSpace(out)}, err
|
|
}
|
|
|
|
func (a *App) SetNvidiaGPUMIG(index int, enabled bool) (ActionResult, error) {
|
|
out, err := a.sat.SetNvidiaGPUMIG(index, enabled)
|
|
title := fmt.Sprintf("Enable MIG on GPU %d", index)
|
|
if !enabled {
|
|
title = fmt.Sprintf("Disable MIG on GPU %d", index)
|
|
}
|
|
return ActionResult{Title: title, Body: strings.TrimSpace(out)}, err
|
|
}
|
|
|
|
func (a *App) SetNvidiaGPUCCMode(index int, enabled bool) (ActionResult, error) {
|
|
out, err := a.sat.SetNvidiaGPUCCMode(index, enabled)
|
|
title := fmt.Sprintf("Enable Confidential Computing on GPU %d", index)
|
|
if !enabled {
|
|
title = fmt.Sprintf("Disable Confidential Computing on GPU %d", index)
|
|
}
|
|
return ActionResult{Title: title, Body: strings.TrimSpace(out)}, err
|
|
}
|
|
|
|
func (a *App) SetNvidiaGPUPowerLimit(index int, watts float64) (ActionResult, error) {
|
|
out, err := a.sat.SetNvidiaGPUPowerLimit(index, watts)
|
|
return ActionResult{Title: fmt.Sprintf("Set power limit on GPU %d", index), Body: strings.TrimSpace(out)}, err
|
|
}
|
|
|
|
func (a *App) ResetNvidiaGPUDefaults() (ActionResult, error) {
|
|
out, err := a.sat.ResetNvidiaGPUDefaults()
|
|
return ActionResult{Title: "Reset all GPUs to factory defaults", Body: strings.TrimSpace(out)}, err
|
|
}
|
|
|
|
func (a *App) RunNvidiaAcceptancePackWithOptions(ctx context.Context, baseDir string, diagLevel int, gpuIndices []int, logFunc func(string)) (ActionResult, error) {
|
|
if strings.TrimSpace(baseDir) == "" {
|
|
baseDir = DefaultSATBaseDir
|
|
}
|
|
path, err := a.sat.RunNvidiaAcceptancePackWithOptions(ctx, baseDir, diagLevel, gpuIndices, logFunc)
|
|
body := "Archive written."
|
|
if path != "" {
|
|
body = "Archive written to " + path
|
|
}
|
|
return ActionResult{Title: "NVIDIA DCGM", Body: body}, err
|
|
}
|
|
|
|
func (a *App) RunNvidiaTargetedStressValidatePack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) {
|
|
if strings.TrimSpace(baseDir) == "" {
|
|
baseDir = DefaultSATBaseDir
|
|
}
|
|
return a.sat.RunNvidiaTargetedStressValidatePack(ctx, baseDir, durationSec, gpuIndices, logFunc)
|
|
}
|
|
|
|
func (a *App) RunNvidiaBenchmark(baseDir string, opts platform.NvidiaBenchmarkOptions, logFunc func(string)) (string, error) {
|
|
return a.RunNvidiaBenchmarkCtx(context.Background(), baseDir, opts, logFunc)
|
|
}
|
|
|
|
func (a *App) RunNvidiaBenchmarkCtx(ctx context.Context, baseDir string, opts platform.NvidiaBenchmarkOptions, logFunc func(string)) (string, error) {
|
|
return a.runNvidiaBenchmarkKind(ctx, baseDir, DefaultBeeBenchPerfDir, opts, "performance", logFunc, a.sat.RunNvidiaBenchmark)
|
|
}
|
|
|
|
func (a *App) RunNvidiaPowerBenchCtx(ctx context.Context, baseDir string, opts platform.NvidiaBenchmarkOptions, logFunc func(string)) (string, error) {
|
|
return a.runNvidiaBenchmarkKind(ctx, baseDir, DefaultBeeBenchPowerDir, opts, "power-fit", logFunc, a.sat.RunNvidiaPowerBench)
|
|
}
|
|
|
|
func (a *App) runNvidiaBenchmarkKind(ctx context.Context, baseDir, defaultBaseDir string, opts platform.NvidiaBenchmarkOptions, benchmarkKind string, logFunc func(string), run func(context.Context, string, platform.NvidiaBenchmarkOptions, func(string)) (string, error)) (string, error) {
|
|
if strings.TrimSpace(baseDir) == "" {
|
|
baseDir = defaultBaseDir
|
|
}
|
|
resolved, err := a.ensureBenchmarkPowerAutotune(ctx, baseDir, opts, benchmarkKind, logFunc)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
opts.ServerPowerSource = resolved.SelectedSource
|
|
return run(ctx, baseDir, opts, logFunc)
|
|
}
|
|
|
|
func (a *App) RunNvidiaPowerSourceAutotuneCtx(ctx context.Context, baseDir string, opts platform.NvidiaBenchmarkOptions, benchmarkKind string, logFunc func(string)) (string, error) {
|
|
if strings.TrimSpace(baseDir) == "" {
|
|
baseDir = DefaultBeeBenchAutotuneDir
|
|
}
|
|
return a.sat.RunNvidiaPowerSourceAutotune(ctx, baseDir, opts, benchmarkKind, logFunc)
|
|
}
|
|
|
|
func (a *App) LoadBenchmarkPowerAutotune() (*platform.BenchmarkPowerAutotuneConfig, error) {
|
|
return platform.LoadBenchmarkPowerAutotuneConfig(DefaultBeeBenchPowerSourceConfigPath)
|
|
}
|
|
|
|
func (a *App) ensureBenchmarkPowerAutotune(ctx context.Context, baseDir string, opts platform.NvidiaBenchmarkOptions, benchmarkKind string, logFunc func(string)) (platform.BenchmarkPowerAutotuneConfig, error) {
|
|
cfgPath := platform.BenchmarkPowerSourceConfigPath(baseDir)
|
|
if cfg, err := platform.LoadBenchmarkPowerAutotuneConfig(cfgPath); err == nil {
|
|
if logFunc != nil {
|
|
logFunc(fmt.Sprintf("benchmark autotune: using saved server power source %s", cfg.SelectedSource))
|
|
}
|
|
return *cfg, nil
|
|
}
|
|
if logFunc != nil {
|
|
logFunc("benchmark autotune: no saved power source config, running autotune first")
|
|
}
|
|
autotuneDir := filepath.Join(filepath.Dir(baseDir), "autotune")
|
|
if _, err := a.RunNvidiaPowerSourceAutotuneCtx(ctx, autotuneDir, opts, benchmarkKind, logFunc); err != nil {
|
|
return platform.BenchmarkPowerAutotuneConfig{}, err
|
|
}
|
|
cfg, err := platform.LoadBenchmarkPowerAutotuneConfig(cfgPath)
|
|
if err != nil {
|
|
return platform.BenchmarkPowerAutotuneConfig{}, err
|
|
}
|
|
return *cfg, nil
|
|
}
|
|
|
|
func (a *App) RunNvidiaOfficialComputePack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, staggerSec int, logFunc func(string)) (string, error) {
|
|
if strings.TrimSpace(baseDir) == "" {
|
|
baseDir = DefaultSATBaseDir
|
|
}
|
|
return a.sat.RunNvidiaOfficialComputePack(ctx, baseDir, durationSec, gpuIndices, staggerSec, logFunc)
|
|
}
|
|
|
|
func (a *App) RunNvidiaTargetedPowerPack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) {
|
|
if strings.TrimSpace(baseDir) == "" {
|
|
baseDir = DefaultSATBaseDir
|
|
}
|
|
return a.sat.RunNvidiaTargetedPowerPack(ctx, baseDir, durationSec, gpuIndices, logFunc)
|
|
}
|
|
|
|
func (a *App) RunNvidiaPulseTestPack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) {
|
|
if strings.TrimSpace(baseDir) == "" {
|
|
baseDir = DefaultSATBaseDir
|
|
}
|
|
return a.sat.RunNvidiaPulseTestPack(ctx, baseDir, durationSec, gpuIndices, logFunc)
|
|
}
|
|
|
|
func (a *App) RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpuIndices []int, fullMatrix bool, logFunc func(string)) (string, error) {
|
|
if strings.TrimSpace(baseDir) == "" {
|
|
baseDir = DefaultSATBaseDir
|
|
}
|
|
return a.sat.RunNvidiaBandwidthPack(ctx, baseDir, gpuIndices, fullMatrix, logFunc)
|
|
}
|
|
|
|
func (a *App) RunNvidiaStressPackCtx(ctx context.Context, baseDir string, opts platform.NvidiaStressOptions, logFunc func(string)) (string, error) {
|
|
if strings.TrimSpace(baseDir) == "" {
|
|
baseDir = DefaultSATBaseDir
|
|
}
|
|
return a.sat.RunNvidiaStressPack(ctx, baseDir, opts, logFunc)
|
|
}
|
|
|
|
func (a *App) RunMemoryAcceptancePack(baseDir string, logFunc func(string)) (string, error) {
|
|
return a.RunMemoryAcceptancePackCtx(context.Background(), baseDir, 256, 1, logFunc)
|
|
}
|
|
|
|
func (a *App) RunMemoryAcceptancePackCtx(ctx context.Context, baseDir string, sizeMB, passes int, logFunc func(string)) (string, error) {
|
|
if strings.TrimSpace(baseDir) == "" {
|
|
baseDir = DefaultSATBaseDir
|
|
}
|
|
return a.sat.RunMemoryAcceptancePack(ctx, baseDir, sizeMB, passes, logFunc)
|
|
}
|
|
|
|
func (a *App) RunMemoryAcceptancePackResult(baseDir string) (ActionResult, error) {
|
|
path, err := a.RunMemoryAcceptancePack(baseDir, nil)
|
|
return ActionResult{Title: "Memory SAT", Body: satResultBody(path)}, err
|
|
}
|
|
|
|
func (a *App) RunCPUAcceptancePackCtx(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
|
|
if strings.TrimSpace(baseDir) == "" {
|
|
baseDir = DefaultSATBaseDir
|
|
}
|
|
return a.sat.RunCPUAcceptancePack(ctx, baseDir, durationSec, logFunc)
|
|
}
|
|
|
|
func (a *App) RunStorageAcceptancePack(baseDir string, logFunc func(string)) (string, error) {
|
|
return a.RunStorageAcceptancePackCtx(context.Background(), baseDir, false, logFunc)
|
|
}
|
|
|
|
func (a *App) RunStorageAcceptancePackCtx(ctx context.Context, baseDir string, extended bool, logFunc func(string)) (string, error) {
|
|
if strings.TrimSpace(baseDir) == "" {
|
|
baseDir = DefaultSATBaseDir
|
|
}
|
|
return a.sat.RunStorageAcceptancePack(ctx, baseDir, extended, logFunc)
|
|
}
|
|
|
|
func (a *App) RunStorageAcceptancePackResult(baseDir string) (ActionResult, error) {
|
|
path, err := a.RunStorageAcceptancePack(baseDir, nil)
|
|
return ActionResult{Title: "Storage SAT", Body: satResultBody(path)}, err
|
|
}
|
|
|
|
func (a *App) RunTPMValidationPackCtx(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
|
|
if strings.TrimSpace(baseDir) == "" {
|
|
baseDir = DefaultSATBaseDir
|
|
}
|
|
return a.sat.RunTPMValidationPack(ctx, baseDir, logFunc)
|
|
}
|
|
|
|
func (a *App) RunNvidiaConfigCheckPackCtx(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
|
|
if strings.TrimSpace(baseDir) == "" {
|
|
baseDir = DefaultSATBaseDir
|
|
}
|
|
return a.sat.RunNvidiaConfigCheckPack(ctx, baseDir, logFunc)
|
|
}
|
|
|
|
func (a *App) RunAMDAcceptancePackCtx(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
|
|
if strings.TrimSpace(baseDir) == "" {
|
|
baseDir = DefaultSATBaseDir
|
|
}
|
|
return a.sat.RunAMDAcceptancePack(ctx, baseDir, logFunc)
|
|
}
|
|
|
|
func (a *App) RunAMDMemIntegrityPackCtx(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
|
|
if strings.TrimSpace(baseDir) == "" {
|
|
baseDir = DefaultSATBaseDir
|
|
}
|
|
return a.sat.RunAMDMemIntegrityPack(ctx, baseDir, logFunc)
|
|
}
|
|
|
|
func (a *App) RunAMDMemBandwidthPackCtx(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
|
|
if strings.TrimSpace(baseDir) == "" {
|
|
baseDir = DefaultSATBaseDir
|
|
}
|
|
return a.sat.RunAMDMemBandwidthPack(ctx, baseDir, logFunc)
|
|
}
|
|
|
|
func (a *App) RunMemoryStressPackCtx(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
|
|
return a.sat.RunMemoryStressPack(ctx, baseDir, durationSec, logFunc)
|
|
}
|
|
|
|
func (a *App) RunSATStressPackCtx(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
|
|
return a.sat.RunSATStressPack(ctx, baseDir, durationSec, logFunc)
|
|
}
|
|
|
|
func (a *App) RunAMDStressPackCtx(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
|
|
if strings.TrimSpace(baseDir) == "" {
|
|
baseDir = DefaultSATBaseDir
|
|
}
|
|
return a.sat.RunAMDStressPack(ctx, baseDir, durationSec, logFunc)
|
|
}
|
|
|
|
func (a *App) RunPlatformStress(ctx context.Context, baseDir string, opts platform.PlatformStressOptions, logFunc func(string)) (string, error) {
|
|
if strings.TrimSpace(baseDir) == "" {
|
|
baseDir = DefaultSATBaseDir
|
|
}
|
|
return a.sat.RunPlatformStress(ctx, baseDir, opts, logFunc)
|
|
}
|
|
|
|
func (a *App) RunFanCheckCtx(ctx context.Context, baseDir string, opts platform.FanCheckOptions, logFunc func(string)) (string, error) {
|
|
if strings.TrimSpace(baseDir) == "" {
|
|
baseDir = DefaultSATBaseDir
|
|
}
|
|
return a.sat.RunFanCheck(ctx, baseDir, opts, logFunc)
|
|
}
|
|
|
|
func (a *App) RunNCCLTests(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) {
|
|
if strings.TrimSpace(baseDir) == "" {
|
|
baseDir = DefaultSATBaseDir
|
|
}
|
|
return a.sat.RunNCCLTests(ctx, baseDir, gpuIndices, logFunc)
|
|
}
|
|
|
|
// satResultBody reads summary.txt from the SAT run directory (archive path without .tar.gz)
|
|
// and returns a formatted human-readable result. Falls back to a plain message if unreadable.
|
|
func satResultBody(archivePath string) string {
|
|
if archivePath == "" {
|
|
return "No output produced."
|
|
}
|
|
runDir := strings.TrimSuffix(archivePath, ".tar.gz")
|
|
raw, err := os.ReadFile(filepath.Join(runDir, "summary.txt"))
|
|
if err != nil {
|
|
return "Archive written to " + archivePath
|
|
}
|
|
return formatSATDetail(strings.TrimSpace(string(raw)))
|
|
}
|