diff --git a/audit/internal/platform/nvidia_stress.go b/audit/internal/platform/nvidia_stress.go index 62bc0c5..6ef735e 100644 --- a/audit/internal/platform/nvidia_stress.go +++ b/audit/internal/platform/nvidia_stress.go @@ -95,9 +95,7 @@ func normalizeNvidiaStressOptions(opts *NvidiaStressOptions) { if opts.DurationSec <= 0 { opts.DurationSec = 300 } - if opts.SizeMB <= 0 { - opts.SizeMB = 64 - } + // SizeMB=0 means "auto" — bee-gpu-burn will query per-GPU memory at runtime. switch strings.TrimSpace(strings.ToLower(opts.Loader)) { case "", NvidiaStressLoaderBuiltin: opts.Loader = NvidiaStressLoaderBuiltin diff --git a/audit/internal/platform/platform_stress.go b/audit/internal/platform/platform_stress.go index 0c03088..d7e909f 100644 --- a/audit/internal/platform/platform_stress.go +++ b/audit/internal/platform/platform_stress.go @@ -26,7 +26,8 @@ type PlatformStressCycle struct { // PlatformStressOptions controls the thermal cycling test. type PlatformStressOptions struct { - Cycles []PlatformStressCycle + Cycles []PlatformStressCycle + Components []string // if empty: run all; values: "cpu", "gpu" } // platformStressRow is one second of telemetry. @@ -68,8 +69,11 @@ func (s *System) RunPlatformStress( return "", fmt.Errorf("mkdir run dir: %w", err) } + hasCPU := len(opts.Components) == 0 || containsComponent(opts.Components, "cpu") + hasGPU := len(opts.Components) == 0 || containsComponent(opts.Components, "gpu") + vendor := s.DetectGPUVendor() - logFunc(fmt.Sprintf("Platform Thermal Cycling — %d cycle(s), GPU vendor: %s", len(opts.Cycles), vendor)) + logFunc(fmt.Sprintf("Platform Thermal Cycling — %d cycle(s), GPU vendor: %s, cpu=%v gpu=%v", len(opts.Cycles), vendor, hasCPU, hasGPU)) var rows []platformStressRow start := time.Now() @@ -88,27 +92,31 @@ func (s *System) RunPlatformStress( var wg sync.WaitGroup // CPU stress - wg.Add(1) - go func() { - defer wg.Done() - cpuCmd, err := buildCPUStressCmd(loadCtx) - if err != nil { - logFunc("CPU stress: " + err.Error()) - return - } - _ = cpuCmd.Wait() // exits when loadCtx times out (SIGKILL) - }() + if hasCPU { + wg.Add(1) + go func() { + defer wg.Done() + cpuCmd, err := buildCPUStressCmd(loadCtx) + if err != nil { + logFunc("CPU stress: " + err.Error()) + return + } + _ = cpuCmd.Wait() // exits when loadCtx times out (SIGKILL) + }() + } // GPU stress - wg.Add(1) - go func() { - defer wg.Done() - gpuCmd := buildGPUStressCmd(loadCtx, vendor) - if gpuCmd == nil { - return - } - _ = gpuCmd.Wait() - }() + if hasGPU { + wg.Add(1) + go func() { + defer wg.Done() + gpuCmd := buildGPUStressCmd(loadCtx, vendor) + if gpuCmd == nil { + return + } + _ = gpuCmd.Wait() + }() + } // Monitoring goroutine for load phase loadRows := collectPhase(loadCtx, cycleNum, "load", start) @@ -439,7 +447,7 @@ func buildNvidiaGPUStressCmd(ctx context.Context) *exec.Cmd { if err != nil { return nil } - cmd := exec.CommandContext(ctx, path, "--seconds", "86400", "--size-mb", "64") + cmd := exec.CommandContext(ctx, path, "--seconds", "86400") cmd.Stdout = nil cmd.Stderr = nil _ = startLowPriorityCmd(cmd, 10) @@ -486,6 +494,15 @@ func platformStressMemoryMB() int { return mb } +func containsComponent(components []string, name string) bool { + for _, c := range components { + if c == name { + return true + } + } + return false +} + func packPlatformDir(dir, dest string) error { f, err := os.Create(dest) if err != nil { diff --git a/audit/internal/webui/api.go b/audit/internal/webui/api.go index b0000b0..6010443 100644 --- a/audit/internal/webui/api.go +++ b/audit/internal/webui/api.go @@ -181,13 +181,14 @@ func (h *handler) handleAPISATRun(target string) http.HandlerFunc { } var body struct { - Duration int `json:"duration"` - DiagLevel int `json:"diag_level"` - GPUIndices []int `json:"gpu_indices"` - ExcludeGPUIndices []int `json:"exclude_gpu_indices"` - Loader string `json:"loader"` - Profile string `json:"profile"` - DisplayName string `json:"display_name"` + Duration int `json:"duration"` + DiagLevel int `json:"diag_level"` + GPUIndices []int `json:"gpu_indices"` + ExcludeGPUIndices []int `json:"exclude_gpu_indices"` + Loader string `json:"loader"` + Profile string `json:"profile"` + DisplayName string `json:"display_name"` + PlatformComponents []string `json:"platform_components"` } if r.Body != nil { if err := json.NewDecoder(r.Body).Decode(&body); err != nil && !errors.Is(err, io.EOF) { @@ -204,13 +205,14 @@ func (h *handler) handleAPISATRun(target string) http.HandlerFunc { Status: TaskPending, CreatedAt: time.Now(), params: taskParams{ - Duration: body.Duration, - DiagLevel: body.DiagLevel, - GPUIndices: body.GPUIndices, - ExcludeGPUIndices: body.ExcludeGPUIndices, - Loader: body.Loader, - BurnProfile: body.Profile, - DisplayName: body.DisplayName, + Duration: body.Duration, + DiagLevel: body.DiagLevel, + GPUIndices: body.GPUIndices, + ExcludeGPUIndices: body.ExcludeGPUIndices, + Loader: body.Loader, + BurnProfile: body.Profile, + DisplayName: body.DisplayName, + PlatformComponents: body.PlatformComponents, }, } if strings.TrimSpace(body.DisplayName) != "" { @@ -512,6 +514,26 @@ func (h *handler) handleAPIGPUPresence(w http.ResponseWriter, r *http.Request) { }) } +// ── GPU tools ───────────────────────────────────────────────────────────────── + +func (h *handler) handleAPIGPUTools(w http.ResponseWriter, _ *http.Request) { + type toolEntry struct { + ID string `json:"id"` + Available bool `json:"available"` + Vendor string `json:"vendor"` // "nvidia" | "amd" + } + _, nvidiaErr := os.Stat("/dev/nvidia0") + _, amdErr := os.Stat("/dev/kfd") + nvidiaUp := nvidiaErr == nil + amdUp := amdErr == nil + writeJSON(w, []toolEntry{ + {ID: "bee-gpu-burn", Available: nvidiaUp, Vendor: "nvidia"}, + {ID: "john", Available: nvidiaUp, Vendor: "nvidia"}, + {ID: "nccl", Available: nvidiaUp, Vendor: "nvidia"}, + {ID: "rvs", Available: amdUp, Vendor: "amd"}, + }) +} + // ── System ──────────────────────────────────────────────────────────────────── func (h *handler) handleAPIRAMStatus(w http.ResponseWriter, r *http.Request) { diff --git a/audit/internal/webui/pages.go b/audit/internal/webui/pages.go index 4d0c220..32678d6 100644 --- a/audit/internal/webui/pages.go +++ b/audit/internal/webui/pages.go @@ -657,96 +657,210 @@ func renderSATCard(id, label, extra string) string { func renderBurn() string { return `
⚠ Warning: Stress tests on this page run hardware at maximum load. Repeated or prolonged use may reduce hardware lifespan (storage endurance, GPU wear). Use only when necessary.

Tasks continue in the background — view progress in Tasks.

-
Burn Profile
-
-

Applied to all tests on this page. NVIDIA SAT on the Validate page still uses DCGM. NVIDIA GPU Stress on this page uses the selected stress loader for the preset duration.

-
-
-
NVIDIA GPU Stress
-
-
-

bee-gpu-burn runs on all detected NVIDIA GPUs by default. NCCL all_reduce_perf is useful for multi-GPU / interconnect load. Use exclusions only when one or more cards must be skipped.

- -
-
CPU Stress
- -
-
AMD GPU Stress
-

Runs ROCm compute stress together with VRAM copy/load activity via RVS GST and records a separate rocm-bandwidth-test snapshot. Missing tools reported as UNSUPPORTED.

- -
-
Memory Stress
-

stress-ng --vm writes and verifies memory patterns across all of RAM. Env: BEE_VM_STRESS_SECONDS (default 300), BEE_VM_STRESS_SIZE_MB (default 80%).

- -
-
SAT Stress (stressapptest)
-

Google stressapptest saturates CPU, memory and cache buses simultaneously. Env: BEE_SAT_STRESS_SECONDS (default 300), BEE_SAT_STRESS_MB (default auto).

- -
-
Platform Thermal Cycling
-

Runs CPU + GPU stress simultaneously across multiple load/idle cycles with varying durations. Detects cooling systems that fail to recover under repeated load cycles. Smoke: 2 cycles ~5 min. Acceptance: 4 cycles ~25 min.

- -
+ +
+
Burn Profile
+
+
+ + +
+ +
+ +
+
GPU Stress
+
+

Tests run on all GPUs in the system. Availability determined by driver status.

+
+ + + + +
+ +
+
+ +
+
Compute Stress
+
+

Select which subsystems to stress. Each checked item runs as a separate task.

+ + + + +
+
+ +
+
Platform Thermal Cycling
+
+

Repeated load+idle cycles. Detects cooling recovery failures and GPU throttle. Smoke: 2×90s. Acceptance: 4×300s.

+

Load components:

+ + + + +
+
+ +
+ + + + -` } diff --git a/audit/internal/webui/server.go b/audit/internal/webui/server.go index d342153..f70be9e 100644 --- a/audit/internal/webui/server.go +++ b/audit/internal/webui/server.go @@ -255,8 +255,9 @@ func NewHandler(opts HandlerOptions) http.Handler { mux.HandleFunc("GET /api/display/resolutions", h.handleAPIDisplayResolutions) mux.HandleFunc("POST /api/display/set", h.handleAPIDisplaySet) - // GPU presence + // GPU presence / tools mux.HandleFunc("GET /api/gpu/presence", h.handleAPIGPUPresence) + mux.HandleFunc("GET /api/gpu/tools", h.handleAPIGPUTools) // System mux.HandleFunc("GET /api/system/ram-status", h.handleAPIRAMStatus) @@ -516,6 +517,7 @@ func (h *handler) handleMetricsChartSVG(w http.ResponseWriter, r *http.Request) case path == "server-power": title = "System Power" vPower, l := h.ringPower.snapshot() + vPower = normalizePowerSeries(vPower) labels = l datasets = [][]float64{vPower} names = []string{"Power W"} @@ -728,9 +730,11 @@ func chartDataFromSamples(path string, samples []platform.LiveMetricSample) ([][ for i, s := range samples { power[i] = s.PowerW } + power = normalizePowerSeries(power) datasets = [][]float64{power} names = []string{"Power W"} - yMin, yMax = autoBounds120(power) + yMin = floatPtr(0) + yMax = autoMax120(power) case path == "server-fans": title = "Fan RPM" @@ -928,6 +932,27 @@ func coalesceDataset(ds []float64, n int) []float64 { return make([]float64, n) } +func normalizePowerSeries(ds []float64) []float64 { + if len(ds) == 0 { + return nil + } + out := make([]float64, len(ds)) + copy(out, ds) + last := 0.0 + haveLast := false + for i, v := range out { + if v > 0 { + last = v + haveLast = true + continue + } + if haveLast { + out[i] = last + } + } + return out +} + // floatPtr returns a pointer to a float64 value. func floatPtr(v float64) *float64 { return &v } diff --git a/audit/internal/webui/tasks.go b/audit/internal/webui/tasks.go index 07825a3..10f8395 100644 --- a/audit/internal/webui/tasks.go +++ b/audit/internal/webui/tasks.go @@ -106,9 +106,10 @@ type taskParams struct { GPUIndices []int `json:"gpu_indices,omitempty"` ExcludeGPUIndices []int `json:"exclude_gpu_indices,omitempty"` Loader string `json:"loader,omitempty"` - BurnProfile string `json:"burn_profile,omitempty"` - DisplayName string `json:"display_name,omitempty"` - Device string `json:"device,omitempty"` // for install + BurnProfile string `json:"burn_profile,omitempty"` + DisplayName string `json:"display_name,omitempty"` + Device string `json:"device,omitempty"` // for install + PlatformComponents []string `json:"platform_components,omitempty"` } type persistedTask struct { @@ -550,6 +551,7 @@ func (q *taskQueue) runTask(t *Task, j *jobState, ctx context.Context) { break } opts := resolvePlatformStressPreset(t.params.BurnProfile) + opts.Components = t.params.PlatformComponents archive, err = a.RunPlatformStress(ctx, "", opts, j.append) case "audit": if a == nil { diff --git a/iso/overlay/usr/local/bin/bee-gpu-burn b/iso/overlay/usr/local/bin/bee-gpu-burn index d269263..00e0b18 100644 --- a/iso/overlay/usr/local/bin/bee-gpu-burn +++ b/iso/overlay/usr/local/bin/bee-gpu-burn @@ -2,7 +2,7 @@ set -eu SECONDS=5 -SIZE_MB=64 +SIZE_MB=0 DEVICES="" EXCLUDE="" WORKER="/usr/local/lib/bee/bee-gpu-burn-worker" @@ -68,8 +68,17 @@ trap 'rm -rf "${TMP_DIR}"' EXIT INT TERM WORKERS="" for id in $(echo "${FINAL}" | tr ',' ' '); do log="${TMP_DIR}/gpu-${id}.log" - echo "starting gpu ${id}" - "${WORKER}" --device "${id}" --seconds "${SECONDS}" --size-mb "${SIZE_MB}" >"${log}" 2>&1 & + gpu_size_mb="${SIZE_MB}" + if [ "${gpu_size_mb}" -le 0 ] 2>/dev/null; then + total_mb=$(nvidia-smi --id="${id}" --query-gpu=memory.total --format=csv,noheader,nounits 2>/dev/null | tr -d '[:space:]') + if [ -n "${total_mb}" ] && [ "${total_mb}" -gt 0 ] 2>/dev/null; then + gpu_size_mb=$(( total_mb * 95 / 100 )) + else + gpu_size_mb=512 + fi + fi + echo "starting gpu ${id} size=${gpu_size_mb}MB" + "${WORKER}" --device "${id}" --seconds "${SECONDS}" --size-mb "${gpu_size_mb}" >"${log}" 2>&1 & pid=$! WORKERS="${WORKERS} ${pid}:${id}:${log}" done