refactor: modularize audit and harden build validation
This commit is contained in:
Mikhail Chusavitin
@@ -0,0 +1,212 @@
|
||||
package webui
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"bee/audit/internal/app"
|
||||
"bee/audit/internal/platform"
|
||||
"bee/audit/internal/schema"
|
||||
)
|
||||
|
||||
// gpuReadyWait bounds how long /api/sat/run-all waits for the GPU driver
|
||||
// stack to come up before it plans the GPU tests. The per-GPU GSP firmware
|
||||
// boot on a multi-GPU box lags the device nodes by tens of seconds.
|
||||
// Overridable from tests.
|
||||
var (
|
||||
gpuReadyWait = 75 * time.Second
|
||||
gpuReadyPollInterval = 3 * time.Second
|
||||
apiRuntimeHealthNow = func(a *app.App) (schema.RuntimeHealth, error) {
|
||||
return a.RuntimeHealthNow()
|
||||
}
|
||||
)
|
||||
|
||||
type satRunAllRequest struct {
|
||||
StressMode bool `json:"stress_mode"`
|
||||
// AMDTargets is the operator's AMD check selection (intent). It is still
|
||||
// gated on an AMD GPU actually being present.
|
||||
AMDTargets []string `json:"amd_targets"`
|
||||
// NvidiaGPUIndices optionally narrows the NVIDIA tests to a subset; empty
|
||||
// means "every GPU the backend enumerates".
|
||||
NvidiaGPUIndices []int `json:"nvidia_gpu_indices"`
|
||||
}
|
||||
|
||||
type satRunAllResponse struct {
|
||||
TaskIDs []string `json:"task_ids"`
|
||||
TaskCount int `json:"task_count"`
|
||||
// Notes carries anything the backend decided to skip or override, so the
|
||||
// page can show it without reasoning about hardware itself.
|
||||
Notes []string `json:"notes,omitempty"`
|
||||
}
|
||||
|
||||
// handleAPISATRunAll plans and enqueues the full validate/check task set
|
||||
// server-side. Hardware presence and readiness are decided here, never in the
|
||||
// browser: the page sends only operator intent (stress toggle, AMD checkbox
|
||||
// selection, an optional GPU subset).
|
||||
func (h *handler) handleAPISATRunAll(w http.ResponseWriter, r *http.Request) {
|
||||
if h.opts.App == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "app not configured")
|
||||
return
|
||||
}
|
||||
var req satRunAllRequest
|
||||
if r.Body != nil {
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil && !errors.Is(err, io.EOF) {
|
||||
writeError(w, http.StatusBadRequest, "invalid request body")
|
||||
return
|
||||
}
|
||||
}
|
||||
specs, notes := h.planSATRunAll(r.Context(), req)
|
||||
|
||||
var ids []string
|
||||
for _, spec := range specs {
|
||||
tasks, err := h.enqueueSATTarget(spec.target, spec.params)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
for _, t := range tasks {
|
||||
if t != nil {
|
||||
ids = append(ids, t.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
slog.Info("sat run-all planned", "tasks", len(ids), "stress", req.StressMode, "notes", len(notes))
|
||||
writeJSON(w, satRunAllResponse{TaskIDs: ids, TaskCount: len(ids), Notes: notes})
|
||||
}
|
||||
|
||||
type satRunAllSpec struct {
|
||||
target string
|
||||
params taskParams
|
||||
}
|
||||
|
||||
func (h *handler) planSATRunAll(ctx context.Context, req satRunAllRequest) ([]satRunAllSpec, []string) {
|
||||
var specs []satRunAllSpec
|
||||
var notes []string
|
||||
skip := func(msg string) {
|
||||
notes = append(notes, msg)
|
||||
slog.Warn("sat run-all: check skipped", "reason", msg)
|
||||
}
|
||||
|
||||
cpuDur := 60
|
||||
if req.StressMode {
|
||||
cpuDur = 1800
|
||||
}
|
||||
specs = append(specs,
|
||||
satRunAllSpec{target: "cpu", params: taskParams{Duration: cpuDur, StressMode: req.StressMode}},
|
||||
satRunAllSpec{target: "memory", params: taskParams{StressMode: req.StressMode}},
|
||||
satRunAllSpec{target: "storage", params: taskParams{StressMode: req.StressMode}},
|
||||
satRunAllSpec{target: "pcie-link", params: taskParams{}},
|
||||
)
|
||||
|
||||
if h.opts.App.TPMPresent() {
|
||||
specs = append(specs, satRunAllSpec{target: "tpm", params: taskParams{}})
|
||||
} else {
|
||||
skip("TPM: no TPM device on this host; check skipped")
|
||||
}
|
||||
|
||||
gp := h.opts.App.DetectGPUPresence()
|
||||
|
||||
if gp.Nvidia || gp.NvidiaInitializing {
|
||||
// nvidia-config only collects inventory and NVLink state; safe to run
|
||||
// even while the compute stack is still coming up.
|
||||
specs = append(specs, satRunAllSpec{target: "nvidia-config", params: taskParams{}})
|
||||
|
||||
health, gpus, ready := h.waitForNvidiaReady(ctx)
|
||||
switch {
|
||||
case health.NvidiaGSPMode == "gsp-stuck":
|
||||
skip("NVIDIA: GSP firmware init is stuck; reboot with GSP=off. GPU compute/interconnect/bandwidth tests skipped")
|
||||
case !ready:
|
||||
skip("NVIDIA: nvidia-smi did not enumerate a GPU after " + gpuReadyWait.String() +
|
||||
". GPU compute/interconnect/bandwidth tests skipped; see the GPU Config check")
|
||||
default:
|
||||
indices := make([]int, 0, len(gpus))
|
||||
for _, g := range gpus {
|
||||
indices = append(indices, g.Index)
|
||||
}
|
||||
if len(req.NvidiaGPUIndices) > 0 {
|
||||
indices = intersectSortedInts(indices, req.NvidiaGPUIndices)
|
||||
}
|
||||
if len(indices) == 0 {
|
||||
skip("NVIDIA: driver ready but no GPU to test (enumeration empty, or the requested subset matched nothing); GPU tests skipped")
|
||||
break
|
||||
}
|
||||
if !health.CUDAReady {
|
||||
notes = append(notes, "NVIDIA: CUDA runtime not confirmed ready; GPU tests queued anyway")
|
||||
}
|
||||
gpuTargets := []string{"nvidia", "nvidia-interconnect", "nvidia-bandwidth", "nvidia-pcie-bandwidth"}
|
||||
if req.StressMode {
|
||||
// Stress tier adds the targeted dcgmi diag load tests.
|
||||
gpuTargets = append(gpuTargets, "nvidia-targeted-stress", "nvidia-targeted-power", "nvidia-pulse")
|
||||
}
|
||||
for _, target := range gpuTargets {
|
||||
specs = append(specs, satRunAllSpec{
|
||||
target: target,
|
||||
params: taskParams{GPUIndices: append([]int(nil), indices...), StressMode: req.StressMode},
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if gp.AMD {
|
||||
for _, target := range req.AMDTargets {
|
||||
switch target {
|
||||
case "amd", "amd-mem", "amd-bandwidth":
|
||||
specs = append(specs, satRunAllSpec{target: target, params: taskParams{StressMode: req.StressMode}})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return specs, notes
|
||||
}
|
||||
|
||||
// waitForNvidiaReady waits until the same nvidia-smi query used by
|
||||
// ListNvidiaGPUs returns at least one GPU. A loaded kernel module alone is
|
||||
// not evidence that NVIDIA user-space commands can address a GPU yet.
|
||||
func (h *handler) waitForNvidiaReady(ctx context.Context) (schema.RuntimeHealth, []platform.NvidiaGPU, bool) {
|
||||
deadline := time.Now().Add(gpuReadyWait)
|
||||
var last schema.RuntimeHealth
|
||||
for {
|
||||
health, err := apiRuntimeHealthNow(h.opts.App)
|
||||
if err == nil {
|
||||
last = health
|
||||
if health.NvidiaGSPMode == "gsp-stuck" {
|
||||
return health, nil, false
|
||||
}
|
||||
}
|
||||
gpus, listErr := apiListNvidiaGPUs(h.opts.App)
|
||||
if listErr == nil && len(gpus) > 0 {
|
||||
return last, gpus, true
|
||||
}
|
||||
if time.Now().After(deadline) || ctx.Err() != nil {
|
||||
return last, nil, false
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return last, nil, false
|
||||
case <-time.After(gpuReadyPollInterval):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// intersectSortedInts returns the ascending-sorted values present in both a
|
||||
// and b.
|
||||
func intersectSortedInts(a, b []int) []int {
|
||||
set := make(map[int]struct{}, len(b))
|
||||
for _, v := range b {
|
||||
set[v] = struct{}{}
|
||||
}
|
||||
out := make([]int, 0, len(a))
|
||||
for _, v := range a {
|
||||
if _, ok := set[v]; ok {
|
||||
out = append(out, v)
|
||||
}
|
||||
}
|
||||
sort.Ints(out)
|
||||
return out
|
||||
}
|
||||
Reference in New Issue
Block a user