bee/audit/internal/webui/api_sat_runall.go

package webui

import (
	"context"
	"encoding/json"
	"errors"
	"io"
	"log/slog"
	"net/http"
	"sort"
	"time"

	"bee/audit/internal/app"
	"bee/audit/internal/platform"
	"bee/audit/internal/schema"
)

// gpuReadyWait bounds how long /api/sat/run-all waits for the GPU driver
// stack to come up before it plans the GPU tests. The per-GPU GSP firmware
// boot on a multi-GPU box lags the device nodes by tens of seconds.
// Overridable from tests.
var (
	gpuReadyWait         = 75 * time.Second
	gpuReadyPollInterval = 3 * time.Second
	apiRuntimeHealthNow  = func(a *app.App) (schema.RuntimeHealth, error) {
		return a.RuntimeHealthNow()
	}
)

type satRunAllRequest struct {
	StressMode bool `json:"stress_mode"`
	// AMDTargets is the operator's AMD check selection (intent). It is still
	// gated on an AMD GPU actually being present.
	AMDTargets []string `json:"amd_targets"`
	// NvidiaGPUIndices optionally narrows the NVIDIA tests to a subset; empty
	// means "every GPU the backend enumerates".
	NvidiaGPUIndices []int `json:"nvidia_gpu_indices"`
}

type satRunAllResponse struct {
	TaskIDs   []string `json:"task_ids"`
	TaskCount int      `json:"task_count"`
	// Notes carries anything the backend decided to skip or override, so the
	// page can show it without reasoning about hardware itself.
	Notes []string `json:"notes,omitempty"`
}

// handleAPISATRunAll plans and enqueues the full validate/check task set
// server-side. Hardware presence and readiness are decided here, never in the
// browser: the page sends only operator intent (stress toggle, AMD checkbox
// selection, an optional GPU subset).
func (h *handler) handleAPISATRunAll(w http.ResponseWriter, r *http.Request) {
	if h.opts.App == nil {
		writeError(w, http.StatusServiceUnavailable, "app not configured")
		return
	}
	var req satRunAllRequest
	if r.Body != nil {
		if err := json.NewDecoder(r.Body).Decode(&req); err != nil && !errors.Is(err, io.EOF) {
			writeError(w, http.StatusBadRequest, "invalid request body")
			return
		}
	}
	specs, notes := h.planSATRunAll(r.Context(), req)

	var ids []string
	for _, spec := range specs {
		tasks, err := h.enqueueSATTarget(spec.target, spec.params)
		if err != nil {
			writeError(w, http.StatusInternalServerError, err.Error())
			return
		}
		for _, t := range tasks {
			if t != nil {
				ids = append(ids, t.ID)
			}
		}
	}
	slog.Info("sat run-all planned", "tasks", len(ids), "stress", req.StressMode, "notes", len(notes))
	writeJSON(w, satRunAllResponse{TaskIDs: ids, TaskCount: len(ids), Notes: notes})
}

type satRunAllSpec struct {
	target string
	params taskParams
}

func (h *handler) planSATRunAll(ctx context.Context, req satRunAllRequest) ([]satRunAllSpec, []string) {
	var specs []satRunAllSpec
	var notes []string
	skip := func(msg string) {
		notes = append(notes, msg)
		slog.Warn("sat run-all: check skipped", "reason", msg)
	}

	cpuDur := 60
	if req.StressMode {
		cpuDur = 1800
	}
	specs = append(specs,
		satRunAllSpec{target: "cpu", params: taskParams{Duration: cpuDur, StressMode: req.StressMode}},
		satRunAllSpec{target: "memory", params: taskParams{StressMode: req.StressMode}},
		satRunAllSpec{target: "storage", params: taskParams{StressMode: req.StressMode}},
	)

	if !req.StressMode {
		if h.opts.App.TPMPresent() {
			specs = append(specs, satRunAllSpec{target: "tpm", params: taskParams{}})
		} else {
			skip("TPM: no TPM device on this host; check skipped")
		}
	} else {
		// Fan ceiling check runs on the Load tier only. It self-cancels as
		// "not applicable" on a host with no fan sensors or no way to load
		// the CPU/GPU, so it is safe to queue unconditionally here.
		specs = append(specs, satRunAllSpec{target: "fan", params: taskParams{StressMode: true}})
	}

	gp := h.opts.App.DetectGPUPresence()

	if gp.Nvidia || gp.NvidiaInitializing {
		// nvidia-config is a read-only Check task, not a load test.
		if !req.StressMode {
			specs = append(specs, satRunAllSpec{target: "nvidia-config", params: taskParams{}})
		}

		health, gpus, ready := h.waitForNvidiaReady(ctx)
		switch {
		case health.NvidiaGSPMode == "gsp-stuck":
			skip("NVIDIA: GSP firmware init is stuck; reboot with GSP=off. GPU compute/interconnect/bandwidth tests skipped")
		case !ready:
			skip("NVIDIA: nvidia-smi did not enumerate a GPU after " + gpuReadyWait.String() +
				". GPU compute/interconnect/bandwidth tests skipped; see the GPU Config check")
		default:
			indices := make([]int, 0, len(gpus))
			for _, g := range gpus {
				indices = append(indices, g.Index)
			}
			if len(req.NvidiaGPUIndices) > 0 {
				indices = intersectSortedInts(indices, req.NvidiaGPUIndices)
			}
			if len(indices) == 0 {
				skip("NVIDIA: driver ready but no GPU to test (enumeration empty, or the requested subset matched nothing); GPU tests skipped")
				break
			}
			if !health.CUDAReady {
				notes = append(notes, "NVIDIA: CUDA runtime not confirmed ready; GPU tests queued anyway")
			}
			gpuTargets := []string{"nvidia", "nvidia-bandwidth"}
			if req.StressMode {
				// Stress tier adds the targeted dcgmi diag load tests.
				gpuTargets = append(gpuTargets, "nvidia-targeted-stress", "nvidia-targeted-power", "nvidia-pulse")
			} else {
				gpuTargets = append(gpuTargets, "nvidia-interconnect")
			}
			for _, target := range gpuTargets {
				specs = append(specs, satRunAllSpec{
					target: target,
					params: taskParams{GPUIndices: append([]int(nil), indices...), StressMode: req.StressMode},
				})
			}
		}
	}

	if gp.AMD && !req.StressMode {
		for _, target := range req.AMDTargets {
			switch target {
			case "amd", "amd-mem", "amd-bandwidth":
				specs = append(specs, satRunAllSpec{target: target, params: taskParams{StressMode: req.StressMode}})
			}
		}
	}

	return specs, notes
}

// waitForNvidiaReady waits until the same nvidia-smi query used by
// ListNvidiaGPUs returns at least one GPU. A loaded kernel module alone is
// not evidence that NVIDIA user-space commands can address a GPU yet.
func (h *handler) waitForNvidiaReady(ctx context.Context) (schema.RuntimeHealth, []platform.NvidiaGPU, bool) {
	deadline := time.Now().Add(gpuReadyWait)
	var last schema.RuntimeHealth
	for {
		health, err := apiRuntimeHealthNow(h.opts.App)
		if err == nil {
			last = health
			if health.NvidiaGSPMode == "gsp-stuck" {
				return health, nil, false
			}
		}
		gpus, listErr := apiListNvidiaGPUs(h.opts.App)
		if listErr == nil && len(gpus) > 0 {
			return last, gpus, true
		}
		if time.Now().After(deadline) || ctx.Err() != nil {
			return last, nil, false
		}
		select {
		case <-ctx.Done():
			return last, nil, false
		case <-time.After(gpuReadyPollInterval):
		}
	}
}

// intersectSortedInts returns the ascending-sorted values present in both a
// and b.
func intersectSortedInts(a, b []int) []int {
	set := make(map[int]struct{}, len(b))
	for _, v := range b {
		set[v] = struct{}{}
	}
	out := make([]int, 0, len(a))
	for _, v := range a {
		if _, ok := set[v]; ok {
			out = append(out, v)
		}
	}
	sort.Ints(out)
	return out
}