reanimator/bee
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

refactor: modularize audit and harden build validation

Browse Source
This commit is contained in:
Mikhail Chusavitin
2026-08-31 21:22:16 +03:00
parent bb22ccfafe
commit ac4bc0b2b7
78 changed files with 13598 additions and 13130 deletions
Download Patch File Download Diff File Expand all files Collapse all files
audit/internal/webui/api.go
-1525
View File
File diff suppressed because it is too large Load Diff
audit/internal/webui/api_gpu_tools.go
+509
View File
@@ -0,0 +1,509 @@
package webui
import (
"encoding/json"
"fmt"
"net/http"
"os"
"os/exec"
"path/filepath"
"regexp"
"strconv"
"strings"
"time"
"bee/audit/internal/platform"
)
func (h *handler) handleAPIGNVIDIAGPUs(w http.ResponseWriter, _ *http.Request) {
if h.opts.App == nil {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
}
gpus, err := h.opts.App.ListNvidiaGPUs()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if gpus == nil {
gpus = []platform.NvidiaGPU{}
}
writeJSON(w, gpus)
}
func (h *handler) handleAPIGNVIDIAGPUStatuses(w http.ResponseWriter, _ *http.Request) {
if h.opts.App == nil {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
}
gpus, err := apiListNvidiaGPUStatuses(h.opts.App)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if gpus == nil {
gpus = []platform.NvidiaGPUStatus{}
}
writeJSON(w, gpus)
}
func (h *handler) handleAPIGNVIDIAReset(w http.ResponseWriter, r *http.Request) {
if h.opts.App == nil {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
}
var req struct {
Index int `json:"index"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid request body")
return
}
result, err := h.opts.App.ResetNvidiaGPU(req.Index)
status := "ok"
if err != nil {
status = "error"
}
writeJSON(w, map[string]string{"status": status, "output": result.Body})
}
// ── GPU settings (ECC / power limit) ──────────────────────────────────────────
func (h *handler) handleAPIGNVIDIAGPUSettings(w http.ResponseWriter, _ *http.Request) {
if h.opts.App == nil {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
}
settings, err := h.opts.App.ListNvidiaGPUSettings()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if settings == nil {
settings = []platform.NvidiaGPUSetting{}
}
writeJSON(w, settings)
}
func (h *handler) handleAPIGNVIDIASetECC(w http.ResponseWriter, r *http.Request) {
if h.opts.App == nil {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
}
var req struct {
Index int `json:"index"`
Enabled bool `json:"enabled"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid request body")
return
}
result, err := h.opts.App.SetNvidiaGPUECC(req.Index, req.Enabled)
status := "ok"
if err != nil {
status = "error"
}
writeJSON(w, map[string]string{"status": status, "output": result.Body})
}
func (h *handler) handleAPIGNVIDIASetMIG(w http.ResponseWriter, r *http.Request) {
if h.opts.App == nil {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
}
var req struct {
Index int `json:"index"`
Enabled bool `json:"enabled"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid request body")
return
}
result, err := h.opts.App.SetNvidiaGPUMIG(req.Index, req.Enabled)
status := "ok"
if err != nil {
status = "error"
}
writeJSON(w, map[string]string{"status": status, "output": result.Body})
}
func (h *handler) handleAPIGNVIDIASetCCMode(w http.ResponseWriter, r *http.Request) {
if h.opts.App == nil {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
}
var req struct {
Index int `json:"index"`
Enabled bool `json:"enabled"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid request body")
return
}
result, err := h.opts.App.SetNvidiaGPUCCMode(req.Index, req.Enabled)
status := "ok"
if err != nil {
status = "error"
}
writeJSON(w, map[string]string{"status": status, "output": result.Body})
}
func (h *handler) handleAPIGNVIDIASetPowerLimit(w http.ResponseWriter, r *http.Request) {
if h.opts.App == nil {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
}
var req struct {
Index int `json:"index"`
Watts float64 `json:"watts"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid request body")
return
}
if req.Watts <= 0 {
writeError(w, http.StatusBadRequest, "watts must be > 0")
return
}
result, err := h.opts.App.SetNvidiaGPUPowerLimit(req.Index, req.Watts)
status := "ok"
if err != nil {
status = "error"
}
writeJSON(w, map[string]string{"status": status, "output": result.Body})
}
func (h *handler) handleAPIGNVIDIAResetDefaults(w http.ResponseWriter, _ *http.Request) {
if h.opts.App == nil {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
}
result, err := h.opts.App.ResetNvidiaGPUDefaults()
status := "ok"
if err != nil {
status = "error"
}
writeJSON(w, map[string]string{"status": status, "output": result.Body})
}
func (h *handler) handleAPIGPUPresence(w http.ResponseWriter, r *http.Request) {
if h.opts.App == nil {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
}
gp := h.opts.App.DetectGPUPresence()
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]bool{
"nvidia": gp.Nvidia,
"amd": gp.AMD,
"nvidia_initializing": gp.NvidiaInitializing,
"amd_initializing": gp.AMDInitializing,
})
}
// ── GPU tools ─────────────────────────────────────────────────────────────────
func (h *handler) handleAPIGPUTools(w http.ResponseWriter, _ *http.Request) {
type toolEntry struct {
ID string `json:"id"`
Available bool `json:"available"`
Vendor string `json:"vendor"` // "nvidia" | "amd"
}
// Single source of truth for GPU presence: see app.DetectGPUPresence.
var nvidiaUp, amdUp bool
if h.opts.App != nil {
gp := h.opts.App.DetectGPUPresence()
nvidiaUp, amdUp = gp.Nvidia, gp.AMD
} else {
_, nvidiaErr := os.Stat("/dev/nvidia0")
_, amdErr := os.Stat("/dev/kfd")
nvidiaUp, amdUp = nvidiaErr == nil, amdErr == nil
}
_, dcgmErr := exec.LookPath("dcgmi")
_, ncclStressErr := exec.LookPath("bee-nccl-gpu-stress")
_, johnErr := exec.LookPath("bee-john-gpu-stress")
_, beeBurnErr := exec.LookPath("bee-gpu-burn")
_, nvBandwidthErr := exec.LookPath("nvbandwidth")
profErr := lookPathAny("dcgmproftester", "dcgmproftester13", "dcgmproftester12", "dcgmproftester11")
writeJSON(w, []toolEntry{
{ID: "nvidia-compute", Available: nvidiaUp && profErr == nil, Vendor: "nvidia"},
{ID: "nvidia-targeted-power", Available: nvidiaUp && dcgmErr == nil, Vendor: "nvidia"},
{ID: "nvidia-pulse", Available: nvidiaUp && dcgmErr == nil, Vendor: "nvidia"},
{ID: "nvidia-interconnect", Available: nvidiaUp && ncclStressErr == nil, Vendor: "nvidia"},
{ID: "nvidia-bandwidth", Available: nvidiaUp && dcgmErr == nil && nvBandwidthErr == nil, Vendor: "nvidia"},
{ID: "bee-gpu-burn", Available: nvidiaUp && beeBurnErr == nil, Vendor: "nvidia"},
{ID: "john", Available: nvidiaUp && johnErr == nil, Vendor: "nvidia"},
{ID: "rvs", Available: amdUp, Vendor: "amd"},
})
}
func lookPathAny(names ...string) error {
for _, name := range names {
if _, err := exec.LookPath(name); err == nil {
return nil
}
}
return exec.ErrNotFound
}
// ── System ────────────────────────────────────────────────────────────────────
func (h *handler) handleAPIRAMStatus(w http.ResponseWriter, r *http.Request) {
if h.opts.App == nil {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
}
status := h.currentRAMStatus()
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(status)
}
type ramStatusResponse struct {
platform.LiveMediaRAMState
InstallTaskActive bool `json:"install_task_active,omitempty"`
CopyTaskActive bool `json:"copy_task_active,omitempty"`
CanStartTask bool `json:"can_start_task,omitempty"`
BlockedReason string `json:"blocked_reason,omitempty"`
}
func (h *handler) currentRAMStatus() ramStatusResponse {
state := h.opts.App.LiveMediaRAMState()
resp := ramStatusResponse{LiveMediaRAMState: state}
if globalQueue.hasActiveTarget("install") {
resp.InstallTaskActive = true
resp.BlockedReason = "install to disk is already running"
return resp
}
if globalQueue.hasActiveTarget("install-to-ram") {
resp.CopyTaskActive = true
resp.BlockedReason = "install to RAM task is already pending or running"
return resp
}
if state.InRAM {
resp.BlockedReason = "system is already running from RAM"
return resp
}
resp.CanStartTask = state.CanStartCopy
if !resp.CanStartTask && resp.BlockedReason == "" {
resp.BlockedReason = state.Message
}
return resp
}
func (h *handler) handleAPIInstallToRAM(w http.ResponseWriter, r *http.Request) {
if h.opts.App == nil {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
}
status := h.currentRAMStatus()
if !status.CanStartTask {
msg := strings.TrimSpace(status.BlockedReason)
if msg == "" {
msg = "install to RAM is not available"
}
writeError(w, http.StatusConflict, msg)
return
}
t := &Task{
ID: newJobID("install-to-ram"),
Name: "Install to RAM",
Target: "install-to-ram",
Priority: defaultTaskPriority("install-to-ram", taskParams{}),
Status: TaskPending,
CreatedAt: time.Now(),
}
globalQueue.enqueue(t)
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]string{"task_id": t.ID})
}
func (h *handler) handleAPISystemReboot(w http.ResponseWriter, r *http.Request) {
if err := exec.Command("systemctl", "reboot").Start(); err != nil {
writeError(w, http.StatusInternalServerError, "reboot failed: "+err.Error())
return
}
writeJSON(w, map[string]string{"status": "rebooting"})
}
func (h *handler) handleAPISystemShutdown(w http.ResponseWriter, r *http.Request) {
if err := exec.Command("systemctl", "poweroff").Start(); err != nil {
writeError(w, http.StatusInternalServerError, "shutdown failed: "+err.Error())
return
}
writeJSON(w, map[string]string{"status": "shutting down"})
}
// timezoneNameRE matches IANA timezone identifiers like "Europe/Moscow" or "UTC".
var timezoneNameRE = regexp.MustCompile(`^[A-Za-z0-9_+\-]+(/[A-Za-z0-9_+\-]+)*$`)
func validTimezoneName(tz string) bool {
if tz == "" || !timezoneNameRE.MatchString(tz) {
return false
}
_, err := os.Stat(filepath.Join("/usr/share/zoneinfo", tz))
return err == nil
}
// handleAPISystemTimeSync sets the host's timezone and wall-clock time from
// values supplied by the client's browser (used when the appliance has no
// network/NTP access to keep its own clock in sync).
func (h *handler) handleAPISystemTimeSync(w http.ResponseWriter, r *http.Request) {
var req struct {
Timezone string `json:"timezone"`
EpochMS int64 `json:"epoch_ms"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid request body")
return
}
if req.EpochMS <= 0 {
writeError(w, http.StatusBadRequest, "epoch_ms required")
return
}
var out strings.Builder
if req.Timezone != "" {
if !validTimezoneName(req.Timezone) {
writeError(w, http.StatusBadRequest, "invalid timezone")
return
}
if b, err := exec.Command("timedatectl", "set-timezone", req.Timezone).CombinedOutput(); err != nil {
writeError(w, http.StatusInternalServerError, "set-timezone failed: "+strings.TrimSpace(string(b)))
return
}
fmt.Fprintf(&out, "timezone set to %s\n", req.Timezone)
}
// Manual time only sticks if NTP sync is off.
_ = exec.Command("timedatectl", "set-ntp", "false").Run()
sec := req.EpochMS / 1000
if b, err := exec.Command("date", "-u", "-s", "@"+strconv.FormatInt(sec, 10)).CombinedOutput(); err != nil {
writeError(w, http.StatusInternalServerError, "set-time failed: "+strings.TrimSpace(string(b)))
return
}
out.WriteString("system clock synced\n")
writeJSON(w, map[string]string{"status": "ok", "output": out.String()})
}
// ── Tools ─────────────────────────────────────────────────────────────────────
var standardTools = []string{
"dmidecode", "smartctl", "nvme", "lspci", "ipmitool",
"tpm2_getcap", "tpm2_pcrread", "tpm2_gettestresult",
"nvidia-smi", "dcgmi", "nv-hostengine", "memtester", "stress-ng", "nvtop",
"mstflint", "saa",
}
func (h *handler) handleAPIToolsCheck(w http.ResponseWriter, r *http.Request) {
if h.opts.App == nil {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
}
statuses := h.opts.App.CheckTools(standardTools)
writeJSON(w, statuses)
}
// ── Preflight ─────────────────────────────────────────────────────────────────
func (h *handler) handleAPIPreflight(w http.ResponseWriter, r *http.Request) {
data, err := loadSnapshot(filepath.Join(h.opts.ExportDir, "runtime-health.json"))
if err != nil {
writeError(w, http.StatusNotFound, "runtime health not found")
return
}
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.Header().Set("Cache-Control", "no-store")
_, _ = w.Write(data)
}
// ── Install ───────────────────────────────────────────────────────────────────
func (h *handler) handleAPIInstallDisks(w http.ResponseWriter, r *http.Request) {
if h.opts.App == nil {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
}
disks, err := h.opts.App.ListInstallDisks()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
type diskJSON struct {
Device string `json:"device"`
Model string `json:"model"`
Size string `json:"size"`
SizeBytes int64 `json:"size_bytes"`
MountedParts []string `json:"mounted_parts"`
Warnings []string `json:"warnings"`
}
result := make([]diskJSON, 0, len(disks))
for _, d := range disks {
result = append(result, diskJSON{
Device: d.Device,
Model: d.Model,
Size: d.Size,
SizeBytes: d.SizeBytes,
MountedParts: d.MountedParts,
Warnings: platform.DiskWarnings(d),
})
}
writeJSON(w, result)
}
func (h *handler) handleAPIInstallRun(w http.ResponseWriter, r *http.Request) {
if h.opts.App == nil {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
}
var req struct {
Device string `json:"device"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.Device == "" {
writeError(w, http.StatusBadRequest, "device is required")
return
}
// Whitelist: only allow devices that ListInstallDisks() returns.
disks, err := h.opts.App.ListInstallDisks()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
allowed := false
for _, d := range disks {
if d.Device == req.Device {
allowed = true
break
}
}
if !allowed {
writeError(w, http.StatusBadRequest, "device not in install candidate list")
return
}
if globalQueue.hasActiveTarget("install-to-ram") {
writeError(w, http.StatusConflict, "install to RAM task is already pending or running")
return
}
if globalQueue.hasActiveTarget("install") {
writeError(w, http.StatusConflict, "install task is already pending or running")
return
}
t := &Task{
ID: newJobID("install"),
Name: "Install to Disk",
Target: "install",
Priority: defaultTaskPriority("install", taskParams{}),
Status: TaskPending,
CreatedAt: time.Now(),
params: taskParams{
Device: req.Device,
},
}
globalQueue.enqueue(t)
writeJSON(w, map[string]string{"task_id": t.ID, "job_id": t.ID})
}
// ── Metrics SSE ───────────────────────────────────────────────────────────────
audit/internal/webui/api_metrics_network.go
+360
View File
@@ -0,0 +1,360 @@
package webui
import (
"encoding/json"
"fmt"
"net/http"
"time"
"bee/audit/internal/app"
"bee/audit/internal/platform"
)
func (h *handler) handleAPIMetricsLatest(w http.ResponseWriter, r *http.Request) {
sample, ok := h.latestMetric()
if !ok {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte("{}"))
return
}
b, err := json.Marshal(sample)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write(b)
}
func (h *handler) handleAPIMetricsStream(w http.ResponseWriter, r *http.Request) {
if !sseStart(w) {
return
}
ticker := time.NewTicker(1 * time.Second)
defer ticker.Stop()
for {
select {
case <-r.Context().Done():
return
case <-ticker.C:
sample, ok := h.latestMetric()
if !ok {
continue
}
b, err := json.Marshal(sample)
if err != nil {
continue
}
if !sseWrite(w, "metrics", string(b)) {
return
}
}
}
}
// feedRings pushes one sample into all in-memory ring buffers.
func (h *handler) feedRings(sample platform.LiveMetricSample) {
for _, t := range sample.Temps {
switch t.Group {
case "cpu":
h.pushNamedMetricRing(&h.cpuTempRings, t.Name, t.Celsius)
case "ambient":
h.pushNamedMetricRing(&h.ambientTempRings, t.Name, t.Celsius)
}
}
h.ringPower.push(sample.PowerW)
h.ringCPULoad.push(sample.CPULoadPct)
h.ringMemLoad.push(sample.MemLoadPct)
h.ringsMu.Lock()
h.pushFanRings(sample.Fans)
for _, gpu := range sample.GPUs {
idx := gpu.GPUIndex
for len(h.gpuRings) <= idx {
h.gpuRings = append(h.gpuRings, &gpuRings{
Temp: newMetricsRing(120),
Util: newMetricsRing(120),
MemUtil: newMetricsRing(120),
Power: newMetricsRing(120),
})
}
h.gpuRings[idx].Temp.push(gpu.TempC)
h.gpuRings[idx].Util.push(gpu.UsagePct)
h.gpuRings[idx].MemUtil.push(gpu.MemUsagePct)
h.gpuRings[idx].Power.push(gpu.PowerW)
}
h.ringsMu.Unlock()
}
func (h *handler) pushFanRings(fans []platform.FanReading) {
if len(fans) == 0 && len(h.ringFans) == 0 {
return
}
fanValues := make(map[string]float64, len(fans))
for _, fan := range fans {
if fan.Name == "" {
continue
}
fanValues[fan.Name] = fan.RPM
found := false
for i, name := range h.fanNames {
if name == fan.Name {
found = true
if i >= len(h.ringFans) {
h.ringFans = append(h.ringFans, newMetricsRing(120))
}
break
}
}
if !found {
h.fanNames = append(h.fanNames, fan.Name)
h.ringFans = append(h.ringFans, newMetricsRing(120))
}
}
for i, ring := range h.ringFans {
if ring == nil {
continue
}
name := ""
if i < len(h.fanNames) {
name = h.fanNames[i]
}
if rpm, ok := fanValues[name]; ok {
ring.push(rpm)
continue
}
if last, ok := ring.latest(); ok {
ring.push(last)
continue
}
ring.push(0)
}
}
func (h *handler) pushNamedMetricRing(dst *[]*namedMetricsRing, name string, value float64) {
if name == "" {
return
}
for _, item := range *dst {
if item != nil && item.Name == name && item.Ring != nil {
item.Ring.push(value)
return
}
}
*dst = append(*dst, &namedMetricsRing{
Name: name,
Ring: newMetricsRing(120),
})
(*dst)[len(*dst)-1].Ring.push(value)
}
// ── Network toggle ────────────────────────────────────────────────────────────
const netRollbackTimeout = 60 * time.Second
func (h *handler) handleAPINetworkToggle(w http.ResponseWriter, r *http.Request) {
if h.opts.App == nil {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
}
var req struct {
Iface string `json:"iface"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.Iface == "" {
writeError(w, http.StatusBadRequest, "iface is required")
return
}
wasUp, err := h.opts.App.GetInterfaceState(req.Iface)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if _, err := h.applyPendingNetworkChange(func() (app.ActionResult, error) {
err := h.opts.App.SetInterfaceState(req.Iface, !wasUp)
return app.ActionResult{}, err
}); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
newState := "up"
if wasUp {
newState = "down"
}
writeJSON(w, map[string]any{
"iface": req.Iface,
"new_state": newState,
"rollback_in": int(netRollbackTimeout.Seconds()),
})
}
func (h *handler) applyPendingNetworkChange(apply func() (app.ActionResult, error)) (app.ActionResult, error) {
if h.opts.App == nil {
return app.ActionResult{}, fmt.Errorf("app not configured")
}
if err := h.rollbackPendingNetworkChange(); err != nil && err.Error() != "no pending network change" {
return app.ActionResult{}, err
}
snapshot, err := h.opts.App.CaptureNetworkSnapshot()
if err != nil {
return app.ActionResult{}, err
}
result, err := apply()
if err != nil {
return result, err
}
pnc := &pendingNetChange{
snapshot: snapshot,
deadline: time.Now().Add(netRollbackTimeout),
}
pnc.timer = time.AfterFunc(netRollbackTimeout, func() {
_ = h.opts.App.RestoreNetworkSnapshot(snapshot)
h.pendingNetMu.Lock()
if h.pendingNet == pnc {
h.pendingNet = nil
}
h.pendingNetMu.Unlock()
})
h.pendingNetMu.Lock()
h.pendingNet = pnc
h.pendingNetMu.Unlock()
return result, nil
}
func (h *handler) hasPendingNetworkChange() bool {
h.pendingNetMu.Lock()
defer h.pendingNetMu.Unlock()
return h.pendingNet != nil
}
func (h *handler) pendingNetworkRollbackIn() int {
h.pendingNetMu.Lock()
defer h.pendingNetMu.Unlock()
if h.pendingNet == nil {
return 0
}
remaining := int(time.Until(h.pendingNet.deadline).Seconds())
if remaining < 1 {
return 1
}
return remaining
}
func (h *handler) handleAPINetworkConfirm(w http.ResponseWriter, _ *http.Request) {
h.pendingNetMu.Lock()
pnc := h.pendingNet
h.pendingNet = nil
h.pendingNetMu.Unlock()
if pnc != nil {
pnc.mu.Lock()
pnc.timer.Stop()
pnc.mu.Unlock()
}
writeJSON(w, map[string]string{"status": "confirmed"})
}
func (h *handler) handleAPINetworkRollback(w http.ResponseWriter, _ *http.Request) {
if err := h.rollbackPendingNetworkChange(); err != nil {
if err.Error() == "no pending network change" {
writeError(w, http.StatusConflict, err.Error())
return
}
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, map[string]string{"status": "rolled back"})
}
func (h *handler) handleAPIBenchmarkResults(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html; charset=utf-8")
fmt.Fprint(w, renderBenchmarkResultsCard(h.opts.ExportDir))
}
// ── Hardware summary / component detail ──────────────────────────────────────
// handleAPIHardwareSummary returns the hardware summary card HTML fragment for
// htmx polling (hx-get="/api/hardware-summary" hx-swap="outerHTML").
func (h *handler) handleAPIHardwareSummary(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "text/html; charset=utf-8")
w.Header().Set("Cache-Control", "no-store")
fmt.Fprint(w, renderHardwareSummaryCard(h.opts))
}
// handleAPIComponentDetail returns an HTML fragment describing the current and
// historical status for one component type (cpu, memory, storage, gpu, psu).
func (h *handler) handleAPIComponentDetail(w http.ResponseWriter, r *http.Request) {
compType := r.PathValue("type")
var exact, prefixes []string
var title string
switch compType {
case "cpu":
title = "CPU"
exact = []string{"cpu:all"}
case "memory":
title = "Memory"
exact = []string{"memory:all"}
prefixes = []string{"memory:"}
case "storage":
title = "Storage"
exact = []string{"storage:all"}
prefixes = []string{"storage:"}
case "gpu":
title = "GPU"
prefixes = []string{"pcie:gpu:"}
case "nic":
title = "NIC"
prefixes = []string{"pcie:nic:"}
case "psu":
title = "PSU"
prefixes = []string{"psu:"}
case "raid":
title = "RAID"
prefixes = []string{"pcie:raid:"}
default:
http.NotFound(w, r)
return
}
var records []app.ComponentStatusRecord
if h.opts.App != nil && h.opts.App.StatusDB != nil {
all := h.opts.App.StatusDB.All()
records = matchedRecords(all, exact, prefixes)
}
fromInventory := false
if len(records) == 0 {
if fallback := inventoryFallbackRecords(compType, h.opts); len(fallback) > 0 {
records = fallback
fromInventory = true
}
}
w.Header().Set("Content-Type", "text/html; charset=utf-8")
w.Header().Set("Cache-Control", "no-store")
fmt.Fprint(w, renderComponentDetail(title, records, fromInventory))
}
func (h *handler) rollbackPendingNetworkChange() error {
h.pendingNetMu.Lock()
pnc := h.pendingNet
h.pendingNet = nil
h.pendingNetMu.Unlock()
if pnc == nil {
return fmt.Errorf("no pending network change")
}
pnc.mu.Lock()
pnc.timer.Stop()
pnc.mu.Unlock()
if h.opts.App != nil {
return h.opts.App.RestoreNetworkSnapshot(pnc.snapshot)
}
return nil
}
audit/internal/webui/api_runs.go
+430
View File
@@ -0,0 +1,430 @@
package webui
import (
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"strings"
"time"
"bee/audit/internal/platform"
)
func (h *handler) handleAPIAuditRun(w http.ResponseWriter, _ *http.Request) {
if h.opts.App == nil {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
}
t := &Task{
ID: newJobID("audit"),
Name: "Audit",
Target: "audit",
Priority: defaultTaskPriority("audit", taskParams{}),
Status: TaskPending,
CreatedAt: time.Now(),
}
globalQueue.enqueue(t)
writeJSON(w, map[string]string{"task_id": t.ID, "job_id": t.ID})
}
func (h *handler) handleAPIAuditStream(w http.ResponseWriter, r *http.Request) {
id := r.URL.Query().Get("job_id")
if id == "" {
id = r.URL.Query().Get("task_id")
}
// Try task queue first, then legacy job manager
if j, ok := globalQueue.findJob(id); ok {
streamJob(w, r, j)
return
}
if j, ok := globalJobs.get(id); ok {
streamJob(w, r, j)
return
}
http.Error(w, "job not found", http.StatusNotFound)
}
// ── SAT ───────────────────────────────────────────────────────────────────────
func (h *handler) handleAPISATRun(target string) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if h.opts.App == nil {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
}
var body struct {
Duration int `json:"duration"`
StressMode bool `json:"stress_mode"`
GPUIndices []int `json:"gpu_indices"`
ExcludeGPUIndices []int `json:"exclude_gpu_indices"`
StaggerGPUStart bool `json:"stagger_gpu_start"`
ParallelGPUs bool `json:"parallel_gpus"`
Loader string `json:"loader"`
Profile string `json:"profile"`
DisplayName string `json:"display_name"`
PlatformComponents []string `json:"platform_components"`
}
if r.Body != nil {
if err := json.NewDecoder(r.Body).Decode(&body); err != nil && !errors.Is(err, io.EOF) {
writeError(w, http.StatusBadRequest, "invalid request body")
return
}
}
params := taskParams{
Duration: body.Duration,
StressMode: body.StressMode,
GPUIndices: body.GPUIndices,
ExcludeGPUIndices: body.ExcludeGPUIndices,
StaggerGPUStart: body.StaggerGPUStart,
ParallelGPUs: body.ParallelGPUs,
Loader: body.Loader,
BurnProfile: body.Profile,
DisplayName: body.DisplayName,
PlatformComponents: body.PlatformComponents,
}
tasks, err := h.enqueueSATTarget(target, params)
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeTaskRunResponse(w, tasks)
}
}
// enqueueSATTarget builds the task set for one SAT target (splitting
// homogeneous multi-GPU NVIDIA targets as needed) and enqueues it. Shared by
// the single-target /api/sat/<target>/run endpoints and /api/sat/run-all.
func (h *handler) enqueueSATTarget(target string, params taskParams) ([]*Task, error) {
name := taskDisplayName(target, params.BurnProfile, params.Loader)
if strings.TrimSpace(params.DisplayName) != "" {
name = params.DisplayName
}
tasks, err := buildNvidiaTaskSet(target, defaultTaskPriority(target, params), time.Now(), params, name, h.opts.App, "sat-"+target)
if err != nil {
return nil, err
}
for _, t := range tasks {
globalQueue.enqueue(t)
}
return tasks, nil
}
// ── Scenario ─────────────────────────────────────────────────────────────────
func (h *handler) handleAPIScenarioList(w http.ResponseWriter, _ *http.Request) {
if h.opts.App == nil {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
}
files, err := h.opts.App.ListAvailableScenarios()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
type scenarioFile struct {
Name string `json:"name"`
Description string `json:"description"`
Device string `json:"device"`
}
out := make([]scenarioFile, 0, len(files))
for _, f := range files {
out = append(out, scenarioFile{Name: f.Name, Description: f.Description, Device: f.Device})
}
writeJSON(w, out)
}
func (h *handler) handleAPIScenarioRun(w http.ResponseWriter, r *http.Request) {
if h.opts.App == nil {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
}
var body struct {
Name string `json:"name"`
}
if r.Body != nil {
if err := json.NewDecoder(r.Body).Decode(&body); err != nil && !errors.Is(err, io.EOF) {
writeError(w, http.StatusBadRequest, "invalid request body")
return
}
}
name := strings.TrimSpace(body.Name)
if name == "" {
writeError(w, http.StatusBadRequest, "scenario name is required")
return
}
t := &Task{
ID: newJobID("scenario"),
Name: "Scenario: " + name,
Target: "scenario",
Priority: defaultTaskPriority("scenario", taskParams{}),
Status: TaskPending,
CreatedAt: time.Now(),
}
t.params.ScenarioName = name
globalQueue.enqueue(t)
writeJSON(w, map[string]string{"task_id": t.ID, "job_id": t.ID})
}
func (h *handler) handleAPIBenchmarkNvidiaRunKind(target string) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if h.opts.App == nil {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
}
var body struct {
Profile string `json:"profile"`
SizeMB int `json:"size_mb"`
GPUIndices []int `json:"gpu_indices"`
ExcludeGPUIndices []int `json:"exclude_gpu_indices"`
RunNCCL *bool `json:"run_nccl"`
ParallelGPUs *bool `json:"parallel_gpus"`
RampUp *bool `json:"ramp_up"`
DisplayName string `json:"display_name"`
}
if r.Body != nil {
if err := json.NewDecoder(r.Body).Decode(&body); err != nil && !errors.Is(err, io.EOF) {
writeError(w, http.StatusBadRequest, "invalid request body")
return
}
}
runNCCL := true
if body.RunNCCL != nil {
runNCCL = *body.RunNCCL
}
parallelGPUs := false
if body.ParallelGPUs != nil {
parallelGPUs = *body.ParallelGPUs
}
rampUp := false
if body.RampUp != nil {
rampUp = *body.RampUp
}
// Build a descriptive base name that includes profile and mode so the task
// list is self-explanatory without opening individual task detail pages.
profile := strings.TrimSpace(body.Profile)
if profile == "" {
profile = "standard"
}
name := taskDisplayName(target, "", "")
if strings.TrimSpace(body.DisplayName) != "" {
name = body.DisplayName
}
// Append profile tag.
name = fmt.Sprintf("%s · %s", name, profile)
if target == "nvidia-bench-power" && parallelGPUs {
writeError(w, http.StatusBadRequest, "power / thermal fit benchmark uses sequential or ramp-up modes only")
return
}
if rampUp && len(body.GPUIndices) > 1 {
// Ramp-up mode: RunNvidiaPowerBench internally ramps from 1 to N GPUs
// in Phase 2 (one additional GPU per step). A single task with all
// selected GPUs is sufficient — spawning N tasks with growing subsets
// would repeat all earlier steps redundantly.
gpus, err := apiListNvidiaGPUs(h.opts.App)
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
resolved, err := expandSelectedGPUIndices(gpus, body.GPUIndices, body.ExcludeGPUIndices)
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
if len(resolved) < 2 {
// Fall through to normal single-task path.
rampUp = false
} else {
now := time.Now()
rampRunID := fmt.Sprintf("ramp-%s", now.UTC().Format("20060102-150405"))
taskName := fmt.Sprintf("%s · ramp 1–%d · GPU %s", name, len(resolved), formatGPUIndexList(resolved))
t := &Task{
ID: newJobID("bee-bench-nvidia"),
Name: taskName,
Target: target,
Priority: defaultTaskPriority(target, taskParams{}),
Status: TaskPending,
CreatedAt: now,
params: taskParams{
GPUIndices: append([]int(nil), resolved...),
SizeMB: body.SizeMB,
BenchmarkProfile: body.Profile,
RunNCCL: runNCCL,
ParallelGPUs: true,
RampTotal: len(resolved),
RampRunID: rampRunID,
DisplayName: taskName,
},
}
globalQueue.enqueue(t)
writeTaskRunResponse(w, []*Task{t})
return
}
}
// For non-ramp tasks append mode tag.
if parallelGPUs {
name = fmt.Sprintf("%s · parallel", name)
} else {
name = fmt.Sprintf("%s · sequential", name)
}
params := taskParams{
GPUIndices: body.GPUIndices,
ExcludeGPUIndices: body.ExcludeGPUIndices,
SizeMB: body.SizeMB,
BenchmarkProfile: body.Profile,
RunNCCL: runNCCL,
ParallelGPUs: parallelGPUs,
DisplayName: body.DisplayName,
}
tasks, err := buildNvidiaTaskSet(target, defaultTaskPriority(target, params), time.Now(), params, name, h.opts.App, "bee-bench-nvidia")
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
for _, t := range tasks {
globalQueue.enqueue(t)
}
writeTaskRunResponse(w, tasks)
}
}
func (h *handler) handleAPIBenchmarkAutotuneRun() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if h.opts.App == nil {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
}
var body struct {
Profile string `json:"profile"`
BenchmarkKind string `json:"benchmark_kind"`
SizeMB int `json:"size_mb"`
}
if r.Body != nil {
if err := json.NewDecoder(r.Body).Decode(&body); err != nil && !errors.Is(err, io.EOF) {
writeError(w, http.StatusBadRequest, "invalid request body")
return
}
}
profile := strings.TrimSpace(body.Profile)
if profile == "" {
profile = "standard"
}
benchmarkKind := strings.TrimSpace(body.BenchmarkKind)
if benchmarkKind == "" {
benchmarkKind = "power-fit"
}
now := time.Now()
taskName := fmt.Sprintf("NVIDIA Benchmark Autotune · %s · %s", profile, benchmarkKind)
t := &Task{
ID: newJobID("bee-bench-autotune"),
Name: taskName,
Target: "nvidia-bench-autotune",
Priority: defaultTaskPriority("nvidia-bench-autotune", taskParams{}),
Status: TaskPending,
CreatedAt: now,
params: taskParams{
BenchmarkProfile: profile,
BenchmarkKind: benchmarkKind,
SizeMB: body.SizeMB,
DisplayName: taskName,
},
}
globalQueue.enqueue(t)
writeTaskRunResponse(w, []*Task{t})
}
}
func (h *handler) handleAPIBenchmarkAutotuneStatus(w http.ResponseWriter, r *http.Request) {
if h.opts.App == nil {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
}
cfg, err := h.opts.App.LoadBenchmarkPowerAutotune()
if err != nil {
if os.IsNotExist(err) {
w.WriteHeader(http.StatusOK)
writeJSON(w, map[string]any{
"configured": false,
"decision": platform.ResolveSystemPowerDecision(h.opts.ExportDir),
})
return
}
writeError(w, http.StatusInternalServerError, err.Error())
return
}
w.WriteHeader(http.StatusOK)
writeJSON(w, map[string]any{
"configured": true,
"config": cfg,
"decision": platform.ResolveSystemPowerDecision(h.opts.ExportDir),
})
}
func (h *handler) handleAPIBenchmarkNvidiaRun(w http.ResponseWriter, r *http.Request) {
h.handleAPIBenchmarkNvidiaRunKind("nvidia-bench-perf").ServeHTTP(w, r)
}
func (h *handler) handleAPISATStream(w http.ResponseWriter, r *http.Request) {
id := r.URL.Query().Get("job_id")
if id == "" {
id = r.URL.Query().Get("task_id")
}
if j, ok := globalQueue.findJob(id); ok {
streamJob(w, r, j)
return
}
if j, ok := globalJobs.get(id); ok {
streamJob(w, r, j)
return
}
http.Error(w, "job not found", http.StatusNotFound)
}
func (h *handler) handleAPISATAbort(w http.ResponseWriter, r *http.Request) {
id := r.URL.Query().Get("job_id")
if id == "" {
id = r.URL.Query().Get("task_id")
}
if t, ok := globalQueue.findByID(id); ok {
globalQueue.mu.Lock()
switch t.Status {
case TaskPending:
t.Status = TaskCancelled
now := time.Now()
t.DoneAt = &now
case TaskRunning:
if t.job == nil || !t.job.abort() {
globalQueue.mu.Unlock()
writeJSON(w, map[string]string{"status": "not_running"})
return
}
globalQueue.mu.Unlock()
writeJSON(w, map[string]string{"status": "aborting"})
return
}
globalQueue.mu.Unlock()
writeJSON(w, map[string]string{"status": "aborted"})
return
}
if j, ok := globalJobs.get(id); ok {
if j.abort() {
writeJSON(w, map[string]string{"status": "aborted"})
} else {
writeJSON(w, map[string]string{"status": "not_running"})
}
return
}
http.Error(w, "job not found", http.StatusNotFound)
}
// ── Services ──────────────────────────────────────────────────────────────────
audit/internal/webui/api_sat_runall.go
+212
View File
@@ -0,0 +1,212 @@
package webui
import (
"context"
"encoding/json"
"errors"
"io"
"log/slog"
"net/http"
"sort"
"time"
"bee/audit/internal/app"
"bee/audit/internal/platform"
"bee/audit/internal/schema"
)
// gpuReadyWait bounds how long /api/sat/run-all waits for the GPU driver
// stack to come up before it plans the GPU tests. The per-GPU GSP firmware
// boot on a multi-GPU box lags the device nodes by tens of seconds.
// Overridable from tests.
var (
gpuReadyWait = 75 * time.Second
gpuReadyPollInterval = 3 * time.Second
apiRuntimeHealthNow = func(a *app.App) (schema.RuntimeHealth, error) {
return a.RuntimeHealthNow()
}
)
type satRunAllRequest struct {
StressMode bool `json:"stress_mode"`
// AMDTargets is the operator's AMD check selection (intent). It is still
// gated on an AMD GPU actually being present.
AMDTargets []string `json:"amd_targets"`
// NvidiaGPUIndices optionally narrows the NVIDIA tests to a subset; empty
// means "every GPU the backend enumerates".
NvidiaGPUIndices []int `json:"nvidia_gpu_indices"`
}
type satRunAllResponse struct {
TaskIDs []string `json:"task_ids"`
TaskCount int `json:"task_count"`
// Notes carries anything the backend decided to skip or override, so the
// page can show it without reasoning about hardware itself.
Notes []string `json:"notes,omitempty"`
}
// handleAPISATRunAll plans and enqueues the full validate/check task set
// server-side. Hardware presence and readiness are decided here, never in the
// browser: the page sends only operator intent (stress toggle, AMD checkbox
// selection, an optional GPU subset).
func (h *handler) handleAPISATRunAll(w http.ResponseWriter, r *http.Request) {
if h.opts.App == nil {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
}
var req satRunAllRequest
if r.Body != nil {
if err := json.NewDecoder(r.Body).Decode(&req); err != nil && !errors.Is(err, io.EOF) {
writeError(w, http.StatusBadRequest, "invalid request body")
return
}
}
specs, notes := h.planSATRunAll(r.Context(), req)
var ids []string
for _, spec := range specs {
tasks, err := h.enqueueSATTarget(spec.target, spec.params)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
for _, t := range tasks {
if t != nil {
ids = append(ids, t.ID)
}
}
}
slog.Info("sat run-all planned", "tasks", len(ids), "stress", req.StressMode, "notes", len(notes))
writeJSON(w, satRunAllResponse{TaskIDs: ids, TaskCount: len(ids), Notes: notes})
}
type satRunAllSpec struct {
target string
params taskParams
}
func (h *handler) planSATRunAll(ctx context.Context, req satRunAllRequest) ([]satRunAllSpec, []string) {
var specs []satRunAllSpec
var notes []string
skip := func(msg string) {
notes = append(notes, msg)
slog.Warn("sat run-all: check skipped", "reason", msg)
}
cpuDur := 60
if req.StressMode {
cpuDur = 1800
}
specs = append(specs,
satRunAllSpec{target: "cpu", params: taskParams{Duration: cpuDur, StressMode: req.StressMode}},
satRunAllSpec{target: "memory", params: taskParams{StressMode: req.StressMode}},
satRunAllSpec{target: "storage", params: taskParams{StressMode: req.StressMode}},
satRunAllSpec{target: "pcie-link", params: taskParams{}},
)
if h.opts.App.TPMPresent() {
specs = append(specs, satRunAllSpec{target: "tpm", params: taskParams{}})
} else {
skip("TPM: no TPM device on this host; check skipped")
}
gp := h.opts.App.DetectGPUPresence()
if gp.Nvidia || gp.NvidiaInitializing {
// nvidia-config only collects inventory and NVLink state; safe to run
// even while the compute stack is still coming up.
specs = append(specs, satRunAllSpec{target: "nvidia-config", params: taskParams{}})
health, gpus, ready := h.waitForNvidiaReady(ctx)
switch {
case health.NvidiaGSPMode == "gsp-stuck":
skip("NVIDIA: GSP firmware init is stuck; reboot with GSP=off. GPU compute/interconnect/bandwidth tests skipped")
case !ready:
skip("NVIDIA: nvidia-smi did not enumerate a GPU after " + gpuReadyWait.String() +
". GPU compute/interconnect/bandwidth tests skipped; see the GPU Config check")
default:
indices := make([]int, 0, len(gpus))
for _, g := range gpus {
indices = append(indices, g.Index)
}
if len(req.NvidiaGPUIndices) > 0 {
indices = intersectSortedInts(indices, req.NvidiaGPUIndices)
}
if len(indices) == 0 {
skip("NVIDIA: driver ready but no GPU to test (enumeration empty, or the requested subset matched nothing); GPU tests skipped")
break
}
if !health.CUDAReady {
notes = append(notes, "NVIDIA: CUDA runtime not confirmed ready; GPU tests queued anyway")
}
gpuTargets := []string{"nvidia", "nvidia-interconnect", "nvidia-bandwidth", "nvidia-pcie-bandwidth"}
if req.StressMode {
// Stress tier adds the targeted dcgmi diag load tests.
gpuTargets = append(gpuTargets, "nvidia-targeted-stress", "nvidia-targeted-power", "nvidia-pulse")
}
for _, target := range gpuTargets {
specs = append(specs, satRunAllSpec{
target: target,
params: taskParams{GPUIndices: append([]int(nil), indices...), StressMode: req.StressMode},
})
}
}
}
if gp.AMD {
for _, target := range req.AMDTargets {
switch target {
case "amd", "amd-mem", "amd-bandwidth":
specs = append(specs, satRunAllSpec{target: target, params: taskParams{StressMode: req.StressMode}})
}
}
}
return specs, notes
}
// waitForNvidiaReady waits until the same nvidia-smi query used by
// ListNvidiaGPUs returns at least one GPU. A loaded kernel module alone is
// not evidence that NVIDIA user-space commands can address a GPU yet.
func (h *handler) waitForNvidiaReady(ctx context.Context) (schema.RuntimeHealth, []platform.NvidiaGPU, bool) {
deadline := time.Now().Add(gpuReadyWait)
var last schema.RuntimeHealth
for {
health, err := apiRuntimeHealthNow(h.opts.App)
if err == nil {
last = health
if health.NvidiaGSPMode == "gsp-stuck" {
return health, nil, false
}
}
gpus, listErr := apiListNvidiaGPUs(h.opts.App)
if listErr == nil && len(gpus) > 0 {
return last, gpus, true
}
if time.Now().After(deadline) || ctx.Err() != nil {
return last, nil, false
}
select {
case <-ctx.Done():
return last, nil, false
case <-time.After(gpuReadyPollInterval):
}
}
}
// intersectSortedInts returns the ascending-sorted values present in both a
// and b.
func intersectSortedInts(a, b []int) []int {
set := make(map[int]struct{}, len(b))
for _, v := range b {
set[v] = struct{}{}
}
out := make([]int, 0, len(a))
for _, v := range a {
if _, ok := set[v]; ok {
out = append(out, v)
}
}
sort.Ints(out)
return out
}
audit/internal/webui/api_sat_runall_test.go
+90
View File
@@ -0,0 +1,90 @@
package webui
import (
"context"
"reflect"
"testing"
"time"
"bee/audit/internal/app"
"bee/audit/internal/platform"
"bee/audit/internal/schema"
)
func TestIntersectSortedInts(t *testing.T) {
got := intersectSortedInts([]int{0, 1, 2, 3, 4, 5, 6, 7}, []int{5, 1, 9})
if want := []int{1, 5}; !reflect.DeepEqual(got, want) {
t.Fatalf("intersectSortedInts=%v want %v", got, want)
}
if got := intersectSortedInts([]int{0, 1}, []int{9}); len(got) != 0 {
t.Fatalf("want empty, got %v", got)
}
}
func TestWaitForNvidiaReadyDoesNotTreatLoadedDriverAsEnumeration(t *testing.T) {
oldWait, oldPoll := gpuReadyWait, gpuReadyPollInterval
oldHealth, oldList := apiRuntimeHealthNow, apiListNvidiaGPUs
gpuReadyWait, gpuReadyPollInterval = 5*time.Millisecond, time.Millisecond
apiRuntimeHealthNow = func(*app.App) (schema.RuntimeHealth, error) {
return schema.RuntimeHealth{DriverReady: true}, nil
}
apiListNvidiaGPUs = func(*app.App) ([]platform.NvidiaGPU, error) { return nil, nil }
t.Cleanup(func() {
gpuReadyWait, gpuReadyPollInterval = oldWait, oldPoll
apiRuntimeHealthNow, apiListNvidiaGPUs = oldHealth, oldList
})
_, gpus, ready := (&handler{opts: HandlerOptions{App: app.New(&platform.System{})}}).waitForNvidiaReady(context.Background())
if ready || len(gpus) != 0 {
t.Fatalf("ready=%v gpus=%v; loaded module without enumerated GPUs must not be ready", ready, gpus)
}
}
func TestWaitForNvidiaReadyReturnsFreshEnumeration(t *testing.T) {
oldWait, oldPoll := gpuReadyWait, gpuReadyPollInterval
oldHealth, oldList := apiRuntimeHealthNow, apiListNvidiaGPUs
gpuReadyWait, gpuReadyPollInterval = 20*time.Millisecond, time.Millisecond
apiRuntimeHealthNow = func(*app.App) (schema.RuntimeHealth, error) {
return schema.RuntimeHealth{DriverReady: true, CUDAReady: true}, nil
}
calls := 0
apiListNvidiaGPUs = func(*app.App) ([]platform.NvidiaGPU, error) {
calls++
if calls < 2 {
return nil, nil
}
return []platform.NvidiaGPU{{Index: 3}}, nil
}
t.Cleanup(func() {
gpuReadyWait, gpuReadyPollInterval = oldWait, oldPoll
apiRuntimeHealthNow, apiListNvidiaGPUs = oldHealth, oldList
})
health, gpus, ready := (&handler{opts: HandlerOptions{App: app.New(&platform.System{})}}).waitForNvidiaReady(context.Background())
if !ready || !health.CUDAReady || len(gpus) != 1 || gpus[0].Index != 3 {
t.Fatalf("ready=%v health=%+v gpus=%v", ready, health, gpus)
}
}
// On a host with no GPU and no TPM the plan is the base checks plus a note
// that TPM was skipped, and no GPU tasks are invented.
func TestPlanSATRunAllNoAcceleratorNoTPM(t *testing.T) {
oldWait := gpuReadyWait
gpuReadyWait = 10 * time.Millisecond
t.Cleanup(func() { gpuReadyWait = oldWait })
h := &handler{opts: HandlerOptions{App: app.New(&platform.System{})}}
specs, notes := h.planSATRunAll(context.Background(), satRunAllRequest{})
var targets []string
for _, s := range specs {
targets = append(targets, s.target)
}
want := []string{"cpu", "memory", "storage", "pcie-link"}
if !reflect.DeepEqual(targets, want) {
t.Fatalf("targets=%v want %v", targets, want)
}
if len(notes) == 0 {
t.Fatalf("expected a note about TPM being skipped")
}
}
audit/internal/webui/api_services_system.go
+260
View File
@@ -0,0 +1,260 @@
package webui
import (
"encoding/json"
"errors"
"net/http"
"os"
"path/filepath"
"strings"
"bee/audit/internal/app"
"bee/audit/internal/platform"
)
func (h *handler) handleAPIServicesList(w http.ResponseWriter, r *http.Request) {
if h.opts.App == nil {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
}
names, err := h.opts.App.ListBeeServices()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
type serviceInfo struct {
Name string `json:"name"`
State string `json:"state"`
Body string `json:"body"`
}
result := make([]serviceInfo, 0, len(names))
for _, name := range names {
state := h.opts.App.ServiceState(name)
body, _ := h.opts.App.ServiceStatus(name)
result = append(result, serviceInfo{Name: name, State: state, Body: body})
}
writeJSON(w, result)
}
func (h *handler) handleAPIServicesAction(w http.ResponseWriter, r *http.Request) {
if h.opts.App == nil {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
}
var req struct {
Name string `json:"name"`
Action string `json:"action"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid request body")
return
}
var action platform.ServiceAction
switch req.Action {
case "start":
action = platform.ServiceStart
case "stop":
action = platform.ServiceStop
case "restart":
action = platform.ServiceRestart
default:
writeError(w, http.StatusBadRequest, "action must be start|stop|restart")
return
}
result, err := h.opts.App.ServiceActionResult(req.Name, action)
status := "ok"
if err != nil {
status = "error"
}
// Always return 200 with output so the frontend can display the actual
// systemctl error message instead of a generic "exit status 1".
writeJSON(w, map[string]string{"status": status, "output": result.Body})
}
// ── Network ───────────────────────────────────────────────────────────────────
func (h *handler) handleAPINetworkStatus(w http.ResponseWriter, r *http.Request) {
if h.opts.App == nil {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
}
ifaces, err := h.opts.App.ListInterfaces()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, map[string]any{
"interfaces": ifaces,
"default_route": h.opts.App.DefaultRoute(),
"pending_change": h.hasPendingNetworkChange(),
"rollback_in": h.pendingNetworkRollbackIn(),
})
}
func (h *handler) handleAPINetworkDHCP(w http.ResponseWriter, r *http.Request) {
if h.opts.App == nil {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
}
var req struct {
Interface string `json:"interface"`
}
_ = json.NewDecoder(r.Body).Decode(&req)
result, err := h.applyPendingNetworkChange(func() (app.ActionResult, error) {
if req.Interface == "" || req.Interface == "all" {
return h.opts.App.DHCPAllResult()
}
return h.opts.App.DHCPOneResult(req.Interface)
})
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, map[string]any{
"status": "ok",
"output": result.Body,
"rollback_in": int(netRollbackTimeout.Seconds()),
})
}
func (h *handler) handleAPINetworkStatic(w http.ResponseWriter, r *http.Request) {
if h.opts.App == nil {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
}
var req struct {
Interface string `json:"interface"`
Address string `json:"address"`
Prefix string `json:"prefix"`
Gateway string `json:"gateway"`
DNS []string `json:"dns"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid request body")
return
}
cfg := platform.StaticIPv4Config{
Interface: req.Interface,
Address: req.Address,
Prefix: req.Prefix,
Gateway: req.Gateway,
DNS: req.DNS,
}
result, err := h.applyPendingNetworkChange(func() (app.ActionResult, error) {
return h.opts.App.SetStaticIPv4Result(cfg)
})
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, map[string]any{
"status": "ok",
"output": result.Body,
"rollback_in": int(netRollbackTimeout.Seconds()),
})
}
// ── Export ────────────────────────────────────────────────────────────────────
func (h *handler) handleAPIExportList(w http.ResponseWriter, r *http.Request) {
entries, err := listExportFiles(h.opts.ExportDir)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, entries)
}
func (h *handler) handleAPIExportUSBTargets(w http.ResponseWriter, _ *http.Request) {
if h.opts.App == nil {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
}
targets, err := h.opts.App.ListRemovableTargets()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if targets == nil {
targets = []platform.RemovableTarget{}
}
writeJSON(w, targets)
}
func (h *handler) handleAPIBlackboxStatus(w http.ResponseWriter, _ *http.Request) {
state, err := app.ReadBlackboxState(filepath.Join(h.opts.ExportDir, "blackbox-state.json"))
if err != nil {
if errors.Is(err, os.ErrNotExist) {
writeJSON(w, app.BlackboxState{Status: "disabled", Targets: []app.BlackboxTargetStatus{}})
return
}
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if state.Targets == nil {
state.Targets = []app.BlackboxTargetStatus{}
}
writeJSON(w, state)
}
func (h *handler) handleAPIBlackboxEnable(w http.ResponseWriter, r *http.Request) {
if h.opts.App == nil {
writeError(w, http.StatusServiceUnavailable, "app not configured")
return
}
var target platform.RemovableTarget
if err := json.NewDecoder(r.Body).Decode(&target); err != nil || strings.TrimSpace(target.Device) == "" {
writeError(w, http.StatusBadRequest, "device is required")
return
}
targets, err := h.opts.App.ListRemovableTargets()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
allowed := false
for _, candidate := range targets {
if candidate.Device == target.Device {
target = candidate
allowed = true
break
}
}
if !allowed {
writeError(w, http.StatusBadRequest, "device not in removable target list")
return
}
marker, err := app.EnableBlackboxTarget(target)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, map[string]any{
"status": "ok",
"message": "Black-box marker written.",
"enrollment_id": marker.EnrollmentID,
})
}
func (h *handler) handleAPIBlackboxDisable(w http.ResponseWriter, r *http.Request) {
var req struct {
Device string `json:"device"`
EnrollmentID string `json:"enrollment_id"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid request body")
return
}
if err := app.DisableBlackboxTarget(req.Device, req.EnrollmentID); err != nil {
if errors.Is(err, os.ErrNotExist) {
writeError(w, http.StatusNotFound, "black-box target not found")
return
}
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, map[string]string{"status": "ok", "message": "Black-box marker removed."})
}
// ── GPU presence ──────────────────────────────────────────────────────────────
audit/internal/webui/huawei_elabel.go
+12 -12
View File
@@ -33,19 +33,19 @@ type huaweiFieldDef struct {
}
var huaweiElabelDefs = []huaweiFieldDef{
{"Device Name", "DeviceName", 0x00, 0x06, 0x01, ""},
{"Device Serial Number", "DeviceSerialNumber", 0x00, 0x06, 0x03, ""},
{"Product Name", "ProductName", 0x00, 0x03, 0x01, ""},
{"Product Serial Number", "ProductSerialNumber", 0x00, 0x03, 0x04, ""},
{"Product Asset Tag", "ProductAssetTag", 0x00, 0x03, 0x05, ""},
{"Product Manufacturer", "ProductManufacturer", 0x00, 0x03, 0x00, ""},
{"Device Name", "DeviceName", 0x00, 0x06, 0x01, ""},
{"Device Serial Number", "DeviceSerialNumber", 0x00, 0x06, 0x03, ""},
{"Product Name", "ProductName", 0x00, 0x03, 0x01, ""},
{"Product Serial Number", "ProductSerialNumber", 0x00, 0x03, 0x04, ""},
{"Product Asset Tag", "ProductAssetTag", 0x00, 0x03, 0x05, ""},
{"Product Manufacturer", "ProductManufacturer", 0x00, 0x03, 0x00, ""},
{"Mainboard Manufacturer", "MainboardManufacturer", 0x00, 0x02, 0x01, ""},
{"Board Product Name", "BoardProductName", 0x00, 0x02, 0x02, ""},
{"Chassis Part Number", "ChassisPartnumber", 0x00, 0x01, 0x01, ""},
{"Chassis Type", "ChassisType", 0x00, 0x01, 0x00, "chassis-type"},
{"IO Chassis Serial", "IOChassisSerialNumber", 0x01, 0x03, 0x04, ""},
{"IO Chassis Asset Tag", "IOChassisAssetTag", 0x01, 0x03, 0x05, ""},
{"GUID", "GUID", 0x00, 0x00, 0x00, "guid"},
{"Board Product Name", "BoardProductName", 0x00, 0x02, 0x02, ""},
{"Chassis Part Number", "ChassisPartnumber", 0x00, 0x01, 0x01, ""},
{"Chassis Type", "ChassisType", 0x00, 0x01, 0x00, "chassis-type"},
{"IO Chassis Serial", "IOChassisSerialNumber", 0x01, 0x03, 0x04, ""},
{"IO Chassis Asset Tag", "IOChassisAssetTag", 0x01, 0x03, 0x05, ""},
{"GUID", "GUID", 0x00, 0x00, 0x00, "guid"},
}
// huaweiGetRaw reads a string elabel field via OEM IPMI raw command.
audit/internal/webui/layout.go
+1 -1
View File
@@ -100,7 +100,7 @@ tbody tr:hover td{background:rgba(0,0,0,.03)}
func layoutNav(active string, buildLabel string) string {
type navItem struct {
id, label, href string
sep bool
sep bool
}
items := []navItem{
{id: "dashboard", label: "Dashboard", href: "/"},
audit/internal/webui/metrics_charts_http.go
+665
View File
@@ -0,0 +1,665 @@
package webui
import (
"fmt"
"html"
"net/http"
"sort"
"strings"
"time"
"bee/audit/internal/platform"
)
func (h *handler) handleMetricsChartSVG(w http.ResponseWriter, r *http.Request) {
path := strings.TrimPrefix(r.URL.Path, "/api/metrics/chart/")
path = strings.TrimSuffix(path, ".svg")
if h.metricsDB == nil {
http.Error(w, "metrics database not available", http.StatusServiceUnavailable)
return
}
samples, err := h.metricsDB.LoadAll()
if err != nil || len(samples) == 0 {
http.Error(w, "metrics history unavailable", http.StatusServiceUnavailable)
return
}
timeline := metricsTimelineSegments(samples, time.Now())
if idx, sub, ok := parseGPUChartPath(path); ok && sub == "overview" {
var overviewOk bool
var buf []byte
buf, overviewOk, err = renderGPUOverviewChartSVG(idx, samples, timeline)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
if !overviewOk {
http.Error(w, "metrics history unavailable", http.StatusServiceUnavailable)
return
}
w.Header().Set("Content-Type", "image/svg+xml")
w.Header().Set("Cache-Control", "no-store")
_, _ = w.Write(buf)
return
}
datasets, names, labels, title, yMin, yMax, stacked, ok := chartDataFromSamples(path, samples)
if !ok {
http.Error(w, "metrics history unavailable", http.StatusServiceUnavailable)
return
}
var buf []byte
if stacked {
buf, err = renderStackedMetricChartSVG(
title,
labels,
sampleTimes(samples),
datasets,
names,
yMax,
chartCanvasHeightForPath(path, len(names)),
timeline,
)
} else {
buf, err = renderMetricChartSVG(
title,
labels,
sampleTimes(samples),
datasets,
names,
yMin,
yMax,
chartCanvasHeightForPath(path, len(names)),
timeline,
)
}
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "image/svg+xml")
w.Header().Set("Cache-Control", "no-store")
_, _ = w.Write(buf)
}
func chartDataFromSamples(path string, samples []platform.LiveMetricSample) (datasets [][]float64, names []string, labels []string, title string, yMin, yMax *float64, stacked bool, ok bool) {
labels = sampleTimeLabels(samples)
switch {
case path == "server-load":
title = "CPU / Memory Load"
cpu := make([]float64, len(samples))
mem := make([]float64, len(samples))
for i, s := range samples {
cpu[i] = s.CPULoadPct
mem[i] = s.MemLoadPct
}
datasets = [][]float64{cpu, mem}
names = []string{"CPU Load %", "Mem Load %"}
yMin = floatPtr(0)
yMax = floatPtr(100)
case path == "server-temp", path == "server-temp-cpu":
title = "CPU Temperature"
datasets, names = namedTempDatasets(samples, "cpu")
yMin = floatPtr(0)
yMax = autoMax120(datasets...)
case path == "server-temp-gpu":
title = "GPU Temperature"
datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.TempC })
yMin = floatPtr(0)
yMax = autoMax120(datasets...)
case path == "server-temp-ambient":
title = "Ambient / Other Sensors"
datasets, names = namedTempDatasets(samples, "ambient")
yMin = floatPtr(0)
yMax = autoMax120(datasets...)
case path == "server-power":
title = "System Power"
power := make([]float64, len(samples))
label := "Power W"
for i, s := range samples {
power[i] = s.PowerW
if strings.TrimSpace(s.PowerSource) != "" {
label = fmt.Sprintf("Power W · %s", s.PowerSource)
if strings.TrimSpace(s.PowerMode) != "" {
label += fmt.Sprintf(" (%s)", s.PowerMode)
}
}
}
power = normalizePowerSeries(power)
datasets = [][]float64{power}
names = []string{label}
yMin = floatPtr(0)
yMax = autoMax120(power)
case path == "server-fans":
title = "Fan RPM"
datasets, names = namedFanDatasets(samples)
yMin, yMax = autoBounds120(datasets...)
case path == "gpu-all-load":
title = "GPU Compute Load"
datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.UsagePct })
yMin = floatPtr(0)
yMax = floatPtr(100)
case path == "gpu-all-memload":
title = "GPU Memory Load"
datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.MemUsagePct })
yMin = floatPtr(0)
yMax = floatPtr(100)
case path == "gpu-all-power":
title = "GPU Power"
datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.PowerW })
yMin, yMax = autoBounds120(datasets...)
case path == "gpu-all-temp":
title = "GPU Temperature"
datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.TempC })
yMin = floatPtr(0)
yMax = autoMax120(datasets...)
case path == "gpu-all-clock":
title = "GPU Core Clock"
datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.ClockMHz })
yMin, yMax = autoBounds120(datasets...)
case path == "gpu-all-memclock":
title = "GPU Memory Clock"
datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.MemClockMHz })
yMin, yMax = autoBounds120(datasets...)
case strings.HasPrefix(path, "gpu/"):
idx, sub, ok := parseGPUChartPath(path)
if !ok {
return nil, nil, nil, "", nil, nil, false, false
}
switch sub {
case "load":
title = gpuDisplayLabel(idx) + " Load"
util := gpuDatasetByIndex(samples, idx, func(g platform.GPUMetricRow) float64 { return g.UsagePct })
mem := gpuDatasetByIndex(samples, idx, func(g platform.GPUMetricRow) float64 { return g.MemUsagePct })
if util == nil && mem == nil {
return nil, nil, nil, "", nil, nil, false, false
}
datasets = [][]float64{coalesceDataset(util, len(samples)), coalesceDataset(mem, len(samples))}
names = []string{"Load %", "Mem %"}
yMin = floatPtr(0)
yMax = floatPtr(100)
case "temp":
title = gpuDisplayLabel(idx) + " Temperature"
temp := gpuDatasetByIndex(samples, idx, func(g platform.GPUMetricRow) float64 { return g.TempC })
if temp == nil {
return nil, nil, nil, "", nil, nil, false, false
}
datasets = [][]float64{temp}
names = []string{"Temp °C"}
yMin = floatPtr(0)
yMax = autoMax120(temp)
case "clock":
title = gpuDisplayLabel(idx) + " Core Clock"
clock := gpuDatasetByIndex(samples, idx, func(g platform.GPUMetricRow) float64 { return g.ClockMHz })
if clock == nil {
return nil, nil, nil, "", nil, nil, false, false
}
datasets = [][]float64{clock}
names = []string{"Core Clock MHz"}
yMin, yMax = autoBounds120(clock)
case "memclock":
title = gpuDisplayLabel(idx) + " Memory Clock"
clock := gpuDatasetByIndex(samples, idx, func(g platform.GPUMetricRow) float64 { return g.MemClockMHz })
if clock == nil {
return nil, nil, nil, "", nil, nil, false, false
}
datasets = [][]float64{clock}
names = []string{"Memory Clock MHz"}
yMin, yMax = autoBounds120(clock)
default:
title = gpuDisplayLabel(idx) + " Power"
power := gpuDatasetByIndex(samples, idx, func(g platform.GPUMetricRow) float64 { return g.PowerW })
if power == nil {
return nil, nil, nil, "", nil, nil, false, false
}
datasets = [][]float64{power}
names = []string{"Power W"}
yMin, yMax = autoBounds120(power)
}
default:
return nil, nil, nil, "", nil, nil, false, false
}
return datasets, names, labels, title, yMin, yMax, stacked, len(datasets) > 0
}
func parseGPUChartPath(path string) (idx int, sub string, ok bool) {
if !strings.HasPrefix(path, "gpu/") {
return 0, "", false
}
rest := strings.TrimPrefix(path, "gpu/")
if rest == "" {
return 0, "", false
}
sub = ""
if i := strings.LastIndex(rest, "-"); i > 0 {
sub = rest[i+1:]
rest = rest[:i]
}
n, err := fmt.Sscanf(rest, "%d", &idx)
if err != nil || n != 1 {
return 0, "", false
}
return idx, sub, true
}
func sampleTimeLabels(samples []platform.LiveMetricSample) []string {
labels := make([]string, len(samples))
if len(samples) == 0 {
return labels
}
times := make([]time.Time, len(samples))
for i, s := range samples {
times[i] = s.Timestamp
}
sameDay := timestampsSameLocalDay(times)
for i, s := range samples {
labels[i] = formatTimelineLabel(s.Timestamp.Local(), sameDay)
}
return labels
}
func namedTempDatasets(samples []platform.LiveMetricSample, group string) ([][]float64, []string) {
seen := map[string]bool{}
var names []string
for _, s := range samples {
for _, t := range s.Temps {
if t.Group == group && !seen[t.Name] {
seen[t.Name] = true
names = append(names, t.Name)
}
}
}
sort.Strings(names)
datasets := make([][]float64, 0, len(names))
for _, name := range names {
ds := make([]float64, len(samples))
for i, s := range samples {
for _, t := range s.Temps {
if t.Group == group && t.Name == name {
ds[i] = t.Celsius
break
}
}
}
datasets = append(datasets, ds)
}
return datasets, names
}
func namedFanDatasets(samples []platform.LiveMetricSample) ([][]float64, []string) {
seen := map[string]bool{}
var names []string
for _, s := range samples {
for _, f := range s.Fans {
if !seen[f.Name] {
seen[f.Name] = true
names = append(names, f.Name)
}
}
}
sort.Strings(names)
datasets := make([][]float64, 0, len(names))
for _, name := range names {
ds := make([]float64, len(samples))
for i, s := range samples {
for _, f := range s.Fans {
if f.Name == name {
ds[i] = f.RPM
break
}
}
}
datasets = append(datasets, normalizeFanSeries(ds))
}
return datasets, names
}
func gpuDatasets(samples []platform.LiveMetricSample, pick func(platform.GPUMetricRow) float64) ([][]float64, []string) {
seen := map[int]bool{}
var indices []int
for _, s := range samples {
for _, g := range s.GPUs {
if !seen[g.GPUIndex] {
seen[g.GPUIndex] = true
indices = append(indices, g.GPUIndex)
}
}
}
sort.Ints(indices)
datasets := make([][]float64, 0, len(indices))
names := make([]string, 0, len(indices))
for _, idx := range indices {
ds := gpuDatasetByIndex(samples, idx, pick)
if ds == nil {
continue
}
datasets = append(datasets, ds)
names = append(names, gpuDisplayLabel(idx))
}
return datasets, names
}
func gpuDatasetByIndex(samples []platform.LiveMetricSample, idx int, pick func(platform.GPUMetricRow) float64) []float64 {
found := false
ds := make([]float64, len(samples))
for i, s := range samples {
for _, g := range s.GPUs {
if g.GPUIndex == idx {
ds[i] = pick(g)
found = true
break
}
}
}
if !found {
return nil
}
return ds
}
func coalesceDataset(ds []float64, n int) []float64 {
if ds != nil {
return ds
}
return make([]float64, n)
}
func normalizePowerSeries(ds []float64) []float64 {
if len(ds) == 0 {
return nil
}
out := make([]float64, len(ds))
copy(out, ds)
last := 0.0
haveLast := false
for i, v := range out {
if v > 0 {
last = v
haveLast = true
continue
}
if haveLast {
out[i] = last
}
}
return out
}
// psuSlotsFromSamples returns the sorted list of PSU slot numbers seen across samples.
func psuSlotsFromSamples(samples []platform.LiveMetricSample) []int {
seen := map[int]struct{}{}
for _, s := range samples {
for _, p := range s.PSUs {
seen[p.Slot] = struct{}{}
}
}
slots := make([]int, 0, len(seen))
for s := range seen {
slots = append(slots, s)
}
sort.Ints(slots)
return slots
}
// psuStackedTotal returns the point-by-point sum of all PSU datasets (for scale calculation).
func psuStackedTotal(datasets [][]float64) []float64 {
if len(datasets) == 0 {
return nil
}
n := len(datasets[0])
total := make([]float64, n)
for _, ds := range datasets {
for i, v := range ds {
total[i] += v
}
}
return total
}
func normalizeFanSeries(ds []float64) []float64 {
if len(ds) == 0 {
return nil
}
out := make([]float64, len(ds))
var lastPositive float64
for i, v := range ds {
if v > 0 {
lastPositive = v
out[i] = v
continue
}
if lastPositive > 0 {
out[i] = lastPositive
continue
}
out[i] = 0
}
return out
}
// floatPtr returns a pointer to a float64 value.
func floatPtr(v float64) *float64 { return &v }
// autoMax120 returns 0→max+20% Y-axis max across all datasets.
func autoMax120(datasets ...[]float64) *float64 {
max := 0.0
for _, ds := range datasets {
for _, v := range ds {
if v > max {
max = v
}
}
}
if max == 0 {
return nil // let library auto-scale
}
v := max * 1.2
return &v
}
func autoBounds120(datasets ...[]float64) (*float64, *float64) {
min := 0.0
max := 0.0
first := true
for _, ds := range datasets {
for _, v := range ds {
if first {
min, max = v, v
first = false
continue
}
if v < min {
min = v
}
if v > max {
max = v
}
}
}
if first {
return nil, nil
}
if max <= 0 {
return floatPtr(0), nil
}
span := max - min
if span <= 0 {
span = max * 0.1
if span <= 0 {
span = 1
}
}
pad := span * 0.2
low := min - pad
if low < 0 {
low = 0
}
high := max + pad
return floatPtr(low), floatPtr(high)
}
func gpuChartLabelIndices(total, target int) []int {
if total <= 0 {
return nil
}
if total == 1 {
return []int{0}
}
step := total / target
if step < 1 {
step = 1
}
var indices []int
for i := 0; i < total; i += step {
indices = append(indices, i)
}
if indices[len(indices)-1] != total-1 {
indices = append(indices, total-1)
}
return indices
}
func chartCanvasHeightForPath(path string, seriesCount int) int {
height := chartCanvasHeight(seriesCount)
if isGPUChartPath(path) {
return height * 2
}
return height
}
func isGPUChartPath(path string) bool {
return strings.HasPrefix(path, "gpu-all-") || strings.HasPrefix(path, "gpu/")
}
func chartLegendVisible(seriesCount int) bool {
return seriesCount <= 8
}
func chartCanvasHeight(seriesCount int) int {
if chartLegendVisible(seriesCount) {
return 360
}
return 288
}
// globalStats returns min, average, and max across all values in all datasets.
func globalStats(datasets [][]float64) (mn, avg, mx float64) {
var sum float64
var count int
first := true
for _, ds := range datasets {
for _, v := range ds {
if first {
mn, mx = v, v
first = false
}
if v < mn {
mn = v
}
if v > mx {
mx = v
}
sum += v
count++
}
}
if count > 0 {
avg = sum / float64(count)
}
return mn, avg, mx
}
func sanitizeChartText(s string) string {
if s == "" {
return ""
}
return html.EscapeString(strings.Map(func(r rune) rune {
if r < 0x20 && r != '\t' && r != '\n' && r != '\r' {
return -1
}
return r
}, s))
}
func snapshotFanRings(rings []*metricsRing, fanNames []string) ([][]float64, []string, []string) {
var datasets [][]float64
var names []string
var labels []string
for i, ring := range rings {
if ring == nil {
continue
}
vals, l := ring.snapshot()
datasets = append(datasets, normalizeFanSeries(vals))
name := "Fan"
if i < len(fanNames) {
name = fanNames[i]
}
names = append(names, name+" RPM")
if len(labels) == 0 {
labels = l
}
}
return datasets, names, labels
}
func chartLegendNumber(v float64) string {
neg := v < 0
if v < 0 {
v = -v
}
var out string
switch {
case v >= 10000:
out = fmt.Sprintf("%dk", int((v+500)/1000))
case v >= 1000:
s := fmt.Sprintf("%.2f", v/1000)
s = strings.TrimRight(strings.TrimRight(s, "0"), ".")
out = strings.ReplaceAll(s, ".", ",") + "k"
default:
out = fmt.Sprintf("%.0f", v)
}
if neg {
return "-" + out
}
return out
}
func chartYAxisNumber(v float64) string {
neg := v < 0
if neg {
v = -v
}
var out string
switch {
case v >= 10000:
out = fmt.Sprintf("%dк", int((v+500)/1000))
case v >= 1000:
// Use one decimal place so ticks like 1400, 1600, 1800 read as
// "1,4к", "1,6к", "1,8к" instead of the ambiguous "1к"/"2к".
s := fmt.Sprintf("%.1f", v/1000)
s = strings.TrimRight(strings.TrimRight(s, "0"), ".")
out = strings.ReplaceAll(s, ".", ",") + "к"
default:
out = fmt.Sprintf("%.0f", v)
}
if neg {
return "-" + out
}
return out
}
audit/internal/webui/metricsdb.go
-4
View File
@@ -455,7 +455,3 @@ func (m *MetricsDB) ExportCSV(w io.Writer) error {
cw.Flush()
return cw.Error()
}
func nullFloat(v float64) sql.NullFloat64 {
return sql.NullFloat64{Float64: v, Valid: true}
}
audit/internal/webui/page_benchmark.go
+1 -1
View File
@@ -613,5 +613,5 @@ func renderPowerBenchmarkResultsCard(exportDir string) string {
}
// renderSpeed and renderEndurance are legacy wrappers; canonical page is 5. Benchmark at /benchmark.
func renderSpeed(opts HandlerOptions) string { return renderBenchmark(opts) }
func renderSpeed(opts HandlerOptions) string { return renderBenchmark(opts) }
func renderEndurance(opts HandlerOptions) string { return renderBenchmark(opts) }
audit/internal/webui/page_topo.go
+1 -770
View File
@@ -11,7 +11,6 @@ import (
"strconv"
"strings"
"bee/audit/internal/app"
"bee/audit/internal/schema"
)
@@ -201,6 +200,7 @@ func parseDIMMBankLocatorNodes(raw string) map[string]int {
// 2. A node number from the Bank Locator via parseDIMMBankLocatorNodes,
// e.g. Locator "DIMM000(A)" whose Bank Locator is
// "_Node1_Channel0_Dimm0" -> 1.
//
// ok=false means neither pattern matched, so this DIMM can't be confidently
// attached to a CPU column and falls back to the unattached Memory row.
func dimmRawNode(mem schema.HardwareMemory, bankNodeByLocator map[string]int) (int, bool) {
@@ -582,772 +582,3 @@ type topoEdge struct {
x1, y1, x2, y2 int
color string
}
func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string {
// A GPU hardware fault that needs a physical reboot (Xid 79 "fallen off
// the bus", Xid 154 "Node Reboot Required" — see gpuNeedsPhysicalReboot)
// isn't visible in dev.Status: the collector only sets that from PCIe
// link-speed checks, not from SAT/kmsg results. Without this, a GPU that
// dropped off the bus mid-test still renders green here even though the
// Hardware Summary card is showing a critical banner for it.
gpuHardwareFault := false
if db, err := app.OpenComponentStatusDB(filepath.Join(exportDir, "component-status.json")); err == nil {
if _, needsReboot := gpuNeedsPhysicalReboot(matchedRecords(db.All(), nil, []string{"pcie:gpu:"})); needsReboot {
gpuHardwareFault = true
}
}
socketIdx := buildSocketIndex(hw.CPUs)
numCols := len(hw.CPUs)
if numCols == 0 {
numCols = 1
}
unknownCol := numCols // extra trailing column for unmatched devices
// Group PCIe devices (GPU/NIC/RAID only — matches the mockup's node
// types) into columns by NUMA node, falling back to the "unknown" bucket.
type placedDevice struct {
dev schema.HardwarePCIeDevice
kind string // "gpu", "nic", "raid"
col int
bdf string
}
var placed []placedDevice
for _, dev := range hw.PCIeDevices {
kind := pcieDeviceKind(dev)
if kind == "" {
continue
}
col := unknownCol
if dev.NUMANode != nil {
if ci, ok := socketIdx[*dev.NUMANode]; ok {
col = ci
}
}
bdf := ""
if dev.Slot != nil {
bdf = normalizeTopoBDF(*dev.Slot)
} else if dev.BDF != nil {
bdf = normalizeTopoBDF(*dev.BDF)
}
placed = append(placed, placedDevice{dev: dev, kind: kind, col: col, bdf: bdf})
}
hasUnknownCol := false
for _, p := range placed {
if p.col == unknownCol {
hasUnknownCol = true
break
}
}
totalCols := numCols
if hasUnknownCol {
totalCols++
}
// GPU index<->BDF map + pairwise NVLink adjacency, read from the
// persisted techdump captured during the last audit cycle, best-effort:
// if the dump is missing (older audit, no NVIDIA GPUs), this is simply
// skipped. Used only to detect the cross-NUMA-bonded-pair anomaly below;
// the pairwise links themselves are drawn in the separate NVLink
// Topology card, since grouping same-kind/same-column devices into one
// stacked card here leaves no single per-GPU anchor point to draw a
// pairwise connector to or from.
bdfToIndex, _ := readNVIDIAIndexByBDF(exportDir)
var pairs []gpuPairLink
if topoMatrix, err := readGPUTopologyMatrix(exportDir); err == nil {
pairs = parseGPUPairAdjacency(topoMatrix)
}
gpuNUMAByIndex := map[int]*int{}
gpuBDFByIndex := map[int]string{}
for _, p := range placed {
if p.kind != "gpu" || p.bdf == "" {
continue
}
if idx, ok := bdfToIndex[p.bdf]; ok {
gpuNUMAByIndex[idx] = p.dev.NUMANode
gpuBDFByIndex[idx] = p.bdf
}
}
// A bonded pair spanning two different NUMA nodes is treated as an
// anomaly (not a neutral fact) per project decision: a bonded pair is
// expected to sit on one NUMA node, so a cross-NUMA bond escalates both
// GPUs' effective severity to at least Warning, regardless of their own
// reported SAT status.
crossNUMAWarnBDF := map[string]bool{}
for _, pair := range pairs {
numaA, okA := gpuNUMAByIndex[pair.GPUA]
numaB, okB := gpuNUMAByIndex[pair.GPUB]
if !okA || !okB || numaA == nil || numaB == nil || *numaA == *numaB {
continue
}
crossNUMAWarnBDF[gpuBDFByIndex[pair.GPUA]] = true
crossNUMAWarnBDF[gpuBDFByIndex[pair.GPUB]] = true
}
kindOrder := []string{"gpu", "nic", "raid"}
kindLabel := map[string]string{"gpu": "GPU", "nic": "NIC", "raid": "RAID"}
// Attach memory DIMMs to their CPU column too, the same way GPU/NIC/RAID
// PCIe devices are attached via NUMANode — memory has no NUMANode field
// in the schema, so this reads the DIMM's own Locator/Bank Locator
// strings instead (see dimmRawNode). DIMMs that can't be confidently
// attached fall back to the unattached "Memory" row below the diagram,
// same as before this existed.
memBankNodes := map[string]int{}
if raw, err := readTopoTechDump(exportDir, "dmidecode-type17.txt"); err == nil {
memBankNodes = parseDIMMBankLocatorNodes(raw)
}
memCol := make([]int, len(hw.Memory))
memMatched := make([]bool, len(hw.Memory))
var memRawNodes []int
for i, m := range hw.Memory {
if node, ok := dimmRawNode(m, memBankNodes); ok {
memCol[i] = node
memMatched[i] = true
memRawNodes = append(memRawNodes, node)
}
}
memColIdx := buildMemoryColumnIndex(memRawNodes)
for i := range hw.Memory {
if !memMatched[i] {
continue
}
col := memColIdx[memCol[i]]
if col >= numCols {
memMatched[i] = false
continue
}
memCol[i] = col
}
var boxes []topoBox
var pcieEdges []topoEdge
for col := 0; col < totalCols; col++ {
colX := (col+1)*24 + col*topoColWidth
if col < len(hw.CPUs) {
cpu := hw.CPUs[col]
model := ""
if cpu.Model != nil {
model = *cpu.Model
}
socket := col
if cpu.Socket != nil {
socket = *cpu.Socket
}
var tally topoStatusTally
tally.add(classifyTopoSeverity(cpu.Status))
fill, stroke, text := topoSeverityColors(tally.worst())
boxes = append(boxes, topoBox{
x: colX, y: topoTopMargin, w: topoBoxWidth, h: topoBoxHeight,
topoCardInfo: topoCardInfo{
label: fmt.Sprintf("CPU %d", socket), sublabel: model, count: 1,
statusLine: tally.line(),
fillVar: fill, strokeVar: stroke, textVar: text,
detailType: "cpu",
},
})
}
y := topoTopMargin + topoBoxHeight + topoDeviceGap*2
// Memory goes first in the chain, directly under the CPU box: DIMMs
// are wired straight to the socket's memory controller, not reached
// over PCIe like the GPU/NIC/RAID chain below it.
var memGroup []schema.HardwareMemory
for i, m := range hw.Memory {
if memMatched[i] && memCol[i] == col {
memGroup = append(memGroup, m)
}
}
if len(memGroup) > 0 {
var tally topoStatusTally
sizeGB := 0
for _, m := range memGroup {
tally.add(classifyTopoSeverity(m.Status))
if m.SizeMB != nil {
sizeGB += *m.SizeMB / 1024
}
}
fill, stroke, text := topoSeverityColors(tally.worst())
sublabel := ""
if sizeGB > 0 {
sublabel = fmt.Sprintf("%d GB total", sizeGB)
}
stackLayers := topoStackLayers(len(memGroup))
boxes = append(boxes, topoBox{
x: colX, y: y, w: topoBoxWidth, h: topoBoxHeight,
topoCardInfo: topoCardInfo{
label: "Memory", sublabel: sublabel, count: len(memGroup),
statusLine: tally.line(),
fillVar: fill, strokeVar: stroke, textVar: text,
detailType: "memory",
},
})
if col < len(hw.CPUs) {
pcieEdges = append(pcieEdges, topoEdge{
x1: colX + topoBoxWidth/2, y1: topoTopMargin + topoBoxHeight,
x2: colX + topoBoxWidth/2, y2: y,
color: "var(--ok-fg)",
})
}
y += topoBoxHeight + topoDeviceGap + stackLayers*topoStackStep
}
for _, kind := range kindOrder {
var group []placedDevice
for _, p := range placed {
if p.col == col && p.kind == kind {
group = append(group, p)
}
}
if len(group) == 0 {
continue
}
var tally topoStatusTally
model := ""
edgeColor := "var(--ok-fg)"
for i, p := range group {
sev := classifyTopoSeverity(p.dev.Status)
if kind == "gpu" && crossNUMAWarnBDF[p.bdf] && sev < 2 {
sev = 2
}
if kind == "gpu" && gpuHardwareFault && sev < 3 {
sev = 3
}
tally.add(sev)
if i == 0 && p.dev.Model != nil {
model = *p.dev.Model
}
if topoEdgeColorVar(p.dev) == "var(--warn-fg)" {
edgeColor = "var(--warn-fg)"
}
}
fill, stroke, text := topoSeverityColors(tally.worst())
stackLayers := topoStackLayers(len(group))
boxes = append(boxes, topoBox{
x: colX, y: y, w: topoBoxWidth, h: topoBoxHeight,
topoCardInfo: topoCardInfo{
label: kindLabel[kind], sublabel: model, count: len(group),
statusLine: tally.line(),
fillVar: fill, strokeVar: stroke, textVar: text,
detailType: kind,
},
})
if col < len(hw.CPUs) {
pcieEdges = append(pcieEdges, topoEdge{
x1: colX + topoBoxWidth/2, y1: topoTopMargin + topoBoxHeight,
x2: colX + topoBoxWidth/2, y2: y,
color: edgeColor,
})
}
y += topoBoxHeight + topoDeviceGap + stackLayers*topoStackStep
}
}
maxDeviceY := topoTopMargin + topoBoxHeight + topoDeviceGap*2
for _, box := range boxes {
bottom := box.y + box.h + topoStackLayers(box.count)*topoStackStep
if bottom > maxDeviceY {
maxDeviceY = bottom
}
}
svgHeight := maxDeviceY + 24
svgWidth := totalCols*topoColWidth + 48
var b strings.Builder
// Wrapped in its own horizontally-scrolling container (matching the
// overflow-x:auto convention used for wide tables elsewhere in webui)
// rather than max-width:100% — squashing a node/edge diagram to fit a
// narrow viewport makes labels and badges illegible, whereas scrolling
// keeps the diagram readable at its natural size on any screen width.
b.WriteString(`<div style="overflow-x:auto">`)
fmt.Fprintf(&b, `<svg width="%d" height="%d" viewBox="0 0 %d %d">`+"\n", svgWidth, svgHeight, svgWidth, svgHeight)
for _, e := range pcieEdges {
fmt.Fprintf(&b, `<line x1="%d" y1="%d" x2="%d" y2="%d" style="stroke:%s;stroke-width:2"/>`+"\n", e.x1, e.y1, e.x2, e.y2, e.color)
}
for _, box := range boxes {
writeTopoBoxSVG(&b, box)
}
b.WriteString(`</svg></div>`)
// Firmware (BMC/BIOS/...) and PSUs have no PCIe/CPU affinity to anchor
// them to a column, and there can be an arbitrary number of any of them
// — so unlike the diagram above, they're plain flex-wrap HTML below the
// SVG rather than absolutely-positioned SVG boxes. A fixed-size SVG
// canvas has no way to wrap overflow onto a new row, which is exactly
// what caused these to pile up and overlap once a board had more
// PSUs/firmware records than fit in one fixed-width row.
//
// Memory DIMMs that were matched to a CPU column above already got a
// box in the SVG diagram; only DIMMs that couldn't be attached to a
// column (see memMatched above) fall back to this row.
var unmatchedMem []schema.HardwareMemory
for i, m := range hw.Memory {
if !memMatched[i] {
unmatchedMem = append(unmatchedMem, m)
}
}
if len(unmatchedMem) > 0 {
var tally topoStatusTally
for _, m := range unmatchedMem {
tally.add(classifyTopoSeverity(m.Status))
}
fill, stroke, text := topoSeverityColors(tally.worst())
sizeGB := 0
for _, m := range unmatchedMem {
if m.SizeMB != nil {
sizeGB += *m.SizeMB / 1024
}
}
sublabel := ""
if sizeGB > 0 {
sublabel = fmt.Sprintf("%d GB total", sizeGB)
}
b.WriteString(renderTopoFlexRow("Memory", []topoCardInfo{{
label: "Memory", sublabel: sublabel, count: len(unmatchedMem),
statusLine: tally.line(),
fillVar: fill, strokeVar: stroke, textVar: text,
detailType: "memory",
}}))
}
var firmwareItems []topoCardInfo
for _, rec := range hw.Firmware {
// Firmware records carry no per-item status in the schema (they are
// identity, not health, facts), so each stays a neutral, uncolored
// card rather than forcing a fake "Unknown" status line.
fillVar, strokeVar, textVar := topoSeverityColors(0)
firmwareItems = append(firmwareItems, topoCardInfo{
label: rec.DeviceName, sublabel: "fw " + rec.Version, count: 1,
fillVar: fillVar, strokeVar: strokeVar, textVar: textVar,
})
}
b.WriteString(renderTopoFlexRow("Firmware", firmwareItems))
if len(hw.PowerSupplies) > 0 {
var tally topoStatusTally
watt := 0
for _, psu := range hw.PowerSupplies {
tally.add(classifyTopoSeverity(psu.Status))
if psu.WattageW != nil {
watt = *psu.WattageW
}
}
fill, stroke, text := topoSeverityColors(tally.worst())
sublabel := ""
if watt > 0 {
sublabel = fmt.Sprintf("%dW each", watt)
}
b.WriteString(renderTopoFlexRow("Power Supplies", []topoCardInfo{{
label: "Power Supplies", sublabel: sublabel, count: len(hw.PowerSupplies),
statusLine: tally.line(),
fillVar: fill, strokeVar: stroke, textVar: text,
detailType: "psu",
}}))
}
return topoCard("Topology", b.String())
}
// renderTopoFlexRow renders a labeled, wrapping row of component cards.
// Returns "" if items is empty (e.g. no PSU data in this audit).
func renderTopoFlexRow(title string, items []topoCardInfo) string {
if len(items) == 0 {
return ""
}
var b strings.Builder
fmt.Fprintf(&b, `<div style="font-size:11px;color:var(--muted);text-transform:uppercase;letter-spacing:.05em;margin:16px 0 6px">%s</div>`,
html.EscapeString(title))
b.WriteString(`<div style="display:flex;flex-wrap:wrap;gap:10px">`)
for _, item := range items {
onclick := ""
cursor := "default"
if item.detailType != "" {
onclick = fmt.Sprintf(` onclick="openComponentDetail('%s')"`, item.detailType)
cursor = "pointer"
}
stackLayers := topoStackLayers(item.count)
// Extra right/bottom padding on the wrapper reserves room for the
// backing layers of the stack effect so they aren't clipped by the
// flex container.
fmt.Fprintf(&b, `<div style="position:relative;padding-right:%dpx;padding-bottom:%dpx">`,
stackLayers*topoStackStep, stackLayers*topoStackStep)
for i := stackLayers; i >= 1; i-- {
off := i * topoStackStep
fmt.Fprintf(&b, `<div style="position:absolute;top:%dpx;left:%dpx;right:0;bottom:0;border-radius:6px;background:%s;border:1px solid %s;opacity:.55"></div>`,
off, off, item.fillVar, item.strokeVar)
}
fmt.Fprintf(&b, `<div%s style="position:relative;cursor:%s;min-width:160px;padding:10px 12px;border-radius:6px;background:%s;border:1px solid %s;color:%s">`,
onclick, cursor, item.fillVar, item.strokeVar, item.textVar)
label := item.label
if item.count > 1 {
label = fmt.Sprintf("%s ×%d", item.label, item.count)
}
fmt.Fprintf(&b, `<div style="font-size:13px;font-weight:700">%s</div>`, html.EscapeString(label))
if item.sublabel != "" {
fmt.Fprintf(&b, `<div style="font-size:11px;opacity:.85">%s</div>`, html.EscapeString(item.sublabel))
}
if item.statusLine != "" {
fmt.Fprintf(&b, `<div style="font-size:11px;font-weight:600;margin-top:4px">%s</div>`, html.EscapeString(item.statusLine))
}
b.WriteString(`</div></div>`)
}
b.WriteString(`</div>`)
return b.String()
}
func writeTopoBoxSVG(b *strings.Builder, box topoBox) {
onclick := ""
cursor := "default"
if box.detailType != "" {
onclick = fmt.Sprintf(` onclick="openComponentDetail('%s')"`, box.detailType)
cursor = "pointer"
}
fmt.Fprintf(b, `<g%s style="cursor:%s">`, onclick, cursor)
// Stack-of-cards effect: faint offset rects behind the front card when
// this box represents more than one physical component (e.g. 4 GPUs in
// one NUMA column), so a group reads as "a deck of N" rather than a
// single item. Peeks toward the bottom-right, into space already
// reserved between this box and the next one in the column.
for i := topoStackLayers(box.count); i >= 1; i-- {
off := i * topoStackStep
fmt.Fprintf(b, `<rect x="%d" y="%d" width="%d" height="%d" rx="6" ry="6" style="fill:%s;stroke:%s;opacity:.55"/>`+"\n",
box.x+off, box.y+off, box.w, box.h, box.fillVar, box.strokeVar)
}
fmt.Fprintf(b, `<rect x="%d" y="%d" width="%d" height="%d" rx="6" ry="6" style="fill:%s;stroke:%s"/>`+"\n",
box.x, box.y, box.w, box.h, box.fillVar, box.strokeVar)
label := box.label
if box.count > 1 {
label = fmt.Sprintf("%s ×%d", box.label, box.count)
}
fmt.Fprintf(b, `<text x="%d" y="%d" style="fill:%s;font-size:13px;font-weight:700">%s</text>`+"\n",
box.x+10, box.y+20, box.textVar, html.EscapeString(label))
if box.sublabel != "" {
fmt.Fprintf(b, `<text x="%d" y="%d" style="fill:%s;font-size:11px;opacity:.85">%s</text>`+"\n",
box.x+10, box.y+36, box.textVar, html.EscapeString(truncateTopoLabel(box.sublabel, 26)))
}
if box.statusLine != "" {
fmt.Fprintf(b, `<text x="%d" y="%d" style="fill:%s;font-size:10px;font-weight:600">%s</text>`+"\n",
box.x+10, box.y+box.h-10, box.textVar, html.EscapeString(box.statusLine))
}
b.WriteString(`</g>` + "\n")
}
func truncateTopoLabel(s string, max int) string {
if len(s) <= max {
return s
}
if max <= 1 {
return s[:max]
}
return s[:max-1] + "…"
}
// ---------------------------------------------------------------------------
// Separate NVLink topology card (read from techdump, not written to any
// ingest contract)
// ---------------------------------------------------------------------------
type topoNVLinkPort struct {
Index int
Active bool
SpeedGBs *float64
ReplayErrors int64
RecoveryErrors int64
CRCErrors int64
}
var (
topoNVLinkGPUHeaderRe = regexp.MustCompile(`^GPU (\d+):`)
topoNVLinkSpeedLineRe = regexp.MustCompile(`^Link (\d+):\s*([\d.]+)\s*GB/s`)
topoNVLinkInactiveRe = regexp.MustCompile(`^Link (\d+):\s*<inactive>`)
topoNVLinkErrorLineRe = regexp.MustCompile(`^Link (\d+):\s*(Replay|Recovery|CRC) Errors:\s*(\d+)`)
)
func readTopoNVLinkStatus(exportDir string) (map[int][]topoNVLinkPort, error) {
raw, err := readTopoTechDump(exportDir, "nvidia-smi-nvlink-status.txt")
if err != nil {
return nil, err
}
return parseTopoNVLinkStatus(raw), nil
}
func parseTopoNVLinkStatus(raw string) map[int][]topoNVLinkPort {
result := map[int][]topoNVLinkPort{}
currentGPU := -1
for _, line := range strings.Split(raw, "\n") {
trimmed := strings.TrimSpace(line)
if m := topoNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
currentGPU, _ = strconv.Atoi(m[1])
continue
}
if currentGPU < 0 {
continue
}
if m := topoNVLinkInactiveRe.FindStringSubmatch(trimmed); m != nil {
idx, _ := strconv.Atoi(m[1])
result[currentGPU] = append(result[currentGPU], topoNVLinkPort{Index: idx, Active: false})
continue
}
if m := topoNVLinkSpeedLineRe.FindStringSubmatch(trimmed); m != nil {
idx, _ := strconv.Atoi(m[1])
port := topoNVLinkPort{Index: idx, Active: true}
if speed, err := strconv.ParseFloat(m[2], 64); err == nil {
port.SpeedGBs = &speed
}
result[currentGPU] = append(result[currentGPU], port)
}
}
return result
}
func readTopoNVLinkErrors(exportDir string) (map[int]map[int][3]int64, error) {
raw, err := readTopoTechDump(exportDir, "nvidia-smi-nvlink-errors.txt")
if err != nil {
return nil, err
}
return parseTopoNVLinkErrors(raw), nil
}
// parseTopoNVLinkErrors returns, per GPU then link index, [replay, recovery, crc].
func parseTopoNVLinkErrors(raw string) map[int]map[int][3]int64 {
result := map[int]map[int][3]int64{}
currentGPU := -1
for _, line := range strings.Split(raw, "\n") {
trimmed := strings.TrimSpace(line)
if m := topoNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
currentGPU, _ = strconv.Atoi(m[1])
continue
}
if currentGPU < 0 {
continue
}
m := topoNVLinkErrorLineRe.FindStringSubmatch(trimmed)
if m == nil {
continue
}
linkIdx, _ := strconv.Atoi(m[1])
count, _ := strconv.ParseInt(m[3], 10, 64)
if result[currentGPU] == nil {
result[currentGPU] = map[int][3]int64{}
}
c := result[currentGPU][linkIdx]
switch m[2] {
case "Replay":
c[0] = count
case "Recovery":
c[1] = count
case "CRC":
c[2] = count
}
result[currentGPU][linkIdx] = c
}
return result
}
// renderTopoNVLinkCard renders the separate NVLink topology card. Returns ""
// if there are fewer than 2 NVIDIA GPUs, or the nvidia-smi nvlink techdump
// wasn't captured (older audit, or nvidia-smi unavailable on that run).
func renderTopoNVLinkCard(hw schema.HardwareSnapshot, exportDir string) string {
gpuCount := 0
for _, dev := range hw.PCIeDevices {
if dev.DeviceClass != nil && isGPUDeviceClass(*dev.DeviceClass) {
gpuCount++
}
}
if gpuCount < 2 {
return ""
}
status, err := readTopoNVLinkStatus(exportDir)
if err != nil || len(status) == 0 {
return topoCard("NVLink Topology", `<span class="badge badge-unknown">nvidia-smi nvlink data unavailable</span>`)
}
errors, _ := readTopoNVLinkErrors(exportDir)
topoMatrix, _ := readGPUTopologyMatrix(exportDir)
pairs := parseGPUPairAdjacency(topoMatrix)
var bodyB strings.Builder
if gpuCount <= 4 && len(pairs) > 0 {
// Small GPU count: per-pair box+line with per-link detail.
for _, pair := range pairs {
activeCount, total, hasError := 0, 0, false
for _, port := range status[pair.GPUA] {
total++
if port.Active {
activeCount++
}
}
for _, counters := range errors[pair.GPUA] {
if counters[0] != 0 || counters[1] != 0 || counters[2] != 0 {
hasError = true
}
}
color := "var(--ok-fg)"
switch {
case hasError:
color = "var(--crit-fg)"
case total > 0 && activeCount < total:
color = "var(--warn-fg)"
}
fmt.Fprintf(&bodyB, `<div style="display:flex;align-items:center;gap:12px;margin-bottom:10px">`+
`<div style="padding:8px 12px;border:1px solid var(--border);border-radius:6px">GPU %d</div>`+
`<div style="flex:1;height:2px;background:%s"></div>`+
`<div style="padding:8px 12px;border:1px solid var(--border);border-radius:6px">GPU %d</div>`+
`<div style="font-size:12px;color:var(--muted)">%d/%d links active%s</div>`+
`</div>`,
pair.GPUA, color, pair.GPUB, activeCount, total, errNoteSuffix(hasError))
}
} else if len(pairs) > 0 {
// Larger GPU counts (NVSwitch fabric): aggregate pair table instead of
// an unreadable all-to-all graph.
bodyB.WriteString(`<table><thead><tr><th>GPU A</th><th>GPU B</th><th>NVLinks</th></tr></thead><tbody>`)
for _, pair := range pairs {
fmt.Fprintf(&bodyB, `<tr><td>GPU %d</td><td>GPU %d</td><td>%d</td></tr>`, pair.GPUA, pair.GPUB, pair.NVLinks)
}
bodyB.WriteString(`</tbody></table>`)
} else {
bodyB.WriteString(`<span class="badge badge-unknown">No NVLink-bonded GPU pairs found</span>`)
}
return topoCard("NVLink Topology", bodyB.String())
}
func errNoteSuffix(hasError bool) string {
if hasError {
return " — errors detected"
}
return ""
}
// ---------------------------------------------------------------------------
// Inventory fallback for the component-detail modal
//
// handleAPIComponentDetail normally sources records from app.ComponentStatusDB,
// which only gains entries once something has actually written a status
// observation (SAT run, watchdog tick, ...). On a freshly booted host that
// hasn't run SAT yet, StatusDB can be entirely empty for a component type even
// though the /topo card for it already shows "N OK" — that card reads
// schema.HardwareComponentStatus.Status straight from the audit snapshot.
// inventoryFallbackRecords bridges that gap by building synthetic records
// from the same snapshot/classifiers the topology card uses, so the two
// views never disagree about how many devices exist or their status.
// ---------------------------------------------------------------------------
// topoSeverityStatus renders classifyTopoSeverity's rank back into the status
// string vocabulary renderComponentDetail/chipLetterClass expect ("OK",
// "Warning", "Critical", "Unknown") — kept in lockstep with classifyTopoSeverity
// so a device the topo card counts as "OK" is never shown here as "Unknown".
func topoSeverityStatus(status *string) string {
switch classifyTopoSeverity(status) {
case 3:
return "Critical"
case 2:
return "Warning"
case 1:
return "OK"
default:
return "Unknown"
}
}
// pcieDeviceKind classifies a PCIe device the same way renderTopoMainDiagram
// does, returning "" for devices that aren't GPU/NIC/RAID.
func pcieDeviceKind(dev schema.HardwarePCIeDevice) string {
switch {
case dev.DeviceClass != nil && isGPUDeviceClass(*dev.DeviceClass):
return "gpu"
case isNICDeviceClassDev(dev):
return "nic"
case dev.DeviceClass != nil && isRAIDControllerClass(*dev.DeviceClass):
return "raid"
default:
return ""
}
}
// pcieDeviceKey builds a stable, human-readable component key for a PCIe
// device: "<kind>:<bdf>" when a slot/BDF is known, else "<kind>:<index>".
func pcieDeviceKey(kind string, index int, dev schema.HardwarePCIeDevice) string {
bdf := ""
if dev.Slot != nil {
bdf = normalizeTopoBDF(*dev.Slot)
} else if dev.BDF != nil {
bdf = normalizeTopoBDF(*dev.BDF)
}
if bdf != "" {
return kind + ":" + bdf
}
return fmt.Sprintf("%s:%d", kind, index)
}
// inventoryFallbackRecords builds ComponentStatusRecord entries straight from
// the audit inventory (bee-audit.json) for the given component type, used
// when ComponentStatusDB has no matching records yet. Records carry only
// ComponentKey/Status — no LastCheckedAt/History — so renderComponentDetail
// renders them without a "checked at" timestamp or sparkline.
func inventoryFallbackRecords(compType string, opts HandlerOptions) []app.ComponentStatusRecord {
data, err := loadSnapshot(opts.AuditPath)
if err != nil {
return nil
}
var ingest schema.HardwareIngestRequest
if err := json.Unmarshal(data, &ingest); err != nil {
return nil
}
hw := ingest.Hardware
var records []app.ComponentStatusRecord
switch compType {
case "cpu":
for i, cpu := range hw.CPUs {
key := fmt.Sprintf("cpu:%d", i)
if cpu.Socket != nil {
key = fmt.Sprintf("cpu:socket%d", *cpu.Socket)
}
records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(cpu.Status)})
}
case "memory":
for i, m := range hw.Memory {
key := fmt.Sprintf("memory:%d", i)
if m.Slot != nil && strings.TrimSpace(*m.Slot) != "" {
key = "memory:" + strings.TrimSpace(*m.Slot)
}
records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(m.Status)})
}
case "storage":
for i, s := range hw.Storage {
key := fmt.Sprintf("storage:%d", i)
if s.Slot != nil && strings.TrimSpace(*s.Slot) != "" {
key = "storage:" + strings.TrimSpace(*s.Slot)
}
records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(s.Status)})
}
case "psu":
for i, p := range hw.PowerSupplies {
key := fmt.Sprintf("psu:%d", i)
if p.Slot != nil && strings.TrimSpace(*p.Slot) != "" {
key = "psu:" + strings.TrimSpace(*p.Slot)
}
records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(p.Status)})
}
case "gpu", "nic", "raid":
for i, dev := range hw.PCIeDevices {
if pcieDeviceKind(dev) != compType {
continue
}
key := pcieDeviceKey(compType, i, dev)
records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(dev.Status)})
}
}
return records
}
audit/internal/webui/page_topo_diagram.go
+783
View File
@@ -0,0 +1,783 @@
package webui
import (
"encoding/json"
"fmt"
"html"
"path/filepath"
"regexp"
"strconv"
"strings"
"bee/audit/internal/app"
"bee/audit/internal/schema"
)
func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string {
// A GPU hardware fault that needs a physical reboot (Xid 79 "fallen off
// the bus", Xid 154 "Node Reboot Required" — see gpuNeedsPhysicalReboot)
// isn't visible in dev.Status: the collector only sets that from PCIe
// link-speed checks, not from SAT/kmsg results. Without this, a GPU that
// dropped off the bus mid-test still renders green here even though the
// Hardware Summary card is showing a critical banner for it.
gpuHardwareFault := false
if db, err := app.OpenComponentStatusDB(filepath.Join(exportDir, "component-status.json")); err == nil {
if _, needsReboot := gpuNeedsPhysicalReboot(matchedRecords(db.All(), nil, []string{"pcie:gpu:"})); needsReboot {
gpuHardwareFault = true
}
}
socketIdx := buildSocketIndex(hw.CPUs)
numCols := len(hw.CPUs)
if numCols == 0 {
numCols = 1
}
unknownCol := numCols // extra trailing column for unmatched devices
// Group PCIe devices (GPU/NIC/RAID only — matches the mockup's node
// types) into columns by NUMA node, falling back to the "unknown" bucket.
type placedDevice struct {
dev schema.HardwarePCIeDevice
kind string // "gpu", "nic", "raid"
col int
bdf string
}
var placed []placedDevice
for _, dev := range hw.PCIeDevices {
kind := pcieDeviceKind(dev)
if kind == "" {
continue
}
col := unknownCol
if dev.NUMANode != nil {
if ci, ok := socketIdx[*dev.NUMANode]; ok {
col = ci
}
}
bdf := ""
if dev.Slot != nil {
bdf = normalizeTopoBDF(*dev.Slot)
} else if dev.BDF != nil {
bdf = normalizeTopoBDF(*dev.BDF)
}
placed = append(placed, placedDevice{dev: dev, kind: kind, col: col, bdf: bdf})
}
hasUnknownCol := false
for _, p := range placed {
if p.col == unknownCol {
hasUnknownCol = true
break
}
}
totalCols := numCols
if hasUnknownCol {
totalCols++
}
// GPU index<->BDF map + pairwise NVLink adjacency, read from the
// persisted techdump captured during the last audit cycle, best-effort:
// if the dump is missing (older audit, no NVIDIA GPUs), this is simply
// skipped. Used only to detect the cross-NUMA-bonded-pair anomaly below;
// the pairwise links themselves are drawn in the separate NVLink
// Topology card, since grouping same-kind/same-column devices into one
// stacked card here leaves no single per-GPU anchor point to draw a
// pairwise connector to or from.
bdfToIndex, _ := readNVIDIAIndexByBDF(exportDir)
var pairs []gpuPairLink
if topoMatrix, err := readGPUTopologyMatrix(exportDir); err == nil {
pairs = parseGPUPairAdjacency(topoMatrix)
}
gpuNUMAByIndex := map[int]*int{}
gpuBDFByIndex := map[int]string{}
for _, p := range placed {
if p.kind != "gpu" || p.bdf == "" {
continue
}
if idx, ok := bdfToIndex[p.bdf]; ok {
gpuNUMAByIndex[idx] = p.dev.NUMANode
gpuBDFByIndex[idx] = p.bdf
}
}
// A bonded pair spanning two different NUMA nodes is treated as an
// anomaly (not a neutral fact) per project decision: a bonded pair is
// expected to sit on one NUMA node, so a cross-NUMA bond escalates both
// GPUs' effective severity to at least Warning, regardless of their own
// reported SAT status.
crossNUMAWarnBDF := map[string]bool{}
for _, pair := range pairs {
numaA, okA := gpuNUMAByIndex[pair.GPUA]
numaB, okB := gpuNUMAByIndex[pair.GPUB]
if !okA || !okB || numaA == nil || numaB == nil || *numaA == *numaB {
continue
}
crossNUMAWarnBDF[gpuBDFByIndex[pair.GPUA]] = true
crossNUMAWarnBDF[gpuBDFByIndex[pair.GPUB]] = true
}
kindOrder := []string{"gpu", "nic", "raid"}
kindLabel := map[string]string{"gpu": "GPU", "nic": "NIC", "raid": "RAID"}
// Attach memory DIMMs to their CPU column too, the same way GPU/NIC/RAID
// PCIe devices are attached via NUMANode — memory has no NUMANode field
// in the schema, so this reads the DIMM's own Locator/Bank Locator
// strings instead (see dimmRawNode). DIMMs that can't be confidently
// attached fall back to the unattached "Memory" row below the diagram,
// same as before this existed.
memBankNodes := map[string]int{}
if raw, err := readTopoTechDump(exportDir, "dmidecode-type17.txt"); err == nil {
memBankNodes = parseDIMMBankLocatorNodes(raw)
}
memCol := make([]int, len(hw.Memory))
memMatched := make([]bool, len(hw.Memory))
var memRawNodes []int
for i, m := range hw.Memory {
if node, ok := dimmRawNode(m, memBankNodes); ok {
memCol[i] = node
memMatched[i] = true
memRawNodes = append(memRawNodes, node)
}
}
memColIdx := buildMemoryColumnIndex(memRawNodes)
for i := range hw.Memory {
if !memMatched[i] {
continue
}
col := memColIdx[memCol[i]]
if col >= numCols {
memMatched[i] = false
continue
}
memCol[i] = col
}
var boxes []topoBox
var pcieEdges []topoEdge
for col := 0; col < totalCols; col++ {
colX := (col+1)*24 + col*topoColWidth
if col < len(hw.CPUs) {
cpu := hw.CPUs[col]
model := ""
if cpu.Model != nil {
model = *cpu.Model
}
socket := col
if cpu.Socket != nil {
socket = *cpu.Socket
}
var tally topoStatusTally
tally.add(classifyTopoSeverity(cpu.Status))
fill, stroke, text := topoSeverityColors(tally.worst())
boxes = append(boxes, topoBox{
x: colX, y: topoTopMargin, w: topoBoxWidth, h: topoBoxHeight,
topoCardInfo: topoCardInfo{
label: fmt.Sprintf("CPU %d", socket), sublabel: model, count: 1,
statusLine: tally.line(),
fillVar: fill, strokeVar: stroke, textVar: text,
detailType: "cpu",
},
})
}
y := topoTopMargin + topoBoxHeight + topoDeviceGap*2
// Memory goes first in the chain, directly under the CPU box: DIMMs
// are wired straight to the socket's memory controller, not reached
// over PCIe like the GPU/NIC/RAID chain below it.
var memGroup []schema.HardwareMemory
for i, m := range hw.Memory {
if memMatched[i] && memCol[i] == col {
memGroup = append(memGroup, m)
}
}
if len(memGroup) > 0 {
var tally topoStatusTally
sizeGB := 0
for _, m := range memGroup {
tally.add(classifyTopoSeverity(m.Status))
if m.SizeMB != nil {
sizeGB += *m.SizeMB / 1024
}
}
fill, stroke, text := topoSeverityColors(tally.worst())
sublabel := ""
if sizeGB > 0 {
sublabel = fmt.Sprintf("%d GB total", sizeGB)
}
stackLayers := topoStackLayers(len(memGroup))
boxes = append(boxes, topoBox{
x: colX, y: y, w: topoBoxWidth, h: topoBoxHeight,
topoCardInfo: topoCardInfo{
label: "Memory", sublabel: sublabel, count: len(memGroup),
statusLine: tally.line(),
fillVar: fill, strokeVar: stroke, textVar: text,
detailType: "memory",
},
})
if col < len(hw.CPUs) {
pcieEdges = append(pcieEdges, topoEdge{
x1: colX + topoBoxWidth/2, y1: topoTopMargin + topoBoxHeight,
x2: colX + topoBoxWidth/2, y2: y,
color: "var(--ok-fg)",
})
}
y += topoBoxHeight + topoDeviceGap + stackLayers*topoStackStep
}
for _, kind := range kindOrder {
var group []placedDevice
for _, p := range placed {
if p.col == col && p.kind == kind {
group = append(group, p)
}
}
if len(group) == 0 {
continue
}
var tally topoStatusTally
model := ""
edgeColor := "var(--ok-fg)"
for i, p := range group {
sev := classifyTopoSeverity(p.dev.Status)
if kind == "gpu" && crossNUMAWarnBDF[p.bdf] && sev < 2 {
sev = 2
}
if kind == "gpu" && gpuHardwareFault && sev < 3 {
sev = 3
}
tally.add(sev)
if i == 0 && p.dev.Model != nil {
model = *p.dev.Model
}
if topoEdgeColorVar(p.dev) == "var(--warn-fg)" {
edgeColor = "var(--warn-fg)"
}
}
fill, stroke, text := topoSeverityColors(tally.worst())
stackLayers := topoStackLayers(len(group))
boxes = append(boxes, topoBox{
x: colX, y: y, w: topoBoxWidth, h: topoBoxHeight,
topoCardInfo: topoCardInfo{
label: kindLabel[kind], sublabel: model, count: len(group),
statusLine: tally.line(),
fillVar: fill, strokeVar: stroke, textVar: text,
detailType: kind,
},
})
if col < len(hw.CPUs) {
pcieEdges = append(pcieEdges, topoEdge{
x1: colX + topoBoxWidth/2, y1: topoTopMargin + topoBoxHeight,
x2: colX + topoBoxWidth/2, y2: y,
color: edgeColor,
})
}
y += topoBoxHeight + topoDeviceGap + stackLayers*topoStackStep
}
}
maxDeviceY := topoTopMargin + topoBoxHeight + topoDeviceGap*2
for _, box := range boxes {
bottom := box.y + box.h + topoStackLayers(box.count)*topoStackStep
if bottom > maxDeviceY {
maxDeviceY = bottom
}
}
svgHeight := maxDeviceY + 24
svgWidth := totalCols*topoColWidth + 48
var b strings.Builder
// Wrapped in its own horizontally-scrolling container (matching the
// overflow-x:auto convention used for wide tables elsewhere in webui)
// rather than max-width:100% — squashing a node/edge diagram to fit a
// narrow viewport makes labels and badges illegible, whereas scrolling
// keeps the diagram readable at its natural size on any screen width.
b.WriteString(`<div style="overflow-x:auto">`)
fmt.Fprintf(&b, `<svg width="%d" height="%d" viewBox="0 0 %d %d">`+"\n", svgWidth, svgHeight, svgWidth, svgHeight)
for _, e := range pcieEdges {
fmt.Fprintf(&b, `<line x1="%d" y1="%d" x2="%d" y2="%d" style="stroke:%s;stroke-width:2"/>`+"\n", e.x1, e.y1, e.x2, e.y2, e.color)
}
for _, box := range boxes {
writeTopoBoxSVG(&b, box)
}
b.WriteString(`</svg></div>`)
// Firmware (BMC/BIOS/...) and PSUs have no PCIe/CPU affinity to anchor
// them to a column, and there can be an arbitrary number of any of them
// — so unlike the diagram above, they're plain flex-wrap HTML below the
// SVG rather than absolutely-positioned SVG boxes. A fixed-size SVG
// canvas has no way to wrap overflow onto a new row, which is exactly
// what caused these to pile up and overlap once a board had more
// PSUs/firmware records than fit in one fixed-width row.
//
// Memory DIMMs that were matched to a CPU column above already got a
// box in the SVG diagram; only DIMMs that couldn't be attached to a
// column (see memMatched above) fall back to this row.
var unmatchedMem []schema.HardwareMemory
for i, m := range hw.Memory {
if !memMatched[i] {
unmatchedMem = append(unmatchedMem, m)
}
}
if len(unmatchedMem) > 0 {
var tally topoStatusTally
for _, m := range unmatchedMem {
tally.add(classifyTopoSeverity(m.Status))
}
fill, stroke, text := topoSeverityColors(tally.worst())
sizeGB := 0
for _, m := range unmatchedMem {
if m.SizeMB != nil {
sizeGB += *m.SizeMB / 1024
}
}
sublabel := ""
if sizeGB > 0 {
sublabel = fmt.Sprintf("%d GB total", sizeGB)
}
b.WriteString(renderTopoFlexRow("Memory", []topoCardInfo{{
label: "Memory", sublabel: sublabel, count: len(unmatchedMem),
statusLine: tally.line(),
fillVar: fill, strokeVar: stroke, textVar: text,
detailType: "memory",
}}))
}
var firmwareItems []topoCardInfo
for _, rec := range hw.Firmware {
// Firmware records carry no per-item status in the schema (they are
// identity, not health, facts), so each stays a neutral, uncolored
// card rather than forcing a fake "Unknown" status line.
fillVar, strokeVar, textVar := topoSeverityColors(0)
firmwareItems = append(firmwareItems, topoCardInfo{
label: rec.DeviceName, sublabel: "fw " + rec.Version, count: 1,
fillVar: fillVar, strokeVar: strokeVar, textVar: textVar,
})
}
b.WriteString(renderTopoFlexRow("Firmware", firmwareItems))
if len(hw.PowerSupplies) > 0 {
var tally topoStatusTally
watt := 0
for _, psu := range hw.PowerSupplies {
tally.add(classifyTopoSeverity(psu.Status))
if psu.WattageW != nil {
watt = *psu.WattageW
}
}
fill, stroke, text := topoSeverityColors(tally.worst())
sublabel := ""
if watt > 0 {
sublabel = fmt.Sprintf("%dW each", watt)
}
b.WriteString(renderTopoFlexRow("Power Supplies", []topoCardInfo{{
label: "Power Supplies", sublabel: sublabel, count: len(hw.PowerSupplies),
statusLine: tally.line(),
fillVar: fill, strokeVar: stroke, textVar: text,
detailType: "psu",
}}))
}
return topoCard("Topology", b.String())
}
// renderTopoFlexRow renders a labeled, wrapping row of component cards.
// Returns "" if items is empty (e.g. no PSU data in this audit).
func renderTopoFlexRow(title string, items []topoCardInfo) string {
if len(items) == 0 {
return ""
}
var b strings.Builder
fmt.Fprintf(&b, `<div style="font-size:11px;color:var(--muted);text-transform:uppercase;letter-spacing:.05em;margin:16px 0 6px">%s</div>`,
html.EscapeString(title))
b.WriteString(`<div style="display:flex;flex-wrap:wrap;gap:10px">`)
for _, item := range items {
onclick := ""
cursor := "default"
if item.detailType != "" {
onclick = fmt.Sprintf(` onclick="openComponentDetail('%s')"`, item.detailType)
cursor = "pointer"
}
stackLayers := topoStackLayers(item.count)
// Extra right/bottom padding on the wrapper reserves room for the
// backing layers of the stack effect so they aren't clipped by the
// flex container.
fmt.Fprintf(&b, `<div style="position:relative;padding-right:%dpx;padding-bottom:%dpx">`,
stackLayers*topoStackStep, stackLayers*topoStackStep)
for i := stackLayers; i >= 1; i-- {
off := i * topoStackStep
fmt.Fprintf(&b, `<div style="position:absolute;top:%dpx;left:%dpx;right:0;bottom:0;border-radius:6px;background:%s;border:1px solid %s;opacity:.55"></div>`,
off, off, item.fillVar, item.strokeVar)
}
fmt.Fprintf(&b, `<div%s style="position:relative;cursor:%s;min-width:160px;padding:10px 12px;border-radius:6px;background:%s;border:1px solid %s;color:%s">`,
onclick, cursor, item.fillVar, item.strokeVar, item.textVar)
label := item.label
if item.count > 1 {
label = fmt.Sprintf("%s ×%d", item.label, item.count)
}
fmt.Fprintf(&b, `<div style="font-size:13px;font-weight:700">%s</div>`, html.EscapeString(label))
if item.sublabel != "" {
fmt.Fprintf(&b, `<div style="font-size:11px;opacity:.85">%s</div>`, html.EscapeString(item.sublabel))
}
if item.statusLine != "" {
fmt.Fprintf(&b, `<div style="font-size:11px;font-weight:600;margin-top:4px">%s</div>`, html.EscapeString(item.statusLine))
}
b.WriteString(`</div></div>`)
}
b.WriteString(`</div>`)
return b.String()
}
func writeTopoBoxSVG(b *strings.Builder, box topoBox) {
onclick := ""
cursor := "default"
if box.detailType != "" {
onclick = fmt.Sprintf(` onclick="openComponentDetail('%s')"`, box.detailType)
cursor = "pointer"
}
fmt.Fprintf(b, `<g%s style="cursor:%s">`, onclick, cursor)
// Stack-of-cards effect: faint offset rects behind the front card when
// this box represents more than one physical component (e.g. 4 GPUs in
// one NUMA column), so a group reads as "a deck of N" rather than a
// single item. Peeks toward the bottom-right, into space already
// reserved between this box and the next one in the column.
for i := topoStackLayers(box.count); i >= 1; i-- {
off := i * topoStackStep
fmt.Fprintf(b, `<rect x="%d" y="%d" width="%d" height="%d" rx="6" ry="6" style="fill:%s;stroke:%s;opacity:.55"/>`+"\n",
box.x+off, box.y+off, box.w, box.h, box.fillVar, box.strokeVar)
}
fmt.Fprintf(b, `<rect x="%d" y="%d" width="%d" height="%d" rx="6" ry="6" style="fill:%s;stroke:%s"/>`+"\n",
box.x, box.y, box.w, box.h, box.fillVar, box.strokeVar)
label := box.label
if box.count > 1 {
label = fmt.Sprintf("%s ×%d", box.label, box.count)
}
fmt.Fprintf(b, `<text x="%d" y="%d" style="fill:%s;font-size:13px;font-weight:700">%s</text>`+"\n",
box.x+10, box.y+20, box.textVar, html.EscapeString(label))
if box.sublabel != "" {
fmt.Fprintf(b, `<text x="%d" y="%d" style="fill:%s;font-size:11px;opacity:.85">%s</text>`+"\n",
box.x+10, box.y+36, box.textVar, html.EscapeString(truncateTopoLabel(box.sublabel, 26)))
}
if box.statusLine != "" {
fmt.Fprintf(b, `<text x="%d" y="%d" style="fill:%s;font-size:10px;font-weight:600">%s</text>`+"\n",
box.x+10, box.y+box.h-10, box.textVar, html.EscapeString(box.statusLine))
}
b.WriteString(`</g>` + "\n")
}
func truncateTopoLabel(s string, max int) string {
if len(s) <= max {
return s
}
if max <= 1 {
return s[:max]
}
return s[:max-1] + "…"
}
// ---------------------------------------------------------------------------
// Separate NVLink topology card (read from techdump, not written to any
// ingest contract)
// ---------------------------------------------------------------------------
type topoNVLinkPort struct {
Index int
Active bool
SpeedGBs *float64
ReplayErrors int64
RecoveryErrors int64
CRCErrors int64
}
var (
topoNVLinkGPUHeaderRe = regexp.MustCompile(`^GPU (\d+):`)
topoNVLinkSpeedLineRe = regexp.MustCompile(`^Link (\d+):\s*([\d.]+)\s*GB/s`)
topoNVLinkInactiveRe = regexp.MustCompile(`^Link (\d+):\s*<inactive>`)
topoNVLinkErrorLineRe = regexp.MustCompile(`^Link (\d+):\s*(Replay|Recovery|CRC) Errors:\s*(\d+)`)
)
func readTopoNVLinkStatus(exportDir string) (map[int][]topoNVLinkPort, error) {
raw, err := readTopoTechDump(exportDir, "nvidia-smi-nvlink-status.txt")
if err != nil {
return nil, err
}
return parseTopoNVLinkStatus(raw), nil
}
func parseTopoNVLinkStatus(raw string) map[int][]topoNVLinkPort {
result := map[int][]topoNVLinkPort{}
currentGPU := -1
for _, line := range strings.Split(raw, "\n") {
trimmed := strings.TrimSpace(line)
if m := topoNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
currentGPU, _ = strconv.Atoi(m[1])
continue
}
if currentGPU < 0 {
continue
}
if m := topoNVLinkInactiveRe.FindStringSubmatch(trimmed); m != nil {
idx, _ := strconv.Atoi(m[1])
result[currentGPU] = append(result[currentGPU], topoNVLinkPort{Index: idx, Active: false})
continue
}
if m := topoNVLinkSpeedLineRe.FindStringSubmatch(trimmed); m != nil {
idx, _ := strconv.Atoi(m[1])
port := topoNVLinkPort{Index: idx, Active: true}
if speed, err := strconv.ParseFloat(m[2], 64); err == nil {
port.SpeedGBs = &speed
}
result[currentGPU] = append(result[currentGPU], port)
}
}
return result
}
func readTopoNVLinkErrors(exportDir string) (map[int]map[int][3]int64, error) {
raw, err := readTopoTechDump(exportDir, "nvidia-smi-nvlink-errors.txt")
if err != nil {
return nil, err
}
return parseTopoNVLinkErrors(raw), nil
}
// parseTopoNVLinkErrors returns, per GPU then link index, [replay, recovery, crc].
func parseTopoNVLinkErrors(raw string) map[int]map[int][3]int64 {
result := map[int]map[int][3]int64{}
currentGPU := -1
for _, line := range strings.Split(raw, "\n") {
trimmed := strings.TrimSpace(line)
if m := topoNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
currentGPU, _ = strconv.Atoi(m[1])
continue
}
if currentGPU < 0 {
continue
}
m := topoNVLinkErrorLineRe.FindStringSubmatch(trimmed)
if m == nil {
continue
}
linkIdx, _ := strconv.Atoi(m[1])
count, _ := strconv.ParseInt(m[3], 10, 64)
if result[currentGPU] == nil {
result[currentGPU] = map[int][3]int64{}
}
c := result[currentGPU][linkIdx]
switch m[2] {
case "Replay":
c[0] = count
case "Recovery":
c[1] = count
case "CRC":
c[2] = count
}
result[currentGPU][linkIdx] = c
}
return result
}
// renderTopoNVLinkCard renders the separate NVLink topology card. Returns ""
// if there are fewer than 2 NVIDIA GPUs, or the nvidia-smi nvlink techdump
// wasn't captured (older audit, or nvidia-smi unavailable on that run).
func renderTopoNVLinkCard(hw schema.HardwareSnapshot, exportDir string) string {
gpuCount := 0
for _, dev := range hw.PCIeDevices {
if dev.DeviceClass != nil && isGPUDeviceClass(*dev.DeviceClass) {
gpuCount++
}
}
if gpuCount < 2 {
return ""
}
status, err := readTopoNVLinkStatus(exportDir)
if err != nil || len(status) == 0 {
return topoCard("NVLink Topology", `<span class="badge badge-unknown">nvidia-smi nvlink data unavailable</span>`)
}
errors, _ := readTopoNVLinkErrors(exportDir)
topoMatrix, _ := readGPUTopologyMatrix(exportDir)
pairs := parseGPUPairAdjacency(topoMatrix)
var bodyB strings.Builder
if gpuCount <= 4 && len(pairs) > 0 {
// Small GPU count: per-pair box+line with per-link detail.
for _, pair := range pairs {
activeCount, total, hasError := 0, 0, false
for _, port := range status[pair.GPUA] {
total++
if port.Active {
activeCount++
}
}
for _, counters := range errors[pair.GPUA] {
if counters[0] != 0 || counters[1] != 0 || counters[2] != 0 {
hasError = true
}
}
color := "var(--ok-fg)"
switch {
case hasError:
color = "var(--crit-fg)"
case total > 0 && activeCount < total:
color = "var(--warn-fg)"
}
fmt.Fprintf(&bodyB, `<div style="display:flex;align-items:center;gap:12px;margin-bottom:10px">`+
`<div style="padding:8px 12px;border:1px solid var(--border);border-radius:6px">GPU %d</div>`+
`<div style="flex:1;height:2px;background:%s"></div>`+
`<div style="padding:8px 12px;border:1px solid var(--border);border-radius:6px">GPU %d</div>`+
`<div style="font-size:12px;color:var(--muted)">%d/%d links active%s</div>`+
`</div>`,
pair.GPUA, color, pair.GPUB, activeCount, total, errNoteSuffix(hasError))
}
} else if len(pairs) > 0 {
// Larger GPU counts (NVSwitch fabric): aggregate pair table instead of
// an unreadable all-to-all graph.
bodyB.WriteString(`<table><thead><tr><th>GPU A</th><th>GPU B</th><th>NVLinks</th></tr></thead><tbody>`)
for _, pair := range pairs {
fmt.Fprintf(&bodyB, `<tr><td>GPU %d</td><td>GPU %d</td><td>%d</td></tr>`, pair.GPUA, pair.GPUB, pair.NVLinks)
}
bodyB.WriteString(`</tbody></table>`)
} else {
bodyB.WriteString(`<span class="badge badge-unknown">No NVLink-bonded GPU pairs found</span>`)
}
return topoCard("NVLink Topology", bodyB.String())
}
func errNoteSuffix(hasError bool) string {
if hasError {
return " — errors detected"
}
return ""
}
// ---------------------------------------------------------------------------
// Inventory fallback for the component-detail modal
//
// handleAPIComponentDetail normally sources records from app.ComponentStatusDB,
// which only gains entries once something has actually written a status
// observation (SAT run, watchdog tick, ...). On a freshly booted host that
// hasn't run SAT yet, StatusDB can be entirely empty for a component type even
// though the /topo card for it already shows "N OK" — that card reads
// schema.HardwareComponentStatus.Status straight from the audit snapshot.
// inventoryFallbackRecords bridges that gap by building synthetic records
// from the same snapshot/classifiers the topology card uses, so the two
// views never disagree about how many devices exist or their status.
// ---------------------------------------------------------------------------
// topoSeverityStatus renders classifyTopoSeverity's rank back into the status
// string vocabulary renderComponentDetail/chipLetterClass expect ("OK",
// "Warning", "Critical", "Unknown") — kept in lockstep with classifyTopoSeverity
// so a device the topo card counts as "OK" is never shown here as "Unknown".
func topoSeverityStatus(status *string) string {
switch classifyTopoSeverity(status) {
case 3:
return "Critical"
case 2:
return "Warning"
case 1:
return "OK"
default:
return "Unknown"
}
}
// pcieDeviceKind classifies a PCIe device the same way renderTopoMainDiagram
// does, returning "" for devices that aren't GPU/NIC/RAID.
func pcieDeviceKind(dev schema.HardwarePCIeDevice) string {
switch {
case dev.DeviceClass != nil && isGPUDeviceClass(*dev.DeviceClass):
return "gpu"
case isNICDeviceClassDev(dev):
return "nic"
case dev.DeviceClass != nil && isRAIDControllerClass(*dev.DeviceClass):
return "raid"
default:
return ""
}
}
// pcieDeviceKey builds a stable, human-readable component key for a PCIe
// device: "<kind>:<bdf>" when a slot/BDF is known, else "<kind>:<index>".
func pcieDeviceKey(kind string, index int, dev schema.HardwarePCIeDevice) string {
bdf := ""
if dev.Slot != nil {
bdf = normalizeTopoBDF(*dev.Slot)
} else if dev.BDF != nil {
bdf = normalizeTopoBDF(*dev.BDF)
}
if bdf != "" {
return kind + ":" + bdf
}
return fmt.Sprintf("%s:%d", kind, index)
}
// inventoryFallbackRecords builds ComponentStatusRecord entries straight from
// the audit inventory (bee-audit.json) for the given component type, used
// when ComponentStatusDB has no matching records yet. Records carry only
// ComponentKey/Status — no LastCheckedAt/History — so renderComponentDetail
// renders them without a "checked at" timestamp or sparkline.
func inventoryFallbackRecords(compType string, opts HandlerOptions) []app.ComponentStatusRecord {
data, err := loadSnapshot(opts.AuditPath)
if err != nil {
return nil
}
var ingest schema.HardwareIngestRequest
if err := json.Unmarshal(data, &ingest); err != nil {
return nil
}
hw := ingest.Hardware
var records []app.ComponentStatusRecord
switch compType {
case "cpu":
for i, cpu := range hw.CPUs {
key := fmt.Sprintf("cpu:%d", i)
if cpu.Socket != nil {
key = fmt.Sprintf("cpu:socket%d", *cpu.Socket)
}
records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(cpu.Status)})
}
case "memory":
for i, m := range hw.Memory {
key := fmt.Sprintf("memory:%d", i)
if m.Slot != nil && strings.TrimSpace(*m.Slot) != "" {
key = "memory:" + strings.TrimSpace(*m.Slot)
}
records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(m.Status)})
}
case "storage":
for i, s := range hw.Storage {
key := fmt.Sprintf("storage:%d", i)
if s.Slot != nil && strings.TrimSpace(*s.Slot) != "" {
key = "storage:" + strings.TrimSpace(*s.Slot)
}
records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(s.Status)})
}
case "psu":
for i, p := range hw.PowerSupplies {
key := fmt.Sprintf("psu:%d", i)
if p.Slot != nil && strings.TrimSpace(*p.Slot) != "" {
key = "psu:" + strings.TrimSpace(*p.Slot)
}
records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(p.Status)})
}
case "gpu", "nic", "raid":
for i, dev := range hw.PCIeDevices {
if pcieDeviceKind(dev) != compType {
continue
}
key := pcieDeviceKey(compType, i, dev)
records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(dev.Status)})
}
}
return records
}
audit/internal/webui/page_validate.go
+53 -94
View File
@@ -286,11 +286,6 @@ function satLoadGPUs() {
satUpdateGPUSelectionNote();
});
}
function satGPUDisplayName(gpu) {
const idx = (gpu && Number.isFinite(Number(gpu.index))) ? Number(gpu.index) : 0;
const name = gpu && gpu.name ? gpu.name : ('GPU ' + idx);
return 'GPU ' + idx + ' — ' + name;
}
function satRequestBody(target, overrides) {
const body = {};
const labels = satLabels();
@@ -358,34 +353,9 @@ function runSATWithOverrides(target, overrides) {
return enqueueSATTarget(target, overrides)
.then(d => streamSATTask(d.task_id, title, false));
}
const nvidiaPerGPUTargets = [];
const nvidiaAllGPUTargets = ['nvidia', 'nvidia-targeted-stress', 'nvidia-targeted-power', 'nvidia-pulse', 'nvidia-interconnect', 'nvidia-bandwidth'];
function satAllGPUIndicesForMulti() {
return Promise.resolve(satSelectedGPUIndices());
}
function expandSATTarget(target) {
if (nvidiaAllGPUTargets.indexOf(target) >= 0) {
return satAllGPUIndicesForMulti().then(function(indices) {
if (!indices.length) return Promise.reject(new Error('No NVIDIA GPUs available.'));
return [{target: target, overrides: {gpu_indices: indices, display_name: satLabels()[target] || target}}];
});
}
if (nvidiaPerGPUTargets.indexOf(target) < 0) {
return Promise.resolve([{target: target}]);
}
const selected = satSelectedGPUIndices();
if (!selected.length) {
return Promise.reject(new Error('Select at least one NVIDIA GPU.'));
}
return loadSatNvidiaGPUs().then(gpus => gpus.filter(gpu => selected.indexOf(Number(gpu.index)) >= 0).map(gpu => ({
target: target,
overrides: {
gpu_indices: [Number(gpu.index)],
display_name: (satLabels()[target] || ('Validate ' + target)) + ' (' + satGPUDisplayName(gpu) + ')'
},
label: satGPUDisplayName(gpu),
})));
}
function runNvidiaFabricValidate(target) {
satAllGPUIndicesForMulti().then(function(indices) {
if (!indices.length) { alert('No NVIDIA GPUs available.'); return; }
@@ -419,52 +389,40 @@ function runAMDValidateSet() {
};
return runNext(0);
}
// runAllSAT hands the whole decision to the backend: which hardware is
// present and ready, and therefore which tasks to enqueue, is decided by
// POST /api/sat/run-all. The page only sends operator intent.
function runAllSAT() {
const cycles = 1;
const status = document.getElementById('sat-all-status');
status.textContent = 'Enqueuing...';
const stressOnlyTargets = ['nvidia-targeted-stress', 'nvidia-targeted-power', 'nvidia-pulse'];
const baseTargets = ['nvidia','nvidia-targeted-stress','nvidia-targeted-power','nvidia-pulse','nvidia-interconnect','nvidia-bandwidth','memory','storage','tpm','cpu'].concat(selectedAMDValidateTargets());
const activeTargets = baseTargets.filter(target => {
if (stressOnlyTargets.indexOf(target) >= 0 && !satStressMode()) return false;
const btn = document.getElementById('sat-btn-' + target);
return !(btn && btn.disabled);
});
Promise.all(activeTargets.map(expandSATTarget)).then(groups => {
const expanded = [];
for (let cycle = 0; cycle < cycles; cycle++) {
groups.forEach(group => group.forEach(item => expanded.push(item)));
}
const total = expanded.length;
let enqueued = 0;
if (!total) {
status.textContent = 'No tasks selected.';
return;
}
const runNext = (idx) => {
if (idx >= expanded.length) { status.textContent = 'Completed ' + total + ' task(s).'; return Promise.resolve(); }
const item = expanded[idx];
status.textContent = 'Running ' + (idx + 1) + '/' + total + '...';
return enqueueSATTarget(item.target, item.overrides)
.then(() => {
enqueued++;
return runNext(idx + 1);
});
};
return runNext(0);
}).catch(err => {
status.textContent = 'Error: ' + err.message;
});
status.textContent = 'Planning on server...';
const body = {
stress_mode: satStressMode(),
amd_targets: selectedAMDValidateTargets(),
};
const gpuSubset = satSelectedGPUIndices();
if (gpuSubset.length) body.nvidia_gpu_indices = gpuSubset;
fetch('/api/sat/run-all', {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify(body),
}).then(r => r.json().then(d => { if (!r.ok) throw new Error(d.error || ('HTTP ' + r.status)); return d; }))
.then(d => {
let msg = 'Enqueued ' + (d.task_count || 0) + ' task(s).';
if (d.notes && d.notes.length) msg += ' ' + d.notes.join('; ');
status.textContent = msg;
})
.catch(err => { status.textContent = 'Error: ' + err.message; });
}
</script>
<script>
fetch('/api/gpu/presence').then(r=>r.json()).then(gp => {
if (!gp.nvidia) disableSATCard('nvidia', 'No NVIDIA GPU detected');
if (!gp.nvidia) disableSATCard('nvidia-targeted-stress', 'No NVIDIA GPU detected');
if (!gp.nvidia) disableSATCard('nvidia-targeted-power', 'No NVIDIA GPU detected');
if (!gp.nvidia) disableSATCard('nvidia-pulse', 'No NVIDIA GPU detected');
if (!gp.nvidia) disableSATCard('nvidia-interconnect', 'No NVIDIA GPU detected');
if (!gp.nvidia) disableSATCard('nvidia-bandwidth', 'No NVIDIA GPU detected');
if (!gp.nvidia) {
const why = gp.nvidia_initializing
? 'NVIDIA GPU present, driver still initializing; Run All will wait for it'
: 'No NVIDIA GPU detected';
['nvidia','nvidia-targeted-stress','nvidia-targeted-power','nvidia-pulse','nvidia-interconnect','nvidia-bandwidth']
.forEach(t => disableSATCard(t, why));
}
if (!gp.amd) disableSATCard('amd', 'No AMD GPU detected');
if (!gp.amd) disableSATAMDOptions('No AMD GPU detected');
});
@@ -915,31 +873,27 @@ function runAMDValidateSet() {
};
return runNext(0);
}
// runAllCheckSAT delegates hardware detection and task planning to the
// backend (POST /api/sat/run-all). The browser no longer decides whether a
// GPU is present: a stale or empty GPU list can no longer silently drop the
// GPU checks.
function runAllCheckSAT() {
const status = document.getElementById('sat-all-status');
status.textContent = 'Enqueuing...';
const nvidiaIndices = satSelectedGPUIndices();
const nvidiaAllTargets = ['nvidia', 'nvidia-interconnect', 'nvidia-bandwidth', 'nvidia-pcie-bandwidth'];
const baseTargets = ['cpu', 'memory', 'storage', 'tpm', 'nvidia-config', 'pcie-link'];
const amdTargets = selectedAMDValidateTargets();
const expanded = [];
baseTargets.forEach(t => expanded.push({target: t}));
if (nvidiaIndices.length) {
nvidiaAllTargets.forEach(t => {
const btn = document.getElementById('sat-btn-' + t);
if (!(btn && btn.disabled)) expanded.push({target: t, overrides: {gpu_indices: nvidiaIndices, display_name: satLabels()[t] || t}});
});
}
amdTargets.forEach(t => expanded.push({target: t}));
if (!expanded.length) { status.textContent = 'No tasks selected.'; return; }
const total = expanded.length;
const runNext = idx => {
if (idx >= expanded.length) { status.textContent = 'Completed ' + total + ' task(s).'; return Promise.resolve(); }
const item = expanded[idx];
status.textContent = 'Running ' + (idx + 1) + '/' + total + '...';
return enqueueSATTarget(item.target, item.overrides).then(() => runNext(idx + 1));
};
runNext(0).catch(err => { status.textContent = 'Error: ' + err.message; });
status.textContent = 'Planning on server...';
const body = {stress_mode: false, amd_targets: selectedAMDValidateTargets()};
const gpuSubset = satSelectedGPUIndices();
if (gpuSubset.length) body.nvidia_gpu_indices = gpuSubset;
fetch('/api/sat/run-all', {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify(body),
}).then(r => r.json().then(d => { if (!r.ok) throw new Error(d.error || ('HTTP ' + r.status)); return d; }))
.then(d => {
let msg = 'Enqueued ' + (d.task_count || 0) + ' task(s).';
if (d.notes && d.notes.length) msg += ' ' + d.notes.join('; ');
status.textContent = msg;
})
.catch(err => { status.textContent = 'Error: ' + err.message; });
}
function disableSATCard(id, reason) {
const btn = document.getElementById('sat-btn-' + id);
@@ -959,7 +913,12 @@ function disableSATCard(id, reason) {
}
}
fetch('/api/gpu/presence').then(r => r.json()).then(gp => {
if (!gp.nvidia) ['nvidia','nvidia-interconnect','nvidia-bandwidth'].forEach(t => disableSATCard(t, 'No NVIDIA GPU detected'));
if (!gp.nvidia) {
const why = gp.nvidia_initializing
? 'NVIDIA GPU present, driver still initializing; Run All will wait for it'
: 'No NVIDIA GPU detected';
['nvidia','nvidia-interconnect','nvidia-bandwidth'].forEach(t => disableSATCard(t, why));
}
if (!gp.amd) {
disableSATCard('amd', 'No AMD GPU detected');
['sat-amd-target','sat-amd-mem-target','sat-amd-bandwidth-target'].forEach(id => {
audit/internal/webui/page_validate_test.go
-1
View File
@@ -37,7 +37,6 @@ func TestRenderCheckIncludesReadOnlyTPMValidation(t *testing.T) {
`tpm2_getcap properties-fixed`,
`tpm2_pcrread`,
`tpm2_gettestresult`,
`'storage', 'tpm', 'nvidia-config'`,
} {
if !strings.Contains(page, want) {
t.Fatalf("check page does not contain %q", want)
audit/internal/webui/pages.go
-684
View File
@@ -5,9 +5,6 @@ import (
"fmt"
"html"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"bee/audit/internal/app"
@@ -645,684 +642,3 @@ function auditModalRun() {
}
</script>`
}
func renderHealthCard(opts HandlerOptions) string {
data, err := loadSnapshot(filepath.Join(opts.ExportDir, "runtime-health.json"))
if err != nil {
return `<div class="card"><div class="card-head">Runtime Health</div><div class="card-body"><span class="badge badge-unknown">No data</span></div></div>`
}
var health schema.RuntimeHealth
if err := json.Unmarshal(data, &health); err != nil {
return `<div class="card"><div class="card-head">Runtime Health</div><div class="card-body"><span class="badge badge-err">Parse error</span></div></div>`
}
status := strings.TrimSpace(health.Status)
if status == "" {
status = "UNKNOWN"
}
badge := "badge-ok"
if status == "PARTIAL" {
badge = "badge-warn"
} else if status == "FAIL" || status == "FAILED" {
badge = "badge-err"
}
var b strings.Builder
b.WriteString(`<div class="card"><div class="card-head">Runtime Health</div><div class="card-body">`)
b.WriteString(fmt.Sprintf(`<div style="margin-bottom:10px"><span class="badge %s">%s</span></div>`, badge, html.EscapeString(status)))
if checkedAt := strings.TrimSpace(health.CheckedAt); checkedAt != "" {
b.WriteString(`<div style="font-size:12px;color:var(--muted);margin-bottom:12px">Checked at: ` + html.EscapeString(checkedAt) + `</div>`)
}
rows := []runtimeHealthRow{
buildRuntimeExportRow(health),
buildRuntimeNetworkRow(health),
buildRuntimeDriverRow(health),
buildRuntimeAccelerationRow(health),
buildRuntimeToolsRow(health),
buildRuntimeServicesRow(health),
buildRuntimeUSBExportRow(health),
buildRuntimeToRAMRow(health),
}
b.WriteString(`<table><thead><tr><th>Check</th><th>Status</th><th>Source</th><th>Issue</th></tr></thead><tbody>`)
for _, row := range rows {
b.WriteString(`<tr><td>` + html.EscapeString(row.Title) + `</td><td>` + runtimeStatusBadge(row.Status) + `</td><td>` + html.EscapeString(row.Source) + `</td><td>` + rowIssueHTML(row.Issue) + `</td></tr>`)
}
b.WriteString(`</tbody></table>`)
b.WriteString(`</div></div>`)
return b.String()
}
type runtimeHealthRow struct {
Title string
Status string
Source string
Issue string
}
func buildRuntimeExportRow(health schema.RuntimeHealth) runtimeHealthRow {
issue := runtimeIssueDescriptions(health.Issues, "export_dir_unavailable")
status := "UNKNOWN"
switch {
case issue != "":
status = "FAILED"
case strings.TrimSpace(health.ExportDir) != "":
status = "OK"
}
source := "os.MkdirAll"
if dir := strings.TrimSpace(health.ExportDir); dir != "" {
source += " " + dir
}
return runtimeHealthRow{Title: "Export Directory", Status: status, Source: source, Issue: issue}
}
func buildRuntimeNetworkRow(health schema.RuntimeHealth) runtimeHealthRow {
status := strings.TrimSpace(health.NetworkStatus)
if status == "" {
status = "UNKNOWN"
}
issue := runtimeIssueDescriptions(health.Issues, "dhcp_failed")
return runtimeHealthRow{Title: "Network", Status: status, Source: "ListInterfaces / DHCP", Issue: issue}
}
func buildRuntimeDriverRow(health schema.RuntimeHealth) runtimeHealthRow {
issue := runtimeIssueDescriptions(health.Issues, "nvidia_kernel_module_missing", "nvidia_modeset_failed", "amdgpu_kernel_module_missing")
status := "UNKNOWN"
switch {
case health.DriverReady && issue == "":
status = "OK"
case health.DriverReady:
status = "PARTIAL"
case issue != "":
status = "FAILED"
}
return runtimeHealthRow{Title: "NVIDIA/AMD Driver", Status: status, Source: "lsmod / vendor probe", Issue: issue}
}
func buildRuntimeAccelerationRow(health schema.RuntimeHealth) runtimeHealthRow {
issue := runtimeIssueDescriptions(health.Issues, "cuda_runtime_not_ready", "rocm_smi_unavailable")
status := "UNKNOWN"
switch {
case health.CUDAReady && issue == "":
status = "OK"
case health.CUDAReady:
status = "PARTIAL"
case issue != "":
status = "FAILED"
}
return runtimeHealthRow{Title: "CUDA / ROCm", Status: status, Source: "bee-gpu-burn / rocm-smi", Issue: issue}
}
func buildRuntimeToolsRow(health schema.RuntimeHealth) runtimeHealthRow {
if len(health.Tools) == 0 {
return runtimeHealthRow{Title: "Required Utilities", Status: "UNKNOWN", Source: "CheckTools", Issue: "No tool status data."}
}
missing := make([]string, 0)
for _, tool := range health.Tools {
if !tool.OK {
missing = append(missing, tool.Name)
}
}
status := "OK"
issue := ""
if len(missing) > 0 {
status = "PARTIAL"
issue = "Missing: " + strings.Join(missing, ", ")
}
return runtimeHealthRow{Title: "Required Utilities", Status: status, Source: "CheckTools", Issue: issue}
}
func buildRuntimeServicesRow(health schema.RuntimeHealth) runtimeHealthRow {
if len(health.Services) == 0 {
return runtimeHealthRow{Title: "Bee Services", Status: "UNKNOWN", Source: "systemctl is-active", Issue: "No service status data."}
}
nonActive := make([]string, 0)
for _, svc := range health.Services {
state := strings.TrimSpace(strings.ToLower(svc.Status))
// "inactive" is OK for oneshot services that have completed successfully
// (bee-sshsetup, bee-preflight, bee-audit, bee-network, etc.).
// Only "failed" is a genuine problem.
switch state {
case "active", "activating", "deactivating", "reloading", "inactive":
// OK — service is running, transitioning normally, or completed successfully
default:
nonActive = append(nonActive, svc.Name+"="+svc.Status)
}
}
status := "OK"
issue := ""
if len(nonActive) > 0 {
status = "PARTIAL"
issue = strings.Join(nonActive, ", ")
}
return runtimeHealthRow{Title: "Bee Services", Status: status, Source: "ServiceState", Issue: issue}
}
func buildRuntimeUSBExportRow(health schema.RuntimeHealth) runtimeHealthRow {
path := strings.TrimSpace(health.USBExportPath)
if path != "" {
return runtimeHealthRow{
Title: "USB Export Drive",
Status: "OK",
Source: "/proc/mounts + lsblk",
Issue: path,
}
}
return runtimeHealthRow{
Title: "USB Export Drive",
Status: "WARNING",
Source: "/proc/mounts + lsblk",
Issue: "No writable USB drive mounted. Plug in a USB drive to enable log export.",
}
}
func buildRuntimeToRAMRow(health schema.RuntimeHealth) runtimeHealthRow {
switch strings.ToLower(strings.TrimSpace(health.ToRAMStatus)) {
case "ok":
return runtimeHealthRow{
Title: "LiveCD in RAM",
Status: "OK",
Source: "live-boot / /proc/mounts",
Issue: "",
}
case "partial":
return runtimeHealthRow{
Title: "LiveCD in RAM",
Status: "WARNING",
Source: "live-boot / /proc/mounts / /dev/shm/bee-live",
Issue: "Partial or staged RAM copy detected. System is not fully running from RAM; Copy to RAM can be retried.",
}
case "failed":
return runtimeHealthRow{
Title: "LiveCD in RAM",
Status: "FAILED",
Source: "live-boot / /proc/mounts",
Issue: "toram boot parameter set but ISO is not mounted from RAM. Copy may have failed.",
}
default:
// toram not active — ISO still on original boot media (USB/CD)
return runtimeHealthRow{
Title: "LiveCD in RAM",
Status: "WARNING",
Source: "live-boot / /proc/mounts",
Issue: "ISO not copied to RAM. Use \u201cCopy to RAM\u201d to free the boot drive and improve performance.",
}
}
}
func buildHardwareComponentRows(exportDir string) []runtimeHealthRow {
path := filepath.Join(exportDir, "component-status.json")
db, err := app.OpenComponentStatusDB(path)
if err != nil {
return []runtimeHealthRow{
{Title: "CPU Component Health", Status: "UNKNOWN", Source: "component-status.json", Issue: "Component status DB not available."},
{Title: "Memory Component Health", Status: "UNKNOWN", Source: "component-status.json", Issue: "Component status DB not available."},
{Title: "Storage Component Health", Status: "UNKNOWN", Source: "component-status.json", Issue: "Component status DB not available."},
{Title: "GPU Component Health", Status: "UNKNOWN", Source: "component-status.json", Issue: "Component status DB not available."},
{Title: "PSU Component Health", Status: "UNKNOWN", Source: "component-status.json", Issue: "No PSU component checks recorded."},
}
}
records := db.All()
return []runtimeHealthRow{
aggregateComponentStatus("CPU", records, []string{"cpu:all"}, nil),
aggregateComponentStatus("Memory", records, []string{"memory:all"}, []string{"memory:"}),
aggregateComponentStatus("Storage", records, []string{"storage:all"}, []string{"storage:"}),
aggregateComponentStatus("GPU", records, nil, []string{"pcie:gpu:"}),
aggregateComponentStatus("PSU", records, nil, []string{"psu:"}),
}
}
// matchedRecords returns all ComponentStatusRecord entries whose key matches
// any exact key or any of the given prefixes. Used for per-device chip rendering.
func firstNonEmpty(vals ...string) string {
for _, v := range vals {
if v != "" {
return v
}
}
return ""
}
func matchedRecords(records []app.ComponentStatusRecord, exact []string, prefixes []string) []app.ComponentStatusRecord {
var matched []app.ComponentStatusRecord
for _, rec := range records {
key := strings.TrimSpace(rec.ComponentKey)
if key == "" {
continue
}
if containsExactKey(key, exact) || hasAnyPrefix(key, prefixes) {
matched = append(matched, rec)
}
}
return matched
}
func aggregateComponentStatus(title string, records []app.ComponentStatusRecord, exact []string, prefixes []string) runtimeHealthRow {
matched := make([]app.ComponentStatusRecord, 0)
for _, rec := range records {
key := strings.TrimSpace(rec.ComponentKey)
if key == "" {
continue
}
if containsExactKey(key, exact) || hasAnyPrefix(key, prefixes) {
matched = append(matched, rec)
}
}
if len(matched) == 0 {
return runtimeHealthRow{Title: title, Status: "UNKNOWN", Source: "component-status.json", Issue: "No component status data."}
}
maxSev := -1
for _, rec := range matched {
if sev := runtimeComponentSeverity(rec.Status); sev > maxSev {
maxSev = sev
}
}
status := "UNKNOWN"
switch maxSev {
case 3:
status = "CRITICAL"
case 2:
status = "WARNING"
case 1:
status = "OK"
}
sources := make([]string, 0)
sourceSeen := map[string]struct{}{}
issues := make([]string, 0)
issueSeen := map[string]struct{}{}
for _, rec := range matched {
if runtimeComponentSeverity(rec.Status) != maxSev {
continue
}
source := latestComponentSource(rec)
if source == "" {
source = "component-status.json"
}
if _, ok := sourceSeen[source]; !ok {
sourceSeen[source] = struct{}{}
sources = append(sources, source)
}
issue := strings.TrimSpace(rec.ErrorSummary)
if issue == "" {
issue = latestComponentDetail(rec)
}
if issue == "" {
continue
}
if _, ok := issueSeen[issue]; ok {
continue
}
issueSeen[issue] = struct{}{}
issues = append(issues, issue)
}
if len(sources) == 0 {
sources = append(sources, "component-status.json")
}
issue := strings.Join(issues, "; ")
if issue == "" {
issue = "—"
}
return runtimeHealthRow{
Title: title,
Status: status,
Source: strings.Join(sources, ", "),
Issue: issue,
}
}
func containsExactKey(key string, exact []string) bool {
for _, candidate := range exact {
if key == candidate {
return true
}
}
return false
}
func hasAnyPrefix(key string, prefixes []string) bool {
for _, prefix := range prefixes {
if strings.HasPrefix(key, prefix) {
return true
}
}
return false
}
func runtimeComponentSeverity(status string) int {
switch strings.TrimSpace(strings.ToLower(status)) {
case "critical":
return 3
case "warning":
return 2
case "ok":
return 1
default:
return 0
}
}
func latestComponentSource(rec app.ComponentStatusRecord) string {
if len(rec.History) == 0 {
return ""
}
return strings.TrimSpace(rec.History[len(rec.History)-1].Source)
}
func latestComponentDetail(rec app.ComponentStatusRecord) string {
if len(rec.History) == 0 {
return ""
}
return strings.TrimSpace(rec.History[len(rec.History)-1].Detail)
}
func runtimeIssueDescriptions(issues []schema.RuntimeIssue, codes ...string) string {
if len(issues) == 0 || len(codes) == 0 {
return ""
}
allowed := make(map[string]struct{}, len(codes))
for _, code := range codes {
allowed[code] = struct{}{}
}
messages := make([]string, 0)
for _, issue := range issues {
if _, ok := allowed[issue.Code]; !ok {
continue
}
desc := strings.TrimSpace(issue.Description)
if desc == "" {
desc = issue.Code
}
messages = append(messages, desc)
}
return strings.Join(messages, "; ")
}
// gpuNeedsPhysicalReboot reports whether any GPU component record carries a
// hardware-fault reason that a driver reset can't clear (e.g. Xid 79 "GPU
// has fallen off the bus", Xid 154 "Node Reboot Required" — see
// collector.xidHardwareFaultMessages). Those errors mean every subsequent
// GPU SAT job will keep failing until the node is physically power-cycled,
// so this drives a dashboard banner that says so up front instead of making
// an operator burn another test cycle to rediscover it.
func gpuNeedsPhysicalReboot(records []app.ComponentStatusRecord) (reason string, needsReboot bool) {
for _, rec := range records {
if !strings.EqualFold(strings.TrimSpace(rec.Status), "Critical") {
continue
}
if strings.Contains(strings.ToLower(rec.ErrorSummary), "reboot") {
return rec.ErrorSummary, true
}
}
return "", false
}
// chipLetterClass maps a component status to a single display letter and CSS class.
func chipLetterClass(status string) (letter, cls string) {
switch strings.ToUpper(strings.TrimSpace(status)) {
case "OK":
return "O", "chip-ok"
case "WARNING", "WARN", "PARTIAL":
return "W", "chip-warn"
case "CRITICAL", "FAIL", "FAILED", "ERROR":
return "F", "chip-fail"
default:
return "?", "chip-unknown"
}
}
// renderComponentChips renders one 20×20 chip per ComponentStatusRecord.
// Hover tooltip shows component key, status, error summary and last check time.
// Falls back to a single unknown chip when no records are available.
func renderComponentChips(matched []app.ComponentStatusRecord) string {
if len(matched) == 0 {
return `<span class="chips"><span class="chip chip-unknown" title="No data">?</span></span>`
}
sort.Slice(matched, func(i, j int) bool {
return matched[i].ComponentKey < matched[j].ComponentKey
})
var b strings.Builder
b.WriteString(`<span class="chips">`)
for _, rec := range matched {
letter, cls := chipLetterClass(rec.Status)
var tooltip strings.Builder
tooltip.WriteString(rec.ComponentKey)
tooltip.WriteString(": ")
tooltip.WriteString(firstNonEmpty(rec.Status, "UNKNOWN"))
if rec.ErrorSummary != "" {
tooltip.WriteString(" — ")
tooltip.WriteString(rec.ErrorSummary)
}
if !rec.LastCheckedAt.IsZero() {
fmt.Fprintf(&tooltip, " (checked %s)", rec.LastCheckedAt.Format("15:04:05"))
}
fmt.Fprintf(&b, `<span class="chip %s" title="%s">%s</span>`,
cls, html.EscapeString(tooltip.String()), letter)
}
b.WriteString(`</span>`)
return b.String()
}
func runtimeStatusBadge(status string) string {
status = strings.ToUpper(strings.TrimSpace(status))
badge := "badge-unknown"
switch status {
case "OK":
badge = "badge-ok"
case "PARTIAL", "WARNING", "WARN":
badge = "badge-warn"
case "FAIL", "FAILED", "CRITICAL":
badge = "badge-err"
}
return `<span class="badge ` + badge + `">` + html.EscapeString(status) + `</span>`
}
func rowIssueHTML(issue string) string {
issue = strings.TrimSpace(issue)
if issue == "" {
return `<span style="color:var(--muted)">—</span>`
}
return html.EscapeString(issue)
}
var aerStatusRe = regexp.MustCompile(`aer_status:\s*0x([0-9a-fA-F]{1,8})`)
// decodeAERStatus parses an AER status hex value from a kernel error detail string
// and returns a human-readable list of set bit names with correctable/uncorrectable label,
// or "" if no AER status is found.
func decodeAERStatus(detail string) string {
m := aerStatusRe.FindStringSubmatch(detail)
if m == nil {
return ""
}
v64, err := strconv.ParseUint(m[1], 16, 32)
if err != nil {
return ""
}
val := uint32(v64)
type bitDef struct {
bit uint32
name string
}
corrBits := []bitDef{
{0, "Receiver Error"}, {6, "Replay Timer Timeout"}, {7, "Advisory Non-Fatal"},
{8, "Corrected Internal Error"}, {9, "Header Log Overflow"},
{13, "Replay Num Rollover"}, {14, "Bad DLLP"}, {15, "Bad TLP"},
}
uncorrBits := []bitDef{
{4, "Data Link Protocol Error"}, {5, "Surprise Down Error"},
{12, "Poisoned TLP Received"}, {13, "Flow Control Protocol Error"},
{14, "Completion Timeout"}, {15, "Completer Abort"}, {16, "Unexpected Completion"},
{17, "Receiver Overflow"}, {18, "Malformed TLP"}, {19, "ECRC Error"},
{20, "Unsupported Request Error"}, {21, "ACS Violation"}, {22, "Uncorrectable Internal Error"},
}
var corrNames, uncorrNames []string
for _, b := range corrBits {
if val&(1<<b.bit) != 0 {
corrNames = append(corrNames, b.name)
}
}
for _, b := range uncorrBits {
if val&(1<<b.bit) != 0 {
uncorrNames = append(uncorrNames, b.name)
}
}
if len(corrNames) >= len(uncorrNames) && len(corrNames) > 0 {
return strings.Join(corrNames, ", ") + " (correctable)"
}
if len(uncorrNames) > 0 {
return strings.Join(uncorrNames, ", ") + " (uncorrectable)"
}
return fmt.Sprintf("unknown bits: 0x%08x", val)
}
// renderSparkline returns a small inline SVG showing non-OK events over time.
// Events are positioned proportionally along the time axis; if all share the same
// timestamp they are spaced evenly. Width is always 100px.
func renderSparkline(history []app.ComponentStatusEntry) string {
const (
svgW = 100
svgH = 20
barW = 3
barH = 14
)
var events []app.ComponentStatusEntry
for _, e := range history {
if e.Status != "OK" {
events = append(events, e)
}
}
if len(events) == 0 {
return ""
}
n := len(events)
barColor := func(status string) string {
if status == "Critical" {
return "#c0392b"
}
return "#d97706"
}
yTop := (svgH - barH) / 2
var bars strings.Builder
if n == 1 {
x := (svgW - barW) / 2
fmt.Fprintf(&bars, `<rect x="%d" y="%d" width="%d" height="%d" fill="%s" rx="1"/>`,
x, yTop, barW, barH, barColor(events[0].Status))
} else {
minT := events[0].At
maxT := events[n-1].At
dur := maxT.Sub(minT).Seconds()
for i, e := range events {
var x int
if dur <= 0 {
step := svgW / n
x = i*step + (step-barW)/2
} else {
frac := e.At.Sub(minT).Seconds() / dur
x = int(frac * float64(svgW-barW))
}
fmt.Fprintf(&bars, `<rect x="%d" y="%d" width="%d" height="%d" fill="%s" rx="1"/>`,
x, yTop, barW, barH, barColor(e.Status))
}
}
return fmt.Sprintf(
`<svg width="%d" height="%d" style="display:inline-block;vertical-align:middle;margin-left:6px;flex-shrink:0" xmlns="http://www.w3.org/2000/svg">`+
`<rect x="0" y="0" width="%d" height="%d" fill="var(--surface-alt,#ebebeb)" rx="3"/>%s</svg>`,
svgW, svgH, svgW, svgH, bars.String())
}
// renderComponentDetail renders a modal content fragment for one component type.
// Called by handleAPIComponentDetail and displayed inside #component-detail-dialog.
// fromInventory marks that records were synthesized from the audit inventory
// snapshot (no ComponentStatusDB history yet) rather than real SAT/watchdog
// observations — see inventoryFallbackRecords.
func renderComponentDetail(title string, records []app.ComponentStatusRecord, fromInventory bool) string {
var b strings.Builder
fmt.Fprintf(&b, `<div style="padding:20px 24px 0">`)
fmt.Fprintf(&b, `<div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:16px">`)
fmt.Fprintf(&b, `<span style="font-size:16px;font-weight:700">%s — Status Detail</span>`, html.EscapeString(title))
b.WriteString(`<button class="btn btn-sm btn-secondary" onclick="document.getElementById('component-detail-dialog').close()">Close</button>`)
b.WriteString(`</div>`)
if len(records) == 0 {
b.WriteString(`<p style="color:var(--muted)">No status data recorded yet for this component type.</p>`)
b.WriteString(`</div>`)
return b.String()
}
if fromInventory {
b.WriteString(`<p style="color:var(--muted);font-size:12px;margin-top:-8px;margin-bottom:16px">No SAT-test history yet — showing latest inventory snapshot.</p>`)
}
sort.Slice(records, func(i, j int) bool {
return records[i].ComponentKey < records[j].ComponentKey
})
for _, rec := range records {
letter, cls := chipLetterClass(rec.Status)
// Count non-OK events across the full history for the badge + sparkline.
warnCount := 0
for _, e := range rec.History {
if e.Status != "OK" {
warnCount++
}
}
fmt.Fprintf(&b, `<div style="margin-bottom:20px">`)
fmt.Fprintf(&b, `<div style="display:flex;align-items:center;gap:8px;margin-bottom:8px;flex-wrap:wrap">`)
fmt.Fprintf(&b, `<span class="chip %s">%s</span>`, cls, letter)
fmt.Fprintf(&b, `<span style="font-weight:700;font-size:13px">%s</span>`, html.EscapeString(rec.ComponentKey))
if !rec.LastCheckedAt.IsZero() {
fmt.Fprintf(&b, `<span style="color:var(--muted);font-size:12px">checked %s</span>`, rec.LastCheckedAt.Format("2006-01-02 15:04:05"))
}
if warnCount > 0 {
noun := "events"
if warnCount == 1 {
noun = "event"
}
fmt.Fprintf(&b,
`<span style="font-size:11px;background:var(--warn-bg,#fffbeb);color:var(--warn-fg,#92400e);border:1px solid var(--warn-border,#fde68a);border-radius:10px;padding:1px 7px;white-space:nowrap">%d %s</span>`,
warnCount, noun)
b.WriteString(renderSparkline(rec.History))
}
b.WriteString(`</div>`)
if rec.ErrorSummary != "" {
fmt.Fprintf(&b, `<div style="font-size:12px;margin-bottom:4px;color:var(--muted)">%s</div>`, html.EscapeString(rec.ErrorSummary))
if decoded := decodeAERStatus(rec.ErrorSummary); decoded != "" {
fmt.Fprintf(&b,
`<div style="font-size:12px;margin-bottom:8px;color:var(--muted)"><span style="background:var(--surface-alt,#f5f5f5);border-radius:4px;padding:1px 6px;font-family:monospace">AER: %s</span></div>`,
html.EscapeString(decoded))
}
}
// History table — newest first, cap at 20 entries.
history := rec.History
if len(history) > 20 {
history = history[len(history)-20:]
}
b.WriteString(`<table style="width:100%;font-size:12px;border-collapse:collapse">`)
b.WriteString(`<tr style="color:var(--muted)"><th style="text-align:left;padding:2px 10px 2px 0;white-space:nowrap">Time</th><th style="text-align:left;padding:2px 10px 2px 0">Status</th><th style="text-align:left;padding:2px 10px 2px 0">Source</th><th style="text-align:left;padding:2px 0">Detail</th></tr>`)
for i := len(history) - 1; i >= 0; i-- {
e := history[i]
eLetter, eCls := chipLetterClass(e.Status)
detail := e.Detail
if detail == "" {
detail = "—"
}
fmt.Fprintf(&b,
`<tr><td style="padding:3px 10px 3px 0;white-space:nowrap;color:var(--muted)">%s</td><td style="padding:3px 10px 3px 0"><span class="chip %s" style="font-size:10px;width:16px;height:16px">%s</span></td><td style="padding:3px 10px 3px 0;white-space:nowrap">%s</td><td style="padding:3px 0;color:var(--muted)">%s</td></tr>`,
html.EscapeString(e.At.Format("2006-01-02 15:04:05")),
eCls, eLetter,
html.EscapeString(e.Source),
html.EscapeString(detail),
)
}
b.WriteString(`</table>`)
b.WriteString(`</div>`)
}
b.WriteString(`</div>`)
return b.String()
}
audit/internal/webui/pages_health.go
+572
View File
@@ -0,0 +1,572 @@
package webui
import (
"encoding/json"
"fmt"
"html"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"bee/audit/internal/app"
"bee/audit/internal/schema"
)
func renderHealthCard(opts HandlerOptions) string {
data, err := loadSnapshot(filepath.Join(opts.ExportDir, "runtime-health.json"))
if err != nil {
return `<div class="card"><div class="card-head">Runtime Health</div><div class="card-body"><span class="badge badge-unknown">No data</span></div></div>`
}
var health schema.RuntimeHealth
if err := json.Unmarshal(data, &health); err != nil {
return `<div class="card"><div class="card-head">Runtime Health</div><div class="card-body"><span class="badge badge-err">Parse error</span></div></div>`
}
status := strings.TrimSpace(health.Status)
if status == "" {
status = "UNKNOWN"
}
badge := "badge-ok"
if status == "PARTIAL" {
badge = "badge-warn"
} else if status == "FAIL" || status == "FAILED" {
badge = "badge-err"
}
var b strings.Builder
b.WriteString(`<div class="card"><div class="card-head">Runtime Health</div><div class="card-body">`)
b.WriteString(fmt.Sprintf(`<div style="margin-bottom:10px"><span class="badge %s">%s</span></div>`, badge, html.EscapeString(status)))
if checkedAt := strings.TrimSpace(health.CheckedAt); checkedAt != "" {
b.WriteString(`<div style="font-size:12px;color:var(--muted);margin-bottom:12px">Checked at: ` + html.EscapeString(checkedAt) + `</div>`)
}
rows := []runtimeHealthRow{
buildRuntimeExportRow(health),
buildRuntimeNetworkRow(health),
buildRuntimeDriverRow(health),
buildRuntimeAccelerationRow(health),
buildRuntimeToolsRow(health),
buildRuntimeServicesRow(health),
buildRuntimeUSBExportRow(health),
buildRuntimeToRAMRow(health),
}
b.WriteString(`<table><thead><tr><th>Check</th><th>Status</th><th>Source</th><th>Issue</th></tr></thead><tbody>`)
for _, row := range rows {
b.WriteString(`<tr><td>` + html.EscapeString(row.Title) + `</td><td>` + runtimeStatusBadge(row.Status) + `</td><td>` + html.EscapeString(row.Source) + `</td><td>` + rowIssueHTML(row.Issue) + `</td></tr>`)
}
b.WriteString(`</tbody></table>`)
b.WriteString(`</div></div>`)
return b.String()
}
type runtimeHealthRow struct {
Title string
Status string
Source string
Issue string
}
func buildRuntimeExportRow(health schema.RuntimeHealth) runtimeHealthRow {
issue := runtimeIssueDescriptions(health.Issues, "export_dir_unavailable")
status := "UNKNOWN"
switch {
case issue != "":
status = "FAILED"
case strings.TrimSpace(health.ExportDir) != "":
status = "OK"
}
source := "os.MkdirAll"
if dir := strings.TrimSpace(health.ExportDir); dir != "" {
source += " " + dir
}
return runtimeHealthRow{Title: "Export Directory", Status: status, Source: source, Issue: issue}
}
func buildRuntimeNetworkRow(health schema.RuntimeHealth) runtimeHealthRow {
status := strings.TrimSpace(health.NetworkStatus)
if status == "" {
status = "UNKNOWN"
}
issue := runtimeIssueDescriptions(health.Issues, "dhcp_failed")
return runtimeHealthRow{Title: "Network", Status: status, Source: "ListInterfaces / DHCP", Issue: issue}
}
func buildRuntimeDriverRow(health schema.RuntimeHealth) runtimeHealthRow {
issue := runtimeIssueDescriptions(health.Issues, "nvidia_kernel_module_missing", "nvidia_modeset_failed", "amdgpu_kernel_module_missing")
status := "UNKNOWN"
switch {
case health.DriverReady && issue == "":
status = "OK"
case health.DriverReady:
status = "PARTIAL"
case issue != "":
status = "FAILED"
}
return runtimeHealthRow{Title: "NVIDIA/AMD Driver", Status: status, Source: "lsmod / vendor probe", Issue: issue}
}
func buildRuntimeAccelerationRow(health schema.RuntimeHealth) runtimeHealthRow {
issue := runtimeIssueDescriptions(health.Issues, "cuda_runtime_not_ready", "rocm_smi_unavailable")
status := "UNKNOWN"
switch {
case health.CUDAReady && issue == "":
status = "OK"
case health.CUDAReady:
status = "PARTIAL"
case issue != "":
status = "FAILED"
}
return runtimeHealthRow{Title: "CUDA / ROCm", Status: status, Source: "bee-gpu-burn / rocm-smi", Issue: issue}
}
func buildRuntimeToolsRow(health schema.RuntimeHealth) runtimeHealthRow {
if len(health.Tools) == 0 {
return runtimeHealthRow{Title: "Required Utilities", Status: "UNKNOWN", Source: "CheckTools", Issue: "No tool status data."}
}
missing := make([]string, 0)
for _, tool := range health.Tools {
if !tool.OK {
missing = append(missing, tool.Name)
}
}
status := "OK"
issue := ""
if len(missing) > 0 {
status = "PARTIAL"
issue = "Missing: " + strings.Join(missing, ", ")
}
return runtimeHealthRow{Title: "Required Utilities", Status: status, Source: "CheckTools", Issue: issue}
}
func buildRuntimeServicesRow(health schema.RuntimeHealth) runtimeHealthRow {
if len(health.Services) == 0 {
return runtimeHealthRow{Title: "Bee Services", Status: "UNKNOWN", Source: "systemctl is-active", Issue: "No service status data."}
}
nonActive := make([]string, 0)
for _, svc := range health.Services {
state := strings.TrimSpace(strings.ToLower(svc.Status))
// "inactive" is OK for oneshot services that have completed successfully
// (bee-sshsetup, bee-preflight, bee-audit, bee-network, etc.).
// Only "failed" is a genuine problem.
switch state {
case "active", "activating", "deactivating", "reloading", "inactive":
// OK — service is running, transitioning normally, or completed successfully
default:
nonActive = append(nonActive, svc.Name+"="+svc.Status)
}
}
status := "OK"
issue := ""
if len(nonActive) > 0 {
status = "PARTIAL"
issue = strings.Join(nonActive, ", ")
}
return runtimeHealthRow{Title: "Bee Services", Status: status, Source: "ServiceState", Issue: issue}
}
func buildRuntimeUSBExportRow(health schema.RuntimeHealth) runtimeHealthRow {
path := strings.TrimSpace(health.USBExportPath)
if path != "" {
return runtimeHealthRow{
Title: "USB Export Drive",
Status: "OK",
Source: "/proc/mounts + lsblk",
Issue: path,
}
}
return runtimeHealthRow{
Title: "USB Export Drive",
Status: "WARNING",
Source: "/proc/mounts + lsblk",
Issue: "No writable USB drive mounted. Plug in a USB drive to enable log export.",
}
}
func buildRuntimeToRAMRow(health schema.RuntimeHealth) runtimeHealthRow {
switch strings.ToLower(strings.TrimSpace(health.ToRAMStatus)) {
case "ok":
return runtimeHealthRow{
Title: "LiveCD in RAM",
Status: "OK",
Source: "live-boot / /proc/mounts",
Issue: "",
}
case "partial":
return runtimeHealthRow{
Title: "LiveCD in RAM",
Status: "WARNING",
Source: "live-boot / /proc/mounts / /dev/shm/bee-live",
Issue: "Partial or staged RAM copy detected. System is not fully running from RAM; Copy to RAM can be retried.",
}
case "failed":
return runtimeHealthRow{
Title: "LiveCD in RAM",
Status: "FAILED",
Source: "live-boot / /proc/mounts",
Issue: "toram boot parameter set but ISO is not mounted from RAM. Copy may have failed.",
}
default:
// toram not active — ISO still on original boot media (USB/CD)
return runtimeHealthRow{
Title: "LiveCD in RAM",
Status: "WARNING",
Source: "live-boot / /proc/mounts",
Issue: "ISO not copied to RAM. Use \u201cCopy to RAM\u201d to free the boot drive and improve performance.",
}
}
}
// matchedRecords returns all ComponentStatusRecord entries whose key matches
// any exact key or any of the given prefixes. Used for per-device chip rendering.
func firstNonEmpty(vals ...string) string {
for _, v := range vals {
if v != "" {
return v
}
}
return ""
}
func matchedRecords(records []app.ComponentStatusRecord, exact []string, prefixes []string) []app.ComponentStatusRecord {
var matched []app.ComponentStatusRecord
for _, rec := range records {
key := strings.TrimSpace(rec.ComponentKey)
if key == "" {
continue
}
if containsExactKey(key, exact) || hasAnyPrefix(key, prefixes) {
matched = append(matched, rec)
}
}
return matched
}
func containsExactKey(key string, exact []string) bool {
for _, candidate := range exact {
if key == candidate {
return true
}
}
return false
}
func hasAnyPrefix(key string, prefixes []string) bool {
for _, prefix := range prefixes {
if strings.HasPrefix(key, prefix) {
return true
}
}
return false
}
func runtimeIssueDescriptions(issues []schema.RuntimeIssue, codes ...string) string {
if len(issues) == 0 || len(codes) == 0 {
return ""
}
allowed := make(map[string]struct{}, len(codes))
for _, code := range codes {
allowed[code] = struct{}{}
}
messages := make([]string, 0)
for _, issue := range issues {
if _, ok := allowed[issue.Code]; !ok {
continue
}
desc := strings.TrimSpace(issue.Description)
if desc == "" {
desc = issue.Code
}
messages = append(messages, desc)
}
return strings.Join(messages, "; ")
}
// gpuNeedsPhysicalReboot reports whether any GPU component record carries a
// hardware-fault reason that a driver reset can't clear (e.g. Xid 79 "GPU
// has fallen off the bus", Xid 154 "Node Reboot Required" — see
// collector.xidHardwareFaultMessages). Those errors mean every subsequent
// GPU SAT job will keep failing until the node is physically power-cycled,
// so this drives a dashboard banner that says so up front instead of making
// an operator burn another test cycle to rediscover it.
func gpuNeedsPhysicalReboot(records []app.ComponentStatusRecord) (reason string, needsReboot bool) {
for _, rec := range records {
if !strings.EqualFold(strings.TrimSpace(rec.Status), "Critical") {
continue
}
if strings.Contains(strings.ToLower(rec.ErrorSummary), "reboot") {
return rec.ErrorSummary, true
}
}
return "", false
}
// chipLetterClass maps a component status to a single display letter and CSS class.
func chipLetterClass(status string) (letter, cls string) {
switch strings.ToUpper(strings.TrimSpace(status)) {
case "OK":
return "O", "chip-ok"
case "WARNING", "WARN", "PARTIAL":
return "W", "chip-warn"
case "CRITICAL", "FAIL", "FAILED", "ERROR":
return "F", "chip-fail"
default:
return "?", "chip-unknown"
}
}
// renderComponentChips renders one 20×20 chip per ComponentStatusRecord.
// Hover tooltip shows component key, status, error summary and last check time.
// Falls back to a single unknown chip when no records are available.
func renderComponentChips(matched []app.ComponentStatusRecord) string {
if len(matched) == 0 {
return `<span class="chips"><span class="chip chip-unknown" title="No data">?</span></span>`
}
sort.Slice(matched, func(i, j int) bool {
return matched[i].ComponentKey < matched[j].ComponentKey
})
var b strings.Builder
b.WriteString(`<span class="chips">`)
for _, rec := range matched {
letter, cls := chipLetterClass(rec.Status)
var tooltip strings.Builder
tooltip.WriteString(rec.ComponentKey)
tooltip.WriteString(": ")
tooltip.WriteString(firstNonEmpty(rec.Status, "UNKNOWN"))
if rec.ErrorSummary != "" {
tooltip.WriteString(" — ")
tooltip.WriteString(rec.ErrorSummary)
}
if !rec.LastCheckedAt.IsZero() {
fmt.Fprintf(&tooltip, " (checked %s)", rec.LastCheckedAt.Format("15:04:05"))
}
fmt.Fprintf(&b, `<span class="chip %s" title="%s">%s</span>`,
cls, html.EscapeString(tooltip.String()), letter)
}
b.WriteString(`</span>`)
return b.String()
}
func runtimeStatusBadge(status string) string {
status = strings.ToUpper(strings.TrimSpace(status))
badge := "badge-unknown"
switch status {
case "OK":
badge = "badge-ok"
case "PARTIAL", "WARNING", "WARN":
badge = "badge-warn"
case "FAIL", "FAILED", "CRITICAL":
badge = "badge-err"
}
return `<span class="badge ` + badge + `">` + html.EscapeString(status) + `</span>`
}
func rowIssueHTML(issue string) string {
issue = strings.TrimSpace(issue)
if issue == "" {
return `<span style="color:var(--muted)">—</span>`
}
return html.EscapeString(issue)
}
var aerStatusRe = regexp.MustCompile(`aer_status:\s*0x([0-9a-fA-F]{1,8})`)
// decodeAERStatus parses an AER status hex value from a kernel error detail string
// and returns a human-readable list of set bit names with correctable/uncorrectable label,
// or "" if no AER status is found.
func decodeAERStatus(detail string) string {
m := aerStatusRe.FindStringSubmatch(detail)
if m == nil {
return ""
}
v64, err := strconv.ParseUint(m[1], 16, 32)
if err != nil {
return ""
}
val := uint32(v64)
type bitDef struct {
bit uint32
name string
}
corrBits := []bitDef{
{0, "Receiver Error"}, {6, "Replay Timer Timeout"}, {7, "Advisory Non-Fatal"},
{8, "Corrected Internal Error"}, {9, "Header Log Overflow"},
{13, "Replay Num Rollover"}, {14, "Bad DLLP"}, {15, "Bad TLP"},
}
uncorrBits := []bitDef{
{4, "Data Link Protocol Error"}, {5, "Surprise Down Error"},
{12, "Poisoned TLP Received"}, {13, "Flow Control Protocol Error"},
{14, "Completion Timeout"}, {15, "Completer Abort"}, {16, "Unexpected Completion"},
{17, "Receiver Overflow"}, {18, "Malformed TLP"}, {19, "ECRC Error"},
{20, "Unsupported Request Error"}, {21, "ACS Violation"}, {22, "Uncorrectable Internal Error"},
}
var corrNames, uncorrNames []string
for _, b := range corrBits {
if val&(1<<b.bit) != 0 {
corrNames = append(corrNames, b.name)
}
}
for _, b := range uncorrBits {
if val&(1<<b.bit) != 0 {
uncorrNames = append(uncorrNames, b.name)
}
}
if len(corrNames) >= len(uncorrNames) && len(corrNames) > 0 {
return strings.Join(corrNames, ", ") + " (correctable)"
}
if len(uncorrNames) > 0 {
return strings.Join(uncorrNames, ", ") + " (uncorrectable)"
}
return fmt.Sprintf("unknown bits: 0x%08x", val)
}
// renderSparkline returns a small inline SVG showing non-OK events over time.
// Events are positioned proportionally along the time axis; if all share the same
// timestamp they are spaced evenly. Width is always 100px.
func renderSparkline(history []app.ComponentStatusEntry) string {
const (
svgW = 100
svgH = 20
barW = 3
barH = 14
)
var events []app.ComponentStatusEntry
for _, e := range history {
if e.Status != "OK" {
events = append(events, e)
}
}
if len(events) == 0 {
return ""
}
n := len(events)
barColor := func(status string) string {
if status == "Critical" {
return "#c0392b"
}
return "#d97706"
}
yTop := (svgH - barH) / 2
var bars strings.Builder
if n == 1 {
x := (svgW - barW) / 2
fmt.Fprintf(&bars, `<rect x="%d" y="%d" width="%d" height="%d" fill="%s" rx="1"/>`,
x, yTop, barW, barH, barColor(events[0].Status))
} else {
minT := events[0].At
maxT := events[n-1].At
dur := maxT.Sub(minT).Seconds()
for i, e := range events {
var x int
if dur <= 0 {
step := svgW / n
x = i*step + (step-barW)/2
} else {
frac := e.At.Sub(minT).Seconds() / dur
x = int(frac * float64(svgW-barW))
}
fmt.Fprintf(&bars, `<rect x="%d" y="%d" width="%d" height="%d" fill="%s" rx="1"/>`,
x, yTop, barW, barH, barColor(e.Status))
}
}
return fmt.Sprintf(
`<svg width="%d" height="%d" style="display:inline-block;vertical-align:middle;margin-left:6px;flex-shrink:0" xmlns="http://www.w3.org/2000/svg">`+
`<rect x="0" y="0" width="%d" height="%d" fill="var(--surface-alt,#ebebeb)" rx="3"/>%s</svg>`,
svgW, svgH, svgW, svgH, bars.String())
}
// renderComponentDetail renders a modal content fragment for one component type.
// Called by handleAPIComponentDetail and displayed inside #component-detail-dialog.
// fromInventory marks that records were synthesized from the audit inventory
// snapshot (no ComponentStatusDB history yet) rather than real SAT/watchdog
// observations — see inventoryFallbackRecords.
func renderComponentDetail(title string, records []app.ComponentStatusRecord, fromInventory bool) string {
var b strings.Builder
fmt.Fprintf(&b, `<div style="padding:20px 24px 0">`)
fmt.Fprintf(&b, `<div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:16px">`)
fmt.Fprintf(&b, `<span style="font-size:16px;font-weight:700">%s — Status Detail</span>`, html.EscapeString(title))
b.WriteString(`<button class="btn btn-sm btn-secondary" onclick="document.getElementById('component-detail-dialog').close()">Close</button>`)
b.WriteString(`</div>`)
if len(records) == 0 {
b.WriteString(`<p style="color:var(--muted)">No status data recorded yet for this component type.</p>`)
b.WriteString(`</div>`)
return b.String()
}
if fromInventory {
b.WriteString(`<p style="color:var(--muted);font-size:12px;margin-top:-8px;margin-bottom:16px">No SAT-test history yet — showing latest inventory snapshot.</p>`)
}
sort.Slice(records, func(i, j int) bool {
return records[i].ComponentKey < records[j].ComponentKey
})
for _, rec := range records {
letter, cls := chipLetterClass(rec.Status)
// Count non-OK events across the full history for the badge + sparkline.
warnCount := 0
for _, e := range rec.History {
if e.Status != "OK" {
warnCount++
}
}
fmt.Fprintf(&b, `<div style="margin-bottom:20px">`)
fmt.Fprintf(&b, `<div style="display:flex;align-items:center;gap:8px;margin-bottom:8px;flex-wrap:wrap">`)
fmt.Fprintf(&b, `<span class="chip %s">%s</span>`, cls, letter)
fmt.Fprintf(&b, `<span style="font-weight:700;font-size:13px">%s</span>`, html.EscapeString(rec.ComponentKey))
if !rec.LastCheckedAt.IsZero() {
fmt.Fprintf(&b, `<span style="color:var(--muted);font-size:12px">checked %s</span>`, rec.LastCheckedAt.Format("2006-01-02 15:04:05"))
}
if warnCount > 0 {
noun := "events"
if warnCount == 1 {
noun = "event"
}
fmt.Fprintf(&b,
`<span style="font-size:11px;background:var(--warn-bg,#fffbeb);color:var(--warn-fg,#92400e);border:1px solid var(--warn-border,#fde68a);border-radius:10px;padding:1px 7px;white-space:nowrap">%d %s</span>`,
warnCount, noun)
b.WriteString(renderSparkline(rec.History))
}
b.WriteString(`</div>`)
if rec.ErrorSummary != "" {
fmt.Fprintf(&b, `<div style="font-size:12px;margin-bottom:4px;color:var(--muted)">%s</div>`, html.EscapeString(rec.ErrorSummary))
if decoded := decodeAERStatus(rec.ErrorSummary); decoded != "" {
fmt.Fprintf(&b,
`<div style="font-size:12px;margin-bottom:8px;color:var(--muted)"><span style="background:var(--surface-alt,#f5f5f5);border-radius:4px;padding:1px 6px;font-family:monospace">AER: %s</span></div>`,
html.EscapeString(decoded))
}
}
// History table — newest first, cap at 20 entries.
history := rec.History
if len(history) > 20 {
history = history[len(history)-20:]
}
b.WriteString(`<table style="width:100%;font-size:12px;border-collapse:collapse">`)
b.WriteString(`<tr style="color:var(--muted)"><th style="text-align:left;padding:2px 10px 2px 0;white-space:nowrap">Time</th><th style="text-align:left;padding:2px 10px 2px 0">Status</th><th style="text-align:left;padding:2px 10px 2px 0">Source</th><th style="text-align:left;padding:2px 0">Detail</th></tr>`)
for i := len(history) - 1; i >= 0; i-- {
e := history[i]
eLetter, eCls := chipLetterClass(e.Status)
detail := e.Detail
if detail == "" {
detail = "—"
}
fmt.Fprintf(&b,
`<tr><td style="padding:3px 10px 3px 0;white-space:nowrap;color:var(--muted)">%s</td><td style="padding:3px 10px 3px 0"><span class="chip %s" style="font-size:10px;width:16px;height:16px">%s</span></td><td style="padding:3px 10px 3px 0;white-space:nowrap">%s</td><td style="padding:3px 0;color:var(--muted)">%s</td></tr>`,
html.EscapeString(e.At.Format("2006-01-02 15:04:05")),
eCls, eLetter,
html.EscapeString(e.Source),
html.EscapeString(detail),
)
}
b.WriteString(`</table>`)
b.WriteString(`</div>`)
}
b.WriteString(`</div>`)
return b.String()
}
audit/internal/webui/raid_actions.go
+707
View File
@@ -0,0 +1,707 @@
package webui
import (
"context"
"encoding/json"
"fmt"
"net/http"
"os/exec"
"strconv"
"strings"
"time"
)
func (h *handler) handleAPIRAIDStatus(w http.ResponseWriter, r *http.Request) {
resp := raidStatusResp{Controllers: []raidControllerInfo{}}
lsi2 := detectStorcli2Controllers()
if lsi := detectLSIControllers(); len(lsi) > 0 {
// storcli64 can enumerate a Tri-Mode controller (SAS3808-iMR/9500
// series) at a basic level but its drive-listing JSON parser finds
// no "Drive Information" for these — a zero-drives entry that
// storcli2 (run above) already covers correctly. Only drop it when
// storcli2 actually found something, so a genuinely drive-populated
// classic controller elsewhere in a mixed setup is never hidden.
for _, c := range lsi {
if len(c.AllDrives) == 0 && len(lsi2) > 0 {
continue
}
resp.Controllers = append(resp.Controllers, c)
}
}
if len(lsi2) > 0 {
resp.Controllers = append(resp.Controllers, lsi2...)
}
if vroc := detectVROCController(); vroc != nil {
resp.Controllers = append(resp.Controllers, *vroc)
}
writeJSON(w, resp)
}
func (h *handler) handleAPIRAIDForeignAction(w http.ResponseWriter, r *http.Request) {
var req struct {
ControllerID string `json:"controller_id"`
Action string `json:"action"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON")
return
}
if req.Action != "import" && req.Action != "clear" {
writeError(w, http.StatusBadRequest, "action must be 'import' or 'clear'")
return
}
ctrlIdx, ok := parseLSIControllerIndex(req.ControllerID)
if !ok {
writeError(w, http.StatusBadRequest, "invalid controller_id")
return
}
target := "raid-foreign-clear"
name := fmt.Sprintf("RAID Foreign Clear (ctrl %d)", ctrlIdx)
if req.Action == "import" {
target = "raid-foreign-import"
name = fmt.Sprintf("RAID Foreign Import (ctrl %d)", ctrlIdx)
}
t := &Task{
ID: newJobID(target),
Name: name,
Target: target,
Priority: defaultTaskPriority(target, taskParams{}),
Status: TaskPending,
CreatedAt: time.Now(),
params: taskParams{RAIDController: ctrlIdx},
}
globalQueue.enqueue(t)
writeJSON(w, map[string]string{"task_id": t.ID})
}
func (h *handler) handleAPIRAIDCreateMirror(w http.ResponseWriter, r *http.Request) {
var req struct {
ControllerID string `json:"controller_id"`
Devices []string `json:"devices"`
ArrayName string `json:"array_name"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON")
return
}
if len(req.Devices) < 2 {
writeError(w, http.StatusBadRequest, "at least 2 devices required")
return
}
var target, name string
var params taskParams
switch {
case strings.HasPrefix(req.ControllerID, "lsi-"):
ctrlIdx, ok := parseLSIControllerIndex(req.ControllerID)
if !ok {
writeError(w, http.StatusBadRequest, "invalid controller_id")
return
}
target = "raid-lsi-create-mirror"
name = fmt.Sprintf("Create RAID 1 Mirror (LSI ctrl %d)", ctrlIdx)
params = taskParams{RAIDController: ctrlIdx, RAIDDevices: req.Devices}
case req.ControllerID == "vroc-0":
arrayName := strings.TrimSpace(req.ArrayName)
if arrayName == "" {
arrayName = "bee-mirror0"
}
target = "raid-vroc-create-mirror"
name = fmt.Sprintf("Create VROC RAID 1 (%s)", arrayName)
params = taskParams{RAIDDevices: req.Devices, RAIDArrayName: arrayName}
default:
writeError(w, http.StatusBadRequest, "unknown controller_id")
return
}
t := &Task{
ID: newJobID(target),
Name: name,
Target: target,
Priority: defaultTaskPriority(target, taskParams{}),
Status: TaskPending,
CreatedAt: time.Now(),
params: params,
}
globalQueue.enqueue(t)
writeJSON(w, map[string]string{"task_id": t.ID})
}
func (h *handler) handleAPIRAIDPrepareDrive(w http.ResponseWriter, r *http.Request) {
var req struct {
ControllerID string `json:"controller_id"`
Slot string `json:"slot"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON")
return
}
ctrlIdx, ok := parseLSIControllerIndex(req.ControllerID)
if !ok {
writeError(w, http.StatusBadRequest, "invalid controller_id")
return
}
if _, _, ok := parseRAIDSlot(req.Slot); !ok {
writeError(w, http.StatusBadRequest, "invalid slot")
return
}
t := &Task{
ID: newJobID("raid-lsi-prepare-drive"),
Name: fmt.Sprintf("Prepare drive %s (LSI ctrl %d)", req.Slot, ctrlIdx),
Target: "raid-lsi-prepare-drive",
Priority: defaultTaskPriority("raid-lsi-prepare-drive", taskParams{}),
Status: TaskPending,
CreatedAt: time.Now(),
params: taskParams{RAIDController: ctrlIdx, RAIDSlot: req.Slot},
}
globalQueue.enqueue(t)
writeJSON(w, map[string]string{"task_id": t.ID})
}
func parseLSIControllerIndex(id string) (int, bool) {
if !strings.HasPrefix(id, "lsi-") {
return 0, false
}
n, err := strconv.Atoi(strings.TrimPrefix(id, "lsi-"))
if err != nil || n < 0 {
return 0, false
}
return n, true
}
// --- Task runner functions ---
func runRAIDForeignClearTask(ctx context.Context, j *jobState, ctrl int) error {
j.append(fmt.Sprintf("Clearing foreign configuration on controller %d...", ctrl))
cmd := exec.CommandContext(ctx, "storcli64", fmt.Sprintf("/c%d/fall", ctrl), "del", "noprompt")
return streamCmdJob(j, cmd)
}
func runRAIDForeignImportTask(ctx context.Context, j *jobState, ctrl int) error {
j.append(fmt.Sprintf("Importing foreign configuration on controller %d...", ctrl))
cmd := exec.CommandContext(ctx, "storcli64", fmt.Sprintf("/c%d/fall", ctrl), "import", "noprompt")
return streamCmdJob(j, cmd)
}
// raidPrepareAction says what (if anything) must be done to a drive in the
// given storcli state before it can join a new VD. Derived from the Broadcom
// StorCLI drive-state matrix: only UGood drives are accepted by "add vd";
// JBOD/UBad convert with "set good force"; hotspares must be released first;
// Frgn/Onln hold configuration data and must not be silently destroyed.
type raidPrepareAction int
const (
raidPrepNone raidPrepareAction = iota // UGood or unknown — try add vd as-is
raidPrepSetGood // JBOD, UBad — "set good force"
raidPrepHotspare // GHS, DHS — "delete hotsparedrive", then set good
raidPrepBlockedFrgn
raidPrepBlockedOnln
)
func classifyRAIDPrepareAction(state string) raidPrepareAction {
switch strings.TrimSpace(state) {
case "JBOD", "UBad":
return raidPrepSetGood
case "GHS", "DHS":
return raidPrepHotspare
case "Frgn":
return raidPrepBlockedFrgn
case "Onln", "Offln":
return raidPrepBlockedOnln
default: // "UGood", "" (state unknown — let add vd decide)
return raidPrepNone
}
}
// raidLSIDriveStates returns EID:Slt -> State for one controller, or nil if
// storcli/parsing fails (callers then fall back to unconditional prepare).
func raidLSIDriveStates(ctx context.Context, ctrl int) map[string]string {
out, err := exec.CommandContext(ctx, "storcli64",
fmt.Sprintf("/c%d/eall/sall", ctrl), "show", "all", "J").Output()
if err != nil {
return nil
}
var doc struct {
Controllers []struct {
ResponseData map[string]json.RawMessage `json:"Response Data"`
} `json:"Controllers"`
}
if err := json.Unmarshal(out, &doc); err != nil || len(doc.Controllers) == 0 {
return nil
}
states := map[string]string{}
for _, c := range doc.Controllers {
for _, d := range parseStorcliResponseDataDrives(c.ResponseData) {
states[strings.TrimSpace(d.EIDSlt)] = strings.TrimSpace(d.State)
}
}
return states
}
func runRAIDLSICreateMirrorTask(ctx context.Context, j *jobState, ctrl int, drives []string) error {
driveList := strings.Join(drives, ",")
states := raidLSIDriveStates(ctx, ctrl)
// Non-UGood drives cannot be added to a VD directly — storcli fails with
// "resources already in use" (exit 11) or similar. Fix what is safely
// fixable (JBOD/UBad/hotspare), refuse what holds data (Frgn/Onln).
for _, drive := range drives {
eid, slt, ok := parseRAIDSlot(drive)
if !ok {
return fmt.Errorf("invalid drive slot %q", drive)
}
state := states[drive]
slotPath := fmt.Sprintf("/c%d/e%d/s%d", ctrl, eid, slt)
switch classifyRAIDPrepareAction(state) {
case raidPrepBlockedFrgn:
return fmt.Errorf("drive %s carries a foreign configuration; run the RAID Foreign Clear (or Import) task first, then retry", drive)
case raidPrepBlockedOnln:
return fmt.Errorf("drive %s is part of an existing virtual drive (state %s); delete that VD first", drive, state)
case raidPrepHotspare:
j.append(fmt.Sprintf("Drive %s is a hotspare (%s); releasing it...", drive, state))
rel := exec.CommandContext(ctx, "storcli64", slotPath, "delete", "hotsparedrive")
if err := streamCmdJob(j, rel); err != nil {
return fmt.Errorf("release hotspare %s: %w", drive, err)
}
case raidPrepSetGood:
j.append(fmt.Sprintf("Drive %s is %s; converting to Unconfigured Good (set good force)...", drive, state))
prep := exec.CommandContext(ctx, "storcli64", slotPath, "set", "good", "force")
if err := streamCmdJob(j, prep); err != nil {
return fmt.Errorf("set good on %s: %w", drive, err)
}
case raidPrepNone:
if state == "" {
// Drive state unknown (storcli query failed) — attempt the
// conversion anyway; harmless on an already-UGood drive with
// force, and add vd below is the real verdict.
j.append(fmt.Sprintf("Preparing drive %s (set good, force)...", drive))
prep := exec.CommandContext(ctx, "storcli64", slotPath, "set", "good", "force")
if err := streamCmdJob(j, prep); err != nil {
j.append(fmt.Sprintf("note: set good on %s: %v (continuing)", drive, err))
}
}
}
}
j.append(fmt.Sprintf("Creating RAID 1 on controller %d with drives: %s", ctrl, driveList))
cmd := exec.CommandContext(ctx, "storcli64",
fmt.Sprintf("/c%d", ctrl),
"add", "vd", "type=raid1",
fmt.Sprintf("drives=%s", driveList),
"pdperarray=2",
)
if err := streamCmdJob(j, cmd); err != nil {
// A blocked add vd is often preserved cache from a dead VD
// ("controller has data in cache for offline or missing virtual
// drives"). Surface it so the log is actionable.
j.append("add vd failed; checking for preserved cache...")
pc := exec.CommandContext(ctx, "storcli64", fmt.Sprintf("/c%d", ctrl), "show", "preservedcache")
_ = streamCmdJob(j, pc)
j.append(fmt.Sprintf("hint: if preserved cache is listed above, clear it with: storcli64 /c%d/vall delete preservedcache (invalidates cached data of dead VDs), then retry", ctrl))
return err
}
return nil
}
// parseRAIDSlot splits a storcli "EID:Slt" identifier (e.g. "252:0") into
// enclosure and slot numbers.
func parseRAIDSlot(slot string) (eid int, slt int, ok bool) {
parts := strings.SplitN(strings.TrimSpace(slot), ":", 2)
if len(parts) != 2 {
return 0, 0, false
}
eid, err1 := strconv.Atoi(strings.TrimSpace(parts[0]))
slt, err2 := strconv.Atoi(strings.TrimSpace(parts[1]))
if err1 != nil || err2 != nil {
return 0, 0, false
}
return eid, slt, true
}
func runRAIDPrepareDriveTask(ctx context.Context, j *jobState, ctrl int, slot string) error {
eid, slt, ok := parseRAIDSlot(slot)
if !ok {
return fmt.Errorf("invalid slot %q", slot)
}
j.append(fmt.Sprintf("Preparing drive %s on controller %d (set good, force)...", slot, ctrl))
cmd := exec.CommandContext(ctx, "storcli64",
fmt.Sprintf("/c%d/e%d/s%d", ctrl, eid, slt),
"set", "good", "force",
)
return streamCmdJob(j, cmd)
}
func runRAIDVROCCreateMirrorTask(ctx context.Context, j *jobState, devices []string, arrayName string) error {
if arrayName == "" {
arrayName = "bee-mirror0"
}
devPath := "/dev/md/" + arrayName
args := []string{
"--create", devPath,
"--level=1",
fmt.Sprintf("--raid-devices=%d", len(devices)),
"--run",
}
args = append(args, devices...)
j.append(fmt.Sprintf("Creating VROC RAID 1 array %s with: %s", devPath, strings.Join(devices, " ")))
cmd := exec.CommandContext(ctx, "mdadm", args...)
return streamCmdJob(j, cmd)
}
// raidParseHumanSizeGB parses storcli size strings like "1.818 TB", "745.211 GB".
func raidParseHumanSizeGB(s string) float64 {
s = strings.TrimSpace(s)
if s == "" {
return 0
}
upper := strings.ToUpper(s)
var mul float64
var numStr string
switch {
case strings.Contains(upper, " TB"):
mul = 1024
numStr = strings.TrimSpace(strings.SplitN(upper, " T", 2)[0])
case strings.Contains(upper, " GB"):
mul = 1
numStr = strings.TrimSpace(strings.SplitN(upper, " G", 2)[0])
case strings.Contains(upper, " MB"):
mul = 1.0 / 1024
numStr = strings.TrimSpace(strings.SplitN(upper, " M", 2)[0])
default:
return 0
}
v, err := strconv.ParseFloat(numStr, 64)
if err != nil {
return 0
}
return v * mul
}
// --- UI card ---
func renderRAIDMgmtCard() string {
return `<div class="card"><div class="card-head card-head-actions">RAID Controller Management<div class="card-head-buttons"><button class="btn btn-sm btn-secondary" onclick="raidLoad()">&#8635; Refresh</button></div></div><div class="card-body">
<div id="raid-status" style="font-size:13px;color:var(--muted);margin-bottom:8px">Loading...</div>
<div id="raid-content"></div>
<div id="raid-out-wrap" style="display:none;margin-top:14px">
<div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:4px">
<span id="raid-out-label" style="font-size:12px;font-weight:600;color:var(--muted)">Output</span>
<span id="raid-out-status" style="font-size:12px"></span>
</div>
<div id="raid-terminal" class="terminal" style="max-height:260px;width:100%;box-sizing:border-box"></div>
</div>
</div></div>
<script>
(function(){
function escHtml(s) {
return String(s||'').replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;').replace(/"/g,'&quot;');
}
var _raidControllers = [];
function raidLoad() {
var status = document.getElementById('raid-status');
var content = document.getElementById('raid-content');
status.textContent = 'Detecting RAID controllers...';
status.style.color = 'var(--muted)';
content.innerHTML = '';
fetch('/api/tools/raid/status', {cache:'no-store'})
.then(function(r) {
if (!r.ok) return r.json().then(function(e) { throw new Error(e.error || r.statusText); });
return r.json();
})
.then(function(data) {
_raidControllers = data.controllers || [];
if (_raidControllers.length === 0) {
status.textContent = 'No RAID controllers detected.';
return;
}
status.textContent = _raidControllers.length + ' controller(s) detected.';
content.innerHTML = _raidControllers.map(function(c, i) {
return raidRenderController(c, i);
}).join('<hr style="margin:16px 0;border:none;border-top:1px solid var(--border)">');
})
.catch(function(e) {
status.textContent = 'Error: ' + e.message;
status.style.color = 'var(--crit-fg)';
});
}
function raidRenderController(c, idx) {
var html = '';
var typeLabel = c.type === 'lsi' ? 'LSI / Broadcom' : 'Intel VROC';
html += '<div style="font-weight:600;font-size:13px;margin-bottom:10px">' + typeLabel + ' &mdash; ' + escHtml(c.model) + '</div>';
if (c.type === 'lsi') {
var foreign = c.foreign_drives || [];
if (foreign.length > 0) {
html += '<div style="background:var(--warn-bg,rgba(240,192,0,0.1));border:1px solid var(--warn-border,#c8a800);border-radius:4px;padding:10px 12px;margin-bottom:12px">';
html += '<div style="font-weight:600;font-size:13px;margin-bottom:6px">&#9888;&#xFE0E; Foreign Configuration Detected (' + foreign.length + ' drive(s))</div>';
html += '<table style="margin-bottom:10px"><tr><th>Slot</th><th>Model</th><th>Size</th><th>State</th></tr>';
foreign.forEach(function(d) {
html += '<tr>'
+ '<td style="font-family:monospace">' + escHtml(d.slot) + '</td>'
+ '<td>' + escHtml(d.model||'—') + '</td>'
+ '<td>' + (d.size_gb > 0 ? Math.round(d.size_gb) + ' GB' : '—') + '</td>'
+ '<td><span class="badge badge-warn">' + escHtml(d.state) + '</span></td>'
+ '</tr>';
});
html += '</table>';
html += '<div style="display:flex;gap:8px;flex-wrap:wrap">';
html += '<button class="btn btn-sm btn-primary" onclick="raidForeignAction(\'' + escHtml(c.id) + '\',\'import\',this)">Import Foreign Config</button>';
html += '<button class="btn btn-sm btn-secondary" style="color:var(--crit-fg)" onclick="raidForeignAction(\'' + escHtml(c.id) + '\',\'clear\',this)">Clear Foreign Config</button>';
html += '</div></div>';
}
html += raidRenderAllDrives(c, idx);
html += raidRenderMirrorSection(c, idx, 'lsi');
}
if (c.type === 'vroc') {
var arrays = c.arrays || [];
if (arrays.length > 0) {
html += '<div style="font-size:12px;font-weight:600;color:var(--muted);margin-bottom:6px;text-transform:uppercase;letter-spacing:.04em">Active Arrays</div>';
html += '<table style="margin-bottom:14px"><tr><th>Name</th><th>Level</th><th>Members</th><th>Status</th></tr>';
arrays.forEach(function(a) {
var badge = a.degraded
? '<span class="badge badge-err">Degraded</span>'
: '<span class="badge badge-ok">OK</span>';
html += '<tr>'
+ '<td style="font-family:monospace">' + escHtml(a.name) + '</td>'
+ '<td>' + escHtml(a.level||'—') + '</td>'
+ '<td style="font-family:monospace;font-size:12px">' + (a.members||[]).map(escHtml).join(', ') + '</td>'
+ '<td>' + badge + '</td>'
+ '</tr>';
});
html += '</table>';
}
html += raidRenderAllDrives(c, idx);
html += raidRenderMirrorSection(c, idx, 'vroc');
}
return html;
}
var RAID_READY_STATES = {'UGood': true, 'JBOD': true, 'available': true};
var RAID_NO_PREPARE_STATES = {'UGood': true, 'JBOD': true, 'Frgn': true, 'Onln': true, 'Msng': true};
function raidRenderAllDrives(c, idx) {
var drives = c.all_drives || [];
var isLSI = c.type === 'lsi';
if (drives.length === 0) {
return '<p style="font-size:13px;color:var(--muted);margin-bottom:12px">No drives detected on this controller.</p>';
}
var html = '<div style="font-size:12px;font-weight:600;color:var(--muted);margin-bottom:6px;text-transform:uppercase;letter-spacing:.04em">All Drives on This Controller</div>';
html += '<table style="margin-bottom:14px"><tr><th>' + (isLSI ? 'Slot' : 'Device') + '</th><th>Model</th><th>Size</th><th>State</th>' + (isLSI ? '<th></th>' : '') + '</tr>';
drives.forEach(function(d) {
var ready = !!RAID_READY_STATES[d.state];
var badgeClass = ready ? 'badge-ok' : 'badge-warn';
var actionCell = '';
if (isLSI && !RAID_NO_PREPARE_STATES[d.state]) {
actionCell = '<td><button class="btn btn-sm btn-secondary" onclick="raidPrepareDrive(\'' + escHtml(c.id) + '\',\'' + escHtml(d.slot) + '\',this)">Prepare</button></td>';
} else if (isLSI) {
actionCell = '<td></td>';
}
html += '<tr>'
+ '<td style="font-family:monospace">' + escHtml(isLSI ? d.slot : d.device) + '</td>'
+ '<td>' + escHtml(d.model||'—') + (d.serial ? ' [' + escHtml(d.serial) + ']' : '') + '</td>'
+ '<td>' + (d.size_gb > 0 ? Math.round(d.size_gb) + ' GB' : '—') + '</td>'
+ '<td><span class="badge ' + badgeClass + '">' + escHtml(d.state||'—') + '</span></td>'
+ actionCell
+ '</tr>';
});
html += '</table>';
return html;
}
function raidPrepareDrive(ctrlID, slot, btn) {
if (!confirm('Prepare drive ' + slot + ' on ' + ctrlID + ' for array creation?\n\nThis forces the drive into Unconfigured Good state. If it currently belongs to a virtual drive or holds data, that data will become inaccessible.')) {
return;
}
var original = btn ? btn.textContent : '';
if (btn) { btn.disabled = true; btn.textContent = 'Preparing...'; }
raidShowOutput('Prepare drive ' + slot, '', '');
fetch('/api/tools/raid/prepare-drive', {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({controller_id: ctrlID, slot: slot})
})
.then(function(r) { return r.json(); })
.then(function(d) {
if (d.error) throw new Error(d.error);
raidStreamTask(d.task_id, 'Prepare drive ' + slot, function() {
if (btn) { btn.disabled = false; btn.textContent = original; }
raidLoad();
});
})
.catch(function(e) {
raidShowOutput('Error', 'failed', e.message);
if (btn) { btn.disabled = false; btn.textContent = original; }
});
}
function raidRenderMirrorSection(c, idx, kind) {
var free = c.free_drives || [];
var html = '<div style="font-size:12px;font-weight:600;color:var(--muted);margin-bottom:6px;text-transform:uppercase;letter-spacing:.04em">Create RAID 1 Mirror</div>';
if (free.length < 2) {
html += '<p style="font-size:13px;color:var(--muted)">No unconfigured drives available (need at least 2).</p>';
return html;
}
html += '<p style="font-size:13px;color:var(--muted);margin-bottom:8px">Select exactly 2 drives:</p>';
html += '<div>';
free.forEach(function(d) {
var val = kind === 'lsi' ? d.slot : d.device;
var label = kind === 'lsi'
? escHtml(d.slot) + (d.model ? ' &mdash; ' + escHtml(d.model) : '') + (d.size_gb > 0 ? ' (' + Math.round(d.size_gb) + ' GB)' : '')
: escHtml(d.device) + (d.model ? ' &mdash; ' + escHtml(d.model) : '') + (d.serial ? ' [' + escHtml(d.serial) + ']' : '');
html += '<label style="display:block;margin-bottom:4px;font-size:13px;cursor:pointer">'
+ '<input type="checkbox" class="raid-mirror-check-' + idx + '" value="' + escHtml(val) + '"> '
+ label + '</label>';
});
html += '</div>';
if (kind === 'vroc') {
html += '<div style="margin-top:10px;display:flex;align-items:center;gap:8px;flex-wrap:wrap">'
+ '<label style="font-size:13px">Array name:&nbsp;<input type="text" id="vroc-arrayname-' + idx + '" value="bee-mirror0" style="font-family:monospace;padding:2px 6px;width:140px"></label>';
} else {
html += '<div style="margin-top:10px;display:flex;gap:8px">';
}
html += '<button class="btn btn-sm btn-primary raid-mirror-btn-' + idx + '" onclick="raidCreateMirror(\'' + escHtml(c.id) + '\',' + idx + ',\'' + kind + '\',this)">Create Mirror</button>';
html += '</div>';
return html;
}
function raidForeignAction(ctrlID, action, btn) {
if (action === 'clear' && !confirm('Clear foreign configuration on ' + ctrlID + '?\n\nThis will DELETE the foreign RAID metadata. Data on those drives may become inaccessible.')) {
return;
}
var original = btn ? btn.textContent : '';
if (btn) { btn.disabled = true; btn.textContent = action === 'import' ? 'Importing...' : 'Clearing...'; }
raidShowOutput('RAID foreign ' + action, '', '');
fetch('/api/tools/raid/foreign', {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({controller_id: ctrlID, action: action})
})
.then(function(r) { return r.json(); })
.then(function(d) {
if (d.error) throw new Error(d.error);
var actionLabel = action === 'import' ? 'Import foreign config' : 'Clear foreign config';
raidStreamTask(d.task_id, actionLabel, function() {
if (btn) { btn.disabled = false; btn.textContent = original; }
raidLoad();
});
})
.catch(function(e) {
raidShowOutput('Error', 'failed', e.message);
if (btn) { btn.disabled = false; btn.textContent = original; }
});
}
function raidCreateMirror(ctrlID, idx, kind, btn) {
var checks = document.querySelectorAll('.raid-mirror-check-' + idx + ':checked');
if (checks.length !== 2) {
alert('Select exactly 2 drives.');
return;
}
var devices = Array.from(checks).map(function(c) { return c.value; });
var arrayName = '';
if (kind === 'vroc') {
var nameEl = document.getElementById('vroc-arrayname-' + idx);
arrayName = nameEl ? nameEl.value.trim() : 'bee-mirror0';
if (!arrayName) arrayName = 'bee-mirror0';
}
var original = btn ? btn.textContent : '';
if (btn) { btn.disabled = true; btn.textContent = 'Creating...'; }
raidShowOutput('Create RAID 1', '', '');
fetch('/api/tools/raid/create-mirror', {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({controller_id: ctrlID, devices: devices, array_name: arrayName})
})
.then(function(r) { return r.json(); })
.then(function(d) {
if (d.error) throw new Error(d.error);
raidStreamTask(d.task_id, 'Create RAID 1 mirror', function() {
if (btn) { btn.disabled = false; btn.textContent = original; }
raidLoad();
});
})
.catch(function(e) {
raidShowOutput('Error', 'failed', e.message);
if (btn) { btn.disabled = false; btn.textContent = original; }
});
}
function raidShowOutput(label, status, text) {
var wrap = document.getElementById('raid-out-wrap');
var labelEl = document.getElementById('raid-out-label');
var statusEl = document.getElementById('raid-out-status');
var term = document.getElementById('raid-terminal');
wrap.style.display = 'block';
labelEl.textContent = label;
if (status === 'ok') {
statusEl.textContent = '✓ done';
statusEl.style.color = 'var(--ok-fg)';
} else if (status === 'failed') {
statusEl.textContent = '✗ failed';
statusEl.style.color = 'var(--crit-fg)';
} else {
statusEl.textContent = status;
statusEl.style.color = 'var(--muted)';
}
if (text !== undefined) {
term.textContent = text;
term.scrollTop = term.scrollHeight;
}
}
function raidStreamTask(taskID, taskName, onDone) {
var term = document.getElementById('raid-terminal');
term.textContent = '';
raidShowOutput(taskName || 'Running…', 'running…', undefined);
var es = new EventSource('/api/tasks/' + taskID + '/stream');
es.onmessage = function(e) {
term.textContent += e.data + '\n';
term.scrollTop = term.scrollHeight;
};
es.addEventListener('done', function(e) {
es.close();
if (!e.data) {
raidShowOutput(taskName, 'ok', undefined);
} else {
raidShowOutput(taskName, 'failed', undefined);
term.textContent += '\nFailed: ' + e.data;
term.scrollTop = term.scrollHeight;
}
if (onDone) onDone();
});
es.onerror = function() {
es.close();
raidShowOutput(taskName, 'failed', undefined);
if (onDone) onDone();
};
}
window.raidLoad = raidLoad;
window.raidForeignAction = raidForeignAction;
window.raidCreateMirror = raidCreateMirror;
window.raidPrepareDrive = raidPrepareDrive;
raidLoad();
})();
</script>`
}
audit/internal/webui/raid_mgmt.go
-699
View File
@@ -1,16 +1,12 @@
package webui
import (
"context"
"encoding/json"
"fmt"
"net/http"
"os"
"os/exec"
"regexp"
"strconv"
"strings"
"time"
)
// --- Response types ---
@@ -395,698 +391,3 @@ func detectVROCController() *raidControllerInfo {
}
// --- API handlers ---
func (h *handler) handleAPIRAIDStatus(w http.ResponseWriter, r *http.Request) {
resp := raidStatusResp{Controllers: []raidControllerInfo{}}
lsi2 := detectStorcli2Controllers()
if lsi := detectLSIControllers(); len(lsi) > 0 {
// storcli64 can enumerate a Tri-Mode controller (SAS3808-iMR/9500
// series) at a basic level but its drive-listing JSON parser finds
// no "Drive Information" for these — a zero-drives entry that
// storcli2 (run above) already covers correctly. Only drop it when
// storcli2 actually found something, so a genuinely drive-populated
// classic controller elsewhere in a mixed setup is never hidden.
for _, c := range lsi {
if len(c.AllDrives) == 0 && len(lsi2) > 0 {
continue
}
resp.Controllers = append(resp.Controllers, c)
}
}
if len(lsi2) > 0 {
resp.Controllers = append(resp.Controllers, lsi2...)
}
if vroc := detectVROCController(); vroc != nil {
resp.Controllers = append(resp.Controllers, *vroc)
}
writeJSON(w, resp)
}
func (h *handler) handleAPIRAIDForeignAction(w http.ResponseWriter, r *http.Request) {
var req struct {
ControllerID string `json:"controller_id"`
Action string `json:"action"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON")
return
}
if req.Action != "import" && req.Action != "clear" {
writeError(w, http.StatusBadRequest, "action must be 'import' or 'clear'")
return
}
ctrlIdx, ok := parseLSIControllerIndex(req.ControllerID)
if !ok {
writeError(w, http.StatusBadRequest, "invalid controller_id")
return
}
target := "raid-foreign-clear"
name := fmt.Sprintf("RAID Foreign Clear (ctrl %d)", ctrlIdx)
if req.Action == "import" {
target = "raid-foreign-import"
name = fmt.Sprintf("RAID Foreign Import (ctrl %d)", ctrlIdx)
}
t := &Task{
ID: newJobID(target),
Name: name,
Target: target,
Priority: defaultTaskPriority(target, taskParams{}),
Status: TaskPending,
CreatedAt: time.Now(),
params: taskParams{RAIDController: ctrlIdx},
}
globalQueue.enqueue(t)
writeJSON(w, map[string]string{"task_id": t.ID})
}
func (h *handler) handleAPIRAIDCreateMirror(w http.ResponseWriter, r *http.Request) {
var req struct {
ControllerID string `json:"controller_id"`
Devices []string `json:"devices"`
ArrayName string `json:"array_name"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON")
return
}
if len(req.Devices) < 2 {
writeError(w, http.StatusBadRequest, "at least 2 devices required")
return
}
var target, name string
var params taskParams
switch {
case strings.HasPrefix(req.ControllerID, "lsi-"):
ctrlIdx, ok := parseLSIControllerIndex(req.ControllerID)
if !ok {
writeError(w, http.StatusBadRequest, "invalid controller_id")
return
}
target = "raid-lsi-create-mirror"
name = fmt.Sprintf("Create RAID 1 Mirror (LSI ctrl %d)", ctrlIdx)
params = taskParams{RAIDController: ctrlIdx, RAIDDevices: req.Devices}
case req.ControllerID == "vroc-0":
arrayName := strings.TrimSpace(req.ArrayName)
if arrayName == "" {
arrayName = "bee-mirror0"
}
target = "raid-vroc-create-mirror"
name = fmt.Sprintf("Create VROC RAID 1 (%s)", arrayName)
params = taskParams{RAIDDevices: req.Devices, RAIDArrayName: arrayName}
default:
writeError(w, http.StatusBadRequest, "unknown controller_id")
return
}
t := &Task{
ID: newJobID(target),
Name: name,
Target: target,
Priority: defaultTaskPriority(target, taskParams{}),
Status: TaskPending,
CreatedAt: time.Now(),
params: params,
}
globalQueue.enqueue(t)
writeJSON(w, map[string]string{"task_id": t.ID})
}
func (h *handler) handleAPIRAIDPrepareDrive(w http.ResponseWriter, r *http.Request) {
var req struct {
ControllerID string `json:"controller_id"`
Slot string `json:"slot"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON")
return
}
ctrlIdx, ok := parseLSIControllerIndex(req.ControllerID)
if !ok {
writeError(w, http.StatusBadRequest, "invalid controller_id")
return
}
if _, _, ok := parseRAIDSlot(req.Slot); !ok {
writeError(w, http.StatusBadRequest, "invalid slot")
return
}
t := &Task{
ID: newJobID("raid-lsi-prepare-drive"),
Name: fmt.Sprintf("Prepare drive %s (LSI ctrl %d)", req.Slot, ctrlIdx),
Target: "raid-lsi-prepare-drive",
Priority: defaultTaskPriority("raid-lsi-prepare-drive", taskParams{}),
Status: TaskPending,
CreatedAt: time.Now(),
params: taskParams{RAIDController: ctrlIdx, RAIDSlot: req.Slot},
}
globalQueue.enqueue(t)
writeJSON(w, map[string]string{"task_id": t.ID})
}
func parseLSIControllerIndex(id string) (int, bool) {
if !strings.HasPrefix(id, "lsi-") {
return 0, false
}
n, err := strconv.Atoi(strings.TrimPrefix(id, "lsi-"))
if err != nil || n < 0 {
return 0, false
}
return n, true
}
// --- Task runner functions ---
func runRAIDForeignClearTask(ctx context.Context, j *jobState, ctrl int) error {
j.append(fmt.Sprintf("Clearing foreign configuration on controller %d...", ctrl))
cmd := exec.CommandContext(ctx, "storcli64", fmt.Sprintf("/c%d/fall", ctrl), "del", "noprompt")
return streamCmdJob(j, cmd)
}
func runRAIDForeignImportTask(ctx context.Context, j *jobState, ctrl int) error {
j.append(fmt.Sprintf("Importing foreign configuration on controller %d...", ctrl))
cmd := exec.CommandContext(ctx, "storcli64", fmt.Sprintf("/c%d/fall", ctrl), "import", "noprompt")
return streamCmdJob(j, cmd)
}
// raidPrepareAction says what (if anything) must be done to a drive in the
// given storcli state before it can join a new VD. Derived from the Broadcom
// StorCLI drive-state matrix: only UGood drives are accepted by "add vd";
// JBOD/UBad convert with "set good force"; hotspares must be released first;
// Frgn/Onln hold configuration data and must not be silently destroyed.
type raidPrepareAction int
const (
raidPrepNone raidPrepareAction = iota // UGood or unknown — try add vd as-is
raidPrepSetGood // JBOD, UBad — "set good force"
raidPrepHotspare // GHS, DHS — "delete hotsparedrive", then set good
raidPrepBlockedFrgn
raidPrepBlockedOnln
)
func classifyRAIDPrepareAction(state string) raidPrepareAction {
switch strings.TrimSpace(state) {
case "JBOD", "UBad":
return raidPrepSetGood
case "GHS", "DHS":
return raidPrepHotspare
case "Frgn":
return raidPrepBlockedFrgn
case "Onln", "Offln":
return raidPrepBlockedOnln
default: // "UGood", "" (state unknown — let add vd decide)
return raidPrepNone
}
}
// raidLSIDriveStates returns EID:Slt -> State for one controller, or nil if
// storcli/parsing fails (callers then fall back to unconditional prepare).
func raidLSIDriveStates(ctx context.Context, ctrl int) map[string]string {
out, err := exec.CommandContext(ctx, "storcli64",
fmt.Sprintf("/c%d/eall/sall", ctrl), "show", "all", "J").Output()
if err != nil {
return nil
}
var doc struct {
Controllers []struct {
ResponseData map[string]json.RawMessage `json:"Response Data"`
} `json:"Controllers"`
}
if err := json.Unmarshal(out, &doc); err != nil || len(doc.Controllers) == 0 {
return nil
}
states := map[string]string{}
for _, c := range doc.Controllers {
for _, d := range parseStorcliResponseDataDrives(c.ResponseData) {
states[strings.TrimSpace(d.EIDSlt)] = strings.TrimSpace(d.State)
}
}
return states
}
func runRAIDLSICreateMirrorTask(ctx context.Context, j *jobState, ctrl int, drives []string) error {
driveList := strings.Join(drives, ",")
states := raidLSIDriveStates(ctx, ctrl)
// Non-UGood drives cannot be added to a VD directly — storcli fails with
// "resources already in use" (exit 11) or similar. Fix what is safely
// fixable (JBOD/UBad/hotspare), refuse what holds data (Frgn/Onln).
for _, drive := range drives {
eid, slt, ok := parseRAIDSlot(drive)
if !ok {
return fmt.Errorf("invalid drive slot %q", drive)
}
state := states[drive]
slotPath := fmt.Sprintf("/c%d/e%d/s%d", ctrl, eid, slt)
switch classifyRAIDPrepareAction(state) {
case raidPrepBlockedFrgn:
return fmt.Errorf("drive %s carries a foreign configuration; run the RAID Foreign Clear (or Import) task first, then retry", drive)
case raidPrepBlockedOnln:
return fmt.Errorf("drive %s is part of an existing virtual drive (state %s); delete that VD first", drive, state)
case raidPrepHotspare:
j.append(fmt.Sprintf("Drive %s is a hotspare (%s); releasing it...", drive, state))
rel := exec.CommandContext(ctx, "storcli64", slotPath, "delete", "hotsparedrive")
if err := streamCmdJob(j, rel); err != nil {
return fmt.Errorf("release hotspare %s: %w", drive, err)
}
case raidPrepSetGood:
j.append(fmt.Sprintf("Drive %s is %s; converting to Unconfigured Good (set good force)...", drive, state))
prep := exec.CommandContext(ctx, "storcli64", slotPath, "set", "good", "force")
if err := streamCmdJob(j, prep); err != nil {
return fmt.Errorf("set good on %s: %w", drive, err)
}
case raidPrepNone:
if state == "" {
// Drive state unknown (storcli query failed) — attempt the
// conversion anyway; harmless on an already-UGood drive with
// force, and add vd below is the real verdict.
j.append(fmt.Sprintf("Preparing drive %s (set good, force)...", drive))
prep := exec.CommandContext(ctx, "storcli64", slotPath, "set", "good", "force")
if err := streamCmdJob(j, prep); err != nil {
j.append(fmt.Sprintf("note: set good on %s: %v (continuing)", drive, err))
}
}
}
}
j.append(fmt.Sprintf("Creating RAID 1 on controller %d with drives: %s", ctrl, driveList))
cmd := exec.CommandContext(ctx, "storcli64",
fmt.Sprintf("/c%d", ctrl),
"add", "vd", "type=raid1",
fmt.Sprintf("drives=%s", driveList),
"pdperarray=2",
)
if err := streamCmdJob(j, cmd); err != nil {
// A blocked add vd is often preserved cache from a dead VD
// ("controller has data in cache for offline or missing virtual
// drives"). Surface it so the log is actionable.
j.append("add vd failed; checking for preserved cache...")
pc := exec.CommandContext(ctx, "storcli64", fmt.Sprintf("/c%d", ctrl), "show", "preservedcache")
_ = streamCmdJob(j, pc)
j.append(fmt.Sprintf("hint: if preserved cache is listed above, clear it with: storcli64 /c%d/vall delete preservedcache (invalidates cached data of dead VDs), then retry", ctrl))
return err
}
return nil
}
// parseRAIDSlot splits a storcli "EID:Slt" identifier (e.g. "252:0") into
// enclosure and slot numbers.
func parseRAIDSlot(slot string) (eid int, slt int, ok bool) {
parts := strings.SplitN(strings.TrimSpace(slot), ":", 2)
if len(parts) != 2 {
return 0, 0, false
}
eid, err1 := strconv.Atoi(strings.TrimSpace(parts[0]))
slt, err2 := strconv.Atoi(strings.TrimSpace(parts[1]))
if err1 != nil || err2 != nil {
return 0, 0, false
}
return eid, slt, true
}
func runRAIDPrepareDriveTask(ctx context.Context, j *jobState, ctrl int, slot string) error {
eid, slt, ok := parseRAIDSlot(slot)
if !ok {
return fmt.Errorf("invalid slot %q", slot)
}
j.append(fmt.Sprintf("Preparing drive %s on controller %d (set good, force)...", slot, ctrl))
cmd := exec.CommandContext(ctx, "storcli64",
fmt.Sprintf("/c%d/e%d/s%d", ctrl, eid, slt),
"set", "good", "force",
)
return streamCmdJob(j, cmd)
}
func runRAIDVROCCreateMirrorTask(ctx context.Context, j *jobState, devices []string, arrayName string) error {
if arrayName == "" {
arrayName = "bee-mirror0"
}
devPath := "/dev/md/" + arrayName
args := []string{
"--create", devPath,
"--level=1",
fmt.Sprintf("--raid-devices=%d", len(devices)),
"--run",
}
args = append(args, devices...)
j.append(fmt.Sprintf("Creating VROC RAID 1 array %s with: %s", devPath, strings.Join(devices, " ")))
cmd := exec.CommandContext(ctx, "mdadm", args...)
return streamCmdJob(j, cmd)
}
// raidParseHumanSizeGB parses storcli size strings like "1.818 TB", "745.211 GB".
func raidParseHumanSizeGB(s string) float64 {
s = strings.TrimSpace(s)
if s == "" {
return 0
}
upper := strings.ToUpper(s)
var mul float64
var numStr string
switch {
case strings.Contains(upper, " TB"):
mul = 1024
numStr = strings.TrimSpace(strings.SplitN(upper, " T", 2)[0])
case strings.Contains(upper, " GB"):
mul = 1
numStr = strings.TrimSpace(strings.SplitN(upper, " G", 2)[0])
case strings.Contains(upper, " MB"):
mul = 1.0 / 1024
numStr = strings.TrimSpace(strings.SplitN(upper, " M", 2)[0])
default:
return 0
}
v, err := strconv.ParseFloat(numStr, 64)
if err != nil {
return 0
}
return v * mul
}
// --- UI card ---
func renderRAIDMgmtCard() string {
return `<div class="card"><div class="card-head card-head-actions">RAID Controller Management<div class="card-head-buttons"><button class="btn btn-sm btn-secondary" onclick="raidLoad()">&#8635; Refresh</button></div></div><div class="card-body">
<div id="raid-status" style="font-size:13px;color:var(--muted);margin-bottom:8px">Loading...</div>
<div id="raid-content"></div>
<div id="raid-out-wrap" style="display:none;margin-top:14px">
<div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:4px">
<span id="raid-out-label" style="font-size:12px;font-weight:600;color:var(--muted)">Output</span>
<span id="raid-out-status" style="font-size:12px"></span>
</div>
<div id="raid-terminal" class="terminal" style="max-height:260px;width:100%;box-sizing:border-box"></div>
</div>
</div></div>
<script>
(function(){
function escHtml(s) {
return String(s||'').replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;').replace(/"/g,'&quot;');
}
var _raidControllers = [];
function raidLoad() {
var status = document.getElementById('raid-status');
var content = document.getElementById('raid-content');
status.textContent = 'Detecting RAID controllers...';
status.style.color = 'var(--muted)';
content.innerHTML = '';
fetch('/api/tools/raid/status', {cache:'no-store'})
.then(function(r) {
if (!r.ok) return r.json().then(function(e) { throw new Error(e.error || r.statusText); });
return r.json();
})
.then(function(data) {
_raidControllers = data.controllers || [];
if (_raidControllers.length === 0) {
status.textContent = 'No RAID controllers detected.';
return;
}
status.textContent = _raidControllers.length + ' controller(s) detected.';
content.innerHTML = _raidControllers.map(function(c, i) {
return raidRenderController(c, i);
}).join('<hr style="margin:16px 0;border:none;border-top:1px solid var(--border)">');
})
.catch(function(e) {
status.textContent = 'Error: ' + e.message;
status.style.color = 'var(--crit-fg)';
});
}
function raidRenderController(c, idx) {
var html = '';
var typeLabel = c.type === 'lsi' ? 'LSI / Broadcom' : 'Intel VROC';
html += '<div style="font-weight:600;font-size:13px;margin-bottom:10px">' + typeLabel + ' &mdash; ' + escHtml(c.model) + '</div>';
if (c.type === 'lsi') {
var foreign = c.foreign_drives || [];
if (foreign.length > 0) {
html += '<div style="background:var(--warn-bg,rgba(240,192,0,0.1));border:1px solid var(--warn-border,#c8a800);border-radius:4px;padding:10px 12px;margin-bottom:12px">';
html += '<div style="font-weight:600;font-size:13px;margin-bottom:6px">&#9888;&#xFE0E; Foreign Configuration Detected (' + foreign.length + ' drive(s))</div>';
html += '<table style="margin-bottom:10px"><tr><th>Slot</th><th>Model</th><th>Size</th><th>State</th></tr>';
foreign.forEach(function(d) {
html += '<tr>'
+ '<td style="font-family:monospace">' + escHtml(d.slot) + '</td>'
+ '<td>' + escHtml(d.model||'—') + '</td>'
+ '<td>' + (d.size_gb > 0 ? Math.round(d.size_gb) + ' GB' : '—') + '</td>'
+ '<td><span class="badge badge-warn">' + escHtml(d.state) + '</span></td>'
+ '</tr>';
});
html += '</table>';
html += '<div style="display:flex;gap:8px;flex-wrap:wrap">';
html += '<button class="btn btn-sm btn-primary" onclick="raidForeignAction(\'' + escHtml(c.id) + '\',\'import\',this)">Import Foreign Config</button>';
html += '<button class="btn btn-sm btn-secondary" style="color:var(--crit-fg)" onclick="raidForeignAction(\'' + escHtml(c.id) + '\',\'clear\',this)">Clear Foreign Config</button>';
html += '</div></div>';
}
html += raidRenderAllDrives(c, idx);
html += raidRenderMirrorSection(c, idx, 'lsi');
}
if (c.type === 'vroc') {
var arrays = c.arrays || [];
if (arrays.length > 0) {
html += '<div style="font-size:12px;font-weight:600;color:var(--muted);margin-bottom:6px;text-transform:uppercase;letter-spacing:.04em">Active Arrays</div>';
html += '<table style="margin-bottom:14px"><tr><th>Name</th><th>Level</th><th>Members</th><th>Status</th></tr>';
arrays.forEach(function(a) {
var badge = a.degraded
? '<span class="badge badge-err">Degraded</span>'
: '<span class="badge badge-ok">OK</span>';
html += '<tr>'
+ '<td style="font-family:monospace">' + escHtml(a.name) + '</td>'
+ '<td>' + escHtml(a.level||'—') + '</td>'
+ '<td style="font-family:monospace;font-size:12px">' + (a.members||[]).map(escHtml).join(', ') + '</td>'
+ '<td>' + badge + '</td>'
+ '</tr>';
});
html += '</table>';
}
html += raidRenderAllDrives(c, idx);
html += raidRenderMirrorSection(c, idx, 'vroc');
}
return html;
}
var RAID_READY_STATES = {'UGood': true, 'JBOD': true, 'available': true};
var RAID_NO_PREPARE_STATES = {'UGood': true, 'JBOD': true, 'Frgn': true, 'Onln': true, 'Msng': true};
function raidRenderAllDrives(c, idx) {
var drives = c.all_drives || [];
var isLSI = c.type === 'lsi';
if (drives.length === 0) {
return '<p style="font-size:13px;color:var(--muted);margin-bottom:12px">No drives detected on this controller.</p>';
}
var html = '<div style="font-size:12px;font-weight:600;color:var(--muted);margin-bottom:6px;text-transform:uppercase;letter-spacing:.04em">All Drives on This Controller</div>';
html += '<table style="margin-bottom:14px"><tr><th>' + (isLSI ? 'Slot' : 'Device') + '</th><th>Model</th><th>Size</th><th>State</th>' + (isLSI ? '<th></th>' : '') + '</tr>';
drives.forEach(function(d) {
var ready = !!RAID_READY_STATES[d.state];
var badgeClass = ready ? 'badge-ok' : 'badge-warn';
var actionCell = '';
if (isLSI && !RAID_NO_PREPARE_STATES[d.state]) {
actionCell = '<td><button class="btn btn-sm btn-secondary" onclick="raidPrepareDrive(\'' + escHtml(c.id) + '\',\'' + escHtml(d.slot) + '\',this)">Prepare</button></td>';
} else if (isLSI) {
actionCell = '<td></td>';
}
html += '<tr>'
+ '<td style="font-family:monospace">' + escHtml(isLSI ? d.slot : d.device) + '</td>'
+ '<td>' + escHtml(d.model||'—') + (d.serial ? ' [' + escHtml(d.serial) + ']' : '') + '</td>'
+ '<td>' + (d.size_gb > 0 ? Math.round(d.size_gb) + ' GB' : '—') + '</td>'
+ '<td><span class="badge ' + badgeClass + '">' + escHtml(d.state||'—') + '</span></td>'
+ actionCell
+ '</tr>';
});
html += '</table>';
return html;
}
function raidPrepareDrive(ctrlID, slot, btn) {
if (!confirm('Prepare drive ' + slot + ' on ' + ctrlID + ' for array creation?\n\nThis forces the drive into Unconfigured Good state. If it currently belongs to a virtual drive or holds data, that data will become inaccessible.')) {
return;
}
var original = btn ? btn.textContent : '';
if (btn) { btn.disabled = true; btn.textContent = 'Preparing...'; }
raidShowOutput('Prepare drive ' + slot, '', '');
fetch('/api/tools/raid/prepare-drive', {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({controller_id: ctrlID, slot: slot})
})
.then(function(r) { return r.json(); })
.then(function(d) {
if (d.error) throw new Error(d.error);
raidStreamTask(d.task_id, 'Prepare drive ' + slot, function() {
if (btn) { btn.disabled = false; btn.textContent = original; }
raidLoad();
});
})
.catch(function(e) {
raidShowOutput('Error', 'failed', e.message);
if (btn) { btn.disabled = false; btn.textContent = original; }
});
}
function raidRenderMirrorSection(c, idx, kind) {
var free = c.free_drives || [];
var html = '<div style="font-size:12px;font-weight:600;color:var(--muted);margin-bottom:6px;text-transform:uppercase;letter-spacing:.04em">Create RAID 1 Mirror</div>';
if (free.length < 2) {
html += '<p style="font-size:13px;color:var(--muted)">No unconfigured drives available (need at least 2).</p>';
return html;
}
html += '<p style="font-size:13px;color:var(--muted);margin-bottom:8px">Select exactly 2 drives:</p>';
html += '<div>';
free.forEach(function(d) {
var val = kind === 'lsi' ? d.slot : d.device;
var label = kind === 'lsi'
? escHtml(d.slot) + (d.model ? ' &mdash; ' + escHtml(d.model) : '') + (d.size_gb > 0 ? ' (' + Math.round(d.size_gb) + ' GB)' : '')
: escHtml(d.device) + (d.model ? ' &mdash; ' + escHtml(d.model) : '') + (d.serial ? ' [' + escHtml(d.serial) + ']' : '');
html += '<label style="display:block;margin-bottom:4px;font-size:13px;cursor:pointer">'
+ '<input type="checkbox" class="raid-mirror-check-' + idx + '" value="' + escHtml(val) + '"> '
+ label + '</label>';
});
html += '</div>';
if (kind === 'vroc') {
html += '<div style="margin-top:10px;display:flex;align-items:center;gap:8px;flex-wrap:wrap">'
+ '<label style="font-size:13px">Array name:&nbsp;<input type="text" id="vroc-arrayname-' + idx + '" value="bee-mirror0" style="font-family:monospace;padding:2px 6px;width:140px"></label>';
} else {
html += '<div style="margin-top:10px;display:flex;gap:8px">';
}
html += '<button class="btn btn-sm btn-primary raid-mirror-btn-' + idx + '" onclick="raidCreateMirror(\'' + escHtml(c.id) + '\',' + idx + ',\'' + kind + '\',this)">Create Mirror</button>';
html += '</div>';
return html;
}
function raidForeignAction(ctrlID, action, btn) {
if (action === 'clear' && !confirm('Clear foreign configuration on ' + ctrlID + '?\n\nThis will DELETE the foreign RAID metadata. Data on those drives may become inaccessible.')) {
return;
}
var original = btn ? btn.textContent : '';
if (btn) { btn.disabled = true; btn.textContent = action === 'import' ? 'Importing...' : 'Clearing...'; }
raidShowOutput('RAID foreign ' + action, '', '');
fetch('/api/tools/raid/foreign', {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({controller_id: ctrlID, action: action})
})
.then(function(r) { return r.json(); })
.then(function(d) {
if (d.error) throw new Error(d.error);
var actionLabel = action === 'import' ? 'Import foreign config' : 'Clear foreign config';
raidStreamTask(d.task_id, actionLabel, function() {
if (btn) { btn.disabled = false; btn.textContent = original; }
raidLoad();
});
})
.catch(function(e) {
raidShowOutput('Error', 'failed', e.message);
if (btn) { btn.disabled = false; btn.textContent = original; }
});
}
function raidCreateMirror(ctrlID, idx, kind, btn) {
var checks = document.querySelectorAll('.raid-mirror-check-' + idx + ':checked');
if (checks.length !== 2) {
alert('Select exactly 2 drives.');
return;
}
var devices = Array.from(checks).map(function(c) { return c.value; });
var arrayName = '';
if (kind === 'vroc') {
var nameEl = document.getElementById('vroc-arrayname-' + idx);
arrayName = nameEl ? nameEl.value.trim() : 'bee-mirror0';
if (!arrayName) arrayName = 'bee-mirror0';
}
var original = btn ? btn.textContent : '';
if (btn) { btn.disabled = true; btn.textContent = 'Creating...'; }
raidShowOutput('Create RAID 1', '', '');
fetch('/api/tools/raid/create-mirror', {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({controller_id: ctrlID, devices: devices, array_name: arrayName})
})
.then(function(r) { return r.json(); })
.then(function(d) {
if (d.error) throw new Error(d.error);
raidStreamTask(d.task_id, 'Create RAID 1 mirror', function() {
if (btn) { btn.disabled = false; btn.textContent = original; }
raidLoad();
});
})
.catch(function(e) {
raidShowOutput('Error', 'failed', e.message);
if (btn) { btn.disabled = false; btn.textContent = original; }
});
}
function raidShowOutput(label, status, text) {
var wrap = document.getElementById('raid-out-wrap');
var labelEl = document.getElementById('raid-out-label');
var statusEl = document.getElementById('raid-out-status');
var term = document.getElementById('raid-terminal');
wrap.style.display = 'block';
labelEl.textContent = label;
if (status === 'ok') {
statusEl.textContent = '✓ done';
statusEl.style.color = 'var(--ok-fg)';
} else if (status === 'failed') {
statusEl.textContent = '✗ failed';
statusEl.style.color = 'var(--crit-fg)';
} else {
statusEl.textContent = status;
statusEl.style.color = 'var(--muted)';
}
if (text !== undefined) {
term.textContent = text;
term.scrollTop = term.scrollHeight;
}
}
function raidStreamTask(taskID, taskName, onDone) {
var term = document.getElementById('raid-terminal');
term.textContent = '';
raidShowOutput(taskName || 'Running…', 'running…', undefined);
var es = new EventSource('/api/tasks/' + taskID + '/stream');
es.onmessage = function(e) {
term.textContent += e.data + '\n';
term.scrollTop = term.scrollHeight;
};
es.addEventListener('done', function(e) {
es.close();
if (!e.data) {
raidShowOutput(taskName, 'ok', undefined);
} else {
raidShowOutput(taskName, 'failed', undefined);
term.textContent += '\nFailed: ' + e.data;
term.scrollTop = term.scrollHeight;
}
if (onDone) onDone();
});
es.onerror = function() {
es.close();
raidShowOutput(taskName, 'failed', undefined);
if (onDone) onDone();
};
}
window.raidLoad = raidLoad;
window.raidForeignAction = raidForeignAction;
window.raidCreateMirror = raidCreateMirror;
window.raidPrepareDrive = raidPrepareDrive;
raidLoad();
})();
</script>`
}
audit/internal/webui/saa_dmi.go
-2
View File
@@ -33,7 +33,6 @@ var (
dmiVersionRE = regexp.MustCompile(`(?i)^version\s*=`)
)
// parseDMIFile parses the DMI.txt produced by "saa GetDmiInfo".
// Real format (from SAA User Guide 4.8.1):
//
@@ -211,4 +210,3 @@ func runSAADMIWriteTask(ctx context.Context, j *jobState, exportDir string, p ta
j.append("Done. Reboot the server for changes to take effect.")
return nil
}
audit/internal/webui/server.go
+1 -673
View File
@@ -5,7 +5,6 @@ import (
"encoding/json"
"errors"
"fmt"
"html"
"io"
"log/slog"
"mime"
@@ -14,7 +13,6 @@ import (
"os"
"path/filepath"
"runtime/debug"
"sort"
"strings"
"sync"
"time"
@@ -276,6 +274,7 @@ func NewHandler(opts HandlerOptions) http.Handler {
mux.HandleFunc("POST /api/sat/memory-stress/run", h.handleAPISATRun("memory-stress"))
mux.HandleFunc("POST /api/sat/sat-stress/run", h.handleAPISATRun("sat-stress"))
mux.HandleFunc("POST /api/sat/platform-stress/run", h.handleAPISATRun("platform-stress"))
mux.HandleFunc("POST /api/sat/run-all", h.handleAPISATRunAll)
mux.HandleFunc("GET /api/sat/stream", h.handleAPISATStream)
mux.HandleFunc("POST /api/sat/abort", h.handleAPISATAbort)
mux.HandleFunc("POST /api/bee-bench/nvidia/perf/run", h.handleAPIBenchmarkNvidiaRunKind("nvidia-bench-perf"))
@@ -619,677 +618,6 @@ func (h *handler) handleViewer(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write(body)
}
func (h *handler) handleMetricsChartSVG(w http.ResponseWriter, r *http.Request) {
path := strings.TrimPrefix(r.URL.Path, "/api/metrics/chart/")
path = strings.TrimSuffix(path, ".svg")
if h.metricsDB == nil {
http.Error(w, "metrics database not available", http.StatusServiceUnavailable)
return
}
samples, err := h.metricsDB.LoadAll()
if err != nil || len(samples) == 0 {
http.Error(w, "metrics history unavailable", http.StatusServiceUnavailable)
return
}
timeline := metricsTimelineSegments(samples, time.Now())
if idx, sub, ok := parseGPUChartPath(path); ok && sub == "overview" {
var overviewOk bool
var buf []byte
buf, overviewOk, err = renderGPUOverviewChartSVG(idx, samples, timeline)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
if !overviewOk {
http.Error(w, "metrics history unavailable", http.StatusServiceUnavailable)
return
}
w.Header().Set("Content-Type", "image/svg+xml")
w.Header().Set("Cache-Control", "no-store")
_, _ = w.Write(buf)
return
}
datasets, names, labels, title, yMin, yMax, stacked, ok := chartDataFromSamples(path, samples)
if !ok {
http.Error(w, "metrics history unavailable", http.StatusServiceUnavailable)
return
}
var buf []byte
if stacked {
buf, err = renderStackedMetricChartSVG(
title,
labels,
sampleTimes(samples),
datasets,
names,
yMax,
chartCanvasHeightForPath(path, len(names)),
timeline,
)
} else {
buf, err = renderMetricChartSVG(
title,
labels,
sampleTimes(samples),
datasets,
names,
yMin,
yMax,
chartCanvasHeightForPath(path, len(names)),
timeline,
)
}
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "image/svg+xml")
w.Header().Set("Cache-Control", "no-store")
_, _ = w.Write(buf)
}
func chartDataFromSamples(path string, samples []platform.LiveMetricSample) (datasets [][]float64, names []string, labels []string, title string, yMin, yMax *float64, stacked bool, ok bool) {
labels = sampleTimeLabels(samples)
switch {
case path == "server-load":
title = "CPU / Memory Load"
cpu := make([]float64, len(samples))
mem := make([]float64, len(samples))
for i, s := range samples {
cpu[i] = s.CPULoadPct
mem[i] = s.MemLoadPct
}
datasets = [][]float64{cpu, mem}
names = []string{"CPU Load %", "Mem Load %"}
yMin = floatPtr(0)
yMax = floatPtr(100)
case path == "server-temp", path == "server-temp-cpu":
title = "CPU Temperature"
datasets, names = namedTempDatasets(samples, "cpu")
yMin = floatPtr(0)
yMax = autoMax120(datasets...)
case path == "server-temp-gpu":
title = "GPU Temperature"
datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.TempC })
yMin = floatPtr(0)
yMax = autoMax120(datasets...)
case path == "server-temp-ambient":
title = "Ambient / Other Sensors"
datasets, names = namedTempDatasets(samples, "ambient")
yMin = floatPtr(0)
yMax = autoMax120(datasets...)
case path == "server-power":
title = "System Power"
power := make([]float64, len(samples))
label := "Power W"
for i, s := range samples {
power[i] = s.PowerW
if strings.TrimSpace(s.PowerSource) != "" {
label = fmt.Sprintf("Power W · %s", s.PowerSource)
if strings.TrimSpace(s.PowerMode) != "" {
label += fmt.Sprintf(" (%s)", s.PowerMode)
}
}
}
power = normalizePowerSeries(power)
datasets = [][]float64{power}
names = []string{label}
yMin = floatPtr(0)
yMax = autoMax120(power)
case path == "server-fans":
title = "Fan RPM"
datasets, names = namedFanDatasets(samples)
yMin, yMax = autoBounds120(datasets...)
case path == "gpu-all-load":
title = "GPU Compute Load"
datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.UsagePct })
yMin = floatPtr(0)
yMax = floatPtr(100)
case path == "gpu-all-memload":
title = "GPU Memory Load"
datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.MemUsagePct })
yMin = floatPtr(0)
yMax = floatPtr(100)
case path == "gpu-all-power":
title = "GPU Power"
datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.PowerW })
yMin, yMax = autoBounds120(datasets...)
case path == "gpu-all-temp":
title = "GPU Temperature"
datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.TempC })
yMin = floatPtr(0)
yMax = autoMax120(datasets...)
case path == "gpu-all-clock":
title = "GPU Core Clock"
datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.ClockMHz })
yMin, yMax = autoBounds120(datasets...)
case path == "gpu-all-memclock":
title = "GPU Memory Clock"
datasets, names = gpuDatasets(samples, func(g platform.GPUMetricRow) float64 { return g.MemClockMHz })
yMin, yMax = autoBounds120(datasets...)
case strings.HasPrefix(path, "gpu/"):
idx, sub, ok := parseGPUChartPath(path)
if !ok {
return nil, nil, nil, "", nil, nil, false, false
}
switch sub {
case "load":
title = gpuDisplayLabel(idx) + " Load"
util := gpuDatasetByIndex(samples, idx, func(g platform.GPUMetricRow) float64 { return g.UsagePct })
mem := gpuDatasetByIndex(samples, idx, func(g platform.GPUMetricRow) float64 { return g.MemUsagePct })
if util == nil && mem == nil {
return nil, nil, nil, "", nil, nil, false, false
}
datasets = [][]float64{coalesceDataset(util, len(samples)), coalesceDataset(mem, len(samples))}
names = []string{"Load %", "Mem %"}
yMin = floatPtr(0)
yMax = floatPtr(100)
case "temp":
title = gpuDisplayLabel(idx) + " Temperature"
temp := gpuDatasetByIndex(samples, idx, func(g platform.GPUMetricRow) float64 { return g.TempC })
if temp == nil {
return nil, nil, nil, "", nil, nil, false, false
}
datasets = [][]float64{temp}
names = []string{"Temp °C"}
yMin = floatPtr(0)
yMax = autoMax120(temp)
case "clock":
title = gpuDisplayLabel(idx) + " Core Clock"
clock := gpuDatasetByIndex(samples, idx, func(g platform.GPUMetricRow) float64 { return g.ClockMHz })
if clock == nil {
return nil, nil, nil, "", nil, nil, false, false
}
datasets = [][]float64{clock}
names = []string{"Core Clock MHz"}
yMin, yMax = autoBounds120(clock)
case "memclock":
title = gpuDisplayLabel(idx) + " Memory Clock"
clock := gpuDatasetByIndex(samples, idx, func(g platform.GPUMetricRow) float64 { return g.MemClockMHz })
if clock == nil {
return nil, nil, nil, "", nil, nil, false, false
}
datasets = [][]float64{clock}
names = []string{"Memory Clock MHz"}
yMin, yMax = autoBounds120(clock)
default:
title = gpuDisplayLabel(idx) + " Power"
power := gpuDatasetByIndex(samples, idx, func(g platform.GPUMetricRow) float64 { return g.PowerW })
if power == nil {
return nil, nil, nil, "", nil, nil, false, false
}
datasets = [][]float64{power}
names = []string{"Power W"}
yMin, yMax = autoBounds120(power)
}
default:
return nil, nil, nil, "", nil, nil, false, false
}
return datasets, names, labels, title, yMin, yMax, stacked, len(datasets) > 0
}
func parseGPUChartPath(path string) (idx int, sub string, ok bool) {
if !strings.HasPrefix(path, "gpu/") {
return 0, "", false
}
rest := strings.TrimPrefix(path, "gpu/")
if rest == "" {
return 0, "", false
}
sub = ""
if i := strings.LastIndex(rest, "-"); i > 0 {
sub = rest[i+1:]
rest = rest[:i]
}
n, err := fmt.Sscanf(rest, "%d", &idx)
if err != nil || n != 1 {
return 0, "", false
}
return idx, sub, true
}
func sampleTimeLabels(samples []platform.LiveMetricSample) []string {
labels := make([]string, len(samples))
if len(samples) == 0 {
return labels
}
times := make([]time.Time, len(samples))
for i, s := range samples {
times[i] = s.Timestamp
}
sameDay := timestampsSameLocalDay(times)
for i, s := range samples {
labels[i] = formatTimelineLabel(s.Timestamp.Local(), sameDay)
}
return labels
}
func namedTempDatasets(samples []platform.LiveMetricSample, group string) ([][]float64, []string) {
seen := map[string]bool{}
var names []string
for _, s := range samples {
for _, t := range s.Temps {
if t.Group == group && !seen[t.Name] {
seen[t.Name] = true
names = append(names, t.Name)
}
}
}
sort.Strings(names)
datasets := make([][]float64, 0, len(names))
for _, name := range names {
ds := make([]float64, len(samples))
for i, s := range samples {
for _, t := range s.Temps {
if t.Group == group && t.Name == name {
ds[i] = t.Celsius
break
}
}
}
datasets = append(datasets, ds)
}
return datasets, names
}
func namedFanDatasets(samples []platform.LiveMetricSample) ([][]float64, []string) {
seen := map[string]bool{}
var names []string
for _, s := range samples {
for _, f := range s.Fans {
if !seen[f.Name] {
seen[f.Name] = true
names = append(names, f.Name)
}
}
}
sort.Strings(names)
datasets := make([][]float64, 0, len(names))
for _, name := range names {
ds := make([]float64, len(samples))
for i, s := range samples {
for _, f := range s.Fans {
if f.Name == name {
ds[i] = f.RPM
break
}
}
}
datasets = append(datasets, normalizeFanSeries(ds))
}
return datasets, names
}
func gpuDatasets(samples []platform.LiveMetricSample, pick func(platform.GPUMetricRow) float64) ([][]float64, []string) {
seen := map[int]bool{}
var indices []int
for _, s := range samples {
for _, g := range s.GPUs {
if !seen[g.GPUIndex] {
seen[g.GPUIndex] = true
indices = append(indices, g.GPUIndex)
}
}
}
sort.Ints(indices)
datasets := make([][]float64, 0, len(indices))
names := make([]string, 0, len(indices))
for _, idx := range indices {
ds := gpuDatasetByIndex(samples, idx, pick)
if ds == nil {
continue
}
datasets = append(datasets, ds)
names = append(names, gpuDisplayLabel(idx))
}
return datasets, names
}
func gpuDatasetByIndex(samples []platform.LiveMetricSample, idx int, pick func(platform.GPUMetricRow) float64) []float64 {
found := false
ds := make([]float64, len(samples))
for i, s := range samples {
for _, g := range s.GPUs {
if g.GPUIndex == idx {
ds[i] = pick(g)
found = true
break
}
}
}
if !found {
return nil
}
return ds
}
func coalesceDataset(ds []float64, n int) []float64 {
if ds != nil {
return ds
}
return make([]float64, n)
}
func normalizePowerSeries(ds []float64) []float64 {
if len(ds) == 0 {
return nil
}
out := make([]float64, len(ds))
copy(out, ds)
last := 0.0
haveLast := false
for i, v := range out {
if v > 0 {
last = v
haveLast = true
continue
}
if haveLast {
out[i] = last
}
}
return out
}
// psuSlotsFromSamples returns the sorted list of PSU slot numbers seen across samples.
func psuSlotsFromSamples(samples []platform.LiveMetricSample) []int {
seen := map[int]struct{}{}
for _, s := range samples {
for _, p := range s.PSUs {
seen[p.Slot] = struct{}{}
}
}
slots := make([]int, 0, len(seen))
for s := range seen {
slots = append(slots, s)
}
sort.Ints(slots)
return slots
}
// psuStackedTotal returns the point-by-point sum of all PSU datasets (for scale calculation).
func psuStackedTotal(datasets [][]float64) []float64 {
if len(datasets) == 0 {
return nil
}
n := len(datasets[0])
total := make([]float64, n)
for _, ds := range datasets {
for i, v := range ds {
total[i] += v
}
}
return total
}
func normalizeFanSeries(ds []float64) []float64 {
if len(ds) == 0 {
return nil
}
out := make([]float64, len(ds))
var lastPositive float64
for i, v := range ds {
if v > 0 {
lastPositive = v
out[i] = v
continue
}
if lastPositive > 0 {
out[i] = lastPositive
continue
}
out[i] = 0
}
return out
}
// floatPtr returns a pointer to a float64 value.
func floatPtr(v float64) *float64 { return &v }
// autoMax120 returns 0→max+20% Y-axis max across all datasets.
func autoMax120(datasets ...[]float64) *float64 {
max := 0.0
for _, ds := range datasets {
for _, v := range ds {
if v > max {
max = v
}
}
}
if max == 0 {
return nil // let library auto-scale
}
v := max * 1.2
return &v
}
func autoBounds120(datasets ...[]float64) (*float64, *float64) {
min := 0.0
max := 0.0
first := true
for _, ds := range datasets {
for _, v := range ds {
if first {
min, max = v, v
first = false
continue
}
if v < min {
min = v
}
if v > max {
max = v
}
}
}
if first {
return nil, nil
}
if max <= 0 {
return floatPtr(0), nil
}
span := max - min
if span <= 0 {
span = max * 0.1
if span <= 0 {
span = 1
}
}
pad := span * 0.2
low := min - pad
if low < 0 {
low = 0
}
high := max + pad
return floatPtr(low), floatPtr(high)
}
func gpuChartLabelIndices(total, target int) []int {
if total <= 0 {
return nil
}
if total == 1 {
return []int{0}
}
step := total / target
if step < 1 {
step = 1
}
var indices []int
for i := 0; i < total; i += step {
indices = append(indices, i)
}
if indices[len(indices)-1] != total-1 {
indices = append(indices, total-1)
}
return indices
}
func chartCanvasHeightForPath(path string, seriesCount int) int {
height := chartCanvasHeight(seriesCount)
if isGPUChartPath(path) {
return height * 2
}
return height
}
func isGPUChartPath(path string) bool {
return strings.HasPrefix(path, "gpu-all-") || strings.HasPrefix(path, "gpu/")
}
func chartLegendVisible(seriesCount int) bool {
return seriesCount <= 8
}
func chartCanvasHeight(seriesCount int) int {
if chartLegendVisible(seriesCount) {
return 360
}
return 288
}
// globalStats returns min, average, and max across all values in all datasets.
func globalStats(datasets [][]float64) (mn, avg, mx float64) {
var sum float64
var count int
first := true
for _, ds := range datasets {
for _, v := range ds {
if first {
mn, mx = v, v
first = false
}
if v < mn {
mn = v
}
if v > mx {
mx = v
}
sum += v
count++
}
}
if count > 0 {
avg = sum / float64(count)
}
return mn, avg, mx
}
func sanitizeChartText(s string) string {
if s == "" {
return ""
}
return html.EscapeString(strings.Map(func(r rune) rune {
if r < 0x20 && r != '\t' && r != '\n' && r != '\r' {
return -1
}
return r
}, s))
}
func snapshotNamedRings(rings []*namedMetricsRing) ([][]float64, []string, []string) {
var datasets [][]float64
var names []string
var labels []string
for _, item := range rings {
if item == nil || item.Ring == nil {
continue
}
vals, l := item.Ring.snapshot()
datasets = append(datasets, vals)
names = append(names, item.Name)
if len(labels) == 0 {
labels = l
}
}
return datasets, names, labels
}
func snapshotFanRings(rings []*metricsRing, fanNames []string) ([][]float64, []string, []string) {
var datasets [][]float64
var names []string
var labels []string
for i, ring := range rings {
if ring == nil {
continue
}
vals, l := ring.snapshot()
datasets = append(datasets, normalizeFanSeries(vals))
name := "Fan"
if i < len(fanNames) {
name = fanNames[i]
}
names = append(names, name+" RPM")
if len(labels) == 0 {
labels = l
}
}
return datasets, names, labels
}
func chartLegendNumber(v float64) string {
neg := v < 0
if v < 0 {
v = -v
}
var out string
switch {
case v >= 10000:
out = fmt.Sprintf("%dk", int((v+500)/1000))
case v >= 1000:
s := fmt.Sprintf("%.2f", v/1000)
s = strings.TrimRight(strings.TrimRight(s, "0"), ".")
out = strings.ReplaceAll(s, ".", ",") + "k"
default:
out = fmt.Sprintf("%.0f", v)
}
if neg {
return "-" + out
}
return out
}
func chartYAxisNumber(v float64) string {
neg := v < 0
if neg {
v = -v
}
var out string
switch {
case v >= 10000:
out = fmt.Sprintf("%dк", int((v+500)/1000))
case v >= 1000:
// Use one decimal place so ticks like 1400, 1600, 1800 read as
// "1,4к", "1,6к", "1,8к" instead of the ambiguous "1к"/"2к".
s := fmt.Sprintf("%.1f", v/1000)
s = strings.TrimRight(strings.TrimRight(s, "0"), ".")
out = strings.ReplaceAll(s, ".", ",") + "к"
default:
out = fmt.Sprintf("%.0f", v)
}
if neg {
return "-" + out
}
return out
}
func (h *handler) handleAPIMetricsExportCSV(w http.ResponseWriter, r *http.Request) {
if h.metricsDB == nil {
http.Error(w, "metrics database not available", http.StatusServiceUnavailable)
audit/internal/webui/server_test.go
+4
View File
@@ -1145,6 +1145,10 @@ func TestMissingAuditJSONReturnsNotFound(t *testing.T) {
func TestSupportBundleEndpointReturnsArchive(t *testing.T) {
dir := t.TempDir()
// Isolate os.TempDir(): BuildSupportBundle stages and writes its archive
// there, and a concurrent support-bundle test in another package would
// otherwise race on the same paths.
t.Setenv("TMPDIR", dir)
exportDir := filepath.Join(dir, "export")
if err := os.MkdirAll(exportDir, 0755); err != nil {
t.Fatal(err)
audit/internal/webui/task_page.go
-8
View File
@@ -2,7 +2,6 @@ package webui
import (
"encoding/json"
"fmt"
"html"
"net/http"
"os"
@@ -225,13 +224,6 @@ func loadTaskReportFragment(task Task) string {
return string(data)
}
func taskArtifactDownloadLink(task Task, absPath string) string {
if strings.TrimSpace(absPath) == "" {
return ""
}
return fmt.Sprintf(`/export/file?path=%s`, absPath)
}
func (h *handler) taskSamplesForRequest(r *http.Request) (Task, []platform.LiveMetricSample, time.Time, time.Time, bool) {
id := r.PathValue("id")
taskPtr, ok := globalQueue.findByID(id)
audit/internal/webui/task_runner.go
+4 -1
View File
@@ -224,7 +224,10 @@ func executeTaskWithOptions(opts *HandlerOptions, t *Task, j *jobState, ctx cont
err = fmt.Errorf("app not configured")
break
}
archive, err = a.RunNvidiaBandwidthPack(ctx, "", t.params.GPUIndices, j.append)
// Validate: one nvbandwidth pass over all selected GPUs. Stress
// (deep): per-socket passes then an all-GPU pass, so a cross-socket
// P2P fault is isolated from a same-socket one.
archive, err = a.RunNvidiaBandwidthPack(ctx, "", t.params.GPUIndices, t.params.StressMode, j.append)
case "nvidia-interconnect":
if a == nil {
err = fmt.Errorf("app not configured")
audit/internal/webui/tasks.go
-401
View File
@@ -2,11 +2,9 @@ package webui
import (
"context"
"encoding/json"
"fmt"
"io"
"log/slog"
"net/http"
"os"
"os/exec"
"path/filepath"
@@ -892,402 +890,3 @@ func splitNL(s string) []string {
}
// ── HTTP handlers ─────────────────────────────────────────────────────────────
func (h *handler) handleAPITasksList(w http.ResponseWriter, _ *http.Request) {
tasks := globalQueue.snapshot()
writeJSON(w, tasks)
}
func (h *handler) handleAPITasksCancel(w http.ResponseWriter, r *http.Request) {
id := r.PathValue("id")
t, ok := globalQueue.findByID(id)
if !ok {
writeError(w, http.StatusNotFound, "task not found")
return
}
globalQueue.mu.Lock()
defer globalQueue.mu.Unlock()
switch t.Status {
case TaskPending:
t.Status = TaskCancelled
now := time.Now()
t.DoneAt = &now
globalQueue.persistLocked()
taskSerialEvent(t, "finished with status="+t.Status)
writeJSON(w, map[string]string{"status": "cancelled"})
case TaskRunning:
if t.job == nil || !t.job.abort() {
writeError(w, http.StatusConflict, "task is not cancellable")
return
}
writeJSON(w, map[string]string{"status": "aborting"})
default:
writeError(w, http.StatusConflict, "task is not running or pending")
}
}
func (h *handler) handleAPITasksPriority(w http.ResponseWriter, r *http.Request) {
id := r.PathValue("id")
t, ok := globalQueue.findByID(id)
if !ok {
writeError(w, http.StatusNotFound, "task not found")
return
}
var req struct {
Delta int `json:"delta"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid body")
return
}
globalQueue.mu.Lock()
defer globalQueue.mu.Unlock()
if t.Status != TaskPending {
writeError(w, http.StatusConflict, "only pending tasks can be reprioritised")
return
}
t.Priority += req.Delta
globalQueue.persistLocked()
writeJSON(w, map[string]int{"priority": t.Priority})
}
func (h *handler) handleAPITasksCancelAll(w http.ResponseWriter, _ *http.Request) {
globalQueue.mu.Lock()
now := time.Now()
n := 0
for _, t := range globalQueue.tasks {
switch t.Status {
case TaskPending:
t.Status = TaskCancelled
t.DoneAt = &now
taskSerialEvent(t, "finished with status="+t.Status)
n++
case TaskRunning:
if t.job != nil {
t.job.abort()
}
n++
}
}
globalQueue.persistLocked()
globalQueue.mu.Unlock()
writeJSON(w, map[string]int{"cancelled": n})
}
func (h *handler) handleAPITasksKillWorkers(w http.ResponseWriter, _ *http.Request) {
// Cancel all queued/running tasks in the queue first.
globalQueue.mu.Lock()
now := time.Now()
cancelled := 0
for _, t := range globalQueue.tasks {
switch t.Status {
case TaskPending:
t.Status = TaskCancelled
t.DoneAt = &now
taskSerialEvent(t, "finished with status="+t.Status)
cancelled++
case TaskRunning:
if t.job != nil {
t.job.abort()
}
if taskMayLeaveOrphanWorkers(t.Target) {
platform.KillTestWorkers()
}
t.Status = TaskCancelled
t.DoneAt = &now
taskSerialEvent(t, "finished with status="+t.Status)
cancelled++
}
}
globalQueue.persistLocked()
globalQueue.mu.Unlock()
// Kill orphaned test worker processes at the OS level.
killed := platform.KillTestWorkers()
writeJSON(w, map[string]any{
"cancelled": cancelled,
"killed": len(killed),
"processes": killed,
})
}
func (h *handler) handleAPITasksStream(w http.ResponseWriter, r *http.Request) {
id := r.PathValue("id")
src, ok := globalQueue.taskStreamSource(id)
if !ok {
http.Error(w, "task not found", http.StatusNotFound)
return
}
if src.job != nil {
streamJob(w, r, src.job)
return
}
if src.status == TaskDone || src.status == TaskFailed || src.status == TaskCancelled {
j := newTaskJobState(src.logPath)
j.finish(src.errMsg)
streamJob(w, r, j)
return
}
if !sseStart(w) {
return
}
sseWrite(w, "", "Task is queued. Waiting for worker...")
ticker := time.NewTicker(200 * time.Millisecond)
defer ticker.Stop()
for {
select {
case <-ticker.C:
src, ok = globalQueue.taskStreamSource(id)
if !ok {
sseWrite(w, "done", "task not found")
return
}
if src.job != nil {
streamSubscribedJob(w, r, src.job)
return
}
if src.status == TaskDone || src.status == TaskFailed || src.status == TaskCancelled {
j := newTaskJobState(src.logPath)
j.finish(src.errMsg)
streamSubscribedJob(w, r, j)
return
}
case <-r.Context().Done():
return
}
}
}
func (q *taskQueue) assignTaskLogPathLocked(t *Task) {
if q.logsDir == "" || t.ID == "" {
return
}
q.ensureTaskArtifactPathsLocked(t)
}
func (q *taskQueue) loadLocked() {
if q.statePath == "" {
return
}
data, err := os.ReadFile(q.statePath)
if err != nil || len(data) == 0 {
return
}
var persisted []persistedTask
if err := json.Unmarshal(data, &persisted); err != nil {
return
}
for _, pt := range persisted {
t := &Task{
ID: pt.ID,
Name: pt.Name,
Target: pt.Target,
Priority: pt.Priority,
Status: pt.Status,
CreatedAt: pt.CreatedAt,
StartedAt: pt.StartedAt,
DoneAt: pt.DoneAt,
ErrMsg: pt.ErrMsg,
LogPath: pt.LogPath,
ArtifactsDir: pt.ArtifactsDir,
ReportJSONPath: pt.ReportJSONPath,
ReportHTMLPath: pt.ReportHTMLPath,
params: pt.Params,
}
q.assignTaskLogPathLocked(t)
if t.Status == TaskRunning {
state, ok := readTaskRunnerState(t)
switch {
case ok && state.Status == TaskRunning && processAlive(state.PID):
t.runnerPID = state.PID
t.job = newTaskJobState(t.LogPath)
case ok && state.Status != TaskRunning:
t.runnerPID = state.PID
t.Status = state.Status
t.ErrMsg = state.Error
now := state.UpdatedAt
if now.IsZero() {
now = time.Now()
}
t.DoneAt = &now
default:
if taskMayLeaveOrphanWorkers(t.Target) {
_ = platform.KillTestWorkers()
}
now := time.Now()
t.Status = TaskFailed
t.DoneAt = &now
t.ErrMsg = "interrupted by bee-web restart"
}
} else if t.Status == TaskPending {
t.StartedAt = nil
t.DoneAt = nil
t.ErrMsg = ""
}
q.tasks = append(q.tasks, t)
}
q.prune()
q.persistLocked()
}
func (q *taskQueue) persistLocked() {
if q.statePath == "" {
return
}
state := make([]persistedTask, 0, len(q.tasks))
for _, t := range q.tasks {
state = append(state, persistedTask{
ID: t.ID,
Name: t.Name,
Target: t.Target,
Priority: t.Priority,
Status: t.Status,
CreatedAt: t.CreatedAt,
StartedAt: t.StartedAt,
DoneAt: t.DoneAt,
ErrMsg: t.ErrMsg,
LogPath: t.LogPath,
ArtifactsDir: t.ArtifactsDir,
ReportJSONPath: t.ReportJSONPath,
ReportHTMLPath: t.ReportHTMLPath,
Params: t.params,
})
}
data, err := json.MarshalIndent(state, "", " ")
if err != nil {
return
}
tmp := q.statePath + ".tmp"
if err := os.WriteFile(tmp, data, 0644); err != nil {
return
}
_ = os.Rename(tmp, q.statePath)
}
func taskElapsedSec(t *Task, now time.Time) int {
if t == nil || t.StartedAt == nil || t.StartedAt.IsZero() {
return 0
}
start := *t.StartedAt
if !t.CreatedAt.IsZero() && start.Before(t.CreatedAt) {
start = t.CreatedAt
}
end := now
if t.DoneAt != nil && !t.DoneAt.IsZero() {
end = *t.DoneAt
}
if end.Before(start) {
return 0
}
return int(end.Sub(start).Round(time.Second) / time.Second)
}
func taskFolderStatus(status string) string {
status = strings.TrimSpace(strings.ToLower(status))
switch status {
case TaskRunning, TaskDone, TaskFailed, TaskCancelled:
return status
default:
return TaskPending
}
}
func sanitizeTaskFolderPart(s string) string {
s = strings.TrimSpace(strings.ToLower(s))
if s == "" {
return "task"
}
var b strings.Builder
lastDash := false
for _, r := range s {
isAlnum := (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9')
if isAlnum {
b.WriteRune(r)
lastDash = false
continue
}
if !lastDash {
b.WriteByte('-')
lastDash = true
}
}
out := strings.Trim(b.String(), "-")
if out == "" {
return "task"
}
return out
}
func taskArtifactsDir(root string, t *Task, status string) string {
if strings.TrimSpace(root) == "" || t == nil {
return ""
}
prefix := taskFolderNumberPrefix(t.ID)
return filepath.Join(root, fmt.Sprintf("%s_%s_%s", prefix, sanitizeTaskFolderPart(t.Name), taskFolderStatus(status)))
}
func taskFolderNumberPrefix(taskID string) string {
taskID = strings.TrimSpace(taskID)
if strings.HasPrefix(taskID, "TASK-") && len(taskID) >= len("TASK-000") {
num := strings.TrimSpace(strings.TrimPrefix(taskID, "TASK-"))
if len(num) == 3 {
allDigits := true
for _, r := range num {
if r < '0' || r > '9' {
allDigits = false
break
}
}
if allDigits {
return num
}
}
}
fallback := sanitizeTaskFolderPart(taskID)
if fallback == "" {
return "000"
}
return fallback
}
func ensureTaskReportPaths(t *Task) {
if t == nil || strings.TrimSpace(t.ArtifactsDir) == "" {
return
}
if t.LogPath == "" || filepath.Base(t.LogPath) == "task.log" {
t.LogPath = filepath.Join(t.ArtifactsDir, "task.log")
}
t.ReportJSONPath = filepath.Join(t.ArtifactsDir, "report.json")
t.ReportHTMLPath = filepath.Join(t.ArtifactsDir, "report.html")
}
func (q *taskQueue) ensureTaskArtifactPathsLocked(t *Task) {
if t == nil || strings.TrimSpace(q.logsDir) == "" || strings.TrimSpace(t.ID) == "" {
return
}
if strings.TrimSpace(t.ArtifactsDir) == "" {
t.ArtifactsDir = taskArtifactsDir(q.logsDir, t, t.Status)
}
if t.ArtifactsDir != "" {
_ = os.MkdirAll(t.ArtifactsDir, 0755)
}
ensureTaskReportPaths(t)
}
func (q *taskQueue) finalizeTaskArtifactPathsLocked(t *Task) {
if t == nil || strings.TrimSpace(q.logsDir) == "" || strings.TrimSpace(t.ID) == "" {
return
}
q.ensureTaskArtifactPathsLocked(t)
dstDir := taskArtifactsDir(q.logsDir, t, t.Status)
if dstDir == "" {
return
}
if t.ArtifactsDir != "" && t.ArtifactsDir != dstDir {
if _, err := os.Stat(dstDir); err != nil {
_ = os.Rename(t.ArtifactsDir, dstDir)
}
t.ArtifactsDir = dstDir
}
ensureTaskReportPaths(t)
}
audit/internal/webui/tasks_http_persistence.go
+412
View File
@@ -0,0 +1,412 @@
package webui
import (
"encoding/json"
"fmt"
"net/http"
"os"
"path/filepath"
"strings"
"time"
"bee/audit/internal/platform"
)
func (h *handler) handleAPITasksList(w http.ResponseWriter, _ *http.Request) {
tasks := globalQueue.snapshot()
writeJSON(w, tasks)
}
func (h *handler) handleAPITasksCancel(w http.ResponseWriter, r *http.Request) {
id := r.PathValue("id")
t, ok := globalQueue.findByID(id)
if !ok {
writeError(w, http.StatusNotFound, "task not found")
return
}
globalQueue.mu.Lock()
defer globalQueue.mu.Unlock()
switch t.Status {
case TaskPending:
t.Status = TaskCancelled
now := time.Now()
t.DoneAt = &now
globalQueue.persistLocked()
taskSerialEvent(t, "finished with status="+t.Status)
writeJSON(w, map[string]string{"status": "cancelled"})
case TaskRunning:
if t.job == nil || !t.job.abort() {
writeError(w, http.StatusConflict, "task is not cancellable")
return
}
writeJSON(w, map[string]string{"status": "aborting"})
default:
writeError(w, http.StatusConflict, "task is not running or pending")
}
}
func (h *handler) handleAPITasksPriority(w http.ResponseWriter, r *http.Request) {
id := r.PathValue("id")
t, ok := globalQueue.findByID(id)
if !ok {
writeError(w, http.StatusNotFound, "task not found")
return
}
var req struct {
Delta int `json:"delta"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid body")
return
}
globalQueue.mu.Lock()
defer globalQueue.mu.Unlock()
if t.Status != TaskPending {
writeError(w, http.StatusConflict, "only pending tasks can be reprioritised")
return
}
t.Priority += req.Delta
globalQueue.persistLocked()
writeJSON(w, map[string]int{"priority": t.Priority})
}
func (h *handler) handleAPITasksCancelAll(w http.ResponseWriter, _ *http.Request) {
globalQueue.mu.Lock()
now := time.Now()
n := 0
for _, t := range globalQueue.tasks {
switch t.Status {
case TaskPending:
t.Status = TaskCancelled
t.DoneAt = &now
taskSerialEvent(t, "finished with status="+t.Status)
n++
case TaskRunning:
if t.job != nil {
t.job.abort()
}
n++
}
}
globalQueue.persistLocked()
globalQueue.mu.Unlock()
writeJSON(w, map[string]int{"cancelled": n})
}
func (h *handler) handleAPITasksKillWorkers(w http.ResponseWriter, _ *http.Request) {
// Cancel all queued/running tasks in the queue first.
globalQueue.mu.Lock()
now := time.Now()
cancelled := 0
for _, t := range globalQueue.tasks {
switch t.Status {
case TaskPending:
t.Status = TaskCancelled
t.DoneAt = &now
taskSerialEvent(t, "finished with status="+t.Status)
cancelled++
case TaskRunning:
if t.job != nil {
t.job.abort()
}
if taskMayLeaveOrphanWorkers(t.Target) {
platform.KillTestWorkers()
}
t.Status = TaskCancelled
t.DoneAt = &now
taskSerialEvent(t, "finished with status="+t.Status)
cancelled++
}
}
globalQueue.persistLocked()
globalQueue.mu.Unlock()
// Kill orphaned test worker processes at the OS level.
killed := platform.KillTestWorkers()
writeJSON(w, map[string]any{
"cancelled": cancelled,
"killed": len(killed),
"processes": killed,
})
}
func (h *handler) handleAPITasksStream(w http.ResponseWriter, r *http.Request) {
id := r.PathValue("id")
src, ok := globalQueue.taskStreamSource(id)
if !ok {
http.Error(w, "task not found", http.StatusNotFound)
return
}
if src.job != nil {
streamJob(w, r, src.job)
return
}
if src.status == TaskDone || src.status == TaskFailed || src.status == TaskCancelled {
j := newTaskJobState(src.logPath)
j.finish(src.errMsg)
streamJob(w, r, j)
return
}
if !sseStart(w) {
return
}
sseWrite(w, "", "Task is queued. Waiting for worker...")
ticker := time.NewTicker(200 * time.Millisecond)
defer ticker.Stop()
for {
select {
case <-ticker.C:
src, ok = globalQueue.taskStreamSource(id)
if !ok {
sseWrite(w, "done", "task not found")
return
}
if src.job != nil {
streamSubscribedJob(w, r, src.job)
return
}
if src.status == TaskDone || src.status == TaskFailed || src.status == TaskCancelled {
j := newTaskJobState(src.logPath)
j.finish(src.errMsg)
streamSubscribedJob(w, r, j)
return
}
case <-r.Context().Done():
return
}
}
}
func (q *taskQueue) assignTaskLogPathLocked(t *Task) {
if q.logsDir == "" || t.ID == "" {
return
}
q.ensureTaskArtifactPathsLocked(t)
}
func (q *taskQueue) loadLocked() {
if q.statePath == "" {
return
}
data, err := os.ReadFile(q.statePath)
if err != nil || len(data) == 0 {
return
}
var persisted []persistedTask
if err := json.Unmarshal(data, &persisted); err != nil {
return
}
for _, pt := range persisted {
t := &Task{
ID: pt.ID,
Name: pt.Name,
Target: pt.Target,
Priority: pt.Priority,
Status: pt.Status,
CreatedAt: pt.CreatedAt,
StartedAt: pt.StartedAt,
DoneAt: pt.DoneAt,
ErrMsg: pt.ErrMsg,
LogPath: pt.LogPath,
ArtifactsDir: pt.ArtifactsDir,
ReportJSONPath: pt.ReportJSONPath,
ReportHTMLPath: pt.ReportHTMLPath,
params: pt.Params,
}
q.assignTaskLogPathLocked(t)
if t.Status == TaskRunning {
state, ok := readTaskRunnerState(t)
switch {
case ok && state.Status == TaskRunning && processAlive(state.PID):
t.runnerPID = state.PID
t.job = newTaskJobState(t.LogPath)
case ok && state.Status != TaskRunning:
t.runnerPID = state.PID
t.Status = state.Status
t.ErrMsg = state.Error
now := state.UpdatedAt
if now.IsZero() {
now = time.Now()
}
t.DoneAt = &now
default:
if taskMayLeaveOrphanWorkers(t.Target) {
_ = platform.KillTestWorkers()
}
now := time.Now()
t.Status = TaskFailed
t.DoneAt = &now
t.ErrMsg = "interrupted by bee-web restart"
}
} else if t.Status == TaskPending {
t.StartedAt = nil
t.DoneAt = nil
t.ErrMsg = ""
}
q.tasks = append(q.tasks, t)
}
q.prune()
q.persistLocked()
}
func (q *taskQueue) persistLocked() {
if q.statePath == "" {
return
}
state := make([]persistedTask, 0, len(q.tasks))
for _, t := range q.tasks {
state = append(state, persistedTask{
ID: t.ID,
Name: t.Name,
Target: t.Target,
Priority: t.Priority,
Status: t.Status,
CreatedAt: t.CreatedAt,
StartedAt: t.StartedAt,
DoneAt: t.DoneAt,
ErrMsg: t.ErrMsg,
LogPath: t.LogPath,
ArtifactsDir: t.ArtifactsDir,
ReportJSONPath: t.ReportJSONPath,
ReportHTMLPath: t.ReportHTMLPath,
Params: t.params,
})
}
data, err := json.MarshalIndent(state, "", " ")
if err != nil {
return
}
tmp := q.statePath + ".tmp"
if err := os.WriteFile(tmp, data, 0644); err != nil {
return
}
_ = os.Rename(tmp, q.statePath)
}
func taskElapsedSec(t *Task, now time.Time) int {
if t == nil || t.StartedAt == nil || t.StartedAt.IsZero() {
return 0
}
start := *t.StartedAt
if !t.CreatedAt.IsZero() && start.Before(t.CreatedAt) {
start = t.CreatedAt
}
end := now
if t.DoneAt != nil && !t.DoneAt.IsZero() {
end = *t.DoneAt
}
if end.Before(start) {
return 0
}
return int(end.Sub(start).Round(time.Second) / time.Second)
}
func taskFolderStatus(status string) string {
status = strings.TrimSpace(strings.ToLower(status))
switch status {
case TaskRunning, TaskDone, TaskFailed, TaskCancelled:
return status
default:
return TaskPending
}
}
func sanitizeTaskFolderPart(s string) string {
s = strings.TrimSpace(strings.ToLower(s))
if s == "" {
return "task"
}
var b strings.Builder
lastDash := false
for _, r := range s {
isAlnum := (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9')
if isAlnum {
b.WriteRune(r)
lastDash = false
continue
}
if !lastDash {
b.WriteByte('-')
lastDash = true
}
}
out := strings.Trim(b.String(), "-")
if out == "" {
return "task"
}
return out
}
func taskArtifactsDir(root string, t *Task, status string) string {
if strings.TrimSpace(root) == "" || t == nil {
return ""
}
prefix := taskFolderNumberPrefix(t.ID)
return filepath.Join(root, fmt.Sprintf("%s_%s_%s", prefix, sanitizeTaskFolderPart(t.Name), taskFolderStatus(status)))
}
func taskFolderNumberPrefix(taskID string) string {
taskID = strings.TrimSpace(taskID)
if strings.HasPrefix(taskID, "TASK-") && len(taskID) >= len("TASK-000") {
num := strings.TrimSpace(strings.TrimPrefix(taskID, "TASK-"))
if len(num) == 3 {
allDigits := true
for _, r := range num {
if r < '0' || r > '9' {
allDigits = false
break
}
}
if allDigits {
return num
}
}
}
fallback := sanitizeTaskFolderPart(taskID)
if fallback == "" {
return "000"
}
return fallback
}
func ensureTaskReportPaths(t *Task) {
if t == nil || strings.TrimSpace(t.ArtifactsDir) == "" {
return
}
if t.LogPath == "" || filepath.Base(t.LogPath) == "task.log" {
t.LogPath = filepath.Join(t.ArtifactsDir, "task.log")
}
t.ReportJSONPath = filepath.Join(t.ArtifactsDir, "report.json")
t.ReportHTMLPath = filepath.Join(t.ArtifactsDir, "report.html")
}
func (q *taskQueue) ensureTaskArtifactPathsLocked(t *Task) {
if t == nil || strings.TrimSpace(q.logsDir) == "" || strings.TrimSpace(t.ID) == "" {
return
}
if strings.TrimSpace(t.ArtifactsDir) == "" {
t.ArtifactsDir = taskArtifactsDir(q.logsDir, t, t.Status)
}
if t.ArtifactsDir != "" {
_ = os.MkdirAll(t.ArtifactsDir, 0755)
}
ensureTaskReportPaths(t)
}
func (q *taskQueue) finalizeTaskArtifactPathsLocked(t *Task) {
if t == nil || strings.TrimSpace(q.logsDir) == "" || strings.TrimSpace(t.ID) == "" {
return
}
q.ensureTaskArtifactPathsLocked(t)
dstDir := taskArtifactsDir(q.logsDir, t, t.Status)
if dstDir == "" {
return
}
if t.ArtifactsDir != "" && t.ArtifactsDir != dstDir {
if _, err := os.Stat(dstDir); err != nil {
_ = os.Rename(t.ArtifactsDir, dstDir)
}
t.ArtifactsDir = dstDir
}
ensureTaskReportPaths(t)
}
audit/internal/webui/tasks_test.go
+28 -11
View File
@@ -16,6 +16,19 @@ import (
"bee/audit/internal/platform"
)
type flushNotifyRecorder struct {
*httptest.ResponseRecorder
flushed chan struct{}
}
func (r *flushNotifyRecorder) Flush() {
r.ResponseRecorder.Flush()
select {
case r.flushed <- struct{}{}:
default:
}
}
func TestTaskQueuePersistsAndRecoversPendingTasks(t *testing.T) {
dir := t.TempDir()
q := &taskQueue{
@@ -275,7 +288,10 @@ func TestHandleAPITasksStreamPendingTaskStartsSSEImmediately(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
req := httptest.NewRequest(http.MethodGet, "/api/tasks/pending-1/stream", nil).WithContext(ctx)
req.SetPathValue("id", "pending-1")
rec := httptest.NewRecorder()
rec := &flushNotifyRecorder{
ResponseRecorder: httptest.NewRecorder(),
flushed: make(chan struct{}, 1),
}
done := make(chan struct{})
go func() {
@@ -284,17 +300,18 @@ func TestHandleAPITasksStreamPendingTaskStartsSSEImmediately(t *testing.T) {
close(done)
}()
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
if strings.Contains(rec.Body.String(), "Task is queued. Waiting for worker...") {
cancel()
<-done
if rec.Code != http.StatusOK {
t.Fatalf("status=%d body=%s", rec.Code, rec.Body.String())
}
return
select {
case <-rec.flushed:
cancel()
<-done
if rec.Code != http.StatusOK {
t.Fatalf("status=%d body=%s", rec.Code, rec.Body.String())
}
time.Sleep(20 * time.Millisecond)
if !strings.Contains(rec.Body.String(), "Task is queued. Waiting for worker...") {
t.Fatalf("missing queued status, body=%q", rec.Body.String())
}
return
case <-time.After(2 * time.Second):
}
cancel()
<-done