refactor: harden diagnostics and consolidate runtime code
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
package webui
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
@@ -16,35 +17,15 @@ import (
|
||||
)
|
||||
|
||||
func (h *handler) handleAPIGNVIDIAGPUs(w http.ResponseWriter, _ *http.Request) {
|
||||
if h.opts.App == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "app not configured")
|
||||
return
|
||||
}
|
||||
gpus, err := h.opts.App.ListNvidiaGPUs()
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
if gpus == nil {
|
||||
gpus = []platform.NvidiaGPU{}
|
||||
}
|
||||
writeJSON(w, gpus)
|
||||
writeAPIListResponse(w, h.opts.App != nil, func() ([]platform.NvidiaGPU, error) {
|
||||
return h.opts.App.ListNvidiaGPUs()
|
||||
})
|
||||
}
|
||||
|
||||
func (h *handler) handleAPIGNVIDIAGPUStatuses(w http.ResponseWriter, _ *http.Request) {
|
||||
if h.opts.App == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "app not configured")
|
||||
return
|
||||
}
|
||||
gpus, err := apiListNvidiaGPUStatuses(h.opts.App)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
if gpus == nil {
|
||||
gpus = []platform.NvidiaGPUStatus{}
|
||||
}
|
||||
writeJSON(w, gpus)
|
||||
writeAPIListResponse(w, h.opts.App != nil, func() ([]platform.NvidiaGPUStatus, error) {
|
||||
return apiListNvidiaGPUStatuses(h.opts.App)
|
||||
})
|
||||
}
|
||||
|
||||
func (h *handler) handleAPIGNVIDIAReset(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -70,64 +51,33 @@ func (h *handler) handleAPIGNVIDIAReset(w http.ResponseWriter, r *http.Request)
|
||||
// ── GPU settings (ECC / power limit) ──────────────────────────────────────────
|
||||
|
||||
func (h *handler) handleAPIGNVIDIAGPUSettings(w http.ResponseWriter, _ *http.Request) {
|
||||
if h.opts.App == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "app not configured")
|
||||
return
|
||||
}
|
||||
settings, err := h.opts.App.ListNvidiaGPUSettings()
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
if settings == nil {
|
||||
settings = []platform.NvidiaGPUSetting{}
|
||||
}
|
||||
writeJSON(w, settings)
|
||||
writeAPIListResponse(w, h.opts.App != nil, func() ([]platform.NvidiaGPUSetting, error) {
|
||||
return h.opts.App.ListNvidiaGPUSettings()
|
||||
})
|
||||
}
|
||||
|
||||
func (h *handler) handleAPIGNVIDIASetECC(w http.ResponseWriter, r *http.Request) {
|
||||
if h.opts.App == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "app not configured")
|
||||
return
|
||||
}
|
||||
var req struct {
|
||||
Index int `json:"index"`
|
||||
Enabled bool `json:"enabled"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid request body")
|
||||
return
|
||||
}
|
||||
result, err := h.opts.App.SetNvidiaGPUECC(req.Index, req.Enabled)
|
||||
status := "ok"
|
||||
if err != nil {
|
||||
status = "error"
|
||||
}
|
||||
writeJSON(w, map[string]string{"status": status, "output": result.Body})
|
||||
h.handleAPIGNVIDIASetBool(w, r, func(index int, enabled bool) (string, error) {
|
||||
result, err := h.opts.App.SetNvidiaGPUECC(index, enabled)
|
||||
return result.Body, err
|
||||
})
|
||||
}
|
||||
|
||||
func (h *handler) handleAPIGNVIDIASetMIG(w http.ResponseWriter, r *http.Request) {
|
||||
if h.opts.App == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "app not configured")
|
||||
return
|
||||
}
|
||||
var req struct {
|
||||
Index int `json:"index"`
|
||||
Enabled bool `json:"enabled"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid request body")
|
||||
return
|
||||
}
|
||||
result, err := h.opts.App.SetNvidiaGPUMIG(req.Index, req.Enabled)
|
||||
status := "ok"
|
||||
if err != nil {
|
||||
status = "error"
|
||||
}
|
||||
writeJSON(w, map[string]string{"status": status, "output": result.Body})
|
||||
h.handleAPIGNVIDIASetBool(w, r, func(index int, enabled bool) (string, error) {
|
||||
result, err := h.opts.App.SetNvidiaGPUMIG(index, enabled)
|
||||
return result.Body, err
|
||||
})
|
||||
}
|
||||
|
||||
func (h *handler) handleAPIGNVIDIASetCCMode(w http.ResponseWriter, r *http.Request) {
|
||||
h.handleAPIGNVIDIASetBool(w, r, func(index int, enabled bool) (string, error) {
|
||||
result, err := h.opts.App.SetNvidiaGPUCCMode(index, enabled)
|
||||
return result.Body, err
|
||||
})
|
||||
}
|
||||
|
||||
func (h *handler) handleAPIGNVIDIASetBool(w http.ResponseWriter, r *http.Request, apply func(int, bool) (string, error)) {
|
||||
if h.opts.App == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "app not configured")
|
||||
return
|
||||
@@ -140,12 +90,12 @@ func (h *handler) handleAPIGNVIDIASetCCMode(w http.ResponseWriter, r *http.Reque
|
||||
writeError(w, http.StatusBadRequest, "invalid request body")
|
||||
return
|
||||
}
|
||||
result, err := h.opts.App.SetNvidiaGPUCCMode(req.Index, req.Enabled)
|
||||
output, err := apply(req.Index, req.Enabled)
|
||||
status := "ok"
|
||||
if err != nil {
|
||||
status = "error"
|
||||
}
|
||||
writeJSON(w, map[string]string{"status": status, "output": result.Body})
|
||||
writeJSON(w, map[string]string{"status": status, "output": output})
|
||||
}
|
||||
|
||||
func (h *handler) handleAPIGNVIDIASetPowerLimit(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -348,6 +298,9 @@ func validTimezoneName(tz string) bool {
|
||||
// values supplied by the client's browser (used when the appliance has no
|
||||
// network/NTP access to keep its own clock in sync).
|
||||
func (h *handler) handleAPISystemTimeSync(w http.ResponseWriter, r *http.Request) {
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
var req struct {
|
||||
Timezone string `json:"timezone"`
|
||||
EpochMS int64 `json:"epoch_ms"`
|
||||
@@ -368,19 +321,25 @@ func (h *handler) handleAPISystemTimeSync(w http.ResponseWriter, r *http.Request
|
||||
writeError(w, http.StatusBadRequest, "invalid timezone")
|
||||
return
|
||||
}
|
||||
if b, err := exec.Command("timedatectl", "set-timezone", req.Timezone).CombinedOutput(); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "set-timezone failed: "+strings.TrimSpace(string(b)))
|
||||
if b, err := exec.CommandContext(ctx, "timedatectl", "set-timezone", req.Timezone).CombinedOutput(); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "set-timezone failed: "+commandFailureDetail(b, err))
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(&out, "timezone set to %s\n", req.Timezone)
|
||||
}
|
||||
|
||||
// Manual time only sticks if NTP sync is off.
|
||||
_ = exec.Command("timedatectl", "set-ntp", "false").Run()
|
||||
if b, err := exec.CommandContext(ctx, "timedatectl", "set-ntp", "false").CombinedOutput(); err != nil {
|
||||
canNTP, canErr := exec.CommandContext(ctx, "timedatectl", "show", "-p", "CanNTP", "--value").Output()
|
||||
if canErr != nil || strings.TrimSpace(string(canNTP)) != "no" {
|
||||
writeError(w, http.StatusInternalServerError, "disable NTP failed: "+commandFailureDetail(b, err))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
sec := req.EpochMS / 1000
|
||||
if b, err := exec.Command("date", "-u", "-s", "@"+strconv.FormatInt(sec, 10)).CombinedOutput(); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "set-time failed: "+strings.TrimSpace(string(b)))
|
||||
if b, err := exec.CommandContext(ctx, "date", "-u", "-s", "@"+strconv.FormatInt(sec, 10)).CombinedOutput(); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "set-time failed: "+commandFailureDetail(b, err))
|
||||
return
|
||||
}
|
||||
out.WriteString("system clock synced\n")
|
||||
@@ -388,6 +347,36 @@ func (h *handler) handleAPISystemTimeSync(w http.ResponseWriter, r *http.Request
|
||||
writeJSON(w, map[string]string{"status": "ok", "output": out.String()})
|
||||
}
|
||||
|
||||
func commandFailureDetail(output []byte, err error) string {
|
||||
if detail := strings.TrimSpace(string(output)); detail != "" {
|
||||
return detail
|
||||
}
|
||||
return err.Error()
|
||||
}
|
||||
|
||||
// handleAPISystemTime reports the host's current wall-clock time and configured
|
||||
// timezone so the dashboard can show them next to the browser's own clock and
|
||||
// flag a drift.
|
||||
func (h *handler) handleAPISystemTime(w http.ResponseWriter, r *http.Request) {
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second)
|
||||
defer cancel()
|
||||
|
||||
tz := ""
|
||||
if b, err := exec.CommandContext(ctx, "timedatectl", "show", "-p", "Timezone", "--value").Output(); err == nil {
|
||||
tz = strings.TrimSpace(string(b))
|
||||
}
|
||||
now := time.Now()
|
||||
if tz == "" {
|
||||
tz, _ = now.Zone()
|
||||
}
|
||||
|
||||
writeJSON(w, map[string]any{
|
||||
"epoch_ms": now.UnixMilli(),
|
||||
"local_time": now.Format("2006-01-02 15:04:05"),
|
||||
"timezone": tz,
|
||||
})
|
||||
}
|
||||
|
||||
// ── Tools ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
var standardTools = []string{
|
||||
|
||||
@@ -35,15 +35,10 @@ func (h *handler) handleAPIAuditStream(w http.ResponseWriter, r *http.Request) {
|
||||
if id == "" {
|
||||
id = r.URL.Query().Get("task_id")
|
||||
}
|
||||
// Try task queue first, then legacy job manager
|
||||
if j, ok := globalQueue.findJob(id); ok {
|
||||
streamJob(w, r, j)
|
||||
return
|
||||
}
|
||||
if j, ok := globalJobs.get(id); ok {
|
||||
streamJob(w, r, j)
|
||||
return
|
||||
}
|
||||
http.Error(w, "job not found", http.StatusNotFound)
|
||||
}
|
||||
|
||||
@@ -383,10 +378,6 @@ func (h *handler) handleAPISATStream(w http.ResponseWriter, r *http.Request) {
|
||||
streamJob(w, r, j)
|
||||
return
|
||||
}
|
||||
if j, ok := globalJobs.get(id); ok {
|
||||
streamJob(w, r, j)
|
||||
return
|
||||
}
|
||||
http.Error(w, "job not found", http.StatusNotFound)
|
||||
}
|
||||
|
||||
@@ -416,14 +407,6 @@ func (h *handler) handleAPISATAbort(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, map[string]string{"status": "aborted"})
|
||||
return
|
||||
}
|
||||
if j, ok := globalJobs.get(id); ok {
|
||||
if j.abort() {
|
||||
writeJSON(w, map[string]string{"status": "aborted"})
|
||||
} else {
|
||||
writeJSON(w, map[string]string{"status": "not_running"})
|
||||
}
|
||||
return
|
||||
}
|
||||
http.Error(w, "job not found", http.StatusNotFound)
|
||||
}
|
||||
|
||||
|
||||
@@ -99,7 +99,10 @@ func (h *handler) handleAPINetworkDHCP(w http.ResponseWriter, r *http.Request) {
|
||||
var req struct {
|
||||
Interface string `json:"interface"`
|
||||
}
|
||||
_ = json.NewDecoder(r.Body).Decode(&req)
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid request body")
|
||||
return
|
||||
}
|
||||
|
||||
result, err := h.applyPendingNetworkChange(func() (app.ActionResult, error) {
|
||||
if req.Interface == "" || req.Interface == "all" {
|
||||
@@ -167,19 +170,9 @@ func (h *handler) handleAPIExportList(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
func (h *handler) handleAPIExportUSBTargets(w http.ResponseWriter, _ *http.Request) {
|
||||
if h.opts.App == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "app not configured")
|
||||
return
|
||||
}
|
||||
targets, err := h.opts.App.ListRemovableTargets()
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
if targets == nil {
|
||||
targets = []platform.RemovableTarget{}
|
||||
}
|
||||
writeJSON(w, targets)
|
||||
writeAPIListResponse(w, h.opts.App != nil, func() ([]platform.RemovableTarget, error) {
|
||||
return h.opts.App.ListRemovableTargets()
|
||||
})
|
||||
}
|
||||
|
||||
func (h *handler) handleAPIBlackboxStatus(w http.ResponseWriter, _ *http.Request) {
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
package webui
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"bee/audit/internal/app"
|
||||
)
|
||||
|
||||
func TestSystemTimeEndpointReportsEpochLocalTimeAndTimezone(t *testing.T) {
|
||||
before := time.Now().Add(-time.Second).UnixMilli()
|
||||
rec := httptest.NewRecorder()
|
||||
NewHandler(HandlerOptions{}).ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/system/time", nil))
|
||||
after := time.Now().Add(time.Second).UnixMilli()
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d", rec.Code, http.StatusOK)
|
||||
}
|
||||
var body struct {
|
||||
EpochMS int64 `json:"epoch_ms"`
|
||||
LocalTime string `json:"local_time"`
|
||||
Timezone string `json:"timezone"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if body.EpochMS < before || body.EpochMS > after {
|
||||
t.Fatalf("epoch_ms = %d, want value in [%d, %d]", body.EpochMS, before, after)
|
||||
}
|
||||
if _, err := time.ParseInLocation("2006-01-02 15:04:05", body.LocalTime, time.Local); err != nil {
|
||||
t.Fatalf("local_time = %q: %v", body.LocalTime, err)
|
||||
}
|
||||
if strings.TrimSpace(body.Timezone) == "" {
|
||||
t.Fatal("timezone must not be empty")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNetworkDHCPRejectsMalformedJSONBeforeChangingNetwork(t *testing.T) {
|
||||
h := &handler{opts: HandlerOptions{App: &app.App{}}}
|
||||
rec := httptest.NewRecorder()
|
||||
h.handleAPINetworkDHCP(rec, httptest.NewRequest(http.MethodPost, "/api/network/dhcp", strings.NewReader("{")))
|
||||
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want %d", rec.Code, http.StatusBadRequest)
|
||||
}
|
||||
}
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// jobState holds the output lines and completion status of an async job.
|
||||
@@ -134,29 +133,6 @@ func (j *jobState) subscribe() ([]string, <-chan string) {
|
||||
return existing, ch
|
||||
}
|
||||
|
||||
// jobManager manages async jobs identified by string IDs.
|
||||
type jobManager struct {
|
||||
mu sync.Mutex
|
||||
jobs map[string]*jobState
|
||||
}
|
||||
|
||||
var globalJobs = &jobManager{jobs: make(map[string]*jobState)}
|
||||
|
||||
func (m *jobManager) create(id string) *jobState {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
j := &jobState{}
|
||||
m.jobs[id] = j
|
||||
// Schedule cleanup after 30 minutes
|
||||
goRecoverOnce("job cleanup", func() {
|
||||
time.Sleep(30 * time.Minute)
|
||||
m.mu.Lock()
|
||||
delete(m.jobs, id)
|
||||
m.mu.Unlock()
|
||||
})
|
||||
return j
|
||||
}
|
||||
|
||||
// isDone returns true if the job has finished (either successfully or with error).
|
||||
func (j *jobState) isDone() bool {
|
||||
j.mu.Lock()
|
||||
@@ -164,13 +140,6 @@ func (j *jobState) isDone() bool {
|
||||
return j.done
|
||||
}
|
||||
|
||||
func (m *jobManager) get(id string) (*jobState, bool) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
j, ok := m.jobs[id]
|
||||
return j, ok
|
||||
}
|
||||
|
||||
func newTaskJobState(logPath string, serialPrefix ...string) *jobState {
|
||||
j := &jobState{logPath: logPath}
|
||||
if len(serialPrefix) > 0 {
|
||||
|
||||
@@ -400,37 +400,6 @@ func normalizePowerSeries(ds []float64) []float64 {
|
||||
return out
|
||||
}
|
||||
|
||||
// psuSlotsFromSamples returns the sorted list of PSU slot numbers seen across samples.
|
||||
func psuSlotsFromSamples(samples []platform.LiveMetricSample) []int {
|
||||
seen := map[int]struct{}{}
|
||||
for _, s := range samples {
|
||||
for _, p := range s.PSUs {
|
||||
seen[p.Slot] = struct{}{}
|
||||
}
|
||||
}
|
||||
slots := make([]int, 0, len(seen))
|
||||
for s := range seen {
|
||||
slots = append(slots, s)
|
||||
}
|
||||
sort.Ints(slots)
|
||||
return slots
|
||||
}
|
||||
|
||||
// psuStackedTotal returns the point-by-point sum of all PSU datasets (for scale calculation).
|
||||
func psuStackedTotal(datasets [][]float64) []float64 {
|
||||
if len(datasets) == 0 {
|
||||
return nil
|
||||
}
|
||||
n := len(datasets[0])
|
||||
total := make([]float64, n)
|
||||
for _, ds := range datasets {
|
||||
for i, v := range ds {
|
||||
total[i] += v
|
||||
}
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
func normalizeFanSeries(ds []float64) []float64 {
|
||||
if len(ds) == 0 {
|
||||
return nil
|
||||
|
||||
@@ -611,7 +611,3 @@ func renderPowerBenchmarkResultsCard(exportDir string) string {
|
||||
b.WriteString(`</div></div>`)
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// renderSpeed and renderEndurance are legacy wrappers; canonical page is 5. Benchmark at /benchmark.
|
||||
func renderSpeed(opts HandlerOptions) string { return renderBenchmark(opts) }
|
||||
func renderEndurance(opts HandlerOptions) string { return renderBenchmark(opts) }
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"bee/audit/internal/platform"
|
||||
"bee/audit/internal/schema"
|
||||
)
|
||||
|
||||
@@ -246,17 +247,7 @@ func buildMemoryColumnIndex(rawNodes []int) map[int]int {
|
||||
// GPU pairwise NVLink adjacency (from a live "nvidia-smi topo -m" query)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type gpuPairLink struct {
|
||||
GPUA, GPUB int
|
||||
NVLinks int
|
||||
}
|
||||
|
||||
var topoNVRe = regexp.MustCompile(`(?i)^NV(\d+)$`)
|
||||
|
||||
// nvidia-smi underlines the topo -m header row with ANSI CSI sequences
|
||||
// (ESC[4m...ESC[0m) even when stdout is not a TTY, so the captured techdump
|
||||
// contains them and "GPU0" is not at the start of the trimmed header line.
|
||||
var topoANSIRe = regexp.MustCompile("\x1b\\[[0-9;]*[A-Za-z]")
|
||||
type gpuPairLink = platform.NvidiaNVLinkBondedPair
|
||||
|
||||
// parseGPUPairAdjacency returns every GPU pair with a nonzero NVLink bond
|
||||
// count from a "nvidia-smi topo -m" matrix. Unlike parseNVIDIATopologyMatrix
|
||||
@@ -264,86 +255,7 @@ var topoANSIRe = regexp.MustCompile("\x1b\\[[0-9;]*[A-Za-z]")
|
||||
// who is bonded to whom — required so GPU-GPU edges are drawn for actually
|
||||
// bonded pairs, not for adjacent boxes in the layout.
|
||||
func parseGPUPairAdjacency(raw string) []gpuPairLink {
|
||||
lines := strings.Split(topoANSIRe.ReplaceAllString(raw, ""), "\n")
|
||||
headerIdx := -1
|
||||
var gpuColIndices []int
|
||||
for i, line := range lines {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if strings.HasPrefix(trimmed, "GPU0") {
|
||||
parts := strings.Fields(trimmed)
|
||||
for j, col := range parts {
|
||||
if strings.HasPrefix(col, "GPU") {
|
||||
gpuColIndices = append(gpuColIndices, j)
|
||||
}
|
||||
}
|
||||
if len(gpuColIndices) >= 2 {
|
||||
headerIdx = i
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
if headerIdx < 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
colIdxToGPU := make(map[int]int, len(gpuColIndices))
|
||||
for gpuIdx, colIdx := range gpuColIndices {
|
||||
colIdxToGPU[colIdx] = gpuIdx
|
||||
}
|
||||
|
||||
seen := map[[2]int]bool{}
|
||||
var pairs []gpuPairLink
|
||||
rowGPU := -1
|
||||
for _, line := range lines[headerIdx+1:] {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if !strings.HasPrefix(trimmed, "GPU") {
|
||||
continue
|
||||
}
|
||||
cells := strings.Fields(trimmed)
|
||||
if len(cells) == 0 {
|
||||
continue
|
||||
}
|
||||
rowLabel := strings.TrimPrefix(cells[0], "GPU")
|
||||
n, err := strconv.Atoi(rowLabel)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
rowGPU = n
|
||||
for colIdx, colGPU := range colIdxToGPU {
|
||||
if colGPU == rowGPU {
|
||||
continue
|
||||
}
|
||||
dataIdx := colIdx + 1
|
||||
if dataIdx >= len(cells) {
|
||||
continue
|
||||
}
|
||||
m := topoNVRe.FindStringSubmatch(cells[dataIdx])
|
||||
if len(m) != 2 {
|
||||
continue
|
||||
}
|
||||
nv, err := strconv.Atoi(m[1])
|
||||
if err != nil || nv <= 0 {
|
||||
continue
|
||||
}
|
||||
a, bGPU := rowGPU, colGPU
|
||||
if a > bGPU {
|
||||
a, bGPU = bGPU, a
|
||||
}
|
||||
key := [2]int{a, bGPU}
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
pairs = append(pairs, gpuPairLink{GPUA: a, GPUB: bGPU, NVLinks: nv})
|
||||
}
|
||||
}
|
||||
sort.Slice(pairs, func(i, j int) bool {
|
||||
if pairs[i].GPUA != pairs[j].GPUA {
|
||||
return pairs[i].GPUA < pairs[j].GPUA
|
||||
}
|
||||
return pairs[i].GPUB < pairs[j].GPUB
|
||||
})
|
||||
return pairs
|
||||
return platform.ParseNvidiaNVLinkBondedPairs(raw)
|
||||
}
|
||||
|
||||
// readTopoTechDump reads a file previously captured into the persistent
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"bee/audit/internal/app"
|
||||
"bee/audit/internal/platform"
|
||||
"bee/audit/internal/schema"
|
||||
)
|
||||
|
||||
@@ -499,7 +500,6 @@ var (
|
||||
topoNVLinkGPUHeaderRe = regexp.MustCompile(`^GPU (\d+):`)
|
||||
topoNVLinkSpeedLineRe = regexp.MustCompile(`^Link (\d+):\s*([\d.]+)\s*GB/s`)
|
||||
topoNVLinkInactiveRe = regexp.MustCompile(`^Link (\d+):\s*<inactive>`)
|
||||
topoNVLinkErrorLineRe = regexp.MustCompile(`^Link (\d+):\s*(Replay|Recovery|CRC) Errors:\s*(\d+)`)
|
||||
)
|
||||
|
||||
func readTopoNVLinkStatus(exportDir string) (map[int][]topoNVLinkPort, error) {
|
||||
@@ -549,38 +549,7 @@ func readTopoNVLinkErrors(exportDir string) (map[int]map[int][3]int64, error) {
|
||||
|
||||
// parseTopoNVLinkErrors returns, per GPU then link index, [replay, recovery, crc].
|
||||
func parseTopoNVLinkErrors(raw string) map[int]map[int][3]int64 {
|
||||
result := map[int]map[int][3]int64{}
|
||||
currentGPU := -1
|
||||
for _, line := range strings.Split(raw, "\n") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if m := topoNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
|
||||
currentGPU, _ = strconv.Atoi(m[1])
|
||||
continue
|
||||
}
|
||||
if currentGPU < 0 {
|
||||
continue
|
||||
}
|
||||
m := topoNVLinkErrorLineRe.FindStringSubmatch(trimmed)
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
linkIdx, _ := strconv.Atoi(m[1])
|
||||
count, _ := strconv.ParseInt(m[3], 10, 64)
|
||||
if result[currentGPU] == nil {
|
||||
result[currentGPU] = map[int][3]int64{}
|
||||
}
|
||||
c := result[currentGPU][linkIdx]
|
||||
switch m[2] {
|
||||
case "Replay":
|
||||
c[0] = count
|
||||
case "Recovery":
|
||||
c[1] = count
|
||||
case "CRC":
|
||||
c[2] = count
|
||||
}
|
||||
result[currentGPU][linkIdx] = c
|
||||
}
|
||||
return result
|
||||
return platform.ParseNvidiaNVLinkErrors(raw)
|
||||
}
|
||||
|
||||
// renderTopoNVLinkCard renders the separate NVLink topology card. Returns ""
|
||||
|
||||
@@ -150,8 +150,9 @@ setInterval(function(){
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// renderTimeSyncCard shows the server's current time and a button that syncs
|
||||
// the host clock and timezone to whatever the client's browser reports.
|
||||
// renderTimeSyncCard shows the server's current clock and timezone next to the
|
||||
// browser's own, highlights any drift, and offers a button that syncs the host
|
||||
// clock and timezone to whatever the client's browser reports.
|
||||
func renderTimeSyncCard() string {
|
||||
return `<div class="card" style="margin-bottom:16px">
|
||||
<div class="card-head card-head-actions">
|
||||
@@ -161,11 +162,66 @@ func renderTimeSyncCard() string {
|
||||
</div>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<table style="font-size:13px;border-collapse:collapse">
|
||||
<tr><td style="padding:2px 16px 2px 0;color:var(--muted)">Server</td>
|
||||
<td style="padding:2px 24px 2px 0" id="time-server-clock">—</td>
|
||||
<td style="padding:2px 0" id="time-server-tz">—</td></tr>
|
||||
<tr><td style="padding:2px 16px 2px 0;color:var(--muted)">Browser</td>
|
||||
<td style="padding:2px 24px 2px 0" id="time-browser-clock">—</td>
|
||||
<td style="padding:2px 0" id="time-browser-tz">—</td></tr>
|
||||
</table>
|
||||
<div id="time-drift-note" style="font-size:13px;margin-top:8px"></div>
|
||||
<span id="time-sync-status" style="font-size:13px;color:var(--muted)"></span>
|
||||
</div>
|
||||
</div>
|
||||
<script>
|
||||
function timeSyncRun() {
|
||||
(function(){
|
||||
var CRIT = 'var(--crit-fg,#9f3a38)';
|
||||
var OK = 'var(--ok-fg,#2c662d)';
|
||||
var refreshPending = false;
|
||||
|
||||
function refreshTime() {
|
||||
var browserTz = Intl.DateTimeFormat().resolvedOptions().timeZone || '';
|
||||
document.getElementById('time-browser-clock').textContent = new Date().toLocaleString();
|
||||
document.getElementById('time-browser-tz').textContent = browserTz;
|
||||
|
||||
if (refreshPending) return Promise.resolve();
|
||||
refreshPending = true;
|
||||
return fetch('/api/system/time', {cache: 'no-store'})
|
||||
.then(function(r){ if(!r.ok) throw new Error(r.statusText); return r.json(); })
|
||||
.then(function(d){
|
||||
var serverMs = d.epoch_ms;
|
||||
var serverTz = d.timezone || '';
|
||||
var skewMs = Math.abs(Date.now() - serverMs);
|
||||
document.getElementById('time-server-clock').textContent = d.local_time || new Date(serverMs).toISOString();
|
||||
document.getElementById('time-server-tz').textContent = serverTz;
|
||||
|
||||
var clockBad = skewMs > 60000;
|
||||
var tzBad = serverTz !== browserTz;
|
||||
document.getElementById('time-server-clock').style.color = clockBad ? CRIT : '';
|
||||
document.getElementById('time-browser-clock').style.color = clockBad ? CRIT : '';
|
||||
document.getElementById('time-server-tz').style.color = tzBad ? CRIT : '';
|
||||
document.getElementById('time-browser-tz').style.color = tzBad ? CRIT : '';
|
||||
|
||||
var note = document.getElementById('time-drift-note');
|
||||
if (clockBad || tzBad) {
|
||||
var parts = [];
|
||||
if (clockBad) parts.push('clock differs by ' + Math.round(skewMs/1000) + 's');
|
||||
if (tzBad) parts.push('timezone mismatch');
|
||||
note.style.color = CRIT;
|
||||
note.textContent = '⚠ ' + parts.join(', ') + ' — click "Sync with this browser"';
|
||||
} else {
|
||||
note.style.color = OK;
|
||||
note.textContent = '✓ server clock and timezone match this browser';
|
||||
}
|
||||
})
|
||||
.catch(function(){
|
||||
document.getElementById('time-server-clock').textContent = 'unavailable';
|
||||
})
|
||||
.finally(function(){ refreshPending = false; });
|
||||
}
|
||||
|
||||
window.timeSyncRun = function() {
|
||||
var btn = document.getElementById('time-sync-btn');
|
||||
var status = document.getElementById('time-sync-status');
|
||||
btn.disabled = true;
|
||||
@@ -177,15 +233,20 @@ function timeSyncRun() {
|
||||
})
|
||||
.then(function(r) { if (!r.ok) return r.text().then(function(t){throw new Error(t || r.statusText);}); return r.json(); })
|
||||
.then(function(d) {
|
||||
status.style.color = 'var(--ok-fg,#2c662d)';
|
||||
status.style.color = OK;
|
||||
status.textContent = '✓ Synced to ' + tz + ' at ' + new Date().toLocaleString();
|
||||
refreshTime();
|
||||
})
|
||||
.catch(function(err) {
|
||||
status.style.color = 'var(--crit-fg,#9f3a38)';
|
||||
status.style.color = CRIT;
|
||||
status.textContent = '✗ Sync failed: ' + err.message;
|
||||
})
|
||||
.finally(function() { btn.disabled = false; });
|
||||
}
|
||||
};
|
||||
|
||||
refreshTime();
|
||||
setInterval(refreshTime, 5000);
|
||||
})();
|
||||
</script>`
|
||||
}
|
||||
|
||||
|
||||
@@ -348,6 +348,7 @@ func NewHandler(opts HandlerOptions) http.Handler {
|
||||
mux.HandleFunc("POST /api/system/install-to-ram", h.handleAPIInstallToRAM)
|
||||
mux.HandleFunc("POST /api/system/reboot", h.handleAPISystemReboot)
|
||||
mux.HandleFunc("POST /api/system/shutdown", h.handleAPISystemShutdown)
|
||||
mux.HandleFunc("GET /api/system/time", h.handleAPISystemTime)
|
||||
mux.HandleFunc("POST /api/system/time-sync", h.handleAPISystemTimeSync)
|
||||
|
||||
// Preflight
|
||||
@@ -813,3 +814,19 @@ func writeError(w http.ResponseWriter, status int, msg string) {
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"error": msg})
|
||||
}
|
||||
|
||||
func writeAPIListResponse[T any](w http.ResponseWriter, configured bool, load func() ([]T, error)) {
|
||||
if !configured {
|
||||
writeError(w, http.StatusServiceUnavailable, "app not configured")
|
||||
return
|
||||
}
|
||||
items, err := load()
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
if items == nil {
|
||||
items = []T{}
|
||||
}
|
||||
writeJSON(w, items)
|
||||
}
|
||||
|
||||
@@ -519,7 +519,9 @@ func (q *taskQueue) startWorker(opts *HandlerOptions) {
|
||||
q.opts = opts
|
||||
q.statePath = filepath.Join(opts.ExportDir, "tasks-state.json")
|
||||
q.logsDir = filepath.Join(opts.ExportDir, "tasks")
|
||||
_ = os.MkdirAll(q.logsDir, 0755)
|
||||
if err := os.MkdirAll(q.logsDir, 0755); err != nil {
|
||||
slog.Error("task queue: create logs directory", "path", q.logsDir, "error", err)
|
||||
}
|
||||
if !q.started {
|
||||
q.loadLocked()
|
||||
q.started = true
|
||||
|
||||
@@ -3,6 +3,7 @@ package webui
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -274,13 +275,17 @@ func (q *taskQueue) persistLocked() {
|
||||
}
|
||||
data, err := json.MarshalIndent(state, "", " ")
|
||||
if err != nil {
|
||||
slog.Error("task queue: marshal persistent state", "path", q.statePath, "error", err)
|
||||
return
|
||||
}
|
||||
tmp := q.statePath + ".tmp"
|
||||
if err := os.WriteFile(tmp, data, 0644); err != nil {
|
||||
slog.Error("task queue: write persistent state", "path", tmp, "error", err)
|
||||
return
|
||||
}
|
||||
_ = os.Rename(tmp, q.statePath)
|
||||
if err := os.Rename(tmp, q.statePath); err != nil {
|
||||
slog.Error("task queue: publish persistent state", "source", tmp, "path", q.statePath, "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func taskElapsedSec(t *Task, now time.Time) int {
|
||||
@@ -388,7 +393,9 @@ func (q *taskQueue) ensureTaskArtifactPathsLocked(t *Task) {
|
||||
t.ArtifactsDir = taskArtifactsDir(q.logsDir, t, t.Status)
|
||||
}
|
||||
if t.ArtifactsDir != "" {
|
||||
_ = os.MkdirAll(t.ArtifactsDir, 0755)
|
||||
if err := os.MkdirAll(t.ArtifactsDir, 0755); err != nil {
|
||||
slog.Error("task queue: create artifacts directory", "path", t.ArtifactsDir, "task_id", t.ID, "error", err)
|
||||
}
|
||||
}
|
||||
ensureTaskReportPaths(t)
|
||||
}
|
||||
@@ -403,10 +410,15 @@ func (q *taskQueue) finalizeTaskArtifactPathsLocked(t *Task) {
|
||||
return
|
||||
}
|
||||
if t.ArtifactsDir != "" && t.ArtifactsDir != dstDir {
|
||||
if _, err := os.Stat(dstDir); err != nil {
|
||||
_ = os.Rename(t.ArtifactsDir, dstDir)
|
||||
if _, err := os.Stat(dstDir); err == nil {
|
||||
slog.Error("task queue: destination artifacts directory already exists", "source", t.ArtifactsDir, "destination", dstDir, "task_id", t.ID)
|
||||
} else if !os.IsNotExist(err) {
|
||||
slog.Error("task queue: inspect destination artifacts directory", "destination", dstDir, "task_id", t.ID, "error", err)
|
||||
} else if err := os.Rename(t.ArtifactsDir, dstDir); err != nil {
|
||||
slog.Error("task queue: move artifacts directory", "source", t.ArtifactsDir, "destination", dstDir, "task_id", t.ID, "error", err)
|
||||
} else {
|
||||
t.ArtifactsDir = dstDir
|
||||
}
|
||||
t.ArtifactsDir = dstDir
|
||||
}
|
||||
ensureTaskReportPaths(t)
|
||||
}
|
||||
|
||||
@@ -227,6 +227,29 @@ func TestTaskArtifactsDirStartsWithTaskNumber(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFinalizeTaskArtifactPathsKeepsSourceWhenDestinationExists(t *testing.T) {
|
||||
logsDir := t.TempDir()
|
||||
sourceDir := filepath.Join(logsDir, "running-artifacts")
|
||||
if err := os.Mkdir(sourceDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
task := &Task{ID: "TASK-007", Name: "NVIDIA Benchmark", Status: TaskDone, ArtifactsDir: sourceDir}
|
||||
destinationDir := taskArtifactsDir(logsDir, task, TaskDone)
|
||||
if err := os.Mkdir(destinationDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
q := &taskQueue{logsDir: logsDir}
|
||||
q.finalizeTaskArtifactPathsLocked(task)
|
||||
|
||||
if task.ArtifactsDir != sourceDir {
|
||||
t.Fatalf("artifacts dir = %q, want retained source %q after destination collision", task.ArtifactsDir, sourceDir)
|
||||
}
|
||||
if task.ReportJSONPath != filepath.Join(sourceDir, "report.json") {
|
||||
t.Fatalf("report path = %q, want source directory", task.ReportJSONPath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleAPITasksStreamReplaysPersistedLogWithoutLiveJob(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
logPath := filepath.Join(dir, "task.log")
|
||||
|
||||
Reference in New Issue
Block a user