package webui import ( "encoding/json" "fmt" "html" "os" "path/filepath" "regexp" "sort" "strconv" "strings" "bee/audit/internal/app" "bee/audit/internal/schema" ) // renderTopo renders the /topo page: a read-only visualization of the server // topology (CPU sockets, NUMA-affine PCIe devices, PSU/BMC) plus a separate // NVLink topology card. It is pure visualization: everything it reads either // already exists in the audit.json contract, or comes from the persisted // techdump captured once per audit cycle (platform.CaptureTechnicalDump) — // nothing here shells out to nvidia-smi itself, writes to // schema.HardwarePCIeDevice or any other contract type, or talks to // Reanimator Core. func renderTopo(opts HandlerOptions) string { data, err := loadSnapshot(opts.AuditPath) if err != nil { return topoCard("Topology", `No audit data`) } var ingest schema.HardwareIngestRequest if err := json.Unmarshal(data, &ingest); err != nil { return topoCard("Topology", `Parse error`) } hw := ingest.Hardware var b strings.Builder b.WriteString(renderTopoMainDiagram(hw, opts.ExportDir)) if nv := renderTopoNVLinkCard(hw, opts.ExportDir); nv != "" { b.WriteString(nv) } return b.String() } func topoCard(title, body string) string { return `
` + html.EscapeString(title) + `
` + body + `
` } // --------------------------------------------------------------------------- // Classification helpers // // webui does not import collector (matches the existing isGPUDeviceClass // precedent in pages.go, which already locally duplicates collector.isGPUClass // instead of importing the package for one classifier). // --------------------------------------------------------------------------- // isNICDeviceClassDev mirrors the classification logic in hwDescribeNIC // (pages.go), applied to a single device instead of aggregated counts. func isNICDeviceClassDev(dev schema.HardwarePCIeDevice) bool { if dev.DeviceClass != nil { c := strings.ToLower(strings.TrimSpace(*dev.DeviceClass)) if c == "ethernetcontroller" || c == "networkcontroller" || strings.Contains(c, "fibrechannel") { return true } } return len(dev.MacAddresses) > 0 } // isRAIDControllerClass matches the canonical class strings produced by // collector.mapPCIeDeviceClass for RAID/storage HBAs. func isRAIDControllerClass(class string) bool { switch strings.TrimSpace(class) { case "MassStorageController", "StorageController": return true default: return false } } // --------------------------------------------------------------------------- // Status / link-speed coloring // --------------------------------------------------------------------------- // pcieGenRank ranks a PCIe generation label ("Gen3", "Gen4", ...) for // comparison. Mirrors collector.pcieLinkSpeedRank's ordering; duplicated // locally rather than exported, per the same "no collector import in webui" // convention used for isGPUDeviceClass/isRAIDControllerClass. func pcieGenRank(gen string) int { gen = strings.ToLower(strings.TrimSpace(gen)) gen = strings.TrimPrefix(gen, "gen") n, err := strconv.Atoi(gen) if err != nil { return 0 } return n } // topoEdgeColorVar computes the CPU->device edge color strictly from // link_speed vs max_link_speed — NOT from dev.Status, since Status can also // be overwritten by SAT/acceptance-test results on the same PCIe device, // which would conflate "link is physically degraded" with "this GPU failed // its stress test" into the same color. func topoEdgeColorVar(dev schema.HardwarePCIeDevice) string { if dev.LinkSpeed == nil || dev.MaxLinkSpeed == nil { return "var(--muted)" } if pcieGenRank(*dev.LinkSpeed) < pcieGenRank(*dev.MaxLinkSpeed) { return "var(--warn-fg)" } return "var(--ok-fg)" } // --------------------------------------------------------------------------- // NUMA node -> CPU socket join (heuristic, no guaranteed hardware mapping) // --------------------------------------------------------------------------- // buildSocketIndex maps a NUMA node number to the index into cpus for the // socket occupying that position in ascending Socket-designation order. // // Linux NUMA node numbering is always 0-based (node0, node1, ...), but // dmidecode's "Socket Designation" is board-defined and frequently 1-based // ("CPU1", "CPU2", ...). Mapping NUMA node N to the CPU whose Socket field // equals N (as an earlier version of this function did) silently fails on // any 1-indexed board: node 0 has no match (dropped into the "unknown" // column) and node 1 wrongly maps to the first CPU. Ranking by Socket value // instead assumes only that node order follows socket order — true for the // common case of N-socket boards — without depending on the numbering base. func buildSocketIndex(cpus []schema.HardwareCPU) map[int]int { order := make([]int, len(cpus)) for i := range cpus { order[i] = i } sort.SliceStable(order, func(a, b int) bool { ca, cb := cpus[order[a]], cpus[order[b]] sa, sb := 0, 0 if ca.Socket != nil { sa = *ca.Socket } if cb.Socket != nil { sb = *cb.Socket } return sa < sb }) idx := map[int]int{} for numaNode, cpuIdx := range order { idx[numaNode] = cpuIdx } return idx } // --------------------------------------------------------------------------- // DIMM -> CPU column attachment // // schema.HardwareMemory has no NUMANode field (unlike HardwarePCIeDevice), so // unlike the GPU/NIC/RAID placement above, a DIMM's CPU affinity has to be // read out of its own Locator/Bank Locator strings — DMI type 17 gives no // other hint. Both patterns below have been observed on real boards. // --------------------------------------------------------------------------- var ( topoMemCPULocatorRe = regexp.MustCompile(`(?i)^cpu\s*0*(\d+)`) topoMemBankNodeRe = regexp.MustCompile(`(?i)node\s*0*(\d+)`) ) // parseDIMMBankLocatorNodes maps a DIMM's Locator (matches // schema.HardwareMemory.Slot) to the node number embedded in its Bank // Locator field (e.g. "_Node1_Channel0_Dimm0"), read from a raw // "dmidecode -t 17" techdump capture. Bank Locator never reaches audit.json // (schema.HardwareMemory.Location is json:"-", used only for internal DIMM // telemetry matching), so boards whose Locator has no CPU number of its own // (e.g. "DIMM000(A)" rather than "CPU0_DIMM_A1") need this fallback to // attach a DIMM to a CPU column at all — matches the existing convention of // reading extra techdump for this page's visualization only (see // readTopoTechDump). func parseDIMMBankLocatorNodes(raw string) map[string]int { result := map[string]int{} for _, sec := range strings.Split(raw, "Memory Device") { var locator string node := -1 for _, line := range strings.Split(sec, "\n") { trimmed := strings.TrimSpace(line) if v, ok := strings.CutPrefix(trimmed, "Locator:"); ok { locator = strings.TrimSpace(v) } if v, ok := strings.CutPrefix(trimmed, "Bank Locator:"); ok { if m := topoMemBankNodeRe.FindStringSubmatch(v); m != nil { if n, err := strconv.Atoi(m[1]); err == nil { node = n } } } } if locator != "" && node >= 0 { result[locator] = node } } return result } // dimmRawNode returns the raw (vendor-numbered, not yet column-mapped) // CPU/NUMA node number for a DIMM, trying two heuristics in order: // 1. A CPU number encoded directly in the Locator itself, e.g. // "CPU0_DIMM_A1". // 2. A node number from the Bank Locator via parseDIMMBankLocatorNodes, // e.g. Locator "DIMM000(A)" whose Bank Locator is // "_Node1_Channel0_Dimm0" -> 1. // ok=false means neither pattern matched, so this DIMM can't be confidently // attached to a CPU column and falls back to the unattached Memory row. func dimmRawNode(mem schema.HardwareMemory, bankNodeByLocator map[string]int) (int, bool) { if mem.Slot == nil { return 0, false } if m := topoMemCPULocatorRe.FindStringSubmatch(*mem.Slot); m != nil { if n, err := strconv.Atoi(m[1]); err == nil { return n, true } } if n, ok := bankNodeByLocator[*mem.Slot]; ok { return n, true } return 0, false } // buildMemoryColumnIndex ranks the distinct raw node numbers seen across all // DIMMs and maps the i-th smallest to column i — the same "node order // follows socket order" assumption buildSocketIndex makes for PCIe NUMA // nodes, but computed independently from PCIe's own numbering: PCIe's // NUMANode is 0-based Linux numbering, while a DIMM's Bank Locator "NodeN" // has been observed 1-based on at least one real board, so the two node // spaces are not guaranteed to share a base. func buildMemoryColumnIndex(rawNodes []int) map[int]int { seen := map[int]bool{} var distinct []int for _, n := range rawNodes { if !seen[n] { seen[n] = true distinct = append(distinct, n) } } sort.Ints(distinct) idx := map[int]int{} for col, n := range distinct { idx[n] = col } return idx } // --------------------------------------------------------------------------- // GPU pairwise NVLink adjacency (from a live "nvidia-smi topo -m" query) // --------------------------------------------------------------------------- type gpuPairLink struct { GPUA, GPUB int NVLinks int } var topoNVRe = regexp.MustCompile(`(?i)^NV(\d+)$`) // nvidia-smi underlines the topo -m header row with ANSI CSI sequences // (ESC[4m...ESC[0m) even when stdout is not a TTY, so the captured techdump // contains them and "GPU0" is not at the start of the trimmed header line. var topoANSIRe = regexp.MustCompile("\x1b\\[[0-9;]*[A-Za-z]") // parseGPUPairAdjacency returns every GPU pair with a nonzero NVLink bond // count from a "nvidia-smi topo -m" matrix. Unlike parseNVIDIATopologyMatrix // (collector package, aggregate-only: min/all-active/count), this returns // who is bonded to whom — required so GPU-GPU edges are drawn for actually // bonded pairs, not for adjacent boxes in the layout. func parseGPUPairAdjacency(raw string) []gpuPairLink { lines := strings.Split(topoANSIRe.ReplaceAllString(raw, ""), "\n") headerIdx := -1 var gpuColIndices []int for i, line := range lines { trimmed := strings.TrimSpace(line) if strings.HasPrefix(trimmed, "GPU0") { parts := strings.Fields(trimmed) for j, col := range parts { if strings.HasPrefix(col, "GPU") { gpuColIndices = append(gpuColIndices, j) } } if len(gpuColIndices) >= 2 { headerIdx = i } break } } if headerIdx < 0 { return nil } colIdxToGPU := make(map[int]int, len(gpuColIndices)) for gpuIdx, colIdx := range gpuColIndices { colIdxToGPU[colIdx] = gpuIdx } seen := map[[2]int]bool{} var pairs []gpuPairLink rowGPU := -1 for _, line := range lines[headerIdx+1:] { trimmed := strings.TrimSpace(line) if !strings.HasPrefix(trimmed, "GPU") { continue } cells := strings.Fields(trimmed) if len(cells) == 0 { continue } rowLabel := strings.TrimPrefix(cells[0], "GPU") n, err := strconv.Atoi(rowLabel) if err != nil { continue } rowGPU = n for colIdx, colGPU := range colIdxToGPU { if colGPU == rowGPU { continue } dataIdx := colIdx + 1 if dataIdx >= len(cells) { continue } m := topoNVRe.FindStringSubmatch(cells[dataIdx]) if len(m) != 2 { continue } nv, err := strconv.Atoi(m[1]) if err != nil || nv <= 0 { continue } a, bGPU := rowGPU, colGPU if a > bGPU { a, bGPU = bGPU, a } key := [2]int{a, bGPU} if seen[key] { continue } seen[key] = true pairs = append(pairs, gpuPairLink{GPUA: a, GPUB: bGPU, NVLinks: nv}) } } sort.Slice(pairs, func(i, j int) bool { if pairs[i].GPUA != pairs[j].GPUA { return pairs[i].GPUA < pairs[j].GPUA } return pairs[i].GPUB < pairs[j].GPUB }) return pairs } // readTopoTechDump reads a file previously captured into the persistent // techdump directory by platform.System.CaptureTechnicalDump (run once per // audit cycle), rather than shelling out to nvidia-smi from the HTTP request // handler — a live call here would block page rendering on a wedged driver, // exactly the failure mode this tool exists to diagnose. func readTopoTechDump(exportDir, name string) (string, error) { out, err := os.ReadFile(filepath.Join(exportDir, "techdump", name)) if err != nil { return "", err } return string(out), nil } func readGPUTopologyMatrix(exportDir string) (string, error) { return readTopoTechDump(exportDir, "nvidia-smi-topo.txt") } // readNVIDIAIndexByBDF parses the persisted nvidia-smi-query.csv techdump // (index,pci.bus_id,...) to map PCI bus address (matching // HardwarePCIeDevice.Slot) to the GPU index nvidia-smi/topo -m reports, so // GPU-GPU edges (keyed by index) can be anchored to the correct box (keyed // by BDF) in the diagram. func readNVIDIAIndexByBDF(exportDir string) (map[string]int, error) { raw, err := readTopoTechDump(exportDir, "nvidia-smi-query.csv") if err != nil { return nil, err } result := map[string]int{} for _, line := range strings.Split(raw, "\n") { line = strings.TrimSpace(line) if line == "" { continue } parts := strings.Split(line, ",") if len(parts) < 2 { continue } idx, err := strconv.Atoi(strings.TrimSpace(parts[0])) if err != nil { continue } bdf := normalizeTopoBDF(strings.TrimSpace(parts[1])) if bdf == "" { continue } result[bdf] = idx } return result, nil } // normalizeTopoBDF normalizes a PCI bus address to "dddd:bb:dd.f" form so // nvidia-smi's "pci.bus_id" output can be matched against // HardwarePCIeDevice.Slot regardless of minor formatting differences // (case, leading domain padding). func normalizeTopoBDF(bdf string) string { bdf = strings.ToLower(strings.TrimSpace(bdf)) if bdf == "" { return "" } parts := strings.Split(bdf, ":") if len(parts) == 3 { domain := parts[0] if len(domain) > 4 { domain = domain[len(domain)-4:] } return domain + ":" + parts[1] + ":" + parts[2] } return bdf } // --------------------------------------------------------------------------- // Card status aggregation // // Every card on this page — whether it represents one component (CPU 1) or a // group of identical ones (GPU ×4) — is colored as a whole by its worst // observed status, with a plain-text summary as the card's last line // (e.g. "4 OK" or "3 OK, 1 Warning"). There is no separate status chip: a // chip needs its own fill, and the SVG boxes previously colored that chip // via CSS classes written for HTML (.badge-ok sets `background`/`color`, // which do nothing on an SVG / — only `fill` does), so every // chip rendered with the SVG default fill of solid black. Coloring the card // itself uses real `fill:var(--ok-bg)` etc. declarations, which sidesteps // that class entirely. // --------------------------------------------------------------------------- // classifyTopoSeverity converts a component's Status pointer to a severity // rank (0=unknown, 1=OK, 2=Warning, 3=Critical), treating nil/unrecognized // the same as "Unknown" — matches topoStatusBadgeClass's classification. func classifyTopoSeverity(status *string) int { if status == nil { return 0 } switch strings.ToUpper(strings.TrimSpace(*status)) { case "OK": return 1 case "WARNING", "WARN", "PARTIAL": return 2 case "CRITICAL", "FAIL", "FAILED", "ERROR": return 3 default: return 0 } } // topoSeverityColors returns the (fill, stroke, text) CSS var() triple a // whole card is painted with for a given worst-observed severity. func topoSeverityColors(sev int) (fill, stroke, text string) { switch sev { case 3: return "var(--crit-bg)", "var(--crit-border)", "var(--crit-fg)" case 2: return "var(--warn-bg)", "#c9ba9b", "var(--warn-fg)" case 1: return "var(--ok-bg)", "#a3c293", "var(--ok-fg)" default: return "var(--surface-2)", "var(--border)", "var(--muted)" } } // topoStatusTally counts how many components in a group fall into each // severity bucket, so a group card can report "3 OK, 1 Warning" rather than // collapsing to a single worst-of value and losing the rest. type topoStatusTally struct { unknown, ok, warn, crit int } func (t *topoStatusTally) add(sev int) { switch sev { case 3: t.crit++ case 2: t.warn++ case 1: t.ok++ default: t.unknown++ } } func (t topoStatusTally) total() int { return t.unknown + t.ok + t.warn + t.crit } func (t topoStatusTally) worst() int { switch { case t.crit > 0: return 3 case t.warn > 0: return 2 case t.ok > 0: return 1 default: return 0 } } // line renders the card's last-line status summary. func (t topoStatusTally) line() string { if t.total() == 0 { return "No data" } if t.total() == 1 { switch { case t.crit > 0: return "Critical" case t.warn > 0: return "Warning" case t.ok > 0: return "OK" default: return "Unknown" } } var parts []string if t.crit > 0 { parts = append(parts, fmt.Sprintf("%d Critical", t.crit)) } if t.warn > 0 { parts = append(parts, fmt.Sprintf("%d Warning", t.warn)) } if t.ok > 0 { parts = append(parts, fmt.Sprintf("%d OK", t.ok)) } if t.unknown > 0 { parts = append(parts, fmt.Sprintf("%d Unknown", t.unknown)) } return strings.Join(parts, ", ") } // topoCardInfo is the shared visual content for one card, rendered either as // an absolutely-positioned SVG box (main diagram) or an HTML flex item // (Memory/Power Supplies rows) by the two writers below. type topoCardInfo struct { label string // e.g. "CPU 1", "GPU", "Power Supplies" sublabel string // representative model/description, "" to omit count int // components represented by this card; >1 draws a stack statusLine string // last line of card text, e.g. "4 OK, 1 Warning" fillVar string strokeVar string textVar string detailType string // "" = not clickable } // topoStackLayers returns how many faint backing cards to draw behind the // front card to read as "a stack of N", capped at 2 — enough to signal // "more than one" without the deck becoming its own visual clutter. func topoStackLayers(count int) int { if count <= 1 { return 0 } if count-1 > 2 { return 2 } return count - 1 } // --------------------------------------------------------------------------- // Main topology diagram // --------------------------------------------------------------------------- const ( topoColWidth = 220 topoBoxWidth = 190 topoBoxHeight = 70 topoDeviceGap = 14 topoTopMargin = 30 topoStackStep = 4 // px offset per backing layer in the card-stack effect ) type topoBox struct { x, y, w, h int topoCardInfo } type topoEdge struct { x1, y1, x2, y2 int color string } func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string { // A GPU hardware fault that needs a physical reboot (Xid 79 "fallen off // the bus", Xid 154 "Node Reboot Required" — see gpuNeedsPhysicalReboot) // isn't visible in dev.Status: the collector only sets that from PCIe // link-speed checks, not from SAT/kmsg results. Without this, a GPU that // dropped off the bus mid-test still renders green here even though the // Hardware Summary card is showing a critical banner for it. gpuHardwareFault := false if db, err := app.OpenComponentStatusDB(filepath.Join(exportDir, "component-status.json")); err == nil { if _, needsReboot := gpuNeedsPhysicalReboot(matchedRecords(db.All(), nil, []string{"pcie:gpu:"})); needsReboot { gpuHardwareFault = true } } socketIdx := buildSocketIndex(hw.CPUs) numCols := len(hw.CPUs) if numCols == 0 { numCols = 1 } unknownCol := numCols // extra trailing column for unmatched devices // Group PCIe devices (GPU/NIC/RAID only — matches the mockup's node // types) into columns by NUMA node, falling back to the "unknown" bucket. type placedDevice struct { dev schema.HardwarePCIeDevice kind string // "gpu", "nic", "raid" col int bdf string } var placed []placedDevice for _, dev := range hw.PCIeDevices { var kind string switch { case dev.DeviceClass != nil && isGPUDeviceClass(*dev.DeviceClass): kind = "gpu" case isNICDeviceClassDev(dev): kind = "nic" case dev.DeviceClass != nil && isRAIDControllerClass(*dev.DeviceClass): kind = "raid" default: continue } col := unknownCol if dev.NUMANode != nil { if ci, ok := socketIdx[*dev.NUMANode]; ok { col = ci } } bdf := "" if dev.Slot != nil { bdf = normalizeTopoBDF(*dev.Slot) } else if dev.BDF != nil { bdf = normalizeTopoBDF(*dev.BDF) } placed = append(placed, placedDevice{dev: dev, kind: kind, col: col, bdf: bdf}) } hasUnknownCol := false for _, p := range placed { if p.col == unknownCol { hasUnknownCol = true break } } totalCols := numCols if hasUnknownCol { totalCols++ } // GPU index<->BDF map + pairwise NVLink adjacency, read from the // persisted techdump captured during the last audit cycle, best-effort: // if the dump is missing (older audit, no NVIDIA GPUs), this is simply // skipped. Used only to detect the cross-NUMA-bonded-pair anomaly below; // the pairwise links themselves are drawn in the separate NVLink // Topology card, since grouping same-kind/same-column devices into one // stacked card here leaves no single per-GPU anchor point to draw a // pairwise connector to or from. bdfToIndex, _ := readNVIDIAIndexByBDF(exportDir) var pairs []gpuPairLink if topoMatrix, err := readGPUTopologyMatrix(exportDir); err == nil { pairs = parseGPUPairAdjacency(topoMatrix) } gpuNUMAByIndex := map[int]*int{} gpuBDFByIndex := map[int]string{} for _, p := range placed { if p.kind != "gpu" || p.bdf == "" { continue } if idx, ok := bdfToIndex[p.bdf]; ok { gpuNUMAByIndex[idx] = p.dev.NUMANode gpuBDFByIndex[idx] = p.bdf } } // A bonded pair spanning two different NUMA nodes is treated as an // anomaly (not a neutral fact) per project decision: a bonded pair is // expected to sit on one NUMA node, so a cross-NUMA bond escalates both // GPUs' effective severity to at least Warning, regardless of their own // reported SAT status. crossNUMAWarnBDF := map[string]bool{} for _, pair := range pairs { numaA, okA := gpuNUMAByIndex[pair.GPUA] numaB, okB := gpuNUMAByIndex[pair.GPUB] if !okA || !okB || numaA == nil || numaB == nil || *numaA == *numaB { continue } crossNUMAWarnBDF[gpuBDFByIndex[pair.GPUA]] = true crossNUMAWarnBDF[gpuBDFByIndex[pair.GPUB]] = true } kindOrder := []string{"gpu", "nic", "raid"} kindLabel := map[string]string{"gpu": "GPU", "nic": "NIC", "raid": "RAID"} // Attach memory DIMMs to their CPU column too, the same way GPU/NIC/RAID // PCIe devices are attached via NUMANode — memory has no NUMANode field // in the schema, so this reads the DIMM's own Locator/Bank Locator // strings instead (see dimmRawNode). DIMMs that can't be confidently // attached fall back to the unattached "Memory" row below the diagram, // same as before this existed. memBankNodes := map[string]int{} if raw, err := readTopoTechDump(exportDir, "dmidecode-type17.txt"); err == nil { memBankNodes = parseDIMMBankLocatorNodes(raw) } memCol := make([]int, len(hw.Memory)) memMatched := make([]bool, len(hw.Memory)) var memRawNodes []int for i, m := range hw.Memory { if node, ok := dimmRawNode(m, memBankNodes); ok { memCol[i] = node memMatched[i] = true memRawNodes = append(memRawNodes, node) } } memColIdx := buildMemoryColumnIndex(memRawNodes) for i := range hw.Memory { if !memMatched[i] { continue } col := memColIdx[memCol[i]] if col >= numCols { memMatched[i] = false continue } memCol[i] = col } var boxes []topoBox var pcieEdges []topoEdge for col := 0; col < totalCols; col++ { colX := (col+1)*24 + col*topoColWidth if col < len(hw.CPUs) { cpu := hw.CPUs[col] model := "" if cpu.Model != nil { model = *cpu.Model } socket := col if cpu.Socket != nil { socket = *cpu.Socket } var tally topoStatusTally tally.add(classifyTopoSeverity(cpu.Status)) fill, stroke, text := topoSeverityColors(tally.worst()) boxes = append(boxes, topoBox{ x: colX, y: topoTopMargin, w: topoBoxWidth, h: topoBoxHeight, topoCardInfo: topoCardInfo{ label: fmt.Sprintf("CPU %d", socket), sublabel: model, count: 1, statusLine: tally.line(), fillVar: fill, strokeVar: stroke, textVar: text, detailType: "cpu", }, }) } y := topoTopMargin + topoBoxHeight + topoDeviceGap*2 // Memory goes first in the chain, directly under the CPU box: DIMMs // are wired straight to the socket's memory controller, not reached // over PCIe like the GPU/NIC/RAID chain below it. var memGroup []schema.HardwareMemory for i, m := range hw.Memory { if memMatched[i] && memCol[i] == col { memGroup = append(memGroup, m) } } if len(memGroup) > 0 { var tally topoStatusTally sizeGB := 0 for _, m := range memGroup { tally.add(classifyTopoSeverity(m.Status)) if m.SizeMB != nil { sizeGB += *m.SizeMB / 1024 } } fill, stroke, text := topoSeverityColors(tally.worst()) sublabel := "" if sizeGB > 0 { sublabel = fmt.Sprintf("%d GB total", sizeGB) } stackLayers := topoStackLayers(len(memGroup)) boxes = append(boxes, topoBox{ x: colX, y: y, w: topoBoxWidth, h: topoBoxHeight, topoCardInfo: topoCardInfo{ label: "Memory", sublabel: sublabel, count: len(memGroup), statusLine: tally.line(), fillVar: fill, strokeVar: stroke, textVar: text, detailType: "memory", }, }) if col < len(hw.CPUs) { pcieEdges = append(pcieEdges, topoEdge{ x1: colX + topoBoxWidth/2, y1: topoTopMargin + topoBoxHeight, x2: colX + topoBoxWidth/2, y2: y, color: "var(--ok-fg)", }) } y += topoBoxHeight + topoDeviceGap + stackLayers*topoStackStep } for _, kind := range kindOrder { var group []placedDevice for _, p := range placed { if p.col == col && p.kind == kind { group = append(group, p) } } if len(group) == 0 { continue } var tally topoStatusTally model := "" edgeColor := "var(--ok-fg)" for i, p := range group { sev := classifyTopoSeverity(p.dev.Status) if kind == "gpu" && crossNUMAWarnBDF[p.bdf] && sev < 2 { sev = 2 } if kind == "gpu" && gpuHardwareFault && sev < 3 { sev = 3 } tally.add(sev) if i == 0 && p.dev.Model != nil { model = *p.dev.Model } if topoEdgeColorVar(p.dev) == "var(--warn-fg)" { edgeColor = "var(--warn-fg)" } } fill, stroke, text := topoSeverityColors(tally.worst()) stackLayers := topoStackLayers(len(group)) boxes = append(boxes, topoBox{ x: colX, y: y, w: topoBoxWidth, h: topoBoxHeight, topoCardInfo: topoCardInfo{ label: kindLabel[kind], sublabel: model, count: len(group), statusLine: tally.line(), fillVar: fill, strokeVar: stroke, textVar: text, detailType: kind, }, }) if col < len(hw.CPUs) { pcieEdges = append(pcieEdges, topoEdge{ x1: colX + topoBoxWidth/2, y1: topoTopMargin + topoBoxHeight, x2: colX + topoBoxWidth/2, y2: y, color: edgeColor, }) } y += topoBoxHeight + topoDeviceGap + stackLayers*topoStackStep } } maxDeviceY := topoTopMargin + topoBoxHeight + topoDeviceGap*2 for _, box := range boxes { bottom := box.y + box.h + topoStackLayers(box.count)*topoStackStep if bottom > maxDeviceY { maxDeviceY = bottom } } svgHeight := maxDeviceY + 24 svgWidth := totalCols*topoColWidth + 48 var b strings.Builder // Wrapped in its own horizontally-scrolling container (matching the // overflow-x:auto convention used for wide tables elsewhere in webui) // rather than max-width:100% — squashing a node/edge diagram to fit a // narrow viewport makes labels and badges illegible, whereas scrolling // keeps the diagram readable at its natural size on any screen width. b.WriteString(`
`) fmt.Fprintf(&b, ``+"\n", svgWidth, svgHeight, svgWidth, svgHeight) for _, e := range pcieEdges { fmt.Fprintf(&b, ``+"\n", e.x1, e.y1, e.x2, e.y2, e.color) } for _, box := range boxes { writeTopoBoxSVG(&b, box) } b.WriteString(`
`) // Firmware (BMC/BIOS/...) and PSUs have no PCIe/CPU affinity to anchor // them to a column, and there can be an arbitrary number of any of them // — so unlike the diagram above, they're plain flex-wrap HTML below the // SVG rather than absolutely-positioned SVG boxes. A fixed-size SVG // canvas has no way to wrap overflow onto a new row, which is exactly // what caused these to pile up and overlap once a board had more // PSUs/firmware records than fit in one fixed-width row. // // Memory DIMMs that were matched to a CPU column above already got a // box in the SVG diagram; only DIMMs that couldn't be attached to a // column (see memMatched above) fall back to this row. var unmatchedMem []schema.HardwareMemory for i, m := range hw.Memory { if !memMatched[i] { unmatchedMem = append(unmatchedMem, m) } } if len(unmatchedMem) > 0 { var tally topoStatusTally for _, m := range unmatchedMem { tally.add(classifyTopoSeverity(m.Status)) } fill, stroke, text := topoSeverityColors(tally.worst()) sizeGB := 0 for _, m := range unmatchedMem { if m.SizeMB != nil { sizeGB += *m.SizeMB / 1024 } } sublabel := "" if sizeGB > 0 { sublabel = fmt.Sprintf("%d GB total", sizeGB) } b.WriteString(renderTopoFlexRow("Memory", []topoCardInfo{{ label: "Memory", sublabel: sublabel, count: len(unmatchedMem), statusLine: tally.line(), fillVar: fill, strokeVar: stroke, textVar: text, detailType: "memory", }})) } var firmwareItems []topoCardInfo for _, rec := range hw.Firmware { // Firmware records carry no per-item status in the schema (they are // identity, not health, facts), so each stays a neutral, uncolored // card rather than forcing a fake "Unknown" status line. fillVar, strokeVar, textVar := topoSeverityColors(0) firmwareItems = append(firmwareItems, topoCardInfo{ label: rec.DeviceName, sublabel: "fw " + rec.Version, count: 1, fillVar: fillVar, strokeVar: strokeVar, textVar: textVar, }) } b.WriteString(renderTopoFlexRow("Firmware", firmwareItems)) if len(hw.PowerSupplies) > 0 { var tally topoStatusTally watt := 0 for _, psu := range hw.PowerSupplies { tally.add(classifyTopoSeverity(psu.Status)) if psu.WattageW != nil { watt = *psu.WattageW } } fill, stroke, text := topoSeverityColors(tally.worst()) sublabel := "" if watt > 0 { sublabel = fmt.Sprintf("%dW each", watt) } b.WriteString(renderTopoFlexRow("Power Supplies", []topoCardInfo{{ label: "Power Supplies", sublabel: sublabel, count: len(hw.PowerSupplies), statusLine: tally.line(), fillVar: fill, strokeVar: stroke, textVar: text, detailType: "psu", }})) } return topoCard("Topology", b.String()) } // renderTopoFlexRow renders a labeled, wrapping row of component cards. // Returns "" if items is empty (e.g. no PSU data in this audit). func renderTopoFlexRow(title string, items []topoCardInfo) string { if len(items) == 0 { return "" } var b strings.Builder fmt.Fprintf(&b, `
%s
`, html.EscapeString(title)) b.WriteString(`
`) for _, item := range items { onclick := "" cursor := "default" if item.detailType != "" { onclick = fmt.Sprintf(` onclick="openComponentDetail('%s')"`, item.detailType) cursor = "pointer" } stackLayers := topoStackLayers(item.count) // Extra right/bottom padding on the wrapper reserves room for the // backing layers of the stack effect so they aren't clipped by the // flex container. fmt.Fprintf(&b, `
`, stackLayers*topoStackStep, stackLayers*topoStackStep) for i := stackLayers; i >= 1; i-- { off := i * topoStackStep fmt.Fprintf(&b, `
`, off, off, item.fillVar, item.strokeVar) } fmt.Fprintf(&b, ``, onclick, cursor, item.fillVar, item.strokeVar, item.textVar) label := item.label if item.count > 1 { label = fmt.Sprintf("%s ×%d", item.label, item.count) } fmt.Fprintf(&b, `
%s
`, html.EscapeString(label)) if item.sublabel != "" { fmt.Fprintf(&b, `
%s
`, html.EscapeString(item.sublabel)) } if item.statusLine != "" { fmt.Fprintf(&b, `
%s
`, html.EscapeString(item.statusLine)) } b.WriteString(`
`) } b.WriteString(``) return b.String() } func writeTopoBoxSVG(b *strings.Builder, box topoBox) { onclick := "" cursor := "default" if box.detailType != "" { onclick = fmt.Sprintf(` onclick="openComponentDetail('%s')"`, box.detailType) cursor = "pointer" } fmt.Fprintf(b, ``, onclick, cursor) // Stack-of-cards effect: faint offset rects behind the front card when // this box represents more than one physical component (e.g. 4 GPUs in // one NUMA column), so a group reads as "a deck of N" rather than a // single item. Peeks toward the bottom-right, into space already // reserved between this box and the next one in the column. for i := topoStackLayers(box.count); i >= 1; i-- { off := i * topoStackStep fmt.Fprintf(b, ``+"\n", box.x+off, box.y+off, box.w, box.h, box.fillVar, box.strokeVar) } fmt.Fprintf(b, ``+"\n", box.x, box.y, box.w, box.h, box.fillVar, box.strokeVar) label := box.label if box.count > 1 { label = fmt.Sprintf("%s ×%d", box.label, box.count) } fmt.Fprintf(b, `%s`+"\n", box.x+10, box.y+20, box.textVar, html.EscapeString(label)) if box.sublabel != "" { fmt.Fprintf(b, `%s`+"\n", box.x+10, box.y+36, box.textVar, html.EscapeString(truncateTopoLabel(box.sublabel, 26))) } if box.statusLine != "" { fmt.Fprintf(b, `%s`+"\n", box.x+10, box.y+box.h-10, box.textVar, html.EscapeString(box.statusLine)) } b.WriteString(`` + "\n") } func truncateTopoLabel(s string, max int) string { if len(s) <= max { return s } if max <= 1 { return s[:max] } return s[:max-1] + "…" } // --------------------------------------------------------------------------- // Separate NVLink topology card (read from techdump, not written to any // ingest contract) // --------------------------------------------------------------------------- type topoNVLinkPort struct { Index int Active bool SpeedGBs *float64 ReplayErrors int64 RecoveryErrors int64 CRCErrors int64 } var ( topoNVLinkGPUHeaderRe = regexp.MustCompile(`^GPU (\d+):`) topoNVLinkSpeedLineRe = regexp.MustCompile(`^Link (\d+):\s*([\d.]+)\s*GB/s`) topoNVLinkInactiveRe = regexp.MustCompile(`^Link (\d+):\s*`) topoNVLinkErrorLineRe = regexp.MustCompile(`^Link (\d+):\s*(Replay|Recovery|CRC) Errors:\s*(\d+)`) ) func readTopoNVLinkStatus(exportDir string) (map[int][]topoNVLinkPort, error) { raw, err := readTopoTechDump(exportDir, "nvidia-smi-nvlink-status.txt") if err != nil { return nil, err } return parseTopoNVLinkStatus(raw), nil } func parseTopoNVLinkStatus(raw string) map[int][]topoNVLinkPort { result := map[int][]topoNVLinkPort{} currentGPU := -1 for _, line := range strings.Split(raw, "\n") { trimmed := strings.TrimSpace(line) if m := topoNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil { currentGPU, _ = strconv.Atoi(m[1]) continue } if currentGPU < 0 { continue } if m := topoNVLinkInactiveRe.FindStringSubmatch(trimmed); m != nil { idx, _ := strconv.Atoi(m[1]) result[currentGPU] = append(result[currentGPU], topoNVLinkPort{Index: idx, Active: false}) continue } if m := topoNVLinkSpeedLineRe.FindStringSubmatch(trimmed); m != nil { idx, _ := strconv.Atoi(m[1]) port := topoNVLinkPort{Index: idx, Active: true} if speed, err := strconv.ParseFloat(m[2], 64); err == nil { port.SpeedGBs = &speed } result[currentGPU] = append(result[currentGPU], port) } } return result } func readTopoNVLinkErrors(exportDir string) (map[int]map[int][3]int64, error) { raw, err := readTopoTechDump(exportDir, "nvidia-smi-nvlink-errors.txt") if err != nil { return nil, err } return parseTopoNVLinkErrors(raw), nil } // parseTopoNVLinkErrors returns, per GPU then link index, [replay, recovery, crc]. func parseTopoNVLinkErrors(raw string) map[int]map[int][3]int64 { result := map[int]map[int][3]int64{} currentGPU := -1 for _, line := range strings.Split(raw, "\n") { trimmed := strings.TrimSpace(line) if m := topoNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil { currentGPU, _ = strconv.Atoi(m[1]) continue } if currentGPU < 0 { continue } m := topoNVLinkErrorLineRe.FindStringSubmatch(trimmed) if m == nil { continue } linkIdx, _ := strconv.Atoi(m[1]) count, _ := strconv.ParseInt(m[3], 10, 64) if result[currentGPU] == nil { result[currentGPU] = map[int][3]int64{} } c := result[currentGPU][linkIdx] switch m[2] { case "Replay": c[0] = count case "Recovery": c[1] = count case "CRC": c[2] = count } result[currentGPU][linkIdx] = c } return result } // renderTopoNVLinkCard renders the separate NVLink topology card. Returns "" // if there are fewer than 2 NVIDIA GPUs, or the nvidia-smi nvlink techdump // wasn't captured (older audit, or nvidia-smi unavailable on that run). func renderTopoNVLinkCard(hw schema.HardwareSnapshot, exportDir string) string { gpuCount := 0 for _, dev := range hw.PCIeDevices { if dev.DeviceClass != nil && isGPUDeviceClass(*dev.DeviceClass) { gpuCount++ } } if gpuCount < 2 { return "" } status, err := readTopoNVLinkStatus(exportDir) if err != nil || len(status) == 0 { return topoCard("NVLink Topology", `nvidia-smi nvlink data unavailable`) } errors, _ := readTopoNVLinkErrors(exportDir) topoMatrix, _ := readGPUTopologyMatrix(exportDir) pairs := parseGPUPairAdjacency(topoMatrix) var bodyB strings.Builder if gpuCount <= 4 && len(pairs) > 0 { // Small GPU count: per-pair box+line with per-link detail. for _, pair := range pairs { activeCount, total, hasError := 0, 0, false for _, port := range status[pair.GPUA] { total++ if port.Active { activeCount++ } } for _, counters := range errors[pair.GPUA] { if counters[0] != 0 || counters[1] != 0 || counters[2] != 0 { hasError = true } } color := "var(--ok-fg)" switch { case hasError: color = "var(--crit-fg)" case total > 0 && activeCount < total: color = "var(--warn-fg)" } fmt.Fprintf(&bodyB, `
`+ `
GPU %d
`+ `
`+ `
GPU %d
`+ `
%d/%d links active%s
`+ `
`, pair.GPUA, color, pair.GPUB, activeCount, total, errNoteSuffix(hasError)) } } else if len(pairs) > 0 { // Larger GPU counts (NVSwitch fabric): aggregate pair table instead of // an unreadable all-to-all graph. bodyB.WriteString(``) for _, pair := range pairs { fmt.Fprintf(&bodyB, ``, pair.GPUA, pair.GPUB, pair.NVLinks) } bodyB.WriteString(`
GPU AGPU BNVLinks
GPU %dGPU %d%d
`) } else { bodyB.WriteString(`No NVLink-bonded GPU pairs found`) } return topoCard("NVLink Topology", bodyB.String()) } func errNoteSuffix(hasError bool) string { if hasError { return " — errors detected" } return "" } // --------------------------------------------------------------------------- // Inventory fallback for the component-detail modal // // handleAPIComponentDetail normally sources records from app.ComponentStatusDB, // which only gains entries once something has actually written a status // observation (SAT run, watchdog tick, ...). On a freshly booted host that // hasn't run SAT yet, StatusDB can be entirely empty for a component type even // though the /topo card for it already shows "N OK" — that card reads // schema.HardwareComponentStatus.Status straight from the audit snapshot. // inventoryFallbackRecords bridges that gap by building synthetic records // from the same snapshot/classifiers the topology card uses, so the two // views never disagree about how many devices exist or their status. // --------------------------------------------------------------------------- // topoSeverityStatus renders classifyTopoSeverity's rank back into the status // string vocabulary renderComponentDetail/chipLetterClass expect ("OK", // "Warning", "Critical", "Unknown") — kept in lockstep with classifyTopoSeverity // so a device the topo card counts as "OK" is never shown here as "Unknown". func topoSeverityStatus(status *string) string { switch classifyTopoSeverity(status) { case 3: return "Critical" case 2: return "Warning" case 1: return "OK" default: return "Unknown" } } // pcieDeviceKind classifies a PCIe device the same way renderTopoMainDiagram // does, returning "" for devices that aren't GPU/NIC/RAID. func pcieDeviceKind(dev schema.HardwarePCIeDevice) string { switch { case dev.DeviceClass != nil && isGPUDeviceClass(*dev.DeviceClass): return "gpu" case isNICDeviceClassDev(dev): return "nic" case dev.DeviceClass != nil && isRAIDControllerClass(*dev.DeviceClass): return "raid" default: return "" } } // pcieDeviceKey builds a stable, human-readable component key for a PCIe // device: ":" when a slot/BDF is known, else ":". func pcieDeviceKey(kind string, index int, dev schema.HardwarePCIeDevice) string { bdf := "" if dev.Slot != nil { bdf = normalizeTopoBDF(*dev.Slot) } else if dev.BDF != nil { bdf = normalizeTopoBDF(*dev.BDF) } if bdf != "" { return kind + ":" + bdf } return fmt.Sprintf("%s:%d", kind, index) } // inventoryFallbackRecords builds ComponentStatusRecord entries straight from // the audit inventory (bee-audit.json) for the given component type, used // when ComponentStatusDB has no matching records yet. Records carry only // ComponentKey/Status — no LastCheckedAt/History — so renderComponentDetail // renders them without a "checked at" timestamp or sparkline. func inventoryFallbackRecords(compType string, opts HandlerOptions) []app.ComponentStatusRecord { data, err := loadSnapshot(opts.AuditPath) if err != nil { return nil } var ingest schema.HardwareIngestRequest if err := json.Unmarshal(data, &ingest); err != nil { return nil } hw := ingest.Hardware var records []app.ComponentStatusRecord switch compType { case "cpu": for i, cpu := range hw.CPUs { key := fmt.Sprintf("cpu:%d", i) if cpu.Socket != nil { key = fmt.Sprintf("cpu:socket%d", *cpu.Socket) } records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(cpu.Status)}) } case "memory": for i, m := range hw.Memory { key := fmt.Sprintf("memory:%d", i) if m.Slot != nil && strings.TrimSpace(*m.Slot) != "" { key = "memory:" + strings.TrimSpace(*m.Slot) } records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(m.Status)}) } case "storage": for i, s := range hw.Storage { key := fmt.Sprintf("storage:%d", i) if s.Slot != nil && strings.TrimSpace(*s.Slot) != "" { key = "storage:" + strings.TrimSpace(*s.Slot) } records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(s.Status)}) } case "psu": for i, p := range hw.PowerSupplies { key := fmt.Sprintf("psu:%d", i) if p.Slot != nil && strings.TrimSpace(*p.Slot) != "" { key = "psu:" + strings.TrimSpace(*p.Slot) } records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(p.Status)}) } case "gpu", "nic", "raid": for i, dev := range hw.PCIeDevices { if pcieDeviceKind(dev) != compType { continue } key := pcieDeviceKey(compType, i, dev) records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(dev.Status)}) } } return records }