Mikhail Chusavitin
1d5c02ebaa
PSU and firmware (BMC/BIOS) boxes were absolutely-positioned SVG rects in a
single fixed-width row with no wrap, so an arbitrary/larger count piled up
and overlapped once a board had more of either than fit in that row. Move
them to plain flex-wrap HTML below the diagram (Firmware row first, then
Power Supplies), which reflows naturally for any count. The main CPU/PCIe/
GPU diagram now scrolls horizontally (overflow-x:auto) instead of being
squashed to fit narrow viewports, matching the wide-table convention used
elsewhere in webui.
Also fixes a crash: renderTopoMainDiagram forced numCols to 1 for layout
purposes when a snapshot has zero CPUs, then unconditionally indexed
hw.CPUs[0], panicking the whole /topo page on any audit without CPU data.
Same-kind/same-column components (e.g. 4 GPUs in one NUMA node) now render
as one stacked card summarizing worst-case status plus a tally line ("3 OK,
1 Warning") instead of one box per component, and card severity coloring
uses real fill/stroke vars instead of HTML badge classes that don't apply
any style to SVG shapes.
1041 lines
33 KiB
Go
1041 lines
33 KiB
Go
package webui
|
||
|
||
import (
|
||
"encoding/json"
|
||
"fmt"
|
||
"html"
|
||
"os"
|
||
"path/filepath"
|
||
"regexp"
|
||
"sort"
|
||
"strconv"
|
||
"strings"
|
||
|
||
"bee/audit/internal/schema"
|
||
)
|
||
|
||
// renderTopo renders the /topo page: a read-only visualization of the server
|
||
// topology (CPU sockets, NUMA-affine PCIe devices, PSU/BMC) plus a separate
|
||
// NVLink topology card. It is pure visualization: everything it reads either
|
||
// already exists in the audit.json contract, or comes from the persisted
|
||
// techdump captured once per audit cycle (platform.CaptureTechnicalDump) —
|
||
// nothing here shells out to nvidia-smi itself, writes to
|
||
// schema.HardwarePCIeDevice or any other contract type, or talks to
|
||
// Reanimator Core.
|
||
func renderTopo(opts HandlerOptions) string {
|
||
data, err := loadSnapshot(opts.AuditPath)
|
||
if err != nil {
|
||
return topoCard("Topology", `<span class="badge badge-unknown">No audit data</span>`)
|
||
}
|
||
var ingest schema.HardwareIngestRequest
|
||
if err := json.Unmarshal(data, &ingest); err != nil {
|
||
return topoCard("Topology", `<span class="badge badge-err">Parse error</span>`)
|
||
}
|
||
hw := ingest.Hardware
|
||
|
||
var b strings.Builder
|
||
b.WriteString(renderTopoMainDiagram(hw, opts.ExportDir))
|
||
if nv := renderTopoNVLinkCard(hw, opts.ExportDir); nv != "" {
|
||
b.WriteString(nv)
|
||
}
|
||
return b.String()
|
||
}
|
||
|
||
func topoCard(title, body string) string {
|
||
return `<div class="card"><div class="card-head">` + html.EscapeString(title) + `</div><div class="card-body">` + body + `</div></div>`
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Classification helpers
|
||
//
|
||
// webui does not import collector (matches the existing isGPUDeviceClass
|
||
// precedent in pages.go, which already locally duplicates collector.isGPUClass
|
||
// instead of importing the package for one classifier).
|
||
// ---------------------------------------------------------------------------
|
||
|
||
// isNICDeviceClassDev mirrors the classification logic in hwDescribeNIC
|
||
// (pages.go), applied to a single device instead of aggregated counts.
|
||
func isNICDeviceClassDev(dev schema.HardwarePCIeDevice) bool {
|
||
if dev.DeviceClass != nil {
|
||
c := strings.ToLower(strings.TrimSpace(*dev.DeviceClass))
|
||
if c == "ethernetcontroller" || c == "networkcontroller" || strings.Contains(c, "fibrechannel") {
|
||
return true
|
||
}
|
||
}
|
||
return len(dev.MacAddresses) > 0
|
||
}
|
||
|
||
// isRAIDControllerClass matches the canonical class strings produced by
|
||
// collector.mapPCIeDeviceClass for RAID/storage HBAs.
|
||
func isRAIDControllerClass(class string) bool {
|
||
switch strings.TrimSpace(class) {
|
||
case "MassStorageController", "StorageController":
|
||
return true
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Status / link-speed coloring
|
||
// ---------------------------------------------------------------------------
|
||
|
||
// pcieGenRank ranks a PCIe generation label ("Gen3", "Gen4", ...) for
|
||
// comparison. Mirrors collector.pcieLinkSpeedRank's ordering; duplicated
|
||
// locally rather than exported, per the same "no collector import in webui"
|
||
// convention used for isGPUDeviceClass/isRAIDControllerClass.
|
||
func pcieGenRank(gen string) int {
|
||
gen = strings.ToLower(strings.TrimSpace(gen))
|
||
gen = strings.TrimPrefix(gen, "gen")
|
||
n, err := strconv.Atoi(gen)
|
||
if err != nil {
|
||
return 0
|
||
}
|
||
return n
|
||
}
|
||
|
||
// topoEdgeColorVar computes the CPU->device edge color strictly from
|
||
// link_speed vs max_link_speed — NOT from dev.Status, since Status can also
|
||
// be overwritten by SAT/acceptance-test results on the same PCIe device,
|
||
// which would conflate "link is physically degraded" with "this GPU failed
|
||
// its stress test" into the same color.
|
||
func topoEdgeColorVar(dev schema.HardwarePCIeDevice) string {
|
||
if dev.LinkSpeed == nil || dev.MaxLinkSpeed == nil {
|
||
return "var(--muted)"
|
||
}
|
||
if pcieGenRank(*dev.LinkSpeed) < pcieGenRank(*dev.MaxLinkSpeed) {
|
||
return "var(--warn-fg)"
|
||
}
|
||
return "var(--ok-fg)"
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// NUMA node -> CPU socket join (heuristic, no guaranteed hardware mapping)
|
||
// ---------------------------------------------------------------------------
|
||
|
||
// buildSocketIndex maps a NUMA node number to the index into cpus for the
|
||
// socket occupying that position in ascending Socket-designation order.
|
||
//
|
||
// Linux NUMA node numbering is always 0-based (node0, node1, ...), but
|
||
// dmidecode's "Socket Designation" is board-defined and frequently 1-based
|
||
// ("CPU1", "CPU2", ...). Mapping NUMA node N to the CPU whose Socket field
|
||
// equals N (as an earlier version of this function did) silently fails on
|
||
// any 1-indexed board: node 0 has no match (dropped into the "unknown"
|
||
// column) and node 1 wrongly maps to the first CPU. Ranking by Socket value
|
||
// instead assumes only that node order follows socket order — true for the
|
||
// common case of N-socket boards — without depending on the numbering base.
|
||
func buildSocketIndex(cpus []schema.HardwareCPU) map[int]int {
|
||
order := make([]int, len(cpus))
|
||
for i := range cpus {
|
||
order[i] = i
|
||
}
|
||
sort.SliceStable(order, func(a, b int) bool {
|
||
ca, cb := cpus[order[a]], cpus[order[b]]
|
||
sa, sb := 0, 0
|
||
if ca.Socket != nil {
|
||
sa = *ca.Socket
|
||
}
|
||
if cb.Socket != nil {
|
||
sb = *cb.Socket
|
||
}
|
||
return sa < sb
|
||
})
|
||
idx := map[int]int{}
|
||
for numaNode, cpuIdx := range order {
|
||
idx[numaNode] = cpuIdx
|
||
}
|
||
return idx
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// GPU pairwise NVLink adjacency (from a live "nvidia-smi topo -m" query)
|
||
// ---------------------------------------------------------------------------
|
||
|
||
type gpuPairLink struct {
|
||
GPUA, GPUB int
|
||
NVLinks int
|
||
}
|
||
|
||
var topoNVRe = regexp.MustCompile(`(?i)^NV(\d+)$`)
|
||
|
||
// nvidia-smi underlines the topo -m header row with ANSI CSI sequences
|
||
// (ESC[4m...ESC[0m) even when stdout is not a TTY, so the captured techdump
|
||
// contains them and "GPU0" is not at the start of the trimmed header line.
|
||
var topoANSIRe = regexp.MustCompile("\x1b\\[[0-9;]*[A-Za-z]")
|
||
|
||
// parseGPUPairAdjacency returns every GPU pair with a nonzero NVLink bond
|
||
// count from a "nvidia-smi topo -m" matrix. Unlike parseNVIDIATopologyMatrix
|
||
// (collector package, aggregate-only: min/all-active/count), this returns
|
||
// who is bonded to whom — required so GPU-GPU edges are drawn for actually
|
||
// bonded pairs, not for adjacent boxes in the layout.
|
||
func parseGPUPairAdjacency(raw string) []gpuPairLink {
|
||
lines := strings.Split(topoANSIRe.ReplaceAllString(raw, ""), "\n")
|
||
headerIdx := -1
|
||
var gpuColIndices []int
|
||
for i, line := range lines {
|
||
trimmed := strings.TrimSpace(line)
|
||
if strings.HasPrefix(trimmed, "GPU0") {
|
||
parts := strings.Fields(trimmed)
|
||
for j, col := range parts {
|
||
if strings.HasPrefix(col, "GPU") {
|
||
gpuColIndices = append(gpuColIndices, j)
|
||
}
|
||
}
|
||
if len(gpuColIndices) >= 2 {
|
||
headerIdx = i
|
||
}
|
||
break
|
||
}
|
||
}
|
||
if headerIdx < 0 {
|
||
return nil
|
||
}
|
||
|
||
colIdxToGPU := make(map[int]int, len(gpuColIndices))
|
||
for gpuIdx, colIdx := range gpuColIndices {
|
||
colIdxToGPU[colIdx] = gpuIdx
|
||
}
|
||
|
||
seen := map[[2]int]bool{}
|
||
var pairs []gpuPairLink
|
||
rowGPU := -1
|
||
for _, line := range lines[headerIdx+1:] {
|
||
trimmed := strings.TrimSpace(line)
|
||
if !strings.HasPrefix(trimmed, "GPU") {
|
||
continue
|
||
}
|
||
cells := strings.Fields(trimmed)
|
||
if len(cells) == 0 {
|
||
continue
|
||
}
|
||
rowLabel := strings.TrimPrefix(cells[0], "GPU")
|
||
n, err := strconv.Atoi(rowLabel)
|
||
if err != nil {
|
||
continue
|
||
}
|
||
rowGPU = n
|
||
for colIdx, colGPU := range colIdxToGPU {
|
||
if colGPU == rowGPU {
|
||
continue
|
||
}
|
||
dataIdx := colIdx + 1
|
||
if dataIdx >= len(cells) {
|
||
continue
|
||
}
|
||
m := topoNVRe.FindStringSubmatch(cells[dataIdx])
|
||
if len(m) != 2 {
|
||
continue
|
||
}
|
||
nv, err := strconv.Atoi(m[1])
|
||
if err != nil || nv <= 0 {
|
||
continue
|
||
}
|
||
a, bGPU := rowGPU, colGPU
|
||
if a > bGPU {
|
||
a, bGPU = bGPU, a
|
||
}
|
||
key := [2]int{a, bGPU}
|
||
if seen[key] {
|
||
continue
|
||
}
|
||
seen[key] = true
|
||
pairs = append(pairs, gpuPairLink{GPUA: a, GPUB: bGPU, NVLinks: nv})
|
||
}
|
||
}
|
||
sort.Slice(pairs, func(i, j int) bool {
|
||
if pairs[i].GPUA != pairs[j].GPUA {
|
||
return pairs[i].GPUA < pairs[j].GPUA
|
||
}
|
||
return pairs[i].GPUB < pairs[j].GPUB
|
||
})
|
||
return pairs
|
||
}
|
||
|
||
// readTopoTechDump reads a file previously captured into the persistent
|
||
// techdump directory by platform.System.CaptureTechnicalDump (run once per
|
||
// audit cycle), rather than shelling out to nvidia-smi from the HTTP request
|
||
// handler — a live call here would block page rendering on a wedged driver,
|
||
// exactly the failure mode this tool exists to diagnose.
|
||
func readTopoTechDump(exportDir, name string) (string, error) {
|
||
out, err := os.ReadFile(filepath.Join(exportDir, "techdump", name))
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
return string(out), nil
|
||
}
|
||
|
||
func readGPUTopologyMatrix(exportDir string) (string, error) {
|
||
return readTopoTechDump(exportDir, "nvidia-smi-topo.txt")
|
||
}
|
||
|
||
// readNVIDIAIndexByBDF parses the persisted nvidia-smi-query.csv techdump
|
||
// (index,pci.bus_id,...) to map PCI bus address (matching
|
||
// HardwarePCIeDevice.Slot) to the GPU index nvidia-smi/topo -m reports, so
|
||
// GPU-GPU edges (keyed by index) can be anchored to the correct box (keyed
|
||
// by BDF) in the diagram.
|
||
func readNVIDIAIndexByBDF(exportDir string) (map[string]int, error) {
|
||
raw, err := readTopoTechDump(exportDir, "nvidia-smi-query.csv")
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
result := map[string]int{}
|
||
for _, line := range strings.Split(raw, "\n") {
|
||
line = strings.TrimSpace(line)
|
||
if line == "" {
|
||
continue
|
||
}
|
||
parts := strings.Split(line, ",")
|
||
if len(parts) < 2 {
|
||
continue
|
||
}
|
||
idx, err := strconv.Atoi(strings.TrimSpace(parts[0]))
|
||
if err != nil {
|
||
continue
|
||
}
|
||
bdf := normalizeTopoBDF(strings.TrimSpace(parts[1]))
|
||
if bdf == "" {
|
||
continue
|
||
}
|
||
result[bdf] = idx
|
||
}
|
||
return result, nil
|
||
}
|
||
|
||
// normalizeTopoBDF normalizes a PCI bus address to "dddd:bb:dd.f" form so
|
||
// nvidia-smi's "pci.bus_id" output can be matched against
|
||
// HardwarePCIeDevice.Slot regardless of minor formatting differences
|
||
// (case, leading domain padding).
|
||
func normalizeTopoBDF(bdf string) string {
|
||
bdf = strings.ToLower(strings.TrimSpace(bdf))
|
||
if bdf == "" {
|
||
return ""
|
||
}
|
||
parts := strings.Split(bdf, ":")
|
||
if len(parts) == 3 {
|
||
domain := parts[0]
|
||
if len(domain) > 4 {
|
||
domain = domain[len(domain)-4:]
|
||
}
|
||
return domain + ":" + parts[1] + ":" + parts[2]
|
||
}
|
||
return bdf
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Card status aggregation
|
||
//
|
||
// Every card on this page — whether it represents one component (CPU 1) or a
|
||
// group of identical ones (GPU ×4) — is colored as a whole by its worst
|
||
// observed status, with a plain-text summary as the card's last line
|
||
// (e.g. "4 OK" or "3 OK, 1 Warning"). There is no separate status chip: a
|
||
// chip needs its own fill, and the SVG boxes previously colored that chip
|
||
// via CSS classes written for HTML (.badge-ok sets `background`/`color`,
|
||
// which do nothing on an SVG <rect>/<text> — only `fill` does), so every
|
||
// chip rendered with the SVG default fill of solid black. Coloring the card
|
||
// itself uses real `fill:var(--ok-bg)` etc. declarations, which sidesteps
|
||
// that class entirely.
|
||
// ---------------------------------------------------------------------------
|
||
|
||
// classifyTopoSeverity converts a component's Status pointer to a severity
|
||
// rank (0=unknown, 1=OK, 2=Warning, 3=Critical), treating nil/unrecognized
|
||
// the same as "Unknown" — matches topoStatusBadgeClass's classification.
|
||
func classifyTopoSeverity(status *string) int {
|
||
if status == nil {
|
||
return 0
|
||
}
|
||
switch strings.ToUpper(strings.TrimSpace(*status)) {
|
||
case "OK":
|
||
return 1
|
||
case "WARNING", "WARN", "PARTIAL":
|
||
return 2
|
||
case "CRITICAL", "FAIL", "FAILED", "ERROR":
|
||
return 3
|
||
default:
|
||
return 0
|
||
}
|
||
}
|
||
|
||
// topoSeverityColors returns the (fill, stroke, text) CSS var() triple a
|
||
// whole card is painted with for a given worst-observed severity.
|
||
func topoSeverityColors(sev int) (fill, stroke, text string) {
|
||
switch sev {
|
||
case 3:
|
||
return "var(--crit-bg)", "var(--crit-border)", "var(--crit-fg)"
|
||
case 2:
|
||
return "var(--warn-bg)", "#c9ba9b", "var(--warn-fg)"
|
||
case 1:
|
||
return "var(--ok-bg)", "#a3c293", "var(--ok-fg)"
|
||
default:
|
||
return "var(--surface-2)", "var(--border)", "var(--muted)"
|
||
}
|
||
}
|
||
|
||
// topoStatusTally counts how many components in a group fall into each
|
||
// severity bucket, so a group card can report "3 OK, 1 Warning" rather than
|
||
// collapsing to a single worst-of value and losing the rest.
|
||
type topoStatusTally struct {
|
||
unknown, ok, warn, crit int
|
||
}
|
||
|
||
func (t *topoStatusTally) add(sev int) {
|
||
switch sev {
|
||
case 3:
|
||
t.crit++
|
||
case 2:
|
||
t.warn++
|
||
case 1:
|
||
t.ok++
|
||
default:
|
||
t.unknown++
|
||
}
|
||
}
|
||
|
||
func (t topoStatusTally) total() int { return t.unknown + t.ok + t.warn + t.crit }
|
||
|
||
func (t topoStatusTally) worst() int {
|
||
switch {
|
||
case t.crit > 0:
|
||
return 3
|
||
case t.warn > 0:
|
||
return 2
|
||
case t.ok > 0:
|
||
return 1
|
||
default:
|
||
return 0
|
||
}
|
||
}
|
||
|
||
// line renders the card's last-line status summary.
|
||
func (t topoStatusTally) line() string {
|
||
if t.total() == 0 {
|
||
return "No data"
|
||
}
|
||
if t.total() == 1 {
|
||
switch {
|
||
case t.crit > 0:
|
||
return "Critical"
|
||
case t.warn > 0:
|
||
return "Warning"
|
||
case t.ok > 0:
|
||
return "OK"
|
||
default:
|
||
return "Unknown"
|
||
}
|
||
}
|
||
var parts []string
|
||
if t.crit > 0 {
|
||
parts = append(parts, fmt.Sprintf("%d Critical", t.crit))
|
||
}
|
||
if t.warn > 0 {
|
||
parts = append(parts, fmt.Sprintf("%d Warning", t.warn))
|
||
}
|
||
if t.ok > 0 {
|
||
parts = append(parts, fmt.Sprintf("%d OK", t.ok))
|
||
}
|
||
if t.unknown > 0 {
|
||
parts = append(parts, fmt.Sprintf("%d Unknown", t.unknown))
|
||
}
|
||
return strings.Join(parts, ", ")
|
||
}
|
||
|
||
// topoCardInfo is the shared visual content for one card, rendered either as
|
||
// an absolutely-positioned SVG box (main diagram) or an HTML flex item
|
||
// (Memory/Power Supplies rows) by the two writers below.
|
||
type topoCardInfo struct {
|
||
label string // e.g. "CPU 1", "GPU", "Power Supplies"
|
||
sublabel string // representative model/description, "" to omit
|
||
count int // components represented by this card; >1 draws a stack
|
||
statusLine string // last line of card text, e.g. "4 OK, 1 Warning"
|
||
fillVar string
|
||
strokeVar string
|
||
textVar string
|
||
detailType string // "" = not clickable
|
||
}
|
||
|
||
// topoStackLayers returns how many faint backing cards to draw behind the
|
||
// front card to read as "a stack of N", capped at 2 — enough to signal
|
||
// "more than one" without the deck becoming its own visual clutter.
|
||
func topoStackLayers(count int) int {
|
||
if count <= 1 {
|
||
return 0
|
||
}
|
||
if count-1 > 2 {
|
||
return 2
|
||
}
|
||
return count - 1
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Main topology diagram
|
||
// ---------------------------------------------------------------------------
|
||
|
||
const (
|
||
topoColWidth = 220
|
||
topoBoxWidth = 190
|
||
topoBoxHeight = 70
|
||
topoDeviceGap = 14
|
||
topoTopMargin = 30
|
||
topoStackStep = 4 // px offset per backing layer in the card-stack effect
|
||
)
|
||
|
||
type topoBox struct {
|
||
x, y, w, h int
|
||
topoCardInfo
|
||
}
|
||
|
||
func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string {
|
||
socketIdx := buildSocketIndex(hw.CPUs)
|
||
numCols := len(hw.CPUs)
|
||
if numCols == 0 {
|
||
numCols = 1
|
||
}
|
||
unknownCol := numCols // extra trailing column for unmatched devices
|
||
|
||
// Group PCIe devices (GPU/NIC/RAID only — matches the mockup's node
|
||
// types) into columns by NUMA node, falling back to the "unknown" bucket.
|
||
type placedDevice struct {
|
||
dev schema.HardwarePCIeDevice
|
||
kind string // "gpu", "nic", "raid"
|
||
col int
|
||
bdf string
|
||
}
|
||
var placed []placedDevice
|
||
for _, dev := range hw.PCIeDevices {
|
||
var kind string
|
||
switch {
|
||
case dev.DeviceClass != nil && isGPUDeviceClass(*dev.DeviceClass):
|
||
kind = "gpu"
|
||
case isNICDeviceClassDev(dev):
|
||
kind = "nic"
|
||
case dev.DeviceClass != nil && isRAIDControllerClass(*dev.DeviceClass):
|
||
kind = "raid"
|
||
default:
|
||
continue
|
||
}
|
||
col := unknownCol
|
||
if dev.NUMANode != nil {
|
||
if ci, ok := socketIdx[*dev.NUMANode]; ok {
|
||
col = ci
|
||
}
|
||
}
|
||
bdf := ""
|
||
if dev.Slot != nil {
|
||
bdf = normalizeTopoBDF(*dev.Slot)
|
||
} else if dev.BDF != nil {
|
||
bdf = normalizeTopoBDF(*dev.BDF)
|
||
}
|
||
placed = append(placed, placedDevice{dev: dev, kind: kind, col: col, bdf: bdf})
|
||
}
|
||
hasUnknownCol := false
|
||
for _, p := range placed {
|
||
if p.col == unknownCol {
|
||
hasUnknownCol = true
|
||
break
|
||
}
|
||
}
|
||
totalCols := numCols
|
||
if hasUnknownCol {
|
||
totalCols++
|
||
}
|
||
|
||
// GPU index<->BDF map + pairwise NVLink adjacency, read from the
|
||
// persisted techdump captured during the last audit cycle, best-effort:
|
||
// if the dump is missing (older audit, no NVIDIA GPUs), this is simply
|
||
// skipped. Used only to detect the cross-NUMA-bonded-pair anomaly below;
|
||
// the pairwise links themselves are drawn in the separate NVLink
|
||
// Topology card, since grouping same-kind/same-column devices into one
|
||
// stacked card here leaves no single per-GPU anchor point to draw a
|
||
// pairwise connector to or from.
|
||
bdfToIndex, _ := readNVIDIAIndexByBDF(exportDir)
|
||
var pairs []gpuPairLink
|
||
if topoMatrix, err := readGPUTopologyMatrix(exportDir); err == nil {
|
||
pairs = parseGPUPairAdjacency(topoMatrix)
|
||
}
|
||
gpuNUMAByIndex := map[int]*int{}
|
||
gpuBDFByIndex := map[int]string{}
|
||
for _, p := range placed {
|
||
if p.kind != "gpu" || p.bdf == "" {
|
||
continue
|
||
}
|
||
if idx, ok := bdfToIndex[p.bdf]; ok {
|
||
gpuNUMAByIndex[idx] = p.dev.NUMANode
|
||
gpuBDFByIndex[idx] = p.bdf
|
||
}
|
||
}
|
||
// A bonded pair spanning two different NUMA nodes is treated as an
|
||
// anomaly (not a neutral fact) per project decision: a bonded pair is
|
||
// expected to sit on one NUMA node, so a cross-NUMA bond escalates both
|
||
// GPUs' effective severity to at least Warning, regardless of their own
|
||
// reported SAT status.
|
||
crossNUMAWarnBDF := map[string]bool{}
|
||
for _, pair := range pairs {
|
||
numaA, okA := gpuNUMAByIndex[pair.GPUA]
|
||
numaB, okB := gpuNUMAByIndex[pair.GPUB]
|
||
if !okA || !okB || numaA == nil || numaB == nil || *numaA == *numaB {
|
||
continue
|
||
}
|
||
crossNUMAWarnBDF[gpuBDFByIndex[pair.GPUA]] = true
|
||
crossNUMAWarnBDF[gpuBDFByIndex[pair.GPUB]] = true
|
||
}
|
||
|
||
kindOrder := []string{"gpu", "nic", "raid"}
|
||
kindLabel := map[string]string{"gpu": "GPU", "nic": "NIC", "raid": "RAID"}
|
||
|
||
var boxes []topoBox
|
||
var pcieEdges []struct {
|
||
x1, y1, x2, y2 int
|
||
color string
|
||
}
|
||
|
||
for col := 0; col < totalCols; col++ {
|
||
colX := (col+1)*24 + col*topoColWidth
|
||
if col < len(hw.CPUs) {
|
||
cpu := hw.CPUs[col]
|
||
model := ""
|
||
if cpu.Model != nil {
|
||
model = *cpu.Model
|
||
}
|
||
socket := col
|
||
if cpu.Socket != nil {
|
||
socket = *cpu.Socket
|
||
}
|
||
var tally topoStatusTally
|
||
tally.add(classifyTopoSeverity(cpu.Status))
|
||
fill, stroke, text := topoSeverityColors(tally.worst())
|
||
boxes = append(boxes, topoBox{
|
||
x: colX, y: topoTopMargin, w: topoBoxWidth, h: topoBoxHeight,
|
||
topoCardInfo: topoCardInfo{
|
||
label: fmt.Sprintf("CPU %d", socket), sublabel: model, count: 1,
|
||
statusLine: tally.line(),
|
||
fillVar: fill, strokeVar: stroke, textVar: text,
|
||
detailType: "cpu",
|
||
},
|
||
})
|
||
}
|
||
|
||
y := topoTopMargin + topoBoxHeight + topoDeviceGap*2
|
||
for _, kind := range kindOrder {
|
||
var group []placedDevice
|
||
for _, p := range placed {
|
||
if p.col == col && p.kind == kind {
|
||
group = append(group, p)
|
||
}
|
||
}
|
||
if len(group) == 0 {
|
||
continue
|
||
}
|
||
|
||
var tally topoStatusTally
|
||
model := ""
|
||
edgeColor := "var(--ok-fg)"
|
||
for i, p := range group {
|
||
sev := classifyTopoSeverity(p.dev.Status)
|
||
if kind == "gpu" && crossNUMAWarnBDF[p.bdf] && sev < 2 {
|
||
sev = 2
|
||
}
|
||
tally.add(sev)
|
||
if i == 0 && p.dev.Model != nil {
|
||
model = *p.dev.Model
|
||
}
|
||
if topoEdgeColorVar(p.dev) == "var(--warn-fg)" {
|
||
edgeColor = "var(--warn-fg)"
|
||
}
|
||
}
|
||
fill, stroke, text := topoSeverityColors(tally.worst())
|
||
stackLayers := topoStackLayers(len(group))
|
||
boxes = append(boxes, topoBox{
|
||
x: colX, y: y, w: topoBoxWidth, h: topoBoxHeight,
|
||
topoCardInfo: topoCardInfo{
|
||
label: kindLabel[kind], sublabel: model, count: len(group),
|
||
statusLine: tally.line(),
|
||
fillVar: fill, strokeVar: stroke, textVar: text,
|
||
detailType: kind,
|
||
},
|
||
})
|
||
|
||
if col < len(hw.CPUs) {
|
||
pcieEdges = append(pcieEdges, struct {
|
||
x1, y1, x2, y2 int
|
||
color string
|
||
}{
|
||
x1: colX + topoBoxWidth/2, y1: topoTopMargin + topoBoxHeight,
|
||
x2: colX + topoBoxWidth/2, y2: y,
|
||
color: edgeColor,
|
||
})
|
||
}
|
||
|
||
y += topoBoxHeight + topoDeviceGap + stackLayers*topoStackStep
|
||
}
|
||
}
|
||
|
||
maxDeviceY := topoTopMargin + topoBoxHeight + topoDeviceGap*2
|
||
for _, box := range boxes {
|
||
bottom := box.y + box.h + topoStackLayers(box.count)*topoStackStep
|
||
if bottom > maxDeviceY {
|
||
maxDeviceY = bottom
|
||
}
|
||
}
|
||
|
||
svgHeight := maxDeviceY + 24
|
||
svgWidth := totalCols*topoColWidth + 48
|
||
|
||
var b strings.Builder
|
||
// Wrapped in its own horizontally-scrolling container (matching the
|
||
// overflow-x:auto convention used for wide tables elsewhere in webui)
|
||
// rather than max-width:100% — squashing a node/edge diagram to fit a
|
||
// narrow viewport makes labels and badges illegible, whereas scrolling
|
||
// keeps the diagram readable at its natural size on any screen width.
|
||
b.WriteString(`<div style="overflow-x:auto">`)
|
||
fmt.Fprintf(&b, `<svg width="%d" height="%d" viewBox="0 0 %d %d">`+"\n", svgWidth, svgHeight, svgWidth, svgHeight)
|
||
for _, e := range pcieEdges {
|
||
fmt.Fprintf(&b, `<line x1="%d" y1="%d" x2="%d" y2="%d" style="stroke:%s;stroke-width:2"/>`+"\n", e.x1, e.y1, e.x2, e.y2, e.color)
|
||
}
|
||
for _, box := range boxes {
|
||
writeTopoBoxSVG(&b, box)
|
||
}
|
||
b.WriteString(`</svg></div>`)
|
||
|
||
// Memory, Firmware (BMC/BIOS/...) and PSUs have no PCIe/CPU affinity to
|
||
// anchor them to a column, and there can be an arbitrary number of any
|
||
// of them — so unlike the diagram above, they're plain flex-wrap HTML
|
||
// below the SVG rather than absolutely-positioned SVG boxes. A
|
||
// fixed-size SVG canvas has no way to wrap overflow onto a new row,
|
||
// which is exactly what caused these to pile up and overlap once a
|
||
// board had more PSUs/firmware records than fit in one fixed-width row.
|
||
if len(hw.Memory) > 0 {
|
||
var tally topoStatusTally
|
||
for _, m := range hw.Memory {
|
||
tally.add(classifyTopoSeverity(m.Status))
|
||
}
|
||
fill, stroke, text := topoSeverityColors(tally.worst())
|
||
sizeGB := 0
|
||
for _, m := range hw.Memory {
|
||
if m.SizeMB != nil {
|
||
sizeGB += *m.SizeMB / 1024
|
||
}
|
||
}
|
||
sublabel := ""
|
||
if sizeGB > 0 {
|
||
sublabel = fmt.Sprintf("%d GB total", sizeGB)
|
||
}
|
||
b.WriteString(renderTopoFlexRow("Memory", []topoCardInfo{{
|
||
label: "Memory", sublabel: sublabel, count: len(hw.Memory),
|
||
statusLine: tally.line(),
|
||
fillVar: fill, strokeVar: stroke, textVar: text,
|
||
detailType: "memory",
|
||
}}))
|
||
}
|
||
|
||
var firmwareItems []topoCardInfo
|
||
for _, rec := range hw.Firmware {
|
||
// Firmware records carry no per-item status in the schema (they are
|
||
// identity, not health, facts), so each stays a neutral, uncolored
|
||
// card rather than forcing a fake "Unknown" status line.
|
||
fillVar, strokeVar, textVar := topoSeverityColors(0)
|
||
firmwareItems = append(firmwareItems, topoCardInfo{
|
||
label: rec.DeviceName, sublabel: "fw " + rec.Version, count: 1,
|
||
fillVar: fillVar, strokeVar: strokeVar, textVar: textVar,
|
||
})
|
||
}
|
||
b.WriteString(renderTopoFlexRow("Firmware", firmwareItems))
|
||
|
||
if len(hw.PowerSupplies) > 0 {
|
||
var tally topoStatusTally
|
||
watt := 0
|
||
for _, psu := range hw.PowerSupplies {
|
||
tally.add(classifyTopoSeverity(psu.Status))
|
||
if psu.WattageW != nil {
|
||
watt = *psu.WattageW
|
||
}
|
||
}
|
||
fill, stroke, text := topoSeverityColors(tally.worst())
|
||
sublabel := ""
|
||
if watt > 0 {
|
||
sublabel = fmt.Sprintf("%dW each", watt)
|
||
}
|
||
b.WriteString(renderTopoFlexRow("Power Supplies", []topoCardInfo{{
|
||
label: "Power Supplies", sublabel: sublabel, count: len(hw.PowerSupplies),
|
||
statusLine: tally.line(),
|
||
fillVar: fill, strokeVar: stroke, textVar: text,
|
||
detailType: "psu",
|
||
}}))
|
||
}
|
||
|
||
return topoCard("Topology", b.String())
|
||
}
|
||
|
||
// renderTopoFlexRow renders a labeled, wrapping row of component cards.
|
||
// Returns "" if items is empty (e.g. no PSU data in this audit).
|
||
func renderTopoFlexRow(title string, items []topoCardInfo) string {
|
||
if len(items) == 0 {
|
||
return ""
|
||
}
|
||
var b strings.Builder
|
||
fmt.Fprintf(&b, `<div style="font-size:11px;color:var(--muted);text-transform:uppercase;letter-spacing:.05em;margin:16px 0 6px">%s</div>`,
|
||
html.EscapeString(title))
|
||
b.WriteString(`<div style="display:flex;flex-wrap:wrap;gap:10px">`)
|
||
for _, item := range items {
|
||
onclick := ""
|
||
cursor := "default"
|
||
if item.detailType != "" {
|
||
onclick = fmt.Sprintf(` onclick="openComponentDetail('%s')"`, item.detailType)
|
||
cursor = "pointer"
|
||
}
|
||
stackLayers := topoStackLayers(item.count)
|
||
// Extra right/bottom padding on the wrapper reserves room for the
|
||
// backing layers of the stack effect so they aren't clipped by the
|
||
// flex container.
|
||
fmt.Fprintf(&b, `<div style="position:relative;padding-right:%dpx;padding-bottom:%dpx">`,
|
||
stackLayers*topoStackStep, stackLayers*topoStackStep)
|
||
for i := stackLayers; i >= 1; i-- {
|
||
off := i * topoStackStep
|
||
fmt.Fprintf(&b, `<div style="position:absolute;top:%dpx;left:%dpx;right:0;bottom:0;border-radius:6px;background:%s;border:1px solid %s;opacity:.55"></div>`,
|
||
off, off, item.fillVar, item.strokeVar)
|
||
}
|
||
fmt.Fprintf(&b, `<div%s style="position:relative;cursor:%s;min-width:160px;padding:10px 12px;border-radius:6px;background:%s;border:1px solid %s;color:%s">`,
|
||
onclick, cursor, item.fillVar, item.strokeVar, item.textVar)
|
||
label := item.label
|
||
if item.count > 1 {
|
||
label = fmt.Sprintf("%s ×%d", item.label, item.count)
|
||
}
|
||
fmt.Fprintf(&b, `<div style="font-size:13px;font-weight:700">%s</div>`, html.EscapeString(label))
|
||
if item.sublabel != "" {
|
||
fmt.Fprintf(&b, `<div style="font-size:11px;opacity:.85">%s</div>`, html.EscapeString(item.sublabel))
|
||
}
|
||
if item.statusLine != "" {
|
||
fmt.Fprintf(&b, `<div style="font-size:11px;font-weight:600;margin-top:4px">%s</div>`, html.EscapeString(item.statusLine))
|
||
}
|
||
b.WriteString(`</div></div>`)
|
||
}
|
||
b.WriteString(`</div>`)
|
||
return b.String()
|
||
}
|
||
|
||
func writeTopoBoxSVG(b *strings.Builder, box topoBox) {
|
||
onclick := ""
|
||
cursor := "default"
|
||
if box.detailType != "" {
|
||
onclick = fmt.Sprintf(` onclick="openComponentDetail('%s')"`, box.detailType)
|
||
cursor = "pointer"
|
||
}
|
||
fmt.Fprintf(b, `<g%s style="cursor:%s">`, onclick, cursor)
|
||
|
||
// Stack-of-cards effect: faint offset rects behind the front card when
|
||
// this box represents more than one physical component (e.g. 4 GPUs in
|
||
// one NUMA column), so a group reads as "a deck of N" rather than a
|
||
// single item. Peeks toward the bottom-right, into space already
|
||
// reserved between this box and the next one in the column.
|
||
for i := topoStackLayers(box.count); i >= 1; i-- {
|
||
off := i * topoStackStep
|
||
fmt.Fprintf(b, `<rect x="%d" y="%d" width="%d" height="%d" rx="6" ry="6" style="fill:%s;stroke:%s;opacity:.55"/>`+"\n",
|
||
box.x+off, box.y+off, box.w, box.h, box.fillVar, box.strokeVar)
|
||
}
|
||
|
||
fmt.Fprintf(b, `<rect x="%d" y="%d" width="%d" height="%d" rx="6" ry="6" style="fill:%s;stroke:%s"/>`+"\n",
|
||
box.x, box.y, box.w, box.h, box.fillVar, box.strokeVar)
|
||
|
||
label := box.label
|
||
if box.count > 1 {
|
||
label = fmt.Sprintf("%s ×%d", box.label, box.count)
|
||
}
|
||
fmt.Fprintf(b, `<text x="%d" y="%d" style="fill:%s;font-size:13px;font-weight:700">%s</text>`+"\n",
|
||
box.x+10, box.y+20, box.textVar, html.EscapeString(label))
|
||
if box.sublabel != "" {
|
||
fmt.Fprintf(b, `<text x="%d" y="%d" style="fill:%s;font-size:11px;opacity:.85">%s</text>`+"\n",
|
||
box.x+10, box.y+36, box.textVar, html.EscapeString(truncateTopoLabel(box.sublabel, 26)))
|
||
}
|
||
if box.statusLine != "" {
|
||
fmt.Fprintf(b, `<text x="%d" y="%d" style="fill:%s;font-size:10px;font-weight:600">%s</text>`+"\n",
|
||
box.x+10, box.y+box.h-10, box.textVar, html.EscapeString(box.statusLine))
|
||
}
|
||
b.WriteString(`</g>` + "\n")
|
||
}
|
||
|
||
func truncateTopoLabel(s string, max int) string {
|
||
if len(s) <= max {
|
||
return s
|
||
}
|
||
if max <= 1 {
|
||
return s[:max]
|
||
}
|
||
return s[:max-1] + "…"
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Separate NVLink topology card (read from techdump, not written to any
|
||
// ingest contract)
|
||
// ---------------------------------------------------------------------------
|
||
|
||
type topoNVLinkPort struct {
|
||
Index int
|
||
Active bool
|
||
SpeedGBs *float64
|
||
ReplayErrors int64
|
||
RecoveryErrors int64
|
||
CRCErrors int64
|
||
}
|
||
|
||
var (
|
||
topoNVLinkGPUHeaderRe = regexp.MustCompile(`^GPU (\d+):`)
|
||
topoNVLinkSpeedLineRe = regexp.MustCompile(`^Link (\d+):\s*([\d.]+)\s*GB/s`)
|
||
topoNVLinkInactiveRe = regexp.MustCompile(`^Link (\d+):\s*<inactive>`)
|
||
topoNVLinkErrorLineRe = regexp.MustCompile(`^Link (\d+):\s*(Replay|Recovery|CRC) Errors:\s*(\d+)`)
|
||
)
|
||
|
||
func readTopoNVLinkStatus(exportDir string) (map[int][]topoNVLinkPort, error) {
|
||
raw, err := readTopoTechDump(exportDir, "nvidia-smi-nvlink-status.txt")
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
return parseTopoNVLinkStatus(raw), nil
|
||
}
|
||
|
||
func parseTopoNVLinkStatus(raw string) map[int][]topoNVLinkPort {
|
||
result := map[int][]topoNVLinkPort{}
|
||
currentGPU := -1
|
||
for _, line := range strings.Split(raw, "\n") {
|
||
trimmed := strings.TrimSpace(line)
|
||
if m := topoNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
|
||
currentGPU, _ = strconv.Atoi(m[1])
|
||
continue
|
||
}
|
||
if currentGPU < 0 {
|
||
continue
|
||
}
|
||
if m := topoNVLinkInactiveRe.FindStringSubmatch(trimmed); m != nil {
|
||
idx, _ := strconv.Atoi(m[1])
|
||
result[currentGPU] = append(result[currentGPU], topoNVLinkPort{Index: idx, Active: false})
|
||
continue
|
||
}
|
||
if m := topoNVLinkSpeedLineRe.FindStringSubmatch(trimmed); m != nil {
|
||
idx, _ := strconv.Atoi(m[1])
|
||
port := topoNVLinkPort{Index: idx, Active: true}
|
||
if speed, err := strconv.ParseFloat(m[2], 64); err == nil {
|
||
port.SpeedGBs = &speed
|
||
}
|
||
result[currentGPU] = append(result[currentGPU], port)
|
||
}
|
||
}
|
||
return result
|
||
}
|
||
|
||
func readTopoNVLinkErrors(exportDir string) (map[int]map[int][3]int64, error) {
|
||
raw, err := readTopoTechDump(exportDir, "nvidia-smi-nvlink-errors.txt")
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
return parseTopoNVLinkErrors(raw), nil
|
||
}
|
||
|
||
// parseTopoNVLinkErrors returns, per GPU then link index, [replay, recovery, crc].
|
||
func parseTopoNVLinkErrors(raw string) map[int]map[int][3]int64 {
|
||
result := map[int]map[int][3]int64{}
|
||
currentGPU := -1
|
||
for _, line := range strings.Split(raw, "\n") {
|
||
trimmed := strings.TrimSpace(line)
|
||
if m := topoNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
|
||
currentGPU, _ = strconv.Atoi(m[1])
|
||
continue
|
||
}
|
||
if currentGPU < 0 {
|
||
continue
|
||
}
|
||
m := topoNVLinkErrorLineRe.FindStringSubmatch(trimmed)
|
||
if m == nil {
|
||
continue
|
||
}
|
||
linkIdx, _ := strconv.Atoi(m[1])
|
||
count, _ := strconv.ParseInt(m[3], 10, 64)
|
||
if result[currentGPU] == nil {
|
||
result[currentGPU] = map[int][3]int64{}
|
||
}
|
||
c := result[currentGPU][linkIdx]
|
||
switch m[2] {
|
||
case "Replay":
|
||
c[0] = count
|
||
case "Recovery":
|
||
c[1] = count
|
||
case "CRC":
|
||
c[2] = count
|
||
}
|
||
result[currentGPU][linkIdx] = c
|
||
}
|
||
return result
|
||
}
|
||
|
||
// renderTopoNVLinkCard renders the separate NVLink topology card. Returns ""
|
||
// if there are fewer than 2 NVIDIA GPUs, or the nvidia-smi nvlink techdump
|
||
// wasn't captured (older audit, or nvidia-smi unavailable on that run).
|
||
func renderTopoNVLinkCard(hw schema.HardwareSnapshot, exportDir string) string {
|
||
gpuCount := 0
|
||
for _, dev := range hw.PCIeDevices {
|
||
if dev.DeviceClass != nil && isGPUDeviceClass(*dev.DeviceClass) {
|
||
gpuCount++
|
||
}
|
||
}
|
||
if gpuCount < 2 {
|
||
return ""
|
||
}
|
||
|
||
status, err := readTopoNVLinkStatus(exportDir)
|
||
if err != nil || len(status) == 0 {
|
||
return topoCard("NVLink Topology", `<span class="badge badge-unknown">nvidia-smi nvlink data unavailable</span>`)
|
||
}
|
||
errors, _ := readTopoNVLinkErrors(exportDir)
|
||
|
||
topoMatrix, _ := readGPUTopologyMatrix(exportDir)
|
||
pairs := parseGPUPairAdjacency(topoMatrix)
|
||
|
||
var bodyB strings.Builder
|
||
if gpuCount <= 4 && len(pairs) > 0 {
|
||
// Small GPU count: per-pair box+line with per-link detail.
|
||
for _, pair := range pairs {
|
||
activeCount, total, hasError := 0, 0, false
|
||
for _, port := range status[pair.GPUA] {
|
||
total++
|
||
if port.Active {
|
||
activeCount++
|
||
}
|
||
}
|
||
for _, counters := range errors[pair.GPUA] {
|
||
if counters[0] != 0 || counters[1] != 0 || counters[2] != 0 {
|
||
hasError = true
|
||
}
|
||
}
|
||
color := "var(--ok-fg)"
|
||
switch {
|
||
case hasError:
|
||
color = "var(--crit-fg)"
|
||
case total > 0 && activeCount < total:
|
||
color = "var(--warn-fg)"
|
||
}
|
||
fmt.Fprintf(&bodyB, `<div style="display:flex;align-items:center;gap:12px;margin-bottom:10px">`+
|
||
`<div style="padding:8px 12px;border:1px solid var(--border);border-radius:6px">GPU %d</div>`+
|
||
`<div style="flex:1;height:2px;background:%s"></div>`+
|
||
`<div style="padding:8px 12px;border:1px solid var(--border);border-radius:6px">GPU %d</div>`+
|
||
`<div style="font-size:12px;color:var(--muted)">%d/%d links active%s</div>`+
|
||
`</div>`,
|
||
pair.GPUA, color, pair.GPUB, activeCount, total, errNoteSuffix(hasError))
|
||
}
|
||
} else if len(pairs) > 0 {
|
||
// Larger GPU counts (NVSwitch fabric): aggregate pair table instead of
|
||
// an unreadable all-to-all graph.
|
||
bodyB.WriteString(`<table><thead><tr><th>GPU A</th><th>GPU B</th><th>NVLinks</th></tr></thead><tbody>`)
|
||
for _, pair := range pairs {
|
||
fmt.Fprintf(&bodyB, `<tr><td>GPU %d</td><td>GPU %d</td><td>%d</td></tr>`, pair.GPUA, pair.GPUB, pair.NVLinks)
|
||
}
|
||
bodyB.WriteString(`</tbody></table>`)
|
||
} else {
|
||
bodyB.WriteString(`<span class="badge badge-unknown">No NVLink-bonded GPU pairs found</span>`)
|
||
}
|
||
|
||
return topoCard("NVLink Topology", bodyB.String())
|
||
}
|
||
|
||
func errNoteSuffix(hasError bool) string {
|
||
if hasError {
|
||
return " — errors detected"
|
||
}
|
||
return ""
|
||
}
|