- nvidia-smi underlines the topo -m header with ANSI CSI codes even when writing to a file; both NVLink matrix parsers failed to find the header and /topo showed "No NVLink-bonded GPU pairs found" on bonded systems. - raid-lsi-create-mirror failed with "resources already in use" (exit 11) on JBOD drives; the task now reads drive states and auto-converts JBOD/UBad (set good force), releases hotspares, refuses Frgn/Onln with actionable messages, and dumps preservedcache info when add vd fails. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
818 lines
25 KiB
Go
818 lines
25 KiB
Go
package webui
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"html"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"bee/audit/internal/schema"
|
|
)
|
|
|
|
// renderTopo renders the /topo page: a read-only visualization of the server
|
|
// topology (CPU sockets, NUMA-affine PCIe devices, PSU/BMC) plus a separate
|
|
// NVLink topology card. It is pure visualization: everything it reads either
|
|
// already exists in the audit.json contract, or comes from the persisted
|
|
// techdump captured once per audit cycle (platform.CaptureTechnicalDump) —
|
|
// nothing here shells out to nvidia-smi itself, writes to
|
|
// schema.HardwarePCIeDevice or any other contract type, or talks to
|
|
// Reanimator Core.
|
|
func renderTopo(opts HandlerOptions) string {
|
|
data, err := loadSnapshot(opts.AuditPath)
|
|
if err != nil {
|
|
return topoCard("Topology", `<span class="badge badge-unknown">No audit data</span>`)
|
|
}
|
|
var ingest schema.HardwareIngestRequest
|
|
if err := json.Unmarshal(data, &ingest); err != nil {
|
|
return topoCard("Topology", `<span class="badge badge-err">Parse error</span>`)
|
|
}
|
|
hw := ingest.Hardware
|
|
|
|
var b strings.Builder
|
|
b.WriteString(renderTopoMainDiagram(hw, opts.ExportDir))
|
|
if nv := renderTopoNVLinkCard(hw, opts.ExportDir); nv != "" {
|
|
b.WriteString(nv)
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
func topoCard(title, body string) string {
|
|
return `<div class="card"><div class="card-head">` + html.EscapeString(title) + `</div><div class="card-body">` + body + `</div></div>`
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Classification helpers
|
|
//
|
|
// webui does not import collector (matches the existing isGPUDeviceClass
|
|
// precedent in pages.go, which already locally duplicates collector.isGPUClass
|
|
// instead of importing the package for one classifier).
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// isNICDeviceClassDev mirrors the classification logic in hwDescribeNIC
|
|
// (pages.go), applied to a single device instead of aggregated counts.
|
|
func isNICDeviceClassDev(dev schema.HardwarePCIeDevice) bool {
|
|
if dev.DeviceClass != nil {
|
|
c := strings.ToLower(strings.TrimSpace(*dev.DeviceClass))
|
|
if c == "ethernetcontroller" || c == "networkcontroller" || strings.Contains(c, "fibrechannel") {
|
|
return true
|
|
}
|
|
}
|
|
return len(dev.MacAddresses) > 0
|
|
}
|
|
|
|
// isRAIDControllerClass matches the canonical class strings produced by
|
|
// collector.mapPCIeDeviceClass for RAID/storage HBAs.
|
|
func isRAIDControllerClass(class string) bool {
|
|
switch strings.TrimSpace(class) {
|
|
case "MassStorageController", "StorageController":
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Status / link-speed coloring
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// topoStatusBadgeClass maps a component's Status pointer to a badge class,
|
|
// treating a nil/absent status the same as literal "Unknown" (matches how
|
|
// the rest of the UI already renders missing status, per chipLetterClass/
|
|
// runtimeStatusBadge in pages.go).
|
|
func topoStatusBadgeClass(status *string) (label, cls string) {
|
|
if status == nil {
|
|
return "?", "badge-unknown"
|
|
}
|
|
switch strings.ToUpper(strings.TrimSpace(*status)) {
|
|
case "OK":
|
|
return "OK", "badge-ok"
|
|
case "WARNING", "WARN", "PARTIAL":
|
|
return "WARN", "badge-warn"
|
|
case "CRITICAL", "FAIL", "FAILED", "ERROR":
|
|
return "CRIT", "badge-err"
|
|
default:
|
|
return "?", "badge-unknown"
|
|
}
|
|
}
|
|
|
|
// pcieGenRank ranks a PCIe generation label ("Gen3", "Gen4", ...) for
|
|
// comparison. Mirrors collector.pcieLinkSpeedRank's ordering; duplicated
|
|
// locally rather than exported, per the same "no collector import in webui"
|
|
// convention used for isGPUDeviceClass/isRAIDControllerClass.
|
|
func pcieGenRank(gen string) int {
|
|
gen = strings.ToLower(strings.TrimSpace(gen))
|
|
gen = strings.TrimPrefix(gen, "gen")
|
|
n, err := strconv.Atoi(gen)
|
|
if err != nil {
|
|
return 0
|
|
}
|
|
return n
|
|
}
|
|
|
|
// topoEdgeColorVar computes the CPU->device edge color strictly from
|
|
// link_speed vs max_link_speed — NOT from dev.Status, since Status can also
|
|
// be overwritten by SAT/acceptance-test results on the same PCIe device,
|
|
// which would conflate "link is physically degraded" with "this GPU failed
|
|
// its stress test" into the same color.
|
|
func topoEdgeColorVar(dev schema.HardwarePCIeDevice) string {
|
|
if dev.LinkSpeed == nil || dev.MaxLinkSpeed == nil {
|
|
return "var(--muted)"
|
|
}
|
|
if pcieGenRank(*dev.LinkSpeed) < pcieGenRank(*dev.MaxLinkSpeed) {
|
|
return "var(--warn-fg)"
|
|
}
|
|
return "var(--ok-fg)"
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// NUMA node -> CPU socket join (heuristic, no guaranteed hardware mapping)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// buildSocketIndex maps a NUMA node number to the index into cpus for the
|
|
// socket occupying that position in ascending Socket-designation order.
|
|
//
|
|
// Linux NUMA node numbering is always 0-based (node0, node1, ...), but
|
|
// dmidecode's "Socket Designation" is board-defined and frequently 1-based
|
|
// ("CPU1", "CPU2", ...). Mapping NUMA node N to the CPU whose Socket field
|
|
// equals N (as an earlier version of this function did) silently fails on
|
|
// any 1-indexed board: node 0 has no match (dropped into the "unknown"
|
|
// column) and node 1 wrongly maps to the first CPU. Ranking by Socket value
|
|
// instead assumes only that node order follows socket order — true for the
|
|
// common case of N-socket boards — without depending on the numbering base.
|
|
func buildSocketIndex(cpus []schema.HardwareCPU) map[int]int {
|
|
order := make([]int, len(cpus))
|
|
for i := range cpus {
|
|
order[i] = i
|
|
}
|
|
sort.SliceStable(order, func(a, b int) bool {
|
|
ca, cb := cpus[order[a]], cpus[order[b]]
|
|
sa, sb := 0, 0
|
|
if ca.Socket != nil {
|
|
sa = *ca.Socket
|
|
}
|
|
if cb.Socket != nil {
|
|
sb = *cb.Socket
|
|
}
|
|
return sa < sb
|
|
})
|
|
idx := map[int]int{}
|
|
for numaNode, cpuIdx := range order {
|
|
idx[numaNode] = cpuIdx
|
|
}
|
|
return idx
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// GPU pairwise NVLink adjacency (from a live "nvidia-smi topo -m" query)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
type gpuPairLink struct {
|
|
GPUA, GPUB int
|
|
NVLinks int
|
|
}
|
|
|
|
var topoNVRe = regexp.MustCompile(`(?i)^NV(\d+)$`)
|
|
|
|
// nvidia-smi underlines the topo -m header row with ANSI CSI sequences
|
|
// (ESC[4m...ESC[0m) even when stdout is not a TTY, so the captured techdump
|
|
// contains them and "GPU0" is not at the start of the trimmed header line.
|
|
var topoANSIRe = regexp.MustCompile("\x1b\\[[0-9;]*[A-Za-z]")
|
|
|
|
// parseGPUPairAdjacency returns every GPU pair with a nonzero NVLink bond
|
|
// count from a "nvidia-smi topo -m" matrix. Unlike parseNVIDIATopologyMatrix
|
|
// (collector package, aggregate-only: min/all-active/count), this returns
|
|
// who is bonded to whom — required so GPU-GPU edges are drawn for actually
|
|
// bonded pairs, not for adjacent boxes in the layout.
|
|
func parseGPUPairAdjacency(raw string) []gpuPairLink {
|
|
lines := strings.Split(topoANSIRe.ReplaceAllString(raw, ""), "\n")
|
|
headerIdx := -1
|
|
var gpuColIndices []int
|
|
for i, line := range lines {
|
|
trimmed := strings.TrimSpace(line)
|
|
if strings.HasPrefix(trimmed, "GPU0") {
|
|
parts := strings.Fields(trimmed)
|
|
for j, col := range parts {
|
|
if strings.HasPrefix(col, "GPU") {
|
|
gpuColIndices = append(gpuColIndices, j)
|
|
}
|
|
}
|
|
if len(gpuColIndices) >= 2 {
|
|
headerIdx = i
|
|
}
|
|
break
|
|
}
|
|
}
|
|
if headerIdx < 0 {
|
|
return nil
|
|
}
|
|
|
|
colIdxToGPU := make(map[int]int, len(gpuColIndices))
|
|
for gpuIdx, colIdx := range gpuColIndices {
|
|
colIdxToGPU[colIdx] = gpuIdx
|
|
}
|
|
|
|
seen := map[[2]int]bool{}
|
|
var pairs []gpuPairLink
|
|
rowGPU := -1
|
|
for _, line := range lines[headerIdx+1:] {
|
|
trimmed := strings.TrimSpace(line)
|
|
if !strings.HasPrefix(trimmed, "GPU") {
|
|
continue
|
|
}
|
|
cells := strings.Fields(trimmed)
|
|
if len(cells) == 0 {
|
|
continue
|
|
}
|
|
rowLabel := strings.TrimPrefix(cells[0], "GPU")
|
|
n, err := strconv.Atoi(rowLabel)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
rowGPU = n
|
|
for colIdx, colGPU := range colIdxToGPU {
|
|
if colGPU == rowGPU {
|
|
continue
|
|
}
|
|
dataIdx := colIdx + 1
|
|
if dataIdx >= len(cells) {
|
|
continue
|
|
}
|
|
m := topoNVRe.FindStringSubmatch(cells[dataIdx])
|
|
if len(m) != 2 {
|
|
continue
|
|
}
|
|
nv, err := strconv.Atoi(m[1])
|
|
if err != nil || nv <= 0 {
|
|
continue
|
|
}
|
|
a, bGPU := rowGPU, colGPU
|
|
if a > bGPU {
|
|
a, bGPU = bGPU, a
|
|
}
|
|
key := [2]int{a, bGPU}
|
|
if seen[key] {
|
|
continue
|
|
}
|
|
seen[key] = true
|
|
pairs = append(pairs, gpuPairLink{GPUA: a, GPUB: bGPU, NVLinks: nv})
|
|
}
|
|
}
|
|
sort.Slice(pairs, func(i, j int) bool {
|
|
if pairs[i].GPUA != pairs[j].GPUA {
|
|
return pairs[i].GPUA < pairs[j].GPUA
|
|
}
|
|
return pairs[i].GPUB < pairs[j].GPUB
|
|
})
|
|
return pairs
|
|
}
|
|
|
|
// readTopoTechDump reads a file previously captured into the persistent
|
|
// techdump directory by platform.System.CaptureTechnicalDump (run once per
|
|
// audit cycle), rather than shelling out to nvidia-smi from the HTTP request
|
|
// handler — a live call here would block page rendering on a wedged driver,
|
|
// exactly the failure mode this tool exists to diagnose.
|
|
func readTopoTechDump(exportDir, name string) (string, error) {
|
|
out, err := os.ReadFile(filepath.Join(exportDir, "techdump", name))
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return string(out), nil
|
|
}
|
|
|
|
func readGPUTopologyMatrix(exportDir string) (string, error) {
|
|
return readTopoTechDump(exportDir, "nvidia-smi-topo.txt")
|
|
}
|
|
|
|
// readNVIDIAIndexByBDF parses the persisted nvidia-smi-query.csv techdump
|
|
// (index,pci.bus_id,...) to map PCI bus address (matching
|
|
// HardwarePCIeDevice.Slot) to the GPU index nvidia-smi/topo -m reports, so
|
|
// GPU-GPU edges (keyed by index) can be anchored to the correct box (keyed
|
|
// by BDF) in the diagram.
|
|
func readNVIDIAIndexByBDF(exportDir string) (map[string]int, error) {
|
|
raw, err := readTopoTechDump(exportDir, "nvidia-smi-query.csv")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
result := map[string]int{}
|
|
for _, line := range strings.Split(raw, "\n") {
|
|
line = strings.TrimSpace(line)
|
|
if line == "" {
|
|
continue
|
|
}
|
|
parts := strings.Split(line, ",")
|
|
if len(parts) < 2 {
|
|
continue
|
|
}
|
|
idx, err := strconv.Atoi(strings.TrimSpace(parts[0]))
|
|
if err != nil {
|
|
continue
|
|
}
|
|
bdf := normalizeTopoBDF(strings.TrimSpace(parts[1]))
|
|
if bdf == "" {
|
|
continue
|
|
}
|
|
result[bdf] = idx
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
// normalizeTopoBDF normalizes a PCI bus address to "dddd:bb:dd.f" form so
|
|
// nvidia-smi's "pci.bus_id" output can be matched against
|
|
// HardwarePCIeDevice.Slot regardless of minor formatting differences
|
|
// (case, leading domain padding).
|
|
func normalizeTopoBDF(bdf string) string {
|
|
bdf = strings.ToLower(strings.TrimSpace(bdf))
|
|
if bdf == "" {
|
|
return ""
|
|
}
|
|
parts := strings.Split(bdf, ":")
|
|
if len(parts) == 3 {
|
|
domain := parts[0]
|
|
if len(domain) > 4 {
|
|
domain = domain[len(domain)-4:]
|
|
}
|
|
return domain + ":" + parts[1] + ":" + parts[2]
|
|
}
|
|
return bdf
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Main topology diagram
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const (
|
|
topoColWidth = 220
|
|
topoBoxWidth = 190
|
|
topoBoxHeight = 56
|
|
topoDeviceGap = 14
|
|
topoTopMargin = 30
|
|
topoEdgeBand = 60
|
|
topoBottomRowH = 90
|
|
)
|
|
|
|
type topoBox struct {
|
|
x, y, w, h int
|
|
label string
|
|
sublabel string
|
|
badgeText string
|
|
badgeCls string
|
|
detailType string // "" = not clickable
|
|
}
|
|
|
|
func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string {
|
|
socketIdx := buildSocketIndex(hw.CPUs)
|
|
numCols := len(hw.CPUs)
|
|
if numCols == 0 {
|
|
numCols = 1
|
|
}
|
|
unknownCol := numCols // extra trailing column for unmatched devices
|
|
|
|
// Group PCIe devices (GPU/NIC/RAID only — matches the mockup's node
|
|
// types) into columns by NUMA node, falling back to the "unknown" bucket.
|
|
type placedDevice struct {
|
|
dev schema.HardwarePCIeDevice
|
|
kind string // "gpu", "nic", "raid"
|
|
col int
|
|
bdf string
|
|
}
|
|
var placed []placedDevice
|
|
for _, dev := range hw.PCIeDevices {
|
|
var kind string
|
|
switch {
|
|
case dev.DeviceClass != nil && isGPUDeviceClass(*dev.DeviceClass):
|
|
kind = "gpu"
|
|
case isNICDeviceClassDev(dev):
|
|
kind = "nic"
|
|
case dev.DeviceClass != nil && isRAIDControllerClass(*dev.DeviceClass):
|
|
kind = "raid"
|
|
default:
|
|
continue
|
|
}
|
|
col := unknownCol
|
|
if dev.NUMANode != nil {
|
|
if ci, ok := socketIdx[*dev.NUMANode]; ok {
|
|
col = ci
|
|
}
|
|
}
|
|
bdf := ""
|
|
if dev.Slot != nil {
|
|
bdf = normalizeTopoBDF(*dev.Slot)
|
|
} else if dev.BDF != nil {
|
|
bdf = normalizeTopoBDF(*dev.BDF)
|
|
}
|
|
placed = append(placed, placedDevice{dev: dev, kind: kind, col: col, bdf: bdf})
|
|
}
|
|
hasUnknownCol := false
|
|
for _, p := range placed {
|
|
if p.col == unknownCol {
|
|
hasUnknownCol = true
|
|
break
|
|
}
|
|
}
|
|
totalCols := numCols
|
|
if hasUnknownCol {
|
|
totalCols++
|
|
}
|
|
|
|
// GPU index<->BDF map + pairwise NVLink adjacency, read from the
|
|
// persisted techdump captured during the last audit cycle, best-effort:
|
|
// if the dump is missing (older audit, no NVIDIA GPUs), GPU-GPU edges are
|
|
// simply omitted.
|
|
bdfToIndex, _ := readNVIDIAIndexByBDF(exportDir)
|
|
var pairs []gpuPairLink
|
|
if topoMatrix, err := readGPUTopologyMatrix(exportDir); err == nil {
|
|
pairs = parseGPUPairAdjacency(topoMatrix)
|
|
}
|
|
|
|
var boxes []topoBox
|
|
var pcieEdges []struct {
|
|
x1, y1, x2, y2 int
|
|
color string
|
|
}
|
|
gpuBoxCenter := map[int][2]int{} // gpu index -> (x, yBottom)
|
|
gpuBoxIndex := map[int]int{} // gpu index -> index into boxes
|
|
gpuNUMANode := map[int]*int{} // gpu index -> its PCIe device's numa_node
|
|
|
|
for col := 0; col < totalCols; col++ {
|
|
colX := (col+1)*24 + col*topoColWidth
|
|
if col < numCols {
|
|
cpu := hw.CPUs[col]
|
|
model := ""
|
|
if cpu.Model != nil {
|
|
model = *cpu.Model
|
|
}
|
|
socket := col
|
|
if cpu.Socket != nil {
|
|
socket = *cpu.Socket
|
|
}
|
|
label, cls := topoStatusBadgeClass(cpu.Status)
|
|
boxes = append(boxes, topoBox{
|
|
x: colX, y: topoTopMargin, w: topoBoxWidth, h: topoBoxHeight,
|
|
label: fmt.Sprintf("CPU %d", socket), sublabel: model,
|
|
badgeText: label, badgeCls: cls, detailType: "cpu",
|
|
})
|
|
}
|
|
|
|
y := topoTopMargin + topoBoxHeight + topoDeviceGap*2
|
|
for _, p := range placed {
|
|
if p.col != col {
|
|
continue
|
|
}
|
|
label, cls := topoStatusBadgeClass(p.dev.Status)
|
|
model := ""
|
|
if p.dev.Model != nil {
|
|
model = *p.dev.Model
|
|
}
|
|
box := topoBox{
|
|
x: colX, y: y, w: topoBoxWidth, h: topoBoxHeight,
|
|
label: strings.ToUpper(p.kind), sublabel: model,
|
|
badgeText: label, badgeCls: cls, detailType: p.kind,
|
|
}
|
|
boxes = append(boxes, box)
|
|
boxIdx := len(boxes) - 1
|
|
|
|
if col < numCols {
|
|
pcieEdges = append(pcieEdges, struct {
|
|
x1, y1, x2, y2 int
|
|
color string
|
|
}{
|
|
x1: colX + topoBoxWidth/2, y1: topoTopMargin + topoBoxHeight,
|
|
x2: colX + topoBoxWidth/2, y2: y,
|
|
color: topoEdgeColorVar(p.dev),
|
|
})
|
|
}
|
|
|
|
if p.kind == "gpu" && p.bdf != "" {
|
|
if idx, ok := bdfToIndex[p.bdf]; ok {
|
|
gpuBoxCenter[idx] = [2]int{colX + topoBoxWidth/2, y + topoBoxHeight}
|
|
gpuBoxIndex[idx] = boxIdx
|
|
gpuNUMANode[idx] = p.dev.NUMANode
|
|
}
|
|
}
|
|
|
|
y += topoBoxHeight + topoDeviceGap
|
|
}
|
|
}
|
|
|
|
maxDeviceY := topoTopMargin + topoBoxHeight + topoDeviceGap*2
|
|
for _, b := range boxes {
|
|
if b.y+b.h > maxDeviceY {
|
|
maxDeviceY = b.y + b.h
|
|
}
|
|
}
|
|
|
|
// GPU-GPU NVLink edges: drawn as an elbow connector through a dedicated
|
|
// band below the device row, kept strictly separate from the vertical
|
|
// CPU->device PCIe edges above so neither visually obscures the other.
|
|
//
|
|
// A bonded pair spanning two different NUMA nodes is treated as an
|
|
// anomaly (not a neutral fact) per project decision: we expect a bonded
|
|
// pair to sit on one NUMA node, so a cross-NUMA bond is flagged Warning
|
|
// on the edge AND on both GPU boxes, regardless of their own SAT status.
|
|
bandY := maxDeviceY + topoEdgeBand/2
|
|
var gpuEdgesSVG strings.Builder
|
|
for _, pair := range pairs {
|
|
c1, ok1 := gpuBoxCenter[pair.GPUA]
|
|
c2, ok2 := gpuBoxCenter[pair.GPUB]
|
|
if !ok1 || !ok2 {
|
|
continue
|
|
}
|
|
color := "var(--ok-fg)"
|
|
title := fmt.Sprintf("NVLink: GPU%d↔GPU%d (%d links)", pair.GPUA, pair.GPUB, pair.NVLinks)
|
|
numaA, numaB := gpuNUMANode[pair.GPUA], gpuNUMANode[pair.GPUB]
|
|
if numaA == nil || numaB == nil {
|
|
color = "var(--muted)"
|
|
} else if *numaA != *numaB {
|
|
color = "var(--warn-fg)"
|
|
title += " — spans NUMA nodes (unexpected)"
|
|
upgradeTopoBoxBadgeToWarn(boxes, gpuBoxIndex[pair.GPUA])
|
|
upgradeTopoBoxBadgeToWarn(boxes, gpuBoxIndex[pair.GPUB])
|
|
}
|
|
fmt.Fprintf(&gpuEdgesSVG,
|
|
`<path d="M %d %d L %d %d L %d %d L %d %d" style="fill:none;stroke:%s;stroke-width:2;stroke-dasharray:4,3"><title>%s</title></path>`+"\n",
|
|
c1[0], c1[1], c1[0], bandY, c2[0], bandY, c2[0], c2[1], color, html.EscapeString(title))
|
|
}
|
|
|
|
svgHeight := bandY + topoEdgeBand/2 + topoBottomRowH
|
|
svgWidth := totalCols*topoColWidth + 48
|
|
|
|
var b strings.Builder
|
|
fmt.Fprintf(&b, `<svg width="%d" height="%d" viewBox="0 0 %d %d" style="max-width:100%%">`+"\n", svgWidth, svgHeight, svgWidth, svgHeight)
|
|
for _, e := range pcieEdges {
|
|
fmt.Fprintf(&b, `<line x1="%d" y1="%d" x2="%d" y2="%d" style="stroke:%s;stroke-width:2"/>`+"\n", e.x1, e.y1, e.x2, e.y2, e.color)
|
|
}
|
|
b.WriteString(gpuEdgesSVG.String())
|
|
for _, box := range boxes {
|
|
writeTopoBoxSVG(&b, box)
|
|
}
|
|
|
|
// PSU / BMC row: standalone boxes, no connecting lines.
|
|
psuY := svgHeight - topoBottomRowH + 20
|
|
psuX := 24
|
|
for _, psu := range hw.PowerSupplies {
|
|
label, cls := topoStatusBadgeClass(psu.Status)
|
|
slot := ""
|
|
if psu.Slot != nil {
|
|
slot = *psu.Slot
|
|
}
|
|
watt := ""
|
|
if psu.WattageW != nil {
|
|
watt = fmt.Sprintf("%dW", *psu.WattageW)
|
|
}
|
|
writeTopoBoxSVG(&b, topoBox{
|
|
x: psuX, y: psuY, w: 150, h: topoBoxHeight,
|
|
label: "PSU " + slot, sublabel: watt,
|
|
badgeText: label, badgeCls: cls, detailType: "psu",
|
|
})
|
|
psuX += 150 + topoDeviceGap
|
|
}
|
|
if bmcVersion, ok := findBMCFirmware(hw.Firmware); ok {
|
|
writeTopoBoxSVG(&b, topoBox{
|
|
x: svgWidth - 200 - 24, y: psuY, w: 200, h: topoBoxHeight,
|
|
label: "BMC", sublabel: "fw " + bmcVersion,
|
|
badgeText: "?", badgeCls: "badge-unknown",
|
|
})
|
|
}
|
|
|
|
b.WriteString(`</svg>`)
|
|
return topoCard("Topology", b.String())
|
|
}
|
|
|
|
// upgradeTopoBoxBadgeToWarn upgrades a box's badge to Warning unless it is
|
|
// already at Critical severity (never downgrades a worse status).
|
|
func upgradeTopoBoxBadgeToWarn(boxes []topoBox, boxIdx int) {
|
|
if boxIdx < 0 || boxIdx >= len(boxes) {
|
|
return
|
|
}
|
|
if boxes[boxIdx].badgeCls == "badge-err" {
|
|
return
|
|
}
|
|
boxes[boxIdx].badgeText = "WARN"
|
|
boxes[boxIdx].badgeCls = "badge-warn"
|
|
}
|
|
|
|
func findBMCFirmware(records []schema.HardwareFirmwareRecord) (string, bool) {
|
|
for _, rec := range records {
|
|
if strings.EqualFold(strings.TrimSpace(rec.DeviceName), "BMC") {
|
|
return rec.Version, true
|
|
}
|
|
}
|
|
return "", false
|
|
}
|
|
|
|
func writeTopoBoxSVG(b *strings.Builder, box topoBox) {
|
|
onclick := ""
|
|
cursor := "default"
|
|
if box.detailType != "" {
|
|
onclick = fmt.Sprintf(` onclick="openComponentDetail('%s')"`, box.detailType)
|
|
cursor = "pointer"
|
|
}
|
|
fmt.Fprintf(b, `<g%s style="cursor:%s">`, onclick, cursor)
|
|
fmt.Fprintf(b, `<rect x="%d" y="%d" width="%d" height="%d" rx="6" ry="6" style="fill:var(--surface);stroke:var(--border)"/>`+"\n",
|
|
box.x, box.y, box.w, box.h)
|
|
fmt.Fprintf(b, `<text x="%d" y="%d" style="fill:var(--ink,#000);font-size:13px;font-weight:700">%s</text>`+"\n",
|
|
box.x+10, box.y+20, html.EscapeString(box.label))
|
|
if box.sublabel != "" {
|
|
fmt.Fprintf(b, `<text x="%d" y="%d" style="fill:var(--muted);font-size:11px">%s</text>`+"\n",
|
|
box.x+10, box.y+36, html.EscapeString(truncateTopoLabel(box.sublabel, 26)))
|
|
}
|
|
fmt.Fprintf(b, `<rect x="%d" y="%d" width="46" height="18" rx="3" ry="3" class="%s"/>`+"\n",
|
|
box.x+box.w-54, box.y+box.h-26, box.badgeCls)
|
|
fmt.Fprintf(b, `<text x="%d" y="%d" style="font-size:10px;font-weight:700" class="%s">%s</text>`+"\n",
|
|
box.x+box.w-49, box.y+box.h-13, box.badgeCls, html.EscapeString(box.badgeText))
|
|
b.WriteString(`</g>` + "\n")
|
|
}
|
|
|
|
func truncateTopoLabel(s string, max int) string {
|
|
if len(s) <= max {
|
|
return s
|
|
}
|
|
if max <= 1 {
|
|
return s[:max]
|
|
}
|
|
return s[:max-1] + "…"
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Separate NVLink topology card (read from techdump, not written to any
|
|
// ingest contract)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
type topoNVLinkPort struct {
|
|
Index int
|
|
Active bool
|
|
SpeedGBs *float64
|
|
ReplayErrors int64
|
|
RecoveryErrors int64
|
|
CRCErrors int64
|
|
}
|
|
|
|
var (
|
|
topoNVLinkGPUHeaderRe = regexp.MustCompile(`^GPU (\d+):`)
|
|
topoNVLinkSpeedLineRe = regexp.MustCompile(`^Link (\d+):\s*([\d.]+)\s*GB/s`)
|
|
topoNVLinkInactiveRe = regexp.MustCompile(`^Link (\d+):\s*<inactive>`)
|
|
topoNVLinkErrorLineRe = regexp.MustCompile(`^Link (\d+):\s*(Replay|Recovery|CRC) Errors:\s*(\d+)`)
|
|
)
|
|
|
|
func readTopoNVLinkStatus(exportDir string) (map[int][]topoNVLinkPort, error) {
|
|
raw, err := readTopoTechDump(exportDir, "nvidia-smi-nvlink-status.txt")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return parseTopoNVLinkStatus(raw), nil
|
|
}
|
|
|
|
func parseTopoNVLinkStatus(raw string) map[int][]topoNVLinkPort {
|
|
result := map[int][]topoNVLinkPort{}
|
|
currentGPU := -1
|
|
for _, line := range strings.Split(raw, "\n") {
|
|
trimmed := strings.TrimSpace(line)
|
|
if m := topoNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
|
|
currentGPU, _ = strconv.Atoi(m[1])
|
|
continue
|
|
}
|
|
if currentGPU < 0 {
|
|
continue
|
|
}
|
|
if m := topoNVLinkInactiveRe.FindStringSubmatch(trimmed); m != nil {
|
|
idx, _ := strconv.Atoi(m[1])
|
|
result[currentGPU] = append(result[currentGPU], topoNVLinkPort{Index: idx, Active: false})
|
|
continue
|
|
}
|
|
if m := topoNVLinkSpeedLineRe.FindStringSubmatch(trimmed); m != nil {
|
|
idx, _ := strconv.Atoi(m[1])
|
|
port := topoNVLinkPort{Index: idx, Active: true}
|
|
if speed, err := strconv.ParseFloat(m[2], 64); err == nil {
|
|
port.SpeedGBs = &speed
|
|
}
|
|
result[currentGPU] = append(result[currentGPU], port)
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
func readTopoNVLinkErrors(exportDir string) (map[int]map[int][3]int64, error) {
|
|
raw, err := readTopoTechDump(exportDir, "nvidia-smi-nvlink-errors.txt")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return parseTopoNVLinkErrors(raw), nil
|
|
}
|
|
|
|
// parseTopoNVLinkErrors returns, per GPU then link index, [replay, recovery, crc].
|
|
func parseTopoNVLinkErrors(raw string) map[int]map[int][3]int64 {
|
|
result := map[int]map[int][3]int64{}
|
|
currentGPU := -1
|
|
for _, line := range strings.Split(raw, "\n") {
|
|
trimmed := strings.TrimSpace(line)
|
|
if m := topoNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
|
|
currentGPU, _ = strconv.Atoi(m[1])
|
|
continue
|
|
}
|
|
if currentGPU < 0 {
|
|
continue
|
|
}
|
|
m := topoNVLinkErrorLineRe.FindStringSubmatch(trimmed)
|
|
if m == nil {
|
|
continue
|
|
}
|
|
linkIdx, _ := strconv.Atoi(m[1])
|
|
count, _ := strconv.ParseInt(m[3], 10, 64)
|
|
if result[currentGPU] == nil {
|
|
result[currentGPU] = map[int][3]int64{}
|
|
}
|
|
c := result[currentGPU][linkIdx]
|
|
switch m[2] {
|
|
case "Replay":
|
|
c[0] = count
|
|
case "Recovery":
|
|
c[1] = count
|
|
case "CRC":
|
|
c[2] = count
|
|
}
|
|
result[currentGPU][linkIdx] = c
|
|
}
|
|
return result
|
|
}
|
|
|
|
// renderTopoNVLinkCard renders the separate NVLink topology card. Returns ""
|
|
// if there are fewer than 2 NVIDIA GPUs, or the nvidia-smi nvlink techdump
|
|
// wasn't captured (older audit, or nvidia-smi unavailable on that run).
|
|
func renderTopoNVLinkCard(hw schema.HardwareSnapshot, exportDir string) string {
|
|
gpuCount := 0
|
|
for _, dev := range hw.PCIeDevices {
|
|
if dev.DeviceClass != nil && isGPUDeviceClass(*dev.DeviceClass) {
|
|
gpuCount++
|
|
}
|
|
}
|
|
if gpuCount < 2 {
|
|
return ""
|
|
}
|
|
|
|
status, err := readTopoNVLinkStatus(exportDir)
|
|
if err != nil || len(status) == 0 {
|
|
return topoCard("NVLink Topology", `<span class="badge badge-unknown">nvidia-smi nvlink data unavailable</span>`)
|
|
}
|
|
errors, _ := readTopoNVLinkErrors(exportDir)
|
|
|
|
topoMatrix, _ := readGPUTopologyMatrix(exportDir)
|
|
pairs := parseGPUPairAdjacency(topoMatrix)
|
|
|
|
var bodyB strings.Builder
|
|
if gpuCount <= 4 && len(pairs) > 0 {
|
|
// Small GPU count: per-pair box+line with per-link detail.
|
|
for _, pair := range pairs {
|
|
activeCount, total, hasError := 0, 0, false
|
|
for _, port := range status[pair.GPUA] {
|
|
total++
|
|
if port.Active {
|
|
activeCount++
|
|
}
|
|
}
|
|
for _, counters := range errors[pair.GPUA] {
|
|
if counters[0] != 0 || counters[1] != 0 || counters[2] != 0 {
|
|
hasError = true
|
|
}
|
|
}
|
|
color := "var(--ok-fg)"
|
|
switch {
|
|
case hasError:
|
|
color = "var(--crit-fg)"
|
|
case total > 0 && activeCount < total:
|
|
color = "var(--warn-fg)"
|
|
}
|
|
fmt.Fprintf(&bodyB, `<div style="display:flex;align-items:center;gap:12px;margin-bottom:10px">`+
|
|
`<div style="padding:8px 12px;border:1px solid var(--border);border-radius:6px">GPU %d</div>`+
|
|
`<div style="flex:1;height:2px;background:%s"></div>`+
|
|
`<div style="padding:8px 12px;border:1px solid var(--border);border-radius:6px">GPU %d</div>`+
|
|
`<div style="font-size:12px;color:var(--muted)">%d/%d links active%s</div>`+
|
|
`</div>`,
|
|
pair.GPUA, color, pair.GPUB, activeCount, total, errNoteSuffix(hasError))
|
|
}
|
|
} else if len(pairs) > 0 {
|
|
// Larger GPU counts (NVSwitch fabric): aggregate pair table instead of
|
|
// an unreadable all-to-all graph.
|
|
bodyB.WriteString(`<table><thead><tr><th>GPU A</th><th>GPU B</th><th>NVLinks</th></tr></thead><tbody>`)
|
|
for _, pair := range pairs {
|
|
fmt.Fprintf(&bodyB, `<tr><td>GPU %d</td><td>GPU %d</td><td>%d</td></tr>`, pair.GPUA, pair.GPUB, pair.NVLinks)
|
|
}
|
|
bodyB.WriteString(`</tbody></table>`)
|
|
} else {
|
|
bodyB.WriteString(`<span class="badge badge-unknown">No NVLink-bonded GPU pairs found</span>`)
|
|
}
|
|
|
|
return topoCard("NVLink Topology", bodyB.String())
|
|
}
|
|
|
|
func errNoteSuffix(hasError bool) string {
|
|
if hasError {
|
|
return " — errors detected"
|
|
}
|
|
return ""
|
|
}
|