webui: add /topo server topology page
New read-only visualization page: CPU sockets as anchor nodes, PCIe
devices (GPU/NIC/RAID) linked to their NUMA-affine socket with edge
color derived strictly from link_speed vs max_link_speed (not from the
device's own Status, which can also be overwritten by SAT-test results
on the same field), PSU/BMC as standalone boxes with no connecting
line, and a separate NVLink Topology card (live nvidia-smi topo -m /
nvlink -s/-e queries, not persisted to any contract).
GPU-GPU edges are drawn strictly from the actual bonded-pair list
parsed out of "nvidia-smi topo -m" (parseGPUPairAdjacency), not from
adjacent box position in the layout — an earlier ASCII mockup drew a
"chain" through unrelated GPUs, which a dedicated regression test now
guards against. A bonded pair spanning two different NUMA nodes is
flagged Warning on the edge and on both GPU boxes, per project
decision that this is an anomaly worth surfacing, not a neutral fact.
Zero changes to the ingest contract: this reverts the HardwareNVLinkPort/
HardwarePCIeDevice.NVLinks field shipped in v11.55 (33d6eee) along with
its collector/nvidia.go enrichment — that field risked a 400 from
Reanimator Core's strict decoder without an RFC, and isn't needed since
the topo page queries nvidia-smi directly instead of reading it from
audit.json. The v11.55 systemd fix (bee-nvidia.service ordering,
nv-hostengine restart) and the nvlink-status/-errors/dcgmi dumps in the
support bundle are untouched.
Also reorganizes support bundle collection per project convention:
system/ is now LiveCD-operational logs only (Xorg, services, console,
network/FS of the host itself); all server-hardware dumps (lspci,
NVIDIA/NVLink/DCGM, fabric manager, PCIe AER, ethtool, mstflint) move
to techdump/, deduplicating two entries already produced by
platform/techdump.go. Adds nvidia-bug-report.sh (previously only
collected inside the NVIDIA SAT pack) and lscpu to the always-on dump.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
33d6eee9cf
commit
a3377083aa
@@ -0,0 +1,777 @@
|
||||
package webui
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"html"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"bee/audit/internal/schema"
|
||||
)
|
||||
|
||||
// renderTopo renders the /topo page: a read-only visualization of the server
|
||||
// topology (CPU sockets, NUMA-affine PCIe devices, PSU/BMC) plus a separate
|
||||
// NVLink topology card. It is pure visualization: everything it reads either
|
||||
// already exists in the audit.json contract, or is a live (non-persisted)
|
||||
// query — nothing here writes to schema.HardwarePCIeDevice or any other
|
||||
// contract type, and nothing here talks to Reanimator Core.
|
||||
func renderTopo(opts HandlerOptions) string {
|
||||
data, err := loadSnapshot(opts.AuditPath)
|
||||
if err != nil {
|
||||
return topoCard("Topology", `<span class="badge badge-unknown">No audit data</span>`)
|
||||
}
|
||||
var ingest schema.HardwareIngestRequest
|
||||
if err := json.Unmarshal(data, &ingest); err != nil {
|
||||
return topoCard("Topology", `<span class="badge badge-err">Parse error</span>`)
|
||||
}
|
||||
hw := ingest.Hardware
|
||||
|
||||
var b strings.Builder
|
||||
b.WriteString(renderTopoMainDiagram(hw))
|
||||
if nv := renderTopoNVLinkCard(hw); nv != "" {
|
||||
b.WriteString(nv)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func topoCard(title, body string) string {
|
||||
return `<div class="card"><div class="card-head">` + html.EscapeString(title) + `</div><div class="card-body">` + body + `</div></div>`
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Classification helpers
|
||||
//
|
||||
// webui does not import collector (matches the existing isGPUDeviceClass
|
||||
// precedent in pages.go, which already locally duplicates collector.isGPUClass
|
||||
// instead of importing the package for one classifier).
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// isNICDeviceClassDev mirrors the classification logic in hwDescribeNIC
|
||||
// (pages.go), applied to a single device instead of aggregated counts.
|
||||
func isNICDeviceClassDev(dev schema.HardwarePCIeDevice) bool {
|
||||
if dev.DeviceClass != nil {
|
||||
c := strings.ToLower(strings.TrimSpace(*dev.DeviceClass))
|
||||
if c == "ethernetcontroller" || c == "networkcontroller" || strings.Contains(c, "fibrechannel") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return len(dev.MacAddresses) > 0
|
||||
}
|
||||
|
||||
// isRAIDControllerClass matches the canonical class strings produced by
|
||||
// collector.mapPCIeDeviceClass for RAID/storage HBAs.
|
||||
func isRAIDControllerClass(class string) bool {
|
||||
switch strings.TrimSpace(class) {
|
||||
case "MassStorageController", "StorageController":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Status / link-speed coloring
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// topoStatusBadgeClass maps a component's Status pointer to a badge class,
|
||||
// treating a nil/absent status the same as literal "Unknown" (matches how
|
||||
// the rest of the UI already renders missing status, per chipLetterClass/
|
||||
// runtimeStatusBadge in pages.go).
|
||||
func topoStatusBadgeClass(status *string) (label, cls string) {
|
||||
if status == nil {
|
||||
return "?", "badge-unknown"
|
||||
}
|
||||
switch strings.ToUpper(strings.TrimSpace(*status)) {
|
||||
case "OK":
|
||||
return "OK", "badge-ok"
|
||||
case "WARNING", "WARN", "PARTIAL":
|
||||
return "WARN", "badge-warn"
|
||||
case "CRITICAL", "FAIL", "FAILED", "ERROR":
|
||||
return "CRIT", "badge-err"
|
||||
default:
|
||||
return "?", "badge-unknown"
|
||||
}
|
||||
}
|
||||
|
||||
// pcieGenRank ranks a PCIe generation label ("Gen3", "Gen4", ...) for
|
||||
// comparison. Mirrors collector.pcieLinkSpeedRank's ordering; duplicated
|
||||
// locally rather than exported, per the same "no collector import in webui"
|
||||
// convention used for isGPUDeviceClass/isRAIDControllerClass.
|
||||
func pcieGenRank(gen string) int {
|
||||
gen = strings.ToLower(strings.TrimSpace(gen))
|
||||
gen = strings.TrimPrefix(gen, "gen")
|
||||
n, err := strconv.Atoi(gen)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// topoEdgeColorVar computes the CPU->device edge color strictly from
|
||||
// link_speed vs max_link_speed — NOT from dev.Status, since Status can also
|
||||
// be overwritten by SAT/acceptance-test results on the same PCIe device,
|
||||
// which would conflate "link is physically degraded" with "this GPU failed
|
||||
// its stress test" into the same color.
|
||||
func topoEdgeColorVar(dev schema.HardwarePCIeDevice) string {
|
||||
if dev.LinkSpeed == nil || dev.MaxLinkSpeed == nil {
|
||||
return "var(--muted)"
|
||||
}
|
||||
if pcieGenRank(*dev.LinkSpeed) < pcieGenRank(*dev.MaxLinkSpeed) {
|
||||
return "var(--warn-fg)"
|
||||
}
|
||||
return "var(--ok-fg)"
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// NUMA node -> CPU socket join (heuristic, no guaranteed hardware mapping)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// buildSocketIndex maps a NUMA node number to the index into cpus whose
|
||||
// Socket field equals that NUMA node number. This is a best-effort heuristic
|
||||
// (NUMANode == Socket) documented as such in the /topo design — Linux NUMA
|
||||
// node numbering and dmidecode socket designation are different numbering
|
||||
// domains with no guaranteed 1:1 mapping, but in practice agree for the
|
||||
// common case of N-socket boards.
|
||||
func buildSocketIndex(cpus []schema.HardwareCPU) map[int]int {
|
||||
idx := map[int]int{}
|
||||
for i, cpu := range cpus {
|
||||
if cpu.Socket != nil {
|
||||
idx[*cpu.Socket] = i
|
||||
}
|
||||
}
|
||||
return idx
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// GPU pairwise NVLink adjacency (from a live "nvidia-smi topo -m" query)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type gpuPairLink struct {
|
||||
GPUA, GPUB int
|
||||
NVLinks int
|
||||
}
|
||||
|
||||
var topoNVRe = regexp.MustCompile(`(?i)^NV(\d+)$`)
|
||||
|
||||
// parseGPUPairAdjacency returns every GPU pair with a nonzero NVLink bond
|
||||
// count from a "nvidia-smi topo -m" matrix. Unlike parseNVIDIATopologyMatrix
|
||||
// (collector package, aggregate-only: min/all-active/count), this returns
|
||||
// who is bonded to whom — required so GPU-GPU edges are drawn for actually
|
||||
// bonded pairs, not for adjacent boxes in the layout.
|
||||
func parseGPUPairAdjacency(raw string) []gpuPairLink {
|
||||
lines := strings.Split(raw, "\n")
|
||||
headerIdx := -1
|
||||
var gpuColIndices []int
|
||||
for i, line := range lines {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if strings.HasPrefix(trimmed, "GPU0") {
|
||||
parts := strings.Fields(trimmed)
|
||||
for j, col := range parts {
|
||||
if strings.HasPrefix(col, "GPU") {
|
||||
gpuColIndices = append(gpuColIndices, j)
|
||||
}
|
||||
}
|
||||
if len(gpuColIndices) >= 2 {
|
||||
headerIdx = i
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
if headerIdx < 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
colIdxToGPU := make(map[int]int, len(gpuColIndices))
|
||||
for gpuIdx, colIdx := range gpuColIndices {
|
||||
colIdxToGPU[colIdx] = gpuIdx
|
||||
}
|
||||
|
||||
seen := map[[2]int]bool{}
|
||||
var pairs []gpuPairLink
|
||||
rowGPU := -1
|
||||
for _, line := range lines[headerIdx+1:] {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if !strings.HasPrefix(trimmed, "GPU") {
|
||||
continue
|
||||
}
|
||||
cells := strings.Fields(trimmed)
|
||||
if len(cells) == 0 {
|
||||
continue
|
||||
}
|
||||
rowLabel := strings.TrimPrefix(cells[0], "GPU")
|
||||
n, err := strconv.Atoi(rowLabel)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
rowGPU = n
|
||||
for colIdx, colGPU := range colIdxToGPU {
|
||||
if colGPU == rowGPU {
|
||||
continue
|
||||
}
|
||||
dataIdx := colIdx + 1
|
||||
if dataIdx >= len(cells) {
|
||||
continue
|
||||
}
|
||||
m := topoNVRe.FindStringSubmatch(cells[dataIdx])
|
||||
if len(m) != 2 {
|
||||
continue
|
||||
}
|
||||
nv, err := strconv.Atoi(m[1])
|
||||
if err != nil || nv <= 0 {
|
||||
continue
|
||||
}
|
||||
a, bGPU := rowGPU, colGPU
|
||||
if a > bGPU {
|
||||
a, bGPU = bGPU, a
|
||||
}
|
||||
key := [2]int{a, bGPU}
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
pairs = append(pairs, gpuPairLink{GPUA: a, GPUB: bGPU, NVLinks: nv})
|
||||
}
|
||||
}
|
||||
sort.Slice(pairs, func(i, j int) bool {
|
||||
if pairs[i].GPUA != pairs[j].GPUA {
|
||||
return pairs[i].GPUA < pairs[j].GPUA
|
||||
}
|
||||
return pairs[i].GPUB < pairs[j].GPUB
|
||||
})
|
||||
return pairs
|
||||
}
|
||||
|
||||
func queryGPUTopologyMatrix() (string, error) {
|
||||
out, err := exec.Command("nvidia-smi", "topo", "-m").Output()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(out), nil
|
||||
}
|
||||
|
||||
// queryNVIDIAIndexByBDF runs a lightweight live nvidia-smi query mapping
|
||||
// PCI bus address (matching HardwarePCIeDevice.Slot) to the GPU index
|
||||
// nvidia-smi/dcgmi/topo -m report, so GPU-GPU edges (keyed by index) can be
|
||||
// anchored to the correct box (keyed by BDF) in the diagram.
|
||||
func queryNVIDIAIndexByBDF() (map[string]int, error) {
|
||||
out, err := exec.Command("nvidia-smi", "--query-gpu=index,pci.bus_id", "--format=csv,noheader").Output()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := map[string]int{}
|
||||
for _, line := range strings.Split(string(out), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
parts := strings.SplitN(line, ",", 2)
|
||||
if len(parts) != 2 {
|
||||
continue
|
||||
}
|
||||
idx, err := strconv.Atoi(strings.TrimSpace(parts[0]))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
bdf := normalizeTopoBDF(strings.TrimSpace(parts[1]))
|
||||
if bdf == "" {
|
||||
continue
|
||||
}
|
||||
result[bdf] = idx
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// normalizeTopoBDF normalizes a PCI bus address to "dddd:bb:dd.f" form so
|
||||
// nvidia-smi's "pci.bus_id" output can be matched against
|
||||
// HardwarePCIeDevice.Slot regardless of minor formatting differences
|
||||
// (case, leading domain padding).
|
||||
func normalizeTopoBDF(bdf string) string {
|
||||
bdf = strings.ToLower(strings.TrimSpace(bdf))
|
||||
if bdf == "" {
|
||||
return ""
|
||||
}
|
||||
parts := strings.Split(bdf, ":")
|
||||
if len(parts) == 3 {
|
||||
domain := parts[0]
|
||||
if len(domain) > 4 {
|
||||
domain = domain[len(domain)-4:]
|
||||
}
|
||||
return domain + ":" + parts[1] + ":" + parts[2]
|
||||
}
|
||||
return bdf
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main topology diagram
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const (
|
||||
topoColWidth = 220
|
||||
topoBoxWidth = 190
|
||||
topoBoxHeight = 56
|
||||
topoDeviceGap = 14
|
||||
topoTopMargin = 30
|
||||
topoEdgeBand = 60
|
||||
topoBottomRowH = 90
|
||||
)
|
||||
|
||||
type topoBox struct {
|
||||
x, y, w, h int
|
||||
label string
|
||||
sublabel string
|
||||
badgeText string
|
||||
badgeCls string
|
||||
detailType string // "" = not clickable
|
||||
}
|
||||
|
||||
func renderTopoMainDiagram(hw schema.HardwareSnapshot) string {
|
||||
socketIdx := buildSocketIndex(hw.CPUs)
|
||||
numCols := len(hw.CPUs)
|
||||
if numCols == 0 {
|
||||
numCols = 1
|
||||
}
|
||||
unknownCol := numCols // extra trailing column for unmatched devices
|
||||
|
||||
// Group PCIe devices (GPU/NIC/RAID only — matches the mockup's node
|
||||
// types) into columns by NUMA node, falling back to the "unknown" bucket.
|
||||
type placedDevice struct {
|
||||
dev schema.HardwarePCIeDevice
|
||||
kind string // "gpu", "nic", "raid"
|
||||
col int
|
||||
bdf string
|
||||
}
|
||||
var placed []placedDevice
|
||||
for _, dev := range hw.PCIeDevices {
|
||||
var kind string
|
||||
switch {
|
||||
case dev.DeviceClass != nil && isGPUDeviceClass(*dev.DeviceClass):
|
||||
kind = "gpu"
|
||||
case isNICDeviceClassDev(dev):
|
||||
kind = "nic"
|
||||
case dev.DeviceClass != nil && isRAIDControllerClass(*dev.DeviceClass):
|
||||
kind = "raid"
|
||||
default:
|
||||
continue
|
||||
}
|
||||
col := unknownCol
|
||||
if dev.NUMANode != nil {
|
||||
if ci, ok := socketIdx[*dev.NUMANode]; ok {
|
||||
col = ci
|
||||
}
|
||||
}
|
||||
bdf := ""
|
||||
if dev.Slot != nil {
|
||||
bdf = normalizeTopoBDF(*dev.Slot)
|
||||
} else if dev.BDF != nil {
|
||||
bdf = normalizeTopoBDF(*dev.BDF)
|
||||
}
|
||||
placed = append(placed, placedDevice{dev: dev, kind: kind, col: col, bdf: bdf})
|
||||
}
|
||||
hasUnknownCol := false
|
||||
for _, p := range placed {
|
||||
if p.col == unknownCol {
|
||||
hasUnknownCol = true
|
||||
break
|
||||
}
|
||||
}
|
||||
totalCols := numCols
|
||||
if hasUnknownCol {
|
||||
totalCols++
|
||||
}
|
||||
|
||||
// Live GPU index<->BDF map + pairwise NVLink adjacency, best-effort:
|
||||
// if nvidia-smi is unavailable, GPU-GPU edges are simply omitted.
|
||||
bdfToIndex, _ := queryNVIDIAIndexByBDF()
|
||||
var pairs []gpuPairLink
|
||||
if topoMatrix, err := queryGPUTopologyMatrix(); err == nil {
|
||||
pairs = parseGPUPairAdjacency(topoMatrix)
|
||||
}
|
||||
|
||||
var boxes []topoBox
|
||||
var pcieEdges []struct {
|
||||
x1, y1, x2, y2 int
|
||||
color string
|
||||
}
|
||||
gpuBoxCenter := map[int][2]int{} // gpu index -> (x, yBottom)
|
||||
gpuBoxIndex := map[int]int{} // gpu index -> index into boxes
|
||||
gpuNUMANode := map[int]*int{} // gpu index -> its PCIe device's numa_node
|
||||
|
||||
for col := 0; col < totalCols; col++ {
|
||||
colX := (col+1)*24 + col*topoColWidth
|
||||
if col < numCols {
|
||||
cpu := hw.CPUs[col]
|
||||
model := ""
|
||||
if cpu.Model != nil {
|
||||
model = *cpu.Model
|
||||
}
|
||||
socket := col
|
||||
if cpu.Socket != nil {
|
||||
socket = *cpu.Socket
|
||||
}
|
||||
label, cls := topoStatusBadgeClass(cpu.Status)
|
||||
boxes = append(boxes, topoBox{
|
||||
x: colX, y: topoTopMargin, w: topoBoxWidth, h: topoBoxHeight,
|
||||
label: fmt.Sprintf("CPU %d", socket), sublabel: model,
|
||||
badgeText: label, badgeCls: cls, detailType: "cpu",
|
||||
})
|
||||
}
|
||||
|
||||
y := topoTopMargin + topoBoxHeight + topoDeviceGap*2
|
||||
for _, p := range placed {
|
||||
if p.col != col {
|
||||
continue
|
||||
}
|
||||
label, cls := topoStatusBadgeClass(p.dev.Status)
|
||||
model := ""
|
||||
if p.dev.Model != nil {
|
||||
model = *p.dev.Model
|
||||
}
|
||||
box := topoBox{
|
||||
x: colX, y: y, w: topoBoxWidth, h: topoBoxHeight,
|
||||
label: strings.ToUpper(p.kind), sublabel: model,
|
||||
badgeText: label, badgeCls: cls, detailType: p.kind,
|
||||
}
|
||||
boxes = append(boxes, box)
|
||||
boxIdx := len(boxes) - 1
|
||||
|
||||
if col < numCols {
|
||||
pcieEdges = append(pcieEdges, struct {
|
||||
x1, y1, x2, y2 int
|
||||
color string
|
||||
}{
|
||||
x1: colX + topoBoxWidth/2, y1: topoTopMargin + topoBoxHeight,
|
||||
x2: colX + topoBoxWidth/2, y2: y,
|
||||
color: topoEdgeColorVar(p.dev),
|
||||
})
|
||||
}
|
||||
|
||||
if p.kind == "gpu" && p.bdf != "" {
|
||||
if idx, ok := bdfToIndex[p.bdf]; ok {
|
||||
gpuBoxCenter[idx] = [2]int{colX + topoBoxWidth/2, y + topoBoxHeight}
|
||||
gpuBoxIndex[idx] = boxIdx
|
||||
gpuNUMANode[idx] = p.dev.NUMANode
|
||||
}
|
||||
}
|
||||
|
||||
y += topoBoxHeight + topoDeviceGap
|
||||
}
|
||||
}
|
||||
|
||||
maxDeviceY := topoTopMargin + topoBoxHeight + topoDeviceGap*2
|
||||
for _, b := range boxes {
|
||||
if b.y+b.h > maxDeviceY {
|
||||
maxDeviceY = b.y + b.h
|
||||
}
|
||||
}
|
||||
|
||||
// GPU-GPU NVLink edges: drawn as an elbow connector through a dedicated
|
||||
// band below the device row, kept strictly separate from the vertical
|
||||
// CPU->device PCIe edges above so neither visually obscures the other.
|
||||
//
|
||||
// A bonded pair spanning two different NUMA nodes is treated as an
|
||||
// anomaly (not a neutral fact) per project decision: we expect a bonded
|
||||
// pair to sit on one NUMA node, so a cross-NUMA bond is flagged Warning
|
||||
// on the edge AND on both GPU boxes, regardless of their own SAT status.
|
||||
bandY := maxDeviceY + topoEdgeBand/2
|
||||
var gpuEdgesSVG strings.Builder
|
||||
for _, pair := range pairs {
|
||||
c1, ok1 := gpuBoxCenter[pair.GPUA]
|
||||
c2, ok2 := gpuBoxCenter[pair.GPUB]
|
||||
if !ok1 || !ok2 {
|
||||
continue
|
||||
}
|
||||
color := "var(--ok-fg)"
|
||||
title := fmt.Sprintf("NVLink: GPU%d↔GPU%d (%d links)", pair.GPUA, pair.GPUB, pair.NVLinks)
|
||||
numaA, numaB := gpuNUMANode[pair.GPUA], gpuNUMANode[pair.GPUB]
|
||||
if numaA == nil || numaB == nil {
|
||||
color = "var(--muted)"
|
||||
} else if *numaA != *numaB {
|
||||
color = "var(--warn-fg)"
|
||||
title += " — spans NUMA nodes (unexpected)"
|
||||
upgradeTopoBoxBadgeToWarn(boxes, gpuBoxIndex[pair.GPUA])
|
||||
upgradeTopoBoxBadgeToWarn(boxes, gpuBoxIndex[pair.GPUB])
|
||||
}
|
||||
fmt.Fprintf(&gpuEdgesSVG,
|
||||
`<path d="M %d %d L %d %d L %d %d L %d %d" style="fill:none;stroke:%s;stroke-width:2;stroke-dasharray:4,3"><title>%s</title></path>`+"\n",
|
||||
c1[0], c1[1], c1[0], bandY, c2[0], bandY, c2[0], c2[1], color, html.EscapeString(title))
|
||||
}
|
||||
|
||||
svgHeight := bandY + topoEdgeBand/2 + topoBottomRowH
|
||||
svgWidth := totalCols*topoColWidth + 48
|
||||
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, `<svg width="%d" height="%d" viewBox="0 0 %d %d" style="max-width:100%%">`+"\n", svgWidth, svgHeight, svgWidth, svgHeight)
|
||||
for _, e := range pcieEdges {
|
||||
fmt.Fprintf(&b, `<line x1="%d" y1="%d" x2="%d" y2="%d" style="stroke:%s;stroke-width:2"/>`+"\n", e.x1, e.y1, e.x2, e.y2, e.color)
|
||||
}
|
||||
b.WriteString(gpuEdgesSVG.String())
|
||||
for _, box := range boxes {
|
||||
writeTopoBoxSVG(&b, box)
|
||||
}
|
||||
|
||||
// PSU / BMC row: standalone boxes, no connecting lines.
|
||||
psuY := svgHeight - topoBottomRowH + 20
|
||||
psuX := 24
|
||||
for _, psu := range hw.PowerSupplies {
|
||||
label, cls := topoStatusBadgeClass(psu.Status)
|
||||
slot := ""
|
||||
if psu.Slot != nil {
|
||||
slot = *psu.Slot
|
||||
}
|
||||
watt := ""
|
||||
if psu.WattageW != nil {
|
||||
watt = fmt.Sprintf("%dW", *psu.WattageW)
|
||||
}
|
||||
writeTopoBoxSVG(&b, topoBox{
|
||||
x: psuX, y: psuY, w: 150, h: topoBoxHeight,
|
||||
label: "PSU " + slot, sublabel: watt,
|
||||
badgeText: label, badgeCls: cls, detailType: "psu",
|
||||
})
|
||||
psuX += 150 + topoDeviceGap
|
||||
}
|
||||
if bmcVersion, ok := findBMCFirmware(hw.Firmware); ok {
|
||||
writeTopoBoxSVG(&b, topoBox{
|
||||
x: svgWidth - 200 - 24, y: psuY, w: 200, h: topoBoxHeight,
|
||||
label: "BMC", sublabel: "fw " + bmcVersion,
|
||||
badgeText: "?", badgeCls: "badge-unknown",
|
||||
})
|
||||
}
|
||||
|
||||
b.WriteString(`</svg>`)
|
||||
return topoCard("Topology", b.String())
|
||||
}
|
||||
|
||||
// upgradeTopoBoxBadgeToWarn upgrades a box's badge to Warning unless it is
|
||||
// already at Critical severity (never downgrades a worse status).
|
||||
func upgradeTopoBoxBadgeToWarn(boxes []topoBox, boxIdx int) {
|
||||
if boxIdx < 0 || boxIdx >= len(boxes) {
|
||||
return
|
||||
}
|
||||
if boxes[boxIdx].badgeCls == "badge-err" {
|
||||
return
|
||||
}
|
||||
boxes[boxIdx].badgeText = "WARN"
|
||||
boxes[boxIdx].badgeCls = "badge-warn"
|
||||
}
|
||||
|
||||
func findBMCFirmware(records []schema.HardwareFirmwareRecord) (string, bool) {
|
||||
for _, rec := range records {
|
||||
if strings.EqualFold(strings.TrimSpace(rec.DeviceName), "BMC") {
|
||||
return rec.Version, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func writeTopoBoxSVG(b *strings.Builder, box topoBox) {
|
||||
onclick := ""
|
||||
cursor := "default"
|
||||
if box.detailType != "" {
|
||||
onclick = fmt.Sprintf(` onclick="openComponentDetail('%s')"`, box.detailType)
|
||||
cursor = "pointer"
|
||||
}
|
||||
fmt.Fprintf(b, `<g%s style="cursor:%s">`, onclick, cursor)
|
||||
fmt.Fprintf(b, `<rect x="%d" y="%d" width="%d" height="%d" rx="6" ry="6" style="fill:var(--surface);stroke:var(--border)"/>`+"\n",
|
||||
box.x, box.y, box.w, box.h)
|
||||
fmt.Fprintf(b, `<text x="%d" y="%d" style="fill:var(--ink,#000);font-size:13px;font-weight:700">%s</text>`+"\n",
|
||||
box.x+10, box.y+20, html.EscapeString(box.label))
|
||||
if box.sublabel != "" {
|
||||
fmt.Fprintf(b, `<text x="%d" y="%d" style="fill:var(--muted);font-size:11px">%s</text>`+"\n",
|
||||
box.x+10, box.y+36, html.EscapeString(truncateTopoLabel(box.sublabel, 26)))
|
||||
}
|
||||
fmt.Fprintf(b, `<rect x="%d" y="%d" width="46" height="18" rx="3" ry="3" class="%s"/>`+"\n",
|
||||
box.x+box.w-54, box.y+box.h-26, box.badgeCls)
|
||||
fmt.Fprintf(b, `<text x="%d" y="%d" style="font-size:10px;font-weight:700" class="%s">%s</text>`+"\n",
|
||||
box.x+box.w-49, box.y+box.h-13, box.badgeCls, html.EscapeString(box.badgeText))
|
||||
b.WriteString(`</g>` + "\n")
|
||||
}
|
||||
|
||||
func truncateTopoLabel(s string, max int) string {
|
||||
if len(s) <= max {
|
||||
return s
|
||||
}
|
||||
if max <= 1 {
|
||||
return s[:max]
|
||||
}
|
||||
return s[:max-1] + "…"
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Separate NVLink topology card (live query, not persisted to any contract)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type topoNVLinkPort struct {
|
||||
Index int
|
||||
Active bool
|
||||
SpeedGBs *float64
|
||||
ReplayErrors int64
|
||||
RecoveryErrors int64
|
||||
CRCErrors int64
|
||||
}
|
||||
|
||||
var (
|
||||
topoNVLinkGPUHeaderRe = regexp.MustCompile(`^GPU (\d+):`)
|
||||
topoNVLinkSpeedLineRe = regexp.MustCompile(`^Link (\d+):\s*([\d.]+)\s*GB/s`)
|
||||
topoNVLinkInactiveRe = regexp.MustCompile(`^Link (\d+):\s*<inactive>`)
|
||||
topoNVLinkErrorLineRe = regexp.MustCompile(`^Link (\d+):\s*(Replay|Recovery|CRC) Errors:\s*(\d+)`)
|
||||
)
|
||||
|
||||
func queryTopoNVLinkStatus() (map[int][]topoNVLinkPort, error) {
|
||||
out, err := exec.Command("nvidia-smi", "nvlink", "-s").Output()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return parseTopoNVLinkStatus(string(out)), nil
|
||||
}
|
||||
|
||||
func parseTopoNVLinkStatus(raw string) map[int][]topoNVLinkPort {
|
||||
result := map[int][]topoNVLinkPort{}
|
||||
currentGPU := -1
|
||||
for _, line := range strings.Split(raw, "\n") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if m := topoNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
|
||||
currentGPU, _ = strconv.Atoi(m[1])
|
||||
continue
|
||||
}
|
||||
if currentGPU < 0 {
|
||||
continue
|
||||
}
|
||||
if m := topoNVLinkInactiveRe.FindStringSubmatch(trimmed); m != nil {
|
||||
idx, _ := strconv.Atoi(m[1])
|
||||
result[currentGPU] = append(result[currentGPU], topoNVLinkPort{Index: idx, Active: false})
|
||||
continue
|
||||
}
|
||||
if m := topoNVLinkSpeedLineRe.FindStringSubmatch(trimmed); m != nil {
|
||||
idx, _ := strconv.Atoi(m[1])
|
||||
port := topoNVLinkPort{Index: idx, Active: true}
|
||||
if speed, err := strconv.ParseFloat(m[2], 64); err == nil {
|
||||
port.SpeedGBs = &speed
|
||||
}
|
||||
result[currentGPU] = append(result[currentGPU], port)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func queryTopoNVLinkErrors() (map[int]map[int][3]int64, error) {
|
||||
out, err := exec.Command("nvidia-smi", "nvlink", "-e").Output()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return parseTopoNVLinkErrors(string(out)), nil
|
||||
}
|
||||
|
||||
// parseTopoNVLinkErrors returns, per GPU then link index, [replay, recovery, crc].
|
||||
func parseTopoNVLinkErrors(raw string) map[int]map[int][3]int64 {
|
||||
result := map[int]map[int][3]int64{}
|
||||
currentGPU := -1
|
||||
for _, line := range strings.Split(raw, "\n") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if m := topoNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
|
||||
currentGPU, _ = strconv.Atoi(m[1])
|
||||
continue
|
||||
}
|
||||
if currentGPU < 0 {
|
||||
continue
|
||||
}
|
||||
m := topoNVLinkErrorLineRe.FindStringSubmatch(trimmed)
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
linkIdx, _ := strconv.Atoi(m[1])
|
||||
count, _ := strconv.ParseInt(m[3], 10, 64)
|
||||
if result[currentGPU] == nil {
|
||||
result[currentGPU] = map[int][3]int64{}
|
||||
}
|
||||
c := result[currentGPU][linkIdx]
|
||||
switch m[2] {
|
||||
case "Replay":
|
||||
c[0] = count
|
||||
case "Recovery":
|
||||
c[1] = count
|
||||
case "CRC":
|
||||
c[2] = count
|
||||
}
|
||||
result[currentGPU][linkIdx] = c
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// renderTopoNVLinkCard renders the separate NVLink topology card. Returns ""
|
||||
// if there are fewer than 2 NVIDIA GPUs or nvidia-smi is unavailable.
|
||||
func renderTopoNVLinkCard(hw schema.HardwareSnapshot) string {
|
||||
gpuCount := 0
|
||||
for _, dev := range hw.PCIeDevices {
|
||||
if dev.DeviceClass != nil && isGPUDeviceClass(*dev.DeviceClass) {
|
||||
gpuCount++
|
||||
}
|
||||
}
|
||||
if gpuCount < 2 {
|
||||
return ""
|
||||
}
|
||||
|
||||
status, err := queryTopoNVLinkStatus()
|
||||
if err != nil || len(status) == 0 {
|
||||
return topoCard("NVLink Topology", `<span class="badge badge-unknown">nvidia-smi nvlink unavailable</span>`)
|
||||
}
|
||||
errors, _ := queryTopoNVLinkErrors()
|
||||
|
||||
topoMatrix, _ := queryGPUTopologyMatrix()
|
||||
pairs := parseGPUPairAdjacency(topoMatrix)
|
||||
|
||||
var bodyB strings.Builder
|
||||
if gpuCount <= 4 && len(pairs) > 0 {
|
||||
// Small GPU count: per-pair box+line with per-link detail.
|
||||
for _, pair := range pairs {
|
||||
activeCount, total, hasError := 0, 0, false
|
||||
for _, port := range status[pair.GPUA] {
|
||||
total++
|
||||
if port.Active {
|
||||
activeCount++
|
||||
}
|
||||
}
|
||||
for _, counters := range errors[pair.GPUA] {
|
||||
if counters[0] != 0 || counters[1] != 0 || counters[2] != 0 {
|
||||
hasError = true
|
||||
}
|
||||
}
|
||||
color := "var(--ok-fg)"
|
||||
switch {
|
||||
case hasError:
|
||||
color = "var(--crit-fg)"
|
||||
case total > 0 && activeCount < total:
|
||||
color = "var(--warn-fg)"
|
||||
}
|
||||
fmt.Fprintf(&bodyB, `<div style="display:flex;align-items:center;gap:12px;margin-bottom:10px">`+
|
||||
`<div style="padding:8px 12px;border:1px solid var(--border);border-radius:6px">GPU %d</div>`+
|
||||
`<div style="flex:1;height:2px;background:%s"></div>`+
|
||||
`<div style="padding:8px 12px;border:1px solid var(--border);border-radius:6px">GPU %d</div>`+
|
||||
`<div style="font-size:12px;color:var(--muted)">%d/%d links active%s</div>`+
|
||||
`</div>`,
|
||||
pair.GPUA, color, pair.GPUB, activeCount, total, errNoteSuffix(hasError))
|
||||
}
|
||||
} else if len(pairs) > 0 {
|
||||
// Larger GPU counts (NVSwitch fabric): aggregate pair table instead of
|
||||
// an unreadable all-to-all graph.
|
||||
bodyB.WriteString(`<table><thead><tr><th>GPU A</th><th>GPU B</th><th>NVLinks</th></tr></thead><tbody>`)
|
||||
for _, pair := range pairs {
|
||||
fmt.Fprintf(&bodyB, `<tr><td>GPU %d</td><td>GPU %d</td><td>%d</td></tr>`, pair.GPUA, pair.GPUB, pair.NVLinks)
|
||||
}
|
||||
bodyB.WriteString(`</tbody></table>`)
|
||||
} else {
|
||||
bodyB.WriteString(`<span class="badge badge-unknown">No NVLink-bonded GPU pairs found</span>`)
|
||||
}
|
||||
|
||||
return topoCard("NVLink Topology", bodyB.String())
|
||||
}
|
||||
|
||||
func errNoteSuffix(hasError bool) string {
|
||||
if hasError {
|
||||
return " — errors detected"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
Reference in New Issue
Block a user