Files
bee/audit/internal/webui/page_topo.go
T
Mikhail ChusavitinandClaude Sonnet 5 2d84ddb577 webui/topo: read from techdump instead of live nvidia-smi, fix socket/NUMA mismatch, fix cross-tier SAT status merge
/topo blocked the HTTP request on live nvidia-smi calls with no timeout
(topo -m, nvlink -s/-e run on every page load), so a wedged driver hung
the page indefinitely. CaptureTechnicalDump now persists these dumps
once per audit cycle; the page reads them from techdump/ instead.

buildSocketIndex mapped NUMA node number to CPU by treating dmidecode's
Socket Designation (often 1-indexed, "CPU1"/"CPU2") as equal to the
NUMA node number (always 0-indexed) — GPUs/NICs on NUMA node 0 fell
into the "unknown" column, others attached to the wrong CPU box. Now
ranks CPUs by Socket value instead of assuming a shared numbering base.

Fixed a bug in ComponentStatusDB/applyComponentStatusDB where GPU SAT
results were keyed per-target ("pcie:gpu:nvidia-stress") instead of
per-vendor, which both broke cross-tier severity tracking (a later
clean "2. Check" run and an earlier failing "3. Load" run never
compared severities) and silently failed to match any real BDF, so the
DB overlay never reached the topology graph at all. GPU keys are now
normalized to vendor ("pcie:gpu:nvidia"/"pcie:gpu:amd"). Also skip
writing to the DB when a SAT task was aborted by the user (ctx
canceled), so a partial run can't stomp a previously recorded status.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-07-07 17:14:03 +03:00

813 lines
25 KiB
Go

package webui
import (
"encoding/json"
"fmt"
"html"
"os"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"bee/audit/internal/schema"
)
// renderTopo renders the /topo page: a read-only visualization of the server
// topology (CPU sockets, NUMA-affine PCIe devices, PSU/BMC) plus a separate
// NVLink topology card. It is pure visualization: everything it reads either
// already exists in the audit.json contract, or comes from the persisted
// techdump captured once per audit cycle (platform.CaptureTechnicalDump) —
// nothing here shells out to nvidia-smi itself, writes to
// schema.HardwarePCIeDevice or any other contract type, or talks to
// Reanimator Core.
func renderTopo(opts HandlerOptions) string {
data, err := loadSnapshot(opts.AuditPath)
if err != nil {
return topoCard("Topology", `<span class="badge badge-unknown">No audit data</span>`)
}
var ingest schema.HardwareIngestRequest
if err := json.Unmarshal(data, &ingest); err != nil {
return topoCard("Topology", `<span class="badge badge-err">Parse error</span>`)
}
hw := ingest.Hardware
var b strings.Builder
b.WriteString(renderTopoMainDiagram(hw, opts.ExportDir))
if nv := renderTopoNVLinkCard(hw, opts.ExportDir); nv != "" {
b.WriteString(nv)
}
return b.String()
}
func topoCard(title, body string) string {
return `<div class="card"><div class="card-head">` + html.EscapeString(title) + `</div><div class="card-body">` + body + `</div></div>`
}
// ---------------------------------------------------------------------------
// Classification helpers
//
// webui does not import collector (matches the existing isGPUDeviceClass
// precedent in pages.go, which already locally duplicates collector.isGPUClass
// instead of importing the package for one classifier).
// ---------------------------------------------------------------------------
// isNICDeviceClassDev mirrors the classification logic in hwDescribeNIC
// (pages.go), applied to a single device instead of aggregated counts.
func isNICDeviceClassDev(dev schema.HardwarePCIeDevice) bool {
if dev.DeviceClass != nil {
c := strings.ToLower(strings.TrimSpace(*dev.DeviceClass))
if c == "ethernetcontroller" || c == "networkcontroller" || strings.Contains(c, "fibrechannel") {
return true
}
}
return len(dev.MacAddresses) > 0
}
// isRAIDControllerClass matches the canonical class strings produced by
// collector.mapPCIeDeviceClass for RAID/storage HBAs.
func isRAIDControllerClass(class string) bool {
switch strings.TrimSpace(class) {
case "MassStorageController", "StorageController":
return true
default:
return false
}
}
// ---------------------------------------------------------------------------
// Status / link-speed coloring
// ---------------------------------------------------------------------------
// topoStatusBadgeClass maps a component's Status pointer to a badge class,
// treating a nil/absent status the same as literal "Unknown" (matches how
// the rest of the UI already renders missing status, per chipLetterClass/
// runtimeStatusBadge in pages.go).
func topoStatusBadgeClass(status *string) (label, cls string) {
if status == nil {
return "?", "badge-unknown"
}
switch strings.ToUpper(strings.TrimSpace(*status)) {
case "OK":
return "OK", "badge-ok"
case "WARNING", "WARN", "PARTIAL":
return "WARN", "badge-warn"
case "CRITICAL", "FAIL", "FAILED", "ERROR":
return "CRIT", "badge-err"
default:
return "?", "badge-unknown"
}
}
// pcieGenRank ranks a PCIe generation label ("Gen3", "Gen4", ...) for
// comparison. Mirrors collector.pcieLinkSpeedRank's ordering; duplicated
// locally rather than exported, per the same "no collector import in webui"
// convention used for isGPUDeviceClass/isRAIDControllerClass.
func pcieGenRank(gen string) int {
gen = strings.ToLower(strings.TrimSpace(gen))
gen = strings.TrimPrefix(gen, "gen")
n, err := strconv.Atoi(gen)
if err != nil {
return 0
}
return n
}
// topoEdgeColorVar computes the CPU->device edge color strictly from
// link_speed vs max_link_speed — NOT from dev.Status, since Status can also
// be overwritten by SAT/acceptance-test results on the same PCIe device,
// which would conflate "link is physically degraded" with "this GPU failed
// its stress test" into the same color.
func topoEdgeColorVar(dev schema.HardwarePCIeDevice) string {
if dev.LinkSpeed == nil || dev.MaxLinkSpeed == nil {
return "var(--muted)"
}
if pcieGenRank(*dev.LinkSpeed) < pcieGenRank(*dev.MaxLinkSpeed) {
return "var(--warn-fg)"
}
return "var(--ok-fg)"
}
// ---------------------------------------------------------------------------
// NUMA node -> CPU socket join (heuristic, no guaranteed hardware mapping)
// ---------------------------------------------------------------------------
// buildSocketIndex maps a NUMA node number to the index into cpus for the
// socket occupying that position in ascending Socket-designation order.
//
// Linux NUMA node numbering is always 0-based (node0, node1, ...), but
// dmidecode's "Socket Designation" is board-defined and frequently 1-based
// ("CPU1", "CPU2", ...). Mapping NUMA node N to the CPU whose Socket field
// equals N (as an earlier version of this function did) silently fails on
// any 1-indexed board: node 0 has no match (dropped into the "unknown"
// column) and node 1 wrongly maps to the first CPU. Ranking by Socket value
// instead assumes only that node order follows socket order — true for the
// common case of N-socket boards — without depending on the numbering base.
func buildSocketIndex(cpus []schema.HardwareCPU) map[int]int {
order := make([]int, len(cpus))
for i := range cpus {
order[i] = i
}
sort.SliceStable(order, func(a, b int) bool {
ca, cb := cpus[order[a]], cpus[order[b]]
sa, sb := 0, 0
if ca.Socket != nil {
sa = *ca.Socket
}
if cb.Socket != nil {
sb = *cb.Socket
}
return sa < sb
})
idx := map[int]int{}
for numaNode, cpuIdx := range order {
idx[numaNode] = cpuIdx
}
return idx
}
// ---------------------------------------------------------------------------
// GPU pairwise NVLink adjacency (from a live "nvidia-smi topo -m" query)
// ---------------------------------------------------------------------------
type gpuPairLink struct {
GPUA, GPUB int
NVLinks int
}
var topoNVRe = regexp.MustCompile(`(?i)^NV(\d+)$`)
// parseGPUPairAdjacency returns every GPU pair with a nonzero NVLink bond
// count from a "nvidia-smi topo -m" matrix. Unlike parseNVIDIATopologyMatrix
// (collector package, aggregate-only: min/all-active/count), this returns
// who is bonded to whom — required so GPU-GPU edges are drawn for actually
// bonded pairs, not for adjacent boxes in the layout.
func parseGPUPairAdjacency(raw string) []gpuPairLink {
lines := strings.Split(raw, "\n")
headerIdx := -1
var gpuColIndices []int
for i, line := range lines {
trimmed := strings.TrimSpace(line)
if strings.HasPrefix(trimmed, "GPU0") {
parts := strings.Fields(trimmed)
for j, col := range parts {
if strings.HasPrefix(col, "GPU") {
gpuColIndices = append(gpuColIndices, j)
}
}
if len(gpuColIndices) >= 2 {
headerIdx = i
}
break
}
}
if headerIdx < 0 {
return nil
}
colIdxToGPU := make(map[int]int, len(gpuColIndices))
for gpuIdx, colIdx := range gpuColIndices {
colIdxToGPU[colIdx] = gpuIdx
}
seen := map[[2]int]bool{}
var pairs []gpuPairLink
rowGPU := -1
for _, line := range lines[headerIdx+1:] {
trimmed := strings.TrimSpace(line)
if !strings.HasPrefix(trimmed, "GPU") {
continue
}
cells := strings.Fields(trimmed)
if len(cells) == 0 {
continue
}
rowLabel := strings.TrimPrefix(cells[0], "GPU")
n, err := strconv.Atoi(rowLabel)
if err != nil {
continue
}
rowGPU = n
for colIdx, colGPU := range colIdxToGPU {
if colGPU == rowGPU {
continue
}
dataIdx := colIdx + 1
if dataIdx >= len(cells) {
continue
}
m := topoNVRe.FindStringSubmatch(cells[dataIdx])
if len(m) != 2 {
continue
}
nv, err := strconv.Atoi(m[1])
if err != nil || nv <= 0 {
continue
}
a, bGPU := rowGPU, colGPU
if a > bGPU {
a, bGPU = bGPU, a
}
key := [2]int{a, bGPU}
if seen[key] {
continue
}
seen[key] = true
pairs = append(pairs, gpuPairLink{GPUA: a, GPUB: bGPU, NVLinks: nv})
}
}
sort.Slice(pairs, func(i, j int) bool {
if pairs[i].GPUA != pairs[j].GPUA {
return pairs[i].GPUA < pairs[j].GPUA
}
return pairs[i].GPUB < pairs[j].GPUB
})
return pairs
}
// readTopoTechDump reads a file previously captured into the persistent
// techdump directory by platform.System.CaptureTechnicalDump (run once per
// audit cycle), rather than shelling out to nvidia-smi from the HTTP request
// handler — a live call here would block page rendering on a wedged driver,
// exactly the failure mode this tool exists to diagnose.
func readTopoTechDump(exportDir, name string) (string, error) {
out, err := os.ReadFile(filepath.Join(exportDir, "techdump", name))
if err != nil {
return "", err
}
return string(out), nil
}
func readGPUTopologyMatrix(exportDir string) (string, error) {
return readTopoTechDump(exportDir, "nvidia-smi-topo.txt")
}
// readNVIDIAIndexByBDF parses the persisted nvidia-smi-query.csv techdump
// (index,pci.bus_id,...) to map PCI bus address (matching
// HardwarePCIeDevice.Slot) to the GPU index nvidia-smi/topo -m reports, so
// GPU-GPU edges (keyed by index) can be anchored to the correct box (keyed
// by BDF) in the diagram.
func readNVIDIAIndexByBDF(exportDir string) (map[string]int, error) {
raw, err := readTopoTechDump(exportDir, "nvidia-smi-query.csv")
if err != nil {
return nil, err
}
result := map[string]int{}
for _, line := range strings.Split(raw, "\n") {
line = strings.TrimSpace(line)
if line == "" {
continue
}
parts := strings.Split(line, ",")
if len(parts) < 2 {
continue
}
idx, err := strconv.Atoi(strings.TrimSpace(parts[0]))
if err != nil {
continue
}
bdf := normalizeTopoBDF(strings.TrimSpace(parts[1]))
if bdf == "" {
continue
}
result[bdf] = idx
}
return result, nil
}
// normalizeTopoBDF normalizes a PCI bus address to "dddd:bb:dd.f" form so
// nvidia-smi's "pci.bus_id" output can be matched against
// HardwarePCIeDevice.Slot regardless of minor formatting differences
// (case, leading domain padding).
func normalizeTopoBDF(bdf string) string {
bdf = strings.ToLower(strings.TrimSpace(bdf))
if bdf == "" {
return ""
}
parts := strings.Split(bdf, ":")
if len(parts) == 3 {
domain := parts[0]
if len(domain) > 4 {
domain = domain[len(domain)-4:]
}
return domain + ":" + parts[1] + ":" + parts[2]
}
return bdf
}
// ---------------------------------------------------------------------------
// Main topology diagram
// ---------------------------------------------------------------------------
const (
topoColWidth = 220
topoBoxWidth = 190
topoBoxHeight = 56
topoDeviceGap = 14
topoTopMargin = 30
topoEdgeBand = 60
topoBottomRowH = 90
)
type topoBox struct {
x, y, w, h int
label string
sublabel string
badgeText string
badgeCls string
detailType string // "" = not clickable
}
func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string {
socketIdx := buildSocketIndex(hw.CPUs)
numCols := len(hw.CPUs)
if numCols == 0 {
numCols = 1
}
unknownCol := numCols // extra trailing column for unmatched devices
// Group PCIe devices (GPU/NIC/RAID only — matches the mockup's node
// types) into columns by NUMA node, falling back to the "unknown" bucket.
type placedDevice struct {
dev schema.HardwarePCIeDevice
kind string // "gpu", "nic", "raid"
col int
bdf string
}
var placed []placedDevice
for _, dev := range hw.PCIeDevices {
var kind string
switch {
case dev.DeviceClass != nil && isGPUDeviceClass(*dev.DeviceClass):
kind = "gpu"
case isNICDeviceClassDev(dev):
kind = "nic"
case dev.DeviceClass != nil && isRAIDControllerClass(*dev.DeviceClass):
kind = "raid"
default:
continue
}
col := unknownCol
if dev.NUMANode != nil {
if ci, ok := socketIdx[*dev.NUMANode]; ok {
col = ci
}
}
bdf := ""
if dev.Slot != nil {
bdf = normalizeTopoBDF(*dev.Slot)
} else if dev.BDF != nil {
bdf = normalizeTopoBDF(*dev.BDF)
}
placed = append(placed, placedDevice{dev: dev, kind: kind, col: col, bdf: bdf})
}
hasUnknownCol := false
for _, p := range placed {
if p.col == unknownCol {
hasUnknownCol = true
break
}
}
totalCols := numCols
if hasUnknownCol {
totalCols++
}
// GPU index<->BDF map + pairwise NVLink adjacency, read from the
// persisted techdump captured during the last audit cycle, best-effort:
// if the dump is missing (older audit, no NVIDIA GPUs), GPU-GPU edges are
// simply omitted.
bdfToIndex, _ := readNVIDIAIndexByBDF(exportDir)
var pairs []gpuPairLink
if topoMatrix, err := readGPUTopologyMatrix(exportDir); err == nil {
pairs = parseGPUPairAdjacency(topoMatrix)
}
var boxes []topoBox
var pcieEdges []struct {
x1, y1, x2, y2 int
color string
}
gpuBoxCenter := map[int][2]int{} // gpu index -> (x, yBottom)
gpuBoxIndex := map[int]int{} // gpu index -> index into boxes
gpuNUMANode := map[int]*int{} // gpu index -> its PCIe device's numa_node
for col := 0; col < totalCols; col++ {
colX := (col+1)*24 + col*topoColWidth
if col < numCols {
cpu := hw.CPUs[col]
model := ""
if cpu.Model != nil {
model = *cpu.Model
}
socket := col
if cpu.Socket != nil {
socket = *cpu.Socket
}
label, cls := topoStatusBadgeClass(cpu.Status)
boxes = append(boxes, topoBox{
x: colX, y: topoTopMargin, w: topoBoxWidth, h: topoBoxHeight,
label: fmt.Sprintf("CPU %d", socket), sublabel: model,
badgeText: label, badgeCls: cls, detailType: "cpu",
})
}
y := topoTopMargin + topoBoxHeight + topoDeviceGap*2
for _, p := range placed {
if p.col != col {
continue
}
label, cls := topoStatusBadgeClass(p.dev.Status)
model := ""
if p.dev.Model != nil {
model = *p.dev.Model
}
box := topoBox{
x: colX, y: y, w: topoBoxWidth, h: topoBoxHeight,
label: strings.ToUpper(p.kind), sublabel: model,
badgeText: label, badgeCls: cls, detailType: p.kind,
}
boxes = append(boxes, box)
boxIdx := len(boxes) - 1
if col < numCols {
pcieEdges = append(pcieEdges, struct {
x1, y1, x2, y2 int
color string
}{
x1: colX + topoBoxWidth/2, y1: topoTopMargin + topoBoxHeight,
x2: colX + topoBoxWidth/2, y2: y,
color: topoEdgeColorVar(p.dev),
})
}
if p.kind == "gpu" && p.bdf != "" {
if idx, ok := bdfToIndex[p.bdf]; ok {
gpuBoxCenter[idx] = [2]int{colX + topoBoxWidth/2, y + topoBoxHeight}
gpuBoxIndex[idx] = boxIdx
gpuNUMANode[idx] = p.dev.NUMANode
}
}
y += topoBoxHeight + topoDeviceGap
}
}
maxDeviceY := topoTopMargin + topoBoxHeight + topoDeviceGap*2
for _, b := range boxes {
if b.y+b.h > maxDeviceY {
maxDeviceY = b.y + b.h
}
}
// GPU-GPU NVLink edges: drawn as an elbow connector through a dedicated
// band below the device row, kept strictly separate from the vertical
// CPU->device PCIe edges above so neither visually obscures the other.
//
// A bonded pair spanning two different NUMA nodes is treated as an
// anomaly (not a neutral fact) per project decision: we expect a bonded
// pair to sit on one NUMA node, so a cross-NUMA bond is flagged Warning
// on the edge AND on both GPU boxes, regardless of their own SAT status.
bandY := maxDeviceY + topoEdgeBand/2
var gpuEdgesSVG strings.Builder
for _, pair := range pairs {
c1, ok1 := gpuBoxCenter[pair.GPUA]
c2, ok2 := gpuBoxCenter[pair.GPUB]
if !ok1 || !ok2 {
continue
}
color := "var(--ok-fg)"
title := fmt.Sprintf("NVLink: GPU%d↔GPU%d (%d links)", pair.GPUA, pair.GPUB, pair.NVLinks)
numaA, numaB := gpuNUMANode[pair.GPUA], gpuNUMANode[pair.GPUB]
if numaA == nil || numaB == nil {
color = "var(--muted)"
} else if *numaA != *numaB {
color = "var(--warn-fg)"
title += " — spans NUMA nodes (unexpected)"
upgradeTopoBoxBadgeToWarn(boxes, gpuBoxIndex[pair.GPUA])
upgradeTopoBoxBadgeToWarn(boxes, gpuBoxIndex[pair.GPUB])
}
fmt.Fprintf(&gpuEdgesSVG,
`<path d="M %d %d L %d %d L %d %d L %d %d" style="fill:none;stroke:%s;stroke-width:2;stroke-dasharray:4,3"><title>%s</title></path>`+"\n",
c1[0], c1[1], c1[0], bandY, c2[0], bandY, c2[0], c2[1], color, html.EscapeString(title))
}
svgHeight := bandY + topoEdgeBand/2 + topoBottomRowH
svgWidth := totalCols*topoColWidth + 48
var b strings.Builder
fmt.Fprintf(&b, `<svg width="%d" height="%d" viewBox="0 0 %d %d" style="max-width:100%%">`+"\n", svgWidth, svgHeight, svgWidth, svgHeight)
for _, e := range pcieEdges {
fmt.Fprintf(&b, `<line x1="%d" y1="%d" x2="%d" y2="%d" style="stroke:%s;stroke-width:2"/>`+"\n", e.x1, e.y1, e.x2, e.y2, e.color)
}
b.WriteString(gpuEdgesSVG.String())
for _, box := range boxes {
writeTopoBoxSVG(&b, box)
}
// PSU / BMC row: standalone boxes, no connecting lines.
psuY := svgHeight - topoBottomRowH + 20
psuX := 24
for _, psu := range hw.PowerSupplies {
label, cls := topoStatusBadgeClass(psu.Status)
slot := ""
if psu.Slot != nil {
slot = *psu.Slot
}
watt := ""
if psu.WattageW != nil {
watt = fmt.Sprintf("%dW", *psu.WattageW)
}
writeTopoBoxSVG(&b, topoBox{
x: psuX, y: psuY, w: 150, h: topoBoxHeight,
label: "PSU " + slot, sublabel: watt,
badgeText: label, badgeCls: cls, detailType: "psu",
})
psuX += 150 + topoDeviceGap
}
if bmcVersion, ok := findBMCFirmware(hw.Firmware); ok {
writeTopoBoxSVG(&b, topoBox{
x: svgWidth - 200 - 24, y: psuY, w: 200, h: topoBoxHeight,
label: "BMC", sublabel: "fw " + bmcVersion,
badgeText: "?", badgeCls: "badge-unknown",
})
}
b.WriteString(`</svg>`)
return topoCard("Topology", b.String())
}
// upgradeTopoBoxBadgeToWarn upgrades a box's badge to Warning unless it is
// already at Critical severity (never downgrades a worse status).
func upgradeTopoBoxBadgeToWarn(boxes []topoBox, boxIdx int) {
if boxIdx < 0 || boxIdx >= len(boxes) {
return
}
if boxes[boxIdx].badgeCls == "badge-err" {
return
}
boxes[boxIdx].badgeText = "WARN"
boxes[boxIdx].badgeCls = "badge-warn"
}
func findBMCFirmware(records []schema.HardwareFirmwareRecord) (string, bool) {
for _, rec := range records {
if strings.EqualFold(strings.TrimSpace(rec.DeviceName), "BMC") {
return rec.Version, true
}
}
return "", false
}
func writeTopoBoxSVG(b *strings.Builder, box topoBox) {
onclick := ""
cursor := "default"
if box.detailType != "" {
onclick = fmt.Sprintf(` onclick="openComponentDetail('%s')"`, box.detailType)
cursor = "pointer"
}
fmt.Fprintf(b, `<g%s style="cursor:%s">`, onclick, cursor)
fmt.Fprintf(b, `<rect x="%d" y="%d" width="%d" height="%d" rx="6" ry="6" style="fill:var(--surface);stroke:var(--border)"/>`+"\n",
box.x, box.y, box.w, box.h)
fmt.Fprintf(b, `<text x="%d" y="%d" style="fill:var(--ink,#000);font-size:13px;font-weight:700">%s</text>`+"\n",
box.x+10, box.y+20, html.EscapeString(box.label))
if box.sublabel != "" {
fmt.Fprintf(b, `<text x="%d" y="%d" style="fill:var(--muted);font-size:11px">%s</text>`+"\n",
box.x+10, box.y+36, html.EscapeString(truncateTopoLabel(box.sublabel, 26)))
}
fmt.Fprintf(b, `<rect x="%d" y="%d" width="46" height="18" rx="3" ry="3" class="%s"/>`+"\n",
box.x+box.w-54, box.y+box.h-26, box.badgeCls)
fmt.Fprintf(b, `<text x="%d" y="%d" style="font-size:10px;font-weight:700" class="%s">%s</text>`+"\n",
box.x+box.w-49, box.y+box.h-13, box.badgeCls, html.EscapeString(box.badgeText))
b.WriteString(`</g>` + "\n")
}
func truncateTopoLabel(s string, max int) string {
if len(s) <= max {
return s
}
if max <= 1 {
return s[:max]
}
return s[:max-1] + "…"
}
// ---------------------------------------------------------------------------
// Separate NVLink topology card (read from techdump, not written to any
// ingest contract)
// ---------------------------------------------------------------------------
type topoNVLinkPort struct {
Index int
Active bool
SpeedGBs *float64
ReplayErrors int64
RecoveryErrors int64
CRCErrors int64
}
var (
topoNVLinkGPUHeaderRe = regexp.MustCompile(`^GPU (\d+):`)
topoNVLinkSpeedLineRe = regexp.MustCompile(`^Link (\d+):\s*([\d.]+)\s*GB/s`)
topoNVLinkInactiveRe = regexp.MustCompile(`^Link (\d+):\s*<inactive>`)
topoNVLinkErrorLineRe = regexp.MustCompile(`^Link (\d+):\s*(Replay|Recovery|CRC) Errors:\s*(\d+)`)
)
func readTopoNVLinkStatus(exportDir string) (map[int][]topoNVLinkPort, error) {
raw, err := readTopoTechDump(exportDir, "nvidia-smi-nvlink-status.txt")
if err != nil {
return nil, err
}
return parseTopoNVLinkStatus(raw), nil
}
func parseTopoNVLinkStatus(raw string) map[int][]topoNVLinkPort {
result := map[int][]topoNVLinkPort{}
currentGPU := -1
for _, line := range strings.Split(raw, "\n") {
trimmed := strings.TrimSpace(line)
if m := topoNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
currentGPU, _ = strconv.Atoi(m[1])
continue
}
if currentGPU < 0 {
continue
}
if m := topoNVLinkInactiveRe.FindStringSubmatch(trimmed); m != nil {
idx, _ := strconv.Atoi(m[1])
result[currentGPU] = append(result[currentGPU], topoNVLinkPort{Index: idx, Active: false})
continue
}
if m := topoNVLinkSpeedLineRe.FindStringSubmatch(trimmed); m != nil {
idx, _ := strconv.Atoi(m[1])
port := topoNVLinkPort{Index: idx, Active: true}
if speed, err := strconv.ParseFloat(m[2], 64); err == nil {
port.SpeedGBs = &speed
}
result[currentGPU] = append(result[currentGPU], port)
}
}
return result
}
func readTopoNVLinkErrors(exportDir string) (map[int]map[int][3]int64, error) {
raw, err := readTopoTechDump(exportDir, "nvidia-smi-nvlink-errors.txt")
if err != nil {
return nil, err
}
return parseTopoNVLinkErrors(raw), nil
}
// parseTopoNVLinkErrors returns, per GPU then link index, [replay, recovery, crc].
func parseTopoNVLinkErrors(raw string) map[int]map[int][3]int64 {
result := map[int]map[int][3]int64{}
currentGPU := -1
for _, line := range strings.Split(raw, "\n") {
trimmed := strings.TrimSpace(line)
if m := topoNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
currentGPU, _ = strconv.Atoi(m[1])
continue
}
if currentGPU < 0 {
continue
}
m := topoNVLinkErrorLineRe.FindStringSubmatch(trimmed)
if m == nil {
continue
}
linkIdx, _ := strconv.Atoi(m[1])
count, _ := strconv.ParseInt(m[3], 10, 64)
if result[currentGPU] == nil {
result[currentGPU] = map[int][3]int64{}
}
c := result[currentGPU][linkIdx]
switch m[2] {
case "Replay":
c[0] = count
case "Recovery":
c[1] = count
case "CRC":
c[2] = count
}
result[currentGPU][linkIdx] = c
}
return result
}
// renderTopoNVLinkCard renders the separate NVLink topology card. Returns ""
// if there are fewer than 2 NVIDIA GPUs, or the nvidia-smi nvlink techdump
// wasn't captured (older audit, or nvidia-smi unavailable on that run).
func renderTopoNVLinkCard(hw schema.HardwareSnapshot, exportDir string) string {
gpuCount := 0
for _, dev := range hw.PCIeDevices {
if dev.DeviceClass != nil && isGPUDeviceClass(*dev.DeviceClass) {
gpuCount++
}
}
if gpuCount < 2 {
return ""
}
status, err := readTopoNVLinkStatus(exportDir)
if err != nil || len(status) == 0 {
return topoCard("NVLink Topology", `<span class="badge badge-unknown">nvidia-smi nvlink data unavailable</span>`)
}
errors, _ := readTopoNVLinkErrors(exportDir)
topoMatrix, _ := readGPUTopologyMatrix(exportDir)
pairs := parseGPUPairAdjacency(topoMatrix)
var bodyB strings.Builder
if gpuCount <= 4 && len(pairs) > 0 {
// Small GPU count: per-pair box+line with per-link detail.
for _, pair := range pairs {
activeCount, total, hasError := 0, 0, false
for _, port := range status[pair.GPUA] {
total++
if port.Active {
activeCount++
}
}
for _, counters := range errors[pair.GPUA] {
if counters[0] != 0 || counters[1] != 0 || counters[2] != 0 {
hasError = true
}
}
color := "var(--ok-fg)"
switch {
case hasError:
color = "var(--crit-fg)"
case total > 0 && activeCount < total:
color = "var(--warn-fg)"
}
fmt.Fprintf(&bodyB, `<div style="display:flex;align-items:center;gap:12px;margin-bottom:10px">`+
`<div style="padding:8px 12px;border:1px solid var(--border);border-radius:6px">GPU %d</div>`+
`<div style="flex:1;height:2px;background:%s"></div>`+
`<div style="padding:8px 12px;border:1px solid var(--border);border-radius:6px">GPU %d</div>`+
`<div style="font-size:12px;color:var(--muted)">%d/%d links active%s</div>`+
`</div>`,
pair.GPUA, color, pair.GPUB, activeCount, total, errNoteSuffix(hasError))
}
} else if len(pairs) > 0 {
// Larger GPU counts (NVSwitch fabric): aggregate pair table instead of
// an unreadable all-to-all graph.
bodyB.WriteString(`<table><thead><tr><th>GPU A</th><th>GPU B</th><th>NVLinks</th></tr></thead><tbody>`)
for _, pair := range pairs {
fmt.Fprintf(&bodyB, `<tr><td>GPU %d</td><td>GPU %d</td><td>%d</td></tr>`, pair.GPUA, pair.GPUB, pair.NVLinks)
}
bodyB.WriteString(`</tbody></table>`)
} else {
bodyB.WriteString(`<span class="badge badge-unknown">No NVLink-bonded GPU pairs found</span>`)
}
return topoCard("NVLink Topology", bodyB.String())
}
func errNoteSuffix(hasError bool) string {
if hasError {
return " — errors detected"
}
return ""
}