Rework the /topo diagram from side-by-side stacked cards into one tall vertical bar per CPU socket with everything attached to it branching off sideways (socket 0 left/branches right, socket 1 right/branches left). Disks are now parented under the storage controller they physically hang off (SATA/AHCI, SAS HBA, RAID) — itself a NUMA-affine PCIe device under one socket — instead of a synthetic catch-all node. The disk->controller link is read from a new storage-controllers.txt techdump (platform.StorageControllerMapScript, a /sys/block walk); disks with no resolvable controller fall back to an "Other" bar. No board/root node. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SYjrG6bVmeG1Z2Wmc8kg3o
495 lines
16 KiB
Go
495 lines
16 KiB
Go
package webui
|
||
|
||
import (
|
||
"encoding/json"
|
||
"fmt"
|
||
"html"
|
||
"os"
|
||
"path/filepath"
|
||
"regexp"
|
||
"sort"
|
||
"strconv"
|
||
"strings"
|
||
|
||
"bee/audit/internal/platform"
|
||
"bee/audit/internal/schema"
|
||
)
|
||
|
||
// renderTopo renders the /topo page: a read-only visualization of the server
|
||
// topology (CPU sockets, NUMA-affine PCIe devices, PSU/BMC) plus a separate
|
||
// NVLink topology card. It is pure visualization: everything it reads either
|
||
// already exists in the audit.json contract, or comes from the persisted
|
||
// techdump captured once per audit cycle (platform.CaptureTechnicalDump) —
|
||
// nothing here shells out to nvidia-smi itself, writes to
|
||
// schema.HardwarePCIeDevice or any other contract type, or talks to
|
||
// Reanimator Core.
|
||
func renderTopo(opts HandlerOptions) string {
|
||
data, err := loadSnapshot(opts.AuditPath)
|
||
if err != nil {
|
||
return topoCard("Topology", `<span class="badge badge-unknown">No audit data</span>`)
|
||
}
|
||
var ingest schema.HardwareIngestRequest
|
||
if err := json.Unmarshal(data, &ingest); err != nil {
|
||
return topoCard("Topology", `<span class="badge badge-err">Parse error</span>`)
|
||
}
|
||
hw := ingest.Hardware
|
||
|
||
var b strings.Builder
|
||
b.WriteString(renderTopoMainDiagram(hw, opts.ExportDir))
|
||
if nv := renderTopoNVLinkCard(hw, opts.ExportDir); nv != "" {
|
||
b.WriteString(nv)
|
||
}
|
||
return b.String()
|
||
}
|
||
|
||
func topoCard(title, body string) string {
|
||
return `<div class="card"><div class="card-head">` + html.EscapeString(title) + `</div><div class="card-body">` + body + `</div></div>`
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Classification helpers
|
||
//
|
||
// webui does not import collector (matches the existing isGPUDeviceClass
|
||
// precedent in pages.go, which already locally duplicates collector.isGPUClass
|
||
// instead of importing the package for one classifier).
|
||
// ---------------------------------------------------------------------------
|
||
|
||
// isNICDeviceClassDev applies isNICDeviceClass (pages.go) to a single device,
|
||
// with a MAC-address fallback for devices lspci doesn't classify as NIC.
|
||
func isNICDeviceClassDev(dev schema.HardwarePCIeDevice) bool {
|
||
if dev.DeviceClass != nil && isNICDeviceClass(*dev.DeviceClass) {
|
||
return true
|
||
}
|
||
return len(dev.MacAddresses) > 0
|
||
}
|
||
|
||
// isRAIDControllerClass matches the canonical class strings produced by
|
||
// collector.mapPCIeDeviceClass for RAID/storage HBAs.
|
||
func isRAIDControllerClass(class string) bool {
|
||
switch strings.TrimSpace(class) {
|
||
case "MassStorageController", "StorageController":
|
||
return true
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Status / link-speed coloring
|
||
// ---------------------------------------------------------------------------
|
||
|
||
// pcieGenRank ranks a PCIe generation label ("Gen3", "Gen4", ...) for
|
||
// comparison. Mirrors collector.pcieLinkSpeedRank's ordering; duplicated
|
||
// locally rather than exported, per the same "no collector import in webui"
|
||
// convention used for isGPUDeviceClass/isRAIDControllerClass.
|
||
func pcieGenRank(gen string) int {
|
||
gen = strings.ToLower(strings.TrimSpace(gen))
|
||
gen = strings.TrimPrefix(gen, "gen")
|
||
n, err := strconv.Atoi(gen)
|
||
if err != nil {
|
||
return 0
|
||
}
|
||
return n
|
||
}
|
||
|
||
// topoEdgeColorVar computes the CPU->device edge color strictly from
|
||
// link_speed vs max_link_speed — NOT from dev.Status, since Status can also
|
||
// be overwritten by SAT/acceptance-test results on the same PCIe device,
|
||
// which would conflate "link is physically degraded" with "this GPU failed
|
||
// its stress test" into the same color.
|
||
func topoEdgeColorVar(dev schema.HardwarePCIeDevice) string {
|
||
if dev.LinkSpeed == nil || dev.MaxLinkSpeed == nil {
|
||
return "var(--muted)"
|
||
}
|
||
if pcieGenRank(*dev.LinkSpeed) < pcieGenRank(*dev.MaxLinkSpeed) {
|
||
return "var(--warn-fg)"
|
||
}
|
||
return "var(--ok-fg)"
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// NUMA node -> CPU socket join (heuristic, no guaranteed hardware mapping)
|
||
// ---------------------------------------------------------------------------
|
||
|
||
// buildSocketIndex maps a NUMA node number to the index into cpus for the
|
||
// socket occupying that position in ascending Socket-designation order.
|
||
//
|
||
// Linux NUMA node numbering is always 0-based (node0, node1, ...), but
|
||
// dmidecode's "Socket Designation" is board-defined and frequently 1-based
|
||
// ("CPU1", "CPU2", ...). Mapping NUMA node N to the CPU whose Socket field
|
||
// equals N (as an earlier version of this function did) silently fails on
|
||
// any 1-indexed board: node 0 has no match (dropped into the "unknown"
|
||
// column) and node 1 wrongly maps to the first CPU. Ranking by Socket value
|
||
// instead assumes only that node order follows socket order — true for the
|
||
// common case of N-socket boards — without depending on the numbering base.
|
||
func buildSocketIndex(cpus []schema.HardwareCPU) map[int]int {
|
||
order := make([]int, len(cpus))
|
||
for i := range cpus {
|
||
order[i] = i
|
||
}
|
||
sort.SliceStable(order, func(a, b int) bool {
|
||
ca, cb := cpus[order[a]], cpus[order[b]]
|
||
sa, sb := 0, 0
|
||
if ca.Socket != nil {
|
||
sa = *ca.Socket
|
||
}
|
||
if cb.Socket != nil {
|
||
sb = *cb.Socket
|
||
}
|
||
return sa < sb
|
||
})
|
||
idx := map[int]int{}
|
||
for numaNode, cpuIdx := range order {
|
||
idx[numaNode] = cpuIdx
|
||
}
|
||
return idx
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// DIMM -> CPU column attachment
|
||
//
|
||
// schema.HardwareMemory has no NUMANode field (unlike HardwarePCIeDevice), so
|
||
// unlike the GPU/NIC/RAID placement above, a DIMM's CPU affinity has to be
|
||
// read out of its own Locator/Bank Locator strings — DMI type 17 gives no
|
||
// other hint. Both patterns below have been observed on real boards.
|
||
// ---------------------------------------------------------------------------
|
||
|
||
var (
|
||
topoMemCPULocatorRe = regexp.MustCompile(`(?i)^cpu\s*0*(\d+)`)
|
||
topoMemBankNodeRe = regexp.MustCompile(`(?i)node\s*0*(\d+)`)
|
||
)
|
||
|
||
// parseDIMMBankLocatorNodes maps a DIMM's Locator (matches
|
||
// schema.HardwareMemory.Slot) to the node number embedded in its Bank
|
||
// Locator field (e.g. "_Node1_Channel0_Dimm0"), read from a raw
|
||
// "dmidecode -t 17" techdump capture. Bank Locator never reaches audit.json
|
||
// (schema.HardwareMemory.Location is json:"-", used only for internal DIMM
|
||
// telemetry matching), so boards whose Locator has no CPU number of its own
|
||
// (e.g. "DIMM000(A)" rather than "CPU0_DIMM_A1") need this fallback to
|
||
// attach a DIMM to a CPU column at all — matches the existing convention of
|
||
// reading extra techdump for this page's visualization only (see
|
||
// readTopoTechDump).
|
||
func parseDIMMBankLocatorNodes(raw string) map[string]int {
|
||
result := map[string]int{}
|
||
for _, sec := range strings.Split(raw, "Memory Device") {
|
||
var locator string
|
||
node := -1
|
||
for _, line := range strings.Split(sec, "\n") {
|
||
trimmed := strings.TrimSpace(line)
|
||
if v, ok := strings.CutPrefix(trimmed, "Locator:"); ok {
|
||
locator = strings.TrimSpace(v)
|
||
}
|
||
if v, ok := strings.CutPrefix(trimmed, "Bank Locator:"); ok {
|
||
if m := topoMemBankNodeRe.FindStringSubmatch(v); m != nil {
|
||
if n, err := strconv.Atoi(m[1]); err == nil {
|
||
node = n
|
||
}
|
||
}
|
||
}
|
||
}
|
||
if locator != "" && node >= 0 {
|
||
result[locator] = node
|
||
}
|
||
}
|
||
return result
|
||
}
|
||
|
||
// dimmRawNode returns the raw (vendor-numbered, not yet column-mapped)
|
||
// CPU/NUMA node number for a DIMM, trying two heuristics in order:
|
||
// 1. A CPU number encoded directly in the Locator itself, e.g.
|
||
// "CPU0_DIMM_A1".
|
||
// 2. A node number from the Bank Locator via parseDIMMBankLocatorNodes,
|
||
// e.g. Locator "DIMM000(A)" whose Bank Locator is
|
||
// "_Node1_Channel0_Dimm0" -> 1.
|
||
//
|
||
// ok=false means neither pattern matched, so this DIMM can't be confidently
|
||
// attached to a CPU column and falls back to the unattached Memory row.
|
||
func dimmRawNode(mem schema.HardwareMemory, bankNodeByLocator map[string]int) (int, bool) {
|
||
if mem.Slot == nil {
|
||
return 0, false
|
||
}
|
||
if m := topoMemCPULocatorRe.FindStringSubmatch(*mem.Slot); m != nil {
|
||
if n, err := strconv.Atoi(m[1]); err == nil {
|
||
return n, true
|
||
}
|
||
}
|
||
if n, ok := bankNodeByLocator[*mem.Slot]; ok {
|
||
return n, true
|
||
}
|
||
return 0, false
|
||
}
|
||
|
||
// buildMemoryColumnIndex ranks the distinct raw node numbers seen across all
|
||
// DIMMs and maps the i-th smallest to column i — the same "node order
|
||
// follows socket order" assumption buildSocketIndex makes for PCIe NUMA
|
||
// nodes, but computed independently from PCIe's own numbering: PCIe's
|
||
// NUMANode is 0-based Linux numbering, while a DIMM's Bank Locator "NodeN"
|
||
// has been observed 1-based on at least one real board, so the two node
|
||
// spaces are not guaranteed to share a base.
|
||
func buildMemoryColumnIndex(rawNodes []int) map[int]int {
|
||
seen := map[int]bool{}
|
||
var distinct []int
|
||
for _, n := range rawNodes {
|
||
if !seen[n] {
|
||
seen[n] = true
|
||
distinct = append(distinct, n)
|
||
}
|
||
}
|
||
sort.Ints(distinct)
|
||
idx := map[int]int{}
|
||
for col, n := range distinct {
|
||
idx[n] = col
|
||
}
|
||
return idx
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// GPU pairwise NVLink adjacency (from a live "nvidia-smi topo -m" query)
|
||
// ---------------------------------------------------------------------------
|
||
|
||
type gpuPairLink = platform.NvidiaNVLinkBondedPair
|
||
|
||
// parseGPUPairAdjacency returns every GPU pair with a nonzero NVLink bond
|
||
// count from a "nvidia-smi topo -m" matrix. Unlike parseNVIDIATopologyMatrix
|
||
// (collector package, aggregate-only: min/all-active/count), this returns
|
||
// who is bonded to whom — required so GPU-GPU edges are drawn for actually
|
||
// bonded pairs, not for adjacent boxes in the layout.
|
||
func parseGPUPairAdjacency(raw string) []gpuPairLink {
|
||
return platform.ParseNvidiaNVLinkBondedPairs(raw)
|
||
}
|
||
|
||
// readTopoTechDump reads a file previously captured into the persistent
|
||
// techdump directory by platform.System.CaptureTechnicalDump (run once per
|
||
// audit cycle), rather than shelling out to nvidia-smi from the HTTP request
|
||
// handler — a live call here would block page rendering on a wedged driver,
|
||
// exactly the failure mode this tool exists to diagnose.
|
||
func readTopoTechDump(exportDir, name string) (string, error) {
|
||
out, err := os.ReadFile(filepath.Join(exportDir, "techdump", name))
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
return string(out), nil
|
||
}
|
||
|
||
func readGPUTopologyMatrix(exportDir string) (string, error) {
|
||
return readTopoTechDump(exportDir, "nvidia-smi-topo.txt")
|
||
}
|
||
|
||
// readNVIDIAIndexByBDF parses the persisted nvidia-smi-query.csv techdump
|
||
// (index,pci.bus_id,...) to map PCI bus address (matching
|
||
// HardwarePCIeDevice.Slot) to the GPU index nvidia-smi/topo -m reports, so
|
||
// GPU-GPU edges (keyed by index) can be anchored to the correct box (keyed
|
||
// by BDF) in the diagram.
|
||
func readNVIDIAIndexByBDF(exportDir string) (map[string]int, error) {
|
||
raw, err := readTopoTechDump(exportDir, "nvidia-smi-query.csv")
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
result := map[string]int{}
|
||
for _, line := range strings.Split(raw, "\n") {
|
||
line = strings.TrimSpace(line)
|
||
if line == "" {
|
||
continue
|
||
}
|
||
parts := strings.Split(line, ",")
|
||
if len(parts) < 2 {
|
||
continue
|
||
}
|
||
idx, err := strconv.Atoi(strings.TrimSpace(parts[0]))
|
||
if err != nil {
|
||
continue
|
||
}
|
||
bdf := normalizeTopoBDF(strings.TrimSpace(parts[1]))
|
||
if bdf == "" {
|
||
continue
|
||
}
|
||
result[bdf] = idx
|
||
}
|
||
return result, nil
|
||
}
|
||
|
||
// normalizeTopoBDF normalizes a PCI bus address to "dddd:bb:dd.f" form so
|
||
// nvidia-smi's "pci.bus_id" output can be matched against
|
||
// HardwarePCIeDevice.Slot regardless of minor formatting differences
|
||
// (case, leading domain padding).
|
||
func normalizeTopoBDF(bdf string) string {
|
||
bdf = strings.ToLower(strings.TrimSpace(bdf))
|
||
if bdf == "" {
|
||
return ""
|
||
}
|
||
parts := strings.Split(bdf, ":")
|
||
if len(parts) == 3 {
|
||
domain := parts[0]
|
||
if len(domain) > 4 {
|
||
domain = domain[len(domain)-4:]
|
||
}
|
||
return domain + ":" + parts[1] + ":" + parts[2]
|
||
}
|
||
return bdf
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Card status aggregation
|
||
//
|
||
// Every card on this page — whether it represents one component (CPU 1) or a
|
||
// group of identical ones (GPU ×4) — is colored as a whole by its worst
|
||
// observed status, with a plain-text summary as the card's last line
|
||
// (e.g. "4 OK" or "3 OK, 1 Warning"). There is no separate status chip: a
|
||
// chip needs its own fill, and the SVG boxes previously colored that chip
|
||
// via CSS classes written for HTML (.badge-ok sets `background`/`color`,
|
||
// which do nothing on an SVG <rect>/<text> — only `fill` does), so every
|
||
// chip rendered with the SVG default fill of solid black. Coloring the card
|
||
// itself uses real `fill:var(--ok-bg)` etc. declarations, which sidesteps
|
||
// that class entirely.
|
||
// ---------------------------------------------------------------------------
|
||
|
||
// classifyTopoSeverity converts a component's Status pointer to a severity
|
||
// rank (0=unknown, 1=OK, 2=Warning, 3=Critical), treating nil/unrecognized
|
||
// the same as "Unknown" — matches topoStatusBadgeClass's classification.
|
||
func classifyTopoSeverity(status *string) int {
|
||
if status == nil {
|
||
return 0
|
||
}
|
||
switch strings.ToUpper(strings.TrimSpace(*status)) {
|
||
case "OK":
|
||
return 1
|
||
case "WARNING", "WARN", "PARTIAL":
|
||
return 2
|
||
case "CRITICAL", "FAIL", "FAILED", "ERROR":
|
||
return 3
|
||
default:
|
||
return 0
|
||
}
|
||
}
|
||
|
||
// topoSeverityColors returns the (fill, stroke, text) CSS var() triple a
|
||
// whole card is painted with for a given worst-observed severity.
|
||
func topoSeverityColors(sev int) (fill, stroke, text string) {
|
||
switch sev {
|
||
case 3:
|
||
return "var(--crit-bg)", "var(--crit-border)", "var(--crit-fg)"
|
||
case 2:
|
||
return "var(--warn-bg)", "#c9ba9b", "var(--warn-fg)"
|
||
case 1:
|
||
return "var(--ok-bg)", "#a3c293", "var(--ok-fg)"
|
||
default:
|
||
return "var(--surface-2)", "var(--border)", "var(--muted)"
|
||
}
|
||
}
|
||
|
||
// topoStatusTally counts how many components in a group fall into each
|
||
// severity bucket, so a group card can report "3 OK, 1 Warning" rather than
|
||
// collapsing to a single worst-of value and losing the rest.
|
||
type topoStatusTally struct {
|
||
unknown, ok, warn, crit int
|
||
}
|
||
|
||
func (t *topoStatusTally) add(sev int) {
|
||
switch sev {
|
||
case 3:
|
||
t.crit++
|
||
case 2:
|
||
t.warn++
|
||
case 1:
|
||
t.ok++
|
||
default:
|
||
t.unknown++
|
||
}
|
||
}
|
||
|
||
func (t topoStatusTally) total() int { return t.unknown + t.ok + t.warn + t.crit }
|
||
|
||
func (t topoStatusTally) worst() int {
|
||
switch {
|
||
case t.crit > 0:
|
||
return 3
|
||
case t.warn > 0:
|
||
return 2
|
||
case t.ok > 0:
|
||
return 1
|
||
default:
|
||
return 0
|
||
}
|
||
}
|
||
|
||
// line renders the card's last-line status summary.
|
||
func (t topoStatusTally) line() string {
|
||
if t.total() == 0 {
|
||
return "No data"
|
||
}
|
||
if t.total() == 1 {
|
||
switch {
|
||
case t.crit > 0:
|
||
return "Critical"
|
||
case t.warn > 0:
|
||
return "Warning"
|
||
case t.ok > 0:
|
||
return "OK"
|
||
default:
|
||
return "Unknown"
|
||
}
|
||
}
|
||
var parts []string
|
||
if t.crit > 0 {
|
||
parts = append(parts, fmt.Sprintf("%d Critical", t.crit))
|
||
}
|
||
if t.warn > 0 {
|
||
parts = append(parts, fmt.Sprintf("%d Warning", t.warn))
|
||
}
|
||
if t.ok > 0 {
|
||
parts = append(parts, fmt.Sprintf("%d OK", t.ok))
|
||
}
|
||
if t.unknown > 0 {
|
||
parts = append(parts, fmt.Sprintf("%d Unknown", t.unknown))
|
||
}
|
||
return strings.Join(parts, ", ")
|
||
}
|
||
|
||
// topoCardInfo is the shared visual content for one card, rendered either as
|
||
// an absolutely-positioned SVG box (main diagram) or an HTML flex item
|
||
// (Memory/Power Supplies rows) by the two writers below.
|
||
type topoCardInfo struct {
|
||
label string // e.g. "CPU 1", "GPU", "Power Supplies"
|
||
sublabel string // representative model/description, "" to omit
|
||
count int // components represented by this card; >1 draws a stack
|
||
statusLine string // last line of card text, e.g. "4 OK, 1 Warning"
|
||
fillVar string
|
||
strokeVar string
|
||
textVar string
|
||
detailType string // "" = not clickable
|
||
}
|
||
|
||
// topoStackLayers returns how many faint backing cards to draw behind the
|
||
// front card to read as "a stack of N", capped at 2 — enough to signal
|
||
// "more than one" without the deck becoming its own visual clutter.
|
||
func topoStackLayers(count int) int {
|
||
if count <= 1 {
|
||
return 0
|
||
}
|
||
if count-1 > 2 {
|
||
return 2
|
||
}
|
||
return count - 1
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Main topology diagram
|
||
// ---------------------------------------------------------------------------
|
||
|
||
const (
|
||
topoBoxHeight = 70
|
||
topoDeviceGap = 14
|
||
topoTopMargin = 30
|
||
topoStackStep = 4 // px offset per backing layer in the card-stack effect
|
||
)
|
||
|
||
type topoBox struct {
|
||
x, y, w, h int
|
||
topoCardInfo
|
||
}
|
||
|
||
type topoEdge struct {
|
||
x1, y1, x2, y2 int
|
||
color string
|
||
}
|