Files
bee/audit/internal/webui/page_topo.go
T

585 lines
18 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package webui
import (
"encoding/json"
"fmt"
"html"
"os"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"bee/audit/internal/schema"
)
// renderTopo renders the /topo page: a read-only visualization of the server
// topology (CPU sockets, NUMA-affine PCIe devices, PSU/BMC) plus a separate
// NVLink topology card. It is pure visualization: everything it reads either
// already exists in the audit.json contract, or comes from the persisted
// techdump captured once per audit cycle (platform.CaptureTechnicalDump) —
// nothing here shells out to nvidia-smi itself, writes to
// schema.HardwarePCIeDevice or any other contract type, or talks to
// Reanimator Core.
func renderTopo(opts HandlerOptions) string {
data, err := loadSnapshot(opts.AuditPath)
if err != nil {
return topoCard("Topology", `<span class="badge badge-unknown">No audit data</span>`)
}
var ingest schema.HardwareIngestRequest
if err := json.Unmarshal(data, &ingest); err != nil {
return topoCard("Topology", `<span class="badge badge-err">Parse error</span>`)
}
hw := ingest.Hardware
var b strings.Builder
b.WriteString(renderTopoMainDiagram(hw, opts.ExportDir))
if nv := renderTopoNVLinkCard(hw, opts.ExportDir); nv != "" {
b.WriteString(nv)
}
return b.String()
}
func topoCard(title, body string) string {
return `<div class="card"><div class="card-head">` + html.EscapeString(title) + `</div><div class="card-body">` + body + `</div></div>`
}
// ---------------------------------------------------------------------------
// Classification helpers
//
// webui does not import collector (matches the existing isGPUDeviceClass
// precedent in pages.go, which already locally duplicates collector.isGPUClass
// instead of importing the package for one classifier).
// ---------------------------------------------------------------------------
// isNICDeviceClassDev applies isNICDeviceClass (pages.go) to a single device,
// with a MAC-address fallback for devices lspci doesn't classify as NIC.
func isNICDeviceClassDev(dev schema.HardwarePCIeDevice) bool {
if dev.DeviceClass != nil && isNICDeviceClass(*dev.DeviceClass) {
return true
}
return len(dev.MacAddresses) > 0
}
// isRAIDControllerClass matches the canonical class strings produced by
// collector.mapPCIeDeviceClass for RAID/storage HBAs.
func isRAIDControllerClass(class string) bool {
switch strings.TrimSpace(class) {
case "MassStorageController", "StorageController":
return true
default:
return false
}
}
// ---------------------------------------------------------------------------
// Status / link-speed coloring
// ---------------------------------------------------------------------------
// pcieGenRank ranks a PCIe generation label ("Gen3", "Gen4", ...) for
// comparison. Mirrors collector.pcieLinkSpeedRank's ordering; duplicated
// locally rather than exported, per the same "no collector import in webui"
// convention used for isGPUDeviceClass/isRAIDControllerClass.
func pcieGenRank(gen string) int {
gen = strings.ToLower(strings.TrimSpace(gen))
gen = strings.TrimPrefix(gen, "gen")
n, err := strconv.Atoi(gen)
if err != nil {
return 0
}
return n
}
// topoEdgeColorVar computes the CPU->device edge color strictly from
// link_speed vs max_link_speed — NOT from dev.Status, since Status can also
// be overwritten by SAT/acceptance-test results on the same PCIe device,
// which would conflate "link is physically degraded" with "this GPU failed
// its stress test" into the same color.
func topoEdgeColorVar(dev schema.HardwarePCIeDevice) string {
if dev.LinkSpeed == nil || dev.MaxLinkSpeed == nil {
return "var(--muted)"
}
if pcieGenRank(*dev.LinkSpeed) < pcieGenRank(*dev.MaxLinkSpeed) {
return "var(--warn-fg)"
}
return "var(--ok-fg)"
}
// ---------------------------------------------------------------------------
// NUMA node -> CPU socket join (heuristic, no guaranteed hardware mapping)
// ---------------------------------------------------------------------------
// buildSocketIndex maps a NUMA node number to the index into cpus for the
// socket occupying that position in ascending Socket-designation order.
//
// Linux NUMA node numbering is always 0-based (node0, node1, ...), but
// dmidecode's "Socket Designation" is board-defined and frequently 1-based
// ("CPU1", "CPU2", ...). Mapping NUMA node N to the CPU whose Socket field
// equals N (as an earlier version of this function did) silently fails on
// any 1-indexed board: node 0 has no match (dropped into the "unknown"
// column) and node 1 wrongly maps to the first CPU. Ranking by Socket value
// instead assumes only that node order follows socket order — true for the
// common case of N-socket boards — without depending on the numbering base.
func buildSocketIndex(cpus []schema.HardwareCPU) map[int]int {
order := make([]int, len(cpus))
for i := range cpus {
order[i] = i
}
sort.SliceStable(order, func(a, b int) bool {
ca, cb := cpus[order[a]], cpus[order[b]]
sa, sb := 0, 0
if ca.Socket != nil {
sa = *ca.Socket
}
if cb.Socket != nil {
sb = *cb.Socket
}
return sa < sb
})
idx := map[int]int{}
for numaNode, cpuIdx := range order {
idx[numaNode] = cpuIdx
}
return idx
}
// ---------------------------------------------------------------------------
// DIMM -> CPU column attachment
//
// schema.HardwareMemory has no NUMANode field (unlike HardwarePCIeDevice), so
// unlike the GPU/NIC/RAID placement above, a DIMM's CPU affinity has to be
// read out of its own Locator/Bank Locator strings — DMI type 17 gives no
// other hint. Both patterns below have been observed on real boards.
// ---------------------------------------------------------------------------
var (
topoMemCPULocatorRe = regexp.MustCompile(`(?i)^cpu\s*0*(\d+)`)
topoMemBankNodeRe = regexp.MustCompile(`(?i)node\s*0*(\d+)`)
)
// parseDIMMBankLocatorNodes maps a DIMM's Locator (matches
// schema.HardwareMemory.Slot) to the node number embedded in its Bank
// Locator field (e.g. "_Node1_Channel0_Dimm0"), read from a raw
// "dmidecode -t 17" techdump capture. Bank Locator never reaches audit.json
// (schema.HardwareMemory.Location is json:"-", used only for internal DIMM
// telemetry matching), so boards whose Locator has no CPU number of its own
// (e.g. "DIMM000(A)" rather than "CPU0_DIMM_A1") need this fallback to
// attach a DIMM to a CPU column at all — matches the existing convention of
// reading extra techdump for this page's visualization only (see
// readTopoTechDump).
func parseDIMMBankLocatorNodes(raw string) map[string]int {
result := map[string]int{}
for _, sec := range strings.Split(raw, "Memory Device") {
var locator string
node := -1
for _, line := range strings.Split(sec, "\n") {
trimmed := strings.TrimSpace(line)
if v, ok := strings.CutPrefix(trimmed, "Locator:"); ok {
locator = strings.TrimSpace(v)
}
if v, ok := strings.CutPrefix(trimmed, "Bank Locator:"); ok {
if m := topoMemBankNodeRe.FindStringSubmatch(v); m != nil {
if n, err := strconv.Atoi(m[1]); err == nil {
node = n
}
}
}
}
if locator != "" && node >= 0 {
result[locator] = node
}
}
return result
}
// dimmRawNode returns the raw (vendor-numbered, not yet column-mapped)
// CPU/NUMA node number for a DIMM, trying two heuristics in order:
// 1. A CPU number encoded directly in the Locator itself, e.g.
// "CPU0_DIMM_A1".
// 2. A node number from the Bank Locator via parseDIMMBankLocatorNodes,
// e.g. Locator "DIMM000(A)" whose Bank Locator is
// "_Node1_Channel0_Dimm0" -> 1.
//
// ok=false means neither pattern matched, so this DIMM can't be confidently
// attached to a CPU column and falls back to the unattached Memory row.
func dimmRawNode(mem schema.HardwareMemory, bankNodeByLocator map[string]int) (int, bool) {
if mem.Slot == nil {
return 0, false
}
if m := topoMemCPULocatorRe.FindStringSubmatch(*mem.Slot); m != nil {
if n, err := strconv.Atoi(m[1]); err == nil {
return n, true
}
}
if n, ok := bankNodeByLocator[*mem.Slot]; ok {
return n, true
}
return 0, false
}
// buildMemoryColumnIndex ranks the distinct raw node numbers seen across all
// DIMMs and maps the i-th smallest to column i — the same "node order
// follows socket order" assumption buildSocketIndex makes for PCIe NUMA
// nodes, but computed independently from PCIe's own numbering: PCIe's
// NUMANode is 0-based Linux numbering, while a DIMM's Bank Locator "NodeN"
// has been observed 1-based on at least one real board, so the two node
// spaces are not guaranteed to share a base.
func buildMemoryColumnIndex(rawNodes []int) map[int]int {
seen := map[int]bool{}
var distinct []int
for _, n := range rawNodes {
if !seen[n] {
seen[n] = true
distinct = append(distinct, n)
}
}
sort.Ints(distinct)
idx := map[int]int{}
for col, n := range distinct {
idx[n] = col
}
return idx
}
// ---------------------------------------------------------------------------
// GPU pairwise NVLink adjacency (from a live "nvidia-smi topo -m" query)
// ---------------------------------------------------------------------------
type gpuPairLink struct {
GPUA, GPUB int
NVLinks int
}
var topoNVRe = regexp.MustCompile(`(?i)^NV(\d+)$`)
// nvidia-smi underlines the topo -m header row with ANSI CSI sequences
// (ESC[4m...ESC[0m) even when stdout is not a TTY, so the captured techdump
// contains them and "GPU0" is not at the start of the trimmed header line.
var topoANSIRe = regexp.MustCompile("\x1b\\[[0-9;]*[A-Za-z]")
// parseGPUPairAdjacency returns every GPU pair with a nonzero NVLink bond
// count from a "nvidia-smi topo -m" matrix. Unlike parseNVIDIATopologyMatrix
// (collector package, aggregate-only: min/all-active/count), this returns
// who is bonded to whom — required so GPU-GPU edges are drawn for actually
// bonded pairs, not for adjacent boxes in the layout.
func parseGPUPairAdjacency(raw string) []gpuPairLink {
lines := strings.Split(topoANSIRe.ReplaceAllString(raw, ""), "\n")
headerIdx := -1
var gpuColIndices []int
for i, line := range lines {
trimmed := strings.TrimSpace(line)
if strings.HasPrefix(trimmed, "GPU0") {
parts := strings.Fields(trimmed)
for j, col := range parts {
if strings.HasPrefix(col, "GPU") {
gpuColIndices = append(gpuColIndices, j)
}
}
if len(gpuColIndices) >= 2 {
headerIdx = i
}
break
}
}
if headerIdx < 0 {
return nil
}
colIdxToGPU := make(map[int]int, len(gpuColIndices))
for gpuIdx, colIdx := range gpuColIndices {
colIdxToGPU[colIdx] = gpuIdx
}
seen := map[[2]int]bool{}
var pairs []gpuPairLink
rowGPU := -1
for _, line := range lines[headerIdx+1:] {
trimmed := strings.TrimSpace(line)
if !strings.HasPrefix(trimmed, "GPU") {
continue
}
cells := strings.Fields(trimmed)
if len(cells) == 0 {
continue
}
rowLabel := strings.TrimPrefix(cells[0], "GPU")
n, err := strconv.Atoi(rowLabel)
if err != nil {
continue
}
rowGPU = n
for colIdx, colGPU := range colIdxToGPU {
if colGPU == rowGPU {
continue
}
dataIdx := colIdx + 1
if dataIdx >= len(cells) {
continue
}
m := topoNVRe.FindStringSubmatch(cells[dataIdx])
if len(m) != 2 {
continue
}
nv, err := strconv.Atoi(m[1])
if err != nil || nv <= 0 {
continue
}
a, bGPU := rowGPU, colGPU
if a > bGPU {
a, bGPU = bGPU, a
}
key := [2]int{a, bGPU}
if seen[key] {
continue
}
seen[key] = true
pairs = append(pairs, gpuPairLink{GPUA: a, GPUB: bGPU, NVLinks: nv})
}
}
sort.Slice(pairs, func(i, j int) bool {
if pairs[i].GPUA != pairs[j].GPUA {
return pairs[i].GPUA < pairs[j].GPUA
}
return pairs[i].GPUB < pairs[j].GPUB
})
return pairs
}
// readTopoTechDump reads a file previously captured into the persistent
// techdump directory by platform.System.CaptureTechnicalDump (run once per
// audit cycle), rather than shelling out to nvidia-smi from the HTTP request
// handler — a live call here would block page rendering on a wedged driver,
// exactly the failure mode this tool exists to diagnose.
func readTopoTechDump(exportDir, name string) (string, error) {
out, err := os.ReadFile(filepath.Join(exportDir, "techdump", name))
if err != nil {
return "", err
}
return string(out), nil
}
func readGPUTopologyMatrix(exportDir string) (string, error) {
return readTopoTechDump(exportDir, "nvidia-smi-topo.txt")
}
// readNVIDIAIndexByBDF parses the persisted nvidia-smi-query.csv techdump
// (index,pci.bus_id,...) to map PCI bus address (matching
// HardwarePCIeDevice.Slot) to the GPU index nvidia-smi/topo -m reports, so
// GPU-GPU edges (keyed by index) can be anchored to the correct box (keyed
// by BDF) in the diagram.
func readNVIDIAIndexByBDF(exportDir string) (map[string]int, error) {
raw, err := readTopoTechDump(exportDir, "nvidia-smi-query.csv")
if err != nil {
return nil, err
}
result := map[string]int{}
for _, line := range strings.Split(raw, "\n") {
line = strings.TrimSpace(line)
if line == "" {
continue
}
parts := strings.Split(line, ",")
if len(parts) < 2 {
continue
}
idx, err := strconv.Atoi(strings.TrimSpace(parts[0]))
if err != nil {
continue
}
bdf := normalizeTopoBDF(strings.TrimSpace(parts[1]))
if bdf == "" {
continue
}
result[bdf] = idx
}
return result, nil
}
// normalizeTopoBDF normalizes a PCI bus address to "dddd:bb:dd.f" form so
// nvidia-smi's "pci.bus_id" output can be matched against
// HardwarePCIeDevice.Slot regardless of minor formatting differences
// (case, leading domain padding).
func normalizeTopoBDF(bdf string) string {
bdf = strings.ToLower(strings.TrimSpace(bdf))
if bdf == "" {
return ""
}
parts := strings.Split(bdf, ":")
if len(parts) == 3 {
domain := parts[0]
if len(domain) > 4 {
domain = domain[len(domain)-4:]
}
return domain + ":" + parts[1] + ":" + parts[2]
}
return bdf
}
// ---------------------------------------------------------------------------
// Card status aggregation
//
// Every card on this page — whether it represents one component (CPU 1) or a
// group of identical ones (GPU ×4) — is colored as a whole by its worst
// observed status, with a plain-text summary as the card's last line
// (e.g. "4 OK" or "3 OK, 1 Warning"). There is no separate status chip: a
// chip needs its own fill, and the SVG boxes previously colored that chip
// via CSS classes written for HTML (.badge-ok sets `background`/`color`,
// which do nothing on an SVG <rect>/<text> — only `fill` does), so every
// chip rendered with the SVG default fill of solid black. Coloring the card
// itself uses real `fill:var(--ok-bg)` etc. declarations, which sidesteps
// that class entirely.
// ---------------------------------------------------------------------------
// classifyTopoSeverity converts a component's Status pointer to a severity
// rank (0=unknown, 1=OK, 2=Warning, 3=Critical), treating nil/unrecognized
// the same as "Unknown" — matches topoStatusBadgeClass's classification.
func classifyTopoSeverity(status *string) int {
if status == nil {
return 0
}
switch strings.ToUpper(strings.TrimSpace(*status)) {
case "OK":
return 1
case "WARNING", "WARN", "PARTIAL":
return 2
case "CRITICAL", "FAIL", "FAILED", "ERROR":
return 3
default:
return 0
}
}
// topoSeverityColors returns the (fill, stroke, text) CSS var() triple a
// whole card is painted with for a given worst-observed severity.
func topoSeverityColors(sev int) (fill, stroke, text string) {
switch sev {
case 3:
return "var(--crit-bg)", "var(--crit-border)", "var(--crit-fg)"
case 2:
return "var(--warn-bg)", "#c9ba9b", "var(--warn-fg)"
case 1:
return "var(--ok-bg)", "#a3c293", "var(--ok-fg)"
default:
return "var(--surface-2)", "var(--border)", "var(--muted)"
}
}
// topoStatusTally counts how many components in a group fall into each
// severity bucket, so a group card can report "3 OK, 1 Warning" rather than
// collapsing to a single worst-of value and losing the rest.
type topoStatusTally struct {
unknown, ok, warn, crit int
}
func (t *topoStatusTally) add(sev int) {
switch sev {
case 3:
t.crit++
case 2:
t.warn++
case 1:
t.ok++
default:
t.unknown++
}
}
func (t topoStatusTally) total() int { return t.unknown + t.ok + t.warn + t.crit }
func (t topoStatusTally) worst() int {
switch {
case t.crit > 0:
return 3
case t.warn > 0:
return 2
case t.ok > 0:
return 1
default:
return 0
}
}
// line renders the card's last-line status summary.
func (t topoStatusTally) line() string {
if t.total() == 0 {
return "No data"
}
if t.total() == 1 {
switch {
case t.crit > 0:
return "Critical"
case t.warn > 0:
return "Warning"
case t.ok > 0:
return "OK"
default:
return "Unknown"
}
}
var parts []string
if t.crit > 0 {
parts = append(parts, fmt.Sprintf("%d Critical", t.crit))
}
if t.warn > 0 {
parts = append(parts, fmt.Sprintf("%d Warning", t.warn))
}
if t.ok > 0 {
parts = append(parts, fmt.Sprintf("%d OK", t.ok))
}
if t.unknown > 0 {
parts = append(parts, fmt.Sprintf("%d Unknown", t.unknown))
}
return strings.Join(parts, ", ")
}
// topoCardInfo is the shared visual content for one card, rendered either as
// an absolutely-positioned SVG box (main diagram) or an HTML flex item
// (Memory/Power Supplies rows) by the two writers below.
type topoCardInfo struct {
label string // e.g. "CPU 1", "GPU", "Power Supplies"
sublabel string // representative model/description, "" to omit
count int // components represented by this card; >1 draws a stack
statusLine string // last line of card text, e.g. "4 OK, 1 Warning"
fillVar string
strokeVar string
textVar string
detailType string // "" = not clickable
}
// topoStackLayers returns how many faint backing cards to draw behind the
// front card to read as "a stack of N", capped at 2 — enough to signal
// "more than one" without the deck becoming its own visual clutter.
func topoStackLayers(count int) int {
if count <= 1 {
return 0
}
if count-1 > 2 {
return 2
}
return count - 1
}
// ---------------------------------------------------------------------------
// Main topology diagram
// ---------------------------------------------------------------------------
const (
topoColWidth = 220
topoBoxWidth = 190
topoBoxHeight = 70
topoDeviceGap = 14
topoTopMargin = 30
topoStackStep = 4 // px offset per backing layer in the card-stack effect
)
type topoBox struct {
x, y, w, h int
topoCardInfo
}
type topoEdge struct {
x1, y1, x2, y2 int
color string
}