Investigating the CG480-S6053 reboot needed a way to run an ad-hoc load
(nvbandwidth across a specific GPU set) while sampling IPMI/nvidia-smi
telemetry in the background — without hardcoding a one-off test into the
SAT pack code for a single investigation.
- audit/internal/platform/scenario.go: ScenarioSpec/ScenarioJob (JSON,
no new dependency) + System.RunScenario. "command" jobs run sequential
or parallel (per-job "parallel" flag); "sampler" jobs run concurrently
in the background on their own interval until every command job
finishes or the scenario's timeout elapses. "{{gpus}}" in a command's
cmd is substituted from that job's gpu_indices. Command jobs are wired
through the same satJobBoundaryHook/satSyncBracketHook seams the SAT
job runner uses, so a scenario run gets the same durability treatment
(evidence that a risky command started/finished reaches blackbox before
a possible crash, not just whatever streamed to the RAM-backed export
dir).
- export.go: ReadScenarioFromRemovableMedia mounts each removable target
looking for scenarios/<name>.json — an air-gapped engineer can author a
scenario elsewhere, drop it under scenarios/ on the same USB stick
already plugged in for blackbox, and run it with no network path onto
the host.
- cmd/bee: new `bee run <file.json|name>` (bare name = looked up on
removable media); `bee scenario run <arg>` kept as a longer alias.
- scenarios/nvbandwidth-all-gpu-power-watch.json: the scenario that
reproduced the actual reboot (full nvbandwidth across all GPUs, which
crashed, vs. clean per-socket passes), with IPMI sensor + GPU power/temp
sampling for a power-delivery correlation check.
Also: webui/page_topo.go — the /topo page's component-status-detail modal
(GET /api/component-detail/{type}) showed "No status data recorded yet"
for any component type ComponentStatusDB has no history for yet (e.g. GPU
before a SAT run this boot), even though the topology card for the same
component already showed "N OK" from the audit inventory snapshot.
inventoryFallbackRecords now synthesizes records from that same inventory
snapshot when StatusDB is empty, using the same device classifiers
(isGPUDeviceClass etc.) and severity mapping (classifyTopoSeverity) the
topology card itself uses, so the two views never disagree.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
1348 lines
44 KiB
Go
1348 lines
44 KiB
Go
package webui
|
||
|
||
import (
|
||
"encoding/json"
|
||
"fmt"
|
||
"html"
|
||
"os"
|
||
"path/filepath"
|
||
"regexp"
|
||
"sort"
|
||
"strconv"
|
||
"strings"
|
||
|
||
"bee/audit/internal/app"
|
||
"bee/audit/internal/schema"
|
||
)
|
||
|
||
// renderTopo renders the /topo page: a read-only visualization of the server
|
||
// topology (CPU sockets, NUMA-affine PCIe devices, PSU/BMC) plus a separate
|
||
// NVLink topology card. It is pure visualization: everything it reads either
|
||
// already exists in the audit.json contract, or comes from the persisted
|
||
// techdump captured once per audit cycle (platform.CaptureTechnicalDump) —
|
||
// nothing here shells out to nvidia-smi itself, writes to
|
||
// schema.HardwarePCIeDevice or any other contract type, or talks to
|
||
// Reanimator Core.
|
||
func renderTopo(opts HandlerOptions) string {
|
||
data, err := loadSnapshot(opts.AuditPath)
|
||
if err != nil {
|
||
return topoCard("Topology", `<span class="badge badge-unknown">No audit data</span>`)
|
||
}
|
||
var ingest schema.HardwareIngestRequest
|
||
if err := json.Unmarshal(data, &ingest); err != nil {
|
||
return topoCard("Topology", `<span class="badge badge-err">Parse error</span>`)
|
||
}
|
||
hw := ingest.Hardware
|
||
|
||
var b strings.Builder
|
||
b.WriteString(renderTopoMainDiagram(hw, opts.ExportDir))
|
||
if nv := renderTopoNVLinkCard(hw, opts.ExportDir); nv != "" {
|
||
b.WriteString(nv)
|
||
}
|
||
return b.String()
|
||
}
|
||
|
||
func topoCard(title, body string) string {
|
||
return `<div class="card"><div class="card-head">` + html.EscapeString(title) + `</div><div class="card-body">` + body + `</div></div>`
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Classification helpers
|
||
//
|
||
// webui does not import collector (matches the existing isGPUDeviceClass
|
||
// precedent in pages.go, which already locally duplicates collector.isGPUClass
|
||
// instead of importing the package for one classifier).
|
||
// ---------------------------------------------------------------------------
|
||
|
||
// isNICDeviceClassDev mirrors the classification logic in hwDescribeNIC
|
||
// (pages.go), applied to a single device instead of aggregated counts.
|
||
func isNICDeviceClassDev(dev schema.HardwarePCIeDevice) bool {
|
||
if dev.DeviceClass != nil {
|
||
c := strings.ToLower(strings.TrimSpace(*dev.DeviceClass))
|
||
if c == "ethernetcontroller" || c == "networkcontroller" || strings.Contains(c, "fibrechannel") {
|
||
return true
|
||
}
|
||
}
|
||
return len(dev.MacAddresses) > 0
|
||
}
|
||
|
||
// isRAIDControllerClass matches the canonical class strings produced by
|
||
// collector.mapPCIeDeviceClass for RAID/storage HBAs.
|
||
func isRAIDControllerClass(class string) bool {
|
||
switch strings.TrimSpace(class) {
|
||
case "MassStorageController", "StorageController":
|
||
return true
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Status / link-speed coloring
|
||
// ---------------------------------------------------------------------------
|
||
|
||
// pcieGenRank ranks a PCIe generation label ("Gen3", "Gen4", ...) for
|
||
// comparison. Mirrors collector.pcieLinkSpeedRank's ordering; duplicated
|
||
// locally rather than exported, per the same "no collector import in webui"
|
||
// convention used for isGPUDeviceClass/isRAIDControllerClass.
|
||
func pcieGenRank(gen string) int {
|
||
gen = strings.ToLower(strings.TrimSpace(gen))
|
||
gen = strings.TrimPrefix(gen, "gen")
|
||
n, err := strconv.Atoi(gen)
|
||
if err != nil {
|
||
return 0
|
||
}
|
||
return n
|
||
}
|
||
|
||
// topoEdgeColorVar computes the CPU->device edge color strictly from
|
||
// link_speed vs max_link_speed — NOT from dev.Status, since Status can also
|
||
// be overwritten by SAT/acceptance-test results on the same PCIe device,
|
||
// which would conflate "link is physically degraded" with "this GPU failed
|
||
// its stress test" into the same color.
|
||
func topoEdgeColorVar(dev schema.HardwarePCIeDevice) string {
|
||
if dev.LinkSpeed == nil || dev.MaxLinkSpeed == nil {
|
||
return "var(--muted)"
|
||
}
|
||
if pcieGenRank(*dev.LinkSpeed) < pcieGenRank(*dev.MaxLinkSpeed) {
|
||
return "var(--warn-fg)"
|
||
}
|
||
return "var(--ok-fg)"
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// NUMA node -> CPU socket join (heuristic, no guaranteed hardware mapping)
|
||
// ---------------------------------------------------------------------------
|
||
|
||
// buildSocketIndex maps a NUMA node number to the index into cpus for the
|
||
// socket occupying that position in ascending Socket-designation order.
|
||
//
|
||
// Linux NUMA node numbering is always 0-based (node0, node1, ...), but
|
||
// dmidecode's "Socket Designation" is board-defined and frequently 1-based
|
||
// ("CPU1", "CPU2", ...). Mapping NUMA node N to the CPU whose Socket field
|
||
// equals N (as an earlier version of this function did) silently fails on
|
||
// any 1-indexed board: node 0 has no match (dropped into the "unknown"
|
||
// column) and node 1 wrongly maps to the first CPU. Ranking by Socket value
|
||
// instead assumes only that node order follows socket order — true for the
|
||
// common case of N-socket boards — without depending on the numbering base.
|
||
func buildSocketIndex(cpus []schema.HardwareCPU) map[int]int {
|
||
order := make([]int, len(cpus))
|
||
for i := range cpus {
|
||
order[i] = i
|
||
}
|
||
sort.SliceStable(order, func(a, b int) bool {
|
||
ca, cb := cpus[order[a]], cpus[order[b]]
|
||
sa, sb := 0, 0
|
||
if ca.Socket != nil {
|
||
sa = *ca.Socket
|
||
}
|
||
if cb.Socket != nil {
|
||
sb = *cb.Socket
|
||
}
|
||
return sa < sb
|
||
})
|
||
idx := map[int]int{}
|
||
for numaNode, cpuIdx := range order {
|
||
idx[numaNode] = cpuIdx
|
||
}
|
||
return idx
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// DIMM -> CPU column attachment
|
||
//
|
||
// schema.HardwareMemory has no NUMANode field (unlike HardwarePCIeDevice), so
|
||
// unlike the GPU/NIC/RAID placement above, a DIMM's CPU affinity has to be
|
||
// read out of its own Locator/Bank Locator strings — DMI type 17 gives no
|
||
// other hint. Both patterns below have been observed on real boards.
|
||
// ---------------------------------------------------------------------------
|
||
|
||
var (
|
||
topoMemCPULocatorRe = regexp.MustCompile(`(?i)^cpu\s*0*(\d+)`)
|
||
topoMemBankNodeRe = regexp.MustCompile(`(?i)node\s*0*(\d+)`)
|
||
)
|
||
|
||
// parseDIMMBankLocatorNodes maps a DIMM's Locator (matches
|
||
// schema.HardwareMemory.Slot) to the node number embedded in its Bank
|
||
// Locator field (e.g. "_Node1_Channel0_Dimm0"), read from a raw
|
||
// "dmidecode -t 17" techdump capture. Bank Locator never reaches audit.json
|
||
// (schema.HardwareMemory.Location is json:"-", used only for internal DIMM
|
||
// telemetry matching), so boards whose Locator has no CPU number of its own
|
||
// (e.g. "DIMM000(A)" rather than "CPU0_DIMM_A1") need this fallback to
|
||
// attach a DIMM to a CPU column at all — matches the existing convention of
|
||
// reading extra techdump for this page's visualization only (see
|
||
// readTopoTechDump).
|
||
func parseDIMMBankLocatorNodes(raw string) map[string]int {
|
||
result := map[string]int{}
|
||
for _, sec := range strings.Split(raw, "Memory Device") {
|
||
var locator string
|
||
node := -1
|
||
for _, line := range strings.Split(sec, "\n") {
|
||
trimmed := strings.TrimSpace(line)
|
||
if v, ok := strings.CutPrefix(trimmed, "Locator:"); ok {
|
||
locator = strings.TrimSpace(v)
|
||
}
|
||
if v, ok := strings.CutPrefix(trimmed, "Bank Locator:"); ok {
|
||
if m := topoMemBankNodeRe.FindStringSubmatch(v); m != nil {
|
||
if n, err := strconv.Atoi(m[1]); err == nil {
|
||
node = n
|
||
}
|
||
}
|
||
}
|
||
}
|
||
if locator != "" && node >= 0 {
|
||
result[locator] = node
|
||
}
|
||
}
|
||
return result
|
||
}
|
||
|
||
// dimmRawNode returns the raw (vendor-numbered, not yet column-mapped)
|
||
// CPU/NUMA node number for a DIMM, trying two heuristics in order:
|
||
// 1. A CPU number encoded directly in the Locator itself, e.g.
|
||
// "CPU0_DIMM_A1".
|
||
// 2. A node number from the Bank Locator via parseDIMMBankLocatorNodes,
|
||
// e.g. Locator "DIMM000(A)" whose Bank Locator is
|
||
// "_Node1_Channel0_Dimm0" -> 1.
|
||
// ok=false means neither pattern matched, so this DIMM can't be confidently
|
||
// attached to a CPU column and falls back to the unattached Memory row.
|
||
func dimmRawNode(mem schema.HardwareMemory, bankNodeByLocator map[string]int) (int, bool) {
|
||
if mem.Slot == nil {
|
||
return 0, false
|
||
}
|
||
if m := topoMemCPULocatorRe.FindStringSubmatch(*mem.Slot); m != nil {
|
||
if n, err := strconv.Atoi(m[1]); err == nil {
|
||
return n, true
|
||
}
|
||
}
|
||
if n, ok := bankNodeByLocator[*mem.Slot]; ok {
|
||
return n, true
|
||
}
|
||
return 0, false
|
||
}
|
||
|
||
// buildMemoryColumnIndex ranks the distinct raw node numbers seen across all
|
||
// DIMMs and maps the i-th smallest to column i — the same "node order
|
||
// follows socket order" assumption buildSocketIndex makes for PCIe NUMA
|
||
// nodes, but computed independently from PCIe's own numbering: PCIe's
|
||
// NUMANode is 0-based Linux numbering, while a DIMM's Bank Locator "NodeN"
|
||
// has been observed 1-based on at least one real board, so the two node
|
||
// spaces are not guaranteed to share a base.
|
||
func buildMemoryColumnIndex(rawNodes []int) map[int]int {
|
||
seen := map[int]bool{}
|
||
var distinct []int
|
||
for _, n := range rawNodes {
|
||
if !seen[n] {
|
||
seen[n] = true
|
||
distinct = append(distinct, n)
|
||
}
|
||
}
|
||
sort.Ints(distinct)
|
||
idx := map[int]int{}
|
||
for col, n := range distinct {
|
||
idx[n] = col
|
||
}
|
||
return idx
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// GPU pairwise NVLink adjacency (from a live "nvidia-smi topo -m" query)
|
||
// ---------------------------------------------------------------------------
|
||
|
||
type gpuPairLink struct {
|
||
GPUA, GPUB int
|
||
NVLinks int
|
||
}
|
||
|
||
var topoNVRe = regexp.MustCompile(`(?i)^NV(\d+)$`)
|
||
|
||
// nvidia-smi underlines the topo -m header row with ANSI CSI sequences
|
||
// (ESC[4m...ESC[0m) even when stdout is not a TTY, so the captured techdump
|
||
// contains them and "GPU0" is not at the start of the trimmed header line.
|
||
var topoANSIRe = regexp.MustCompile("\x1b\\[[0-9;]*[A-Za-z]")
|
||
|
||
// parseGPUPairAdjacency returns every GPU pair with a nonzero NVLink bond
|
||
// count from a "nvidia-smi topo -m" matrix. Unlike parseNVIDIATopologyMatrix
|
||
// (collector package, aggregate-only: min/all-active/count), this returns
|
||
// who is bonded to whom — required so GPU-GPU edges are drawn for actually
|
||
// bonded pairs, not for adjacent boxes in the layout.
|
||
func parseGPUPairAdjacency(raw string) []gpuPairLink {
|
||
lines := strings.Split(topoANSIRe.ReplaceAllString(raw, ""), "\n")
|
||
headerIdx := -1
|
||
var gpuColIndices []int
|
||
for i, line := range lines {
|
||
trimmed := strings.TrimSpace(line)
|
||
if strings.HasPrefix(trimmed, "GPU0") {
|
||
parts := strings.Fields(trimmed)
|
||
for j, col := range parts {
|
||
if strings.HasPrefix(col, "GPU") {
|
||
gpuColIndices = append(gpuColIndices, j)
|
||
}
|
||
}
|
||
if len(gpuColIndices) >= 2 {
|
||
headerIdx = i
|
||
}
|
||
break
|
||
}
|
||
}
|
||
if headerIdx < 0 {
|
||
return nil
|
||
}
|
||
|
||
colIdxToGPU := make(map[int]int, len(gpuColIndices))
|
||
for gpuIdx, colIdx := range gpuColIndices {
|
||
colIdxToGPU[colIdx] = gpuIdx
|
||
}
|
||
|
||
seen := map[[2]int]bool{}
|
||
var pairs []gpuPairLink
|
||
rowGPU := -1
|
||
for _, line := range lines[headerIdx+1:] {
|
||
trimmed := strings.TrimSpace(line)
|
||
if !strings.HasPrefix(trimmed, "GPU") {
|
||
continue
|
||
}
|
||
cells := strings.Fields(trimmed)
|
||
if len(cells) == 0 {
|
||
continue
|
||
}
|
||
rowLabel := strings.TrimPrefix(cells[0], "GPU")
|
||
n, err := strconv.Atoi(rowLabel)
|
||
if err != nil {
|
||
continue
|
||
}
|
||
rowGPU = n
|
||
for colIdx, colGPU := range colIdxToGPU {
|
||
if colGPU == rowGPU {
|
||
continue
|
||
}
|
||
dataIdx := colIdx + 1
|
||
if dataIdx >= len(cells) {
|
||
continue
|
||
}
|
||
m := topoNVRe.FindStringSubmatch(cells[dataIdx])
|
||
if len(m) != 2 {
|
||
continue
|
||
}
|
||
nv, err := strconv.Atoi(m[1])
|
||
if err != nil || nv <= 0 {
|
||
continue
|
||
}
|
||
a, bGPU := rowGPU, colGPU
|
||
if a > bGPU {
|
||
a, bGPU = bGPU, a
|
||
}
|
||
key := [2]int{a, bGPU}
|
||
if seen[key] {
|
||
continue
|
||
}
|
||
seen[key] = true
|
||
pairs = append(pairs, gpuPairLink{GPUA: a, GPUB: bGPU, NVLinks: nv})
|
||
}
|
||
}
|
||
sort.Slice(pairs, func(i, j int) bool {
|
||
if pairs[i].GPUA != pairs[j].GPUA {
|
||
return pairs[i].GPUA < pairs[j].GPUA
|
||
}
|
||
return pairs[i].GPUB < pairs[j].GPUB
|
||
})
|
||
return pairs
|
||
}
|
||
|
||
// readTopoTechDump reads a file previously captured into the persistent
|
||
// techdump directory by platform.System.CaptureTechnicalDump (run once per
|
||
// audit cycle), rather than shelling out to nvidia-smi from the HTTP request
|
||
// handler — a live call here would block page rendering on a wedged driver,
|
||
// exactly the failure mode this tool exists to diagnose.
|
||
func readTopoTechDump(exportDir, name string) (string, error) {
|
||
out, err := os.ReadFile(filepath.Join(exportDir, "techdump", name))
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
return string(out), nil
|
||
}
|
||
|
||
func readGPUTopologyMatrix(exportDir string) (string, error) {
|
||
return readTopoTechDump(exportDir, "nvidia-smi-topo.txt")
|
||
}
|
||
|
||
// readNVIDIAIndexByBDF parses the persisted nvidia-smi-query.csv techdump
|
||
// (index,pci.bus_id,...) to map PCI bus address (matching
|
||
// HardwarePCIeDevice.Slot) to the GPU index nvidia-smi/topo -m reports, so
|
||
// GPU-GPU edges (keyed by index) can be anchored to the correct box (keyed
|
||
// by BDF) in the diagram.
|
||
func readNVIDIAIndexByBDF(exportDir string) (map[string]int, error) {
|
||
raw, err := readTopoTechDump(exportDir, "nvidia-smi-query.csv")
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
result := map[string]int{}
|
||
for _, line := range strings.Split(raw, "\n") {
|
||
line = strings.TrimSpace(line)
|
||
if line == "" {
|
||
continue
|
||
}
|
||
parts := strings.Split(line, ",")
|
||
if len(parts) < 2 {
|
||
continue
|
||
}
|
||
idx, err := strconv.Atoi(strings.TrimSpace(parts[0]))
|
||
if err != nil {
|
||
continue
|
||
}
|
||
bdf := normalizeTopoBDF(strings.TrimSpace(parts[1]))
|
||
if bdf == "" {
|
||
continue
|
||
}
|
||
result[bdf] = idx
|
||
}
|
||
return result, nil
|
||
}
|
||
|
||
// normalizeTopoBDF normalizes a PCI bus address to "dddd:bb:dd.f" form so
|
||
// nvidia-smi's "pci.bus_id" output can be matched against
|
||
// HardwarePCIeDevice.Slot regardless of minor formatting differences
|
||
// (case, leading domain padding).
|
||
func normalizeTopoBDF(bdf string) string {
|
||
bdf = strings.ToLower(strings.TrimSpace(bdf))
|
||
if bdf == "" {
|
||
return ""
|
||
}
|
||
parts := strings.Split(bdf, ":")
|
||
if len(parts) == 3 {
|
||
domain := parts[0]
|
||
if len(domain) > 4 {
|
||
domain = domain[len(domain)-4:]
|
||
}
|
||
return domain + ":" + parts[1] + ":" + parts[2]
|
||
}
|
||
return bdf
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Card status aggregation
|
||
//
|
||
// Every card on this page — whether it represents one component (CPU 1) or a
|
||
// group of identical ones (GPU ×4) — is colored as a whole by its worst
|
||
// observed status, with a plain-text summary as the card's last line
|
||
// (e.g. "4 OK" or "3 OK, 1 Warning"). There is no separate status chip: a
|
||
// chip needs its own fill, and the SVG boxes previously colored that chip
|
||
// via CSS classes written for HTML (.badge-ok sets `background`/`color`,
|
||
// which do nothing on an SVG <rect>/<text> — only `fill` does), so every
|
||
// chip rendered with the SVG default fill of solid black. Coloring the card
|
||
// itself uses real `fill:var(--ok-bg)` etc. declarations, which sidesteps
|
||
// that class entirely.
|
||
// ---------------------------------------------------------------------------
|
||
|
||
// classifyTopoSeverity converts a component's Status pointer to a severity
|
||
// rank (0=unknown, 1=OK, 2=Warning, 3=Critical), treating nil/unrecognized
|
||
// the same as "Unknown" — matches topoStatusBadgeClass's classification.
|
||
func classifyTopoSeverity(status *string) int {
|
||
if status == nil {
|
||
return 0
|
||
}
|
||
switch strings.ToUpper(strings.TrimSpace(*status)) {
|
||
case "OK":
|
||
return 1
|
||
case "WARNING", "WARN", "PARTIAL":
|
||
return 2
|
||
case "CRITICAL", "FAIL", "FAILED", "ERROR":
|
||
return 3
|
||
default:
|
||
return 0
|
||
}
|
||
}
|
||
|
||
// topoSeverityColors returns the (fill, stroke, text) CSS var() triple a
|
||
// whole card is painted with for a given worst-observed severity.
|
||
func topoSeverityColors(sev int) (fill, stroke, text string) {
|
||
switch sev {
|
||
case 3:
|
||
return "var(--crit-bg)", "var(--crit-border)", "var(--crit-fg)"
|
||
case 2:
|
||
return "var(--warn-bg)", "#c9ba9b", "var(--warn-fg)"
|
||
case 1:
|
||
return "var(--ok-bg)", "#a3c293", "var(--ok-fg)"
|
||
default:
|
||
return "var(--surface-2)", "var(--border)", "var(--muted)"
|
||
}
|
||
}
|
||
|
||
// topoStatusTally counts how many components in a group fall into each
|
||
// severity bucket, so a group card can report "3 OK, 1 Warning" rather than
|
||
// collapsing to a single worst-of value and losing the rest.
|
||
type topoStatusTally struct {
|
||
unknown, ok, warn, crit int
|
||
}
|
||
|
||
func (t *topoStatusTally) add(sev int) {
|
||
switch sev {
|
||
case 3:
|
||
t.crit++
|
||
case 2:
|
||
t.warn++
|
||
case 1:
|
||
t.ok++
|
||
default:
|
||
t.unknown++
|
||
}
|
||
}
|
||
|
||
func (t topoStatusTally) total() int { return t.unknown + t.ok + t.warn + t.crit }
|
||
|
||
func (t topoStatusTally) worst() int {
|
||
switch {
|
||
case t.crit > 0:
|
||
return 3
|
||
case t.warn > 0:
|
||
return 2
|
||
case t.ok > 0:
|
||
return 1
|
||
default:
|
||
return 0
|
||
}
|
||
}
|
||
|
||
// line renders the card's last-line status summary.
|
||
func (t topoStatusTally) line() string {
|
||
if t.total() == 0 {
|
||
return "No data"
|
||
}
|
||
if t.total() == 1 {
|
||
switch {
|
||
case t.crit > 0:
|
||
return "Critical"
|
||
case t.warn > 0:
|
||
return "Warning"
|
||
case t.ok > 0:
|
||
return "OK"
|
||
default:
|
||
return "Unknown"
|
||
}
|
||
}
|
||
var parts []string
|
||
if t.crit > 0 {
|
||
parts = append(parts, fmt.Sprintf("%d Critical", t.crit))
|
||
}
|
||
if t.warn > 0 {
|
||
parts = append(parts, fmt.Sprintf("%d Warning", t.warn))
|
||
}
|
||
if t.ok > 0 {
|
||
parts = append(parts, fmt.Sprintf("%d OK", t.ok))
|
||
}
|
||
if t.unknown > 0 {
|
||
parts = append(parts, fmt.Sprintf("%d Unknown", t.unknown))
|
||
}
|
||
return strings.Join(parts, ", ")
|
||
}
|
||
|
||
// topoCardInfo is the shared visual content for one card, rendered either as
|
||
// an absolutely-positioned SVG box (main diagram) or an HTML flex item
|
||
// (Memory/Power Supplies rows) by the two writers below.
|
||
type topoCardInfo struct {
|
||
label string // e.g. "CPU 1", "GPU", "Power Supplies"
|
||
sublabel string // representative model/description, "" to omit
|
||
count int // components represented by this card; >1 draws a stack
|
||
statusLine string // last line of card text, e.g. "4 OK, 1 Warning"
|
||
fillVar string
|
||
strokeVar string
|
||
textVar string
|
||
detailType string // "" = not clickable
|
||
}
|
||
|
||
// topoStackLayers returns how many faint backing cards to draw behind the
|
||
// front card to read as "a stack of N", capped at 2 — enough to signal
|
||
// "more than one" without the deck becoming its own visual clutter.
|
||
func topoStackLayers(count int) int {
|
||
if count <= 1 {
|
||
return 0
|
||
}
|
||
if count-1 > 2 {
|
||
return 2
|
||
}
|
||
return count - 1
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Main topology diagram
|
||
// ---------------------------------------------------------------------------
|
||
|
||
const (
|
||
topoColWidth = 220
|
||
topoBoxWidth = 190
|
||
topoBoxHeight = 70
|
||
topoDeviceGap = 14
|
||
topoTopMargin = 30
|
||
topoStackStep = 4 // px offset per backing layer in the card-stack effect
|
||
)
|
||
|
||
type topoBox struct {
|
||
x, y, w, h int
|
||
topoCardInfo
|
||
}
|
||
|
||
type topoEdge struct {
|
||
x1, y1, x2, y2 int
|
||
color string
|
||
}
|
||
|
||
func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string {
|
||
socketIdx := buildSocketIndex(hw.CPUs)
|
||
numCols := len(hw.CPUs)
|
||
if numCols == 0 {
|
||
numCols = 1
|
||
}
|
||
unknownCol := numCols // extra trailing column for unmatched devices
|
||
|
||
// Group PCIe devices (GPU/NIC/RAID only — matches the mockup's node
|
||
// types) into columns by NUMA node, falling back to the "unknown" bucket.
|
||
type placedDevice struct {
|
||
dev schema.HardwarePCIeDevice
|
||
kind string // "gpu", "nic", "raid"
|
||
col int
|
||
bdf string
|
||
}
|
||
var placed []placedDevice
|
||
for _, dev := range hw.PCIeDevices {
|
||
var kind string
|
||
switch {
|
||
case dev.DeviceClass != nil && isGPUDeviceClass(*dev.DeviceClass):
|
||
kind = "gpu"
|
||
case isNICDeviceClassDev(dev):
|
||
kind = "nic"
|
||
case dev.DeviceClass != nil && isRAIDControllerClass(*dev.DeviceClass):
|
||
kind = "raid"
|
||
default:
|
||
continue
|
||
}
|
||
col := unknownCol
|
||
if dev.NUMANode != nil {
|
||
if ci, ok := socketIdx[*dev.NUMANode]; ok {
|
||
col = ci
|
||
}
|
||
}
|
||
bdf := ""
|
||
if dev.Slot != nil {
|
||
bdf = normalizeTopoBDF(*dev.Slot)
|
||
} else if dev.BDF != nil {
|
||
bdf = normalizeTopoBDF(*dev.BDF)
|
||
}
|
||
placed = append(placed, placedDevice{dev: dev, kind: kind, col: col, bdf: bdf})
|
||
}
|
||
hasUnknownCol := false
|
||
for _, p := range placed {
|
||
if p.col == unknownCol {
|
||
hasUnknownCol = true
|
||
break
|
||
}
|
||
}
|
||
totalCols := numCols
|
||
if hasUnknownCol {
|
||
totalCols++
|
||
}
|
||
|
||
// GPU index<->BDF map + pairwise NVLink adjacency, read from the
|
||
// persisted techdump captured during the last audit cycle, best-effort:
|
||
// if the dump is missing (older audit, no NVIDIA GPUs), this is simply
|
||
// skipped. Used only to detect the cross-NUMA-bonded-pair anomaly below;
|
||
// the pairwise links themselves are drawn in the separate NVLink
|
||
// Topology card, since grouping same-kind/same-column devices into one
|
||
// stacked card here leaves no single per-GPU anchor point to draw a
|
||
// pairwise connector to or from.
|
||
bdfToIndex, _ := readNVIDIAIndexByBDF(exportDir)
|
||
var pairs []gpuPairLink
|
||
if topoMatrix, err := readGPUTopologyMatrix(exportDir); err == nil {
|
||
pairs = parseGPUPairAdjacency(topoMatrix)
|
||
}
|
||
gpuNUMAByIndex := map[int]*int{}
|
||
gpuBDFByIndex := map[int]string{}
|
||
for _, p := range placed {
|
||
if p.kind != "gpu" || p.bdf == "" {
|
||
continue
|
||
}
|
||
if idx, ok := bdfToIndex[p.bdf]; ok {
|
||
gpuNUMAByIndex[idx] = p.dev.NUMANode
|
||
gpuBDFByIndex[idx] = p.bdf
|
||
}
|
||
}
|
||
// A bonded pair spanning two different NUMA nodes is treated as an
|
||
// anomaly (not a neutral fact) per project decision: a bonded pair is
|
||
// expected to sit on one NUMA node, so a cross-NUMA bond escalates both
|
||
// GPUs' effective severity to at least Warning, regardless of their own
|
||
// reported SAT status.
|
||
crossNUMAWarnBDF := map[string]bool{}
|
||
for _, pair := range pairs {
|
||
numaA, okA := gpuNUMAByIndex[pair.GPUA]
|
||
numaB, okB := gpuNUMAByIndex[pair.GPUB]
|
||
if !okA || !okB || numaA == nil || numaB == nil || *numaA == *numaB {
|
||
continue
|
||
}
|
||
crossNUMAWarnBDF[gpuBDFByIndex[pair.GPUA]] = true
|
||
crossNUMAWarnBDF[gpuBDFByIndex[pair.GPUB]] = true
|
||
}
|
||
|
||
kindOrder := []string{"gpu", "nic", "raid"}
|
||
kindLabel := map[string]string{"gpu": "GPU", "nic": "NIC", "raid": "RAID"}
|
||
|
||
// Attach memory DIMMs to their CPU column too, the same way GPU/NIC/RAID
|
||
// PCIe devices are attached via NUMANode — memory has no NUMANode field
|
||
// in the schema, so this reads the DIMM's own Locator/Bank Locator
|
||
// strings instead (see dimmRawNode). DIMMs that can't be confidently
|
||
// attached fall back to the unattached "Memory" row below the diagram,
|
||
// same as before this existed.
|
||
memBankNodes := map[string]int{}
|
||
if raw, err := readTopoTechDump(exportDir, "dmidecode-type17.txt"); err == nil {
|
||
memBankNodes = parseDIMMBankLocatorNodes(raw)
|
||
}
|
||
memCol := make([]int, len(hw.Memory))
|
||
memMatched := make([]bool, len(hw.Memory))
|
||
var memRawNodes []int
|
||
for i, m := range hw.Memory {
|
||
if node, ok := dimmRawNode(m, memBankNodes); ok {
|
||
memCol[i] = node
|
||
memMatched[i] = true
|
||
memRawNodes = append(memRawNodes, node)
|
||
}
|
||
}
|
||
memColIdx := buildMemoryColumnIndex(memRawNodes)
|
||
for i := range hw.Memory {
|
||
if !memMatched[i] {
|
||
continue
|
||
}
|
||
col := memColIdx[memCol[i]]
|
||
if col >= numCols {
|
||
memMatched[i] = false
|
||
continue
|
||
}
|
||
memCol[i] = col
|
||
}
|
||
|
||
var boxes []topoBox
|
||
var pcieEdges []topoEdge
|
||
|
||
for col := 0; col < totalCols; col++ {
|
||
colX := (col+1)*24 + col*topoColWidth
|
||
if col < len(hw.CPUs) {
|
||
cpu := hw.CPUs[col]
|
||
model := ""
|
||
if cpu.Model != nil {
|
||
model = *cpu.Model
|
||
}
|
||
socket := col
|
||
if cpu.Socket != nil {
|
||
socket = *cpu.Socket
|
||
}
|
||
var tally topoStatusTally
|
||
tally.add(classifyTopoSeverity(cpu.Status))
|
||
fill, stroke, text := topoSeverityColors(tally.worst())
|
||
boxes = append(boxes, topoBox{
|
||
x: colX, y: topoTopMargin, w: topoBoxWidth, h: topoBoxHeight,
|
||
topoCardInfo: topoCardInfo{
|
||
label: fmt.Sprintf("CPU %d", socket), sublabel: model, count: 1,
|
||
statusLine: tally.line(),
|
||
fillVar: fill, strokeVar: stroke, textVar: text,
|
||
detailType: "cpu",
|
||
},
|
||
})
|
||
}
|
||
|
||
y := topoTopMargin + topoBoxHeight + topoDeviceGap*2
|
||
|
||
// Memory goes first in the chain, directly under the CPU box: DIMMs
|
||
// are wired straight to the socket's memory controller, not reached
|
||
// over PCIe like the GPU/NIC/RAID chain below it.
|
||
var memGroup []schema.HardwareMemory
|
||
for i, m := range hw.Memory {
|
||
if memMatched[i] && memCol[i] == col {
|
||
memGroup = append(memGroup, m)
|
||
}
|
||
}
|
||
if len(memGroup) > 0 {
|
||
var tally topoStatusTally
|
||
sizeGB := 0
|
||
for _, m := range memGroup {
|
||
tally.add(classifyTopoSeverity(m.Status))
|
||
if m.SizeMB != nil {
|
||
sizeGB += *m.SizeMB / 1024
|
||
}
|
||
}
|
||
fill, stroke, text := topoSeverityColors(tally.worst())
|
||
sublabel := ""
|
||
if sizeGB > 0 {
|
||
sublabel = fmt.Sprintf("%d GB total", sizeGB)
|
||
}
|
||
stackLayers := topoStackLayers(len(memGroup))
|
||
boxes = append(boxes, topoBox{
|
||
x: colX, y: y, w: topoBoxWidth, h: topoBoxHeight,
|
||
topoCardInfo: topoCardInfo{
|
||
label: "Memory", sublabel: sublabel, count: len(memGroup),
|
||
statusLine: tally.line(),
|
||
fillVar: fill, strokeVar: stroke, textVar: text,
|
||
detailType: "memory",
|
||
},
|
||
})
|
||
if col < len(hw.CPUs) {
|
||
pcieEdges = append(pcieEdges, topoEdge{
|
||
x1: colX + topoBoxWidth/2, y1: topoTopMargin + topoBoxHeight,
|
||
x2: colX + topoBoxWidth/2, y2: y,
|
||
color: "var(--ok-fg)",
|
||
})
|
||
}
|
||
y += topoBoxHeight + topoDeviceGap + stackLayers*topoStackStep
|
||
}
|
||
|
||
for _, kind := range kindOrder {
|
||
var group []placedDevice
|
||
for _, p := range placed {
|
||
if p.col == col && p.kind == kind {
|
||
group = append(group, p)
|
||
}
|
||
}
|
||
if len(group) == 0 {
|
||
continue
|
||
}
|
||
|
||
var tally topoStatusTally
|
||
model := ""
|
||
edgeColor := "var(--ok-fg)"
|
||
for i, p := range group {
|
||
sev := classifyTopoSeverity(p.dev.Status)
|
||
if kind == "gpu" && crossNUMAWarnBDF[p.bdf] && sev < 2 {
|
||
sev = 2
|
||
}
|
||
tally.add(sev)
|
||
if i == 0 && p.dev.Model != nil {
|
||
model = *p.dev.Model
|
||
}
|
||
if topoEdgeColorVar(p.dev) == "var(--warn-fg)" {
|
||
edgeColor = "var(--warn-fg)"
|
||
}
|
||
}
|
||
fill, stroke, text := topoSeverityColors(tally.worst())
|
||
stackLayers := topoStackLayers(len(group))
|
||
boxes = append(boxes, topoBox{
|
||
x: colX, y: y, w: topoBoxWidth, h: topoBoxHeight,
|
||
topoCardInfo: topoCardInfo{
|
||
label: kindLabel[kind], sublabel: model, count: len(group),
|
||
statusLine: tally.line(),
|
||
fillVar: fill, strokeVar: stroke, textVar: text,
|
||
detailType: kind,
|
||
},
|
||
})
|
||
|
||
if col < len(hw.CPUs) {
|
||
pcieEdges = append(pcieEdges, topoEdge{
|
||
x1: colX + topoBoxWidth/2, y1: topoTopMargin + topoBoxHeight,
|
||
x2: colX + topoBoxWidth/2, y2: y,
|
||
color: edgeColor,
|
||
})
|
||
}
|
||
|
||
y += topoBoxHeight + topoDeviceGap + stackLayers*topoStackStep
|
||
}
|
||
}
|
||
|
||
maxDeviceY := topoTopMargin + topoBoxHeight + topoDeviceGap*2
|
||
for _, box := range boxes {
|
||
bottom := box.y + box.h + topoStackLayers(box.count)*topoStackStep
|
||
if bottom > maxDeviceY {
|
||
maxDeviceY = bottom
|
||
}
|
||
}
|
||
|
||
svgHeight := maxDeviceY + 24
|
||
svgWidth := totalCols*topoColWidth + 48
|
||
|
||
var b strings.Builder
|
||
// Wrapped in its own horizontally-scrolling container (matching the
|
||
// overflow-x:auto convention used for wide tables elsewhere in webui)
|
||
// rather than max-width:100% — squashing a node/edge diagram to fit a
|
||
// narrow viewport makes labels and badges illegible, whereas scrolling
|
||
// keeps the diagram readable at its natural size on any screen width.
|
||
b.WriteString(`<div style="overflow-x:auto">`)
|
||
fmt.Fprintf(&b, `<svg width="%d" height="%d" viewBox="0 0 %d %d">`+"\n", svgWidth, svgHeight, svgWidth, svgHeight)
|
||
for _, e := range pcieEdges {
|
||
fmt.Fprintf(&b, `<line x1="%d" y1="%d" x2="%d" y2="%d" style="stroke:%s;stroke-width:2"/>`+"\n", e.x1, e.y1, e.x2, e.y2, e.color)
|
||
}
|
||
for _, box := range boxes {
|
||
writeTopoBoxSVG(&b, box)
|
||
}
|
||
b.WriteString(`</svg></div>`)
|
||
|
||
// Firmware (BMC/BIOS/...) and PSUs have no PCIe/CPU affinity to anchor
|
||
// them to a column, and there can be an arbitrary number of any of them
|
||
// — so unlike the diagram above, they're plain flex-wrap HTML below the
|
||
// SVG rather than absolutely-positioned SVG boxes. A fixed-size SVG
|
||
// canvas has no way to wrap overflow onto a new row, which is exactly
|
||
// what caused these to pile up and overlap once a board had more
|
||
// PSUs/firmware records than fit in one fixed-width row.
|
||
//
|
||
// Memory DIMMs that were matched to a CPU column above already got a
|
||
// box in the SVG diagram; only DIMMs that couldn't be attached to a
|
||
// column (see memMatched above) fall back to this row.
|
||
var unmatchedMem []schema.HardwareMemory
|
||
for i, m := range hw.Memory {
|
||
if !memMatched[i] {
|
||
unmatchedMem = append(unmatchedMem, m)
|
||
}
|
||
}
|
||
if len(unmatchedMem) > 0 {
|
||
var tally topoStatusTally
|
||
for _, m := range unmatchedMem {
|
||
tally.add(classifyTopoSeverity(m.Status))
|
||
}
|
||
fill, stroke, text := topoSeverityColors(tally.worst())
|
||
sizeGB := 0
|
||
for _, m := range unmatchedMem {
|
||
if m.SizeMB != nil {
|
||
sizeGB += *m.SizeMB / 1024
|
||
}
|
||
}
|
||
sublabel := ""
|
||
if sizeGB > 0 {
|
||
sublabel = fmt.Sprintf("%d GB total", sizeGB)
|
||
}
|
||
b.WriteString(renderTopoFlexRow("Memory", []topoCardInfo{{
|
||
label: "Memory", sublabel: sublabel, count: len(unmatchedMem),
|
||
statusLine: tally.line(),
|
||
fillVar: fill, strokeVar: stroke, textVar: text,
|
||
detailType: "memory",
|
||
}}))
|
||
}
|
||
|
||
var firmwareItems []topoCardInfo
|
||
for _, rec := range hw.Firmware {
|
||
// Firmware records carry no per-item status in the schema (they are
|
||
// identity, not health, facts), so each stays a neutral, uncolored
|
||
// card rather than forcing a fake "Unknown" status line.
|
||
fillVar, strokeVar, textVar := topoSeverityColors(0)
|
||
firmwareItems = append(firmwareItems, topoCardInfo{
|
||
label: rec.DeviceName, sublabel: "fw " + rec.Version, count: 1,
|
||
fillVar: fillVar, strokeVar: strokeVar, textVar: textVar,
|
||
})
|
||
}
|
||
b.WriteString(renderTopoFlexRow("Firmware", firmwareItems))
|
||
|
||
if len(hw.PowerSupplies) > 0 {
|
||
var tally topoStatusTally
|
||
watt := 0
|
||
for _, psu := range hw.PowerSupplies {
|
||
tally.add(classifyTopoSeverity(psu.Status))
|
||
if psu.WattageW != nil {
|
||
watt = *psu.WattageW
|
||
}
|
||
}
|
||
fill, stroke, text := topoSeverityColors(tally.worst())
|
||
sublabel := ""
|
||
if watt > 0 {
|
||
sublabel = fmt.Sprintf("%dW each", watt)
|
||
}
|
||
b.WriteString(renderTopoFlexRow("Power Supplies", []topoCardInfo{{
|
||
label: "Power Supplies", sublabel: sublabel, count: len(hw.PowerSupplies),
|
||
statusLine: tally.line(),
|
||
fillVar: fill, strokeVar: stroke, textVar: text,
|
||
detailType: "psu",
|
||
}}))
|
||
}
|
||
|
||
return topoCard("Topology", b.String())
|
||
}
|
||
|
||
// renderTopoFlexRow renders a labeled, wrapping row of component cards.
|
||
// Returns "" if items is empty (e.g. no PSU data in this audit).
|
||
func renderTopoFlexRow(title string, items []topoCardInfo) string {
|
||
if len(items) == 0 {
|
||
return ""
|
||
}
|
||
var b strings.Builder
|
||
fmt.Fprintf(&b, `<div style="font-size:11px;color:var(--muted);text-transform:uppercase;letter-spacing:.05em;margin:16px 0 6px">%s</div>`,
|
||
html.EscapeString(title))
|
||
b.WriteString(`<div style="display:flex;flex-wrap:wrap;gap:10px">`)
|
||
for _, item := range items {
|
||
onclick := ""
|
||
cursor := "default"
|
||
if item.detailType != "" {
|
||
onclick = fmt.Sprintf(` onclick="openComponentDetail('%s')"`, item.detailType)
|
||
cursor = "pointer"
|
||
}
|
||
stackLayers := topoStackLayers(item.count)
|
||
// Extra right/bottom padding on the wrapper reserves room for the
|
||
// backing layers of the stack effect so they aren't clipped by the
|
||
// flex container.
|
||
fmt.Fprintf(&b, `<div style="position:relative;padding-right:%dpx;padding-bottom:%dpx">`,
|
||
stackLayers*topoStackStep, stackLayers*topoStackStep)
|
||
for i := stackLayers; i >= 1; i-- {
|
||
off := i * topoStackStep
|
||
fmt.Fprintf(&b, `<div style="position:absolute;top:%dpx;left:%dpx;right:0;bottom:0;border-radius:6px;background:%s;border:1px solid %s;opacity:.55"></div>`,
|
||
off, off, item.fillVar, item.strokeVar)
|
||
}
|
||
fmt.Fprintf(&b, `<div%s style="position:relative;cursor:%s;min-width:160px;padding:10px 12px;border-radius:6px;background:%s;border:1px solid %s;color:%s">`,
|
||
onclick, cursor, item.fillVar, item.strokeVar, item.textVar)
|
||
label := item.label
|
||
if item.count > 1 {
|
||
label = fmt.Sprintf("%s ×%d", item.label, item.count)
|
||
}
|
||
fmt.Fprintf(&b, `<div style="font-size:13px;font-weight:700">%s</div>`, html.EscapeString(label))
|
||
if item.sublabel != "" {
|
||
fmt.Fprintf(&b, `<div style="font-size:11px;opacity:.85">%s</div>`, html.EscapeString(item.sublabel))
|
||
}
|
||
if item.statusLine != "" {
|
||
fmt.Fprintf(&b, `<div style="font-size:11px;font-weight:600;margin-top:4px">%s</div>`, html.EscapeString(item.statusLine))
|
||
}
|
||
b.WriteString(`</div></div>`)
|
||
}
|
||
b.WriteString(`</div>`)
|
||
return b.String()
|
||
}
|
||
|
||
func writeTopoBoxSVG(b *strings.Builder, box topoBox) {
|
||
onclick := ""
|
||
cursor := "default"
|
||
if box.detailType != "" {
|
||
onclick = fmt.Sprintf(` onclick="openComponentDetail('%s')"`, box.detailType)
|
||
cursor = "pointer"
|
||
}
|
||
fmt.Fprintf(b, `<g%s style="cursor:%s">`, onclick, cursor)
|
||
|
||
// Stack-of-cards effect: faint offset rects behind the front card when
|
||
// this box represents more than one physical component (e.g. 4 GPUs in
|
||
// one NUMA column), so a group reads as "a deck of N" rather than a
|
||
// single item. Peeks toward the bottom-right, into space already
|
||
// reserved between this box and the next one in the column.
|
||
for i := topoStackLayers(box.count); i >= 1; i-- {
|
||
off := i * topoStackStep
|
||
fmt.Fprintf(b, `<rect x="%d" y="%d" width="%d" height="%d" rx="6" ry="6" style="fill:%s;stroke:%s;opacity:.55"/>`+"\n",
|
||
box.x+off, box.y+off, box.w, box.h, box.fillVar, box.strokeVar)
|
||
}
|
||
|
||
fmt.Fprintf(b, `<rect x="%d" y="%d" width="%d" height="%d" rx="6" ry="6" style="fill:%s;stroke:%s"/>`+"\n",
|
||
box.x, box.y, box.w, box.h, box.fillVar, box.strokeVar)
|
||
|
||
label := box.label
|
||
if box.count > 1 {
|
||
label = fmt.Sprintf("%s ×%d", box.label, box.count)
|
||
}
|
||
fmt.Fprintf(b, `<text x="%d" y="%d" style="fill:%s;font-size:13px;font-weight:700">%s</text>`+"\n",
|
||
box.x+10, box.y+20, box.textVar, html.EscapeString(label))
|
||
if box.sublabel != "" {
|
||
fmt.Fprintf(b, `<text x="%d" y="%d" style="fill:%s;font-size:11px;opacity:.85">%s</text>`+"\n",
|
||
box.x+10, box.y+36, box.textVar, html.EscapeString(truncateTopoLabel(box.sublabel, 26)))
|
||
}
|
||
if box.statusLine != "" {
|
||
fmt.Fprintf(b, `<text x="%d" y="%d" style="fill:%s;font-size:10px;font-weight:600">%s</text>`+"\n",
|
||
box.x+10, box.y+box.h-10, box.textVar, html.EscapeString(box.statusLine))
|
||
}
|
||
b.WriteString(`</g>` + "\n")
|
||
}
|
||
|
||
func truncateTopoLabel(s string, max int) string {
|
||
if len(s) <= max {
|
||
return s
|
||
}
|
||
if max <= 1 {
|
||
return s[:max]
|
||
}
|
||
return s[:max-1] + "…"
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Separate NVLink topology card (read from techdump, not written to any
|
||
// ingest contract)
|
||
// ---------------------------------------------------------------------------
|
||
|
||
type topoNVLinkPort struct {
|
||
Index int
|
||
Active bool
|
||
SpeedGBs *float64
|
||
ReplayErrors int64
|
||
RecoveryErrors int64
|
||
CRCErrors int64
|
||
}
|
||
|
||
var (
|
||
topoNVLinkGPUHeaderRe = regexp.MustCompile(`^GPU (\d+):`)
|
||
topoNVLinkSpeedLineRe = regexp.MustCompile(`^Link (\d+):\s*([\d.]+)\s*GB/s`)
|
||
topoNVLinkInactiveRe = regexp.MustCompile(`^Link (\d+):\s*<inactive>`)
|
||
topoNVLinkErrorLineRe = regexp.MustCompile(`^Link (\d+):\s*(Replay|Recovery|CRC) Errors:\s*(\d+)`)
|
||
)
|
||
|
||
func readTopoNVLinkStatus(exportDir string) (map[int][]topoNVLinkPort, error) {
|
||
raw, err := readTopoTechDump(exportDir, "nvidia-smi-nvlink-status.txt")
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
return parseTopoNVLinkStatus(raw), nil
|
||
}
|
||
|
||
func parseTopoNVLinkStatus(raw string) map[int][]topoNVLinkPort {
|
||
result := map[int][]topoNVLinkPort{}
|
||
currentGPU := -1
|
||
for _, line := range strings.Split(raw, "\n") {
|
||
trimmed := strings.TrimSpace(line)
|
||
if m := topoNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
|
||
currentGPU, _ = strconv.Atoi(m[1])
|
||
continue
|
||
}
|
||
if currentGPU < 0 {
|
||
continue
|
||
}
|
||
if m := topoNVLinkInactiveRe.FindStringSubmatch(trimmed); m != nil {
|
||
idx, _ := strconv.Atoi(m[1])
|
||
result[currentGPU] = append(result[currentGPU], topoNVLinkPort{Index: idx, Active: false})
|
||
continue
|
||
}
|
||
if m := topoNVLinkSpeedLineRe.FindStringSubmatch(trimmed); m != nil {
|
||
idx, _ := strconv.Atoi(m[1])
|
||
port := topoNVLinkPort{Index: idx, Active: true}
|
||
if speed, err := strconv.ParseFloat(m[2], 64); err == nil {
|
||
port.SpeedGBs = &speed
|
||
}
|
||
result[currentGPU] = append(result[currentGPU], port)
|
||
}
|
||
}
|
||
return result
|
||
}
|
||
|
||
func readTopoNVLinkErrors(exportDir string) (map[int]map[int][3]int64, error) {
|
||
raw, err := readTopoTechDump(exportDir, "nvidia-smi-nvlink-errors.txt")
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
return parseTopoNVLinkErrors(raw), nil
|
||
}
|
||
|
||
// parseTopoNVLinkErrors returns, per GPU then link index, [replay, recovery, crc].
|
||
func parseTopoNVLinkErrors(raw string) map[int]map[int][3]int64 {
|
||
result := map[int]map[int][3]int64{}
|
||
currentGPU := -1
|
||
for _, line := range strings.Split(raw, "\n") {
|
||
trimmed := strings.TrimSpace(line)
|
||
if m := topoNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
|
||
currentGPU, _ = strconv.Atoi(m[1])
|
||
continue
|
||
}
|
||
if currentGPU < 0 {
|
||
continue
|
||
}
|
||
m := topoNVLinkErrorLineRe.FindStringSubmatch(trimmed)
|
||
if m == nil {
|
||
continue
|
||
}
|
||
linkIdx, _ := strconv.Atoi(m[1])
|
||
count, _ := strconv.ParseInt(m[3], 10, 64)
|
||
if result[currentGPU] == nil {
|
||
result[currentGPU] = map[int][3]int64{}
|
||
}
|
||
c := result[currentGPU][linkIdx]
|
||
switch m[2] {
|
||
case "Replay":
|
||
c[0] = count
|
||
case "Recovery":
|
||
c[1] = count
|
||
case "CRC":
|
||
c[2] = count
|
||
}
|
||
result[currentGPU][linkIdx] = c
|
||
}
|
||
return result
|
||
}
|
||
|
||
// renderTopoNVLinkCard renders the separate NVLink topology card. Returns ""
|
||
// if there are fewer than 2 NVIDIA GPUs, or the nvidia-smi nvlink techdump
|
||
// wasn't captured (older audit, or nvidia-smi unavailable on that run).
|
||
func renderTopoNVLinkCard(hw schema.HardwareSnapshot, exportDir string) string {
|
||
gpuCount := 0
|
||
for _, dev := range hw.PCIeDevices {
|
||
if dev.DeviceClass != nil && isGPUDeviceClass(*dev.DeviceClass) {
|
||
gpuCount++
|
||
}
|
||
}
|
||
if gpuCount < 2 {
|
||
return ""
|
||
}
|
||
|
||
status, err := readTopoNVLinkStatus(exportDir)
|
||
if err != nil || len(status) == 0 {
|
||
return topoCard("NVLink Topology", `<span class="badge badge-unknown">nvidia-smi nvlink data unavailable</span>`)
|
||
}
|
||
errors, _ := readTopoNVLinkErrors(exportDir)
|
||
|
||
topoMatrix, _ := readGPUTopologyMatrix(exportDir)
|
||
pairs := parseGPUPairAdjacency(topoMatrix)
|
||
|
||
var bodyB strings.Builder
|
||
if gpuCount <= 4 && len(pairs) > 0 {
|
||
// Small GPU count: per-pair box+line with per-link detail.
|
||
for _, pair := range pairs {
|
||
activeCount, total, hasError := 0, 0, false
|
||
for _, port := range status[pair.GPUA] {
|
||
total++
|
||
if port.Active {
|
||
activeCount++
|
||
}
|
||
}
|
||
for _, counters := range errors[pair.GPUA] {
|
||
if counters[0] != 0 || counters[1] != 0 || counters[2] != 0 {
|
||
hasError = true
|
||
}
|
||
}
|
||
color := "var(--ok-fg)"
|
||
switch {
|
||
case hasError:
|
||
color = "var(--crit-fg)"
|
||
case total > 0 && activeCount < total:
|
||
color = "var(--warn-fg)"
|
||
}
|
||
fmt.Fprintf(&bodyB, `<div style="display:flex;align-items:center;gap:12px;margin-bottom:10px">`+
|
||
`<div style="padding:8px 12px;border:1px solid var(--border);border-radius:6px">GPU %d</div>`+
|
||
`<div style="flex:1;height:2px;background:%s"></div>`+
|
||
`<div style="padding:8px 12px;border:1px solid var(--border);border-radius:6px">GPU %d</div>`+
|
||
`<div style="font-size:12px;color:var(--muted)">%d/%d links active%s</div>`+
|
||
`</div>`,
|
||
pair.GPUA, color, pair.GPUB, activeCount, total, errNoteSuffix(hasError))
|
||
}
|
||
} else if len(pairs) > 0 {
|
||
// Larger GPU counts (NVSwitch fabric): aggregate pair table instead of
|
||
// an unreadable all-to-all graph.
|
||
bodyB.WriteString(`<table><thead><tr><th>GPU A</th><th>GPU B</th><th>NVLinks</th></tr></thead><tbody>`)
|
||
for _, pair := range pairs {
|
||
fmt.Fprintf(&bodyB, `<tr><td>GPU %d</td><td>GPU %d</td><td>%d</td></tr>`, pair.GPUA, pair.GPUB, pair.NVLinks)
|
||
}
|
||
bodyB.WriteString(`</tbody></table>`)
|
||
} else {
|
||
bodyB.WriteString(`<span class="badge badge-unknown">No NVLink-bonded GPU pairs found</span>`)
|
||
}
|
||
|
||
return topoCard("NVLink Topology", bodyB.String())
|
||
}
|
||
|
||
func errNoteSuffix(hasError bool) string {
|
||
if hasError {
|
||
return " — errors detected"
|
||
}
|
||
return ""
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Inventory fallback for the component-detail modal
|
||
//
|
||
// handleAPIComponentDetail normally sources records from app.ComponentStatusDB,
|
||
// which only gains entries once something has actually written a status
|
||
// observation (SAT run, watchdog tick, ...). On a freshly booted host that
|
||
// hasn't run SAT yet, StatusDB can be entirely empty for a component type even
|
||
// though the /topo card for it already shows "N OK" — that card reads
|
||
// schema.HardwareComponentStatus.Status straight from the audit snapshot.
|
||
// inventoryFallbackRecords bridges that gap by building synthetic records
|
||
// from the same snapshot/classifiers the topology card uses, so the two
|
||
// views never disagree about how many devices exist or their status.
|
||
// ---------------------------------------------------------------------------
|
||
|
||
// topoSeverityStatus renders classifyTopoSeverity's rank back into the status
|
||
// string vocabulary renderComponentDetail/chipLetterClass expect ("OK",
|
||
// "Warning", "Critical", "Unknown") — kept in lockstep with classifyTopoSeverity
|
||
// so a device the topo card counts as "OK" is never shown here as "Unknown".
|
||
func topoSeverityStatus(status *string) string {
|
||
switch classifyTopoSeverity(status) {
|
||
case 3:
|
||
return "Critical"
|
||
case 2:
|
||
return "Warning"
|
||
case 1:
|
||
return "OK"
|
||
default:
|
||
return "Unknown"
|
||
}
|
||
}
|
||
|
||
// pcieDeviceKind classifies a PCIe device the same way renderTopoMainDiagram
|
||
// does, returning "" for devices that aren't GPU/NIC/RAID.
|
||
func pcieDeviceKind(dev schema.HardwarePCIeDevice) string {
|
||
switch {
|
||
case dev.DeviceClass != nil && isGPUDeviceClass(*dev.DeviceClass):
|
||
return "gpu"
|
||
case isNICDeviceClassDev(dev):
|
||
return "nic"
|
||
case dev.DeviceClass != nil && isRAIDControllerClass(*dev.DeviceClass):
|
||
return "raid"
|
||
default:
|
||
return ""
|
||
}
|
||
}
|
||
|
||
// pcieDeviceKey builds a stable, human-readable component key for a PCIe
|
||
// device: "<kind>:<bdf>" when a slot/BDF is known, else "<kind>:<index>".
|
||
func pcieDeviceKey(kind string, index int, dev schema.HardwarePCIeDevice) string {
|
||
bdf := ""
|
||
if dev.Slot != nil {
|
||
bdf = normalizeTopoBDF(*dev.Slot)
|
||
} else if dev.BDF != nil {
|
||
bdf = normalizeTopoBDF(*dev.BDF)
|
||
}
|
||
if bdf != "" {
|
||
return kind + ":" + bdf
|
||
}
|
||
return fmt.Sprintf("%s:%d", kind, index)
|
||
}
|
||
|
||
// inventoryFallbackRecords builds ComponentStatusRecord entries straight from
|
||
// the audit inventory (bee-audit.json) for the given component type, used
|
||
// when ComponentStatusDB has no matching records yet. Records carry only
|
||
// ComponentKey/Status — no LastCheckedAt/History — so renderComponentDetail
|
||
// renders them without a "checked at" timestamp or sparkline.
|
||
func inventoryFallbackRecords(compType string, opts HandlerOptions) []app.ComponentStatusRecord {
|
||
data, err := loadSnapshot(opts.AuditPath)
|
||
if err != nil {
|
||
return nil
|
||
}
|
||
var ingest schema.HardwareIngestRequest
|
||
if err := json.Unmarshal(data, &ingest); err != nil {
|
||
return nil
|
||
}
|
||
hw := ingest.Hardware
|
||
|
||
var records []app.ComponentStatusRecord
|
||
switch compType {
|
||
case "cpu":
|
||
for i, cpu := range hw.CPUs {
|
||
key := fmt.Sprintf("cpu:%d", i)
|
||
if cpu.Socket != nil {
|
||
key = fmt.Sprintf("cpu:socket%d", *cpu.Socket)
|
||
}
|
||
records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(cpu.Status)})
|
||
}
|
||
case "memory":
|
||
for i, m := range hw.Memory {
|
||
key := fmt.Sprintf("memory:%d", i)
|
||
if m.Slot != nil && strings.TrimSpace(*m.Slot) != "" {
|
||
key = "memory:" + strings.TrimSpace(*m.Slot)
|
||
}
|
||
records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(m.Status)})
|
||
}
|
||
case "storage":
|
||
for i, s := range hw.Storage {
|
||
key := fmt.Sprintf("storage:%d", i)
|
||
if s.Slot != nil && strings.TrimSpace(*s.Slot) != "" {
|
||
key = "storage:" + strings.TrimSpace(*s.Slot)
|
||
}
|
||
records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(s.Status)})
|
||
}
|
||
case "psu":
|
||
for i, p := range hw.PowerSupplies {
|
||
key := fmt.Sprintf("psu:%d", i)
|
||
if p.Slot != nil && strings.TrimSpace(*p.Slot) != "" {
|
||
key = "psu:" + strings.TrimSpace(*p.Slot)
|
||
}
|
||
records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(p.Status)})
|
||
}
|
||
case "gpu", "nic", "raid":
|
||
for i, dev := range hw.PCIeDevices {
|
||
if pcieDeviceKind(dev) != compType {
|
||
continue
|
||
}
|
||
key := pcieDeviceKey(compType, i, dev)
|
||
records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(dev.Status)})
|
||
}
|
||
}
|
||
return records
|
||
}
|