webui/topo: attach memory DIMMs to their CPU column in the topology diagram
Memory used to render as one unattached row below the whole diagram
regardless of which socket it belonged to. schema.HardwareMemory has no
NUMANode field (unlike PCIe devices), so CPU affinity is instead read out of
the DIMM's own Locator string: either a CPU number encoded directly in it
("CPU0_DIMM_A1"), or — when the Locator has no CPU number of its own, e.g.
"DIMM000(A)" — a node number from Bank Locator ("_Node1_Channel0_Dimm0"),
read from the persisted dmidecode-type17.txt techdump the same way the
NVLink card already reads extra techdump for visualization only.
DIMMs that can't be attached to a column via either heuristic still fall
back to the old unattached "Memory" row so nothing silently disappears.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
c1f0f7824e
commit
41f683de2b
@@ -147,6 +147,103 @@ func buildSocketIndex(cpus []schema.HardwareCPU) map[int]int {
|
||||
return idx
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// DIMM -> CPU column attachment
|
||||
//
|
||||
// schema.HardwareMemory has no NUMANode field (unlike HardwarePCIeDevice), so
|
||||
// unlike the GPU/NIC/RAID placement above, a DIMM's CPU affinity has to be
|
||||
// read out of its own Locator/Bank Locator strings — DMI type 17 gives no
|
||||
// other hint. Both patterns below have been observed on real boards.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
var (
|
||||
topoMemCPULocatorRe = regexp.MustCompile(`(?i)^cpu\s*0*(\d+)`)
|
||||
topoMemBankNodeRe = regexp.MustCompile(`(?i)node\s*0*(\d+)`)
|
||||
)
|
||||
|
||||
// parseDIMMBankLocatorNodes maps a DIMM's Locator (matches
|
||||
// schema.HardwareMemory.Slot) to the node number embedded in its Bank
|
||||
// Locator field (e.g. "_Node1_Channel0_Dimm0"), read from a raw
|
||||
// "dmidecode -t 17" techdump capture. Bank Locator never reaches audit.json
|
||||
// (schema.HardwareMemory.Location is json:"-", used only for internal DIMM
|
||||
// telemetry matching), so boards whose Locator has no CPU number of its own
|
||||
// (e.g. "DIMM000(A)" rather than "CPU0_DIMM_A1") need this fallback to
|
||||
// attach a DIMM to a CPU column at all — matches the existing convention of
|
||||
// reading extra techdump for this page's visualization only (see
|
||||
// readTopoTechDump).
|
||||
func parseDIMMBankLocatorNodes(raw string) map[string]int {
|
||||
result := map[string]int{}
|
||||
for _, sec := range strings.Split(raw, "Memory Device") {
|
||||
var locator string
|
||||
node := -1
|
||||
for _, line := range strings.Split(sec, "\n") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if v, ok := strings.CutPrefix(trimmed, "Locator:"); ok {
|
||||
locator = strings.TrimSpace(v)
|
||||
}
|
||||
if v, ok := strings.CutPrefix(trimmed, "Bank Locator:"); ok {
|
||||
if m := topoMemBankNodeRe.FindStringSubmatch(v); m != nil {
|
||||
if n, err := strconv.Atoi(m[1]); err == nil {
|
||||
node = n
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if locator != "" && node >= 0 {
|
||||
result[locator] = node
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// dimmRawNode returns the raw (vendor-numbered, not yet column-mapped)
|
||||
// CPU/NUMA node number for a DIMM, trying two heuristics in order:
|
||||
// 1. A CPU number encoded directly in the Locator itself, e.g.
|
||||
// "CPU0_DIMM_A1".
|
||||
// 2. A node number from the Bank Locator via parseDIMMBankLocatorNodes,
|
||||
// e.g. Locator "DIMM000(A)" whose Bank Locator is
|
||||
// "_Node1_Channel0_Dimm0" -> 1.
|
||||
// ok=false means neither pattern matched, so this DIMM can't be confidently
|
||||
// attached to a CPU column and falls back to the unattached Memory row.
|
||||
func dimmRawNode(mem schema.HardwareMemory, bankNodeByLocator map[string]int) (int, bool) {
|
||||
if mem.Slot == nil {
|
||||
return 0, false
|
||||
}
|
||||
if m := topoMemCPULocatorRe.FindStringSubmatch(*mem.Slot); m != nil {
|
||||
if n, err := strconv.Atoi(m[1]); err == nil {
|
||||
return n, true
|
||||
}
|
||||
}
|
||||
if n, ok := bankNodeByLocator[*mem.Slot]; ok {
|
||||
return n, true
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// buildMemoryColumnIndex ranks the distinct raw node numbers seen across all
|
||||
// DIMMs and maps the i-th smallest to column i — the same "node order
|
||||
// follows socket order" assumption buildSocketIndex makes for PCIe NUMA
|
||||
// nodes, but computed independently from PCIe's own numbering: PCIe's
|
||||
// NUMANode is 0-based Linux numbering, while a DIMM's Bank Locator "NodeN"
|
||||
// has been observed 1-based on at least one real board, so the two node
|
||||
// spaces are not guaranteed to share a base.
|
||||
func buildMemoryColumnIndex(rawNodes []int) map[int]int {
|
||||
seen := map[int]bool{}
|
||||
var distinct []int
|
||||
for _, n := range rawNodes {
|
||||
if !seen[n] {
|
||||
seen[n] = true
|
||||
distinct = append(distinct, n)
|
||||
}
|
||||
}
|
||||
sort.Ints(distinct)
|
||||
idx := map[int]int{}
|
||||
for col, n := range distinct {
|
||||
idx[n] = col
|
||||
}
|
||||
return idx
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// GPU pairwise NVLink adjacency (from a live "nvidia-smi topo -m" query)
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -483,6 +580,11 @@ type topoBox struct {
|
||||
topoCardInfo
|
||||
}
|
||||
|
||||
type topoEdge struct {
|
||||
x1, y1, x2, y2 int
|
||||
color string
|
||||
}
|
||||
|
||||
func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string {
|
||||
socketIdx := buildSocketIndex(hw.CPUs)
|
||||
numCols := len(hw.CPUs)
|
||||
@@ -581,11 +683,41 @@ func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string
|
||||
kindOrder := []string{"gpu", "nic", "raid"}
|
||||
kindLabel := map[string]string{"gpu": "GPU", "nic": "NIC", "raid": "RAID"}
|
||||
|
||||
var boxes []topoBox
|
||||
var pcieEdges []struct {
|
||||
x1, y1, x2, y2 int
|
||||
color string
|
||||
// Attach memory DIMMs to their CPU column too, the same way GPU/NIC/RAID
|
||||
// PCIe devices are attached via NUMANode — memory has no NUMANode field
|
||||
// in the schema, so this reads the DIMM's own Locator/Bank Locator
|
||||
// strings instead (see dimmRawNode). DIMMs that can't be confidently
|
||||
// attached fall back to the unattached "Memory" row below the diagram,
|
||||
// same as before this existed.
|
||||
memBankNodes := map[string]int{}
|
||||
if raw, err := readTopoTechDump(exportDir, "dmidecode-type17.txt"); err == nil {
|
||||
memBankNodes = parseDIMMBankLocatorNodes(raw)
|
||||
}
|
||||
memCol := make([]int, len(hw.Memory))
|
||||
memMatched := make([]bool, len(hw.Memory))
|
||||
var memRawNodes []int
|
||||
for i, m := range hw.Memory {
|
||||
if node, ok := dimmRawNode(m, memBankNodes); ok {
|
||||
memCol[i] = node
|
||||
memMatched[i] = true
|
||||
memRawNodes = append(memRawNodes, node)
|
||||
}
|
||||
}
|
||||
memColIdx := buildMemoryColumnIndex(memRawNodes)
|
||||
for i := range hw.Memory {
|
||||
if !memMatched[i] {
|
||||
continue
|
||||
}
|
||||
col := memColIdx[memCol[i]]
|
||||
if col >= numCols {
|
||||
memMatched[i] = false
|
||||
continue
|
||||
}
|
||||
memCol[i] = col
|
||||
}
|
||||
|
||||
var boxes []topoBox
|
||||
var pcieEdges []topoEdge
|
||||
|
||||
for col := 0; col < totalCols; col++ {
|
||||
colX := (col+1)*24 + col*topoColWidth
|
||||
@@ -614,6 +746,50 @@ func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string
|
||||
}
|
||||
|
||||
y := topoTopMargin + topoBoxHeight + topoDeviceGap*2
|
||||
|
||||
// Memory goes first in the chain, directly under the CPU box: DIMMs
|
||||
// are wired straight to the socket's memory controller, not reached
|
||||
// over PCIe like the GPU/NIC/RAID chain below it.
|
||||
var memGroup []schema.HardwareMemory
|
||||
for i, m := range hw.Memory {
|
||||
if memMatched[i] && memCol[i] == col {
|
||||
memGroup = append(memGroup, m)
|
||||
}
|
||||
}
|
||||
if len(memGroup) > 0 {
|
||||
var tally topoStatusTally
|
||||
sizeGB := 0
|
||||
for _, m := range memGroup {
|
||||
tally.add(classifyTopoSeverity(m.Status))
|
||||
if m.SizeMB != nil {
|
||||
sizeGB += *m.SizeMB / 1024
|
||||
}
|
||||
}
|
||||
fill, stroke, text := topoSeverityColors(tally.worst())
|
||||
sublabel := ""
|
||||
if sizeGB > 0 {
|
||||
sublabel = fmt.Sprintf("%d GB total", sizeGB)
|
||||
}
|
||||
stackLayers := topoStackLayers(len(memGroup))
|
||||
boxes = append(boxes, topoBox{
|
||||
x: colX, y: y, w: topoBoxWidth, h: topoBoxHeight,
|
||||
topoCardInfo: topoCardInfo{
|
||||
label: "Memory", sublabel: sublabel, count: len(memGroup),
|
||||
statusLine: tally.line(),
|
||||
fillVar: fill, strokeVar: stroke, textVar: text,
|
||||
detailType: "memory",
|
||||
},
|
||||
})
|
||||
if col < len(hw.CPUs) {
|
||||
pcieEdges = append(pcieEdges, topoEdge{
|
||||
x1: colX + topoBoxWidth/2, y1: topoTopMargin + topoBoxHeight,
|
||||
x2: colX + topoBoxWidth/2, y2: y,
|
||||
color: "var(--ok-fg)",
|
||||
})
|
||||
}
|
||||
y += topoBoxHeight + topoDeviceGap + stackLayers*topoStackStep
|
||||
}
|
||||
|
||||
for _, kind := range kindOrder {
|
||||
var group []placedDevice
|
||||
for _, p := range placed {
|
||||
@@ -654,10 +830,7 @@ func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string
|
||||
})
|
||||
|
||||
if col < len(hw.CPUs) {
|
||||
pcieEdges = append(pcieEdges, struct {
|
||||
x1, y1, x2, y2 int
|
||||
color string
|
||||
}{
|
||||
pcieEdges = append(pcieEdges, topoEdge{
|
||||
x1: colX + topoBoxWidth/2, y1: topoTopMargin + topoBoxHeight,
|
||||
x2: colX + topoBoxWidth/2, y2: y,
|
||||
color: edgeColor,
|
||||
@@ -695,21 +868,31 @@ func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string
|
||||
}
|
||||
b.WriteString(`</svg></div>`)
|
||||
|
||||
// Memory, Firmware (BMC/BIOS/...) and PSUs have no PCIe/CPU affinity to
|
||||
// anchor them to a column, and there can be an arbitrary number of any
|
||||
// of them — so unlike the diagram above, they're plain flex-wrap HTML
|
||||
// below the SVG rather than absolutely-positioned SVG boxes. A
|
||||
// fixed-size SVG canvas has no way to wrap overflow onto a new row,
|
||||
// which is exactly what caused these to pile up and overlap once a
|
||||
// board had more PSUs/firmware records than fit in one fixed-width row.
|
||||
if len(hw.Memory) > 0 {
|
||||
// Firmware (BMC/BIOS/...) and PSUs have no PCIe/CPU affinity to anchor
|
||||
// them to a column, and there can be an arbitrary number of any of them
|
||||
// — so unlike the diagram above, they're plain flex-wrap HTML below the
|
||||
// SVG rather than absolutely-positioned SVG boxes. A fixed-size SVG
|
||||
// canvas has no way to wrap overflow onto a new row, which is exactly
|
||||
// what caused these to pile up and overlap once a board had more
|
||||
// PSUs/firmware records than fit in one fixed-width row.
|
||||
//
|
||||
// Memory DIMMs that were matched to a CPU column above already got a
|
||||
// box in the SVG diagram; only DIMMs that couldn't be attached to a
|
||||
// column (see memMatched above) fall back to this row.
|
||||
var unmatchedMem []schema.HardwareMemory
|
||||
for i, m := range hw.Memory {
|
||||
if !memMatched[i] {
|
||||
unmatchedMem = append(unmatchedMem, m)
|
||||
}
|
||||
}
|
||||
if len(unmatchedMem) > 0 {
|
||||
var tally topoStatusTally
|
||||
for _, m := range hw.Memory {
|
||||
for _, m := range unmatchedMem {
|
||||
tally.add(classifyTopoSeverity(m.Status))
|
||||
}
|
||||
fill, stroke, text := topoSeverityColors(tally.worst())
|
||||
sizeGB := 0
|
||||
for _, m := range hw.Memory {
|
||||
for _, m := range unmatchedMem {
|
||||
if m.SizeMB != nil {
|
||||
sizeGB += *m.SizeMB / 1024
|
||||
}
|
||||
@@ -719,7 +902,7 @@ func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string
|
||||
sublabel = fmt.Sprintf("%d GB total", sizeGB)
|
||||
}
|
||||
b.WriteString(renderTopoFlexRow("Memory", []topoCardInfo{{
|
||||
label: "Memory", sublabel: sublabel, count: len(hw.Memory),
|
||||
label: "Memory", sublabel: sublabel, count: len(unmatchedMem),
|
||||
statusLine: tally.line(),
|
||||
fillVar: fill, strokeVar: stroke, textVar: text,
|
||||
detailType: "memory",
|
||||
|
||||
Reference in New Issue
Block a user