Files
logpile/internal/topology/build.go
T
Mikhail ChusavitinandClaude Opus 5.5 c4870cfb36 fix(topology): attach SNC/NPS NUMA nodes to their real socket
Topology mapped one NUMA node per CPU, so on a 2-socket SNC2 host
(nodes 0-3) node 1 went to CPU1 and nodes 2-3 got no CPU: the second
BlueField-3, four GPUs and several NICs rendered unattached.

Use numa-nodes.txt (node=N package=P), now carried by parser v26 as
TopologyEvidence.NUMANodes; without it, fall back to contiguous
per-socket blocks of zero-based node ids. Pin parser v26.0.0. ADL-082.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-19 13:41:22 +03:00

560 lines
16 KiB
Go

package topology
import (
"encoding/json"
"fmt"
"regexp"
"sort"
"strconv"
"strings"
"git.mchus.pro/mchus/logpile/internal/exporter"
"git.mchus.pro/reanimator/parser/v26/models"
)
const ContractVersion = "1.0"
var (
cpuLocatorRE = regexp.MustCompile(`(?i)(?:^|[_ -])(?:cpu|proc(?:essor)?)\s*0*(\d+)`)
bankNodeRE = regexp.MustCompile(`(?i)node\s*0*(\d+)`)
nvLinkRE = regexp.MustCompile(`(?i)^NV(\d+)$`)
storcliVDRE = regexp.MustCompile(`^PDs for VD (\d+)$`)
storcliPDRE = regexp.MustCompile(`^Drive /c(\d+)/e\d+/s\d+$`)
numaNodeRE = regexp.MustCompile(`node=(\d+)\s+package=(\d+)`)
)
// PCI vendor IDs of BMC-embedded VGA functions (Matrox G200e*, ASPEED AST*).
var bmcVideoVendorIDs = map[int]bool{0x102b: true, 0x1a03: true}
// Build creates the standalone topology.json document from the same
// normalized projection used by the chart, augmented by optional static Bee
// support-bundle evidence. It never executes commands or reads live state.
// controllers are the onboard storage controllers the Reanimator projection
// omits; one becomes a node only when a disk resolves to it.
func Build(snapshot *exporter.ReanimatorExport, evidence *models.TopologyEvidence, controllers ...exporter.ReanimatorPCIe) *models.TopologyDocument {
doc := &models.TopologyDocument{Version: ContractVersion}
if snapshot == nil {
return doc
}
doc.Title = topologyTitle(snapshot)
hw := snapshot.Hardware
cpuIDs := make([]string, 0, len(hw.CPUs))
socketByCPU := make([]int, 0, len(hw.CPUs))
for i, cpu := range hw.CPUs {
socket := cpu.Socket
id := fmt.Sprintf("cpu:%d", socket)
if containsNode(doc.Nodes, id) {
id = fmt.Sprintf("cpu:%d:%d", socket, i)
}
s := socket
doc.Nodes = append(doc.Nodes, models.TopologyNode{ID: id, Kind: "cpu", Label: fmt.Sprintf("CPU %d", socket), Model: cleanCPUModel(cpu.Model), Status: cpu.Status, Socket: &s})
cpuIDs = append(cpuIDs, id)
socketByCPU = append(socketByCPU, socket)
}
diskController := resolveDiskControllers(evidence)
pcieDevices := append([]exporter.ReanimatorPCIe(nil), hw.PCIeDevices...)
for _, ctrl := range controllers {
bdf := normalizeBDF(firstNonEmpty(ctrl.Slot, ctrl.BDF))
for _, disk := range hw.Storage {
if bdf != "" && diskController[strings.TrimSpace(disk.Slot)] == bdf {
pcieDevices = append(pcieDevices, ctrl)
break
}
}
}
numaToCPU := buildNUMAMap(pcieDevices, cpuIDs, socketByCPU)
for node, pkg := range parseNUMAPackages(evidenceString(evidence, func(e *models.TopologyEvidence) string { return e.NUMANodes })) {
if cpuID := memoryCPU(pkg, cpuIDs, socketByCPU); cpuID != "" {
numaToCPU[node] = cpuID
}
}
bdfToNode := map[string]string{}
gpuNodeByBDF := map[string]string{}
for i, dev := range pcieDevices {
kind := pcieKind(dev)
if kind == "" {
continue
}
bdf := normalizeBDF(firstNonEmpty(dev.Slot, dev.BDF))
if bdfToNode[bdf] != "" {
continue
}
id := fmt.Sprintf("%s:%s", kind, bdf)
if bdf == "" {
id = fmt.Sprintf("%s:%d", kind, i)
}
node := models.TopologyNode{ID: id, Kind: kind, Label: strings.ToUpper(kind), Model: dev.Model, Status: dev.Status, NUMANode: dev.NUMANode, BDF: bdf, LinkSpeed: dev.LinkSpeed, MaxLinkSpeed: dev.MaxLinkSpeed}
doc.Nodes = append(doc.Nodes, node)
if bdf != "" {
bdfToNode[bdf] = id
if kind == "gpu" {
gpuNodeByBDF[bdf] = id
}
}
if dev.NUMANode != nil {
if cpuID := numaToCPU[*dev.NUMANode]; cpuID != "" {
doc.Edges = append(doc.Edges, models.TopologyEdge{From: cpuID, To: id, Kind: "pcie", Status: linkStatus(dev.LinkSpeed, dev.MaxLinkSpeed)})
}
}
}
bankNodes := parseDIMMBankNodes(evidenceString(evidence, func(e *models.TopologyEvidence) string { return e.DIMMType17 }))
for i, mem := range hw.Memory {
id := fmt.Sprintf("memory:%s", strings.TrimSpace(mem.Slot))
if strings.TrimSpace(mem.Slot) == "" || containsNode(doc.Nodes, id) {
id = fmt.Sprintf("memory:%d", i)
}
node := models.TopologyNode{ID: id, Kind: "memory", Label: "Memory", Model: mem.PartNumber, Status: mem.Status, SizeMB: mem.SizeMB}
doc.Nodes = append(doc.Nodes, node)
if rawNode, ok := memoryNode(mem.Slot, mem.Location, bankNodes); ok {
if cpuID := memoryCPU(rawNode, cpuIDs, socketByCPU); cpuID != "" {
doc.Edges = append(doc.Edges, models.TopologyEdge{From: cpuID, To: id, Kind: "memory"})
}
}
}
for i, disk := range hw.Storage {
id := fmt.Sprintf("storage:%s", strings.TrimSpace(disk.Slot))
if strings.TrimSpace(disk.Slot) == "" || containsNode(doc.Nodes, id) {
id = fmt.Sprintf("storage:%d", i)
}
doc.Nodes = append(doc.Nodes, models.TopologyNode{ID: id, Kind: "storage", Label: firstNonEmpty(disk.Type, "Disk"), Model: disk.Model, Status: disk.Status})
if controller := bdfToNode[diskController[strings.TrimSpace(disk.Slot)]]; controller != "" {
doc.Edges = append(doc.Edges, models.TopologyEdge{From: controller, To: id, Kind: "storage"})
}
}
for i, psu := range hw.PowerSupplies {
label := strings.TrimSpace(psu.Slot)
if label == "" {
label = fmt.Sprintf("PSU %d", i+1)
} else if _, err := strconv.Atoi(label); err == nil {
label = "PSU " + label
}
doc.Nodes = append(doc.Nodes, models.TopologyNode{ID: fmt.Sprintf("psu:%d", i), Kind: "psu", Label: label, Model: psu.Model, Status: psu.Status, WattageW: psu.WattageW})
}
for i, fw := range hw.Firmware {
doc.Nodes = append(doc.Nodes, models.TopologyNode{ID: fmt.Sprintf("firmware:%d", i), Kind: "firmware", Label: fw.DeviceName, Model: "fw " + fw.Version})
}
addNVLinkEdges(doc, evidence, gpuNodeByBDF)
return doc
}
func topologyTitle(s *exporter.ReanimatorExport) string {
parts := []string{strings.TrimSpace(s.Hardware.Board.ProductName), strings.TrimSpace(s.Hardware.Board.SerialNumber)}
var out []string
for _, p := range parts {
if p != "" {
out = append(out, p)
}
}
if len(out) == 0 {
return "Server topology"
}
return strings.Join(out, " - ")
}
func buildNUMAMap(devices []exporter.ReanimatorPCIe, cpuIDs []string, sockets []int) map[int]string {
result := map[int]string{}
if len(cpuIDs) == 0 {
return result
}
seen, hasZero := map[int]bool{}, false
for _, d := range devices {
if d.NUMANode != nil {
seen[*d.NUMANode] = true
hasZero = hasZero || *d.NUMANode == 0
}
}
var nodes []int
for n := range seen {
nodes = append(nodes, n)
}
sort.Ints(nodes)
// Zero-based NUMA ids: with Sub-NUMA Clustering a socket owns several
// consecutive nodes (SNC2 on 2 sockets: 0,1 -> CPU0, 2,3 -> CPU1).
perSocket := 1
if hasZero {
perSocket = (nodes[len(nodes)-1] + len(cpuIDs)) / len(cpuIDs)
}
for rank, n := range nodes {
if hasZero && n/perSocket < len(cpuIDs) {
result[n] = cpuIDs[n/perSocket]
continue
}
for i, socket := range sockets {
if socket == n {
result[n] = cpuIDs[i]
break
}
}
if result[n] == "" && rank < len(cpuIDs) {
result[n] = cpuIDs[rank]
}
}
return result
}
// parseNUMAPackages reads numa-nodes.txt "node=N package=P" lines: the exact
// socket of every NUMA node, which the guess above lacks under SNC/NPS.
func parseNUMAPackages(raw string) map[int]int {
out := map[int]int{}
for _, m := range numaNodeRE.FindAllStringSubmatch(raw, -1) {
node, _ := strconv.Atoi(m[1])
pkg, _ := strconv.Atoi(m[2])
out[node] = pkg
}
return out
}
func pcieKind(d exporter.ReanimatorPCIe) string {
c := strings.ToLower(strings.TrimSpace(d.DeviceClass))
model := strings.ToLower(d.Model + " " + d.Manufacturer)
video := strings.Contains(c, "video") || strings.Contains(c, "display")
switch {
case video && bmcVideoVendorIDs[d.VendorID]:
// The BMC's own VGA function is not a GPU of the server.
return ""
case video, strings.Contains(c, "3d"), strings.Contains(c, "processingaccelerator"), strings.Contains(model, "nvidia") && strings.Contains(model, "gpu"):
return "gpu"
case strings.Contains(c, "network"), strings.Contains(c, "ethernet"), strings.Contains(c, "fibrechannel"), len(d.MACAddresses) > 0:
return "nic"
case strings.Contains(c, "storage"), strings.Contains(c, "raid"):
return "raid"
default:
return ""
}
}
func linkStatus(current, maximum string) string {
a, b := speedRank(current), speedRank(maximum)
if a == 0 || b == 0 {
return "Unknown"
}
if a < b {
return "Warning"
}
return "OK"
}
func speedRank(v string) float64 {
v = strings.ToLower(strings.TrimSpace(v))
v = strings.TrimPrefix(v, "gen")
for _, suffix := range []string{" gt/s", "gts", "gtps"} {
v = strings.TrimSuffix(v, suffix)
}
n, _ := strconv.ParseFloat(strings.TrimSpace(v), 64)
return n
}
func parseDIMMBankNodes(raw string) map[string]int {
out := map[string]int{}
for _, sec := range strings.Split(raw, "Memory Device") {
locator, node := "", -1
for _, line := range strings.Split(sec, "\n") {
line = strings.TrimSpace(line)
if v, ok := strings.CutPrefix(line, "Locator:"); ok {
locator = strings.TrimSpace(v)
}
if v, ok := strings.CutPrefix(line, "Bank Locator:"); ok {
if m := bankNodeRE.FindStringSubmatch(v); m != nil {
node, _ = strconv.Atoi(m[1])
}
}
}
if locator != "" && node >= 0 {
out[locator] = node
}
}
return out
}
func memoryNode(slot, location string, bank map[string]int) (int, bool) {
for _, value := range []string{slot, location} {
if m := cpuLocatorRE.FindStringSubmatch(strings.TrimSpace(value)); m != nil {
n, _ := strconv.Atoi(m[1])
return n, true
}
if n, ok := bank[strings.TrimSpace(value)]; ok {
return n, true
}
if m := bankNodeRE.FindStringSubmatch(value); m != nil {
n, _ := strconv.Atoi(m[1])
return n, true
}
}
return 0, false
}
func memoryCPU(node int, cpuIDs []string, sockets []int) string {
for i, s := range sockets {
if s == node {
return cpuIDs[i]
}
}
if node >= 0 && node < len(cpuIDs) {
return cpuIDs[node]
}
if node > 0 && node <= len(cpuIDs) {
return cpuIDs[node-1]
}
return ""
}
// resolveDiskControllers maps a storage slot to the BDF of its controller.
// Directly attached disks are keyed by SCSI HCTL. Physical drives behind a
// RAID controller have no block device of their own, so their storcli
// "EID:Slt" slot is resolved through the virtual drive they belong to:
// PD -> VD -> "OS Drive Name" -> block device -> controller.
func resolveDiskControllers(evidence *models.TopologyEvidence) map[string]string {
if evidence == nil {
return map[string]string{}
}
out, byBlockDev := parseStorageMap(evidence.StorageMap)
byStorcliCtrl := map[int]string{}
for _, ctrl := range storcliControllers(evidence.StorcliVDs) {
for key, raw := range ctrl.data {
m := storcliVDRE.FindStringSubmatch(key)
if m == nil {
continue
}
var props map[string]any
_ = json.Unmarshal(ctrl.data["VD"+m[1]+" Properties"], &props)
osDrive, _ := props["OS Drive Name"].(string)
bdf := byBlockDev[strings.TrimPrefix(strings.TrimSpace(osDrive), "/dev/")]
if bdf == "" {
continue
}
byStorcliCtrl[ctrl.index] = bdf
for _, slot := range storcliSlots(raw) {
if out[slot] == "" {
out[slot] = bdf
}
}
}
}
// Drives outside any virtual drive (unconfigured, JBOD, spare) still sit on
// a controller whose BDF a sibling virtual drive has revealed.
for _, ctrl := range storcliControllers(evidence.StorcliDrives) {
for key, raw := range ctrl.data {
m := storcliPDRE.FindStringSubmatch(key)
if m == nil {
continue
}
index, _ := strconv.Atoi(m[1])
bdf := byStorcliCtrl[index]
if bdf == "" {
continue
}
for _, slot := range storcliSlots(raw) {
if out[slot] == "" {
out[slot] = bdf
}
}
}
}
return out
}
type storcliController struct {
index int
data map[string]json.RawMessage
}
func storcliControllers(raw string) []storcliController {
var doc struct {
Controllers []struct {
Status struct {
Controller int `json:"Controller"`
} `json:"Command Status"`
Data map[string]json.RawMessage `json:"Response Data"`
} `json:"Controllers"`
}
if strings.TrimSpace(raw) == "" || json.Unmarshal([]byte(raw), &doc) != nil {
return nil
}
out := make([]storcliController, 0, len(doc.Controllers))
for _, c := range doc.Controllers {
out = append(out, storcliController{index: c.Status.Controller, data: c.Data})
}
return out
}
func storcliSlots(raw json.RawMessage) []string {
var rows []map[string]any
if json.Unmarshal(raw, &rows) != nil {
return nil
}
var out []string
for _, row := range rows {
if slot, _ := row["EID:Slt"].(string); strings.TrimSpace(slot) != "" {
out = append(out, strings.TrimSpace(slot))
}
}
return out
}
// parseStorageMap reads "sda hctl=0:3:111:0 ctrl=0000:6d:00.0" lines into
// controller BDFs keyed by HCTL and by block device name.
func parseStorageMap(raw string) (byHCTL, byBlockDev map[string]string) {
byHCTL, byBlockDev = map[string]string{}, map[string]string{}
for _, line := range strings.Split(raw, "\n") {
hctl, ctrl := "", ""
fields := strings.Fields(line)
for _, field := range fields {
if v, ok := strings.CutPrefix(field, "hctl="); ok {
hctl = v
}
if v, ok := strings.CutPrefix(field, "ctrl="); ok {
ctrl = normalizeBDF(v)
}
}
if ctrl == "" {
continue
}
if hctl != "" {
byHCTL[hctl] = ctrl
}
if !strings.Contains(fields[0], "=") {
byBlockDev[fields[0]] = ctrl
}
}
return byHCTL, byBlockDev
}
func addNVLinkEdges(doc *models.TopologyDocument, evidence *models.TopologyEvidence, gpuByBDF map[string]string) {
if evidence == nil || evidence.NVIDIATopology == "" || evidence.NVIDIAQueryCSV == "" {
return
}
idxToNode := map[int]string{}
for _, line := range strings.Split(evidence.NVIDIAQueryCSV, "\n") {
parts := strings.Split(line, ",")
if len(parts) < 2 {
continue
}
idx, err := strconv.Atoi(strings.TrimSpace(parts[0]))
if err != nil {
continue
}
if id := gpuByBDF[normalizeBDF(parts[1])]; id != "" {
idxToNode[idx] = id
}
}
for idx, id := range idxToNode {
for i := range doc.Nodes {
if doc.Nodes[i].ID == id {
doc.Nodes[i].Label = fmt.Sprintf("GPU %d", idx)
break
}
}
}
lines := strings.Split(evidence.NVIDIATopology, "\n")
var gpuCols []int
header := -1
for i, line := range lines {
fields := strings.Fields(line)
if len(fields) > 1 && fields[0] == "GPU0" {
for _, f := range fields {
if strings.HasPrefix(f, "GPU") {
n, err := strconv.Atoi(strings.TrimPrefix(f, "GPU"))
if err == nil {
gpuCols = append(gpuCols, n)
}
}
}
header = i
break
}
}
if header < 0 {
return
}
seen := map[[2]int]bool{}
nvStatus := "OK"
if strings.Contains(strings.ToLower(evidence.NVLinkStatus), "<inactive>") {
nvStatus = "Warning"
}
if nvlinkHasErrors(evidence.NVLinkErrors) {
nvStatus = "Critical"
}
for _, line := range lines[header+1:] {
fields := strings.Fields(line)
if len(fields) < 2 || !strings.HasPrefix(fields[0], "GPU") {
continue
}
row, err := strconv.Atoi(strings.TrimPrefix(fields[0], "GPU"))
if err != nil {
continue
}
for col, gpu := range gpuCols {
if row == gpu || col+1 >= len(fields) {
continue
}
m := nvLinkRE.FindStringSubmatch(fields[col+1])
if m == nil {
continue
}
links, _ := strconv.Atoi(m[1])
a, b := row, gpu
if a > b {
a, b = b, a
}
key := [2]int{a, b}
if seen[key] || idxToNode[a] == "" || idxToNode[b] == "" {
continue
}
seen[key] = true
doc.Edges = append(doc.Edges, models.TopologyEdge{From: idxToNode[a], To: idxToNode[b], Kind: "nvlink", Status: nvStatus, Label: fmt.Sprintf("NV%d", links), Links: links})
}
}
}
func nvlinkHasErrors(raw string) bool {
for _, line := range strings.Split(raw, "\n") {
lower := strings.ToLower(line)
if !strings.Contains(lower, "error") {
continue
}
fields := strings.Fields(line)
if len(fields) == 0 {
continue
}
last := strings.Trim(fields[len(fields)-1], " ,")
if n, err := strconv.ParseInt(last, 10, 64); err == nil && n > 0 {
return true
}
}
return false
}
func evidenceString(e *models.TopologyEvidence, get func(*models.TopologyEvidence) string) string {
if e == nil {
return ""
}
return get(e)
}
func containsNode(nodes []models.TopologyNode, id string) bool {
for _, n := range nodes {
if n.ID == id {
return true
}
}
return false
}
func normalizeBDF(v string) string { return strings.ToLower(strings.TrimSpace(v)) }
func firstNonEmpty(values ...string) string {
for _, v := range values {
if strings.TrimSpace(v) != "" {
return strings.TrimSpace(v)
}
}
return ""
}
func cleanCPUModel(v string) string {
return strings.Join(strings.Fields(strings.NewReplacer("(R)", "", "(TM)", "", " CPU", "", " Processor", "").Replace(v)), " ")
}