fix(topology): attach HPE DIMMs and RAID member disks; import easy-bee event logs
HPE DL380 Gen11 BEE-SP v14.23 blackbox bundle (ADL-077): - topology: accept "PROC n" DIMM locators, show the storage controller filtered from the export by ADL-062 when disks resolve to it, resolve storcli EID:Slt drives via VD -> OS drive -> controller BDF, hide BMC VGA, label numeric PSU slots "PSU n" - easy_bee: parse SEL / dmesg -T event times, keep message_id and component_ref, pass platform_config through - exporter: map ipmi-sel -> bmc and dmesg/kernel -> host in event_logs - privacy: treat journald/syslog kernel lines as noise (linux.it false positive) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
9f5f7a98be
commit
ff5b99fa5c
15 files changed
+349
-42
No files matched your search
+143
-13
@@ -1,6 +1,7 @@
|
||||
package topology
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
@@ -14,15 +15,22 @@ import (
|
||||
const ContractVersion = "1.0"
|
||||
|
||||
var (
|
||||
cpuLocatorRE = regexp.MustCompile(`(?i)(?:^|[_ -])cpu\s*0*(\d+)`)
|
||||
cpuLocatorRE = regexp.MustCompile(`(?i)(?:^|[_ -])(?:cpu|proc(?:essor)?)\s*0*(\d+)`)
|
||||
bankNodeRE = regexp.MustCompile(`(?i)node\s*0*(\d+)`)
|
||||
nvLinkRE = regexp.MustCompile(`(?i)^NV(\d+)$`)
|
||||
storcliVDRE = regexp.MustCompile(`^PDs for VD (\d+)$`)
|
||||
storcliPDRE = regexp.MustCompile(`^Drive /c(\d+)/e\d+/s\d+$`)
|
||||
)
|
||||
|
||||
// PCI vendor IDs of BMC-embedded VGA functions (Matrox G200e*, ASPEED AST*).
|
||||
var bmcVideoVendorIDs = map[int]bool{0x102b: true, 0x1a03: true}
|
||||
|
||||
// Build creates the standalone topology.json document from the same
|
||||
// normalized projection used by the chart, augmented by optional static Bee
|
||||
// support-bundle evidence. It never executes commands or reads live state.
|
||||
func Build(snapshot *exporter.ReanimatorExport, evidence *models.TopologyEvidence) *models.TopologyDocument {
|
||||
// controllers are the onboard storage controllers the Reanimator projection
|
||||
// omits; one becomes a node only when a disk resolves to it.
|
||||
func Build(snapshot *exporter.ReanimatorExport, evidence *models.TopologyEvidence, controllers ...exporter.ReanimatorPCIe) *models.TopologyDocument {
|
||||
doc := &models.TopologyDocument{Version: ContractVersion}
|
||||
if snapshot == nil {
|
||||
return doc
|
||||
@@ -44,15 +52,30 @@ func Build(snapshot *exporter.ReanimatorExport, evidence *models.TopologyEvidenc
|
||||
socketByCPU = append(socketByCPU, socket)
|
||||
}
|
||||
|
||||
numaToCPU := buildNUMAMap(hw.PCIeDevices, cpuIDs, socketByCPU)
|
||||
diskController := resolveDiskControllers(evidence)
|
||||
pcieDevices := append([]exporter.ReanimatorPCIe(nil), hw.PCIeDevices...)
|
||||
for _, ctrl := range controllers {
|
||||
bdf := normalizeBDF(firstNonEmpty(ctrl.Slot, ctrl.BDF))
|
||||
for _, disk := range hw.Storage {
|
||||
if bdf != "" && diskController[strings.TrimSpace(disk.Slot)] == bdf {
|
||||
pcieDevices = append(pcieDevices, ctrl)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
numaToCPU := buildNUMAMap(pcieDevices, cpuIDs, socketByCPU)
|
||||
bdfToNode := map[string]string{}
|
||||
gpuNodeByBDF := map[string]string{}
|
||||
for i, dev := range hw.PCIeDevices {
|
||||
for i, dev := range pcieDevices {
|
||||
kind := pcieKind(dev)
|
||||
if kind == "" {
|
||||
continue
|
||||
}
|
||||
bdf := normalizeBDF(firstNonEmpty(dev.Slot, dev.BDF))
|
||||
if bdfToNode[bdf] != "" {
|
||||
continue
|
||||
}
|
||||
id := fmt.Sprintf("%s:%s", kind, bdf)
|
||||
if bdf == "" {
|
||||
id = fmt.Sprintf("%s:%d", kind, i)
|
||||
@@ -87,14 +110,13 @@ func Build(snapshot *exporter.ReanimatorExport, evidence *models.TopologyEvidenc
|
||||
}
|
||||
}
|
||||
|
||||
controllerByHCTL := parseStorageMap(evidenceString(evidence, func(e *models.TopologyEvidence) string { return e.StorageMap }))
|
||||
for i, disk := range hw.Storage {
|
||||
id := fmt.Sprintf("storage:%s", strings.TrimSpace(disk.Slot))
|
||||
if strings.TrimSpace(disk.Slot) == "" || containsNode(doc.Nodes, id) {
|
||||
id = fmt.Sprintf("storage:%d", i)
|
||||
}
|
||||
doc.Nodes = append(doc.Nodes, models.TopologyNode{ID: id, Kind: "storage", Label: firstNonEmpty(disk.Type, "Disk"), Model: disk.Model, Status: disk.Status})
|
||||
if controller := bdfToNode[controllerByHCTL[strings.TrimSpace(disk.Slot)]]; controller != "" {
|
||||
if controller := bdfToNode[diskController[strings.TrimSpace(disk.Slot)]]; controller != "" {
|
||||
doc.Edges = append(doc.Edges, models.TopologyEdge{From: controller, To: id, Kind: "storage"})
|
||||
}
|
||||
}
|
||||
@@ -103,6 +125,8 @@ func Build(snapshot *exporter.ReanimatorExport, evidence *models.TopologyEvidenc
|
||||
label := strings.TrimSpace(psu.Slot)
|
||||
if label == "" {
|
||||
label = fmt.Sprintf("PSU %d", i+1)
|
||||
} else if _, err := strconv.Atoi(label); err == nil {
|
||||
label = "PSU " + label
|
||||
}
|
||||
doc.Nodes = append(doc.Nodes, models.TopologyNode{ID: fmt.Sprintf("psu:%d", i), Kind: "psu", Label: label, Model: psu.Model, Status: psu.Status, WattageW: psu.WattageW})
|
||||
}
|
||||
@@ -166,8 +190,12 @@ func buildNUMAMap(devices []exporter.ReanimatorPCIe, cpuIDs []string, sockets []
|
||||
func pcieKind(d exporter.ReanimatorPCIe) string {
|
||||
c := strings.ToLower(strings.TrimSpace(d.DeviceClass))
|
||||
model := strings.ToLower(d.Model + " " + d.Manufacturer)
|
||||
video := strings.Contains(c, "video") || strings.Contains(c, "display")
|
||||
switch {
|
||||
case strings.Contains(c, "video"), strings.Contains(c, "display"), strings.Contains(c, "3d"), strings.Contains(c, "processingaccelerator"), strings.Contains(model, "nvidia") && strings.Contains(model, "gpu"):
|
||||
case video && bmcVideoVendorIDs[d.VendorID]:
|
||||
// The BMC's own VGA function is not a GPU of the server.
|
||||
return ""
|
||||
case video, strings.Contains(c, "3d"), strings.Contains(c, "processingaccelerator"), strings.Contains(model, "nvidia") && strings.Contains(model, "gpu"):
|
||||
return "gpu"
|
||||
case strings.Contains(c, "network"), strings.Contains(c, "ethernet"), strings.Contains(c, "fibrechannel"), len(d.MACAddresses) > 0:
|
||||
return "nic"
|
||||
@@ -253,11 +281,107 @@ func memoryCPU(node int, cpuIDs []string, sockets []int) string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func parseStorageMap(raw string) map[string]string {
|
||||
out := map[string]string{}
|
||||
// resolveDiskControllers maps a storage slot to the BDF of its controller.
|
||||
// Directly attached disks are keyed by SCSI HCTL. Physical drives behind a
|
||||
// RAID controller have no block device of their own, so their storcli
|
||||
// "EID:Slt" slot is resolved through the virtual drive they belong to:
|
||||
// PD -> VD -> "OS Drive Name" -> block device -> controller.
|
||||
func resolveDiskControllers(evidence *models.TopologyEvidence) map[string]string {
|
||||
if evidence == nil {
|
||||
return map[string]string{}
|
||||
}
|
||||
out, byBlockDev := parseStorageMap(evidence.StorageMap)
|
||||
byStorcliCtrl := map[int]string{}
|
||||
for _, ctrl := range storcliControllers(evidence.StorcliVDs) {
|
||||
for key, raw := range ctrl.data {
|
||||
m := storcliVDRE.FindStringSubmatch(key)
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
var props map[string]any
|
||||
_ = json.Unmarshal(ctrl.data["VD"+m[1]+" Properties"], &props)
|
||||
osDrive, _ := props["OS Drive Name"].(string)
|
||||
bdf := byBlockDev[strings.TrimPrefix(strings.TrimSpace(osDrive), "/dev/")]
|
||||
if bdf == "" {
|
||||
continue
|
||||
}
|
||||
byStorcliCtrl[ctrl.index] = bdf
|
||||
for _, slot := range storcliSlots(raw) {
|
||||
if out[slot] == "" {
|
||||
out[slot] = bdf
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Drives outside any virtual drive (unconfigured, JBOD, spare) still sit on
|
||||
// a controller whose BDF a sibling virtual drive has revealed.
|
||||
for _, ctrl := range storcliControllers(evidence.StorcliDrives) {
|
||||
for key, raw := range ctrl.data {
|
||||
m := storcliPDRE.FindStringSubmatch(key)
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
index, _ := strconv.Atoi(m[1])
|
||||
bdf := byStorcliCtrl[index]
|
||||
if bdf == "" {
|
||||
continue
|
||||
}
|
||||
for _, slot := range storcliSlots(raw) {
|
||||
if out[slot] == "" {
|
||||
out[slot] = bdf
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type storcliController struct {
|
||||
index int
|
||||
data map[string]json.RawMessage
|
||||
}
|
||||
|
||||
func storcliControllers(raw string) []storcliController {
|
||||
var doc struct {
|
||||
Controllers []struct {
|
||||
Status struct {
|
||||
Controller int `json:"Controller"`
|
||||
} `json:"Command Status"`
|
||||
Data map[string]json.RawMessage `json:"Response Data"`
|
||||
} `json:"Controllers"`
|
||||
}
|
||||
if strings.TrimSpace(raw) == "" || json.Unmarshal([]byte(raw), &doc) != nil {
|
||||
return nil
|
||||
}
|
||||
out := make([]storcliController, 0, len(doc.Controllers))
|
||||
for _, c := range doc.Controllers {
|
||||
out = append(out, storcliController{index: c.Status.Controller, data: c.Data})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func storcliSlots(raw json.RawMessage) []string {
|
||||
var rows []map[string]any
|
||||
if json.Unmarshal(raw, &rows) != nil {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, row := range rows {
|
||||
if slot, _ := row["EID:Slt"].(string); strings.TrimSpace(slot) != "" {
|
||||
out = append(out, strings.TrimSpace(slot))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// parseStorageMap reads "sda hctl=0:3:111:0 ctrl=0000:6d:00.0" lines into
|
||||
// controller BDFs keyed by HCTL and by block device name.
|
||||
func parseStorageMap(raw string) (byHCTL, byBlockDev map[string]string) {
|
||||
byHCTL, byBlockDev = map[string]string{}, map[string]string{}
|
||||
for _, line := range strings.Split(raw, "\n") {
|
||||
hctl, ctrl := "", ""
|
||||
for _, field := range strings.Fields(line) {
|
||||
fields := strings.Fields(line)
|
||||
for _, field := range fields {
|
||||
if v, ok := strings.CutPrefix(field, "hctl="); ok {
|
||||
hctl = v
|
||||
}
|
||||
@@ -265,11 +389,17 @@ func parseStorageMap(raw string) map[string]string {
|
||||
ctrl = normalizeBDF(v)
|
||||
}
|
||||
}
|
||||
if hctl != "" && ctrl != "" {
|
||||
out[hctl] = ctrl
|
||||
if ctrl == "" {
|
||||
continue
|
||||
}
|
||||
if hctl != "" {
|
||||
byHCTL[hctl] = ctrl
|
||||
}
|
||||
if !strings.Contains(fields[0], "=") {
|
||||
byBlockDev[fields[0]] = ctrl
|
||||
}
|
||||
}
|
||||
return out
|
||||
return byHCTL, byBlockDev
|
||||
}
|
||||
|
||||
func addNVLinkEdges(doc *models.TopologyDocument, evidence *models.TopologyEvidence, gpuByBDF map[string]string) {
|
||||
|
||||
@@ -82,3 +82,64 @@ func TestRenderHTMLIsStaticAndEscapesLabels(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// HPE ProLiant DL380 Gen11 (BEE-SP v14.23): DIMM locators read "PROC 1 DIMM 10",
|
||||
// the MegaRAID controller is not part of the Reanimator projection, and its
|
||||
// physical drives carry storcli "EID:Slt" slots instead of an HCTL.
|
||||
func TestBuildAttachesHPEMemoryAndRAIDMemberDisks(t *testing.T) {
|
||||
snapshot := &exporter.ReanimatorExport{Hardware: exporter.ReanimatorHardware{
|
||||
Board: exporter.ReanimatorBoard{SerialNumber: "SN1"},
|
||||
CPUs: []exporter.ReanimatorCPU{{Socket: 1}},
|
||||
Memory: []exporter.ReanimatorMemory{{Slot: "PROC 1 DIMM 10", SizeMB: 32768}},
|
||||
Storage: []exporter.ReanimatorStorage{
|
||||
{Slot: "252:1", Model: "Micron_7450"},
|
||||
{Slot: "252:2", Model: "Micron_7450"},
|
||||
{Slot: "252:3", Model: "Micron_7450"},
|
||||
},
|
||||
PCIeDevices: []exporter.ReanimatorPCIe{
|
||||
{Slot: "0000:01:00.1", VendorID: 0x102b, DeviceClass: "VideoController", Model: "MGA G200eH3", NUMANode: intp(0)},
|
||||
{Slot: "0000:6c:00.0", DeviceClass: "NetworkController", NUMANode: intp(0)},
|
||||
},
|
||||
}}
|
||||
e := &models.TopologyEvidence{
|
||||
StorageMap: "sda hctl=0:3:111:0 ctrl=0000:6d:00.0\nsdb hctl=1:0:0:0 ctrl=0000:00:14.0\n",
|
||||
StorcliVDs: `{"Controllers":[{"Command Status":{"Controller":0},"Response Data":{
|
||||
"/c0/v239":[{"DG/VD":"0/239","TYPE":"RAID1"}],
|
||||
"PDs for VD 239":[{"EID:Slt":"252:1"},{"EID:Slt":"252:2"}],
|
||||
"VD239 Properties":{"OS Drive Name":"/dev/sda"}}}]}`,
|
||||
StorcliDrives: `{"Controllers":[{"Command Status":{"Controller":0},"Response Data":{
|
||||
"Drive /c0/e252/s1":[{"EID:Slt":"252:1"}],
|
||||
"Drive /c0/e252/s3":[{"EID:Slt":"252:3"}],
|
||||
"Drive /c0/e252/s3 - Detailed Information":{}}}]}`,
|
||||
}
|
||||
controllers := []exporter.ReanimatorPCIe{
|
||||
{Slot: "0000:6d:00.0", DeviceClass: "MassStorageController", Model: "MegaRAID", NUMANode: intp(0)},
|
||||
{Slot: "0000:00:17.0", DeviceClass: "SATAController", Model: "AHCI", NUMANode: intp(0)},
|
||||
}
|
||||
doc := Build(snapshot, e, controllers...)
|
||||
|
||||
edges := map[string]string{}
|
||||
for _, edge := range doc.Edges {
|
||||
edges[edge.To] = edge.From
|
||||
}
|
||||
want := map[string]string{
|
||||
"memory:PROC 1 DIMM 10": "cpu:1",
|
||||
"raid:0000:6d:00.0": "cpu:1",
|
||||
"storage:252:1": "raid:0000:6d:00.0",
|
||||
"storage:252:2": "raid:0000:6d:00.0",
|
||||
"storage:252:3": "raid:0000:6d:00.0",
|
||||
}
|
||||
for to, from := range want {
|
||||
if edges[to] != from {
|
||||
t.Errorf("edge to %s: got from %q, want %q", to, edges[to], from)
|
||||
}
|
||||
}
|
||||
for _, n := range doc.Nodes {
|
||||
if n.Kind == "gpu" {
|
||||
t.Errorf("BMC VGA rendered as a GPU: %+v", n)
|
||||
}
|
||||
if n.BDF == "0000:00:17.0" {
|
||||
t.Errorf("diskless onboard controller must stay hidden: %+v", n)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user