refactor: modularize audit and harden build validation
This commit is contained in:
Mikhail Chusavitin
@@ -11,7 +11,6 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"bee/audit/internal/app"
|
||||
"bee/audit/internal/schema"
|
||||
)
|
||||
|
||||
@@ -201,6 +200,7 @@ func parseDIMMBankLocatorNodes(raw string) map[string]int {
|
||||
// 2. A node number from the Bank Locator via parseDIMMBankLocatorNodes,
|
||||
// e.g. Locator "DIMM000(A)" whose Bank Locator is
|
||||
// "_Node1_Channel0_Dimm0" -> 1.
|
||||
//
|
||||
// ok=false means neither pattern matched, so this DIMM can't be confidently
|
||||
// attached to a CPU column and falls back to the unattached Memory row.
|
||||
func dimmRawNode(mem schema.HardwareMemory, bankNodeByLocator map[string]int) (int, bool) {
|
||||
@@ -582,772 +582,3 @@ type topoEdge struct {
|
||||
x1, y1, x2, y2 int
|
||||
color string
|
||||
}
|
||||
|
||||
func renderTopoMainDiagram(hw schema.HardwareSnapshot, exportDir string) string {
|
||||
// A GPU hardware fault that needs a physical reboot (Xid 79 "fallen off
|
||||
// the bus", Xid 154 "Node Reboot Required" — see gpuNeedsPhysicalReboot)
|
||||
// isn't visible in dev.Status: the collector only sets that from PCIe
|
||||
// link-speed checks, not from SAT/kmsg results. Without this, a GPU that
|
||||
// dropped off the bus mid-test still renders green here even though the
|
||||
// Hardware Summary card is showing a critical banner for it.
|
||||
gpuHardwareFault := false
|
||||
if db, err := app.OpenComponentStatusDB(filepath.Join(exportDir, "component-status.json")); err == nil {
|
||||
if _, needsReboot := gpuNeedsPhysicalReboot(matchedRecords(db.All(), nil, []string{"pcie:gpu:"})); needsReboot {
|
||||
gpuHardwareFault = true
|
||||
}
|
||||
}
|
||||
|
||||
socketIdx := buildSocketIndex(hw.CPUs)
|
||||
numCols := len(hw.CPUs)
|
||||
if numCols == 0 {
|
||||
numCols = 1
|
||||
}
|
||||
unknownCol := numCols // extra trailing column for unmatched devices
|
||||
|
||||
// Group PCIe devices (GPU/NIC/RAID only — matches the mockup's node
|
||||
// types) into columns by NUMA node, falling back to the "unknown" bucket.
|
||||
type placedDevice struct {
|
||||
dev schema.HardwarePCIeDevice
|
||||
kind string // "gpu", "nic", "raid"
|
||||
col int
|
||||
bdf string
|
||||
}
|
||||
var placed []placedDevice
|
||||
for _, dev := range hw.PCIeDevices {
|
||||
kind := pcieDeviceKind(dev)
|
||||
if kind == "" {
|
||||
continue
|
||||
}
|
||||
col := unknownCol
|
||||
if dev.NUMANode != nil {
|
||||
if ci, ok := socketIdx[*dev.NUMANode]; ok {
|
||||
col = ci
|
||||
}
|
||||
}
|
||||
bdf := ""
|
||||
if dev.Slot != nil {
|
||||
bdf = normalizeTopoBDF(*dev.Slot)
|
||||
} else if dev.BDF != nil {
|
||||
bdf = normalizeTopoBDF(*dev.BDF)
|
||||
}
|
||||
placed = append(placed, placedDevice{dev: dev, kind: kind, col: col, bdf: bdf})
|
||||
}
|
||||
hasUnknownCol := false
|
||||
for _, p := range placed {
|
||||
if p.col == unknownCol {
|
||||
hasUnknownCol = true
|
||||
break
|
||||
}
|
||||
}
|
||||
totalCols := numCols
|
||||
if hasUnknownCol {
|
||||
totalCols++
|
||||
}
|
||||
|
||||
// GPU index<->BDF map + pairwise NVLink adjacency, read from the
|
||||
// persisted techdump captured during the last audit cycle, best-effort:
|
||||
// if the dump is missing (older audit, no NVIDIA GPUs), this is simply
|
||||
// skipped. Used only to detect the cross-NUMA-bonded-pair anomaly below;
|
||||
// the pairwise links themselves are drawn in the separate NVLink
|
||||
// Topology card, since grouping same-kind/same-column devices into one
|
||||
// stacked card here leaves no single per-GPU anchor point to draw a
|
||||
// pairwise connector to or from.
|
||||
bdfToIndex, _ := readNVIDIAIndexByBDF(exportDir)
|
||||
var pairs []gpuPairLink
|
||||
if topoMatrix, err := readGPUTopologyMatrix(exportDir); err == nil {
|
||||
pairs = parseGPUPairAdjacency(topoMatrix)
|
||||
}
|
||||
gpuNUMAByIndex := map[int]*int{}
|
||||
gpuBDFByIndex := map[int]string{}
|
||||
for _, p := range placed {
|
||||
if p.kind != "gpu" || p.bdf == "" {
|
||||
continue
|
||||
}
|
||||
if idx, ok := bdfToIndex[p.bdf]; ok {
|
||||
gpuNUMAByIndex[idx] = p.dev.NUMANode
|
||||
gpuBDFByIndex[idx] = p.bdf
|
||||
}
|
||||
}
|
||||
// A bonded pair spanning two different NUMA nodes is treated as an
|
||||
// anomaly (not a neutral fact) per project decision: a bonded pair is
|
||||
// expected to sit on one NUMA node, so a cross-NUMA bond escalates both
|
||||
// GPUs' effective severity to at least Warning, regardless of their own
|
||||
// reported SAT status.
|
||||
crossNUMAWarnBDF := map[string]bool{}
|
||||
for _, pair := range pairs {
|
||||
numaA, okA := gpuNUMAByIndex[pair.GPUA]
|
||||
numaB, okB := gpuNUMAByIndex[pair.GPUB]
|
||||
if !okA || !okB || numaA == nil || numaB == nil || *numaA == *numaB {
|
||||
continue
|
||||
}
|
||||
crossNUMAWarnBDF[gpuBDFByIndex[pair.GPUA]] = true
|
||||
crossNUMAWarnBDF[gpuBDFByIndex[pair.GPUB]] = true
|
||||
}
|
||||
|
||||
kindOrder := []string{"gpu", "nic", "raid"}
|
||||
kindLabel := map[string]string{"gpu": "GPU", "nic": "NIC", "raid": "RAID"}
|
||||
|
||||
// Attach memory DIMMs to their CPU column too, the same way GPU/NIC/RAID
|
||||
// PCIe devices are attached via NUMANode — memory has no NUMANode field
|
||||
// in the schema, so this reads the DIMM's own Locator/Bank Locator
|
||||
// strings instead (see dimmRawNode). DIMMs that can't be confidently
|
||||
// attached fall back to the unattached "Memory" row below the diagram,
|
||||
// same as before this existed.
|
||||
memBankNodes := map[string]int{}
|
||||
if raw, err := readTopoTechDump(exportDir, "dmidecode-type17.txt"); err == nil {
|
||||
memBankNodes = parseDIMMBankLocatorNodes(raw)
|
||||
}
|
||||
memCol := make([]int, len(hw.Memory))
|
||||
memMatched := make([]bool, len(hw.Memory))
|
||||
var memRawNodes []int
|
||||
for i, m := range hw.Memory {
|
||||
if node, ok := dimmRawNode(m, memBankNodes); ok {
|
||||
memCol[i] = node
|
||||
memMatched[i] = true
|
||||
memRawNodes = append(memRawNodes, node)
|
||||
}
|
||||
}
|
||||
memColIdx := buildMemoryColumnIndex(memRawNodes)
|
||||
for i := range hw.Memory {
|
||||
if !memMatched[i] {
|
||||
continue
|
||||
}
|
||||
col := memColIdx[memCol[i]]
|
||||
if col >= numCols {
|
||||
memMatched[i] = false
|
||||
continue
|
||||
}
|
||||
memCol[i] = col
|
||||
}
|
||||
|
||||
var boxes []topoBox
|
||||
var pcieEdges []topoEdge
|
||||
|
||||
for col := 0; col < totalCols; col++ {
|
||||
colX := (col+1)*24 + col*topoColWidth
|
||||
if col < len(hw.CPUs) {
|
||||
cpu := hw.CPUs[col]
|
||||
model := ""
|
||||
if cpu.Model != nil {
|
||||
model = *cpu.Model
|
||||
}
|
||||
socket := col
|
||||
if cpu.Socket != nil {
|
||||
socket = *cpu.Socket
|
||||
}
|
||||
var tally topoStatusTally
|
||||
tally.add(classifyTopoSeverity(cpu.Status))
|
||||
fill, stroke, text := topoSeverityColors(tally.worst())
|
||||
boxes = append(boxes, topoBox{
|
||||
x: colX, y: topoTopMargin, w: topoBoxWidth, h: topoBoxHeight,
|
||||
topoCardInfo: topoCardInfo{
|
||||
label: fmt.Sprintf("CPU %d", socket), sublabel: model, count: 1,
|
||||
statusLine: tally.line(),
|
||||
fillVar: fill, strokeVar: stroke, textVar: text,
|
||||
detailType: "cpu",
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
y := topoTopMargin + topoBoxHeight + topoDeviceGap*2
|
||||
|
||||
// Memory goes first in the chain, directly under the CPU box: DIMMs
|
||||
// are wired straight to the socket's memory controller, not reached
|
||||
// over PCIe like the GPU/NIC/RAID chain below it.
|
||||
var memGroup []schema.HardwareMemory
|
||||
for i, m := range hw.Memory {
|
||||
if memMatched[i] && memCol[i] == col {
|
||||
memGroup = append(memGroup, m)
|
||||
}
|
||||
}
|
||||
if len(memGroup) > 0 {
|
||||
var tally topoStatusTally
|
||||
sizeGB := 0
|
||||
for _, m := range memGroup {
|
||||
tally.add(classifyTopoSeverity(m.Status))
|
||||
if m.SizeMB != nil {
|
||||
sizeGB += *m.SizeMB / 1024
|
||||
}
|
||||
}
|
||||
fill, stroke, text := topoSeverityColors(tally.worst())
|
||||
sublabel := ""
|
||||
if sizeGB > 0 {
|
||||
sublabel = fmt.Sprintf("%d GB total", sizeGB)
|
||||
}
|
||||
stackLayers := topoStackLayers(len(memGroup))
|
||||
boxes = append(boxes, topoBox{
|
||||
x: colX, y: y, w: topoBoxWidth, h: topoBoxHeight,
|
||||
topoCardInfo: topoCardInfo{
|
||||
label: "Memory", sublabel: sublabel, count: len(memGroup),
|
||||
statusLine: tally.line(),
|
||||
fillVar: fill, strokeVar: stroke, textVar: text,
|
||||
detailType: "memory",
|
||||
},
|
||||
})
|
||||
if col < len(hw.CPUs) {
|
||||
pcieEdges = append(pcieEdges, topoEdge{
|
||||
x1: colX + topoBoxWidth/2, y1: topoTopMargin + topoBoxHeight,
|
||||
x2: colX + topoBoxWidth/2, y2: y,
|
||||
color: "var(--ok-fg)",
|
||||
})
|
||||
}
|
||||
y += topoBoxHeight + topoDeviceGap + stackLayers*topoStackStep
|
||||
}
|
||||
|
||||
for _, kind := range kindOrder {
|
||||
var group []placedDevice
|
||||
for _, p := range placed {
|
||||
if p.col == col && p.kind == kind {
|
||||
group = append(group, p)
|
||||
}
|
||||
}
|
||||
if len(group) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
var tally topoStatusTally
|
||||
model := ""
|
||||
edgeColor := "var(--ok-fg)"
|
||||
for i, p := range group {
|
||||
sev := classifyTopoSeverity(p.dev.Status)
|
||||
if kind == "gpu" && crossNUMAWarnBDF[p.bdf] && sev < 2 {
|
||||
sev = 2
|
||||
}
|
||||
if kind == "gpu" && gpuHardwareFault && sev < 3 {
|
||||
sev = 3
|
||||
}
|
||||
tally.add(sev)
|
||||
if i == 0 && p.dev.Model != nil {
|
||||
model = *p.dev.Model
|
||||
}
|
||||
if topoEdgeColorVar(p.dev) == "var(--warn-fg)" {
|
||||
edgeColor = "var(--warn-fg)"
|
||||
}
|
||||
}
|
||||
fill, stroke, text := topoSeverityColors(tally.worst())
|
||||
stackLayers := topoStackLayers(len(group))
|
||||
boxes = append(boxes, topoBox{
|
||||
x: colX, y: y, w: topoBoxWidth, h: topoBoxHeight,
|
||||
topoCardInfo: topoCardInfo{
|
||||
label: kindLabel[kind], sublabel: model, count: len(group),
|
||||
statusLine: tally.line(),
|
||||
fillVar: fill, strokeVar: stroke, textVar: text,
|
||||
detailType: kind,
|
||||
},
|
||||
})
|
||||
|
||||
if col < len(hw.CPUs) {
|
||||
pcieEdges = append(pcieEdges, topoEdge{
|
||||
x1: colX + topoBoxWidth/2, y1: topoTopMargin + topoBoxHeight,
|
||||
x2: colX + topoBoxWidth/2, y2: y,
|
||||
color: edgeColor,
|
||||
})
|
||||
}
|
||||
|
||||
y += topoBoxHeight + topoDeviceGap + stackLayers*topoStackStep
|
||||
}
|
||||
}
|
||||
|
||||
maxDeviceY := topoTopMargin + topoBoxHeight + topoDeviceGap*2
|
||||
for _, box := range boxes {
|
||||
bottom := box.y + box.h + topoStackLayers(box.count)*topoStackStep
|
||||
if bottom > maxDeviceY {
|
||||
maxDeviceY = bottom
|
||||
}
|
||||
}
|
||||
|
||||
svgHeight := maxDeviceY + 24
|
||||
svgWidth := totalCols*topoColWidth + 48
|
||||
|
||||
var b strings.Builder
|
||||
// Wrapped in its own horizontally-scrolling container (matching the
|
||||
// overflow-x:auto convention used for wide tables elsewhere in webui)
|
||||
// rather than max-width:100% — squashing a node/edge diagram to fit a
|
||||
// narrow viewport makes labels and badges illegible, whereas scrolling
|
||||
// keeps the diagram readable at its natural size on any screen width.
|
||||
b.WriteString(`<div style="overflow-x:auto">`)
|
||||
fmt.Fprintf(&b, `<svg width="%d" height="%d" viewBox="0 0 %d %d">`+"\n", svgWidth, svgHeight, svgWidth, svgHeight)
|
||||
for _, e := range pcieEdges {
|
||||
fmt.Fprintf(&b, `<line x1="%d" y1="%d" x2="%d" y2="%d" style="stroke:%s;stroke-width:2"/>`+"\n", e.x1, e.y1, e.x2, e.y2, e.color)
|
||||
}
|
||||
for _, box := range boxes {
|
||||
writeTopoBoxSVG(&b, box)
|
||||
}
|
||||
b.WriteString(`</svg></div>`)
|
||||
|
||||
// Firmware (BMC/BIOS/...) and PSUs have no PCIe/CPU affinity to anchor
|
||||
// them to a column, and there can be an arbitrary number of any of them
|
||||
// — so unlike the diagram above, they're plain flex-wrap HTML below the
|
||||
// SVG rather than absolutely-positioned SVG boxes. A fixed-size SVG
|
||||
// canvas has no way to wrap overflow onto a new row, which is exactly
|
||||
// what caused these to pile up and overlap once a board had more
|
||||
// PSUs/firmware records than fit in one fixed-width row.
|
||||
//
|
||||
// Memory DIMMs that were matched to a CPU column above already got a
|
||||
// box in the SVG diagram; only DIMMs that couldn't be attached to a
|
||||
// column (see memMatched above) fall back to this row.
|
||||
var unmatchedMem []schema.HardwareMemory
|
||||
for i, m := range hw.Memory {
|
||||
if !memMatched[i] {
|
||||
unmatchedMem = append(unmatchedMem, m)
|
||||
}
|
||||
}
|
||||
if len(unmatchedMem) > 0 {
|
||||
var tally topoStatusTally
|
||||
for _, m := range unmatchedMem {
|
||||
tally.add(classifyTopoSeverity(m.Status))
|
||||
}
|
||||
fill, stroke, text := topoSeverityColors(tally.worst())
|
||||
sizeGB := 0
|
||||
for _, m := range unmatchedMem {
|
||||
if m.SizeMB != nil {
|
||||
sizeGB += *m.SizeMB / 1024
|
||||
}
|
||||
}
|
||||
sublabel := ""
|
||||
if sizeGB > 0 {
|
||||
sublabel = fmt.Sprintf("%d GB total", sizeGB)
|
||||
}
|
||||
b.WriteString(renderTopoFlexRow("Memory", []topoCardInfo{{
|
||||
label: "Memory", sublabel: sublabel, count: len(unmatchedMem),
|
||||
statusLine: tally.line(),
|
||||
fillVar: fill, strokeVar: stroke, textVar: text,
|
||||
detailType: "memory",
|
||||
}}))
|
||||
}
|
||||
|
||||
var firmwareItems []topoCardInfo
|
||||
for _, rec := range hw.Firmware {
|
||||
// Firmware records carry no per-item status in the schema (they are
|
||||
// identity, not health, facts), so each stays a neutral, uncolored
|
||||
// card rather than forcing a fake "Unknown" status line.
|
||||
fillVar, strokeVar, textVar := topoSeverityColors(0)
|
||||
firmwareItems = append(firmwareItems, topoCardInfo{
|
||||
label: rec.DeviceName, sublabel: "fw " + rec.Version, count: 1,
|
||||
fillVar: fillVar, strokeVar: strokeVar, textVar: textVar,
|
||||
})
|
||||
}
|
||||
b.WriteString(renderTopoFlexRow("Firmware", firmwareItems))
|
||||
|
||||
if len(hw.PowerSupplies) > 0 {
|
||||
var tally topoStatusTally
|
||||
watt := 0
|
||||
for _, psu := range hw.PowerSupplies {
|
||||
tally.add(classifyTopoSeverity(psu.Status))
|
||||
if psu.WattageW != nil {
|
||||
watt = *psu.WattageW
|
||||
}
|
||||
}
|
||||
fill, stroke, text := topoSeverityColors(tally.worst())
|
||||
sublabel := ""
|
||||
if watt > 0 {
|
||||
sublabel = fmt.Sprintf("%dW each", watt)
|
||||
}
|
||||
b.WriteString(renderTopoFlexRow("Power Supplies", []topoCardInfo{{
|
||||
label: "Power Supplies", sublabel: sublabel, count: len(hw.PowerSupplies),
|
||||
statusLine: tally.line(),
|
||||
fillVar: fill, strokeVar: stroke, textVar: text,
|
||||
detailType: "psu",
|
||||
}}))
|
||||
}
|
||||
|
||||
return topoCard("Topology", b.String())
|
||||
}
|
||||
|
||||
// renderTopoFlexRow renders a labeled, wrapping row of component cards.
|
||||
// Returns "" if items is empty (e.g. no PSU data in this audit).
|
||||
func renderTopoFlexRow(title string, items []topoCardInfo) string {
|
||||
if len(items) == 0 {
|
||||
return ""
|
||||
}
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, `<div style="font-size:11px;color:var(--muted);text-transform:uppercase;letter-spacing:.05em;margin:16px 0 6px">%s</div>`,
|
||||
html.EscapeString(title))
|
||||
b.WriteString(`<div style="display:flex;flex-wrap:wrap;gap:10px">`)
|
||||
for _, item := range items {
|
||||
onclick := ""
|
||||
cursor := "default"
|
||||
if item.detailType != "" {
|
||||
onclick = fmt.Sprintf(` onclick="openComponentDetail('%s')"`, item.detailType)
|
||||
cursor = "pointer"
|
||||
}
|
||||
stackLayers := topoStackLayers(item.count)
|
||||
// Extra right/bottom padding on the wrapper reserves room for the
|
||||
// backing layers of the stack effect so they aren't clipped by the
|
||||
// flex container.
|
||||
fmt.Fprintf(&b, `<div style="position:relative;padding-right:%dpx;padding-bottom:%dpx">`,
|
||||
stackLayers*topoStackStep, stackLayers*topoStackStep)
|
||||
for i := stackLayers; i >= 1; i-- {
|
||||
off := i * topoStackStep
|
||||
fmt.Fprintf(&b, `<div style="position:absolute;top:%dpx;left:%dpx;right:0;bottom:0;border-radius:6px;background:%s;border:1px solid %s;opacity:.55"></div>`,
|
||||
off, off, item.fillVar, item.strokeVar)
|
||||
}
|
||||
fmt.Fprintf(&b, `<div%s style="position:relative;cursor:%s;min-width:160px;padding:10px 12px;border-radius:6px;background:%s;border:1px solid %s;color:%s">`,
|
||||
onclick, cursor, item.fillVar, item.strokeVar, item.textVar)
|
||||
label := item.label
|
||||
if item.count > 1 {
|
||||
label = fmt.Sprintf("%s ×%d", item.label, item.count)
|
||||
}
|
||||
fmt.Fprintf(&b, `<div style="font-size:13px;font-weight:700">%s</div>`, html.EscapeString(label))
|
||||
if item.sublabel != "" {
|
||||
fmt.Fprintf(&b, `<div style="font-size:11px;opacity:.85">%s</div>`, html.EscapeString(item.sublabel))
|
||||
}
|
||||
if item.statusLine != "" {
|
||||
fmt.Fprintf(&b, `<div style="font-size:11px;font-weight:600;margin-top:4px">%s</div>`, html.EscapeString(item.statusLine))
|
||||
}
|
||||
b.WriteString(`</div></div>`)
|
||||
}
|
||||
b.WriteString(`</div>`)
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func writeTopoBoxSVG(b *strings.Builder, box topoBox) {
|
||||
onclick := ""
|
||||
cursor := "default"
|
||||
if box.detailType != "" {
|
||||
onclick = fmt.Sprintf(` onclick="openComponentDetail('%s')"`, box.detailType)
|
||||
cursor = "pointer"
|
||||
}
|
||||
fmt.Fprintf(b, `<g%s style="cursor:%s">`, onclick, cursor)
|
||||
|
||||
// Stack-of-cards effect: faint offset rects behind the front card when
|
||||
// this box represents more than one physical component (e.g. 4 GPUs in
|
||||
// one NUMA column), so a group reads as "a deck of N" rather than a
|
||||
// single item. Peeks toward the bottom-right, into space already
|
||||
// reserved between this box and the next one in the column.
|
||||
for i := topoStackLayers(box.count); i >= 1; i-- {
|
||||
off := i * topoStackStep
|
||||
fmt.Fprintf(b, `<rect x="%d" y="%d" width="%d" height="%d" rx="6" ry="6" style="fill:%s;stroke:%s;opacity:.55"/>`+"\n",
|
||||
box.x+off, box.y+off, box.w, box.h, box.fillVar, box.strokeVar)
|
||||
}
|
||||
|
||||
fmt.Fprintf(b, `<rect x="%d" y="%d" width="%d" height="%d" rx="6" ry="6" style="fill:%s;stroke:%s"/>`+"\n",
|
||||
box.x, box.y, box.w, box.h, box.fillVar, box.strokeVar)
|
||||
|
||||
label := box.label
|
||||
if box.count > 1 {
|
||||
label = fmt.Sprintf("%s ×%d", box.label, box.count)
|
||||
}
|
||||
fmt.Fprintf(b, `<text x="%d" y="%d" style="fill:%s;font-size:13px;font-weight:700">%s</text>`+"\n",
|
||||
box.x+10, box.y+20, box.textVar, html.EscapeString(label))
|
||||
if box.sublabel != "" {
|
||||
fmt.Fprintf(b, `<text x="%d" y="%d" style="fill:%s;font-size:11px;opacity:.85">%s</text>`+"\n",
|
||||
box.x+10, box.y+36, box.textVar, html.EscapeString(truncateTopoLabel(box.sublabel, 26)))
|
||||
}
|
||||
if box.statusLine != "" {
|
||||
fmt.Fprintf(b, `<text x="%d" y="%d" style="fill:%s;font-size:10px;font-weight:600">%s</text>`+"\n",
|
||||
box.x+10, box.y+box.h-10, box.textVar, html.EscapeString(box.statusLine))
|
||||
}
|
||||
b.WriteString(`</g>` + "\n")
|
||||
}
|
||||
|
||||
func truncateTopoLabel(s string, max int) string {
|
||||
if len(s) <= max {
|
||||
return s
|
||||
}
|
||||
if max <= 1 {
|
||||
return s[:max]
|
||||
}
|
||||
return s[:max-1] + "…"
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Separate NVLink topology card (read from techdump, not written to any
|
||||
// ingest contract)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type topoNVLinkPort struct {
|
||||
Index int
|
||||
Active bool
|
||||
SpeedGBs *float64
|
||||
ReplayErrors int64
|
||||
RecoveryErrors int64
|
||||
CRCErrors int64
|
||||
}
|
||||
|
||||
var (
|
||||
topoNVLinkGPUHeaderRe = regexp.MustCompile(`^GPU (\d+):`)
|
||||
topoNVLinkSpeedLineRe = regexp.MustCompile(`^Link (\d+):\s*([\d.]+)\s*GB/s`)
|
||||
topoNVLinkInactiveRe = regexp.MustCompile(`^Link (\d+):\s*<inactive>`)
|
||||
topoNVLinkErrorLineRe = regexp.MustCompile(`^Link (\d+):\s*(Replay|Recovery|CRC) Errors:\s*(\d+)`)
|
||||
)
|
||||
|
||||
func readTopoNVLinkStatus(exportDir string) (map[int][]topoNVLinkPort, error) {
|
||||
raw, err := readTopoTechDump(exportDir, "nvidia-smi-nvlink-status.txt")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return parseTopoNVLinkStatus(raw), nil
|
||||
}
|
||||
|
||||
func parseTopoNVLinkStatus(raw string) map[int][]topoNVLinkPort {
|
||||
result := map[int][]topoNVLinkPort{}
|
||||
currentGPU := -1
|
||||
for _, line := range strings.Split(raw, "\n") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if m := topoNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
|
||||
currentGPU, _ = strconv.Atoi(m[1])
|
||||
continue
|
||||
}
|
||||
if currentGPU < 0 {
|
||||
continue
|
||||
}
|
||||
if m := topoNVLinkInactiveRe.FindStringSubmatch(trimmed); m != nil {
|
||||
idx, _ := strconv.Atoi(m[1])
|
||||
result[currentGPU] = append(result[currentGPU], topoNVLinkPort{Index: idx, Active: false})
|
||||
continue
|
||||
}
|
||||
if m := topoNVLinkSpeedLineRe.FindStringSubmatch(trimmed); m != nil {
|
||||
idx, _ := strconv.Atoi(m[1])
|
||||
port := topoNVLinkPort{Index: idx, Active: true}
|
||||
if speed, err := strconv.ParseFloat(m[2], 64); err == nil {
|
||||
port.SpeedGBs = &speed
|
||||
}
|
||||
result[currentGPU] = append(result[currentGPU], port)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func readTopoNVLinkErrors(exportDir string) (map[int]map[int][3]int64, error) {
|
||||
raw, err := readTopoTechDump(exportDir, "nvidia-smi-nvlink-errors.txt")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return parseTopoNVLinkErrors(raw), nil
|
||||
}
|
||||
|
||||
// parseTopoNVLinkErrors returns, per GPU then link index, [replay, recovery, crc].
|
||||
func parseTopoNVLinkErrors(raw string) map[int]map[int][3]int64 {
|
||||
result := map[int]map[int][3]int64{}
|
||||
currentGPU := -1
|
||||
for _, line := range strings.Split(raw, "\n") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if m := topoNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
|
||||
currentGPU, _ = strconv.Atoi(m[1])
|
||||
continue
|
||||
}
|
||||
if currentGPU < 0 {
|
||||
continue
|
||||
}
|
||||
m := topoNVLinkErrorLineRe.FindStringSubmatch(trimmed)
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
linkIdx, _ := strconv.Atoi(m[1])
|
||||
count, _ := strconv.ParseInt(m[3], 10, 64)
|
||||
if result[currentGPU] == nil {
|
||||
result[currentGPU] = map[int][3]int64{}
|
||||
}
|
||||
c := result[currentGPU][linkIdx]
|
||||
switch m[2] {
|
||||
case "Replay":
|
||||
c[0] = count
|
||||
case "Recovery":
|
||||
c[1] = count
|
||||
case "CRC":
|
||||
c[2] = count
|
||||
}
|
||||
result[currentGPU][linkIdx] = c
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// renderTopoNVLinkCard renders the separate NVLink topology card. Returns ""
|
||||
// if there are fewer than 2 NVIDIA GPUs, or the nvidia-smi nvlink techdump
|
||||
// wasn't captured (older audit, or nvidia-smi unavailable on that run).
|
||||
func renderTopoNVLinkCard(hw schema.HardwareSnapshot, exportDir string) string {
|
||||
gpuCount := 0
|
||||
for _, dev := range hw.PCIeDevices {
|
||||
if dev.DeviceClass != nil && isGPUDeviceClass(*dev.DeviceClass) {
|
||||
gpuCount++
|
||||
}
|
||||
}
|
||||
if gpuCount < 2 {
|
||||
return ""
|
||||
}
|
||||
|
||||
status, err := readTopoNVLinkStatus(exportDir)
|
||||
if err != nil || len(status) == 0 {
|
||||
return topoCard("NVLink Topology", `<span class="badge badge-unknown">nvidia-smi nvlink data unavailable</span>`)
|
||||
}
|
||||
errors, _ := readTopoNVLinkErrors(exportDir)
|
||||
|
||||
topoMatrix, _ := readGPUTopologyMatrix(exportDir)
|
||||
pairs := parseGPUPairAdjacency(topoMatrix)
|
||||
|
||||
var bodyB strings.Builder
|
||||
if gpuCount <= 4 && len(pairs) > 0 {
|
||||
// Small GPU count: per-pair box+line with per-link detail.
|
||||
for _, pair := range pairs {
|
||||
activeCount, total, hasError := 0, 0, false
|
||||
for _, port := range status[pair.GPUA] {
|
||||
total++
|
||||
if port.Active {
|
||||
activeCount++
|
||||
}
|
||||
}
|
||||
for _, counters := range errors[pair.GPUA] {
|
||||
if counters[0] != 0 || counters[1] != 0 || counters[2] != 0 {
|
||||
hasError = true
|
||||
}
|
||||
}
|
||||
color := "var(--ok-fg)"
|
||||
switch {
|
||||
case hasError:
|
||||
color = "var(--crit-fg)"
|
||||
case total > 0 && activeCount < total:
|
||||
color = "var(--warn-fg)"
|
||||
}
|
||||
fmt.Fprintf(&bodyB, `<div style="display:flex;align-items:center;gap:12px;margin-bottom:10px">`+
|
||||
`<div style="padding:8px 12px;border:1px solid var(--border);border-radius:6px">GPU %d</div>`+
|
||||
`<div style="flex:1;height:2px;background:%s"></div>`+
|
||||
`<div style="padding:8px 12px;border:1px solid var(--border);border-radius:6px">GPU %d</div>`+
|
||||
`<div style="font-size:12px;color:var(--muted)">%d/%d links active%s</div>`+
|
||||
`</div>`,
|
||||
pair.GPUA, color, pair.GPUB, activeCount, total, errNoteSuffix(hasError))
|
||||
}
|
||||
} else if len(pairs) > 0 {
|
||||
// Larger GPU counts (NVSwitch fabric): aggregate pair table instead of
|
||||
// an unreadable all-to-all graph.
|
||||
bodyB.WriteString(`<table><thead><tr><th>GPU A</th><th>GPU B</th><th>NVLinks</th></tr></thead><tbody>`)
|
||||
for _, pair := range pairs {
|
||||
fmt.Fprintf(&bodyB, `<tr><td>GPU %d</td><td>GPU %d</td><td>%d</td></tr>`, pair.GPUA, pair.GPUB, pair.NVLinks)
|
||||
}
|
||||
bodyB.WriteString(`</tbody></table>`)
|
||||
} else {
|
||||
bodyB.WriteString(`<span class="badge badge-unknown">No NVLink-bonded GPU pairs found</span>`)
|
||||
}
|
||||
|
||||
return topoCard("NVLink Topology", bodyB.String())
|
||||
}
|
||||
|
||||
func errNoteSuffix(hasError bool) string {
|
||||
if hasError {
|
||||
return " — errors detected"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Inventory fallback for the component-detail modal
|
||||
//
|
||||
// handleAPIComponentDetail normally sources records from app.ComponentStatusDB,
|
||||
// which only gains entries once something has actually written a status
|
||||
// observation (SAT run, watchdog tick, ...). On a freshly booted host that
|
||||
// hasn't run SAT yet, StatusDB can be entirely empty for a component type even
|
||||
// though the /topo card for it already shows "N OK" — that card reads
|
||||
// schema.HardwareComponentStatus.Status straight from the audit snapshot.
|
||||
// inventoryFallbackRecords bridges that gap by building synthetic records
|
||||
// from the same snapshot/classifiers the topology card uses, so the two
|
||||
// views never disagree about how many devices exist or their status.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// topoSeverityStatus renders classifyTopoSeverity's rank back into the status
|
||||
// string vocabulary renderComponentDetail/chipLetterClass expect ("OK",
|
||||
// "Warning", "Critical", "Unknown") — kept in lockstep with classifyTopoSeverity
|
||||
// so a device the topo card counts as "OK" is never shown here as "Unknown".
|
||||
func topoSeverityStatus(status *string) string {
|
||||
switch classifyTopoSeverity(status) {
|
||||
case 3:
|
||||
return "Critical"
|
||||
case 2:
|
||||
return "Warning"
|
||||
case 1:
|
||||
return "OK"
|
||||
default:
|
||||
return "Unknown"
|
||||
}
|
||||
}
|
||||
|
||||
// pcieDeviceKind classifies a PCIe device the same way renderTopoMainDiagram
|
||||
// does, returning "" for devices that aren't GPU/NIC/RAID.
|
||||
func pcieDeviceKind(dev schema.HardwarePCIeDevice) string {
|
||||
switch {
|
||||
case dev.DeviceClass != nil && isGPUDeviceClass(*dev.DeviceClass):
|
||||
return "gpu"
|
||||
case isNICDeviceClassDev(dev):
|
||||
return "nic"
|
||||
case dev.DeviceClass != nil && isRAIDControllerClass(*dev.DeviceClass):
|
||||
return "raid"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// pcieDeviceKey builds a stable, human-readable component key for a PCIe
|
||||
// device: "<kind>:<bdf>" when a slot/BDF is known, else "<kind>:<index>".
|
||||
func pcieDeviceKey(kind string, index int, dev schema.HardwarePCIeDevice) string {
|
||||
bdf := ""
|
||||
if dev.Slot != nil {
|
||||
bdf = normalizeTopoBDF(*dev.Slot)
|
||||
} else if dev.BDF != nil {
|
||||
bdf = normalizeTopoBDF(*dev.BDF)
|
||||
}
|
||||
if bdf != "" {
|
||||
return kind + ":" + bdf
|
||||
}
|
||||
return fmt.Sprintf("%s:%d", kind, index)
|
||||
}
|
||||
|
||||
// inventoryFallbackRecords builds ComponentStatusRecord entries straight from
|
||||
// the audit inventory (bee-audit.json) for the given component type, used
|
||||
// when ComponentStatusDB has no matching records yet. Records carry only
|
||||
// ComponentKey/Status — no LastCheckedAt/History — so renderComponentDetail
|
||||
// renders them without a "checked at" timestamp or sparkline.
|
||||
func inventoryFallbackRecords(compType string, opts HandlerOptions) []app.ComponentStatusRecord {
|
||||
data, err := loadSnapshot(opts.AuditPath)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var ingest schema.HardwareIngestRequest
|
||||
if err := json.Unmarshal(data, &ingest); err != nil {
|
||||
return nil
|
||||
}
|
||||
hw := ingest.Hardware
|
||||
|
||||
var records []app.ComponentStatusRecord
|
||||
switch compType {
|
||||
case "cpu":
|
||||
for i, cpu := range hw.CPUs {
|
||||
key := fmt.Sprintf("cpu:%d", i)
|
||||
if cpu.Socket != nil {
|
||||
key = fmt.Sprintf("cpu:socket%d", *cpu.Socket)
|
||||
}
|
||||
records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(cpu.Status)})
|
||||
}
|
||||
case "memory":
|
||||
for i, m := range hw.Memory {
|
||||
key := fmt.Sprintf("memory:%d", i)
|
||||
if m.Slot != nil && strings.TrimSpace(*m.Slot) != "" {
|
||||
key = "memory:" + strings.TrimSpace(*m.Slot)
|
||||
}
|
||||
records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(m.Status)})
|
||||
}
|
||||
case "storage":
|
||||
for i, s := range hw.Storage {
|
||||
key := fmt.Sprintf("storage:%d", i)
|
||||
if s.Slot != nil && strings.TrimSpace(*s.Slot) != "" {
|
||||
key = "storage:" + strings.TrimSpace(*s.Slot)
|
||||
}
|
||||
records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(s.Status)})
|
||||
}
|
||||
case "psu":
|
||||
for i, p := range hw.PowerSupplies {
|
||||
key := fmt.Sprintf("psu:%d", i)
|
||||
if p.Slot != nil && strings.TrimSpace(*p.Slot) != "" {
|
||||
key = "psu:" + strings.TrimSpace(*p.Slot)
|
||||
}
|
||||
records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(p.Status)})
|
||||
}
|
||||
case "gpu", "nic", "raid":
|
||||
for i, dev := range hw.PCIeDevices {
|
||||
if pcieDeviceKind(dev) != compType {
|
||||
continue
|
||||
}
|
||||
key := pcieDeviceKey(compType, i, dev)
|
||||
records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(dev.Status)})
|
||||
}
|
||||
}
|
||||
return records
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user