reanimator/bee
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

webui: add /topo server topology page

Browse Source 
New read-only visualization page: CPU sockets as anchor nodes, PCIe
devices (GPU/NIC/RAID) linked to their NUMA-affine socket with edge
color derived strictly from link_speed vs max_link_speed (not from the
device's own Status, which can also be overwritten by SAT-test results
on the same field), PSU/BMC as standalone boxes with no connecting
line, and a separate NVLink Topology card (live nvidia-smi topo -m /
nvlink -s/-e queries, not persisted to any contract).

GPU-GPU edges are drawn strictly from the actual bonded-pair list
parsed out of "nvidia-smi topo -m" (parseGPUPairAdjacency), not from
adjacent box position in the layout — an earlier ASCII mockup drew a
"chain" through unrelated GPUs, which a dedicated regression test now
guards against. A bonded pair spanning two different NUMA nodes is
flagged Warning on the edge and on both GPU boxes, per project
decision that this is an anomaly worth surfacing, not a neutral fact.

Zero changes to the ingest contract: this reverts the HardwareNVLinkPort/
HardwarePCIeDevice.NVLinks field shipped in v11.55 (33d6eee) along with
its collector/nvidia.go enrichment — that field risked a 400 from
Reanimator Core's strict decoder without an RFC, and isn't needed since
the topo page queries nvidia-smi directly instead of reading it from
audit.json. The v11.55 systemd fix (bee-nvidia.service ordering,
nv-hostengine restart) and the nvlink-status/-errors/dcgmi dumps in the
support bundle are untouched.

Also reorganizes support bundle collection per project convention:
system/ is now LiveCD-operational logs only (Xorg, services, console,
network/FS of the host itself); all server-hardware dumps (lspci,
NVIDIA/NVLink/DCGM, fabric manager, PCIe AER, ethtool, mstflint) move
to techdump/, deduplicating two entries already produced by
platform/techdump.go. Adds nvidia-bug-report.sh (previously only
collected inside the NVIDIA SAT pack) and lscpu to the always-on dump.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-07-07 11:50:33 +03:00
co-authored by Claude Sonnet 5
parent 33d6eee9cf
commit a3377083aa
11 changed files with 1110 additions and 322 deletions
Download Patch File Download Diff File Expand all files Collapse all files
audit/internal/collector/nvidia.go
+1 -147
View File
@@ -6,7 +6,6 @@ import (
"fmt"
"log/slog"
"os/exec"
"regexp"
"strconv"
"strings"
)
@@ -39,48 +38,7 @@ func enrichPCIeWithNVIDIA(devs []schema.HardwarePCIeDevice) []schema.HardwarePCI
slog.Info("nvidia: enrichment skipped", "err", err)
return enrichPCIeWithNVIDIAData(devs, nil, false)
}
devs = enrichPCIeWithNVIDIAData(devs, gpuByBDF, true)
return enrichPCIeWithNVIDIANVLinks(devs)
}
// enrichPCIeWithNVIDIANVLinks attaches per-link NVLink status (nvidia-smi
// nvlink -s) and error counters (nvidia-smi nvlink -e) to each GPU's
// HardwarePCIeDevice entry, keyed by the "nvidia_gpu_index" telemetry set by
// enrichPCIeWithNVIDIAData. Independent of NVSwitch/fabric-manager detection
// so it also covers direct GPU-to-GPU bridge boards with no switch present.
func enrichPCIeWithNVIDIANVLinks(devs []schema.HardwarePCIeDevice) []schema.HardwarePCIeDevice {
statusByGPU, statusErr := nvlinkStatusFn()
if statusErr != nil {
slog.Info("nvidia: nvlink -s unavailable, skipping nvlink enrichment", "err", statusErr)
return devs
}
errorsByGPU, errorsErr := nvlinkErrorsFn()
if errorsErr != nil {
slog.Info("nvidia: nvlink -e unavailable", "err", errorsErr)
}
for i := range devs {
if devs[i].Telemetry == nil {
continue
}
idx, ok := devs[i].Telemetry["nvidia_gpu_index"].(int)
if !ok {
continue
}
ports, ok := statusByGPU[idx]
if !ok {
continue
}
for j := range ports {
if counters, ok := errorsByGPU[idx][ports[j].Index]; ok {
ports[j].ReplayErrors = &counters.Replay
ports[j].RecoveryErrors = &counters.Recovery
ports[j].CRCErrors = &counters.CRC
}
}
devs[i].NVLinks = ports
}
return devs
return enrichPCIeWithNVIDIAData(devs, gpuByBDF, true)
}
func hasNVIDIADevices(devs []schema.HardwarePCIeDevice) bool {
@@ -327,107 +285,3 @@ func injectNVIDIATelemetry(dev *schema.HardwarePCIeDevice, info nvidiaGPUInfo) {
dev.MaxLinkWidth = info.PCIeLinkWidthMax
}
}
var (
nvlinkGPUHeaderRe = regexp.MustCompile(`^GPU (\d+):`)
nvlinkSpeedLineRe = regexp.MustCompile(`^Link (\d+):\s*([\d.]+)\s*GB/s`)
nvlinkInactiveRe = regexp.MustCompile(`^Link (\d+):\s*<inactive>`)
nvlinkErrorCounterRe = regexp.MustCompile(`^Link (\d+):\s*(Replay|Recovery|CRC) Errors:\s*(\d+)`)
)
// nvlinkErrorCounters holds the per-link error counters reported by
// "nvidia-smi nvlink -e" for one GPU.
type nvlinkErrorCounters struct {
Replay, Recovery, CRC int64
}
// nvlinkStatusFn and nvlinkErrorsFn are swappable for testing.
var (
nvlinkStatusFn = queryNVIDIANVLinkStatusByGPU
nvlinkErrorsFn = queryNVIDIANVLinkErrorsByGPU
)
// queryNVIDIANVLinkStatusByGPU runs "nvidia-smi nvlink -s" and returns each
// GPU's NVLink ports keyed by GPU index (as printed in the "GPU N:" header,
// matching the index nvidia-smi --query-gpu also reports).
func queryNVIDIANVLinkStatusByGPU() (map[int][]schema.HardwareNVLinkPort, error) {
out, err := exec.Command("nvidia-smi", "nvlink", "-s").Output()
if err != nil {
return nil, err
}
return parseNVIDIANVLinkStatusByGPU(string(out)), nil
}
func parseNVIDIANVLinkStatusByGPU(raw string) map[int][]schema.HardwareNVLinkPort {
result := map[int][]schema.HardwareNVLinkPort{}
currentGPU := -1
for _, line := range strings.Split(raw, "\n") {
trimmed := strings.TrimSpace(line)
if m := nvlinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
currentGPU, _ = strconv.Atoi(m[1])
continue
}
if currentGPU < 0 {
continue
}
if m := nvlinkInactiveRe.FindStringSubmatch(trimmed); m != nil {
idx, _ := strconv.Atoi(m[1])
result[currentGPU] = append(result[currentGPU], schema.HardwareNVLinkPort{Index: idx, Active: false})
continue
}
if m := nvlinkSpeedLineRe.FindStringSubmatch(trimmed); m != nil {
idx, _ := strconv.Atoi(m[1])
port := schema.HardwareNVLinkPort{Index: idx, Active: true}
if speed, err := strconv.ParseFloat(m[2], 64); err == nil {
port.SpeedGBs = &speed
}
result[currentGPU] = append(result[currentGPU], port)
}
}
return result
}
// queryNVIDIANVLinkErrorsByGPU runs "nvidia-smi nvlink -e" and returns
// per-link error counters keyed by GPU index then link index.
func queryNVIDIANVLinkErrorsByGPU() (map[int]map[int]nvlinkErrorCounters, error) {
out, err := exec.Command("nvidia-smi", "nvlink", "-e").Output()
if err != nil {
return nil, err
}
return parseNVIDIANVLinkErrorsByGPU(string(out)), nil
}
func parseNVIDIANVLinkErrorsByGPU(raw string) map[int]map[int]nvlinkErrorCounters {
result := map[int]map[int]nvlinkErrorCounters{}
currentGPU := -1
for _, line := range strings.Split(raw, "\n") {
trimmed := strings.TrimSpace(line)
if m := nvlinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
currentGPU, _ = strconv.Atoi(m[1])
continue
}
if currentGPU < 0 {
continue
}
m := nvlinkErrorCounterRe.FindStringSubmatch(trimmed)
if m == nil {
continue
}
linkIdx, _ := strconv.Atoi(m[1])
count, _ := strconv.ParseInt(m[3], 10, 64)
if result[currentGPU] == nil {
result[currentGPU] = map[int]nvlinkErrorCounters{}
}
c := result[currentGPU][linkIdx]
switch m[2] {
case "Replay":
c.Replay = count
case "Recovery":
c.Recovery = count
case "CRC":
c.CRC = count
}
result[currentGPU][linkIdx] = c
}
return result
}