reanimator/bee
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

webui: add /topo server topology page

Browse Source 
New read-only visualization page: CPU sockets as anchor nodes, PCIe
devices (GPU/NIC/RAID) linked to their NUMA-affine socket with edge
color derived strictly from link_speed vs max_link_speed (not from the
device's own Status, which can also be overwritten by SAT-test results
on the same field), PSU/BMC as standalone boxes with no connecting
line, and a separate NVLink Topology card (live nvidia-smi topo -m /
nvlink -s/-e queries, not persisted to any contract).

GPU-GPU edges are drawn strictly from the actual bonded-pair list
parsed out of "nvidia-smi topo -m" (parseGPUPairAdjacency), not from
adjacent box position in the layout — an earlier ASCII mockup drew a
"chain" through unrelated GPUs, which a dedicated regression test now
guards against. A bonded pair spanning two different NUMA nodes is
flagged Warning on the edge and on both GPU boxes, per project
decision that this is an anomaly worth surfacing, not a neutral fact.

Zero changes to the ingest contract: this reverts the HardwareNVLinkPort/
HardwarePCIeDevice.NVLinks field shipped in v11.55 (33d6eee) along with
its collector/nvidia.go enrichment — that field risked a 400 from
Reanimator Core's strict decoder without an RFC, and isn't needed since
the topo page queries nvidia-smi directly instead of reading it from
audit.json. The v11.55 systemd fix (bee-nvidia.service ordering,
nv-hostengine restart) and the nvlink-status/-errors/dcgmi dumps in the
support bundle are untouched.

Also reorganizes support bundle collection per project convention:
system/ is now LiveCD-operational logs only (Xorg, services, console,
network/FS of the host itself); all server-hardware dumps (lspci,
NVIDIA/NVLink/DCGM, fabric manager, PCIe AER, ethtool, mstflint) move
to techdump/, deduplicating two entries already produced by
platform/techdump.go. Adds nvidia-bug-report.sh (previously only
collected inside the NVIDIA SAT pack) and lscpu to the always-on dump.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-07-07 11:50:33 +03:00
co-authored by Claude Sonnet 5
parent 33d6eee9cf
commit a3377083aa
11 changed files with 1110 additions and 322 deletions
Download Patch File Download Diff File Expand all files Collapse all files
audit/internal/collector/nvidia.go
+1 -147
View File
@@ -6,7 +6,6 @@ import (
"fmt"
"log/slog"
"os/exec"
"regexp"
"strconv"
"strings"
)
@@ -39,48 +38,7 @@ func enrichPCIeWithNVIDIA(devs []schema.HardwarePCIeDevice) []schema.HardwarePCI
slog.Info("nvidia: enrichment skipped", "err", err)
return enrichPCIeWithNVIDIAData(devs, nil, false)
}
devs = enrichPCIeWithNVIDIAData(devs, gpuByBDF, true)
return enrichPCIeWithNVIDIANVLinks(devs)
}
// enrichPCIeWithNVIDIANVLinks attaches per-link NVLink status (nvidia-smi
// nvlink -s) and error counters (nvidia-smi nvlink -e) to each GPU's
// HardwarePCIeDevice entry, keyed by the "nvidia_gpu_index" telemetry set by
// enrichPCIeWithNVIDIAData. Independent of NVSwitch/fabric-manager detection
// so it also covers direct GPU-to-GPU bridge boards with no switch present.
func enrichPCIeWithNVIDIANVLinks(devs []schema.HardwarePCIeDevice) []schema.HardwarePCIeDevice {
statusByGPU, statusErr := nvlinkStatusFn()
if statusErr != nil {
slog.Info("nvidia: nvlink -s unavailable, skipping nvlink enrichment", "err", statusErr)
return devs
}
errorsByGPU, errorsErr := nvlinkErrorsFn()
if errorsErr != nil {
slog.Info("nvidia: nvlink -e unavailable", "err", errorsErr)
}
for i := range devs {
if devs[i].Telemetry == nil {
continue
}
idx, ok := devs[i].Telemetry["nvidia_gpu_index"].(int)
if !ok {
continue
}
ports, ok := statusByGPU[idx]
if !ok {
continue
}
for j := range ports {
if counters, ok := errorsByGPU[idx][ports[j].Index]; ok {
ports[j].ReplayErrors = &counters.Replay
ports[j].RecoveryErrors = &counters.Recovery
ports[j].CRCErrors = &counters.CRC
}
}
devs[i].NVLinks = ports
}
return devs
return enrichPCIeWithNVIDIAData(devs, gpuByBDF, true)
}
func hasNVIDIADevices(devs []schema.HardwarePCIeDevice) bool {
@@ -327,107 +285,3 @@ func injectNVIDIATelemetry(dev *schema.HardwarePCIeDevice, info nvidiaGPUInfo) {
dev.MaxLinkWidth = info.PCIeLinkWidthMax
}
}
var (
nvlinkGPUHeaderRe = regexp.MustCompile(`^GPU (\d+):`)
nvlinkSpeedLineRe = regexp.MustCompile(`^Link (\d+):\s*([\d.]+)\s*GB/s`)
nvlinkInactiveRe = regexp.MustCompile(`^Link (\d+):\s*<inactive>`)
nvlinkErrorCounterRe = regexp.MustCompile(`^Link (\d+):\s*(Replay|Recovery|CRC) Errors:\s*(\d+)`)
)
// nvlinkErrorCounters holds the per-link error counters reported by
// "nvidia-smi nvlink -e" for one GPU.
type nvlinkErrorCounters struct {
Replay, Recovery, CRC int64
}
// nvlinkStatusFn and nvlinkErrorsFn are swappable for testing.
var (
nvlinkStatusFn = queryNVIDIANVLinkStatusByGPU
nvlinkErrorsFn = queryNVIDIANVLinkErrorsByGPU
)
// queryNVIDIANVLinkStatusByGPU runs "nvidia-smi nvlink -s" and returns each
// GPU's NVLink ports keyed by GPU index (as printed in the "GPU N:" header,
// matching the index nvidia-smi --query-gpu also reports).
func queryNVIDIANVLinkStatusByGPU() (map[int][]schema.HardwareNVLinkPort, error) {
out, err := exec.Command("nvidia-smi", "nvlink", "-s").Output()
if err != nil {
return nil, err
}
return parseNVIDIANVLinkStatusByGPU(string(out)), nil
}
func parseNVIDIANVLinkStatusByGPU(raw string) map[int][]schema.HardwareNVLinkPort {
result := map[int][]schema.HardwareNVLinkPort{}
currentGPU := -1
for _, line := range strings.Split(raw, "\n") {
trimmed := strings.TrimSpace(line)
if m := nvlinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
currentGPU, _ = strconv.Atoi(m[1])
continue
}
if currentGPU < 0 {
continue
}
if m := nvlinkInactiveRe.FindStringSubmatch(trimmed); m != nil {
idx, _ := strconv.Atoi(m[1])
result[currentGPU] = append(result[currentGPU], schema.HardwareNVLinkPort{Index: idx, Active: false})
continue
}
if m := nvlinkSpeedLineRe.FindStringSubmatch(trimmed); m != nil {
idx, _ := strconv.Atoi(m[1])
port := schema.HardwareNVLinkPort{Index: idx, Active: true}
if speed, err := strconv.ParseFloat(m[2], 64); err == nil {
port.SpeedGBs = &speed
}
result[currentGPU] = append(result[currentGPU], port)
}
}
return result
}
// queryNVIDIANVLinkErrorsByGPU runs "nvidia-smi nvlink -e" and returns
// per-link error counters keyed by GPU index then link index.
func queryNVIDIANVLinkErrorsByGPU() (map[int]map[int]nvlinkErrorCounters, error) {
out, err := exec.Command("nvidia-smi", "nvlink", "-e").Output()
if err != nil {
return nil, err
}
return parseNVIDIANVLinkErrorsByGPU(string(out)), nil
}
func parseNVIDIANVLinkErrorsByGPU(raw string) map[int]map[int]nvlinkErrorCounters {
result := map[int]map[int]nvlinkErrorCounters{}
currentGPU := -1
for _, line := range strings.Split(raw, "\n") {
trimmed := strings.TrimSpace(line)
if m := nvlinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
currentGPU, _ = strconv.Atoi(m[1])
continue
}
if currentGPU < 0 {
continue
}
m := nvlinkErrorCounterRe.FindStringSubmatch(trimmed)
if m == nil {
continue
}
linkIdx, _ := strconv.Atoi(m[1])
count, _ := strconv.ParseInt(m[3], 10, 64)
if result[currentGPU] == nil {
result[currentGPU] = map[int]nvlinkErrorCounters{}
}
c := result[currentGPU][linkIdx]
switch m[2] {
case "Replay":
c.Replay = count
case "Recovery":
c.Recovery = count
case "CRC":
c.CRC = count
}
result[currentGPU][linkIdx] = c
}
return result
}
audit/internal/collector/nvidia_test.go
-89
View File
@@ -126,92 +126,3 @@ func TestEnrichPCIeWithNVIDIAData_driverMissingFallback(t *testing.T) {
func ptrInt64(v int64) *int64 { return &v }
func ptrFloat(v float64) *float64 { return &v }
func TestParseNVIDIANVLinkStatusByGPU(t *testing.T) {
// Real-world 2-GPU direct-bridge H100 SXM output: link 15 inactive on both GPUs.
input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140)
Link 0: 26.562 GB/s
Link 15: <inactive>
Link 17: 26.562 GB/s
GPU 1: NVIDIA H100 80GB HBM3 (UUID: GPU-603fe750-0516-9db5-86ec-ea61af3fce35)
Link 0: 26.562 GB/s
Link 15: <inactive>
`
got := parseNVIDIANVLinkStatusByGPU(input)
if len(got[0]) != 3 {
t.Fatalf("gpu0 ports=%d want 3 (%#v)", len(got[0]), got[0])
}
if got[0][1].Index != 15 || got[0][1].Active {
t.Fatalf("gpu0 link15=%#v want inactive", got[0][1])
}
if got[0][0].SpeedGBs == nil || *got[0][0].SpeedGBs != 26.562 {
t.Fatalf("gpu0 link0 speed=%#v want 26.562", got[0][0].SpeedGBs)
}
if len(got[1]) != 2 {
t.Fatalf("gpu1 ports=%d want 2 (%#v)", len(got[1]), got[1])
}
if got[1][1].Active {
t.Fatalf("gpu1 link15 should be inactive: %#v", got[1][1])
}
}
func TestParseNVIDIANVLinkErrorsByGPU(t *testing.T) {
input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140)
Link 0: Replay Errors: 0
Link 0: Recovery Errors: 0
Link 0: CRC Errors: 0
Link 1: Replay Errors: 3
Link 1: Recovery Errors: 1
Link 1: CRC Errors: 2
GPU 1: NVIDIA H100 80GB HBM3 (UUID: GPU-603fe750-0516-9db5-86ec-ea61af3fce35)
Link 0: Replay Errors: 0
Link 0: Recovery Errors: 0
Link 0: CRC Errors: 0
`
got := parseNVIDIANVLinkErrorsByGPU(input)
c := got[0][1]
if c.Replay != 3 || c.Recovery != 1 || c.CRC != 2 {
t.Fatalf("gpu0 link1 counters=%#v want {3,1,2}", c)
}
zero := got[0][0]
if zero.Replay != 0 || zero.Recovery != 0 || zero.CRC != 0 {
t.Fatalf("gpu0 link0 counters=%#v want all zero", zero)
}
if _, ok := got[1][0]; !ok {
t.Fatalf("expected gpu1 link0 entry present")
}
}
func TestEnrichPCIeWithNVIDIANVLinksAttachesPortsByIndex(t *testing.T) {
oldStatus, oldErrors := nvlinkStatusFn, nvlinkErrorsFn
t.Cleanup(func() { nvlinkStatusFn, nvlinkErrorsFn = oldStatus, oldErrors })
nvlinkStatusFn = func() (map[int][]schema.HardwareNVLinkPort, error) {
return map[int][]schema.HardwareNVLinkPort{
0: {{Index: 0, Active: true, SpeedGBs: ptrFloat(26.562)}, {Index: 15, Active: false}},
}, nil
}
nvlinkErrorsFn = func() (map[int]map[int]nvlinkErrorCounters, error) {
return map[int]map[int]nvlinkErrorCounters{
0: {0: {Replay: 1}},
}, nil
}
devices := []schema.HardwarePCIeDevice{
{Telemetry: map[string]any{"nvidia_gpu_index": 0}},
}
out := enrichPCIeWithNVIDIANVLinks(devices)
if len(out[0].NVLinks) != 2 {
t.Fatalf("nvlinks=%d want 2", len(out[0].NVLinks))
}
if out[0].NVLinks[0].ReplayErrors == nil || *out[0].NVLinks[0].ReplayErrors != 1 {
t.Fatalf("link0 replay errors=%#v want 1", out[0].NVLinks[0].ReplayErrors)
}
if out[0].NVLinks[1].Active {
t.Fatalf("link15 should stay inactive")
}
}