reanimator/bee
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

webui: add /topo server topology page

Browse Source 
New read-only visualization page: CPU sockets as anchor nodes, PCIe
devices (GPU/NIC/RAID) linked to their NUMA-affine socket with edge
color derived strictly from link_speed vs max_link_speed (not from the
device's own Status, which can also be overwritten by SAT-test results
on the same field), PSU/BMC as standalone boxes with no connecting
line, and a separate NVLink Topology card (live nvidia-smi topo -m /
nvlink -s/-e queries, not persisted to any contract).

GPU-GPU edges are drawn strictly from the actual bonded-pair list
parsed out of "nvidia-smi topo -m" (parseGPUPairAdjacency), not from
adjacent box position in the layout — an earlier ASCII mockup drew a
"chain" through unrelated GPUs, which a dedicated regression test now
guards against. A bonded pair spanning two different NUMA nodes is
flagged Warning on the edge and on both GPU boxes, per project
decision that this is an anomaly worth surfacing, not a neutral fact.

Zero changes to the ingest contract: this reverts the HardwareNVLinkPort/
HardwarePCIeDevice.NVLinks field shipped in v11.55 (33d6eee) along with
its collector/nvidia.go enrichment — that field risked a 400 from
Reanimator Core's strict decoder without an RFC, and isn't needed since
the topo page queries nvidia-smi directly instead of reading it from
audit.json. The v11.55 systemd fix (bee-nvidia.service ordering,
nv-hostengine restart) and the nvlink-status/-errors/dcgmi dumps in the
support bundle are untouched.

Also reorganizes support bundle collection per project convention:
system/ is now LiveCD-operational logs only (Xorg, services, console,
network/FS of the host itself); all server-hardware dumps (lspci,
NVIDIA/NVLink/DCGM, fabric manager, PCIe AER, ethtool, mstflint) move
to techdump/, deduplicating two entries already produced by
platform/techdump.go. Adds nvidia-bug-report.sh (previously only
collected inside the NVIDIA SAT pack) and lscpu to the always-on dump.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-07-07 11:50:33 +03:00
co-authored by Claude Sonnet 5
parent 33d6eee9cf
commit a3377083aa
11 changed files with 1110 additions and 322 deletions
Download Patch File Download Diff File Expand all files Collapse all files
audit/internal/collector/nvidia_test.go
-89
View File
@@ -126,92 +126,3 @@ func TestEnrichPCIeWithNVIDIAData_driverMissingFallback(t *testing.T) {
func ptrInt64(v int64) *int64 { return &v }
func ptrFloat(v float64) *float64 { return &v }
func TestParseNVIDIANVLinkStatusByGPU(t *testing.T) {
// Real-world 2-GPU direct-bridge H100 SXM output: link 15 inactive on both GPUs.
input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140)
Link 0: 26.562 GB/s
Link 15: <inactive>
Link 17: 26.562 GB/s
GPU 1: NVIDIA H100 80GB HBM3 (UUID: GPU-603fe750-0516-9db5-86ec-ea61af3fce35)
Link 0: 26.562 GB/s
Link 15: <inactive>
`
got := parseNVIDIANVLinkStatusByGPU(input)
if len(got[0]) != 3 {
t.Fatalf("gpu0 ports=%d want 3 (%#v)", len(got[0]), got[0])
}
if got[0][1].Index != 15 || got[0][1].Active {
t.Fatalf("gpu0 link15=%#v want inactive", got[0][1])
}
if got[0][0].SpeedGBs == nil || *got[0][0].SpeedGBs != 26.562 {
t.Fatalf("gpu0 link0 speed=%#v want 26.562", got[0][0].SpeedGBs)
}
if len(got[1]) != 2 {
t.Fatalf("gpu1 ports=%d want 2 (%#v)", len(got[1]), got[1])
}
if got[1][1].Active {
t.Fatalf("gpu1 link15 should be inactive: %#v", got[1][1])
}
}
func TestParseNVIDIANVLinkErrorsByGPU(t *testing.T) {
input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140)
Link 0: Replay Errors: 0
Link 0: Recovery Errors: 0
Link 0: CRC Errors: 0
Link 1: Replay Errors: 3
Link 1: Recovery Errors: 1
Link 1: CRC Errors: 2
GPU 1: NVIDIA H100 80GB HBM3 (UUID: GPU-603fe750-0516-9db5-86ec-ea61af3fce35)
Link 0: Replay Errors: 0
Link 0: Recovery Errors: 0
Link 0: CRC Errors: 0
`
got := parseNVIDIANVLinkErrorsByGPU(input)
c := got[0][1]
if c.Replay != 3 || c.Recovery != 1 || c.CRC != 2 {
t.Fatalf("gpu0 link1 counters=%#v want {3,1,2}", c)
}
zero := got[0][0]
if zero.Replay != 0 || zero.Recovery != 0 || zero.CRC != 0 {
t.Fatalf("gpu0 link0 counters=%#v want all zero", zero)
}
if _, ok := got[1][0]; !ok {
t.Fatalf("expected gpu1 link0 entry present")
}
}
func TestEnrichPCIeWithNVIDIANVLinksAttachesPortsByIndex(t *testing.T) {
oldStatus, oldErrors := nvlinkStatusFn, nvlinkErrorsFn
t.Cleanup(func() { nvlinkStatusFn, nvlinkErrorsFn = oldStatus, oldErrors })
nvlinkStatusFn = func() (map[int][]schema.HardwareNVLinkPort, error) {
return map[int][]schema.HardwareNVLinkPort{
0: {{Index: 0, Active: true, SpeedGBs: ptrFloat(26.562)}, {Index: 15, Active: false}},
}, nil
}
nvlinkErrorsFn = func() (map[int]map[int]nvlinkErrorCounters, error) {
return map[int]map[int]nvlinkErrorCounters{
0: {0: {Replay: 1}},
}, nil
}
devices := []schema.HardwarePCIeDevice{
{Telemetry: map[string]any{"nvidia_gpu_index": 0}},
}
out := enrichPCIeWithNVIDIANVLinks(devices)
if len(out[0].NVLinks) != 2 {
t.Fatalf("nvlinks=%d want 2", len(out[0].NVLinks))
}
if out[0].NVLinks[0].ReplayErrors == nil || *out[0].NVLinks[0].ReplayErrors != 1 {
t.Fatalf("link0 replay errors=%#v want 1", out[0].NVLinks[0].ReplayErrors)
}
if out[0].NVLinks[1].Active {
t.Fatalf("link15 should stay inactive")
}
}