reanimator/bee
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

webui: add /topo server topology page

Browse Source 
New read-only visualization page: CPU sockets as anchor nodes, PCIe
devices (GPU/NIC/RAID) linked to their NUMA-affine socket with edge
color derived strictly from link_speed vs max_link_speed (not from the
device's own Status, which can also be overwritten by SAT-test results
on the same field), PSU/BMC as standalone boxes with no connecting
line, and a separate NVLink Topology card (live nvidia-smi topo -m /
nvlink -s/-e queries, not persisted to any contract).

GPU-GPU edges are drawn strictly from the actual bonded-pair list
parsed out of "nvidia-smi topo -m" (parseGPUPairAdjacency), not from
adjacent box position in the layout — an earlier ASCII mockup drew a
"chain" through unrelated GPUs, which a dedicated regression test now
guards against. A bonded pair spanning two different NUMA nodes is
flagged Warning on the edge and on both GPU boxes, per project
decision that this is an anomaly worth surfacing, not a neutral fact.

Zero changes to the ingest contract: this reverts the HardwareNVLinkPort/
HardwarePCIeDevice.NVLinks field shipped in v11.55 (33d6eee) along with
its collector/nvidia.go enrichment — that field risked a 400 from
Reanimator Core's strict decoder without an RFC, and isn't needed since
the topo page queries nvidia-smi directly instead of reading it from
audit.json. The v11.55 systemd fix (bee-nvidia.service ordering,
nv-hostengine restart) and the nvlink-status/-errors/dcgmi dumps in the
support bundle are untouched.

Also reorganizes support bundle collection per project convention:
system/ is now LiveCD-operational logs only (Xorg, services, console,
network/FS of the host itself); all server-hardware dumps (lspci,
NVIDIA/NVLink/DCGM, fabric manager, PCIe AER, ethtool, mstflint) move
to techdump/, deduplicating two entries already produced by
platform/techdump.go. Adds nvidia-bug-report.sh (previously only
collected inside the NVIDIA SAT pack) and lscpu to the always-on dump.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-07-07 11:50:33 +03:00
co-authored by Claude Sonnet 5
parent 33d6eee9cf
commit a3377083aa
11 changed files with 1110 additions and 322 deletions
Download Patch File Download Diff File Expand all files Collapse all files
audit/internal/app/app_test.go
+7 -7
View File
@@ -884,13 +884,13 @@ func TestBuildSupportBundleIncludesExportDirContents(t *testing.T) {
for _, want := range []string{
"/system/ip-link.txt",
"/system/ip-link-stats.txt",
"/system/kernel-aer-nvidia.txt",
"/system/lspci-nvidia-bridges-vv.txt",
"/system/pcie-aer-sysfs.txt",
"/system/ethtool-info.txt",
"/system/ethtool-link.txt",
"/system/ethtool-module.txt",
"/system/mstflint-query.txt",
"/techdump/kernel-aer-nvidia.txt",
"/techdump/lspci-nvidia-bridges-vv.txt",
"/techdump/pcie-aer-sysfs.txt",
"/techdump/ethtool-info.txt",
"/techdump/ethtool-link.txt",
"/techdump/ethtool-module.txt",
"/techdump/mstflint-query.txt",
} {
var found bool
for _, name := range names {
audit/internal/app/support_bundle.go
+29 -22
View File
@@ -37,8 +37,7 @@ var supportBundleCommands = []struct {
{name: "system/uname.txt", cmd: []string{"uname", "-a"}},
{name: "system/cmdline.txt", cmd: []string{"cat", "/proc/cmdline"}},
{name: "system/lsmod.txt", cmd: []string{"lsmod"}},
{name: "system/lspci-nn.txt", cmd: []string{"lspci", "-nn"}},
{name: "system/lspci-vvv.txt", cmd: []string{"lspci", "-vvv"}},
{name: "techdump/lspci-nn.txt", cmd: []string{"lspci", "-nn"}},
{name: "system/ip-addr.txt", cmd: []string{"ip", "addr"}},
{name: "system/ip-link.txt", cmd: []string{"ip", "-details", "link", "show"}},
{name: "system/ip-link-stats.txt", cmd: []string{"ip", "-s", "link", "show"}},
@@ -53,7 +52,7 @@ else
echo "dmesg not found"
fi
`}},
{name: "system/kernel-aer-nvidia.txt", cmd: []string{"sh", "-c", `
{name: "techdump/kernel-aer-nvidia.txt", cmd: []string{"sh", "-c", `
if command -v dmesg >/dev/null 2>&1; then
dmesg | grep -iE 'AER|NVRM|Xid|pcieport|nvidia' || echo "no AER/NVRM/Xid kernel messages found"
else
@@ -83,7 +82,7 @@ fi
{name: "system/ps-gui.txt", cmd: []string{"sh", "-c", `
ps -ef | grep -iE 'lightdm|Xorg|X$|openbox|chromium|chrome|xinit|xsession' | grep -v grep || echo "no GUI processes found"
`}},
{name: "system/lspci-video-vv.txt", cmd: []string{"sh", "-c", `
{name: "techdump/lspci-video-vv.txt", cmd: []string{"sh", "-c", `
if ! command -v lspci >/dev/null 2>&1; then
echo "lspci not found"
exit 0
@@ -169,36 +168,44 @@ echo
echo "=== failed units ==="
systemctl --failed --no-pager 2>&1 | grep -iE 'lightdm|display-manager|Xorg' || echo "no failed GUI units"
`}},
{name: "system/nvidia-smi-q.txt", cmd: []string{"nvidia-smi", "-q"}},
{name: "system/nvidia-smi-topo.txt", cmd: []string{"sh", "-c", `
{name: "techdump/nvidia-smi-topo.txt", cmd: []string{"sh", "-c", `
if command -v nvidia-smi >/dev/null 2>&1; then
nvidia-smi topo -m 2>&1 || true
else
echo "nvidia-smi not found"
fi
`}},
{name: "system/nvidia-smi-nvlink-status.txt", cmd: []string{"sh", "-c", `
{name: "techdump/nvidia-smi-nvlink-status.txt", cmd: []string{"sh", "-c", `
if command -v nvidia-smi >/dev/null 2>&1; then
nvidia-smi nvlink -s 2>&1 || true
else
echo "nvidia-smi not found"
fi
`}},
{name: "system/nvidia-smi-nvlink-errors.txt", cmd: []string{"sh", "-c", `
{name: "techdump/nvidia-smi-nvlink-errors.txt", cmd: []string{"sh", "-c", `
if command -v nvidia-smi >/dev/null 2>&1; then
nvidia-smi nvlink -e 2>&1 || true
else
echo "nvidia-smi not found"
fi
`}},
{name: "system/dcgmi-nvlink-status.txt", cmd: []string{"sh", "-c", `
{name: "techdump/dcgmi-nvlink-status.txt", cmd: []string{"sh", "-c", `
if command -v dcgmi >/dev/null 2>&1; then
dcgmi nvlink --link-status 2>&1 || true
else
echo "dcgmi not found"
fi
`}},
{name: "system/systemctl-nvidia-units.txt", cmd: []string{"sh", "-c", `
{name: "techdump/nvidia-bug-report.txt", cmd: []string{"sh", "-c", `
if command -v nvidia-bug-report.sh >/dev/null 2>&1; then
nvidia-bug-report.sh --output-file /tmp/bee-nvidia-bug-report.log >/dev/null 2>&1 \
&& cat /tmp/bee-nvidia-bug-report.log \
&& rm -f /tmp/bee-nvidia-bug-report.log
else
echo "nvidia-bug-report.sh not found"
fi
`}},
{name: "techdump/systemctl-nvidia-units.txt", cmd: []string{"sh", "-c", `
if ! command -v systemctl >/dev/null 2>&1; then
echo "systemctl not found"
exit 0
@@ -212,7 +219,7 @@ echo
echo "=== failed units ==="
systemctl --failed --no-pager 2>&1 | grep -iE 'nvidia|fabric' || echo "no failed nvidia/fabric units"
`}},
{name: "system/fabric-manager-paths.txt", cmd: []string{"sh", "-c", `
{name: "techdump/fabric-manager-paths.txt", cmd: []string{"sh", "-c", `
for candidate in \
/usr/bin/nvidia-fabricmanager \
/usr/bin/nv-fabricmanager \
@@ -228,7 +235,7 @@ if ! ls /usr/bin/nvidia-fabricmanager /usr/bin/nv-fabricmanager /usr/bin/nvidia-
echo "no fabric manager binaries found"
fi
`}},
{name: "system/lspci-nvidia-bridges-vv.txt", cmd: []string{"sh", "-c", `
{name: "techdump/lspci-nvidia-bridges-vv.txt", cmd: []string{"sh", "-c", `
if ! command -v lspci >/dev/null 2>&1; then
echo "lspci not found"
exit 0
@@ -250,7 +257,7 @@ if [ "$found" -eq 0 ]; then
echo "no NVIDIA PCI devices found"
fi
`}},
{name: "system/pcie-nvidia-link.txt", cmd: []string{"sh", "-c", `
{name: "techdump/pcie-nvidia-link.txt", cmd: []string{"sh", "-c", `
for d in /sys/bus/pci/devices/*/; do
vendor=$(cat "$d/vendor" 2>/dev/null)
[ "$vendor" = "0x10de" ] || continue
@@ -266,7 +273,7 @@ for d in /sys/bus/pci/devices/*/; do
done
done
`}},
{name: "system/pcie-aer-sysfs.txt", cmd: []string{"sh", "-c", `
{name: "techdump/pcie-aer-sysfs.txt", cmd: []string{"sh", "-c", `
found=0
for dev in /sys/bus/pci/devices/*; do
[ -e "$dev" ] || continue
@@ -290,7 +297,7 @@ if [ "$found" -eq 0 ]; then
echo "no PCIe AER sysfs counters found"
fi
`}},
{name: "system/ethtool-info.txt", cmd: []string{"sh", "-c", `
{name: "techdump/ethtool-info.txt", cmd: []string{"sh", "-c", `
if ! command -v ethtool >/dev/null 2>&1; then
echo "ethtool not found"
exit 0
@@ -309,7 +316,7 @@ if [ "$found" -eq 0 ]; then
echo "no interfaces found"
fi
`}},
{name: "system/ethtool-link.txt", cmd: []string{"sh", "-c", `
{name: "techdump/ethtool-link.txt", cmd: []string{"sh", "-c", `
if ! command -v ethtool >/dev/null 2>&1; then
echo "ethtool not found"
exit 0
@@ -328,7 +335,7 @@ if [ "$found" -eq 0 ]; then
echo "no interfaces found"
fi
`}},
{name: "system/ethtool-module.txt", cmd: []string{"sh", "-c", `
{name: "techdump/ethtool-module.txt", cmd: []string{"sh", "-c", `
if ! command -v ethtool >/dev/null 2>&1; then
echo "ethtool not found"
exit 0
@@ -347,7 +354,7 @@ if [ "$found" -eq 0 ]; then
echo "no interfaces found"
fi
`}},
{name: "system/mstflint-query.txt", cmd: []string{"sh", "-c", `
{name: "techdump/mstflint-query.txt", cmd: []string{"sh", "-c", `
if ! command -v mstflint >/dev/null 2>&1; then
echo "mstflint not found"
exit 0
@@ -382,10 +389,10 @@ var supportBundleOptionalFiles = []struct {
{name: "system/lightdm/x-0-greeter.log", src: "/var/log/lightdm/x-0-greeter.log"},
{name: "system/home-bee-xsession-errors.log", src: "/home/bee/.xsession-errors"},
{name: "system/home-bee-chromium-debug.log", src: "/tmp/bee-chrome/chrome_debug.log"},
{name: "system/fabricmanager.log", src: "/var/log/fabricmanager.log"},
{name: "system/nvlsm.log", src: "/var/log/nvlsm.log"},
{name: "system/fabricmanager/fabricmanager.log", src: "/var/log/fabricmanager/fabricmanager.log"},
{name: "system/fabricmanager/nvlsm.log", src: "/var/log/fabricmanager/nvlsm.log"},
{name: "techdump/fabricmanager.log", src: "/var/log/fabricmanager.log"},
{name: "techdump/nvlsm.log", src: "/var/log/nvlsm.log"},
{name: "techdump/fabricmanager/fabricmanager.log", src: "/var/log/fabricmanager/fabricmanager.log"},
{name: "techdump/fabricmanager/nvlsm.log", src: "/var/log/fabricmanager/nvlsm.log"},
}
const supportBundleGlob = "????-??-?? (BEE-SP*)*.tar.gz"