webui: add /topo server topology page
New read-only visualization page: CPU sockets as anchor nodes, PCIe
devices (GPU/NIC/RAID) linked to their NUMA-affine socket with edge
color derived strictly from link_speed vs max_link_speed (not from the
device's own Status, which can also be overwritten by SAT-test results
on the same field), PSU/BMC as standalone boxes with no connecting
line, and a separate NVLink Topology card (live nvidia-smi topo -m /
nvlink -s/-e queries, not persisted to any contract).
GPU-GPU edges are drawn strictly from the actual bonded-pair list
parsed out of "nvidia-smi topo -m" (parseGPUPairAdjacency), not from
adjacent box position in the layout — an earlier ASCII mockup drew a
"chain" through unrelated GPUs, which a dedicated regression test now
guards against. A bonded pair spanning two different NUMA nodes is
flagged Warning on the edge and on both GPU boxes, per project
decision that this is an anomaly worth surfacing, not a neutral fact.
Zero changes to the ingest contract: this reverts the HardwareNVLinkPort/
HardwarePCIeDevice.NVLinks field shipped in v11.55 (33d6eee) along with
its collector/nvidia.go enrichment — that field risked a 400 from
Reanimator Core's strict decoder without an RFC, and isn't needed since
the topo page queries nvidia-smi directly instead of reading it from
audit.json. The v11.55 systemd fix (bee-nvidia.service ordering,
nv-hostengine restart) and the nvlink-status/-errors/dcgmi dumps in the
support bundle are untouched.
Also reorganizes support bundle collection per project convention:
system/ is now LiveCD-operational logs only (Xorg, services, console,
network/FS of the host itself); all server-hardware dumps (lspci,
NVIDIA/NVLink/DCGM, fabric manager, PCIe AER, ethtool, mstflint) move
to techdump/, deduplicating two entries already produced by
platform/techdump.go. Adds nvidia-bug-report.sh (previously only
collected inside the NVIDIA SAT pack) and lscpu to the always-on dump.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
co-authored by
Claude Sonnet 5
Claude Sonnet 5
parent
33d6eee9cf
commit
a3377083aa
@@ -884,13 +884,13 @@ func TestBuildSupportBundleIncludesExportDirContents(t *testing.T) {
|
||||
for _, want := range []string{
|
||||
"/system/ip-link.txt",
|
||||
"/system/ip-link-stats.txt",
|
||||
"/system/kernel-aer-nvidia.txt",
|
||||
"/system/lspci-nvidia-bridges-vv.txt",
|
||||
"/system/pcie-aer-sysfs.txt",
|
||||
"/system/ethtool-info.txt",
|
||||
"/system/ethtool-link.txt",
|
||||
"/system/ethtool-module.txt",
|
||||
"/system/mstflint-query.txt",
|
||||
"/techdump/kernel-aer-nvidia.txt",
|
||||
"/techdump/lspci-nvidia-bridges-vv.txt",
|
||||
"/techdump/pcie-aer-sysfs.txt",
|
||||
"/techdump/ethtool-info.txt",
|
||||
"/techdump/ethtool-link.txt",
|
||||
"/techdump/ethtool-module.txt",
|
||||
"/techdump/mstflint-query.txt",
|
||||
} {
|
||||
var found bool
|
||||
for _, name := range names {
|
||||
|
||||
@@ -37,8 +37,7 @@ var supportBundleCommands = []struct {
|
||||
{name: "system/uname.txt", cmd: []string{"uname", "-a"}},
|
||||
{name: "system/cmdline.txt", cmd: []string{"cat", "/proc/cmdline"}},
|
||||
{name: "system/lsmod.txt", cmd: []string{"lsmod"}},
|
||||
{name: "system/lspci-nn.txt", cmd: []string{"lspci", "-nn"}},
|
||||
{name: "system/lspci-vvv.txt", cmd: []string{"lspci", "-vvv"}},
|
||||
{name: "techdump/lspci-nn.txt", cmd: []string{"lspci", "-nn"}},
|
||||
{name: "system/ip-addr.txt", cmd: []string{"ip", "addr"}},
|
||||
{name: "system/ip-link.txt", cmd: []string{"ip", "-details", "link", "show"}},
|
||||
{name: "system/ip-link-stats.txt", cmd: []string{"ip", "-s", "link", "show"}},
|
||||
@@ -53,7 +52,7 @@ else
|
||||
echo "dmesg not found"
|
||||
fi
|
||||
`}},
|
||||
{name: "system/kernel-aer-nvidia.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "techdump/kernel-aer-nvidia.txt", cmd: []string{"sh", "-c", `
|
||||
if command -v dmesg >/dev/null 2>&1; then
|
||||
dmesg | grep -iE 'AER|NVRM|Xid|pcieport|nvidia' || echo "no AER/NVRM/Xid kernel messages found"
|
||||
else
|
||||
@@ -83,7 +82,7 @@ fi
|
||||
{name: "system/ps-gui.txt", cmd: []string{"sh", "-c", `
|
||||
ps -ef | grep -iE 'lightdm|Xorg|X$|openbox|chromium|chrome|xinit|xsession' | grep -v grep || echo "no GUI processes found"
|
||||
`}},
|
||||
{name: "system/lspci-video-vv.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "techdump/lspci-video-vv.txt", cmd: []string{"sh", "-c", `
|
||||
if ! command -v lspci >/dev/null 2>&1; then
|
||||
echo "lspci not found"
|
||||
exit 0
|
||||
@@ -169,36 +168,44 @@ echo
|
||||
echo "=== failed units ==="
|
||||
systemctl --failed --no-pager 2>&1 | grep -iE 'lightdm|display-manager|Xorg' || echo "no failed GUI units"
|
||||
`}},
|
||||
{name: "system/nvidia-smi-q.txt", cmd: []string{"nvidia-smi", "-q"}},
|
||||
{name: "system/nvidia-smi-topo.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "techdump/nvidia-smi-topo.txt", cmd: []string{"sh", "-c", `
|
||||
if command -v nvidia-smi >/dev/null 2>&1; then
|
||||
nvidia-smi topo -m 2>&1 || true
|
||||
else
|
||||
echo "nvidia-smi not found"
|
||||
fi
|
||||
`}},
|
||||
{name: "system/nvidia-smi-nvlink-status.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "techdump/nvidia-smi-nvlink-status.txt", cmd: []string{"sh", "-c", `
|
||||
if command -v nvidia-smi >/dev/null 2>&1; then
|
||||
nvidia-smi nvlink -s 2>&1 || true
|
||||
else
|
||||
echo "nvidia-smi not found"
|
||||
fi
|
||||
`}},
|
||||
{name: "system/nvidia-smi-nvlink-errors.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "techdump/nvidia-smi-nvlink-errors.txt", cmd: []string{"sh", "-c", `
|
||||
if command -v nvidia-smi >/dev/null 2>&1; then
|
||||
nvidia-smi nvlink -e 2>&1 || true
|
||||
else
|
||||
echo "nvidia-smi not found"
|
||||
fi
|
||||
`}},
|
||||
{name: "system/dcgmi-nvlink-status.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "techdump/dcgmi-nvlink-status.txt", cmd: []string{"sh", "-c", `
|
||||
if command -v dcgmi >/dev/null 2>&1; then
|
||||
dcgmi nvlink --link-status 2>&1 || true
|
||||
else
|
||||
echo "dcgmi not found"
|
||||
fi
|
||||
`}},
|
||||
{name: "system/systemctl-nvidia-units.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "techdump/nvidia-bug-report.txt", cmd: []string{"sh", "-c", `
|
||||
if command -v nvidia-bug-report.sh >/dev/null 2>&1; then
|
||||
nvidia-bug-report.sh --output-file /tmp/bee-nvidia-bug-report.log >/dev/null 2>&1 \
|
||||
&& cat /tmp/bee-nvidia-bug-report.log \
|
||||
&& rm -f /tmp/bee-nvidia-bug-report.log
|
||||
else
|
||||
echo "nvidia-bug-report.sh not found"
|
||||
fi
|
||||
`}},
|
||||
{name: "techdump/systemctl-nvidia-units.txt", cmd: []string{"sh", "-c", `
|
||||
if ! command -v systemctl >/dev/null 2>&1; then
|
||||
echo "systemctl not found"
|
||||
exit 0
|
||||
@@ -212,7 +219,7 @@ echo
|
||||
echo "=== failed units ==="
|
||||
systemctl --failed --no-pager 2>&1 | grep -iE 'nvidia|fabric' || echo "no failed nvidia/fabric units"
|
||||
`}},
|
||||
{name: "system/fabric-manager-paths.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "techdump/fabric-manager-paths.txt", cmd: []string{"sh", "-c", `
|
||||
for candidate in \
|
||||
/usr/bin/nvidia-fabricmanager \
|
||||
/usr/bin/nv-fabricmanager \
|
||||
@@ -228,7 +235,7 @@ if ! ls /usr/bin/nvidia-fabricmanager /usr/bin/nv-fabricmanager /usr/bin/nvidia-
|
||||
echo "no fabric manager binaries found"
|
||||
fi
|
||||
`}},
|
||||
{name: "system/lspci-nvidia-bridges-vv.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "techdump/lspci-nvidia-bridges-vv.txt", cmd: []string{"sh", "-c", `
|
||||
if ! command -v lspci >/dev/null 2>&1; then
|
||||
echo "lspci not found"
|
||||
exit 0
|
||||
@@ -250,7 +257,7 @@ if [ "$found" -eq 0 ]; then
|
||||
echo "no NVIDIA PCI devices found"
|
||||
fi
|
||||
`}},
|
||||
{name: "system/pcie-nvidia-link.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "techdump/pcie-nvidia-link.txt", cmd: []string{"sh", "-c", `
|
||||
for d in /sys/bus/pci/devices/*/; do
|
||||
vendor=$(cat "$d/vendor" 2>/dev/null)
|
||||
[ "$vendor" = "0x10de" ] || continue
|
||||
@@ -266,7 +273,7 @@ for d in /sys/bus/pci/devices/*/; do
|
||||
done
|
||||
done
|
||||
`}},
|
||||
{name: "system/pcie-aer-sysfs.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "techdump/pcie-aer-sysfs.txt", cmd: []string{"sh", "-c", `
|
||||
found=0
|
||||
for dev in /sys/bus/pci/devices/*; do
|
||||
[ -e "$dev" ] || continue
|
||||
@@ -290,7 +297,7 @@ if [ "$found" -eq 0 ]; then
|
||||
echo "no PCIe AER sysfs counters found"
|
||||
fi
|
||||
`}},
|
||||
{name: "system/ethtool-info.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "techdump/ethtool-info.txt", cmd: []string{"sh", "-c", `
|
||||
if ! command -v ethtool >/dev/null 2>&1; then
|
||||
echo "ethtool not found"
|
||||
exit 0
|
||||
@@ -309,7 +316,7 @@ if [ "$found" -eq 0 ]; then
|
||||
echo "no interfaces found"
|
||||
fi
|
||||
`}},
|
||||
{name: "system/ethtool-link.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "techdump/ethtool-link.txt", cmd: []string{"sh", "-c", `
|
||||
if ! command -v ethtool >/dev/null 2>&1; then
|
||||
echo "ethtool not found"
|
||||
exit 0
|
||||
@@ -328,7 +335,7 @@ if [ "$found" -eq 0 ]; then
|
||||
echo "no interfaces found"
|
||||
fi
|
||||
`}},
|
||||
{name: "system/ethtool-module.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "techdump/ethtool-module.txt", cmd: []string{"sh", "-c", `
|
||||
if ! command -v ethtool >/dev/null 2>&1; then
|
||||
echo "ethtool not found"
|
||||
exit 0
|
||||
@@ -347,7 +354,7 @@ if [ "$found" -eq 0 ]; then
|
||||
echo "no interfaces found"
|
||||
fi
|
||||
`}},
|
||||
{name: "system/mstflint-query.txt", cmd: []string{"sh", "-c", `
|
||||
{name: "techdump/mstflint-query.txt", cmd: []string{"sh", "-c", `
|
||||
if ! command -v mstflint >/dev/null 2>&1; then
|
||||
echo "mstflint not found"
|
||||
exit 0
|
||||
@@ -382,10 +389,10 @@ var supportBundleOptionalFiles = []struct {
|
||||
{name: "system/lightdm/x-0-greeter.log", src: "/var/log/lightdm/x-0-greeter.log"},
|
||||
{name: "system/home-bee-xsession-errors.log", src: "/home/bee/.xsession-errors"},
|
||||
{name: "system/home-bee-chromium-debug.log", src: "/tmp/bee-chrome/chrome_debug.log"},
|
||||
{name: "system/fabricmanager.log", src: "/var/log/fabricmanager.log"},
|
||||
{name: "system/nvlsm.log", src: "/var/log/nvlsm.log"},
|
||||
{name: "system/fabricmanager/fabricmanager.log", src: "/var/log/fabricmanager/fabricmanager.log"},
|
||||
{name: "system/fabricmanager/nvlsm.log", src: "/var/log/fabricmanager/nvlsm.log"},
|
||||
{name: "techdump/fabricmanager.log", src: "/var/log/fabricmanager.log"},
|
||||
{name: "techdump/nvlsm.log", src: "/var/log/nvlsm.log"},
|
||||
{name: "techdump/fabricmanager/fabricmanager.log", src: "/var/log/fabricmanager/fabricmanager.log"},
|
||||
{name: "techdump/fabricmanager/nvlsm.log", src: "/var/log/fabricmanager/nvlsm.log"},
|
||||
}
|
||||
|
||||
const supportBundleGlob = "????-??-?? (BEE-SP*)*.tar.gz"
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
@@ -39,48 +38,7 @@ func enrichPCIeWithNVIDIA(devs []schema.HardwarePCIeDevice) []schema.HardwarePCI
|
||||
slog.Info("nvidia: enrichment skipped", "err", err)
|
||||
return enrichPCIeWithNVIDIAData(devs, nil, false)
|
||||
}
|
||||
devs = enrichPCIeWithNVIDIAData(devs, gpuByBDF, true)
|
||||
return enrichPCIeWithNVIDIANVLinks(devs)
|
||||
}
|
||||
|
||||
// enrichPCIeWithNVIDIANVLinks attaches per-link NVLink status (nvidia-smi
|
||||
// nvlink -s) and error counters (nvidia-smi nvlink -e) to each GPU's
|
||||
// HardwarePCIeDevice entry, keyed by the "nvidia_gpu_index" telemetry set by
|
||||
// enrichPCIeWithNVIDIAData. Independent of NVSwitch/fabric-manager detection
|
||||
// so it also covers direct GPU-to-GPU bridge boards with no switch present.
|
||||
func enrichPCIeWithNVIDIANVLinks(devs []schema.HardwarePCIeDevice) []schema.HardwarePCIeDevice {
|
||||
statusByGPU, statusErr := nvlinkStatusFn()
|
||||
if statusErr != nil {
|
||||
slog.Info("nvidia: nvlink -s unavailable, skipping nvlink enrichment", "err", statusErr)
|
||||
return devs
|
||||
}
|
||||
errorsByGPU, errorsErr := nvlinkErrorsFn()
|
||||
if errorsErr != nil {
|
||||
slog.Info("nvidia: nvlink -e unavailable", "err", errorsErr)
|
||||
}
|
||||
|
||||
for i := range devs {
|
||||
if devs[i].Telemetry == nil {
|
||||
continue
|
||||
}
|
||||
idx, ok := devs[i].Telemetry["nvidia_gpu_index"].(int)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
ports, ok := statusByGPU[idx]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for j := range ports {
|
||||
if counters, ok := errorsByGPU[idx][ports[j].Index]; ok {
|
||||
ports[j].ReplayErrors = &counters.Replay
|
||||
ports[j].RecoveryErrors = &counters.Recovery
|
||||
ports[j].CRCErrors = &counters.CRC
|
||||
}
|
||||
}
|
||||
devs[i].NVLinks = ports
|
||||
}
|
||||
return devs
|
||||
return enrichPCIeWithNVIDIAData(devs, gpuByBDF, true)
|
||||
}
|
||||
|
||||
func hasNVIDIADevices(devs []schema.HardwarePCIeDevice) bool {
|
||||
@@ -327,107 +285,3 @@ func injectNVIDIATelemetry(dev *schema.HardwarePCIeDevice, info nvidiaGPUInfo) {
|
||||
dev.MaxLinkWidth = info.PCIeLinkWidthMax
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
nvlinkGPUHeaderRe = regexp.MustCompile(`^GPU (\d+):`)
|
||||
nvlinkSpeedLineRe = regexp.MustCompile(`^Link (\d+):\s*([\d.]+)\s*GB/s`)
|
||||
nvlinkInactiveRe = regexp.MustCompile(`^Link (\d+):\s*<inactive>`)
|
||||
nvlinkErrorCounterRe = regexp.MustCompile(`^Link (\d+):\s*(Replay|Recovery|CRC) Errors:\s*(\d+)`)
|
||||
)
|
||||
|
||||
// nvlinkErrorCounters holds the per-link error counters reported by
|
||||
// "nvidia-smi nvlink -e" for one GPU.
|
||||
type nvlinkErrorCounters struct {
|
||||
Replay, Recovery, CRC int64
|
||||
}
|
||||
|
||||
// nvlinkStatusFn and nvlinkErrorsFn are swappable for testing.
|
||||
var (
|
||||
nvlinkStatusFn = queryNVIDIANVLinkStatusByGPU
|
||||
nvlinkErrorsFn = queryNVIDIANVLinkErrorsByGPU
|
||||
)
|
||||
|
||||
// queryNVIDIANVLinkStatusByGPU runs "nvidia-smi nvlink -s" and returns each
|
||||
// GPU's NVLink ports keyed by GPU index (as printed in the "GPU N:" header,
|
||||
// matching the index nvidia-smi --query-gpu also reports).
|
||||
func queryNVIDIANVLinkStatusByGPU() (map[int][]schema.HardwareNVLinkPort, error) {
|
||||
out, err := exec.Command("nvidia-smi", "nvlink", "-s").Output()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return parseNVIDIANVLinkStatusByGPU(string(out)), nil
|
||||
}
|
||||
|
||||
func parseNVIDIANVLinkStatusByGPU(raw string) map[int][]schema.HardwareNVLinkPort {
|
||||
result := map[int][]schema.HardwareNVLinkPort{}
|
||||
currentGPU := -1
|
||||
for _, line := range strings.Split(raw, "\n") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if m := nvlinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
|
||||
currentGPU, _ = strconv.Atoi(m[1])
|
||||
continue
|
||||
}
|
||||
if currentGPU < 0 {
|
||||
continue
|
||||
}
|
||||
if m := nvlinkInactiveRe.FindStringSubmatch(trimmed); m != nil {
|
||||
idx, _ := strconv.Atoi(m[1])
|
||||
result[currentGPU] = append(result[currentGPU], schema.HardwareNVLinkPort{Index: idx, Active: false})
|
||||
continue
|
||||
}
|
||||
if m := nvlinkSpeedLineRe.FindStringSubmatch(trimmed); m != nil {
|
||||
idx, _ := strconv.Atoi(m[1])
|
||||
port := schema.HardwareNVLinkPort{Index: idx, Active: true}
|
||||
if speed, err := strconv.ParseFloat(m[2], 64); err == nil {
|
||||
port.SpeedGBs = &speed
|
||||
}
|
||||
result[currentGPU] = append(result[currentGPU], port)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// queryNVIDIANVLinkErrorsByGPU runs "nvidia-smi nvlink -e" and returns
|
||||
// per-link error counters keyed by GPU index then link index.
|
||||
func queryNVIDIANVLinkErrorsByGPU() (map[int]map[int]nvlinkErrorCounters, error) {
|
||||
out, err := exec.Command("nvidia-smi", "nvlink", "-e").Output()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return parseNVIDIANVLinkErrorsByGPU(string(out)), nil
|
||||
}
|
||||
|
||||
func parseNVIDIANVLinkErrorsByGPU(raw string) map[int]map[int]nvlinkErrorCounters {
|
||||
result := map[int]map[int]nvlinkErrorCounters{}
|
||||
currentGPU := -1
|
||||
for _, line := range strings.Split(raw, "\n") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if m := nvlinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
|
||||
currentGPU, _ = strconv.Atoi(m[1])
|
||||
continue
|
||||
}
|
||||
if currentGPU < 0 {
|
||||
continue
|
||||
}
|
||||
m := nvlinkErrorCounterRe.FindStringSubmatch(trimmed)
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
linkIdx, _ := strconv.Atoi(m[1])
|
||||
count, _ := strconv.ParseInt(m[3], 10, 64)
|
||||
if result[currentGPU] == nil {
|
||||
result[currentGPU] = map[int]nvlinkErrorCounters{}
|
||||
}
|
||||
c := result[currentGPU][linkIdx]
|
||||
switch m[2] {
|
||||
case "Replay":
|
||||
c.Replay = count
|
||||
case "Recovery":
|
||||
c.Recovery = count
|
||||
case "CRC":
|
||||
c.CRC = count
|
||||
}
|
||||
result[currentGPU][linkIdx] = c
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
@@ -126,92 +126,3 @@ func TestEnrichPCIeWithNVIDIAData_driverMissingFallback(t *testing.T) {
|
||||
|
||||
func ptrInt64(v int64) *int64 { return &v }
|
||||
func ptrFloat(v float64) *float64 { return &v }
|
||||
|
||||
func TestParseNVIDIANVLinkStatusByGPU(t *testing.T) {
|
||||
// Real-world 2-GPU direct-bridge H100 SXM output: link 15 inactive on both GPUs.
|
||||
input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140)
|
||||
Link 0: 26.562 GB/s
|
||||
Link 15: <inactive>
|
||||
Link 17: 26.562 GB/s
|
||||
GPU 1: NVIDIA H100 80GB HBM3 (UUID: GPU-603fe750-0516-9db5-86ec-ea61af3fce35)
|
||||
Link 0: 26.562 GB/s
|
||||
Link 15: <inactive>
|
||||
`
|
||||
got := parseNVIDIANVLinkStatusByGPU(input)
|
||||
|
||||
if len(got[0]) != 3 {
|
||||
t.Fatalf("gpu0 ports=%d want 3 (%#v)", len(got[0]), got[0])
|
||||
}
|
||||
if got[0][1].Index != 15 || got[0][1].Active {
|
||||
t.Fatalf("gpu0 link15=%#v want inactive", got[0][1])
|
||||
}
|
||||
if got[0][0].SpeedGBs == nil || *got[0][0].SpeedGBs != 26.562 {
|
||||
t.Fatalf("gpu0 link0 speed=%#v want 26.562", got[0][0].SpeedGBs)
|
||||
}
|
||||
if len(got[1]) != 2 {
|
||||
t.Fatalf("gpu1 ports=%d want 2 (%#v)", len(got[1]), got[1])
|
||||
}
|
||||
if got[1][1].Active {
|
||||
t.Fatalf("gpu1 link15 should be inactive: %#v", got[1][1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseNVIDIANVLinkErrorsByGPU(t *testing.T) {
|
||||
input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140)
|
||||
Link 0: Replay Errors: 0
|
||||
Link 0: Recovery Errors: 0
|
||||
Link 0: CRC Errors: 0
|
||||
Link 1: Replay Errors: 3
|
||||
Link 1: Recovery Errors: 1
|
||||
Link 1: CRC Errors: 2
|
||||
GPU 1: NVIDIA H100 80GB HBM3 (UUID: GPU-603fe750-0516-9db5-86ec-ea61af3fce35)
|
||||
Link 0: Replay Errors: 0
|
||||
Link 0: Recovery Errors: 0
|
||||
Link 0: CRC Errors: 0
|
||||
`
|
||||
got := parseNVIDIANVLinkErrorsByGPU(input)
|
||||
|
||||
c := got[0][1]
|
||||
if c.Replay != 3 || c.Recovery != 1 || c.CRC != 2 {
|
||||
t.Fatalf("gpu0 link1 counters=%#v want {3,1,2}", c)
|
||||
}
|
||||
zero := got[0][0]
|
||||
if zero.Replay != 0 || zero.Recovery != 0 || zero.CRC != 0 {
|
||||
t.Fatalf("gpu0 link0 counters=%#v want all zero", zero)
|
||||
}
|
||||
if _, ok := got[1][0]; !ok {
|
||||
t.Fatalf("expected gpu1 link0 entry present")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnrichPCIeWithNVIDIANVLinksAttachesPortsByIndex(t *testing.T) {
|
||||
oldStatus, oldErrors := nvlinkStatusFn, nvlinkErrorsFn
|
||||
t.Cleanup(func() { nvlinkStatusFn, nvlinkErrorsFn = oldStatus, oldErrors })
|
||||
|
||||
nvlinkStatusFn = func() (map[int][]schema.HardwareNVLinkPort, error) {
|
||||
return map[int][]schema.HardwareNVLinkPort{
|
||||
0: {{Index: 0, Active: true, SpeedGBs: ptrFloat(26.562)}, {Index: 15, Active: false}},
|
||||
}, nil
|
||||
}
|
||||
nvlinkErrorsFn = func() (map[int]map[int]nvlinkErrorCounters, error) {
|
||||
return map[int]map[int]nvlinkErrorCounters{
|
||||
0: {0: {Replay: 1}},
|
||||
}, nil
|
||||
}
|
||||
|
||||
devices := []schema.HardwarePCIeDevice{
|
||||
{Telemetry: map[string]any{"nvidia_gpu_index": 0}},
|
||||
}
|
||||
|
||||
out := enrichPCIeWithNVIDIANVLinks(devices)
|
||||
|
||||
if len(out[0].NVLinks) != 2 {
|
||||
t.Fatalf("nvlinks=%d want 2", len(out[0].NVLinks))
|
||||
}
|
||||
if out[0].NVLinks[0].ReplayErrors == nil || *out[0].NVLinks[0].ReplayErrors != 1 {
|
||||
t.Fatalf("link0 replay errors=%#v want 1", out[0].NVLinks[0].ReplayErrors)
|
||||
}
|
||||
if out[0].NVLinks[1].Active {
|
||||
t.Fatalf("link15 should stay inactive")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ var techDumpFixedCommands = []struct {
|
||||
{Name: "dmidecode", Args: []string{"-t", "17"}, File: "dmidecode-type17.txt"},
|
||||
{Name: "lspci", Args: []string{"-vmm", "-D"}, File: "lspci-vmm.txt"},
|
||||
{Name: "lspci", Args: []string{"-vvv"}, File: "lspci-vvv.txt"},
|
||||
{Name: "lscpu", Args: nil, File: "lscpu.txt"},
|
||||
{Name: "lsblk", Args: []string{"-J", "-d", "-o", "NAME,TYPE,SIZE,SERIAL,MODEL,TRAN,HCTL"}, File: "lsblk.json"},
|
||||
{Name: "sensors", Args: []string{"-j"}, File: "sensors.json"},
|
||||
{Name: "ipmitool", Args: []string{"fru", "print"}, File: "ipmitool-fru.txt"},
|
||||
|
||||
@@ -219,24 +219,9 @@ type HardwarePCIeDevice struct {
|
||||
MacAddresses []string `json:"mac_addresses,omitempty"`
|
||||
Present *bool `json:"present,omitempty"`
|
||||
IOMMUGroup *int `json:"iommu_group,omitempty"`
|
||||
NVLinks []HardwareNVLinkPort `json:"nvlinks,omitempty"`
|
||||
Telemetry map[string]any `json:"-"`
|
||||
}
|
||||
|
||||
// HardwareNVLinkPort describes a single NVLink lane on a GPU, as reported by
|
||||
// "nvidia-smi nvlink -s" (speed/active state) and "nvidia-smi nvlink -e"
|
||||
// (per-link error counters). Only populated on GPU-class HardwarePCIeDevice
|
||||
// entries. An inactive link is not necessarily a fault: some GPU SKUs/boards
|
||||
// reserve lanes as standby failover paths by design.
|
||||
type HardwareNVLinkPort struct {
|
||||
Index int `json:"index"`
|
||||
Active bool `json:"active"`
|
||||
SpeedGBs *float64 `json:"speed_gbs,omitempty"`
|
||||
ReplayErrors *int64 `json:"replay_errors,omitempty"`
|
||||
RecoveryErrors *int64 `json:"recovery_errors,omitempty"`
|
||||
CRCErrors *int64 `json:"crc_errors,omitempty"`
|
||||
}
|
||||
|
||||
type HardwarePowerSupply struct {
|
||||
HardwareComponentStatus
|
||||
Slot *string `json:"slot,omitempty"`
|
||||
|
||||
@@ -1726,6 +1726,9 @@ func (h *handler) handleAPIComponentDetail(w http.ResponseWriter, r *http.Reques
|
||||
case "gpu":
|
||||
title = "GPU"
|
||||
prefixes = []string{"pcie:gpu:"}
|
||||
case "nic":
|
||||
title = "NIC"
|
||||
prefixes = []string{"pcie:nic:"}
|
||||
case "psu":
|
||||
title = "PSU"
|
||||
prefixes = []string{"psu:"}
|
||||
|
||||
@@ -111,6 +111,7 @@ func layoutNav(active string, buildLabel string) string {
|
||||
{sep: true},
|
||||
{id: "tasks", label: "Tasks", href: "/tasks"},
|
||||
{id: "tools", label: "Tools", href: "/tools"},
|
||||
{id: "topo", label: "Topology", href: "/topo"},
|
||||
{id: "settings", label: "Settings", href: "/settings"},
|
||||
}
|
||||
var b strings.Builder
|
||||
|
||||
@@ -0,0 +1,777 @@
|
||||
package webui
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"html"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"bee/audit/internal/schema"
|
||||
)
|
||||
|
||||
// renderTopo renders the /topo page: a read-only visualization of the server
|
||||
// topology (CPU sockets, NUMA-affine PCIe devices, PSU/BMC) plus a separate
|
||||
// NVLink topology card. It is pure visualization: everything it reads either
|
||||
// already exists in the audit.json contract, or is a live (non-persisted)
|
||||
// query — nothing here writes to schema.HardwarePCIeDevice or any other
|
||||
// contract type, and nothing here talks to Reanimator Core.
|
||||
func renderTopo(opts HandlerOptions) string {
|
||||
data, err := loadSnapshot(opts.AuditPath)
|
||||
if err != nil {
|
||||
return topoCard("Topology", `<span class="badge badge-unknown">No audit data</span>`)
|
||||
}
|
||||
var ingest schema.HardwareIngestRequest
|
||||
if err := json.Unmarshal(data, &ingest); err != nil {
|
||||
return topoCard("Topology", `<span class="badge badge-err">Parse error</span>`)
|
||||
}
|
||||
hw := ingest.Hardware
|
||||
|
||||
var b strings.Builder
|
||||
b.WriteString(renderTopoMainDiagram(hw))
|
||||
if nv := renderTopoNVLinkCard(hw); nv != "" {
|
||||
b.WriteString(nv)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func topoCard(title, body string) string {
|
||||
return `<div class="card"><div class="card-head">` + html.EscapeString(title) + `</div><div class="card-body">` + body + `</div></div>`
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Classification helpers
|
||||
//
|
||||
// webui does not import collector (matches the existing isGPUDeviceClass
|
||||
// precedent in pages.go, which already locally duplicates collector.isGPUClass
|
||||
// instead of importing the package for one classifier).
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// isNICDeviceClassDev mirrors the classification logic in hwDescribeNIC
|
||||
// (pages.go), applied to a single device instead of aggregated counts.
|
||||
func isNICDeviceClassDev(dev schema.HardwarePCIeDevice) bool {
|
||||
if dev.DeviceClass != nil {
|
||||
c := strings.ToLower(strings.TrimSpace(*dev.DeviceClass))
|
||||
if c == "ethernetcontroller" || c == "networkcontroller" || strings.Contains(c, "fibrechannel") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return len(dev.MacAddresses) > 0
|
||||
}
|
||||
|
||||
// isRAIDControllerClass matches the canonical class strings produced by
|
||||
// collector.mapPCIeDeviceClass for RAID/storage HBAs.
|
||||
func isRAIDControllerClass(class string) bool {
|
||||
switch strings.TrimSpace(class) {
|
||||
case "MassStorageController", "StorageController":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Status / link-speed coloring
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// topoStatusBadgeClass maps a component's Status pointer to a badge class,
|
||||
// treating a nil/absent status the same as literal "Unknown" (matches how
|
||||
// the rest of the UI already renders missing status, per chipLetterClass/
|
||||
// runtimeStatusBadge in pages.go).
|
||||
func topoStatusBadgeClass(status *string) (label, cls string) {
|
||||
if status == nil {
|
||||
return "?", "badge-unknown"
|
||||
}
|
||||
switch strings.ToUpper(strings.TrimSpace(*status)) {
|
||||
case "OK":
|
||||
return "OK", "badge-ok"
|
||||
case "WARNING", "WARN", "PARTIAL":
|
||||
return "WARN", "badge-warn"
|
||||
case "CRITICAL", "FAIL", "FAILED", "ERROR":
|
||||
return "CRIT", "badge-err"
|
||||
default:
|
||||
return "?", "badge-unknown"
|
||||
}
|
||||
}
|
||||
|
||||
// pcieGenRank ranks a PCIe generation label ("Gen3", "Gen4", ...) for
|
||||
// comparison. Mirrors collector.pcieLinkSpeedRank's ordering; duplicated
|
||||
// locally rather than exported, per the same "no collector import in webui"
|
||||
// convention used for isGPUDeviceClass/isRAIDControllerClass.
|
||||
func pcieGenRank(gen string) int {
|
||||
gen = strings.ToLower(strings.TrimSpace(gen))
|
||||
gen = strings.TrimPrefix(gen, "gen")
|
||||
n, err := strconv.Atoi(gen)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// topoEdgeColorVar computes the CPU->device edge color strictly from
|
||||
// link_speed vs max_link_speed — NOT from dev.Status, since Status can also
|
||||
// be overwritten by SAT/acceptance-test results on the same PCIe device,
|
||||
// which would conflate "link is physically degraded" with "this GPU failed
|
||||
// its stress test" into the same color.
|
||||
func topoEdgeColorVar(dev schema.HardwarePCIeDevice) string {
|
||||
if dev.LinkSpeed == nil || dev.MaxLinkSpeed == nil {
|
||||
return "var(--muted)"
|
||||
}
|
||||
if pcieGenRank(*dev.LinkSpeed) < pcieGenRank(*dev.MaxLinkSpeed) {
|
||||
return "var(--warn-fg)"
|
||||
}
|
||||
return "var(--ok-fg)"
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// NUMA node -> CPU socket join (heuristic, no guaranteed hardware mapping)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// buildSocketIndex maps a NUMA node number to the index into cpus whose
|
||||
// Socket field equals that NUMA node number. This is a best-effort heuristic
|
||||
// (NUMANode == Socket) documented as such in the /topo design — Linux NUMA
|
||||
// node numbering and dmidecode socket designation are different numbering
|
||||
// domains with no guaranteed 1:1 mapping, but in practice agree for the
|
||||
// common case of N-socket boards.
|
||||
func buildSocketIndex(cpus []schema.HardwareCPU) map[int]int {
|
||||
idx := map[int]int{}
|
||||
for i, cpu := range cpus {
|
||||
if cpu.Socket != nil {
|
||||
idx[*cpu.Socket] = i
|
||||
}
|
||||
}
|
||||
return idx
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// GPU pairwise NVLink adjacency (from a live "nvidia-smi topo -m" query)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type gpuPairLink struct {
|
||||
GPUA, GPUB int
|
||||
NVLinks int
|
||||
}
|
||||
|
||||
var topoNVRe = regexp.MustCompile(`(?i)^NV(\d+)$`)
|
||||
|
||||
// parseGPUPairAdjacency returns every GPU pair with a nonzero NVLink bond
|
||||
// count from a "nvidia-smi topo -m" matrix. Unlike parseNVIDIATopologyMatrix
|
||||
// (collector package, aggregate-only: min/all-active/count), this returns
|
||||
// who is bonded to whom — required so GPU-GPU edges are drawn for actually
|
||||
// bonded pairs, not for adjacent boxes in the layout.
|
||||
func parseGPUPairAdjacency(raw string) []gpuPairLink {
|
||||
lines := strings.Split(raw, "\n")
|
||||
headerIdx := -1
|
||||
var gpuColIndices []int
|
||||
for i, line := range lines {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if strings.HasPrefix(trimmed, "GPU0") {
|
||||
parts := strings.Fields(trimmed)
|
||||
for j, col := range parts {
|
||||
if strings.HasPrefix(col, "GPU") {
|
||||
gpuColIndices = append(gpuColIndices, j)
|
||||
}
|
||||
}
|
||||
if len(gpuColIndices) >= 2 {
|
||||
headerIdx = i
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
if headerIdx < 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
colIdxToGPU := make(map[int]int, len(gpuColIndices))
|
||||
for gpuIdx, colIdx := range gpuColIndices {
|
||||
colIdxToGPU[colIdx] = gpuIdx
|
||||
}
|
||||
|
||||
seen := map[[2]int]bool{}
|
||||
var pairs []gpuPairLink
|
||||
rowGPU := -1
|
||||
for _, line := range lines[headerIdx+1:] {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if !strings.HasPrefix(trimmed, "GPU") {
|
||||
continue
|
||||
}
|
||||
cells := strings.Fields(trimmed)
|
||||
if len(cells) == 0 {
|
||||
continue
|
||||
}
|
||||
rowLabel := strings.TrimPrefix(cells[0], "GPU")
|
||||
n, err := strconv.Atoi(rowLabel)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
rowGPU = n
|
||||
for colIdx, colGPU := range colIdxToGPU {
|
||||
if colGPU == rowGPU {
|
||||
continue
|
||||
}
|
||||
dataIdx := colIdx + 1
|
||||
if dataIdx >= len(cells) {
|
||||
continue
|
||||
}
|
||||
m := topoNVRe.FindStringSubmatch(cells[dataIdx])
|
||||
if len(m) != 2 {
|
||||
continue
|
||||
}
|
||||
nv, err := strconv.Atoi(m[1])
|
||||
if err != nil || nv <= 0 {
|
||||
continue
|
||||
}
|
||||
a, bGPU := rowGPU, colGPU
|
||||
if a > bGPU {
|
||||
a, bGPU = bGPU, a
|
||||
}
|
||||
key := [2]int{a, bGPU}
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
pairs = append(pairs, gpuPairLink{GPUA: a, GPUB: bGPU, NVLinks: nv})
|
||||
}
|
||||
}
|
||||
sort.Slice(pairs, func(i, j int) bool {
|
||||
if pairs[i].GPUA != pairs[j].GPUA {
|
||||
return pairs[i].GPUA < pairs[j].GPUA
|
||||
}
|
||||
return pairs[i].GPUB < pairs[j].GPUB
|
||||
})
|
||||
return pairs
|
||||
}
|
||||
|
||||
func queryGPUTopologyMatrix() (string, error) {
|
||||
out, err := exec.Command("nvidia-smi", "topo", "-m").Output()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(out), nil
|
||||
}
|
||||
|
||||
// queryNVIDIAIndexByBDF runs a lightweight live nvidia-smi query mapping
|
||||
// PCI bus address (matching HardwarePCIeDevice.Slot) to the GPU index
|
||||
// nvidia-smi/dcgmi/topo -m report, so GPU-GPU edges (keyed by index) can be
|
||||
// anchored to the correct box (keyed by BDF) in the diagram.
|
||||
func queryNVIDIAIndexByBDF() (map[string]int, error) {
|
||||
out, err := exec.Command("nvidia-smi", "--query-gpu=index,pci.bus_id", "--format=csv,noheader").Output()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result := map[string]int{}
|
||||
for _, line := range strings.Split(string(out), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
parts := strings.SplitN(line, ",", 2)
|
||||
if len(parts) != 2 {
|
||||
continue
|
||||
}
|
||||
idx, err := strconv.Atoi(strings.TrimSpace(parts[0]))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
bdf := normalizeTopoBDF(strings.TrimSpace(parts[1]))
|
||||
if bdf == "" {
|
||||
continue
|
||||
}
|
||||
result[bdf] = idx
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// normalizeTopoBDF normalizes a PCI bus address to "dddd:bb:dd.f" form so
|
||||
// nvidia-smi's "pci.bus_id" output can be matched against
|
||||
// HardwarePCIeDevice.Slot regardless of minor formatting differences
|
||||
// (case, leading domain padding).
|
||||
func normalizeTopoBDF(bdf string) string {
|
||||
bdf = strings.ToLower(strings.TrimSpace(bdf))
|
||||
if bdf == "" {
|
||||
return ""
|
||||
}
|
||||
parts := strings.Split(bdf, ":")
|
||||
if len(parts) == 3 {
|
||||
domain := parts[0]
|
||||
if len(domain) > 4 {
|
||||
domain = domain[len(domain)-4:]
|
||||
}
|
||||
return domain + ":" + parts[1] + ":" + parts[2]
|
||||
}
|
||||
return bdf
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main topology diagram
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const (
|
||||
topoColWidth = 220
|
||||
topoBoxWidth = 190
|
||||
topoBoxHeight = 56
|
||||
topoDeviceGap = 14
|
||||
topoTopMargin = 30
|
||||
topoEdgeBand = 60
|
||||
topoBottomRowH = 90
|
||||
)
|
||||
|
||||
type topoBox struct {
|
||||
x, y, w, h int
|
||||
label string
|
||||
sublabel string
|
||||
badgeText string
|
||||
badgeCls string
|
||||
detailType string // "" = not clickable
|
||||
}
|
||||
|
||||
func renderTopoMainDiagram(hw schema.HardwareSnapshot) string {
|
||||
socketIdx := buildSocketIndex(hw.CPUs)
|
||||
numCols := len(hw.CPUs)
|
||||
if numCols == 0 {
|
||||
numCols = 1
|
||||
}
|
||||
unknownCol := numCols // extra trailing column for unmatched devices
|
||||
|
||||
// Group PCIe devices (GPU/NIC/RAID only — matches the mockup's node
|
||||
// types) into columns by NUMA node, falling back to the "unknown" bucket.
|
||||
type placedDevice struct {
|
||||
dev schema.HardwarePCIeDevice
|
||||
kind string // "gpu", "nic", "raid"
|
||||
col int
|
||||
bdf string
|
||||
}
|
||||
var placed []placedDevice
|
||||
for _, dev := range hw.PCIeDevices {
|
||||
var kind string
|
||||
switch {
|
||||
case dev.DeviceClass != nil && isGPUDeviceClass(*dev.DeviceClass):
|
||||
kind = "gpu"
|
||||
case isNICDeviceClassDev(dev):
|
||||
kind = "nic"
|
||||
case dev.DeviceClass != nil && isRAIDControllerClass(*dev.DeviceClass):
|
||||
kind = "raid"
|
||||
default:
|
||||
continue
|
||||
}
|
||||
col := unknownCol
|
||||
if dev.NUMANode != nil {
|
||||
if ci, ok := socketIdx[*dev.NUMANode]; ok {
|
||||
col = ci
|
||||
}
|
||||
}
|
||||
bdf := ""
|
||||
if dev.Slot != nil {
|
||||
bdf = normalizeTopoBDF(*dev.Slot)
|
||||
} else if dev.BDF != nil {
|
||||
bdf = normalizeTopoBDF(*dev.BDF)
|
||||
}
|
||||
placed = append(placed, placedDevice{dev: dev, kind: kind, col: col, bdf: bdf})
|
||||
}
|
||||
hasUnknownCol := false
|
||||
for _, p := range placed {
|
||||
if p.col == unknownCol {
|
||||
hasUnknownCol = true
|
||||
break
|
||||
}
|
||||
}
|
||||
totalCols := numCols
|
||||
if hasUnknownCol {
|
||||
totalCols++
|
||||
}
|
||||
|
||||
// Live GPU index<->BDF map + pairwise NVLink adjacency, best-effort:
|
||||
// if nvidia-smi is unavailable, GPU-GPU edges are simply omitted.
|
||||
bdfToIndex, _ := queryNVIDIAIndexByBDF()
|
||||
var pairs []gpuPairLink
|
||||
if topoMatrix, err := queryGPUTopologyMatrix(); err == nil {
|
||||
pairs = parseGPUPairAdjacency(topoMatrix)
|
||||
}
|
||||
|
||||
var boxes []topoBox
|
||||
var pcieEdges []struct {
|
||||
x1, y1, x2, y2 int
|
||||
color string
|
||||
}
|
||||
gpuBoxCenter := map[int][2]int{} // gpu index -> (x, yBottom)
|
||||
gpuBoxIndex := map[int]int{} // gpu index -> index into boxes
|
||||
gpuNUMANode := map[int]*int{} // gpu index -> its PCIe device's numa_node
|
||||
|
||||
for col := 0; col < totalCols; col++ {
|
||||
colX := (col+1)*24 + col*topoColWidth
|
||||
if col < numCols {
|
||||
cpu := hw.CPUs[col]
|
||||
model := ""
|
||||
if cpu.Model != nil {
|
||||
model = *cpu.Model
|
||||
}
|
||||
socket := col
|
||||
if cpu.Socket != nil {
|
||||
socket = *cpu.Socket
|
||||
}
|
||||
label, cls := topoStatusBadgeClass(cpu.Status)
|
||||
boxes = append(boxes, topoBox{
|
||||
x: colX, y: topoTopMargin, w: topoBoxWidth, h: topoBoxHeight,
|
||||
label: fmt.Sprintf("CPU %d", socket), sublabel: model,
|
||||
badgeText: label, badgeCls: cls, detailType: "cpu",
|
||||
})
|
||||
}
|
||||
|
||||
y := topoTopMargin + topoBoxHeight + topoDeviceGap*2
|
||||
for _, p := range placed {
|
||||
if p.col != col {
|
||||
continue
|
||||
}
|
||||
label, cls := topoStatusBadgeClass(p.dev.Status)
|
||||
model := ""
|
||||
if p.dev.Model != nil {
|
||||
model = *p.dev.Model
|
||||
}
|
||||
box := topoBox{
|
||||
x: colX, y: y, w: topoBoxWidth, h: topoBoxHeight,
|
||||
label: strings.ToUpper(p.kind), sublabel: model,
|
||||
badgeText: label, badgeCls: cls, detailType: p.kind,
|
||||
}
|
||||
boxes = append(boxes, box)
|
||||
boxIdx := len(boxes) - 1
|
||||
|
||||
if col < numCols {
|
||||
pcieEdges = append(pcieEdges, struct {
|
||||
x1, y1, x2, y2 int
|
||||
color string
|
||||
}{
|
||||
x1: colX + topoBoxWidth/2, y1: topoTopMargin + topoBoxHeight,
|
||||
x2: colX + topoBoxWidth/2, y2: y,
|
||||
color: topoEdgeColorVar(p.dev),
|
||||
})
|
||||
}
|
||||
|
||||
if p.kind == "gpu" && p.bdf != "" {
|
||||
if idx, ok := bdfToIndex[p.bdf]; ok {
|
||||
gpuBoxCenter[idx] = [2]int{colX + topoBoxWidth/2, y + topoBoxHeight}
|
||||
gpuBoxIndex[idx] = boxIdx
|
||||
gpuNUMANode[idx] = p.dev.NUMANode
|
||||
}
|
||||
}
|
||||
|
||||
y += topoBoxHeight + topoDeviceGap
|
||||
}
|
||||
}
|
||||
|
||||
maxDeviceY := topoTopMargin + topoBoxHeight + topoDeviceGap*2
|
||||
for _, b := range boxes {
|
||||
if b.y+b.h > maxDeviceY {
|
||||
maxDeviceY = b.y + b.h
|
||||
}
|
||||
}
|
||||
|
||||
// GPU-GPU NVLink edges: drawn as an elbow connector through a dedicated
|
||||
// band below the device row, kept strictly separate from the vertical
|
||||
// CPU->device PCIe edges above so neither visually obscures the other.
|
||||
//
|
||||
// A bonded pair spanning two different NUMA nodes is treated as an
|
||||
// anomaly (not a neutral fact) per project decision: we expect a bonded
|
||||
// pair to sit on one NUMA node, so a cross-NUMA bond is flagged Warning
|
||||
// on the edge AND on both GPU boxes, regardless of their own SAT status.
|
||||
bandY := maxDeviceY + topoEdgeBand/2
|
||||
var gpuEdgesSVG strings.Builder
|
||||
for _, pair := range pairs {
|
||||
c1, ok1 := gpuBoxCenter[pair.GPUA]
|
||||
c2, ok2 := gpuBoxCenter[pair.GPUB]
|
||||
if !ok1 || !ok2 {
|
||||
continue
|
||||
}
|
||||
color := "var(--ok-fg)"
|
||||
title := fmt.Sprintf("NVLink: GPU%d↔GPU%d (%d links)", pair.GPUA, pair.GPUB, pair.NVLinks)
|
||||
numaA, numaB := gpuNUMANode[pair.GPUA], gpuNUMANode[pair.GPUB]
|
||||
if numaA == nil || numaB == nil {
|
||||
color = "var(--muted)"
|
||||
} else if *numaA != *numaB {
|
||||
color = "var(--warn-fg)"
|
||||
title += " — spans NUMA nodes (unexpected)"
|
||||
upgradeTopoBoxBadgeToWarn(boxes, gpuBoxIndex[pair.GPUA])
|
||||
upgradeTopoBoxBadgeToWarn(boxes, gpuBoxIndex[pair.GPUB])
|
||||
}
|
||||
fmt.Fprintf(&gpuEdgesSVG,
|
||||
`<path d="M %d %d L %d %d L %d %d L %d %d" style="fill:none;stroke:%s;stroke-width:2;stroke-dasharray:4,3"><title>%s</title></path>`+"\n",
|
||||
c1[0], c1[1], c1[0], bandY, c2[0], bandY, c2[0], c2[1], color, html.EscapeString(title))
|
||||
}
|
||||
|
||||
svgHeight := bandY + topoEdgeBand/2 + topoBottomRowH
|
||||
svgWidth := totalCols*topoColWidth + 48
|
||||
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, `<svg width="%d" height="%d" viewBox="0 0 %d %d" style="max-width:100%%">`+"\n", svgWidth, svgHeight, svgWidth, svgHeight)
|
||||
for _, e := range pcieEdges {
|
||||
fmt.Fprintf(&b, `<line x1="%d" y1="%d" x2="%d" y2="%d" style="stroke:%s;stroke-width:2"/>`+"\n", e.x1, e.y1, e.x2, e.y2, e.color)
|
||||
}
|
||||
b.WriteString(gpuEdgesSVG.String())
|
||||
for _, box := range boxes {
|
||||
writeTopoBoxSVG(&b, box)
|
||||
}
|
||||
|
||||
// PSU / BMC row: standalone boxes, no connecting lines.
|
||||
psuY := svgHeight - topoBottomRowH + 20
|
||||
psuX := 24
|
||||
for _, psu := range hw.PowerSupplies {
|
||||
label, cls := topoStatusBadgeClass(psu.Status)
|
||||
slot := ""
|
||||
if psu.Slot != nil {
|
||||
slot = *psu.Slot
|
||||
}
|
||||
watt := ""
|
||||
if psu.WattageW != nil {
|
||||
watt = fmt.Sprintf("%dW", *psu.WattageW)
|
||||
}
|
||||
writeTopoBoxSVG(&b, topoBox{
|
||||
x: psuX, y: psuY, w: 150, h: topoBoxHeight,
|
||||
label: "PSU " + slot, sublabel: watt,
|
||||
badgeText: label, badgeCls: cls, detailType: "psu",
|
||||
})
|
||||
psuX += 150 + topoDeviceGap
|
||||
}
|
||||
if bmcVersion, ok := findBMCFirmware(hw.Firmware); ok {
|
||||
writeTopoBoxSVG(&b, topoBox{
|
||||
x: svgWidth - 200 - 24, y: psuY, w: 200, h: topoBoxHeight,
|
||||
label: "BMC", sublabel: "fw " + bmcVersion,
|
||||
badgeText: "?", badgeCls: "badge-unknown",
|
||||
})
|
||||
}
|
||||
|
||||
b.WriteString(`</svg>`)
|
||||
return topoCard("Topology", b.String())
|
||||
}
|
||||
|
||||
// upgradeTopoBoxBadgeToWarn upgrades a box's badge to Warning unless it is
|
||||
// already at Critical severity (never downgrades a worse status).
|
||||
func upgradeTopoBoxBadgeToWarn(boxes []topoBox, boxIdx int) {
|
||||
if boxIdx < 0 || boxIdx >= len(boxes) {
|
||||
return
|
||||
}
|
||||
if boxes[boxIdx].badgeCls == "badge-err" {
|
||||
return
|
||||
}
|
||||
boxes[boxIdx].badgeText = "WARN"
|
||||
boxes[boxIdx].badgeCls = "badge-warn"
|
||||
}
|
||||
|
||||
func findBMCFirmware(records []schema.HardwareFirmwareRecord) (string, bool) {
|
||||
for _, rec := range records {
|
||||
if strings.EqualFold(strings.TrimSpace(rec.DeviceName), "BMC") {
|
||||
return rec.Version, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func writeTopoBoxSVG(b *strings.Builder, box topoBox) {
|
||||
onclick := ""
|
||||
cursor := "default"
|
||||
if box.detailType != "" {
|
||||
onclick = fmt.Sprintf(` onclick="openComponentDetail('%s')"`, box.detailType)
|
||||
cursor = "pointer"
|
||||
}
|
||||
fmt.Fprintf(b, `<g%s style="cursor:%s">`, onclick, cursor)
|
||||
fmt.Fprintf(b, `<rect x="%d" y="%d" width="%d" height="%d" rx="6" ry="6" style="fill:var(--surface);stroke:var(--border)"/>`+"\n",
|
||||
box.x, box.y, box.w, box.h)
|
||||
fmt.Fprintf(b, `<text x="%d" y="%d" style="fill:var(--ink,#000);font-size:13px;font-weight:700">%s</text>`+"\n",
|
||||
box.x+10, box.y+20, html.EscapeString(box.label))
|
||||
if box.sublabel != "" {
|
||||
fmt.Fprintf(b, `<text x="%d" y="%d" style="fill:var(--muted);font-size:11px">%s</text>`+"\n",
|
||||
box.x+10, box.y+36, html.EscapeString(truncateTopoLabel(box.sublabel, 26)))
|
||||
}
|
||||
fmt.Fprintf(b, `<rect x="%d" y="%d" width="46" height="18" rx="3" ry="3" class="%s"/>`+"\n",
|
||||
box.x+box.w-54, box.y+box.h-26, box.badgeCls)
|
||||
fmt.Fprintf(b, `<text x="%d" y="%d" style="font-size:10px;font-weight:700" class="%s">%s</text>`+"\n",
|
||||
box.x+box.w-49, box.y+box.h-13, box.badgeCls, html.EscapeString(box.badgeText))
|
||||
b.WriteString(`</g>` + "\n")
|
||||
}
|
||||
|
||||
func truncateTopoLabel(s string, max int) string {
|
||||
if len(s) <= max {
|
||||
return s
|
||||
}
|
||||
if max <= 1 {
|
||||
return s[:max]
|
||||
}
|
||||
return s[:max-1] + "…"
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Separate NVLink topology card (live query, not persisted to any contract)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type topoNVLinkPort struct {
|
||||
Index int
|
||||
Active bool
|
||||
SpeedGBs *float64
|
||||
ReplayErrors int64
|
||||
RecoveryErrors int64
|
||||
CRCErrors int64
|
||||
}
|
||||
|
||||
var (
|
||||
topoNVLinkGPUHeaderRe = regexp.MustCompile(`^GPU (\d+):`)
|
||||
topoNVLinkSpeedLineRe = regexp.MustCompile(`^Link (\d+):\s*([\d.]+)\s*GB/s`)
|
||||
topoNVLinkInactiveRe = regexp.MustCompile(`^Link (\d+):\s*<inactive>`)
|
||||
topoNVLinkErrorLineRe = regexp.MustCompile(`^Link (\d+):\s*(Replay|Recovery|CRC) Errors:\s*(\d+)`)
|
||||
)
|
||||
|
||||
func queryTopoNVLinkStatus() (map[int][]topoNVLinkPort, error) {
|
||||
out, err := exec.Command("nvidia-smi", "nvlink", "-s").Output()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return parseTopoNVLinkStatus(string(out)), nil
|
||||
}
|
||||
|
||||
func parseTopoNVLinkStatus(raw string) map[int][]topoNVLinkPort {
|
||||
result := map[int][]topoNVLinkPort{}
|
||||
currentGPU := -1
|
||||
for _, line := range strings.Split(raw, "\n") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if m := topoNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
|
||||
currentGPU, _ = strconv.Atoi(m[1])
|
||||
continue
|
||||
}
|
||||
if currentGPU < 0 {
|
||||
continue
|
||||
}
|
||||
if m := topoNVLinkInactiveRe.FindStringSubmatch(trimmed); m != nil {
|
||||
idx, _ := strconv.Atoi(m[1])
|
||||
result[currentGPU] = append(result[currentGPU], topoNVLinkPort{Index: idx, Active: false})
|
||||
continue
|
||||
}
|
||||
if m := topoNVLinkSpeedLineRe.FindStringSubmatch(trimmed); m != nil {
|
||||
idx, _ := strconv.Atoi(m[1])
|
||||
port := topoNVLinkPort{Index: idx, Active: true}
|
||||
if speed, err := strconv.ParseFloat(m[2], 64); err == nil {
|
||||
port.SpeedGBs = &speed
|
||||
}
|
||||
result[currentGPU] = append(result[currentGPU], port)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func queryTopoNVLinkErrors() (map[int]map[int][3]int64, error) {
|
||||
out, err := exec.Command("nvidia-smi", "nvlink", "-e").Output()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return parseTopoNVLinkErrors(string(out)), nil
|
||||
}
|
||||
|
||||
// parseTopoNVLinkErrors returns, per GPU then link index, [replay, recovery, crc].
|
||||
func parseTopoNVLinkErrors(raw string) map[int]map[int][3]int64 {
|
||||
result := map[int]map[int][3]int64{}
|
||||
currentGPU := -1
|
||||
for _, line := range strings.Split(raw, "\n") {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if m := topoNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
|
||||
currentGPU, _ = strconv.Atoi(m[1])
|
||||
continue
|
||||
}
|
||||
if currentGPU < 0 {
|
||||
continue
|
||||
}
|
||||
m := topoNVLinkErrorLineRe.FindStringSubmatch(trimmed)
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
linkIdx, _ := strconv.Atoi(m[1])
|
||||
count, _ := strconv.ParseInt(m[3], 10, 64)
|
||||
if result[currentGPU] == nil {
|
||||
result[currentGPU] = map[int][3]int64{}
|
||||
}
|
||||
c := result[currentGPU][linkIdx]
|
||||
switch m[2] {
|
||||
case "Replay":
|
||||
c[0] = count
|
||||
case "Recovery":
|
||||
c[1] = count
|
||||
case "CRC":
|
||||
c[2] = count
|
||||
}
|
||||
result[currentGPU][linkIdx] = c
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// renderTopoNVLinkCard renders the separate NVLink topology card. Returns ""
|
||||
// if there are fewer than 2 NVIDIA GPUs or nvidia-smi is unavailable.
|
||||
func renderTopoNVLinkCard(hw schema.HardwareSnapshot) string {
|
||||
gpuCount := 0
|
||||
for _, dev := range hw.PCIeDevices {
|
||||
if dev.DeviceClass != nil && isGPUDeviceClass(*dev.DeviceClass) {
|
||||
gpuCount++
|
||||
}
|
||||
}
|
||||
if gpuCount < 2 {
|
||||
return ""
|
||||
}
|
||||
|
||||
status, err := queryTopoNVLinkStatus()
|
||||
if err != nil || len(status) == 0 {
|
||||
return topoCard("NVLink Topology", `<span class="badge badge-unknown">nvidia-smi nvlink unavailable</span>`)
|
||||
}
|
||||
errors, _ := queryTopoNVLinkErrors()
|
||||
|
||||
topoMatrix, _ := queryGPUTopologyMatrix()
|
||||
pairs := parseGPUPairAdjacency(topoMatrix)
|
||||
|
||||
var bodyB strings.Builder
|
||||
if gpuCount <= 4 && len(pairs) > 0 {
|
||||
// Small GPU count: per-pair box+line with per-link detail.
|
||||
for _, pair := range pairs {
|
||||
activeCount, total, hasError := 0, 0, false
|
||||
for _, port := range status[pair.GPUA] {
|
||||
total++
|
||||
if port.Active {
|
||||
activeCount++
|
||||
}
|
||||
}
|
||||
for _, counters := range errors[pair.GPUA] {
|
||||
if counters[0] != 0 || counters[1] != 0 || counters[2] != 0 {
|
||||
hasError = true
|
||||
}
|
||||
}
|
||||
color := "var(--ok-fg)"
|
||||
switch {
|
||||
case hasError:
|
||||
color = "var(--crit-fg)"
|
||||
case total > 0 && activeCount < total:
|
||||
color = "var(--warn-fg)"
|
||||
}
|
||||
fmt.Fprintf(&bodyB, `<div style="display:flex;align-items:center;gap:12px;margin-bottom:10px">`+
|
||||
`<div style="padding:8px 12px;border:1px solid var(--border);border-radius:6px">GPU %d</div>`+
|
||||
`<div style="flex:1;height:2px;background:%s"></div>`+
|
||||
`<div style="padding:8px 12px;border:1px solid var(--border);border-radius:6px">GPU %d</div>`+
|
||||
`<div style="font-size:12px;color:var(--muted)">%d/%d links active%s</div>`+
|
||||
`</div>`,
|
||||
pair.GPUA, color, pair.GPUB, activeCount, total, errNoteSuffix(hasError))
|
||||
}
|
||||
} else if len(pairs) > 0 {
|
||||
// Larger GPU counts (NVSwitch fabric): aggregate pair table instead of
|
||||
// an unreadable all-to-all graph.
|
||||
bodyB.WriteString(`<table><thead><tr><th>GPU A</th><th>GPU B</th><th>NVLinks</th></tr></thead><tbody>`)
|
||||
for _, pair := range pairs {
|
||||
fmt.Fprintf(&bodyB, `<tr><td>GPU %d</td><td>GPU %d</td><td>%d</td></tr>`, pair.GPUA, pair.GPUB, pair.NVLinks)
|
||||
}
|
||||
bodyB.WriteString(`</tbody></table>`)
|
||||
} else {
|
||||
bodyB.WriteString(`<span class="badge badge-unknown">No NVLink-bonded GPU pairs found</span>`)
|
||||
}
|
||||
|
||||
return topoCard("NVLink Topology", bodyB.String())
|
||||
}
|
||||
|
||||
func errNoteSuffix(hasError bool) string {
|
||||
if hasError {
|
||||
return " — errors detected"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,245 @@
|
||||
package webui
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"bee/audit/internal/schema"
|
||||
)
|
||||
|
||||
func TestTopoPageNoAuditDataGracefulFallback(t *testing.T) {
|
||||
handler := NewHandler(HandlerOptions{})
|
||||
rec := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/topo", nil))
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status=%d", rec.Code)
|
||||
}
|
||||
body := rec.Body.String()
|
||||
if !strings.Contains(body, "No audit data") {
|
||||
t.Fatalf("topo page missing no-audit-data fallback: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTopoPageRendersCPUAndDegradedPCIeLink(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "audit.json")
|
||||
|
||||
socket := 0
|
||||
numaNode := 0
|
||||
gen3, gen4 := "Gen3", "Gen4"
|
||||
deviceClass := "VideoController"
|
||||
model := "NVIDIA H100 80GB HBM3"
|
||||
cpuModel := "Intel Xeon 6530"
|
||||
okStatus := "OK"
|
||||
|
||||
ingest := schema.HardwareIngestRequest{
|
||||
CollectedAt: "2026-03-15T00:00:00Z",
|
||||
Hardware: schema.HardwareSnapshot{
|
||||
CPUs: []schema.HardwareCPU{
|
||||
{
|
||||
HardwareComponentStatus: schema.HardwareComponentStatus{Status: &okStatus},
|
||||
Socket: &socket,
|
||||
Model: &cpuModel,
|
||||
},
|
||||
},
|
||||
PCIeDevices: []schema.HardwarePCIeDevice{
|
||||
{
|
||||
DeviceClass: &deviceClass,
|
||||
Model: &model,
|
||||
NUMANode: &numaNode,
|
||||
LinkSpeed: &gen3,
|
||||
MaxLinkSpeed: &gen4,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
data, err := json.Marshal(ingest)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, data, 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
handler := NewHandler(HandlerOptions{AuditPath: path})
|
||||
rec := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/topo", nil))
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status=%d", rec.Code)
|
||||
}
|
||||
body := rec.Body.String()
|
||||
if !strings.Contains(body, "CPU 0") {
|
||||
t.Fatalf("topo page missing CPU 0 box: %s", body)
|
||||
}
|
||||
if !strings.Contains(body, "GPU") {
|
||||
t.Fatalf("topo page missing GPU box: %s", body)
|
||||
}
|
||||
if !strings.Contains(body, "var(--warn-fg)") {
|
||||
t.Fatalf("topo page missing degraded-link warn edge color: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTopoPageLinkedFromNav(t *testing.T) {
|
||||
handler := NewHandler(HandlerOptions{})
|
||||
rec := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/", nil))
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status=%d", rec.Code)
|
||||
}
|
||||
if !strings.Contains(rec.Body.String(), `href="/topo"`) {
|
||||
t.Fatalf("nav missing /topo link")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseGPUPairAdjacencyRealTwoGPUDump(t *testing.T) {
|
||||
// Real system/nvidia-smi-topo.txt from a support bundle for this exact
|
||||
// server: two H100s directly bridged (NV17), spanning two NUMA nodes.
|
||||
input := "\tGPU0\tGPU1\tNIC0\tNIC1\tCPU Affinity\tNUMA Affinity\tGPU NUMA ID\n" +
|
||||
"GPU0\t X \tNV17\tSYS\tSYS\t0-23,48-71\t0\t\tN/A\n" +
|
||||
"GPU1\tNV17\t X \tNODE\tNODE\t24-47,72-95\t1\t\tN/A\n" +
|
||||
"NIC0\tSYS\tNODE\t X \tPIX\t\t\t\n" +
|
||||
"NIC1\tSYS\tNODE\tPIX\t X \t\t\t\n"
|
||||
|
||||
pairs := parseGPUPairAdjacency(input)
|
||||
if len(pairs) != 1 {
|
||||
t.Fatalf("pairs=%d want 1 (%#v)", len(pairs), pairs)
|
||||
}
|
||||
if pairs[0].GPUA != 0 || pairs[0].GPUB != 1 || pairs[0].NVLinks != 17 {
|
||||
t.Fatalf("pair=%#v want {0,1,17}", pairs[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseGPUPairAdjacencyDoesNotChainUnrelatedGPUs(t *testing.T) {
|
||||
// 4 GPUs: only (0,1) and (2,3) are actually bonded. GPU1 and GPU2 must
|
||||
// NOT get an edge just because they're adjacent in the layout.
|
||||
input := "\tGPU0\tGPU1\tGPU2\tGPU3\n" +
|
||||
"GPU0\t X \tNV18\tSYS\tSYS\n" +
|
||||
"GPU1\tNV18\t X \tSYS\tSYS\n" +
|
||||
"GPU2\tSYS\tSYS\t X \tNV18\n" +
|
||||
"GPU3\tSYS\tSYS\tNV18\t X \n"
|
||||
|
||||
pairs := parseGPUPairAdjacency(input)
|
||||
if len(pairs) != 2 {
|
||||
t.Fatalf("pairs=%d want 2 (%#v)", len(pairs), pairs)
|
||||
}
|
||||
want := map[[2]int]bool{{0, 1}: true, {2, 3}: true}
|
||||
for _, p := range pairs {
|
||||
if !want[[2]int{p.GPUA, p.GPUB}] {
|
||||
t.Fatalf("unexpected pair %#v (GPU1-GPU2 chaining bug?)", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseGPUPairAdjacencyEmptyOnNoMatrix(t *testing.T) {
|
||||
if pairs := parseGPUPairAdjacency("no gpus here"); pairs != nil {
|
||||
t.Fatalf("pairs=%v want nil", pairs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPcieGenRank(t *testing.T) {
|
||||
if pcieGenRank("Gen5") <= pcieGenRank("Gen4") {
|
||||
t.Fatalf("Gen5 should rank higher than Gen4")
|
||||
}
|
||||
if pcieGenRank("bogus") != 0 {
|
||||
t.Fatalf("unparseable gen should rank 0")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTopoEdgeColorVar(t *testing.T) {
|
||||
gen3, gen4 := "Gen3", "Gen4"
|
||||
degraded := schema.HardwarePCIeDevice{LinkSpeed: &gen3, MaxLinkSpeed: &gen4}
|
||||
if got := topoEdgeColorVar(degraded); got != "var(--warn-fg)" {
|
||||
t.Fatalf("degraded color=%q want warn", got)
|
||||
}
|
||||
full := schema.HardwarePCIeDevice{LinkSpeed: &gen4, MaxLinkSpeed: &gen4}
|
||||
if got := topoEdgeColorVar(full); got != "var(--ok-fg)" {
|
||||
t.Fatalf("full-speed color=%q want ok", got)
|
||||
}
|
||||
unknown := schema.HardwarePCIeDevice{}
|
||||
if got := topoEdgeColorVar(unknown); got != "var(--muted)" {
|
||||
t.Fatalf("unknown color=%q want muted", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSocketIndex(t *testing.T) {
|
||||
s0, s1 := 0, 1
|
||||
cpus := []schema.HardwareCPU{{Socket: &s1}, {Socket: &s0}}
|
||||
idx := buildSocketIndex(cpus)
|
||||
if idx[0] != 1 || idx[1] != 0 {
|
||||
t.Fatalf("idx=%#v want {0:1, 1:0}", idx)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsRAIDControllerClass(t *testing.T) {
|
||||
if !isRAIDControllerClass("StorageController") || !isRAIDControllerClass("MassStorageController") {
|
||||
t.Fatalf("expected known RAID/storage classes to match")
|
||||
}
|
||||
if isRAIDControllerClass("VideoController") {
|
||||
t.Fatalf("GPU class should not match RAID classifier")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsNICDeviceClassDev(t *testing.T) {
|
||||
class := "EthernetController"
|
||||
nic := schema.HardwarePCIeDevice{DeviceClass: &class}
|
||||
if !isNICDeviceClassDev(nic) {
|
||||
t.Fatalf("expected EthernetController to classify as NIC")
|
||||
}
|
||||
withMac := schema.HardwarePCIeDevice{MacAddresses: []string{"aa:bb:cc:dd:ee:ff"}}
|
||||
if !isNICDeviceClassDev(withMac) {
|
||||
t.Fatalf("expected device with MAC address to classify as NIC")
|
||||
}
|
||||
other := schema.HardwarePCIeDevice{}
|
||||
if isNICDeviceClassDev(other) {
|
||||
t.Fatalf("expected empty device to not classify as NIC")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTopoStatusBadgeClassNilIsUnknown(t *testing.T) {
|
||||
label, cls := topoStatusBadgeClass(nil)
|
||||
if label != "?" || cls != "badge-unknown" {
|
||||
t.Fatalf("nil status = (%q,%q) want (?, badge-unknown)", label, cls)
|
||||
}
|
||||
ok := "OK"
|
||||
label, cls = topoStatusBadgeClass(&ok)
|
||||
if label != "OK" || cls != "badge-ok" {
|
||||
t.Fatalf("OK status = (%q,%q) want (OK, badge-ok)", label, cls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTopoNVLinkStatus(t *testing.T) {
|
||||
input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140)
|
||||
Link 0: 26.562 GB/s
|
||||
Link 15: <inactive>
|
||||
GPU 1: NVIDIA H100 80GB HBM3 (UUID: GPU-603fe750-0516-9db5-86ec-ea61af3fce35)
|
||||
Link 0: 26.562 GB/s
|
||||
`
|
||||
got := parseTopoNVLinkStatus(input)
|
||||
if len(got[0]) != 2 || got[0][1].Active {
|
||||
t.Fatalf("gpu0=%#v want link15 inactive", got[0])
|
||||
}
|
||||
if len(got[1]) != 1 || got[1][0].SpeedGBs == nil || *got[1][0].SpeedGBs != 26.562 {
|
||||
t.Fatalf("gpu1=%#v want link0 26.562 GB/s", got[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTopoNVLinkErrors(t *testing.T) {
|
||||
input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140)
|
||||
Link 0: Replay Errors: 0
|
||||
Link 0: Recovery Errors: 0
|
||||
Link 0: CRC Errors: 0
|
||||
Link 1: Replay Errors: 3
|
||||
Link 1: Recovery Errors: 1
|
||||
Link 1: CRC Errors: 2
|
||||
`
|
||||
got := parseTopoNVLinkErrors(input)
|
||||
c := got[0][1]
|
||||
if c[0] != 3 || c[1] != 1 || c[2] != 2 {
|
||||
t.Fatalf("link1 counters=%#v want {3,1,2}", c)
|
||||
}
|
||||
}
|
||||
@@ -46,6 +46,10 @@ func renderPage(page string, opts HandlerOptions) string {
|
||||
pageID = "tools"
|
||||
title = "Tools"
|
||||
body = renderTools()
|
||||
case "topo":
|
||||
pageID = "topo"
|
||||
title = "Topology"
|
||||
body = renderTopo(opts)
|
||||
case "settings":
|
||||
pageID = "settings"
|
||||
title = "Settings"
|
||||
|
||||
Reference in New Issue
Block a user