diff --git a/audit/internal/app/app_test.go b/audit/internal/app/app_test.go index 6ec3869..363beaa 100644 --- a/audit/internal/app/app_test.go +++ b/audit/internal/app/app_test.go @@ -884,13 +884,13 @@ func TestBuildSupportBundleIncludesExportDirContents(t *testing.T) { for _, want := range []string{ "/system/ip-link.txt", "/system/ip-link-stats.txt", - "/system/kernel-aer-nvidia.txt", - "/system/lspci-nvidia-bridges-vv.txt", - "/system/pcie-aer-sysfs.txt", - "/system/ethtool-info.txt", - "/system/ethtool-link.txt", - "/system/ethtool-module.txt", - "/system/mstflint-query.txt", + "/techdump/kernel-aer-nvidia.txt", + "/techdump/lspci-nvidia-bridges-vv.txt", + "/techdump/pcie-aer-sysfs.txt", + "/techdump/ethtool-info.txt", + "/techdump/ethtool-link.txt", + "/techdump/ethtool-module.txt", + "/techdump/mstflint-query.txt", } { var found bool for _, name := range names { diff --git a/audit/internal/app/support_bundle.go b/audit/internal/app/support_bundle.go index cf20ac2..0b7549e 100644 --- a/audit/internal/app/support_bundle.go +++ b/audit/internal/app/support_bundle.go @@ -37,8 +37,7 @@ var supportBundleCommands = []struct { {name: "system/uname.txt", cmd: []string{"uname", "-a"}}, {name: "system/cmdline.txt", cmd: []string{"cat", "/proc/cmdline"}}, {name: "system/lsmod.txt", cmd: []string{"lsmod"}}, - {name: "system/lspci-nn.txt", cmd: []string{"lspci", "-nn"}}, - {name: "system/lspci-vvv.txt", cmd: []string{"lspci", "-vvv"}}, + {name: "techdump/lspci-nn.txt", cmd: []string{"lspci", "-nn"}}, {name: "system/ip-addr.txt", cmd: []string{"ip", "addr"}}, {name: "system/ip-link.txt", cmd: []string{"ip", "-details", "link", "show"}}, {name: "system/ip-link-stats.txt", cmd: []string{"ip", "-s", "link", "show"}}, @@ -53,7 +52,7 @@ else echo "dmesg not found" fi `}}, - {name: "system/kernel-aer-nvidia.txt", cmd: []string{"sh", "-c", ` + {name: "techdump/kernel-aer-nvidia.txt", cmd: []string{"sh", "-c", ` if command -v dmesg >/dev/null 2>&1; then dmesg | grep -iE 'AER|NVRM|Xid|pcieport|nvidia' || echo "no AER/NVRM/Xid kernel messages found" else @@ -83,7 +82,7 @@ fi {name: "system/ps-gui.txt", cmd: []string{"sh", "-c", ` ps -ef | grep -iE 'lightdm|Xorg|X$|openbox|chromium|chrome|xinit|xsession' | grep -v grep || echo "no GUI processes found" `}}, - {name: "system/lspci-video-vv.txt", cmd: []string{"sh", "-c", ` + {name: "techdump/lspci-video-vv.txt", cmd: []string{"sh", "-c", ` if ! command -v lspci >/dev/null 2>&1; then echo "lspci not found" exit 0 @@ -169,36 +168,44 @@ echo echo "=== failed units ===" systemctl --failed --no-pager 2>&1 | grep -iE 'lightdm|display-manager|Xorg' || echo "no failed GUI units" `}}, - {name: "system/nvidia-smi-q.txt", cmd: []string{"nvidia-smi", "-q"}}, - {name: "system/nvidia-smi-topo.txt", cmd: []string{"sh", "-c", ` + {name: "techdump/nvidia-smi-topo.txt", cmd: []string{"sh", "-c", ` if command -v nvidia-smi >/dev/null 2>&1; then nvidia-smi topo -m 2>&1 || true else echo "nvidia-smi not found" fi `}}, - {name: "system/nvidia-smi-nvlink-status.txt", cmd: []string{"sh", "-c", ` + {name: "techdump/nvidia-smi-nvlink-status.txt", cmd: []string{"sh", "-c", ` if command -v nvidia-smi >/dev/null 2>&1; then nvidia-smi nvlink -s 2>&1 || true else echo "nvidia-smi not found" fi `}}, - {name: "system/nvidia-smi-nvlink-errors.txt", cmd: []string{"sh", "-c", ` + {name: "techdump/nvidia-smi-nvlink-errors.txt", cmd: []string{"sh", "-c", ` if command -v nvidia-smi >/dev/null 2>&1; then nvidia-smi nvlink -e 2>&1 || true else echo "nvidia-smi not found" fi `}}, - {name: "system/dcgmi-nvlink-status.txt", cmd: []string{"sh", "-c", ` + {name: "techdump/dcgmi-nvlink-status.txt", cmd: []string{"sh", "-c", ` if command -v dcgmi >/dev/null 2>&1; then dcgmi nvlink --link-status 2>&1 || true else echo "dcgmi not found" fi `}}, - {name: "system/systemctl-nvidia-units.txt", cmd: []string{"sh", "-c", ` + {name: "techdump/nvidia-bug-report.txt", cmd: []string{"sh", "-c", ` +if command -v nvidia-bug-report.sh >/dev/null 2>&1; then + nvidia-bug-report.sh --output-file /tmp/bee-nvidia-bug-report.log >/dev/null 2>&1 \ + && cat /tmp/bee-nvidia-bug-report.log \ + && rm -f /tmp/bee-nvidia-bug-report.log +else + echo "nvidia-bug-report.sh not found" +fi +`}}, + {name: "techdump/systemctl-nvidia-units.txt", cmd: []string{"sh", "-c", ` if ! command -v systemctl >/dev/null 2>&1; then echo "systemctl not found" exit 0 @@ -212,7 +219,7 @@ echo echo "=== failed units ===" systemctl --failed --no-pager 2>&1 | grep -iE 'nvidia|fabric' || echo "no failed nvidia/fabric units" `}}, - {name: "system/fabric-manager-paths.txt", cmd: []string{"sh", "-c", ` + {name: "techdump/fabric-manager-paths.txt", cmd: []string{"sh", "-c", ` for candidate in \ /usr/bin/nvidia-fabricmanager \ /usr/bin/nv-fabricmanager \ @@ -228,7 +235,7 @@ if ! ls /usr/bin/nvidia-fabricmanager /usr/bin/nv-fabricmanager /usr/bin/nvidia- echo "no fabric manager binaries found" fi `}}, - {name: "system/lspci-nvidia-bridges-vv.txt", cmd: []string{"sh", "-c", ` + {name: "techdump/lspci-nvidia-bridges-vv.txt", cmd: []string{"sh", "-c", ` if ! command -v lspci >/dev/null 2>&1; then echo "lspci not found" exit 0 @@ -250,7 +257,7 @@ if [ "$found" -eq 0 ]; then echo "no NVIDIA PCI devices found" fi `}}, - {name: "system/pcie-nvidia-link.txt", cmd: []string{"sh", "-c", ` + {name: "techdump/pcie-nvidia-link.txt", cmd: []string{"sh", "-c", ` for d in /sys/bus/pci/devices/*/; do vendor=$(cat "$d/vendor" 2>/dev/null) [ "$vendor" = "0x10de" ] || continue @@ -266,7 +273,7 @@ for d in /sys/bus/pci/devices/*/; do done done `}}, - {name: "system/pcie-aer-sysfs.txt", cmd: []string{"sh", "-c", ` + {name: "techdump/pcie-aer-sysfs.txt", cmd: []string{"sh", "-c", ` found=0 for dev in /sys/bus/pci/devices/*; do [ -e "$dev" ] || continue @@ -290,7 +297,7 @@ if [ "$found" -eq 0 ]; then echo "no PCIe AER sysfs counters found" fi `}}, - {name: "system/ethtool-info.txt", cmd: []string{"sh", "-c", ` + {name: "techdump/ethtool-info.txt", cmd: []string{"sh", "-c", ` if ! command -v ethtool >/dev/null 2>&1; then echo "ethtool not found" exit 0 @@ -309,7 +316,7 @@ if [ "$found" -eq 0 ]; then echo "no interfaces found" fi `}}, - {name: "system/ethtool-link.txt", cmd: []string{"sh", "-c", ` + {name: "techdump/ethtool-link.txt", cmd: []string{"sh", "-c", ` if ! command -v ethtool >/dev/null 2>&1; then echo "ethtool not found" exit 0 @@ -328,7 +335,7 @@ if [ "$found" -eq 0 ]; then echo "no interfaces found" fi `}}, - {name: "system/ethtool-module.txt", cmd: []string{"sh", "-c", ` + {name: "techdump/ethtool-module.txt", cmd: []string{"sh", "-c", ` if ! command -v ethtool >/dev/null 2>&1; then echo "ethtool not found" exit 0 @@ -347,7 +354,7 @@ if [ "$found" -eq 0 ]; then echo "no interfaces found" fi `}}, - {name: "system/mstflint-query.txt", cmd: []string{"sh", "-c", ` + {name: "techdump/mstflint-query.txt", cmd: []string{"sh", "-c", ` if ! command -v mstflint >/dev/null 2>&1; then echo "mstflint not found" exit 0 @@ -382,10 +389,10 @@ var supportBundleOptionalFiles = []struct { {name: "system/lightdm/x-0-greeter.log", src: "/var/log/lightdm/x-0-greeter.log"}, {name: "system/home-bee-xsession-errors.log", src: "/home/bee/.xsession-errors"}, {name: "system/home-bee-chromium-debug.log", src: "/tmp/bee-chrome/chrome_debug.log"}, - {name: "system/fabricmanager.log", src: "/var/log/fabricmanager.log"}, - {name: "system/nvlsm.log", src: "/var/log/nvlsm.log"}, - {name: "system/fabricmanager/fabricmanager.log", src: "/var/log/fabricmanager/fabricmanager.log"}, - {name: "system/fabricmanager/nvlsm.log", src: "/var/log/fabricmanager/nvlsm.log"}, + {name: "techdump/fabricmanager.log", src: "/var/log/fabricmanager.log"}, + {name: "techdump/nvlsm.log", src: "/var/log/nvlsm.log"}, + {name: "techdump/fabricmanager/fabricmanager.log", src: "/var/log/fabricmanager/fabricmanager.log"}, + {name: "techdump/fabricmanager/nvlsm.log", src: "/var/log/fabricmanager/nvlsm.log"}, } const supportBundleGlob = "????-??-?? (BEE-SP*)*.tar.gz" diff --git a/audit/internal/collector/nvidia.go b/audit/internal/collector/nvidia.go index 99c9a47..25639d5 100644 --- a/audit/internal/collector/nvidia.go +++ b/audit/internal/collector/nvidia.go @@ -6,7 +6,6 @@ import ( "fmt" "log/slog" "os/exec" - "regexp" "strconv" "strings" ) @@ -39,48 +38,7 @@ func enrichPCIeWithNVIDIA(devs []schema.HardwarePCIeDevice) []schema.HardwarePCI slog.Info("nvidia: enrichment skipped", "err", err) return enrichPCIeWithNVIDIAData(devs, nil, false) } - devs = enrichPCIeWithNVIDIAData(devs, gpuByBDF, true) - return enrichPCIeWithNVIDIANVLinks(devs) -} - -// enrichPCIeWithNVIDIANVLinks attaches per-link NVLink status (nvidia-smi -// nvlink -s) and error counters (nvidia-smi nvlink -e) to each GPU's -// HardwarePCIeDevice entry, keyed by the "nvidia_gpu_index" telemetry set by -// enrichPCIeWithNVIDIAData. Independent of NVSwitch/fabric-manager detection -// so it also covers direct GPU-to-GPU bridge boards with no switch present. -func enrichPCIeWithNVIDIANVLinks(devs []schema.HardwarePCIeDevice) []schema.HardwarePCIeDevice { - statusByGPU, statusErr := nvlinkStatusFn() - if statusErr != nil { - slog.Info("nvidia: nvlink -s unavailable, skipping nvlink enrichment", "err", statusErr) - return devs - } - errorsByGPU, errorsErr := nvlinkErrorsFn() - if errorsErr != nil { - slog.Info("nvidia: nvlink -e unavailable", "err", errorsErr) - } - - for i := range devs { - if devs[i].Telemetry == nil { - continue - } - idx, ok := devs[i].Telemetry["nvidia_gpu_index"].(int) - if !ok { - continue - } - ports, ok := statusByGPU[idx] - if !ok { - continue - } - for j := range ports { - if counters, ok := errorsByGPU[idx][ports[j].Index]; ok { - ports[j].ReplayErrors = &counters.Replay - ports[j].RecoveryErrors = &counters.Recovery - ports[j].CRCErrors = &counters.CRC - } - } - devs[i].NVLinks = ports - } - return devs + return enrichPCIeWithNVIDIAData(devs, gpuByBDF, true) } func hasNVIDIADevices(devs []schema.HardwarePCIeDevice) bool { @@ -327,107 +285,3 @@ func injectNVIDIATelemetry(dev *schema.HardwarePCIeDevice, info nvidiaGPUInfo) { dev.MaxLinkWidth = info.PCIeLinkWidthMax } } - -var ( - nvlinkGPUHeaderRe = regexp.MustCompile(`^GPU (\d+):`) - nvlinkSpeedLineRe = regexp.MustCompile(`^Link (\d+):\s*([\d.]+)\s*GB/s`) - nvlinkInactiveRe = regexp.MustCompile(`^Link (\d+):\s*`) - nvlinkErrorCounterRe = regexp.MustCompile(`^Link (\d+):\s*(Replay|Recovery|CRC) Errors:\s*(\d+)`) -) - -// nvlinkErrorCounters holds the per-link error counters reported by -// "nvidia-smi nvlink -e" for one GPU. -type nvlinkErrorCounters struct { - Replay, Recovery, CRC int64 -} - -// nvlinkStatusFn and nvlinkErrorsFn are swappable for testing. -var ( - nvlinkStatusFn = queryNVIDIANVLinkStatusByGPU - nvlinkErrorsFn = queryNVIDIANVLinkErrorsByGPU -) - -// queryNVIDIANVLinkStatusByGPU runs "nvidia-smi nvlink -s" and returns each -// GPU's NVLink ports keyed by GPU index (as printed in the "GPU N:" header, -// matching the index nvidia-smi --query-gpu also reports). -func queryNVIDIANVLinkStatusByGPU() (map[int][]schema.HardwareNVLinkPort, error) { - out, err := exec.Command("nvidia-smi", "nvlink", "-s").Output() - if err != nil { - return nil, err - } - return parseNVIDIANVLinkStatusByGPU(string(out)), nil -} - -func parseNVIDIANVLinkStatusByGPU(raw string) map[int][]schema.HardwareNVLinkPort { - result := map[int][]schema.HardwareNVLinkPort{} - currentGPU := -1 - for _, line := range strings.Split(raw, "\n") { - trimmed := strings.TrimSpace(line) - if m := nvlinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil { - currentGPU, _ = strconv.Atoi(m[1]) - continue - } - if currentGPU < 0 { - continue - } - if m := nvlinkInactiveRe.FindStringSubmatch(trimmed); m != nil { - idx, _ := strconv.Atoi(m[1]) - result[currentGPU] = append(result[currentGPU], schema.HardwareNVLinkPort{Index: idx, Active: false}) - continue - } - if m := nvlinkSpeedLineRe.FindStringSubmatch(trimmed); m != nil { - idx, _ := strconv.Atoi(m[1]) - port := schema.HardwareNVLinkPort{Index: idx, Active: true} - if speed, err := strconv.ParseFloat(m[2], 64); err == nil { - port.SpeedGBs = &speed - } - result[currentGPU] = append(result[currentGPU], port) - } - } - return result -} - -// queryNVIDIANVLinkErrorsByGPU runs "nvidia-smi nvlink -e" and returns -// per-link error counters keyed by GPU index then link index. -func queryNVIDIANVLinkErrorsByGPU() (map[int]map[int]nvlinkErrorCounters, error) { - out, err := exec.Command("nvidia-smi", "nvlink", "-e").Output() - if err != nil { - return nil, err - } - return parseNVIDIANVLinkErrorsByGPU(string(out)), nil -} - -func parseNVIDIANVLinkErrorsByGPU(raw string) map[int]map[int]nvlinkErrorCounters { - result := map[int]map[int]nvlinkErrorCounters{} - currentGPU := -1 - for _, line := range strings.Split(raw, "\n") { - trimmed := strings.TrimSpace(line) - if m := nvlinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil { - currentGPU, _ = strconv.Atoi(m[1]) - continue - } - if currentGPU < 0 { - continue - } - m := nvlinkErrorCounterRe.FindStringSubmatch(trimmed) - if m == nil { - continue - } - linkIdx, _ := strconv.Atoi(m[1]) - count, _ := strconv.ParseInt(m[3], 10, 64) - if result[currentGPU] == nil { - result[currentGPU] = map[int]nvlinkErrorCounters{} - } - c := result[currentGPU][linkIdx] - switch m[2] { - case "Replay": - c.Replay = count - case "Recovery": - c.Recovery = count - case "CRC": - c.CRC = count - } - result[currentGPU][linkIdx] = c - } - return result -} diff --git a/audit/internal/collector/nvidia_test.go b/audit/internal/collector/nvidia_test.go index d157f38..781d574 100644 --- a/audit/internal/collector/nvidia_test.go +++ b/audit/internal/collector/nvidia_test.go @@ -126,92 +126,3 @@ func TestEnrichPCIeWithNVIDIAData_driverMissingFallback(t *testing.T) { func ptrInt64(v int64) *int64 { return &v } func ptrFloat(v float64) *float64 { return &v } - -func TestParseNVIDIANVLinkStatusByGPU(t *testing.T) { - // Real-world 2-GPU direct-bridge H100 SXM output: link 15 inactive on both GPUs. - input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140) - Link 0: 26.562 GB/s - Link 15: - Link 17: 26.562 GB/s -GPU 1: NVIDIA H100 80GB HBM3 (UUID: GPU-603fe750-0516-9db5-86ec-ea61af3fce35) - Link 0: 26.562 GB/s - Link 15: -` - got := parseNVIDIANVLinkStatusByGPU(input) - - if len(got[0]) != 3 { - t.Fatalf("gpu0 ports=%d want 3 (%#v)", len(got[0]), got[0]) - } - if got[0][1].Index != 15 || got[0][1].Active { - t.Fatalf("gpu0 link15=%#v want inactive", got[0][1]) - } - if got[0][0].SpeedGBs == nil || *got[0][0].SpeedGBs != 26.562 { - t.Fatalf("gpu0 link0 speed=%#v want 26.562", got[0][0].SpeedGBs) - } - if len(got[1]) != 2 { - t.Fatalf("gpu1 ports=%d want 2 (%#v)", len(got[1]), got[1]) - } - if got[1][1].Active { - t.Fatalf("gpu1 link15 should be inactive: %#v", got[1][1]) - } -} - -func TestParseNVIDIANVLinkErrorsByGPU(t *testing.T) { - input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140) - Link 0: Replay Errors: 0 - Link 0: Recovery Errors: 0 - Link 0: CRC Errors: 0 - Link 1: Replay Errors: 3 - Link 1: Recovery Errors: 1 - Link 1: CRC Errors: 2 -GPU 1: NVIDIA H100 80GB HBM3 (UUID: GPU-603fe750-0516-9db5-86ec-ea61af3fce35) - Link 0: Replay Errors: 0 - Link 0: Recovery Errors: 0 - Link 0: CRC Errors: 0 -` - got := parseNVIDIANVLinkErrorsByGPU(input) - - c := got[0][1] - if c.Replay != 3 || c.Recovery != 1 || c.CRC != 2 { - t.Fatalf("gpu0 link1 counters=%#v want {3,1,2}", c) - } - zero := got[0][0] - if zero.Replay != 0 || zero.Recovery != 0 || zero.CRC != 0 { - t.Fatalf("gpu0 link0 counters=%#v want all zero", zero) - } - if _, ok := got[1][0]; !ok { - t.Fatalf("expected gpu1 link0 entry present") - } -} - -func TestEnrichPCIeWithNVIDIANVLinksAttachesPortsByIndex(t *testing.T) { - oldStatus, oldErrors := nvlinkStatusFn, nvlinkErrorsFn - t.Cleanup(func() { nvlinkStatusFn, nvlinkErrorsFn = oldStatus, oldErrors }) - - nvlinkStatusFn = func() (map[int][]schema.HardwareNVLinkPort, error) { - return map[int][]schema.HardwareNVLinkPort{ - 0: {{Index: 0, Active: true, SpeedGBs: ptrFloat(26.562)}, {Index: 15, Active: false}}, - }, nil - } - nvlinkErrorsFn = func() (map[int]map[int]nvlinkErrorCounters, error) { - return map[int]map[int]nvlinkErrorCounters{ - 0: {0: {Replay: 1}}, - }, nil - } - - devices := []schema.HardwarePCIeDevice{ - {Telemetry: map[string]any{"nvidia_gpu_index": 0}}, - } - - out := enrichPCIeWithNVIDIANVLinks(devices) - - if len(out[0].NVLinks) != 2 { - t.Fatalf("nvlinks=%d want 2", len(out[0].NVLinks)) - } - if out[0].NVLinks[0].ReplayErrors == nil || *out[0].NVLinks[0].ReplayErrors != 1 { - t.Fatalf("link0 replay errors=%#v want 1", out[0].NVLinks[0].ReplayErrors) - } - if out[0].NVLinks[1].Active { - t.Fatalf("link15 should stay inactive") - } -} diff --git a/audit/internal/platform/techdump.go b/audit/internal/platform/techdump.go index 298be1a..8126f75 100644 --- a/audit/internal/platform/techdump.go +++ b/audit/internal/platform/techdump.go @@ -21,6 +21,7 @@ var techDumpFixedCommands = []struct { {Name: "dmidecode", Args: []string{"-t", "17"}, File: "dmidecode-type17.txt"}, {Name: "lspci", Args: []string{"-vmm", "-D"}, File: "lspci-vmm.txt"}, {Name: "lspci", Args: []string{"-vvv"}, File: "lspci-vvv.txt"}, + {Name: "lscpu", Args: nil, File: "lscpu.txt"}, {Name: "lsblk", Args: []string{"-J", "-d", "-o", "NAME,TYPE,SIZE,SERIAL,MODEL,TRAN,HCTL"}, File: "lsblk.json"}, {Name: "sensors", Args: []string{"-j"}, File: "sensors.json"}, {Name: "ipmitool", Args: []string{"fru", "print"}, File: "ipmitool-fru.txt"}, diff --git a/audit/internal/schema/hardware.go b/audit/internal/schema/hardware.go index 66d5c28..ceb99b6 100644 --- a/audit/internal/schema/hardware.go +++ b/audit/internal/schema/hardware.go @@ -178,63 +178,48 @@ type HardwareStorage struct { type HardwarePCIeDevice struct { HardwareComponentStatus - Slot *string `json:"slot,omitempty"` - VendorID *int `json:"vendor_id,omitempty"` - DeviceID *int `json:"device_id,omitempty"` - NUMANode *int `json:"numa_node,omitempty"` - TemperatureC *float64 `json:"temperature_c,omitempty"` - PowerW *float64 `json:"power_w,omitempty"` - LifeRemainingPct *float64 `json:"life_remaining_pct,omitempty"` - LifeUsedPct *float64 `json:"life_used_pct,omitempty"` - ECCCorrectedTotal *int64 `json:"ecc_corrected_total,omitempty"` - ECCUncorrectedTotal *int64 `json:"ecc_uncorrected_total,omitempty"` - HWSlowdown *bool `json:"hw_slowdown,omitempty"` - BatteryChargePct *float64 `json:"battery_charge_pct,omitempty"` - BatteryHealthPct *float64 `json:"battery_health_pct,omitempty"` - BatteryTemperatureC *float64 `json:"battery_temperature_c,omitempty"` - BatteryVoltageV *float64 `json:"battery_voltage_v,omitempty"` - BatteryReplaceRequired *bool `json:"battery_replace_required,omitempty"` - SFPPresent *bool `json:"sfp_present,omitempty"` - SFPIdentifier *string `json:"sfp_identifier,omitempty"` - SFPConnector *string `json:"sfp_connector,omitempty"` - SFPVendor *string `json:"sfp_vendor,omitempty"` - SFPPartNumber *string `json:"sfp_part_number,omitempty"` - SFPSerialNumber *string `json:"sfp_serial_number,omitempty"` - SFPWavelengthNM *float64 `json:"sfp_wavelength_nm,omitempty"` - SFPTemperatureC *float64 `json:"sfp_temperature_c,omitempty"` - SFPTXPowerDBM *float64 `json:"sfp_tx_power_dbm,omitempty"` - SFPRXPowerDBM *float64 `json:"sfp_rx_power_dbm,omitempty"` - SFPVoltageV *float64 `json:"sfp_voltage_v,omitempty"` - SFPBiasMA *float64 `json:"sfp_bias_ma,omitempty"` - BDF *string `json:"-"` - DeviceClass *string `json:"device_class,omitempty"` - Manufacturer *string `json:"manufacturer,omitempty"` - Model *string `json:"model,omitempty"` - LinkWidth *int `json:"link_width,omitempty"` - LinkSpeed *string `json:"link_speed,omitempty"` - MaxLinkWidth *int `json:"max_link_width,omitempty"` - MaxLinkSpeed *string `json:"max_link_speed,omitempty"` - SerialNumber *string `json:"serial_number,omitempty"` - Firmware *string `json:"firmware,omitempty"` - MacAddresses []string `json:"mac_addresses,omitempty"` - Present *bool `json:"present,omitempty"` - IOMMUGroup *int `json:"iommu_group,omitempty"` - NVLinks []HardwareNVLinkPort `json:"nvlinks,omitempty"` - Telemetry map[string]any `json:"-"` -} - -// HardwareNVLinkPort describes a single NVLink lane on a GPU, as reported by -// "nvidia-smi nvlink -s" (speed/active state) and "nvidia-smi nvlink -e" -// (per-link error counters). Only populated on GPU-class HardwarePCIeDevice -// entries. An inactive link is not necessarily a fault: some GPU SKUs/boards -// reserve lanes as standby failover paths by design. -type HardwareNVLinkPort struct { - Index int `json:"index"` - Active bool `json:"active"` - SpeedGBs *float64 `json:"speed_gbs,omitempty"` - ReplayErrors *int64 `json:"replay_errors,omitempty"` - RecoveryErrors *int64 `json:"recovery_errors,omitempty"` - CRCErrors *int64 `json:"crc_errors,omitempty"` + Slot *string `json:"slot,omitempty"` + VendorID *int `json:"vendor_id,omitempty"` + DeviceID *int `json:"device_id,omitempty"` + NUMANode *int `json:"numa_node,omitempty"` + TemperatureC *float64 `json:"temperature_c,omitempty"` + PowerW *float64 `json:"power_w,omitempty"` + LifeRemainingPct *float64 `json:"life_remaining_pct,omitempty"` + LifeUsedPct *float64 `json:"life_used_pct,omitempty"` + ECCCorrectedTotal *int64 `json:"ecc_corrected_total,omitempty"` + ECCUncorrectedTotal *int64 `json:"ecc_uncorrected_total,omitempty"` + HWSlowdown *bool `json:"hw_slowdown,omitempty"` + BatteryChargePct *float64 `json:"battery_charge_pct,omitempty"` + BatteryHealthPct *float64 `json:"battery_health_pct,omitempty"` + BatteryTemperatureC *float64 `json:"battery_temperature_c,omitempty"` + BatteryVoltageV *float64 `json:"battery_voltage_v,omitempty"` + BatteryReplaceRequired *bool `json:"battery_replace_required,omitempty"` + SFPPresent *bool `json:"sfp_present,omitempty"` + SFPIdentifier *string `json:"sfp_identifier,omitempty"` + SFPConnector *string `json:"sfp_connector,omitempty"` + SFPVendor *string `json:"sfp_vendor,omitempty"` + SFPPartNumber *string `json:"sfp_part_number,omitempty"` + SFPSerialNumber *string `json:"sfp_serial_number,omitempty"` + SFPWavelengthNM *float64 `json:"sfp_wavelength_nm,omitempty"` + SFPTemperatureC *float64 `json:"sfp_temperature_c,omitempty"` + SFPTXPowerDBM *float64 `json:"sfp_tx_power_dbm,omitempty"` + SFPRXPowerDBM *float64 `json:"sfp_rx_power_dbm,omitempty"` + SFPVoltageV *float64 `json:"sfp_voltage_v,omitempty"` + SFPBiasMA *float64 `json:"sfp_bias_ma,omitempty"` + BDF *string `json:"-"` + DeviceClass *string `json:"device_class,omitempty"` + Manufacturer *string `json:"manufacturer,omitempty"` + Model *string `json:"model,omitempty"` + LinkWidth *int `json:"link_width,omitempty"` + LinkSpeed *string `json:"link_speed,omitempty"` + MaxLinkWidth *int `json:"max_link_width,omitempty"` + MaxLinkSpeed *string `json:"max_link_speed,omitempty"` + SerialNumber *string `json:"serial_number,omitempty"` + Firmware *string `json:"firmware,omitempty"` + MacAddresses []string `json:"mac_addresses,omitempty"` + Present *bool `json:"present,omitempty"` + IOMMUGroup *int `json:"iommu_group,omitempty"` + Telemetry map[string]any `json:"-"` } type HardwarePowerSupply struct { diff --git a/audit/internal/webui/api.go b/audit/internal/webui/api.go index 19672d2..88e1888 100644 --- a/audit/internal/webui/api.go +++ b/audit/internal/webui/api.go @@ -1726,6 +1726,9 @@ func (h *handler) handleAPIComponentDetail(w http.ResponseWriter, r *http.Reques case "gpu": title = "GPU" prefixes = []string{"pcie:gpu:"} + case "nic": + title = "NIC" + prefixes = []string{"pcie:nic:"} case "psu": title = "PSU" prefixes = []string{"psu:"} diff --git a/audit/internal/webui/layout.go b/audit/internal/webui/layout.go index f9c6d68..ddb5714 100644 --- a/audit/internal/webui/layout.go +++ b/audit/internal/webui/layout.go @@ -111,6 +111,7 @@ func layoutNav(active string, buildLabel string) string { {sep: true}, {id: "tasks", label: "Tasks", href: "/tasks"}, {id: "tools", label: "Tools", href: "/tools"}, + {id: "topo", label: "Topology", href: "/topo"}, {id: "settings", label: "Settings", href: "/settings"}, } var b strings.Builder diff --git a/audit/internal/webui/page_topo.go b/audit/internal/webui/page_topo.go new file mode 100644 index 0000000..3eca577 --- /dev/null +++ b/audit/internal/webui/page_topo.go @@ -0,0 +1,777 @@ +package webui + +import ( + "encoding/json" + "fmt" + "html" + "os/exec" + "regexp" + "sort" + "strconv" + "strings" + + "bee/audit/internal/schema" +) + +// renderTopo renders the /topo page: a read-only visualization of the server +// topology (CPU sockets, NUMA-affine PCIe devices, PSU/BMC) plus a separate +// NVLink topology card. It is pure visualization: everything it reads either +// already exists in the audit.json contract, or is a live (non-persisted) +// query — nothing here writes to schema.HardwarePCIeDevice or any other +// contract type, and nothing here talks to Reanimator Core. +func renderTopo(opts HandlerOptions) string { + data, err := loadSnapshot(opts.AuditPath) + if err != nil { + return topoCard("Topology", `No audit data`) + } + var ingest schema.HardwareIngestRequest + if err := json.Unmarshal(data, &ingest); err != nil { + return topoCard("Topology", `Parse error`) + } + hw := ingest.Hardware + + var b strings.Builder + b.WriteString(renderTopoMainDiagram(hw)) + if nv := renderTopoNVLinkCard(hw); nv != "" { + b.WriteString(nv) + } + return b.String() +} + +func topoCard(title, body string) string { + return `
` + html.EscapeString(title) + `
` + body + `
` +} + +// --------------------------------------------------------------------------- +// Classification helpers +// +// webui does not import collector (matches the existing isGPUDeviceClass +// precedent in pages.go, which already locally duplicates collector.isGPUClass +// instead of importing the package for one classifier). +// --------------------------------------------------------------------------- + +// isNICDeviceClassDev mirrors the classification logic in hwDescribeNIC +// (pages.go), applied to a single device instead of aggregated counts. +func isNICDeviceClassDev(dev schema.HardwarePCIeDevice) bool { + if dev.DeviceClass != nil { + c := strings.ToLower(strings.TrimSpace(*dev.DeviceClass)) + if c == "ethernetcontroller" || c == "networkcontroller" || strings.Contains(c, "fibrechannel") { + return true + } + } + return len(dev.MacAddresses) > 0 +} + +// isRAIDControllerClass matches the canonical class strings produced by +// collector.mapPCIeDeviceClass for RAID/storage HBAs. +func isRAIDControllerClass(class string) bool { + switch strings.TrimSpace(class) { + case "MassStorageController", "StorageController": + return true + default: + return false + } +} + +// --------------------------------------------------------------------------- +// Status / link-speed coloring +// --------------------------------------------------------------------------- + +// topoStatusBadgeClass maps a component's Status pointer to a badge class, +// treating a nil/absent status the same as literal "Unknown" (matches how +// the rest of the UI already renders missing status, per chipLetterClass/ +// runtimeStatusBadge in pages.go). +func topoStatusBadgeClass(status *string) (label, cls string) { + if status == nil { + return "?", "badge-unknown" + } + switch strings.ToUpper(strings.TrimSpace(*status)) { + case "OK": + return "OK", "badge-ok" + case "WARNING", "WARN", "PARTIAL": + return "WARN", "badge-warn" + case "CRITICAL", "FAIL", "FAILED", "ERROR": + return "CRIT", "badge-err" + default: + return "?", "badge-unknown" + } +} + +// pcieGenRank ranks a PCIe generation label ("Gen3", "Gen4", ...) for +// comparison. Mirrors collector.pcieLinkSpeedRank's ordering; duplicated +// locally rather than exported, per the same "no collector import in webui" +// convention used for isGPUDeviceClass/isRAIDControllerClass. +func pcieGenRank(gen string) int { + gen = strings.ToLower(strings.TrimSpace(gen)) + gen = strings.TrimPrefix(gen, "gen") + n, err := strconv.Atoi(gen) + if err != nil { + return 0 + } + return n +} + +// topoEdgeColorVar computes the CPU->device edge color strictly from +// link_speed vs max_link_speed — NOT from dev.Status, since Status can also +// be overwritten by SAT/acceptance-test results on the same PCIe device, +// which would conflate "link is physically degraded" with "this GPU failed +// its stress test" into the same color. +func topoEdgeColorVar(dev schema.HardwarePCIeDevice) string { + if dev.LinkSpeed == nil || dev.MaxLinkSpeed == nil { + return "var(--muted)" + } + if pcieGenRank(*dev.LinkSpeed) < pcieGenRank(*dev.MaxLinkSpeed) { + return "var(--warn-fg)" + } + return "var(--ok-fg)" +} + +// --------------------------------------------------------------------------- +// NUMA node -> CPU socket join (heuristic, no guaranteed hardware mapping) +// --------------------------------------------------------------------------- + +// buildSocketIndex maps a NUMA node number to the index into cpus whose +// Socket field equals that NUMA node number. This is a best-effort heuristic +// (NUMANode == Socket) documented as such in the /topo design — Linux NUMA +// node numbering and dmidecode socket designation are different numbering +// domains with no guaranteed 1:1 mapping, but in practice agree for the +// common case of N-socket boards. +func buildSocketIndex(cpus []schema.HardwareCPU) map[int]int { + idx := map[int]int{} + for i, cpu := range cpus { + if cpu.Socket != nil { + idx[*cpu.Socket] = i + } + } + return idx +} + +// --------------------------------------------------------------------------- +// GPU pairwise NVLink adjacency (from a live "nvidia-smi topo -m" query) +// --------------------------------------------------------------------------- + +type gpuPairLink struct { + GPUA, GPUB int + NVLinks int +} + +var topoNVRe = regexp.MustCompile(`(?i)^NV(\d+)$`) + +// parseGPUPairAdjacency returns every GPU pair with a nonzero NVLink bond +// count from a "nvidia-smi topo -m" matrix. Unlike parseNVIDIATopologyMatrix +// (collector package, aggregate-only: min/all-active/count), this returns +// who is bonded to whom — required so GPU-GPU edges are drawn for actually +// bonded pairs, not for adjacent boxes in the layout. +func parseGPUPairAdjacency(raw string) []gpuPairLink { + lines := strings.Split(raw, "\n") + headerIdx := -1 + var gpuColIndices []int + for i, line := range lines { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "GPU0") { + parts := strings.Fields(trimmed) + for j, col := range parts { + if strings.HasPrefix(col, "GPU") { + gpuColIndices = append(gpuColIndices, j) + } + } + if len(gpuColIndices) >= 2 { + headerIdx = i + } + break + } + } + if headerIdx < 0 { + return nil + } + + colIdxToGPU := make(map[int]int, len(gpuColIndices)) + for gpuIdx, colIdx := range gpuColIndices { + colIdxToGPU[colIdx] = gpuIdx + } + + seen := map[[2]int]bool{} + var pairs []gpuPairLink + rowGPU := -1 + for _, line := range lines[headerIdx+1:] { + trimmed := strings.TrimSpace(line) + if !strings.HasPrefix(trimmed, "GPU") { + continue + } + cells := strings.Fields(trimmed) + if len(cells) == 0 { + continue + } + rowLabel := strings.TrimPrefix(cells[0], "GPU") + n, err := strconv.Atoi(rowLabel) + if err != nil { + continue + } + rowGPU = n + for colIdx, colGPU := range colIdxToGPU { + if colGPU == rowGPU { + continue + } + dataIdx := colIdx + 1 + if dataIdx >= len(cells) { + continue + } + m := topoNVRe.FindStringSubmatch(cells[dataIdx]) + if len(m) != 2 { + continue + } + nv, err := strconv.Atoi(m[1]) + if err != nil || nv <= 0 { + continue + } + a, bGPU := rowGPU, colGPU + if a > bGPU { + a, bGPU = bGPU, a + } + key := [2]int{a, bGPU} + if seen[key] { + continue + } + seen[key] = true + pairs = append(pairs, gpuPairLink{GPUA: a, GPUB: bGPU, NVLinks: nv}) + } + } + sort.Slice(pairs, func(i, j int) bool { + if pairs[i].GPUA != pairs[j].GPUA { + return pairs[i].GPUA < pairs[j].GPUA + } + return pairs[i].GPUB < pairs[j].GPUB + }) + return pairs +} + +func queryGPUTopologyMatrix() (string, error) { + out, err := exec.Command("nvidia-smi", "topo", "-m").Output() + if err != nil { + return "", err + } + return string(out), nil +} + +// queryNVIDIAIndexByBDF runs a lightweight live nvidia-smi query mapping +// PCI bus address (matching HardwarePCIeDevice.Slot) to the GPU index +// nvidia-smi/dcgmi/topo -m report, so GPU-GPU edges (keyed by index) can be +// anchored to the correct box (keyed by BDF) in the diagram. +func queryNVIDIAIndexByBDF() (map[string]int, error) { + out, err := exec.Command("nvidia-smi", "--query-gpu=index,pci.bus_id", "--format=csv,noheader").Output() + if err != nil { + return nil, err + } + result := map[string]int{} + for _, line := range strings.Split(string(out), "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + parts := strings.SplitN(line, ",", 2) + if len(parts) != 2 { + continue + } + idx, err := strconv.Atoi(strings.TrimSpace(parts[0])) + if err != nil { + continue + } + bdf := normalizeTopoBDF(strings.TrimSpace(parts[1])) + if bdf == "" { + continue + } + result[bdf] = idx + } + return result, nil +} + +// normalizeTopoBDF normalizes a PCI bus address to "dddd:bb:dd.f" form so +// nvidia-smi's "pci.bus_id" output can be matched against +// HardwarePCIeDevice.Slot regardless of minor formatting differences +// (case, leading domain padding). +func normalizeTopoBDF(bdf string) string { + bdf = strings.ToLower(strings.TrimSpace(bdf)) + if bdf == "" { + return "" + } + parts := strings.Split(bdf, ":") + if len(parts) == 3 { + domain := parts[0] + if len(domain) > 4 { + domain = domain[len(domain)-4:] + } + return domain + ":" + parts[1] + ":" + parts[2] + } + return bdf +} + +// --------------------------------------------------------------------------- +// Main topology diagram +// --------------------------------------------------------------------------- + +const ( + topoColWidth = 220 + topoBoxWidth = 190 + topoBoxHeight = 56 + topoDeviceGap = 14 + topoTopMargin = 30 + topoEdgeBand = 60 + topoBottomRowH = 90 +) + +type topoBox struct { + x, y, w, h int + label string + sublabel string + badgeText string + badgeCls string + detailType string // "" = not clickable +} + +func renderTopoMainDiagram(hw schema.HardwareSnapshot) string { + socketIdx := buildSocketIndex(hw.CPUs) + numCols := len(hw.CPUs) + if numCols == 0 { + numCols = 1 + } + unknownCol := numCols // extra trailing column for unmatched devices + + // Group PCIe devices (GPU/NIC/RAID only — matches the mockup's node + // types) into columns by NUMA node, falling back to the "unknown" bucket. + type placedDevice struct { + dev schema.HardwarePCIeDevice + kind string // "gpu", "nic", "raid" + col int + bdf string + } + var placed []placedDevice + for _, dev := range hw.PCIeDevices { + var kind string + switch { + case dev.DeviceClass != nil && isGPUDeviceClass(*dev.DeviceClass): + kind = "gpu" + case isNICDeviceClassDev(dev): + kind = "nic" + case dev.DeviceClass != nil && isRAIDControllerClass(*dev.DeviceClass): + kind = "raid" + default: + continue + } + col := unknownCol + if dev.NUMANode != nil { + if ci, ok := socketIdx[*dev.NUMANode]; ok { + col = ci + } + } + bdf := "" + if dev.Slot != nil { + bdf = normalizeTopoBDF(*dev.Slot) + } else if dev.BDF != nil { + bdf = normalizeTopoBDF(*dev.BDF) + } + placed = append(placed, placedDevice{dev: dev, kind: kind, col: col, bdf: bdf}) + } + hasUnknownCol := false + for _, p := range placed { + if p.col == unknownCol { + hasUnknownCol = true + break + } + } + totalCols := numCols + if hasUnknownCol { + totalCols++ + } + + // Live GPU index<->BDF map + pairwise NVLink adjacency, best-effort: + // if nvidia-smi is unavailable, GPU-GPU edges are simply omitted. + bdfToIndex, _ := queryNVIDIAIndexByBDF() + var pairs []gpuPairLink + if topoMatrix, err := queryGPUTopologyMatrix(); err == nil { + pairs = parseGPUPairAdjacency(topoMatrix) + } + + var boxes []topoBox + var pcieEdges []struct { + x1, y1, x2, y2 int + color string + } + gpuBoxCenter := map[int][2]int{} // gpu index -> (x, yBottom) + gpuBoxIndex := map[int]int{} // gpu index -> index into boxes + gpuNUMANode := map[int]*int{} // gpu index -> its PCIe device's numa_node + + for col := 0; col < totalCols; col++ { + colX := (col+1)*24 + col*topoColWidth + if col < numCols { + cpu := hw.CPUs[col] + model := "" + if cpu.Model != nil { + model = *cpu.Model + } + socket := col + if cpu.Socket != nil { + socket = *cpu.Socket + } + label, cls := topoStatusBadgeClass(cpu.Status) + boxes = append(boxes, topoBox{ + x: colX, y: topoTopMargin, w: topoBoxWidth, h: topoBoxHeight, + label: fmt.Sprintf("CPU %d", socket), sublabel: model, + badgeText: label, badgeCls: cls, detailType: "cpu", + }) + } + + y := topoTopMargin + topoBoxHeight + topoDeviceGap*2 + for _, p := range placed { + if p.col != col { + continue + } + label, cls := topoStatusBadgeClass(p.dev.Status) + model := "" + if p.dev.Model != nil { + model = *p.dev.Model + } + box := topoBox{ + x: colX, y: y, w: topoBoxWidth, h: topoBoxHeight, + label: strings.ToUpper(p.kind), sublabel: model, + badgeText: label, badgeCls: cls, detailType: p.kind, + } + boxes = append(boxes, box) + boxIdx := len(boxes) - 1 + + if col < numCols { + pcieEdges = append(pcieEdges, struct { + x1, y1, x2, y2 int + color string + }{ + x1: colX + topoBoxWidth/2, y1: topoTopMargin + topoBoxHeight, + x2: colX + topoBoxWidth/2, y2: y, + color: topoEdgeColorVar(p.dev), + }) + } + + if p.kind == "gpu" && p.bdf != "" { + if idx, ok := bdfToIndex[p.bdf]; ok { + gpuBoxCenter[idx] = [2]int{colX + topoBoxWidth/2, y + topoBoxHeight} + gpuBoxIndex[idx] = boxIdx + gpuNUMANode[idx] = p.dev.NUMANode + } + } + + y += topoBoxHeight + topoDeviceGap + } + } + + maxDeviceY := topoTopMargin + topoBoxHeight + topoDeviceGap*2 + for _, b := range boxes { + if b.y+b.h > maxDeviceY { + maxDeviceY = b.y + b.h + } + } + + // GPU-GPU NVLink edges: drawn as an elbow connector through a dedicated + // band below the device row, kept strictly separate from the vertical + // CPU->device PCIe edges above so neither visually obscures the other. + // + // A bonded pair spanning two different NUMA nodes is treated as an + // anomaly (not a neutral fact) per project decision: we expect a bonded + // pair to sit on one NUMA node, so a cross-NUMA bond is flagged Warning + // on the edge AND on both GPU boxes, regardless of their own SAT status. + bandY := maxDeviceY + topoEdgeBand/2 + var gpuEdgesSVG strings.Builder + for _, pair := range pairs { + c1, ok1 := gpuBoxCenter[pair.GPUA] + c2, ok2 := gpuBoxCenter[pair.GPUB] + if !ok1 || !ok2 { + continue + } + color := "var(--ok-fg)" + title := fmt.Sprintf("NVLink: GPU%d↔GPU%d (%d links)", pair.GPUA, pair.GPUB, pair.NVLinks) + numaA, numaB := gpuNUMANode[pair.GPUA], gpuNUMANode[pair.GPUB] + if numaA == nil || numaB == nil { + color = "var(--muted)" + } else if *numaA != *numaB { + color = "var(--warn-fg)" + title += " — spans NUMA nodes (unexpected)" + upgradeTopoBoxBadgeToWarn(boxes, gpuBoxIndex[pair.GPUA]) + upgradeTopoBoxBadgeToWarn(boxes, gpuBoxIndex[pair.GPUB]) + } + fmt.Fprintf(&gpuEdgesSVG, + `%s`+"\n", + c1[0], c1[1], c1[0], bandY, c2[0], bandY, c2[0], c2[1], color, html.EscapeString(title)) + } + + svgHeight := bandY + topoEdgeBand/2 + topoBottomRowH + svgWidth := totalCols*topoColWidth + 48 + + var b strings.Builder + fmt.Fprintf(&b, ``+"\n", svgWidth, svgHeight, svgWidth, svgHeight) + for _, e := range pcieEdges { + fmt.Fprintf(&b, ``+"\n", e.x1, e.y1, e.x2, e.y2, e.color) + } + b.WriteString(gpuEdgesSVG.String()) + for _, box := range boxes { + writeTopoBoxSVG(&b, box) + } + + // PSU / BMC row: standalone boxes, no connecting lines. + psuY := svgHeight - topoBottomRowH + 20 + psuX := 24 + for _, psu := range hw.PowerSupplies { + label, cls := topoStatusBadgeClass(psu.Status) + slot := "" + if psu.Slot != nil { + slot = *psu.Slot + } + watt := "" + if psu.WattageW != nil { + watt = fmt.Sprintf("%dW", *psu.WattageW) + } + writeTopoBoxSVG(&b, topoBox{ + x: psuX, y: psuY, w: 150, h: topoBoxHeight, + label: "PSU " + slot, sublabel: watt, + badgeText: label, badgeCls: cls, detailType: "psu", + }) + psuX += 150 + topoDeviceGap + } + if bmcVersion, ok := findBMCFirmware(hw.Firmware); ok { + writeTopoBoxSVG(&b, topoBox{ + x: svgWidth - 200 - 24, y: psuY, w: 200, h: topoBoxHeight, + label: "BMC", sublabel: "fw " + bmcVersion, + badgeText: "?", badgeCls: "badge-unknown", + }) + } + + b.WriteString(``) + return topoCard("Topology", b.String()) +} + +// upgradeTopoBoxBadgeToWarn upgrades a box's badge to Warning unless it is +// already at Critical severity (never downgrades a worse status). +func upgradeTopoBoxBadgeToWarn(boxes []topoBox, boxIdx int) { + if boxIdx < 0 || boxIdx >= len(boxes) { + return + } + if boxes[boxIdx].badgeCls == "badge-err" { + return + } + boxes[boxIdx].badgeText = "WARN" + boxes[boxIdx].badgeCls = "badge-warn" +} + +func findBMCFirmware(records []schema.HardwareFirmwareRecord) (string, bool) { + for _, rec := range records { + if strings.EqualFold(strings.TrimSpace(rec.DeviceName), "BMC") { + return rec.Version, true + } + } + return "", false +} + +func writeTopoBoxSVG(b *strings.Builder, box topoBox) { + onclick := "" + cursor := "default" + if box.detailType != "" { + onclick = fmt.Sprintf(` onclick="openComponentDetail('%s')"`, box.detailType) + cursor = "pointer" + } + fmt.Fprintf(b, ``, onclick, cursor) + fmt.Fprintf(b, ``+"\n", + box.x, box.y, box.w, box.h) + fmt.Fprintf(b, `%s`+"\n", + box.x+10, box.y+20, html.EscapeString(box.label)) + if box.sublabel != "" { + fmt.Fprintf(b, `%s`+"\n", + box.x+10, box.y+36, html.EscapeString(truncateTopoLabel(box.sublabel, 26))) + } + fmt.Fprintf(b, ``+"\n", + box.x+box.w-54, box.y+box.h-26, box.badgeCls) + fmt.Fprintf(b, `%s`+"\n", + box.x+box.w-49, box.y+box.h-13, box.badgeCls, html.EscapeString(box.badgeText)) + b.WriteString(`` + "\n") +} + +func truncateTopoLabel(s string, max int) string { + if len(s) <= max { + return s + } + if max <= 1 { + return s[:max] + } + return s[:max-1] + "…" +} + +// --------------------------------------------------------------------------- +// Separate NVLink topology card (live query, not persisted to any contract) +// --------------------------------------------------------------------------- + +type topoNVLinkPort struct { + Index int + Active bool + SpeedGBs *float64 + ReplayErrors int64 + RecoveryErrors int64 + CRCErrors int64 +} + +var ( + topoNVLinkGPUHeaderRe = regexp.MustCompile(`^GPU (\d+):`) + topoNVLinkSpeedLineRe = regexp.MustCompile(`^Link (\d+):\s*([\d.]+)\s*GB/s`) + topoNVLinkInactiveRe = regexp.MustCompile(`^Link (\d+):\s*`) + topoNVLinkErrorLineRe = regexp.MustCompile(`^Link (\d+):\s*(Replay|Recovery|CRC) Errors:\s*(\d+)`) +) + +func queryTopoNVLinkStatus() (map[int][]topoNVLinkPort, error) { + out, err := exec.Command("nvidia-smi", "nvlink", "-s").Output() + if err != nil { + return nil, err + } + return parseTopoNVLinkStatus(string(out)), nil +} + +func parseTopoNVLinkStatus(raw string) map[int][]topoNVLinkPort { + result := map[int][]topoNVLinkPort{} + currentGPU := -1 + for _, line := range strings.Split(raw, "\n") { + trimmed := strings.TrimSpace(line) + if m := topoNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil { + currentGPU, _ = strconv.Atoi(m[1]) + continue + } + if currentGPU < 0 { + continue + } + if m := topoNVLinkInactiveRe.FindStringSubmatch(trimmed); m != nil { + idx, _ := strconv.Atoi(m[1]) + result[currentGPU] = append(result[currentGPU], topoNVLinkPort{Index: idx, Active: false}) + continue + } + if m := topoNVLinkSpeedLineRe.FindStringSubmatch(trimmed); m != nil { + idx, _ := strconv.Atoi(m[1]) + port := topoNVLinkPort{Index: idx, Active: true} + if speed, err := strconv.ParseFloat(m[2], 64); err == nil { + port.SpeedGBs = &speed + } + result[currentGPU] = append(result[currentGPU], port) + } + } + return result +} + +func queryTopoNVLinkErrors() (map[int]map[int][3]int64, error) { + out, err := exec.Command("nvidia-smi", "nvlink", "-e").Output() + if err != nil { + return nil, err + } + return parseTopoNVLinkErrors(string(out)), nil +} + +// parseTopoNVLinkErrors returns, per GPU then link index, [replay, recovery, crc]. +func parseTopoNVLinkErrors(raw string) map[int]map[int][3]int64 { + result := map[int]map[int][3]int64{} + currentGPU := -1 + for _, line := range strings.Split(raw, "\n") { + trimmed := strings.TrimSpace(line) + if m := topoNVLinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil { + currentGPU, _ = strconv.Atoi(m[1]) + continue + } + if currentGPU < 0 { + continue + } + m := topoNVLinkErrorLineRe.FindStringSubmatch(trimmed) + if m == nil { + continue + } + linkIdx, _ := strconv.Atoi(m[1]) + count, _ := strconv.ParseInt(m[3], 10, 64) + if result[currentGPU] == nil { + result[currentGPU] = map[int][3]int64{} + } + c := result[currentGPU][linkIdx] + switch m[2] { + case "Replay": + c[0] = count + case "Recovery": + c[1] = count + case "CRC": + c[2] = count + } + result[currentGPU][linkIdx] = c + } + return result +} + +// renderTopoNVLinkCard renders the separate NVLink topology card. Returns "" +// if there are fewer than 2 NVIDIA GPUs or nvidia-smi is unavailable. +func renderTopoNVLinkCard(hw schema.HardwareSnapshot) string { + gpuCount := 0 + for _, dev := range hw.PCIeDevices { + if dev.DeviceClass != nil && isGPUDeviceClass(*dev.DeviceClass) { + gpuCount++ + } + } + if gpuCount < 2 { + return "" + } + + status, err := queryTopoNVLinkStatus() + if err != nil || len(status) == 0 { + return topoCard("NVLink Topology", `nvidia-smi nvlink unavailable`) + } + errors, _ := queryTopoNVLinkErrors() + + topoMatrix, _ := queryGPUTopologyMatrix() + pairs := parseGPUPairAdjacency(topoMatrix) + + var bodyB strings.Builder + if gpuCount <= 4 && len(pairs) > 0 { + // Small GPU count: per-pair box+line with per-link detail. + for _, pair := range pairs { + activeCount, total, hasError := 0, 0, false + for _, port := range status[pair.GPUA] { + total++ + if port.Active { + activeCount++ + } + } + for _, counters := range errors[pair.GPUA] { + if counters[0] != 0 || counters[1] != 0 || counters[2] != 0 { + hasError = true + } + } + color := "var(--ok-fg)" + switch { + case hasError: + color = "var(--crit-fg)" + case total > 0 && activeCount < total: + color = "var(--warn-fg)" + } + fmt.Fprintf(&bodyB, `
`+ + `
GPU %d
`+ + `
`+ + `
GPU %d
`+ + `
%d/%d links active%s
`+ + `
`, + pair.GPUA, color, pair.GPUB, activeCount, total, errNoteSuffix(hasError)) + } + } else if len(pairs) > 0 { + // Larger GPU counts (NVSwitch fabric): aggregate pair table instead of + // an unreadable all-to-all graph. + bodyB.WriteString(``) + for _, pair := range pairs { + fmt.Fprintf(&bodyB, ``, pair.GPUA, pair.GPUB, pair.NVLinks) + } + bodyB.WriteString(`
GPU AGPU BNVLinks
GPU %dGPU %d%d
`) + } else { + bodyB.WriteString(`No NVLink-bonded GPU pairs found`) + } + + return topoCard("NVLink Topology", bodyB.String()) +} + +func errNoteSuffix(hasError bool) string { + if hasError { + return " — errors detected" + } + return "" +} diff --git a/audit/internal/webui/page_topo_test.go b/audit/internal/webui/page_topo_test.go new file mode 100644 index 0000000..74935c9 --- /dev/null +++ b/audit/internal/webui/page_topo_test.go @@ -0,0 +1,245 @@ +package webui + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "bee/audit/internal/schema" +) + +func TestTopoPageNoAuditDataGracefulFallback(t *testing.T) { + handler := NewHandler(HandlerOptions{}) + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/topo", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("status=%d", rec.Code) + } + body := rec.Body.String() + if !strings.Contains(body, "No audit data") { + t.Fatalf("topo page missing no-audit-data fallback: %s", body) + } +} + +func TestTopoPageRendersCPUAndDegradedPCIeLink(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "audit.json") + + socket := 0 + numaNode := 0 + gen3, gen4 := "Gen3", "Gen4" + deviceClass := "VideoController" + model := "NVIDIA H100 80GB HBM3" + cpuModel := "Intel Xeon 6530" + okStatus := "OK" + + ingest := schema.HardwareIngestRequest{ + CollectedAt: "2026-03-15T00:00:00Z", + Hardware: schema.HardwareSnapshot{ + CPUs: []schema.HardwareCPU{ + { + HardwareComponentStatus: schema.HardwareComponentStatus{Status: &okStatus}, + Socket: &socket, + Model: &cpuModel, + }, + }, + PCIeDevices: []schema.HardwarePCIeDevice{ + { + DeviceClass: &deviceClass, + Model: &model, + NUMANode: &numaNode, + LinkSpeed: &gen3, + MaxLinkSpeed: &gen4, + }, + }, + }, + } + data, err := json.Marshal(ingest) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0644); err != nil { + t.Fatal(err) + } + + handler := NewHandler(HandlerOptions{AuditPath: path}) + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/topo", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("status=%d", rec.Code) + } + body := rec.Body.String() + if !strings.Contains(body, "CPU 0") { + t.Fatalf("topo page missing CPU 0 box: %s", body) + } + if !strings.Contains(body, "GPU") { + t.Fatalf("topo page missing GPU box: %s", body) + } + if !strings.Contains(body, "var(--warn-fg)") { + t.Fatalf("topo page missing degraded-link warn edge color: %s", body) + } +} + +func TestTopoPageLinkedFromNav(t *testing.T) { + handler := NewHandler(HandlerOptions{}) + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("status=%d", rec.Code) + } + if !strings.Contains(rec.Body.String(), `href="/topo"`) { + t.Fatalf("nav missing /topo link") + } +} + +func TestParseGPUPairAdjacencyRealTwoGPUDump(t *testing.T) { + // Real system/nvidia-smi-topo.txt from a support bundle for this exact + // server: two H100s directly bridged (NV17), spanning two NUMA nodes. + input := "\tGPU0\tGPU1\tNIC0\tNIC1\tCPU Affinity\tNUMA Affinity\tGPU NUMA ID\n" + + "GPU0\t X \tNV17\tSYS\tSYS\t0-23,48-71\t0\t\tN/A\n" + + "GPU1\tNV17\t X \tNODE\tNODE\t24-47,72-95\t1\t\tN/A\n" + + "NIC0\tSYS\tNODE\t X \tPIX\t\t\t\n" + + "NIC1\tSYS\tNODE\tPIX\t X \t\t\t\n" + + pairs := parseGPUPairAdjacency(input) + if len(pairs) != 1 { + t.Fatalf("pairs=%d want 1 (%#v)", len(pairs), pairs) + } + if pairs[0].GPUA != 0 || pairs[0].GPUB != 1 || pairs[0].NVLinks != 17 { + t.Fatalf("pair=%#v want {0,1,17}", pairs[0]) + } +} + +func TestParseGPUPairAdjacencyDoesNotChainUnrelatedGPUs(t *testing.T) { + // 4 GPUs: only (0,1) and (2,3) are actually bonded. GPU1 and GPU2 must + // NOT get an edge just because they're adjacent in the layout. + input := "\tGPU0\tGPU1\tGPU2\tGPU3\n" + + "GPU0\t X \tNV18\tSYS\tSYS\n" + + "GPU1\tNV18\t X \tSYS\tSYS\n" + + "GPU2\tSYS\tSYS\t X \tNV18\n" + + "GPU3\tSYS\tSYS\tNV18\t X \n" + + pairs := parseGPUPairAdjacency(input) + if len(pairs) != 2 { + t.Fatalf("pairs=%d want 2 (%#v)", len(pairs), pairs) + } + want := map[[2]int]bool{{0, 1}: true, {2, 3}: true} + for _, p := range pairs { + if !want[[2]int{p.GPUA, p.GPUB}] { + t.Fatalf("unexpected pair %#v (GPU1-GPU2 chaining bug?)", p) + } + } +} + +func TestParseGPUPairAdjacencyEmptyOnNoMatrix(t *testing.T) { + if pairs := parseGPUPairAdjacency("no gpus here"); pairs != nil { + t.Fatalf("pairs=%v want nil", pairs) + } +} + +func TestPcieGenRank(t *testing.T) { + if pcieGenRank("Gen5") <= pcieGenRank("Gen4") { + t.Fatalf("Gen5 should rank higher than Gen4") + } + if pcieGenRank("bogus") != 0 { + t.Fatalf("unparseable gen should rank 0") + } +} + +func TestTopoEdgeColorVar(t *testing.T) { + gen3, gen4 := "Gen3", "Gen4" + degraded := schema.HardwarePCIeDevice{LinkSpeed: &gen3, MaxLinkSpeed: &gen4} + if got := topoEdgeColorVar(degraded); got != "var(--warn-fg)" { + t.Fatalf("degraded color=%q want warn", got) + } + full := schema.HardwarePCIeDevice{LinkSpeed: &gen4, MaxLinkSpeed: &gen4} + if got := topoEdgeColorVar(full); got != "var(--ok-fg)" { + t.Fatalf("full-speed color=%q want ok", got) + } + unknown := schema.HardwarePCIeDevice{} + if got := topoEdgeColorVar(unknown); got != "var(--muted)" { + t.Fatalf("unknown color=%q want muted", got) + } +} + +func TestBuildSocketIndex(t *testing.T) { + s0, s1 := 0, 1 + cpus := []schema.HardwareCPU{{Socket: &s1}, {Socket: &s0}} + idx := buildSocketIndex(cpus) + if idx[0] != 1 || idx[1] != 0 { + t.Fatalf("idx=%#v want {0:1, 1:0}", idx) + } +} + +func TestIsRAIDControllerClass(t *testing.T) { + if !isRAIDControllerClass("StorageController") || !isRAIDControllerClass("MassStorageController") { + t.Fatalf("expected known RAID/storage classes to match") + } + if isRAIDControllerClass("VideoController") { + t.Fatalf("GPU class should not match RAID classifier") + } +} + +func TestIsNICDeviceClassDev(t *testing.T) { + class := "EthernetController" + nic := schema.HardwarePCIeDevice{DeviceClass: &class} + if !isNICDeviceClassDev(nic) { + t.Fatalf("expected EthernetController to classify as NIC") + } + withMac := schema.HardwarePCIeDevice{MacAddresses: []string{"aa:bb:cc:dd:ee:ff"}} + if !isNICDeviceClassDev(withMac) { + t.Fatalf("expected device with MAC address to classify as NIC") + } + other := schema.HardwarePCIeDevice{} + if isNICDeviceClassDev(other) { + t.Fatalf("expected empty device to not classify as NIC") + } +} + +func TestTopoStatusBadgeClassNilIsUnknown(t *testing.T) { + label, cls := topoStatusBadgeClass(nil) + if label != "?" || cls != "badge-unknown" { + t.Fatalf("nil status = (%q,%q) want (?, badge-unknown)", label, cls) + } + ok := "OK" + label, cls = topoStatusBadgeClass(&ok) + if label != "OK" || cls != "badge-ok" { + t.Fatalf("OK status = (%q,%q) want (OK, badge-ok)", label, cls) + } +} + +func TestParseTopoNVLinkStatus(t *testing.T) { + input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140) + Link 0: 26.562 GB/s + Link 15: +GPU 1: NVIDIA H100 80GB HBM3 (UUID: GPU-603fe750-0516-9db5-86ec-ea61af3fce35) + Link 0: 26.562 GB/s +` + got := parseTopoNVLinkStatus(input) + if len(got[0]) != 2 || got[0][1].Active { + t.Fatalf("gpu0=%#v want link15 inactive", got[0]) + } + if len(got[1]) != 1 || got[1][0].SpeedGBs == nil || *got[1][0].SpeedGBs != 26.562 { + t.Fatalf("gpu1=%#v want link0 26.562 GB/s", got[1]) + } +} + +func TestParseTopoNVLinkErrors(t *testing.T) { + input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140) + Link 0: Replay Errors: 0 + Link 0: Recovery Errors: 0 + Link 0: CRC Errors: 0 + Link 1: Replay Errors: 3 + Link 1: Recovery Errors: 1 + Link 1: CRC Errors: 2 +` + got := parseTopoNVLinkErrors(input) + c := got[0][1] + if c[0] != 3 || c[1] != 1 || c[2] != 2 { + t.Fatalf("link1 counters=%#v want {3,1,2}", c) + } +} diff --git a/audit/internal/webui/pages.go b/audit/internal/webui/pages.go index 4aa4b5b..800fafc 100644 --- a/audit/internal/webui/pages.go +++ b/audit/internal/webui/pages.go @@ -46,6 +46,10 @@ func renderPage(page string, opts HandlerOptions) string { pageID = "tools" title = "Tools" body = renderTools() + case "topo": + pageID = "topo" + title = "Topology" + body = renderTopo(opts) case "settings": pageID = "settings" title = "Settings"