diff --git a/audit/internal/app/support_bundle.go b/audit/internal/app/support_bundle.go index 96bfa3e..cf20ac2 100644 --- a/audit/internal/app/support_bundle.go +++ b/audit/internal/app/support_bundle.go @@ -176,6 +176,27 @@ if command -v nvidia-smi >/dev/null 2>&1; then else echo "nvidia-smi not found" fi +`}}, + {name: "system/nvidia-smi-nvlink-status.txt", cmd: []string{"sh", "-c", ` +if command -v nvidia-smi >/dev/null 2>&1; then + nvidia-smi nvlink -s 2>&1 || true +else + echo "nvidia-smi not found" +fi +`}}, + {name: "system/nvidia-smi-nvlink-errors.txt", cmd: []string{"sh", "-c", ` +if command -v nvidia-smi >/dev/null 2>&1; then + nvidia-smi nvlink -e 2>&1 || true +else + echo "nvidia-smi not found" +fi +`}}, + {name: "system/dcgmi-nvlink-status.txt", cmd: []string{"sh", "-c", ` +if command -v dcgmi >/dev/null 2>&1; then + dcgmi nvlink --link-status 2>&1 || true +else + echo "dcgmi not found" +fi `}}, {name: "system/systemctl-nvidia-units.txt", cmd: []string{"sh", "-c", ` if ! command -v systemctl >/dev/null 2>&1; then diff --git a/audit/internal/collector/nvidia.go b/audit/internal/collector/nvidia.go index 25639d5..99c9a47 100644 --- a/audit/internal/collector/nvidia.go +++ b/audit/internal/collector/nvidia.go @@ -6,6 +6,7 @@ import ( "fmt" "log/slog" "os/exec" + "regexp" "strconv" "strings" ) @@ -38,7 +39,48 @@ func enrichPCIeWithNVIDIA(devs []schema.HardwarePCIeDevice) []schema.HardwarePCI slog.Info("nvidia: enrichment skipped", "err", err) return enrichPCIeWithNVIDIAData(devs, nil, false) } - return enrichPCIeWithNVIDIAData(devs, gpuByBDF, true) + devs = enrichPCIeWithNVIDIAData(devs, gpuByBDF, true) + return enrichPCIeWithNVIDIANVLinks(devs) +} + +// enrichPCIeWithNVIDIANVLinks attaches per-link NVLink status (nvidia-smi +// nvlink -s) and error counters (nvidia-smi nvlink -e) to each GPU's +// HardwarePCIeDevice entry, keyed by the "nvidia_gpu_index" telemetry set by +// enrichPCIeWithNVIDIAData. Independent of NVSwitch/fabric-manager detection +// so it also covers direct GPU-to-GPU bridge boards with no switch present. +func enrichPCIeWithNVIDIANVLinks(devs []schema.HardwarePCIeDevice) []schema.HardwarePCIeDevice { + statusByGPU, statusErr := nvlinkStatusFn() + if statusErr != nil { + slog.Info("nvidia: nvlink -s unavailable, skipping nvlink enrichment", "err", statusErr) + return devs + } + errorsByGPU, errorsErr := nvlinkErrorsFn() + if errorsErr != nil { + slog.Info("nvidia: nvlink -e unavailable", "err", errorsErr) + } + + for i := range devs { + if devs[i].Telemetry == nil { + continue + } + idx, ok := devs[i].Telemetry["nvidia_gpu_index"].(int) + if !ok { + continue + } + ports, ok := statusByGPU[idx] + if !ok { + continue + } + for j := range ports { + if counters, ok := errorsByGPU[idx][ports[j].Index]; ok { + ports[j].ReplayErrors = &counters.Replay + ports[j].RecoveryErrors = &counters.Recovery + ports[j].CRCErrors = &counters.CRC + } + } + devs[i].NVLinks = ports + } + return devs } func hasNVIDIADevices(devs []schema.HardwarePCIeDevice) bool { @@ -285,3 +327,107 @@ func injectNVIDIATelemetry(dev *schema.HardwarePCIeDevice, info nvidiaGPUInfo) { dev.MaxLinkWidth = info.PCIeLinkWidthMax } } + +var ( + nvlinkGPUHeaderRe = regexp.MustCompile(`^GPU (\d+):`) + nvlinkSpeedLineRe = regexp.MustCompile(`^Link (\d+):\s*([\d.]+)\s*GB/s`) + nvlinkInactiveRe = regexp.MustCompile(`^Link (\d+):\s*`) + nvlinkErrorCounterRe = regexp.MustCompile(`^Link (\d+):\s*(Replay|Recovery|CRC) Errors:\s*(\d+)`) +) + +// nvlinkErrorCounters holds the per-link error counters reported by +// "nvidia-smi nvlink -e" for one GPU. +type nvlinkErrorCounters struct { + Replay, Recovery, CRC int64 +} + +// nvlinkStatusFn and nvlinkErrorsFn are swappable for testing. +var ( + nvlinkStatusFn = queryNVIDIANVLinkStatusByGPU + nvlinkErrorsFn = queryNVIDIANVLinkErrorsByGPU +) + +// queryNVIDIANVLinkStatusByGPU runs "nvidia-smi nvlink -s" and returns each +// GPU's NVLink ports keyed by GPU index (as printed in the "GPU N:" header, +// matching the index nvidia-smi --query-gpu also reports). +func queryNVIDIANVLinkStatusByGPU() (map[int][]schema.HardwareNVLinkPort, error) { + out, err := exec.Command("nvidia-smi", "nvlink", "-s").Output() + if err != nil { + return nil, err + } + return parseNVIDIANVLinkStatusByGPU(string(out)), nil +} + +func parseNVIDIANVLinkStatusByGPU(raw string) map[int][]schema.HardwareNVLinkPort { + result := map[int][]schema.HardwareNVLinkPort{} + currentGPU := -1 + for _, line := range strings.Split(raw, "\n") { + trimmed := strings.TrimSpace(line) + if m := nvlinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil { + currentGPU, _ = strconv.Atoi(m[1]) + continue + } + if currentGPU < 0 { + continue + } + if m := nvlinkInactiveRe.FindStringSubmatch(trimmed); m != nil { + idx, _ := strconv.Atoi(m[1]) + result[currentGPU] = append(result[currentGPU], schema.HardwareNVLinkPort{Index: idx, Active: false}) + continue + } + if m := nvlinkSpeedLineRe.FindStringSubmatch(trimmed); m != nil { + idx, _ := strconv.Atoi(m[1]) + port := schema.HardwareNVLinkPort{Index: idx, Active: true} + if speed, err := strconv.ParseFloat(m[2], 64); err == nil { + port.SpeedGBs = &speed + } + result[currentGPU] = append(result[currentGPU], port) + } + } + return result +} + +// queryNVIDIANVLinkErrorsByGPU runs "nvidia-smi nvlink -e" and returns +// per-link error counters keyed by GPU index then link index. +func queryNVIDIANVLinkErrorsByGPU() (map[int]map[int]nvlinkErrorCounters, error) { + out, err := exec.Command("nvidia-smi", "nvlink", "-e").Output() + if err != nil { + return nil, err + } + return parseNVIDIANVLinkErrorsByGPU(string(out)), nil +} + +func parseNVIDIANVLinkErrorsByGPU(raw string) map[int]map[int]nvlinkErrorCounters { + result := map[int]map[int]nvlinkErrorCounters{} + currentGPU := -1 + for _, line := range strings.Split(raw, "\n") { + trimmed := strings.TrimSpace(line) + if m := nvlinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil { + currentGPU, _ = strconv.Atoi(m[1]) + continue + } + if currentGPU < 0 { + continue + } + m := nvlinkErrorCounterRe.FindStringSubmatch(trimmed) + if m == nil { + continue + } + linkIdx, _ := strconv.Atoi(m[1]) + count, _ := strconv.ParseInt(m[3], 10, 64) + if result[currentGPU] == nil { + result[currentGPU] = map[int]nvlinkErrorCounters{} + } + c := result[currentGPU][linkIdx] + switch m[2] { + case "Replay": + c.Replay = count + case "Recovery": + c.Recovery = count + case "CRC": + c.CRC = count + } + result[currentGPU][linkIdx] = c + } + return result +} diff --git a/audit/internal/collector/nvidia_test.go b/audit/internal/collector/nvidia_test.go index 781d574..d157f38 100644 --- a/audit/internal/collector/nvidia_test.go +++ b/audit/internal/collector/nvidia_test.go @@ -126,3 +126,92 @@ func TestEnrichPCIeWithNVIDIAData_driverMissingFallback(t *testing.T) { func ptrInt64(v int64) *int64 { return &v } func ptrFloat(v float64) *float64 { return &v } + +func TestParseNVIDIANVLinkStatusByGPU(t *testing.T) { + // Real-world 2-GPU direct-bridge H100 SXM output: link 15 inactive on both GPUs. + input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140) + Link 0: 26.562 GB/s + Link 15: + Link 17: 26.562 GB/s +GPU 1: NVIDIA H100 80GB HBM3 (UUID: GPU-603fe750-0516-9db5-86ec-ea61af3fce35) + Link 0: 26.562 GB/s + Link 15: +` + got := parseNVIDIANVLinkStatusByGPU(input) + + if len(got[0]) != 3 { + t.Fatalf("gpu0 ports=%d want 3 (%#v)", len(got[0]), got[0]) + } + if got[0][1].Index != 15 || got[0][1].Active { + t.Fatalf("gpu0 link15=%#v want inactive", got[0][1]) + } + if got[0][0].SpeedGBs == nil || *got[0][0].SpeedGBs != 26.562 { + t.Fatalf("gpu0 link0 speed=%#v want 26.562", got[0][0].SpeedGBs) + } + if len(got[1]) != 2 { + t.Fatalf("gpu1 ports=%d want 2 (%#v)", len(got[1]), got[1]) + } + if got[1][1].Active { + t.Fatalf("gpu1 link15 should be inactive: %#v", got[1][1]) + } +} + +func TestParseNVIDIANVLinkErrorsByGPU(t *testing.T) { + input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140) + Link 0: Replay Errors: 0 + Link 0: Recovery Errors: 0 + Link 0: CRC Errors: 0 + Link 1: Replay Errors: 3 + Link 1: Recovery Errors: 1 + Link 1: CRC Errors: 2 +GPU 1: NVIDIA H100 80GB HBM3 (UUID: GPU-603fe750-0516-9db5-86ec-ea61af3fce35) + Link 0: Replay Errors: 0 + Link 0: Recovery Errors: 0 + Link 0: CRC Errors: 0 +` + got := parseNVIDIANVLinkErrorsByGPU(input) + + c := got[0][1] + if c.Replay != 3 || c.Recovery != 1 || c.CRC != 2 { + t.Fatalf("gpu0 link1 counters=%#v want {3,1,2}", c) + } + zero := got[0][0] + if zero.Replay != 0 || zero.Recovery != 0 || zero.CRC != 0 { + t.Fatalf("gpu0 link0 counters=%#v want all zero", zero) + } + if _, ok := got[1][0]; !ok { + t.Fatalf("expected gpu1 link0 entry present") + } +} + +func TestEnrichPCIeWithNVIDIANVLinksAttachesPortsByIndex(t *testing.T) { + oldStatus, oldErrors := nvlinkStatusFn, nvlinkErrorsFn + t.Cleanup(func() { nvlinkStatusFn, nvlinkErrorsFn = oldStatus, oldErrors }) + + nvlinkStatusFn = func() (map[int][]schema.HardwareNVLinkPort, error) { + return map[int][]schema.HardwareNVLinkPort{ + 0: {{Index: 0, Active: true, SpeedGBs: ptrFloat(26.562)}, {Index: 15, Active: false}}, + }, nil + } + nvlinkErrorsFn = func() (map[int]map[int]nvlinkErrorCounters, error) { + return map[int]map[int]nvlinkErrorCounters{ + 0: {0: {Replay: 1}}, + }, nil + } + + devices := []schema.HardwarePCIeDevice{ + {Telemetry: map[string]any{"nvidia_gpu_index": 0}}, + } + + out := enrichPCIeWithNVIDIANVLinks(devices) + + if len(out[0].NVLinks) != 2 { + t.Fatalf("nvlinks=%d want 2", len(out[0].NVLinks)) + } + if out[0].NVLinks[0].ReplayErrors == nil || *out[0].NVLinks[0].ReplayErrors != 1 { + t.Fatalf("link0 replay errors=%#v want 1", out[0].NVLinks[0].ReplayErrors) + } + if out[0].NVLinks[1].Active { + t.Fatalf("link15 should stay inactive") + } +} diff --git a/audit/internal/schema/hardware.go b/audit/internal/schema/hardware.go index d12bf8e..66d5c28 100644 --- a/audit/internal/schema/hardware.go +++ b/audit/internal/schema/hardware.go @@ -59,13 +59,13 @@ type RuntimeInterface struct { } type HardwareSnapshot struct { - Board HardwareBoard `json:"board"` - Firmware []HardwareFirmwareRecord `json:"firmware,omitempty"` - CPUs []HardwareCPU `json:"cpus,omitempty"` - Memory []HardwareMemory `json:"memory,omitempty"` - Storage []HardwareStorage `json:"storage,omitempty"` - PCIeDevices []HardwarePCIeDevice `json:"pcie_devices,omitempty"` - PowerSupplies []HardwarePowerSupply `json:"power_supplies,omitempty"` + Board HardwareBoard `json:"board"` + Firmware []HardwareFirmwareRecord `json:"firmware,omitempty"` + CPUs []HardwareCPU `json:"cpus,omitempty"` + Memory []HardwareMemory `json:"memory,omitempty"` + Storage []HardwareStorage `json:"storage,omitempty"` + PCIeDevices []HardwarePCIeDevice `json:"pcie_devices,omitempty"` + PowerSupplies []HardwarePowerSupply `json:"power_supplies,omitempty"` Sensors *HardwareSensors `json:"sensors,omitempty"` EventLogs []HardwareEventLog `json:"event_logs,omitempty"` PlatformConfig *json.RawMessage `json:"platform_config,omitempty"` @@ -178,48 +178,63 @@ type HardwareStorage struct { type HardwarePCIeDevice struct { HardwareComponentStatus - Slot *string `json:"slot,omitempty"` - VendorID *int `json:"vendor_id,omitempty"` - DeviceID *int `json:"device_id,omitempty"` - NUMANode *int `json:"numa_node,omitempty"` - TemperatureC *float64 `json:"temperature_c,omitempty"` - PowerW *float64 `json:"power_w,omitempty"` - LifeRemainingPct *float64 `json:"life_remaining_pct,omitempty"` - LifeUsedPct *float64 `json:"life_used_pct,omitempty"` - ECCCorrectedTotal *int64 `json:"ecc_corrected_total,omitempty"` - ECCUncorrectedTotal *int64 `json:"ecc_uncorrected_total,omitempty"` - HWSlowdown *bool `json:"hw_slowdown,omitempty"` - BatteryChargePct *float64 `json:"battery_charge_pct,omitempty"` - BatteryHealthPct *float64 `json:"battery_health_pct,omitempty"` - BatteryTemperatureC *float64 `json:"battery_temperature_c,omitempty"` - BatteryVoltageV *float64 `json:"battery_voltage_v,omitempty"` - BatteryReplaceRequired *bool `json:"battery_replace_required,omitempty"` - SFPPresent *bool `json:"sfp_present,omitempty"` - SFPIdentifier *string `json:"sfp_identifier,omitempty"` - SFPConnector *string `json:"sfp_connector,omitempty"` - SFPVendor *string `json:"sfp_vendor,omitempty"` - SFPPartNumber *string `json:"sfp_part_number,omitempty"` - SFPSerialNumber *string `json:"sfp_serial_number,omitempty"` - SFPWavelengthNM *float64 `json:"sfp_wavelength_nm,omitempty"` - SFPTemperatureC *float64 `json:"sfp_temperature_c,omitempty"` - SFPTXPowerDBM *float64 `json:"sfp_tx_power_dbm,omitempty"` - SFPRXPowerDBM *float64 `json:"sfp_rx_power_dbm,omitempty"` - SFPVoltageV *float64 `json:"sfp_voltage_v,omitempty"` - SFPBiasMA *float64 `json:"sfp_bias_ma,omitempty"` - BDF *string `json:"-"` - DeviceClass *string `json:"device_class,omitempty"` - Manufacturer *string `json:"manufacturer,omitempty"` - Model *string `json:"model,omitempty"` - LinkWidth *int `json:"link_width,omitempty"` - LinkSpeed *string `json:"link_speed,omitempty"` - MaxLinkWidth *int `json:"max_link_width,omitempty"` - MaxLinkSpeed *string `json:"max_link_speed,omitempty"` - SerialNumber *string `json:"serial_number,omitempty"` - Firmware *string `json:"firmware,omitempty"` - MacAddresses []string `json:"mac_addresses,omitempty"` - Present *bool `json:"present,omitempty"` - IOMMUGroup *int `json:"iommu_group,omitempty"` - Telemetry map[string]any `json:"-"` + Slot *string `json:"slot,omitempty"` + VendorID *int `json:"vendor_id,omitempty"` + DeviceID *int `json:"device_id,omitempty"` + NUMANode *int `json:"numa_node,omitempty"` + TemperatureC *float64 `json:"temperature_c,omitempty"` + PowerW *float64 `json:"power_w,omitempty"` + LifeRemainingPct *float64 `json:"life_remaining_pct,omitempty"` + LifeUsedPct *float64 `json:"life_used_pct,omitempty"` + ECCCorrectedTotal *int64 `json:"ecc_corrected_total,omitempty"` + ECCUncorrectedTotal *int64 `json:"ecc_uncorrected_total,omitempty"` + HWSlowdown *bool `json:"hw_slowdown,omitempty"` + BatteryChargePct *float64 `json:"battery_charge_pct,omitempty"` + BatteryHealthPct *float64 `json:"battery_health_pct,omitempty"` + BatteryTemperatureC *float64 `json:"battery_temperature_c,omitempty"` + BatteryVoltageV *float64 `json:"battery_voltage_v,omitempty"` + BatteryReplaceRequired *bool `json:"battery_replace_required,omitempty"` + SFPPresent *bool `json:"sfp_present,omitempty"` + SFPIdentifier *string `json:"sfp_identifier,omitempty"` + SFPConnector *string `json:"sfp_connector,omitempty"` + SFPVendor *string `json:"sfp_vendor,omitempty"` + SFPPartNumber *string `json:"sfp_part_number,omitempty"` + SFPSerialNumber *string `json:"sfp_serial_number,omitempty"` + SFPWavelengthNM *float64 `json:"sfp_wavelength_nm,omitempty"` + SFPTemperatureC *float64 `json:"sfp_temperature_c,omitempty"` + SFPTXPowerDBM *float64 `json:"sfp_tx_power_dbm,omitempty"` + SFPRXPowerDBM *float64 `json:"sfp_rx_power_dbm,omitempty"` + SFPVoltageV *float64 `json:"sfp_voltage_v,omitempty"` + SFPBiasMA *float64 `json:"sfp_bias_ma,omitempty"` + BDF *string `json:"-"` + DeviceClass *string `json:"device_class,omitempty"` + Manufacturer *string `json:"manufacturer,omitempty"` + Model *string `json:"model,omitempty"` + LinkWidth *int `json:"link_width,omitempty"` + LinkSpeed *string `json:"link_speed,omitempty"` + MaxLinkWidth *int `json:"max_link_width,omitempty"` + MaxLinkSpeed *string `json:"max_link_speed,omitempty"` + SerialNumber *string `json:"serial_number,omitempty"` + Firmware *string `json:"firmware,omitempty"` + MacAddresses []string `json:"mac_addresses,omitempty"` + Present *bool `json:"present,omitempty"` + IOMMUGroup *int `json:"iommu_group,omitempty"` + NVLinks []HardwareNVLinkPort `json:"nvlinks,omitempty"` + Telemetry map[string]any `json:"-"` +} + +// HardwareNVLinkPort describes a single NVLink lane on a GPU, as reported by +// "nvidia-smi nvlink -s" (speed/active state) and "nvidia-smi nvlink -e" +// (per-link error counters). Only populated on GPU-class HardwarePCIeDevice +// entries. An inactive link is not necessarily a fault: some GPU SKUs/boards +// reserve lanes as standby failover paths by design. +type HardwareNVLinkPort struct { + Index int `json:"index"` + Active bool `json:"active"` + SpeedGBs *float64 `json:"speed_gbs,omitempty"` + ReplayErrors *int64 `json:"replay_errors,omitempty"` + RecoveryErrors *int64 `json:"recovery_errors,omitempty"` + CRCErrors *int64 `json:"crc_errors,omitempty"` } type HardwarePowerSupply struct { diff --git a/iso/overlay/etc/systemd/system/bee-nvidia.service b/iso/overlay/etc/systemd/system/bee-nvidia.service index 4b85f4f..0a4e245 100644 --- a/iso/overlay/etc/systemd/system/bee-nvidia.service +++ b/iso/overlay/etc/systemd/system/bee-nvidia.service @@ -1,7 +1,7 @@ [Unit] Description=Bee: load NVIDIA kernel modules and create device nodes After=local-fs.target udev.service bee-blackbox.service -Before=bee-audit.service +Before=bee-audit.service nvidia-dcgm.service nvidia-fabricmanager.service # Skip silently if bee-nvidia-load is absent (non-nvidia builds). ConditionPathExists=/usr/local/bin/bee-nvidia-load diff --git a/iso/overlay/usr/local/bin/bee-nvidia-load b/iso/overlay/usr/local/bin/bee-nvidia-load index 6da8281..106b2b5 100755 --- a/iso/overlay/usr/local/bin/bee-nvidia-load +++ b/iso/overlay/usr/local/bin/bee-nvidia-load @@ -274,38 +274,25 @@ else log "WARN: nvidia-fabricmanager.service not installed" fi -# Start DCGM host engine so dcgmi can discover GPUs. -# nv-hostengine must run after the NVIDIA modules and device nodes are ready. -# If it started too early (for example via systemd before bee-nvidia-load), it can -# keep a stale empty inventory and dcgmi diag later reports no testable entities. -if command -v nv-hostengine >/dev/null 2>&1; then - if pgrep -x nv-hostengine >/dev/null 2>&1; then - if command -v pkill >/dev/null 2>&1; then - pkill -x nv-hostengine >/dev/null 2>&1 || true - tries=0 - while pgrep -x nv-hostengine >/dev/null 2>&1; do - tries=$((tries + 1)) - if [ "${tries}" -ge 10 ]; then - log "WARN: nv-hostengine is still running after restart request" - break - fi - sleep 1 - done - if pgrep -x nv-hostengine >/dev/null 2>&1; then - log "WARN: keeping existing nv-hostengine process" - else - log "nv-hostengine restarted" - fi - else - log "WARN: pkill not found — cannot refresh nv-hostengine inventory" - fi - fi - if ! pgrep -x nv-hostengine >/dev/null 2>&1; then - nv-hostengine - log "nv-hostengine started" +# Restart the DCGM host engine so dcgmi can discover GPUs. nv-hostengine +# enumerates GPUs once at startup and never rescans; bee-nvidia.service now +# orders itself Before=nvidia-dcgm.service so systemd shouldn't start it until +# modules/device nodes exist, but restart here too in case the unit was +# already active from a previous boot/reload with a stale empty inventory. +# Use systemctl (not a raw nv-hostengine invocation) so systemd's own +# supervision of nvidia-dcgm.service stays authoritative and we don't end up +# with two host engines racing for the same port. +if command -v systemctl >/dev/null 2>&1 && systemctl list-unit-files --no-legend 2>/dev/null | grep -q '^nvidia-dcgm\.service'; then + if systemctl restart nvidia-dcgm.service >/dev/null 2>&1; then + log "nvidia-dcgm restarted" + elif systemctl start nvidia-dcgm.service >/dev/null 2>&1; then + log "nvidia-dcgm started" + else + log "WARN: failed to start nvidia-dcgm.service" + systemctl status nvidia-dcgm.service --no-pager 2>&1 | sed 's/^/ nvidia-dcgm: /' || true fi else - log "WARN: nv-hostengine not found — dcgmi diagnostics will not work" + log "WARN: nvidia-dcgm.service not installed" fi log "done"