sat: collect NVLink status/errors, fix nv-hostengine restart via systemd

Add NVLink port status (nvidia-smi nvlink -s), error counters (nvidia-smi nvlink -e), and dcgmi nvlink status to the support bundle, and enrich HardwarePCIeDevice entries with per-link telemetry. Replace the raw nv-hostengine pkill/restart dance in bee-nvidia-load with systemctl restart/start of nvidia-dcgm.service, and order bee-nvidia.service Before= nvidia-dcgm.service and nvidia-fabricmanager.service so modules/device nodes exist before those units start.
2026-07-07 10:21:56 +03:00
parent a98721b038
commit 33d6eee9cf
6 changed files with 339 additions and 81 deletions
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"log/slog"
 	"os/exec"
+	"regexp"
 	"strconv"
 	"strings"
 )
@@ -38,7 +39,48 @@ func enrichPCIeWithNVIDIA(devs []schema.HardwarePCIeDevice) []schema.HardwarePCI
 		slog.Info("nvidia: enrichment skipped", "err", err)
 		return enrichPCIeWithNVIDIAData(devs, nil, false)
 	}
-	return enrichPCIeWithNVIDIAData(devs, gpuByBDF, true)
+	devs = enrichPCIeWithNVIDIAData(devs, gpuByBDF, true)
+	return enrichPCIeWithNVIDIANVLinks(devs)
+}
+
+// enrichPCIeWithNVIDIANVLinks attaches per-link NVLink status (nvidia-smi
+// nvlink -s) and error counters (nvidia-smi nvlink -e) to each GPU's
+// HardwarePCIeDevice entry, keyed by the "nvidia_gpu_index" telemetry set by
+// enrichPCIeWithNVIDIAData. Independent of NVSwitch/fabric-manager detection
+// so it also covers direct GPU-to-GPU bridge boards with no switch present.
+func enrichPCIeWithNVIDIANVLinks(devs []schema.HardwarePCIeDevice) []schema.HardwarePCIeDevice {
+	statusByGPU, statusErr := nvlinkStatusFn()
+	if statusErr != nil {
+		slog.Info("nvidia: nvlink -s unavailable, skipping nvlink enrichment", "err", statusErr)
+		return devs
+	}
+	errorsByGPU, errorsErr := nvlinkErrorsFn()
+	if errorsErr != nil {
+		slog.Info("nvidia: nvlink -e unavailable", "err", errorsErr)
+	}
+
+	for i := range devs {
+		if devs[i].Telemetry == nil {
+			continue
+		}
+		idx, ok := devs[i].Telemetry["nvidia_gpu_index"].(int)
+		if !ok {
+			continue
+		}
+		ports, ok := statusByGPU[idx]
+		if !ok {
+			continue
+		}
+		for j := range ports {
+			if counters, ok := errorsByGPU[idx][ports[j].Index]; ok {
+				ports[j].ReplayErrors = &counters.Replay
+				ports[j].RecoveryErrors = &counters.Recovery
+				ports[j].CRCErrors = &counters.CRC
+			}
+		}
+		devs[i].NVLinks = ports
+	}
+	return devs
 }

 func hasNVIDIADevices(devs []schema.HardwarePCIeDevice) bool {
@@ -285,3 +327,107 @@ func injectNVIDIATelemetry(dev *schema.HardwarePCIeDevice, info nvidiaGPUInfo) {
 		dev.MaxLinkWidth = info.PCIeLinkWidthMax
 	}
 }
+
+var (
+	nvlinkGPUHeaderRe    = regexp.MustCompile(`^GPU (\d+):`)
+	nvlinkSpeedLineRe    = regexp.MustCompile(`^Link (\d+):\s*([\d.]+)\s*GB/s`)
+	nvlinkInactiveRe     = regexp.MustCompile(`^Link (\d+):\s*<inactive>`)
+	nvlinkErrorCounterRe = regexp.MustCompile(`^Link (\d+):\s*(Replay|Recovery|CRC) Errors:\s*(\d+)`)
+)
+
+// nvlinkErrorCounters holds the per-link error counters reported by
+// "nvidia-smi nvlink -e" for one GPU.
+type nvlinkErrorCounters struct {
+	Replay, Recovery, CRC int64
+}
+
+// nvlinkStatusFn and nvlinkErrorsFn are swappable for testing.
+var (
+	nvlinkStatusFn = queryNVIDIANVLinkStatusByGPU
+	nvlinkErrorsFn = queryNVIDIANVLinkErrorsByGPU
+)
+
+// queryNVIDIANVLinkStatusByGPU runs "nvidia-smi nvlink -s" and returns each
+// GPU's NVLink ports keyed by GPU index (as printed in the "GPU N:" header,
+// matching the index nvidia-smi --query-gpu also reports).
+func queryNVIDIANVLinkStatusByGPU() (map[int][]schema.HardwareNVLinkPort, error) {
+	out, err := exec.Command("nvidia-smi", "nvlink", "-s").Output()
+	if err != nil {
+		return nil, err
+	}
+	return parseNVIDIANVLinkStatusByGPU(string(out)), nil
+}
+
+func parseNVIDIANVLinkStatusByGPU(raw string) map[int][]schema.HardwareNVLinkPort {
+	result := map[int][]schema.HardwareNVLinkPort{}
+	currentGPU := -1
+	for _, line := range strings.Split(raw, "\n") {
+		trimmed := strings.TrimSpace(line)
+		if m := nvlinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
+			currentGPU, _ = strconv.Atoi(m[1])
+			continue
+		}
+		if currentGPU < 0 {
+			continue
+		}
+		if m := nvlinkInactiveRe.FindStringSubmatch(trimmed); m != nil {
+			idx, _ := strconv.Atoi(m[1])
+			result[currentGPU] = append(result[currentGPU], schema.HardwareNVLinkPort{Index: idx, Active: false})
+			continue
+		}
+		if m := nvlinkSpeedLineRe.FindStringSubmatch(trimmed); m != nil {
+			idx, _ := strconv.Atoi(m[1])
+			port := schema.HardwareNVLinkPort{Index: idx, Active: true}
+			if speed, err := strconv.ParseFloat(m[2], 64); err == nil {
+				port.SpeedGBs = &speed
+			}
+			result[currentGPU] = append(result[currentGPU], port)
+		}
+	}
+	return result
+}
+
+// queryNVIDIANVLinkErrorsByGPU runs "nvidia-smi nvlink -e" and returns
+// per-link error counters keyed by GPU index then link index.
+func queryNVIDIANVLinkErrorsByGPU() (map[int]map[int]nvlinkErrorCounters, error) {
+	out, err := exec.Command("nvidia-smi", "nvlink", "-e").Output()
+	if err != nil {
+		return nil, err
+	}
+	return parseNVIDIANVLinkErrorsByGPU(string(out)), nil
+}
+
+func parseNVIDIANVLinkErrorsByGPU(raw string) map[int]map[int]nvlinkErrorCounters {
+	result := map[int]map[int]nvlinkErrorCounters{}
+	currentGPU := -1
+	for _, line := range strings.Split(raw, "\n") {
+		trimmed := strings.TrimSpace(line)
+		if m := nvlinkGPUHeaderRe.FindStringSubmatch(trimmed); m != nil {
+			currentGPU, _ = strconv.Atoi(m[1])
+			continue
+		}
+		if currentGPU < 0 {
+			continue
+		}
+		m := nvlinkErrorCounterRe.FindStringSubmatch(trimmed)
+		if m == nil {
+			continue
+		}
+		linkIdx, _ := strconv.Atoi(m[1])
+		count, _ := strconv.ParseInt(m[3], 10, 64)
+		if result[currentGPU] == nil {
+			result[currentGPU] = map[int]nvlinkErrorCounters{}
+		}
+		c := result[currentGPU][linkIdx]
+		switch m[2] {
+		case "Replay":
+			c.Replay = count
+		case "Recovery":
+			c.Recovery = count
+		case "CRC":
+			c.CRC = count
+		}
+		result[currentGPU][linkIdx] = c
+	}
+	return result
+}