sat: collect NVLink status/errors, fix nv-hostengine restart via systemd

Add NVLink port status (nvidia-smi nvlink -s), error counters
(nvidia-smi nvlink -e), and dcgmi nvlink status to the support bundle,
and enrich HardwarePCIeDevice entries with per-link telemetry.

Replace the raw nv-hostengine pkill/restart dance in bee-nvidia-load
with systemctl restart/start of nvidia-dcgm.service, and order
bee-nvidia.service Before= nvidia-dcgm.service and
nvidia-fabricmanager.service so modules/device nodes exist before
those units start.

iso/overlay/etc/systemd/system/bee-nvidia.service

@@ -1,7 +1,7 @@
 [Unit]
 Description=Bee: load NVIDIA kernel modules and create device nodes
 After=local-fs.target udev.service bee-blackbox.service
-Before=bee-audit.service
+Before=bee-audit.service nvidia-dcgm.service nvidia-fabricmanager.service
 # Skip silently if bee-nvidia-load is absent (non-nvidia builds).
 ConditionPathExists=/usr/local/bin/bee-nvidia-load
 

iso/overlay/usr/local/bin/bee-nvidia-load

@@ -274,38 +274,25 @@ else
     log "WARN: nvidia-fabricmanager.service not installed"
 fi
 
-# Start DCGM host engine so dcgmi can discover GPUs.
-# nv-hostengine must run after the NVIDIA modules and device nodes are ready.
-# If it started too early (for example via systemd before bee-nvidia-load), it can
-# keep a stale empty inventory and dcgmi diag later reports no testable entities.
-if command -v nv-hostengine >/dev/null 2>&1; then
-    if pgrep -x nv-hostengine >/dev/null 2>&1; then
-        if command -v pkill >/dev/null 2>&1; then
-            pkill -x nv-hostengine >/dev/null 2>&1 || true
-            tries=0
-            while pgrep -x nv-hostengine >/dev/null 2>&1; do
-                tries=$((tries + 1))
-                if [ "${tries}" -ge 10 ]; then
-                    log "WARN: nv-hostengine is still running after restart request"
-                    break
-                fi
-                sleep 1
-            done
-            if pgrep -x nv-hostengine >/dev/null 2>&1; then
-                log "WARN: keeping existing nv-hostengine process"
-            else
-                log "nv-hostengine restarted"
-            fi
-        else
-            log "WARN: pkill not found — cannot refresh nv-hostengine inventory"
-        fi
-    fi
-    if ! pgrep -x nv-hostengine >/dev/null 2>&1; then
-        nv-hostengine
-        log "nv-hostengine started"
+# Restart the DCGM host engine so dcgmi can discover GPUs. nv-hostengine
+# enumerates GPUs once at startup and never rescans; bee-nvidia.service now
+# orders itself Before=nvidia-dcgm.service so systemd shouldn't start it until
+# modules/device nodes exist, but restart here too in case the unit was
+# already active from a previous boot/reload with a stale empty inventory.
+# Use systemctl (not a raw nv-hostengine invocation) so systemd's own
+# supervision of nvidia-dcgm.service stays authoritative and we don't end up
+# with two host engines racing for the same port.
+if command -v systemctl >/dev/null 2>&1 && systemctl list-unit-files --no-legend 2>/dev/null | grep -q '^nvidia-dcgm\.service'; then
+    if systemctl restart nvidia-dcgm.service >/dev/null 2>&1; then
+        log "nvidia-dcgm restarted"
+    elif systemctl start nvidia-dcgm.service >/dev/null 2>&1; then
+        log "nvidia-dcgm started"
+    else
+        log "WARN: failed to start nvidia-dcgm.service"
+        systemctl status nvidia-dcgm.service --no-pager 2>&1 | sed 's/^/  nvidia-dcgm: /' || true
     fi
 else
-    log "WARN: nv-hostengine not found — dcgmi diagnostics will not work"
+    log "WARN: nvidia-dcgm.service not installed"
 fi
 
 log "done"