iso: harden fast-path staleness check and add DCGM version verification

needs_full_build() compared mtimes of VERSIONS/package-lists/hooks against
a marker file, which is fragile: a failed build attempt never touches the
marker, and git checkouts/rsync can reorder mtimes relative to real edit
order. This let the fast-path silently reuse a stale squashfs (baked
against old package pins) with no error — root-caused after a driver/DCGM
pin change (590/DCGM 4.6.0 -> 580/DCGM 4.5.2) produced a ~290MB larger ISO
that only shrank back down with --clean-build.

Switch to a content hash (VERSIONS + package-lists + hooks + archives +
bootloaders + auto/config + Dockerfile) instead of mtimes, and add a real
version check in validate_iso_nvidia_runtime: extract dpkg status from the
squashfs and confirm the installed datacenter-gpu-manager-4-core version
matches DCGM_VERSION. The previous check only confirmed dcgmi/nv-hostengine
/dcgmproftester exist, which stays true across DCGM versions and can't
catch this class of staleness.

iso/builder/build.sh

@@ -609,36 +609,77 @@ validate_iso_nvidia_runtime() {
     squashfs_tmp="$(mktemp)"
     squashfs_list="$(mktemp)"
     iso_files="$(mktemp)"
+    dpkg_status_dir="$(mktemp -d)"
     iso_list_files "$iso_path" > "$iso_files" || {
         rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files"
+        rm -rf "$dpkg_status_dir"
         nvidia_runtime_fail "failed to list ISO files for NVIDIA runtime validation"
     }
     grep '^live/.*\.squashfs$' "$iso_files" | while IFS= read -r squashfs_member; do
         iso_read_member "$iso_path" "$squashfs_member" "$squashfs_tmp" || {
             rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files"
+            rm -rf "$dpkg_status_dir"
             nvidia_runtime_fail "failed to extract $squashfs_member from ISO"
         }
         unsquashfs -ll "$squashfs_tmp" >> "$squashfs_list" 2>/dev/null || {
             rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files"
+            rm -rf "$dpkg_status_dir"
             nvidia_runtime_fail "failed to inspect $squashfs_member from ISO"
         }
+        # var/lib/dpkg/status lives in whichever squashfs layer has the base
+        # rootfs (not the usr/firmware split-off layers); harmless no-op on
+        # the others.
+        unsquashfs -d "${dpkg_status_dir}/extract" -f "$squashfs_tmp" var/lib/dpkg/status >/dev/null 2>&1 || true
         : > "$squashfs_tmp"
     done
 
     grep -Eq 'usr/bin/dcgmi$' "$squashfs_list" || {
         rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files"
+        rm -rf "$dpkg_status_dir"
         nvidia_runtime_fail "dcgmi missing from final NVIDIA ISO"
     }
     grep -Eq 'usr/bin/nv-hostengine$' "$squashfs_list" || {
         rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files"
+        rm -rf "$dpkg_status_dir"
         nvidia_runtime_fail "nv-hostengine missing from final NVIDIA ISO"
     }
     grep -Eq 'usr/bin/dcgmproftester([0-9]+)?$' "$squashfs_list" || {
         rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files"
+        rm -rf "$dpkg_status_dir"
         nvidia_runtime_fail "dcgmproftester missing from final NVIDIA ISO"
     }
 
+    # Cross-check the DCGM package version actually baked into the squashfs
+    # against VERSIONS. dcgmi/nv-hostengine/dcgmproftester presence alone
+    # doesn't catch a stale squashfs served by a mis-detected fast-path build
+    # (dcgmi stays present across DCGM versions) — this does.
+    dpkg_status_file="${dpkg_status_dir}/extract/var/lib/dpkg/status"
+    if [ -f "$dpkg_status_file" ]; then
+        _installed_dcgm_version="$(awk '
+            /^Package: datacenter-gpu-manager-4-core$/ { in_pkg=1; next }
+            /^Package: / { in_pkg=0 }
+            in_pkg && /^Version: / { sub(/^Version: /, ""); print; exit }
+        ' "$dpkg_status_file")"
+        # dpkg records an epoch-qualified version ("1:4.5.2-1"); VERSIONS
+        # pins the same value without the epoch, so match by suffix.
+        case "$_installed_dcgm_version" in
+            *"${DCGM_VERSION}")
+                ;;
+            "")
+                echo "=== WARNING: datacenter-gpu-manager-4-core not found in ISO dpkg status; skipping DCGM version check ==="
+                ;;
+            *)
                 rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files"
+                rm -rf "$dpkg_status_dir"
+                nvidia_runtime_fail "DCGM version mismatch: VERSIONS pins ${DCGM_VERSION} but ISO has ${_installed_dcgm_version} (stale squashfs — retry with --clean-build)"
+                ;;
+        esac
+    else
+        echo "=== WARNING: could not read dpkg status from ISO; skipping DCGM version check ==="
+    fi
+
+    rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files"
+    rm -rf "$dpkg_status_dir"
     echo "=== NVIDIA runtime validation OK ==="
 }
 
@@ -885,8 +926,30 @@ reset_live_build_stage() {
     done
 }
 
-# Marker written after every successful full lb build for this variant
+# Marker + content hash written after every successful full lb build for this variant
 FULL_BUILD_MARKER="${BUILD_WORK_DIR}/.bee-full-build-marker"
+FULL_BUILD_HASH_FILE="${BUILD_WORK_DIR}/.bee-full-build-hash"
+
+# Hashes the content of every "heavy" config input (VERSIONS, package lists,
+# hooks, archives, bootloaders, auto/config, Dockerfile). Deliberately content-
+# based rather than mtime-based: mtimes get reset by git checkouts, rsync, and
+# retried builds in ways that don't track "did this content actually change
+# since the last full build", which previously let needs_full_build() silently
+# take the fast-path (reusing an old squashfs built against different package
+# pins) with no error.
+hash_heavy_config() {
+    {
+        [ -f "${BUILDER_DIR}/VERSIONS" ] && cat "${BUILDER_DIR}/VERSIONS"
+        [ -f "${BUILDER_DIR}/auto/config" ] && cat "${BUILDER_DIR}/auto/config"
+        [ -f "${BUILDER_DIR}/Dockerfile" ] && cat "${BUILDER_DIR}/Dockerfile"
+        find \
+            "${BUILDER_DIR}/config/package-lists" \
+            "${BUILDER_DIR}/config/hooks" \
+            "${BUILDER_DIR}/config/archives" \
+            "${BUILDER_DIR}/config/bootloaders" \
+            -type f 2>/dev/null | sort | xargs -r cat
+    } | sha256sum | awk '{print $1}'
+}
 
 # Returns 0 if full lb build is needed, 1 if fast-path is safe.
 # Fast-path is safe when only light files changed since the last full build
@@ -894,24 +957,18 @@ FULL_BUILD_MARKER="${BUILD_WORK_DIR}/.bee-full-build-marker"
 # hooks, archives, Dockerfile, auto/config) require a full lb build.
 needs_full_build() {
     [ -f "${FULL_BUILD_MARKER}" ]                                        || return 0
+    [ -f "${FULL_BUILD_HASH_FILE}" ]                                     || return 0
     [ -f "${BUILD_WORK_DIR}/live-image-amd64.hybrid.iso" ]               || return 0
     # Accept any versioned squashfs (filesystem-v*.squashfs or legacy filesystem.squashfs)
     _any_sq=$(find "${BUILD_WORK_DIR}/binary/live" -maxdepth 1 \
         -name 'filesystem*.squashfs' 2>/dev/null | head -1)
     [ -n "$_any_sq" ]                                                    || return 0
 
-    _heavy=$(find \
+    _new_hash="$(hash_heavy_config)"
-        "${BUILDER_DIR}/VERSIONS" \
+    _old_hash="$(cat "${FULL_BUILD_HASH_FILE}" 2>/dev/null)"
-        "${BUILDER_DIR}/auto/config" \
-        "${BUILDER_DIR}/Dockerfile" \
-        "${BUILDER_DIR}/config/package-lists" \
-        "${BUILDER_DIR}/config/hooks" \
-        "${BUILDER_DIR}/config/archives" \
-        "${BUILDER_DIR}/config/bootloaders" \
-        -newer "${FULL_BUILD_MARKER}" 2>/dev/null | head -1)
 
-    if [ -n "$_heavy" ]; then
+    if [ "$_new_hash" != "$_old_hash" ]; then
-        echo "=== full build required: heavy config changed: $(basename "$_heavy") ==="
+        echo "=== full build required: heavy config content changed since last full build ==="
         return 0
     fi
 
@@ -1765,6 +1822,7 @@ if [ -f "$ISO_RAW" ]; then
     validate_iso_nvidia_runtime "$ISO_RAW"
     cp "$ISO_RAW" "$ISO_OUT"
     touch "${FULL_BUILD_MARKER}"
+    hash_heavy_config > "${FULL_BUILD_HASH_FILE}"
     echo ""
     echo "=== done (${BUILD_VARIANT}) ==="
     echo "ISO: $ISO_OUT"