diff --git a/iso/builder/build.sh b/iso/builder/build.sh index d20e8bc..d9f4d9b 100755 --- a/iso/builder/build.sh +++ b/iso/builder/build.sh @@ -609,36 +609,77 @@ validate_iso_nvidia_runtime() { squashfs_tmp="$(mktemp)" squashfs_list="$(mktemp)" iso_files="$(mktemp)" + dpkg_status_dir="$(mktemp -d)" iso_list_files "$iso_path" > "$iso_files" || { rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files" + rm -rf "$dpkg_status_dir" nvidia_runtime_fail "failed to list ISO files for NVIDIA runtime validation" } grep '^live/.*\.squashfs$' "$iso_files" | while IFS= read -r squashfs_member; do iso_read_member "$iso_path" "$squashfs_member" "$squashfs_tmp" || { rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files" + rm -rf "$dpkg_status_dir" nvidia_runtime_fail "failed to extract $squashfs_member from ISO" } unsquashfs -ll "$squashfs_tmp" >> "$squashfs_list" 2>/dev/null || { rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files" + rm -rf "$dpkg_status_dir" nvidia_runtime_fail "failed to inspect $squashfs_member from ISO" } + # var/lib/dpkg/status lives in whichever squashfs layer has the base + # rootfs (not the usr/firmware split-off layers); harmless no-op on + # the others. + unsquashfs -d "${dpkg_status_dir}/extract" -f "$squashfs_tmp" var/lib/dpkg/status >/dev/null 2>&1 || true : > "$squashfs_tmp" done grep -Eq 'usr/bin/dcgmi$' "$squashfs_list" || { rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files" + rm -rf "$dpkg_status_dir" nvidia_runtime_fail "dcgmi missing from final NVIDIA ISO" } grep -Eq 'usr/bin/nv-hostengine$' "$squashfs_list" || { rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files" + rm -rf "$dpkg_status_dir" nvidia_runtime_fail "nv-hostengine missing from final NVIDIA ISO" } grep -Eq 'usr/bin/dcgmproftester([0-9]+)?$' "$squashfs_list" || { rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files" + rm -rf "$dpkg_status_dir" nvidia_runtime_fail "dcgmproftester missing from final NVIDIA ISO" } + # Cross-check the DCGM package version actually baked into the squashfs + # against VERSIONS. dcgmi/nv-hostengine/dcgmproftester presence alone + # doesn't catch a stale squashfs served by a mis-detected fast-path build + # (dcgmi stays present across DCGM versions) — this does. + dpkg_status_file="${dpkg_status_dir}/extract/var/lib/dpkg/status" + if [ -f "$dpkg_status_file" ]; then + _installed_dcgm_version="$(awk ' + /^Package: datacenter-gpu-manager-4-core$/ { in_pkg=1; next } + /^Package: / { in_pkg=0 } + in_pkg && /^Version: / { sub(/^Version: /, ""); print; exit } + ' "$dpkg_status_file")" + # dpkg records an epoch-qualified version ("1:4.5.2-1"); VERSIONS + # pins the same value without the epoch, so match by suffix. + case "$_installed_dcgm_version" in + *"${DCGM_VERSION}") + ;; + "") + echo "=== WARNING: datacenter-gpu-manager-4-core not found in ISO dpkg status; skipping DCGM version check ===" + ;; + *) + rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files" + rm -rf "$dpkg_status_dir" + nvidia_runtime_fail "DCGM version mismatch: VERSIONS pins ${DCGM_VERSION} but ISO has ${_installed_dcgm_version} (stale squashfs — retry with --clean-build)" + ;; + esac + else + echo "=== WARNING: could not read dpkg status from ISO; skipping DCGM version check ===" + fi + rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files" + rm -rf "$dpkg_status_dir" echo "=== NVIDIA runtime validation OK ===" } @@ -885,8 +926,30 @@ reset_live_build_stage() { done } -# Marker written after every successful full lb build for this variant +# Marker + content hash written after every successful full lb build for this variant FULL_BUILD_MARKER="${BUILD_WORK_DIR}/.bee-full-build-marker" +FULL_BUILD_HASH_FILE="${BUILD_WORK_DIR}/.bee-full-build-hash" + +# Hashes the content of every "heavy" config input (VERSIONS, package lists, +# hooks, archives, bootloaders, auto/config, Dockerfile). Deliberately content- +# based rather than mtime-based: mtimes get reset by git checkouts, rsync, and +# retried builds in ways that don't track "did this content actually change +# since the last full build", which previously let needs_full_build() silently +# take the fast-path (reusing an old squashfs built against different package +# pins) with no error. +hash_heavy_config() { + { + [ -f "${BUILDER_DIR}/VERSIONS" ] && cat "${BUILDER_DIR}/VERSIONS" + [ -f "${BUILDER_DIR}/auto/config" ] && cat "${BUILDER_DIR}/auto/config" + [ -f "${BUILDER_DIR}/Dockerfile" ] && cat "${BUILDER_DIR}/Dockerfile" + find \ + "${BUILDER_DIR}/config/package-lists" \ + "${BUILDER_DIR}/config/hooks" \ + "${BUILDER_DIR}/config/archives" \ + "${BUILDER_DIR}/config/bootloaders" \ + -type f 2>/dev/null | sort | xargs -r cat + } | sha256sum | awk '{print $1}' +} # Returns 0 if full lb build is needed, 1 if fast-path is safe. # Fast-path is safe when only light files changed since the last full build @@ -894,24 +957,18 @@ FULL_BUILD_MARKER="${BUILD_WORK_DIR}/.bee-full-build-marker" # hooks, archives, Dockerfile, auto/config) require a full lb build. needs_full_build() { [ -f "${FULL_BUILD_MARKER}" ] || return 0 + [ -f "${FULL_BUILD_HASH_FILE}" ] || return 0 [ -f "${BUILD_WORK_DIR}/live-image-amd64.hybrid.iso" ] || return 0 # Accept any versioned squashfs (filesystem-v*.squashfs or legacy filesystem.squashfs) _any_sq=$(find "${BUILD_WORK_DIR}/binary/live" -maxdepth 1 \ -name 'filesystem*.squashfs' 2>/dev/null | head -1) [ -n "$_any_sq" ] || return 0 - _heavy=$(find \ - "${BUILDER_DIR}/VERSIONS" \ - "${BUILDER_DIR}/auto/config" \ - "${BUILDER_DIR}/Dockerfile" \ - "${BUILDER_DIR}/config/package-lists" \ - "${BUILDER_DIR}/config/hooks" \ - "${BUILDER_DIR}/config/archives" \ - "${BUILDER_DIR}/config/bootloaders" \ - -newer "${FULL_BUILD_MARKER}" 2>/dev/null | head -1) + _new_hash="$(hash_heavy_config)" + _old_hash="$(cat "${FULL_BUILD_HASH_FILE}" 2>/dev/null)" - if [ -n "$_heavy" ]; then - echo "=== full build required: heavy config changed: $(basename "$_heavy") ===" + if [ "$_new_hash" != "$_old_hash" ]; then + echo "=== full build required: heavy config content changed since last full build ===" return 0 fi @@ -1765,6 +1822,7 @@ if [ -f "$ISO_RAW" ]; then validate_iso_nvidia_runtime "$ISO_RAW" cp "$ISO_RAW" "$ISO_OUT" touch "${FULL_BUILD_MARKER}" + hash_heavy_config > "${FULL_BUILD_HASH_FILE}" echo "" echo "=== done (${BUILD_VARIANT}) ===" echo "ISO: $ISO_OUT"