iso: harden fast-path staleness check and add DCGM version verification
needs_full_build() compared mtimes of VERSIONS/package-lists/hooks against a marker file, which is fragile: a failed build attempt never touches the marker, and git checkouts/rsync can reorder mtimes relative to real edit order. This let the fast-path silently reuse a stale squashfs (baked against old package pins) with no error — root-caused after a driver/DCGM pin change (590/DCGM 4.6.0 -> 580/DCGM 4.5.2) produced a ~290MB larger ISO that only shrank back down with --clean-build. Switch to a content hash (VERSIONS + package-lists + hooks + archives + bootloaders + auto/config + Dockerfile) instead of mtimes, and add a real version check in validate_iso_nvidia_runtime: extract dpkg status from the squashfs and confirm the installed datacenter-gpu-manager-4-core version matches DCGM_VERSION. The previous check only confirmed dcgmi/nv-hostengine /dcgmproftester exist, which stays true across DCGM versions and can't catch this class of staleness.
This commit is contained in:
Mikhail Chusavitin
+70
-12
@@ -609,36 +609,77 @@ validate_iso_nvidia_runtime() {
|
||||
squashfs_tmp="$(mktemp)"
|
||||
squashfs_list="$(mktemp)"
|
||||
iso_files="$(mktemp)"
|
||||
dpkg_status_dir="$(mktemp -d)"
|
||||
iso_list_files "$iso_path" > "$iso_files" || {
|
||||
rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files"
|
||||
rm -rf "$dpkg_status_dir"
|
||||
nvidia_runtime_fail "failed to list ISO files for NVIDIA runtime validation"
|
||||
}
|
||||
grep '^live/.*\.squashfs$' "$iso_files" | while IFS= read -r squashfs_member; do
|
||||
iso_read_member "$iso_path" "$squashfs_member" "$squashfs_tmp" || {
|
||||
rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files"
|
||||
rm -rf "$dpkg_status_dir"
|
||||
nvidia_runtime_fail "failed to extract $squashfs_member from ISO"
|
||||
}
|
||||
unsquashfs -ll "$squashfs_tmp" >> "$squashfs_list" 2>/dev/null || {
|
||||
rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files"
|
||||
rm -rf "$dpkg_status_dir"
|
||||
nvidia_runtime_fail "failed to inspect $squashfs_member from ISO"
|
||||
}
|
||||
# var/lib/dpkg/status lives in whichever squashfs layer has the base
|
||||
# rootfs (not the usr/firmware split-off layers); harmless no-op on
|
||||
# the others.
|
||||
unsquashfs -d "${dpkg_status_dir}/extract" -f "$squashfs_tmp" var/lib/dpkg/status >/dev/null 2>&1 || true
|
||||
: > "$squashfs_tmp"
|
||||
done
|
||||
|
||||
grep -Eq 'usr/bin/dcgmi$' "$squashfs_list" || {
|
||||
rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files"
|
||||
rm -rf "$dpkg_status_dir"
|
||||
nvidia_runtime_fail "dcgmi missing from final NVIDIA ISO"
|
||||
}
|
||||
grep -Eq 'usr/bin/nv-hostengine$' "$squashfs_list" || {
|
||||
rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files"
|
||||
rm -rf "$dpkg_status_dir"
|
||||
nvidia_runtime_fail "nv-hostengine missing from final NVIDIA ISO"
|
||||
}
|
||||
grep -Eq 'usr/bin/dcgmproftester([0-9]+)?$' "$squashfs_list" || {
|
||||
rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files"
|
||||
rm -rf "$dpkg_status_dir"
|
||||
nvidia_runtime_fail "dcgmproftester missing from final NVIDIA ISO"
|
||||
}
|
||||
|
||||
# Cross-check the DCGM package version actually baked into the squashfs
|
||||
# against VERSIONS. dcgmi/nv-hostengine/dcgmproftester presence alone
|
||||
# doesn't catch a stale squashfs served by a mis-detected fast-path build
|
||||
# (dcgmi stays present across DCGM versions) — this does.
|
||||
dpkg_status_file="${dpkg_status_dir}/extract/var/lib/dpkg/status"
|
||||
if [ -f "$dpkg_status_file" ]; then
|
||||
_installed_dcgm_version="$(awk '
|
||||
/^Package: datacenter-gpu-manager-4-core$/ { in_pkg=1; next }
|
||||
/^Package: / { in_pkg=0 }
|
||||
in_pkg && /^Version: / { sub(/^Version: /, ""); print; exit }
|
||||
' "$dpkg_status_file")"
|
||||
# dpkg records an epoch-qualified version ("1:4.5.2-1"); VERSIONS
|
||||
# pins the same value without the epoch, so match by suffix.
|
||||
case "$_installed_dcgm_version" in
|
||||
*"${DCGM_VERSION}")
|
||||
;;
|
||||
"")
|
||||
echo "=== WARNING: datacenter-gpu-manager-4-core not found in ISO dpkg status; skipping DCGM version check ==="
|
||||
;;
|
||||
*)
|
||||
rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files"
|
||||
rm -rf "$dpkg_status_dir"
|
||||
nvidia_runtime_fail "DCGM version mismatch: VERSIONS pins ${DCGM_VERSION} but ISO has ${_installed_dcgm_version} (stale squashfs — retry with --clean-build)"
|
||||
;;
|
||||
esac
|
||||
else
|
||||
echo "=== WARNING: could not read dpkg status from ISO; skipping DCGM version check ==="
|
||||
fi
|
||||
|
||||
rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files"
|
||||
rm -rf "$dpkg_status_dir"
|
||||
echo "=== NVIDIA runtime validation OK ==="
|
||||
}
|
||||
|
||||
@@ -885,8 +926,30 @@ reset_live_build_stage() {
|
||||
done
|
||||
}
|
||||
|
||||
# Marker written after every successful full lb build for this variant
|
||||
# Marker + content hash written after every successful full lb build for this variant
|
||||
FULL_BUILD_MARKER="${BUILD_WORK_DIR}/.bee-full-build-marker"
|
||||
FULL_BUILD_HASH_FILE="${BUILD_WORK_DIR}/.bee-full-build-hash"
|
||||
|
||||
# Hashes the content of every "heavy" config input (VERSIONS, package lists,
|
||||
# hooks, archives, bootloaders, auto/config, Dockerfile). Deliberately content-
|
||||
# based rather than mtime-based: mtimes get reset by git checkouts, rsync, and
|
||||
# retried builds in ways that don't track "did this content actually change
|
||||
# since the last full build", which previously let needs_full_build() silently
|
||||
# take the fast-path (reusing an old squashfs built against different package
|
||||
# pins) with no error.
|
||||
hash_heavy_config() {
|
||||
{
|
||||
[ -f "${BUILDER_DIR}/VERSIONS" ] && cat "${BUILDER_DIR}/VERSIONS"
|
||||
[ -f "${BUILDER_DIR}/auto/config" ] && cat "${BUILDER_DIR}/auto/config"
|
||||
[ -f "${BUILDER_DIR}/Dockerfile" ] && cat "${BUILDER_DIR}/Dockerfile"
|
||||
find \
|
||||
"${BUILDER_DIR}/config/package-lists" \
|
||||
"${BUILDER_DIR}/config/hooks" \
|
||||
"${BUILDER_DIR}/config/archives" \
|
||||
"${BUILDER_DIR}/config/bootloaders" \
|
||||
-type f 2>/dev/null | sort | xargs -r cat
|
||||
} | sha256sum | awk '{print $1}'
|
||||
}
|
||||
|
||||
# Returns 0 if full lb build is needed, 1 if fast-path is safe.
|
||||
# Fast-path is safe when only light files changed since the last full build
|
||||
@@ -894,24 +957,18 @@ FULL_BUILD_MARKER="${BUILD_WORK_DIR}/.bee-full-build-marker"
|
||||
# hooks, archives, Dockerfile, auto/config) require a full lb build.
|
||||
needs_full_build() {
|
||||
[ -f "${FULL_BUILD_MARKER}" ] || return 0
|
||||
[ -f "${FULL_BUILD_HASH_FILE}" ] || return 0
|
||||
[ -f "${BUILD_WORK_DIR}/live-image-amd64.hybrid.iso" ] || return 0
|
||||
# Accept any versioned squashfs (filesystem-v*.squashfs or legacy filesystem.squashfs)
|
||||
_any_sq=$(find "${BUILD_WORK_DIR}/binary/live" -maxdepth 1 \
|
||||
-name 'filesystem*.squashfs' 2>/dev/null | head -1)
|
||||
[ -n "$_any_sq" ] || return 0
|
||||
|
||||
_heavy=$(find \
|
||||
"${BUILDER_DIR}/VERSIONS" \
|
||||
"${BUILDER_DIR}/auto/config" \
|
||||
"${BUILDER_DIR}/Dockerfile" \
|
||||
"${BUILDER_DIR}/config/package-lists" \
|
||||
"${BUILDER_DIR}/config/hooks" \
|
||||
"${BUILDER_DIR}/config/archives" \
|
||||
"${BUILDER_DIR}/config/bootloaders" \
|
||||
-newer "${FULL_BUILD_MARKER}" 2>/dev/null | head -1)
|
||||
_new_hash="$(hash_heavy_config)"
|
||||
_old_hash="$(cat "${FULL_BUILD_HASH_FILE}" 2>/dev/null)"
|
||||
|
||||
if [ -n "$_heavy" ]; then
|
||||
echo "=== full build required: heavy config changed: $(basename "$_heavy") ==="
|
||||
if [ "$_new_hash" != "$_old_hash" ]; then
|
||||
echo "=== full build required: heavy config content changed since last full build ==="
|
||||
return 0
|
||||
fi
|
||||
|
||||
@@ -1765,6 +1822,7 @@ if [ -f "$ISO_RAW" ]; then
|
||||
validate_iso_nvidia_runtime "$ISO_RAW"
|
||||
cp "$ISO_RAW" "$ISO_OUT"
|
||||
touch "${FULL_BUILD_MARKER}"
|
||||
hash_heavy_config > "${FULL_BUILD_HASH_FILE}"
|
||||
echo ""
|
||||
echo "=== done (${BUILD_VARIANT}) ==="
|
||||
echo "ISO: $ISO_OUT"
|
||||
|
||||
Reference in New Issue
Block a user