From eae55707304a54143da0ee9be8de3a9a213cc739 Mon Sep 17 00:00:00 2001 From: Mikhail Chusavitin Date: Thu, 3 Sep 2026 15:29:48 +0300 Subject: [PATCH] fix(iso): stop fast-path repack from clobbering merged-usr /lib The NVIDIA GSP firmware was staged into the overlay at a bare lib/firmware/nvidia//. On a merged-usr Debian root /lib is a symlink to usr/lib, so the fast-path repack's `rsync -a` (overlay stage -> the unsquashfs'd tree) replaced that symlink with a plain directory holding only the firmware. Everything else reachable only through /lib (/lib/systemd/systemd, /lib/x86_64-linux-gnu/ld-linux + libc, ...) was orphaned. The squashfs still packed and mounted, but boot panicked: run-init: can't execute '/sbin/init': No such file or directory run-init: can't execute '/bin/sh': No such file or directory Kernel panic - not syncing: Attempted to kill init Full `lb build` was unaffected (live-build copies includes.chroot with cp -a, which follows the /lib symlink), so only fast-path ISOs were bad. Three fixes: - build.sh: stage GSP firmware under usr/lib/firmware/nvidia//, the canonical merged-usr path. Changing the overlay path also makes overlay_paths_were_removed() force one full build on the next run. - fast-path.sh: add `rsync --keep-dirlinks` so a real directory in the overlay stage can never again replace a symlink-to-directory in the root. Verified: firmware lands in usr/lib/firmware, /lib stays a symlink. - iso-validation.sh: new validate_iso_rootfs_layout, run for every variant on both build paths. Fails the build if the squashfs has a plain-dir /bin|/sbin|/lib|/lib64, or if /usr/sbin/init is present without a resolvable /usr/lib/systemd/systemd and /lib symlink. Verified it flags the broken v14.02-1-g642e686 ISO and passes a correct layout. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01H2LLuid8PFhBqBPcxXxkQU --- iso/builder/build.sh | 16 ++++++-- iso/builder/lib/fast-path.sh | 7 +++- iso/builder/lib/iso-validation.sh | 61 +++++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 5 deletions(-) diff --git a/iso/builder/build.sh b/iso/builder/build.sh index f28dd33..06a5fbd 100755 --- a/iso/builder/build.sh +++ b/iso/builder/build.sh @@ -452,11 +452,17 @@ if [ "$BEE_GPU_VENDOR" = "nvidia" ]; then mkdir -p "${OVERLAY_STAGE_DIR}/etc/OpenCL/vendors" printf 'libnvidia-opencl.so.1\n' > "${OVERLAY_STAGE_DIR}/etc/OpenCL/vendors/nvidia.icd" - # Inject GSP firmware into /lib/firmware/nvidia// + # Inject GSP firmware into /usr/lib/firmware/nvidia//. + # + # It MUST be staged under usr/lib, never a bare lib/. On a merged-usr Debian + # root /lib is a symlink to usr/lib; staging a real lib/ directory here makes + # `rsync -a` (fast-path repack) replace that symlink with a plain directory, + # which orphans /lib/systemd/systemd, /lib/x86_64-linux-gnu/* and friends and + # panics the boot with "run-init: can't execute '/sbin/init'". if [ -d "${NVIDIA_CACHE}/firmware" ] && [ "$(ls -A "${NVIDIA_CACHE}/firmware" 2>/dev/null)" ]; then - mkdir -p "${OVERLAY_STAGE_DIR}/lib/firmware/nvidia/${NVIDIA_DRIVER_VERSION}" - cp "${NVIDIA_CACHE}/firmware/"* "${OVERLAY_STAGE_DIR}/lib/firmware/nvidia/${NVIDIA_DRIVER_VERSION}/" - echo "=== firmware: $(ls "${OVERLAY_STAGE_DIR}/lib/firmware/nvidia/${NVIDIA_DRIVER_VERSION}/" | wc -l) files injected ===" + mkdir -p "${OVERLAY_STAGE_DIR}/usr/lib/firmware/nvidia/${NVIDIA_DRIVER_VERSION}" + cp "${NVIDIA_CACHE}/firmware/"* "${OVERLAY_STAGE_DIR}/usr/lib/firmware/nvidia/${NVIDIA_DRIVER_VERSION}/" + echo "=== firmware: $(ls "${OVERLAY_STAGE_DIR}/usr/lib/firmware/nvidia/${NVIDIA_DRIVER_VERSION}/" | wc -l) files injected ===" fi # --- build / download NCCL --- @@ -642,6 +648,7 @@ if ! needs_full_build; then validate_iso_live_boot_entries "$ISO_RAW" validate_iso_grub_assets "$ISO_RAW" validate_iso_nvidia_runtime "$ISO_RAW" + validate_iso_rootfs_layout "$ISO_RAW" cp "$ISO_RAW" "$ISO_OUT" echo "" echo "=== done (${BUILD_VARIANT}, fast-path) ===" @@ -703,6 +710,7 @@ if [ -f "$ISO_RAW" ]; then validate_iso_live_boot_entries "$ISO_RAW" validate_iso_grub_assets "$ISO_RAW" validate_iso_nvidia_runtime "$ISO_RAW" + validate_iso_rootfs_layout "$ISO_RAW" cp "$ISO_RAW" "$ISO_OUT" hash_heavy_config > "${FULL_BUILD_HASH_FILE}.new" printf '%s\n' "${DEBIAN_KERNEL_ABI}" > "${FULL_BUILD_ABI_FILE}.new" diff --git a/iso/builder/lib/fast-path.sh b/iso/builder/lib/fast-path.sh index 03ffcf0..855c711 100755 --- a/iso/builder/lib/fast-path.sh +++ b/iso/builder/lib/fast-path.sh @@ -135,7 +135,12 @@ fast_path_repack_squashfs() ( echo "=== fast-path: unsquash $(basename "$_old_sq") ($(du -sh "$_old_sq" | cut -f1) compressed) ===" unsquashfs -d "$_tmp" "$_old_sq" echo "=== fast-path: syncing overlay stage ===" - rsync -a --checksum "${OVERLAY_STAGE_DIR}/" "$_tmp/" + # --keep-dirlinks: when the extracted root has a symlink-to-directory (e.g. + # merged-usr /lib -> usr/lib) and the overlay stage carries a real directory + # of the same name, follow the symlink instead of replacing it with a plain + # directory. Without this rsync silently orphans everything reachable only + # through that symlink and the resulting image panics at boot. + rsync -a --keep-dirlinks --checksum "${OVERLAY_STAGE_DIR}/" "$_tmp/" echo "=== fast-path: repacking as ${SQUASHFS_FILENAME} ===" _sq_new="${_sq}.new" rm -f "$_sq_new" diff --git a/iso/builder/lib/iso-validation.sh b/iso/builder/lib/iso-validation.sh index 71a0289..c5a492f 100755 --- a/iso/builder/lib/iso-validation.sh +++ b/iso/builder/lib/iso-validation.sh @@ -639,3 +639,64 @@ validate_iso_nvidia_runtime() { rm -rf "$dpkg_status_dir" echo "=== NVIDIA runtime validation OK ===" } + +rootfs_layout_fail() { + echo "ERROR: $1" >&2 + exit 1 +} + +# Guard against a corrupted merged-usr layout in the live rootfs. On Debian +# bookworm /bin, /sbin, /lib and /lib64 are symlinks into /usr; if a build step +# stages a real directory of that name into the overlay, `rsync -a` replaces the +# symlink with a plain directory and orphans everything reachable only through +# it (/lib/systemd/systemd, /lib/x86_64-linux-gnu/*, ...). The image still packs +# and mounts fine but panics at boot: "run-init: can't execute '/sbin/init'". +validate_iso_rootfs_layout() { + iso_path="$1" + echo "=== validating rootfs merged-usr layout in ISO ===" + + [ -f "$iso_path" ] || rootfs_layout_fail "ISO not found for rootfs layout validation: $iso_path" + require_iso_reader "$iso_path" >/dev/null 2>&1 || rootfs_layout_fail "ISO reader unavailable for rootfs layout validation" + command -v unsquashfs >/dev/null 2>&1 || rootfs_layout_fail "unsquashfs is required for rootfs layout validation" + + _sq_tmp="$(mktemp)" + _sq_list="$(mktemp)" + _iso_files="$(mktemp)" + + iso_list_files "$iso_path" > "$_iso_files" \ + || rootfs_layout_fail "failed to list ISO files for rootfs layout validation" + + _saw_rootfs=0 + while IFS= read -r _member; do + [ -n "$_member" ] || continue + iso_read_member "$iso_path" "$_member" "$_sq_tmp" \ + || rootfs_layout_fail "failed to extract $_member from ISO" + : > "$_sq_list" + unsquashfs -ll "$_sq_tmp" > "$_sq_list" 2>/dev/null \ + || rootfs_layout_fail "failed to inspect $_member from ISO" + + # A plain-directory top-level bin/sbin/lib/lib64 is always wrong here. + _bad="$(grep -E '^d[rwxsStT-]{9} .* squashfs-root/(bin|sbin|lib|lib64)$' "$_sq_list" || true)" + if [ -n "$_bad" ]; then + echo "$_bad" >&2 + rootfs_layout_fail "$_member: /bin|/sbin|/lib|/lib64 is a plain directory, not a merged-usr symlink (overlay clobbered it; stage under usr/lib instead)" + fi + + # In the layer that carries the base rootfs, assert init is resolvable. + if grep -Eq ' squashfs-root/usr/sbin/init( ->|$)' "$_sq_list"; then + _saw_rootfs=1 + grep -Eq ' squashfs-root/usr/lib/systemd/systemd$' "$_sq_list" \ + || rootfs_layout_fail "$_member: /usr/sbin/init present but /usr/lib/systemd/systemd missing" + grep -Eq ' squashfs-root/lib -> ' "$_sq_list" \ + || rootfs_layout_fail "$_member: /lib is not a symlink into /usr (merged-usr broken)" + fi + done <