fix(iso): let empty directories survive the squashfs layer split
bee_layer_classify only ever tracked regular files and symlinks (`find ... -type f -o -type l`), so any directory that is empty at build time — like /var/log/nvidia-dcgm, correctly created and chowned by the datacenter-gpu-manager postinst — was silently dropped from every layer's rsync --files-from list and never reached the built ISO. This is why bbc6fb1's78d1b9bfollow-up (seeding a marker file just for that one path) kept the directory alive: it was a targeted workaround for a general gap in the classifier, not a fix of it. Replace that workaround with the general mechanism: classify also walks every directory, computes the subset that is empty all the way down (no file or symlink anywhere in its subtree — a directory that does hold files needs no entry, rsync already recreates it as an implied parent), and assigns each one to a layer via the same dpkg-ownership / injected-rule precedence used for files. Add an injected rule routing /var/log/nvidia-dcgm to 20-nvidia-platform, alongside the DCGM binaries that actually use it, instead of letting it fall through to base by default. bee_layer_build folds each layer's empty-dir list into the same rsync --files-from call; recursion into a directory that is by-construction empty copies nothing extra. Revert the 9000/9999 hook changes from78d1b9bnow that they're redundant, and cover the new path with test-squashfs-layers.sh (ruled, unruled, and nested-empty directories, asserted present in the merged rootfs after a real mksquashfs/unsquashfs round-trip). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 5
@@ -135,6 +135,7 @@ bee_layer_injected_rules() {
|
||||
^usr/local/bin/bee-check-nvswitch$ 10-nvidia-driver
|
||||
^etc/systemd/system/bee-nvidia\.service$ 10-nvidia-driver
|
||||
^etc/systemd/system/nvidia-fabricmanager\.service\.d/ 20-nvidia-platform
|
||||
^var/log/nvidia-dcgm$ 20-nvidia-platform
|
||||
^usr/lib/libnccl\.so(\..*)?$ 30-nvidia-cuda-libs
|
||||
^usr/lib/libcublas\.so(\..*)?$ 30-nvidia-cuda-libs
|
||||
^usr/lib/libcublasLt\.so(\..*)?$ 30-nvidia-cuda-libs
|
||||
@@ -161,7 +162,11 @@ bee_layer_canon_paths() {
|
||||
|
||||
# bee_layer_classify <rootfs_dir> <variant> <work_dir>
|
||||
# Partition every regular file and symlink under <rootfs_dir> into exactly one
|
||||
# layer. Writes <work_dir>/<slug>.files (LC_ALL=C sorted) and
|
||||
# layer, plus every directory that is empty all the way down (no file or
|
||||
# symlink anywhere in its subtree) - a directory that DOES contain files
|
||||
# needs no entry of its own, since rsync recreates it as an implied parent
|
||||
# of whichever layer(s) its files land in. Writes <work_dir>/<slug>.files and
|
||||
# <work_dir>/<slug>.dirs (both LC_ALL=C sorted) and
|
||||
# <work_dir>/classify-report.txt. Runs as a subshell: internal state cannot
|
||||
# leak into the caller, and any step failure aborts with a non-zero exit.
|
||||
bee_layer_classify() (
|
||||
@@ -182,7 +187,22 @@ bee_layer_classify() (
|
||||
( cd "$_root" && find . -mindepth 1 \( -type f -o -type l \) -printf '%P\n' ) \
|
||||
| LC_ALL=C sort > "$_wd/all.files"
|
||||
|
||||
# 2. dpkg-owned files that belong to a non-base layer.
|
||||
# 1b. Every directory, and the subset of those that are empty all the way
|
||||
# down. A directory holding files elsewhere in its subtree is already
|
||||
# covered by rsync's implied-parent handling in bee_layer_build; only
|
||||
# a fully-empty branch needs an explicit entry, or it is silently
|
||||
# dropped from every layer (it owns no file or symlink for step 1 to
|
||||
# ever see).
|
||||
( cd "$_root" && find . -mindepth 1 -type d -printf '%P\n' ) \
|
||||
| LC_ALL=C sort > "$_wd/all.dirs"
|
||||
awk -F/ '{
|
||||
p = $1
|
||||
print p
|
||||
for (i = 2; i < NF; i++) { p = p "/" $i; print p }
|
||||
}' "$_wd/all.files" | LC_ALL=C sort -u > "$_wd/nonempty.dirs"
|
||||
LC_ALL=C comm -23 "$_wd/all.dirs" "$_wd/nonempty.dirs" > "$_wd/empty.dirs"
|
||||
|
||||
# 2. dpkg-owned files (and directories) that belong to a non-base layer.
|
||||
: > "$_wd/dpkg.map"
|
||||
for _list in "$_root"/var/lib/dpkg/info/*.list; do
|
||||
[ -f "$_list" ] || continue
|
||||
@@ -195,11 +215,11 @@ bee_layer_classify() (
|
||||
| awk -v l="$_layer" -v t="$_tab" 'NF { print $0 t l }' >> "$_wd/dpkg.map"
|
||||
done
|
||||
|
||||
# 3. Injected (no-deb) files, by explicit rule. These override dpkg.
|
||||
# 3. Injected (no-deb) files/directories, by explicit rule. These override dpkg.
|
||||
: > "$_wd/injected.map"
|
||||
bee_layer_injected_rules | while IFS="$_tab" read -r _re _layer; do
|
||||
[ -n "$_re" ] || continue
|
||||
LC_ALL=C grep -E "$_re" "$_wd/all.files" \
|
||||
LC_ALL=C grep -hE "$_re" "$_wd/all.files" "$_wd/all.dirs" \
|
||||
| awk -v l="$_layer" -v t="$_tab" '{ print $0 t l }' >> "$_wd/injected.map" || true
|
||||
done
|
||||
|
||||
@@ -207,23 +227,33 @@ bee_layer_classify() (
|
||||
# Drop the merged-usr compat symlinks (bin/sbin/lib/lib64) unconditionally
|
||||
# - a bare "/lib" entry appears in some dpkg .list files (firmware-*), and
|
||||
# those symlinks must always stay in the base layer.
|
||||
# Then keep only rows that name a real file or symlink in the tree: dpkg
|
||||
# .list files also record bare directories, which must never reach an
|
||||
# rsync --files-from list (rsync would copy the directory recursively).
|
||||
# candidate.map covers files and directories alike; it is joined
|
||||
# against all.files below for the file assignment, and separately
|
||||
# against empty.dirs for the directory assignment. A dpkg .list
|
||||
# directory entry that isn't empty simply fails that second join and
|
||||
# is dropped - it never reaches an rsync --files-from list (rsync
|
||||
# would otherwise copy it, and everything under it, recursively).
|
||||
cat "$_wd/injected.map" "$_wd/dpkg.map" \
|
||||
| awk -F'\t' '$1!="bin" && $1!="sbin" && $1!="lib" && $1!="lib64" && !seen[$1]++ { print }' \
|
||||
| LC_ALL=C sort -t "$_tab" -k1,1 > "$_wd/candidate.map"
|
||||
LC_ALL=C join -t "$_tab" -j 1 "$_wd/all.files" "$_wd/candidate.map" \
|
||||
> "$_wd/assigned.map" || true
|
||||
LC_ALL=C join -t "$_tab" -j 1 "$_wd/empty.dirs" "$_wd/candidate.map" \
|
||||
> "$_wd/assigned-dirs.map" || true
|
||||
|
||||
# 5. Split into per-layer file lists; base gets the remainder.
|
||||
for _slug in $_slugs; do : > "$_wd/$_slug.files"; done
|
||||
# 5. Split into per-layer file/dir lists; base gets the remainder of each.
|
||||
for _slug in $_slugs; do : > "$_wd/$_slug.files"; : > "$_wd/$_slug.dirs"; done
|
||||
awk -F'\t' -v wd="$_wd" '{ print $1 >> (wd "/" $2 ".files") }' "$_wd/assigned.map"
|
||||
awk -F'\t' -v wd="$_wd" '{ print $1 >> (wd "/" $2 ".dirs") }' "$_wd/assigned-dirs.map"
|
||||
for _slug in $_slugs; do
|
||||
LC_ALL=C sort -o "$_wd/$_slug.files" "$_wd/$_slug.files"
|
||||
LC_ALL=C sort -o "$_wd/$_slug.dirs" "$_wd/$_slug.dirs"
|
||||
done
|
||||
cut -f1 "$_wd/assigned.map" | LC_ALL=C sort > "$_wd/assigned.paths"
|
||||
LC_ALL=C comm -23 "$_wd/all.files" "$_wd/assigned.paths" > "$_wd/00-base.files"
|
||||
cut -f1 "$_wd/assigned-dirs.map" | LC_ALL=C sort > "$_wd/assigned-dirs.paths"
|
||||
LC_ALL=C comm -23 "$_wd/empty.dirs" "$_wd/assigned-dirs.paths" >> "$_wd/00-base.dirs"
|
||||
LC_ALL=C sort -o "$_wd/00-base.dirs" "$_wd/00-base.dirs"
|
||||
|
||||
# 6. Completeness / disjointness.
|
||||
_total="$(wc -l < "$_wd/all.files")"
|
||||
@@ -231,8 +261,9 @@ bee_layer_classify() (
|
||||
: > "$_wd/classify-report.txt"
|
||||
for _slug in $_slugs; do
|
||||
_n="$(wc -l < "$_wd/$_slug.files")"
|
||||
_nd="$(wc -l < "$_wd/$_slug.dirs")"
|
||||
_sum=$((_sum + _n))
|
||||
printf '%-20s %8d files\n' "$_slug" "$_n" >> "$_wd/classify-report.txt"
|
||||
printf '%-20s %8d files %6d empty dirs\n' "$_slug" "$_n" "$_nd" >> "$_wd/classify-report.txt"
|
||||
done
|
||||
printf '%-20s %8d files\n' "TOTAL" "$_sum" >> "$_wd/classify-report.txt"
|
||||
printf '%-20s %8d files\n' "tree" "$_total" >> "$_wd/classify-report.txt"
|
||||
@@ -249,6 +280,15 @@ bee_layer_classify() (
|
||||
printf '%s\n' "$_dups" | head >&2
|
||||
exit 1
|
||||
fi
|
||||
_dirtotal="$(wc -l < "$_wd/empty.dirs")"
|
||||
_dirsum=0
|
||||
for _slug in $_slugs; do
|
||||
_dirsum=$((_dirsum + $(wc -l < "$_wd/$_slug.dirs")))
|
||||
done
|
||||
if [ "$_dirsum" -ne "$_dirtotal" ]; then
|
||||
echo "bee_layer_classify: partition covers $_dirsum of $_dirtotal empty dirs" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 7. merged-usr guard.
|
||||
for _link in bin sbin lib lib64; do
|
||||
@@ -278,6 +318,7 @@ bee_layer_build() (
|
||||
|
||||
for _slug in $_slugs; do
|
||||
_list="$_wd/$_slug.files"
|
||||
_dirlist="$_wd/$_slug.dirs"
|
||||
[ -s "$_list" ] || { echo "bee_layer_build: empty file list for $_slug" >&2; exit 1; }
|
||||
_stage="$_wd/stage-$_slug"
|
||||
_sq="$_out/filesystem-v${_ver}-${_slug}.squashfs"
|
||||
@@ -286,8 +327,11 @@ bee_layer_build() (
|
||||
# --files-from with -a recreates implied parent directories from the
|
||||
# source but copies only listed entries. --link-dest hardlinks unchanged
|
||||
# files from the source tree (same fs) so a 5 GB rootfs is not physically
|
||||
# copied once per layer.
|
||||
rsync -a --link-dest="$_root" --files-from="$_list" "$_root/" "$_stage/"
|
||||
# copied once per layer. $_dirlist entries are directories that are
|
||||
# empty all the way down (bee_layer_classify guarantees this), so
|
||||
# -a's implicit recursion into them copies nothing extra.
|
||||
cat "$_list" "$_dirlist" > "$_wd/$_slug.rsync-list"
|
||||
rsync -a --link-dest="$_root" --files-from="$_wd/$_slug.rsync-list" "$_root/" "$_stage/"
|
||||
rm -f "$_sq"
|
||||
# shellcheck disable=SC2086
|
||||
mksquashfs "$_stage" "$_sq" $BEE_LAYER_MKSQUASHFS_OPTS
|
||||
|
||||
Reference in New Issue
Block a user