fix(iso): let empty directories survive the squashfs layer split
bee_layer_classify only ever tracked regular files and symlinks (`find ... -type f -o -type l`), so any directory that is empty at build time — like /var/log/nvidia-dcgm, correctly created and chowned by the datacenter-gpu-manager postinst — was silently dropped from every layer's rsync --files-from list and never reached the built ISO. This is why bbc6fb1's78d1b9bfollow-up (seeding a marker file just for that one path) kept the directory alive: it was a targeted workaround for a general gap in the classifier, not a fix of it. Replace that workaround with the general mechanism: classify also walks every directory, computes the subset that is empty all the way down (no file or symlink anywhere in its subtree — a directory that does hold files needs no entry, rsync already recreates it as an implied parent), and assigns each one to a layer via the same dpkg-ownership / injected-rule precedence used for files. Add an injected rule routing /var/log/nvidia-dcgm to 20-nvidia-platform, alongside the DCGM binaries that actually use it, instead of letting it fall through to base by default. bee_layer_build folds each layer's empty-dir list into the same rsync --files-from call; recursion into a directory that is by-construction empty copies nothing extra. Revert the 9000/9999 hook changes from78d1b9bnow that they're redundant, and cover the new path with test-squashfs-layers.sh (ruled, unruled, and nested-empty directories, asserted present in the merged rootfs after a real mksquashfs/unsquashfs round-trip). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 5
@@ -46,25 +46,6 @@ if [ "$GPU_VENDOR" = "nvidia" ]; then
|
||||
systemctl enable nvidia-dcgm.service 2>/dev/null || true
|
||||
systemctl enable nvidia-fabricmanager.service 2>/dev/null || true
|
||||
systemctl enable bee-nvidia.service
|
||||
|
||||
# nvidia-dcgm.service runs `nv-hostengine --service-account nvidia-dcgm`,
|
||||
# which drops privileges to that account for all file I/O and writes
|
||||
# diagnostics to DCGM_HOME_DIR (/var/log/nvidia-dcgm, per the packaged
|
||||
# unit's Environment=). The package's own postinst already creates and
|
||||
# chowns this directory correctly — but it's empty at that point, and
|
||||
# lib/squashfs-layers.sh's classifier only tracks regular files and
|
||||
# symlinks (`find ... -type f -o -type l`), so an empty directory is
|
||||
# silently dropped from every layer's rsync --files-from list and never
|
||||
# reaches the built ISO at all. `dcgmi diag`'s deployment check then
|
||||
# fails with DCGM_FR_FILE_CREATE_PERMISSIONS at boot because the
|
||||
# directory doesn't exist, regardless of ownership. Re-create it and
|
||||
# seed one real file so it rides along the classifier and rsync -a
|
||||
# recreates the parent directory (with this ownership) as an implied
|
||||
# parent.
|
||||
if id nvidia-dcgm >/dev/null 2>&1; then
|
||||
install -d -o nvidia-dcgm -g nvidia-dcgm -m 0755 /var/log/nvidia-dcgm
|
||||
install -o nvidia-dcgm -g nvidia-dcgm -m 0644 /dev/null /var/log/nvidia-dcgm/.keep
|
||||
fi
|
||||
elif [ "$GPU_VENDOR" = "amd" ]; then
|
||||
# ROCm symlinks (packages install to /opt/rocm-*/bin/)
|
||||
for tool in rocm-smi rocm-bandwidth-test rvs; do
|
||||
|
||||
@@ -27,9 +27,6 @@ rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# ── Misc ──────────────────────────────────────────────────────────────────────
|
||||
rm -rf /tmp/* /var/tmp/* 2>/dev/null || true
|
||||
# /var/log/nvidia-dcgm/.keep is intentionally seeded by 9000-bee-setup so an
|
||||
# otherwise-empty DCGM_HOME_DIR survives the squashfs layer classifier (which
|
||||
# only tracks files/symlinks, not empty directories) — never sweep it up here.
|
||||
find /var/log -type f ! -path '/var/log/nvidia-dcgm/.keep' -delete 2>/dev/null || true
|
||||
find /var/log -type f -delete 2>/dev/null || true
|
||||
|
||||
echo "=== slim: done ==="
|
||||
|
||||
@@ -135,6 +135,7 @@ bee_layer_injected_rules() {
|
||||
^usr/local/bin/bee-check-nvswitch$ 10-nvidia-driver
|
||||
^etc/systemd/system/bee-nvidia\.service$ 10-nvidia-driver
|
||||
^etc/systemd/system/nvidia-fabricmanager\.service\.d/ 20-nvidia-platform
|
||||
^var/log/nvidia-dcgm$ 20-nvidia-platform
|
||||
^usr/lib/libnccl\.so(\..*)?$ 30-nvidia-cuda-libs
|
||||
^usr/lib/libcublas\.so(\..*)?$ 30-nvidia-cuda-libs
|
||||
^usr/lib/libcublasLt\.so(\..*)?$ 30-nvidia-cuda-libs
|
||||
@@ -161,7 +162,11 @@ bee_layer_canon_paths() {
|
||||
|
||||
# bee_layer_classify <rootfs_dir> <variant> <work_dir>
|
||||
# Partition every regular file and symlink under <rootfs_dir> into exactly one
|
||||
# layer. Writes <work_dir>/<slug>.files (LC_ALL=C sorted) and
|
||||
# layer, plus every directory that is empty all the way down (no file or
|
||||
# symlink anywhere in its subtree) - a directory that DOES contain files
|
||||
# needs no entry of its own, since rsync recreates it as an implied parent
|
||||
# of whichever layer(s) its files land in. Writes <work_dir>/<slug>.files and
|
||||
# <work_dir>/<slug>.dirs (both LC_ALL=C sorted) and
|
||||
# <work_dir>/classify-report.txt. Runs as a subshell: internal state cannot
|
||||
# leak into the caller, and any step failure aborts with a non-zero exit.
|
||||
bee_layer_classify() (
|
||||
@@ -182,7 +187,22 @@ bee_layer_classify() (
|
||||
( cd "$_root" && find . -mindepth 1 \( -type f -o -type l \) -printf '%P\n' ) \
|
||||
| LC_ALL=C sort > "$_wd/all.files"
|
||||
|
||||
# 2. dpkg-owned files that belong to a non-base layer.
|
||||
# 1b. Every directory, and the subset of those that are empty all the way
|
||||
# down. A directory holding files elsewhere in its subtree is already
|
||||
# covered by rsync's implied-parent handling in bee_layer_build; only
|
||||
# a fully-empty branch needs an explicit entry, or it is silently
|
||||
# dropped from every layer (it owns no file or symlink for step 1 to
|
||||
# ever see).
|
||||
( cd "$_root" && find . -mindepth 1 -type d -printf '%P\n' ) \
|
||||
| LC_ALL=C sort > "$_wd/all.dirs"
|
||||
awk -F/ '{
|
||||
p = $1
|
||||
print p
|
||||
for (i = 2; i < NF; i++) { p = p "/" $i; print p }
|
||||
}' "$_wd/all.files" | LC_ALL=C sort -u > "$_wd/nonempty.dirs"
|
||||
LC_ALL=C comm -23 "$_wd/all.dirs" "$_wd/nonempty.dirs" > "$_wd/empty.dirs"
|
||||
|
||||
# 2. dpkg-owned files (and directories) that belong to a non-base layer.
|
||||
: > "$_wd/dpkg.map"
|
||||
for _list in "$_root"/var/lib/dpkg/info/*.list; do
|
||||
[ -f "$_list" ] || continue
|
||||
@@ -195,11 +215,11 @@ bee_layer_classify() (
|
||||
| awk -v l="$_layer" -v t="$_tab" 'NF { print $0 t l }' >> "$_wd/dpkg.map"
|
||||
done
|
||||
|
||||
# 3. Injected (no-deb) files, by explicit rule. These override dpkg.
|
||||
# 3. Injected (no-deb) files/directories, by explicit rule. These override dpkg.
|
||||
: > "$_wd/injected.map"
|
||||
bee_layer_injected_rules | while IFS="$_tab" read -r _re _layer; do
|
||||
[ -n "$_re" ] || continue
|
||||
LC_ALL=C grep -E "$_re" "$_wd/all.files" \
|
||||
LC_ALL=C grep -hE "$_re" "$_wd/all.files" "$_wd/all.dirs" \
|
||||
| awk -v l="$_layer" -v t="$_tab" '{ print $0 t l }' >> "$_wd/injected.map" || true
|
||||
done
|
||||
|
||||
@@ -207,23 +227,33 @@ bee_layer_classify() (
|
||||
# Drop the merged-usr compat symlinks (bin/sbin/lib/lib64) unconditionally
|
||||
# - a bare "/lib" entry appears in some dpkg .list files (firmware-*), and
|
||||
# those symlinks must always stay in the base layer.
|
||||
# Then keep only rows that name a real file or symlink in the tree: dpkg
|
||||
# .list files also record bare directories, which must never reach an
|
||||
# rsync --files-from list (rsync would copy the directory recursively).
|
||||
# candidate.map covers files and directories alike; it is joined
|
||||
# against all.files below for the file assignment, and separately
|
||||
# against empty.dirs for the directory assignment. A dpkg .list
|
||||
# directory entry that isn't empty simply fails that second join and
|
||||
# is dropped - it never reaches an rsync --files-from list (rsync
|
||||
# would otherwise copy it, and everything under it, recursively).
|
||||
cat "$_wd/injected.map" "$_wd/dpkg.map" \
|
||||
| awk -F'\t' '$1!="bin" && $1!="sbin" && $1!="lib" && $1!="lib64" && !seen[$1]++ { print }' \
|
||||
| LC_ALL=C sort -t "$_tab" -k1,1 > "$_wd/candidate.map"
|
||||
LC_ALL=C join -t "$_tab" -j 1 "$_wd/all.files" "$_wd/candidate.map" \
|
||||
> "$_wd/assigned.map" || true
|
||||
LC_ALL=C join -t "$_tab" -j 1 "$_wd/empty.dirs" "$_wd/candidate.map" \
|
||||
> "$_wd/assigned-dirs.map" || true
|
||||
|
||||
# 5. Split into per-layer file lists; base gets the remainder.
|
||||
for _slug in $_slugs; do : > "$_wd/$_slug.files"; done
|
||||
# 5. Split into per-layer file/dir lists; base gets the remainder of each.
|
||||
for _slug in $_slugs; do : > "$_wd/$_slug.files"; : > "$_wd/$_slug.dirs"; done
|
||||
awk -F'\t' -v wd="$_wd" '{ print $1 >> (wd "/" $2 ".files") }' "$_wd/assigned.map"
|
||||
awk -F'\t' -v wd="$_wd" '{ print $1 >> (wd "/" $2 ".dirs") }' "$_wd/assigned-dirs.map"
|
||||
for _slug in $_slugs; do
|
||||
LC_ALL=C sort -o "$_wd/$_slug.files" "$_wd/$_slug.files"
|
||||
LC_ALL=C sort -o "$_wd/$_slug.dirs" "$_wd/$_slug.dirs"
|
||||
done
|
||||
cut -f1 "$_wd/assigned.map" | LC_ALL=C sort > "$_wd/assigned.paths"
|
||||
LC_ALL=C comm -23 "$_wd/all.files" "$_wd/assigned.paths" > "$_wd/00-base.files"
|
||||
cut -f1 "$_wd/assigned-dirs.map" | LC_ALL=C sort > "$_wd/assigned-dirs.paths"
|
||||
LC_ALL=C comm -23 "$_wd/empty.dirs" "$_wd/assigned-dirs.paths" >> "$_wd/00-base.dirs"
|
||||
LC_ALL=C sort -o "$_wd/00-base.dirs" "$_wd/00-base.dirs"
|
||||
|
||||
# 6. Completeness / disjointness.
|
||||
_total="$(wc -l < "$_wd/all.files")"
|
||||
@@ -231,8 +261,9 @@ bee_layer_classify() (
|
||||
: > "$_wd/classify-report.txt"
|
||||
for _slug in $_slugs; do
|
||||
_n="$(wc -l < "$_wd/$_slug.files")"
|
||||
_nd="$(wc -l < "$_wd/$_slug.dirs")"
|
||||
_sum=$((_sum + _n))
|
||||
printf '%-20s %8d files\n' "$_slug" "$_n" >> "$_wd/classify-report.txt"
|
||||
printf '%-20s %8d files %6d empty dirs\n' "$_slug" "$_n" "$_nd" >> "$_wd/classify-report.txt"
|
||||
done
|
||||
printf '%-20s %8d files\n' "TOTAL" "$_sum" >> "$_wd/classify-report.txt"
|
||||
printf '%-20s %8d files\n' "tree" "$_total" >> "$_wd/classify-report.txt"
|
||||
@@ -249,6 +280,15 @@ bee_layer_classify() (
|
||||
printf '%s\n' "$_dups" | head >&2
|
||||
exit 1
|
||||
fi
|
||||
_dirtotal="$(wc -l < "$_wd/empty.dirs")"
|
||||
_dirsum=0
|
||||
for _slug in $_slugs; do
|
||||
_dirsum=$((_dirsum + $(wc -l < "$_wd/$_slug.dirs")))
|
||||
done
|
||||
if [ "$_dirsum" -ne "$_dirtotal" ]; then
|
||||
echo "bee_layer_classify: partition covers $_dirsum of $_dirtotal empty dirs" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 7. merged-usr guard.
|
||||
for _link in bin sbin lib lib64; do
|
||||
@@ -278,6 +318,7 @@ bee_layer_build() (
|
||||
|
||||
for _slug in $_slugs; do
|
||||
_list="$_wd/$_slug.files"
|
||||
_dirlist="$_wd/$_slug.dirs"
|
||||
[ -s "$_list" ] || { echo "bee_layer_build: empty file list for $_slug" >&2; exit 1; }
|
||||
_stage="$_wd/stage-$_slug"
|
||||
_sq="$_out/filesystem-v${_ver}-${_slug}.squashfs"
|
||||
@@ -286,8 +327,11 @@ bee_layer_build() (
|
||||
# --files-from with -a recreates implied parent directories from the
|
||||
# source but copies only listed entries. --link-dest hardlinks unchanged
|
||||
# files from the source tree (same fs) so a 5 GB rootfs is not physically
|
||||
# copied once per layer.
|
||||
rsync -a --link-dest="$_root" --files-from="$_list" "$_root/" "$_stage/"
|
||||
# copied once per layer. $_dirlist entries are directories that are
|
||||
# empty all the way down (bee_layer_classify guarantees this), so
|
||||
# -a's implicit recursion into them copies nothing extra.
|
||||
cat "$_list" "$_dirlist" > "$_wd/$_slug.rsync-list"
|
||||
rsync -a --link-dest="$_root" --files-from="$_wd/$_slug.rsync-list" "$_root/" "$_stage/"
|
||||
rm -f "$_sq"
|
||||
# shellcheck disable=SC2086
|
||||
mksquashfs "$_stage" "$_sq" $BEE_LAYER_MKSQUASHFS_OPTS
|
||||
|
||||
@@ -98,8 +98,15 @@ mk usr/local/bin/bee-dcgmproftester-staggered "#!stag"
|
||||
# unruled file under an nvidia dir stays in base.
|
||||
mk usr/share/doc/nvidia-random/README "notes"
|
||||
|
||||
# --- empty directories: no file/symlink anywhere in their subtree, so they
|
||||
# never appear in all.files and must be classified/carried separately.
|
||||
mkdir -p "$ROOT/var/log/nvidia-dcgm" # ruled -> 20-nvidia-platform
|
||||
mkdir -p "$ROOT/var/lib/bee-empty-unruled" # unruled -> base
|
||||
mkdir -p "$ROOT/var/lib/bee-empty-nested/still-empty" # nested empty branch -> base
|
||||
|
||||
fail() { echo "FAIL: $*" >&2; exit 1; }
|
||||
in_layer() { LC_ALL=C grep -qx "$2" "$WD/$1.files" || fail "expected $2 in layer $1"; }
|
||||
in_dir_layer() { LC_ALL=C grep -qx "$2" "$WD/$1.dirs" || fail "expected empty dir $2 in layer $1"; }
|
||||
|
||||
MONO="$T/filesystem-v14.99.squashfs"
|
||||
mksquashfs "$ROOT" "$MONO" -comp zstd -noappend -no-progress -no-xattrs >/dev/null
|
||||
@@ -146,11 +153,20 @@ in_layer 40-nvidia-dcgm-cuda usr/bin/dcgmproftester12
|
||||
in_layer 40-nvidia-dcgm-cuda usr/local/bin/bee-dcgmproftester-staggered
|
||||
in_layer 00-base usr/share/doc/nvidia-random/README
|
||||
|
||||
in_dir_layer 20-nvidia-platform var/log/nvidia-dcgm
|
||||
in_dir_layer 00-base var/lib/bee-empty-unruled
|
||||
in_dir_layer 00-base var/lib/bee-empty-nested
|
||||
in_dir_layer 00-base var/lib/bee-empty-nested/still-empty
|
||||
|
||||
OUT="$T/out"
|
||||
bee_layer_build "$R2" "$WD" "$OUT" 14.99 nvidia
|
||||
bee_layer_verify_each "$OUT" 14.99 nvidia
|
||||
bee_layer_merge "$OUT" 14.99 nvidia "$T/merged"
|
||||
|
||||
[ -d "$T/merged/var/log/nvidia-dcgm" ] || fail "empty dir var/log/nvidia-dcgm missing from merged rootfs"
|
||||
[ -d "$T/merged/var/lib/bee-empty-unruled" ] || fail "empty dir var/lib/bee-empty-unruled missing from merged rootfs"
|
||||
[ -d "$T/merged/var/lib/bee-empty-nested/still-empty" ] || fail "nested empty dir missing from merged rootfs"
|
||||
|
||||
MODLIVE="$T/live"; mkdir -p "$MODLIVE"
|
||||
cp "$OUT"/filesystem-v14.99-*.squashfs "$MODLIVE/"
|
||||
bee_layer_write_module_file "$MODLIVE" 14.99 nvidia
|
||||
|
||||
Reference in New Issue
Block a user