diff --git a/iso/builder/config/hooks/normal/9000-bee-setup.hook.chroot b/iso/builder/config/hooks/normal/9000-bee-setup.hook.chroot index a245eac..7e2ba00 100755 --- a/iso/builder/config/hooks/normal/9000-bee-setup.hook.chroot +++ b/iso/builder/config/hooks/normal/9000-bee-setup.hook.chroot @@ -46,25 +46,6 @@ if [ "$GPU_VENDOR" = "nvidia" ]; then systemctl enable nvidia-dcgm.service 2>/dev/null || true systemctl enable nvidia-fabricmanager.service 2>/dev/null || true systemctl enable bee-nvidia.service - - # nvidia-dcgm.service runs `nv-hostengine --service-account nvidia-dcgm`, - # which drops privileges to that account for all file I/O and writes - # diagnostics to DCGM_HOME_DIR (/var/log/nvidia-dcgm, per the packaged - # unit's Environment=). The package's own postinst already creates and - # chowns this directory correctly — but it's empty at that point, and - # lib/squashfs-layers.sh's classifier only tracks regular files and - # symlinks (`find ... -type f -o -type l`), so an empty directory is - # silently dropped from every layer's rsync --files-from list and never - # reaches the built ISO at all. `dcgmi diag`'s deployment check then - # fails with DCGM_FR_FILE_CREATE_PERMISSIONS at boot because the - # directory doesn't exist, regardless of ownership. Re-create it and - # seed one real file so it rides along the classifier and rsync -a - # recreates the parent directory (with this ownership) as an implied - # parent. - if id nvidia-dcgm >/dev/null 2>&1; then - install -d -o nvidia-dcgm -g nvidia-dcgm -m 0755 /var/log/nvidia-dcgm - install -o nvidia-dcgm -g nvidia-dcgm -m 0644 /dev/null /var/log/nvidia-dcgm/.keep - fi elif [ "$GPU_VENDOR" = "amd" ]; then # ROCm symlinks (packages install to /opt/rocm-*/bin/) for tool in rocm-smi rocm-bandwidth-test rvs; do diff --git a/iso/builder/config/hooks/normal/9999-slim.hook.chroot b/iso/builder/config/hooks/normal/9999-slim.hook.chroot index 25642d3..f9bea21 100755 --- a/iso/builder/config/hooks/normal/9999-slim.hook.chroot +++ b/iso/builder/config/hooks/normal/9999-slim.hook.chroot @@ -27,9 +27,6 @@ rm -rf /var/lib/apt/lists/* # ── Misc ────────────────────────────────────────────────────────────────────── rm -rf /tmp/* /var/tmp/* 2>/dev/null || true -# /var/log/nvidia-dcgm/.keep is intentionally seeded by 9000-bee-setup so an -# otherwise-empty DCGM_HOME_DIR survives the squashfs layer classifier (which -# only tracks files/symlinks, not empty directories) — never sweep it up here. -find /var/log -type f ! -path '/var/log/nvidia-dcgm/.keep' -delete 2>/dev/null || true +find /var/log -type f -delete 2>/dev/null || true echo "=== slim: done ===" diff --git a/iso/builder/lib/squashfs-layers.sh b/iso/builder/lib/squashfs-layers.sh index 42d43de..9d825f6 100644 --- a/iso/builder/lib/squashfs-layers.sh +++ b/iso/builder/lib/squashfs-layers.sh @@ -135,6 +135,7 @@ bee_layer_injected_rules() { ^usr/local/bin/bee-check-nvswitch$ 10-nvidia-driver ^etc/systemd/system/bee-nvidia\.service$ 10-nvidia-driver ^etc/systemd/system/nvidia-fabricmanager\.service\.d/ 20-nvidia-platform +^var/log/nvidia-dcgm$ 20-nvidia-platform ^usr/lib/libnccl\.so(\..*)?$ 30-nvidia-cuda-libs ^usr/lib/libcublas\.so(\..*)?$ 30-nvidia-cuda-libs ^usr/lib/libcublasLt\.so(\..*)?$ 30-nvidia-cuda-libs @@ -161,7 +162,11 @@ bee_layer_canon_paths() { # bee_layer_classify # Partition every regular file and symlink under into exactly one -# layer. Writes /.files (LC_ALL=C sorted) and +# layer, plus every directory that is empty all the way down (no file or +# symlink anywhere in its subtree) - a directory that DOES contain files +# needs no entry of its own, since rsync recreates it as an implied parent +# of whichever layer(s) its files land in. Writes /.files and +# /.dirs (both LC_ALL=C sorted) and # /classify-report.txt. Runs as a subshell: internal state cannot # leak into the caller, and any step failure aborts with a non-zero exit. bee_layer_classify() ( @@ -182,7 +187,22 @@ bee_layer_classify() ( ( cd "$_root" && find . -mindepth 1 \( -type f -o -type l \) -printf '%P\n' ) \ | LC_ALL=C sort > "$_wd/all.files" - # 2. dpkg-owned files that belong to a non-base layer. + # 1b. Every directory, and the subset of those that are empty all the way + # down. A directory holding files elsewhere in its subtree is already + # covered by rsync's implied-parent handling in bee_layer_build; only + # a fully-empty branch needs an explicit entry, or it is silently + # dropped from every layer (it owns no file or symlink for step 1 to + # ever see). + ( cd "$_root" && find . -mindepth 1 -type d -printf '%P\n' ) \ + | LC_ALL=C sort > "$_wd/all.dirs" + awk -F/ '{ + p = $1 + print p + for (i = 2; i < NF; i++) { p = p "/" $i; print p } + }' "$_wd/all.files" | LC_ALL=C sort -u > "$_wd/nonempty.dirs" + LC_ALL=C comm -23 "$_wd/all.dirs" "$_wd/nonempty.dirs" > "$_wd/empty.dirs" + + # 2. dpkg-owned files (and directories) that belong to a non-base layer. : > "$_wd/dpkg.map" for _list in "$_root"/var/lib/dpkg/info/*.list; do [ -f "$_list" ] || continue @@ -195,11 +215,11 @@ bee_layer_classify() ( | awk -v l="$_layer" -v t="$_tab" 'NF { print $0 t l }' >> "$_wd/dpkg.map" done - # 3. Injected (no-deb) files, by explicit rule. These override dpkg. + # 3. Injected (no-deb) files/directories, by explicit rule. These override dpkg. : > "$_wd/injected.map" bee_layer_injected_rules | while IFS="$_tab" read -r _re _layer; do [ -n "$_re" ] || continue - LC_ALL=C grep -E "$_re" "$_wd/all.files" \ + LC_ALL=C grep -hE "$_re" "$_wd/all.files" "$_wd/all.dirs" \ | awk -v l="$_layer" -v t="$_tab" '{ print $0 t l }' >> "$_wd/injected.map" || true done @@ -207,23 +227,33 @@ bee_layer_classify() ( # Drop the merged-usr compat symlinks (bin/sbin/lib/lib64) unconditionally # - a bare "/lib" entry appears in some dpkg .list files (firmware-*), and # those symlinks must always stay in the base layer. - # Then keep only rows that name a real file or symlink in the tree: dpkg - # .list files also record bare directories, which must never reach an - # rsync --files-from list (rsync would copy the directory recursively). + # candidate.map covers files and directories alike; it is joined + # against all.files below for the file assignment, and separately + # against empty.dirs for the directory assignment. A dpkg .list + # directory entry that isn't empty simply fails that second join and + # is dropped - it never reaches an rsync --files-from list (rsync + # would otherwise copy it, and everything under it, recursively). cat "$_wd/injected.map" "$_wd/dpkg.map" \ | awk -F'\t' '$1!="bin" && $1!="sbin" && $1!="lib" && $1!="lib64" && !seen[$1]++ { print }' \ | LC_ALL=C sort -t "$_tab" -k1,1 > "$_wd/candidate.map" LC_ALL=C join -t "$_tab" -j 1 "$_wd/all.files" "$_wd/candidate.map" \ > "$_wd/assigned.map" || true + LC_ALL=C join -t "$_tab" -j 1 "$_wd/empty.dirs" "$_wd/candidate.map" \ + > "$_wd/assigned-dirs.map" || true - # 5. Split into per-layer file lists; base gets the remainder. - for _slug in $_slugs; do : > "$_wd/$_slug.files"; done + # 5. Split into per-layer file/dir lists; base gets the remainder of each. + for _slug in $_slugs; do : > "$_wd/$_slug.files"; : > "$_wd/$_slug.dirs"; done awk -F'\t' -v wd="$_wd" '{ print $1 >> (wd "/" $2 ".files") }' "$_wd/assigned.map" + awk -F'\t' -v wd="$_wd" '{ print $1 >> (wd "/" $2 ".dirs") }' "$_wd/assigned-dirs.map" for _slug in $_slugs; do LC_ALL=C sort -o "$_wd/$_slug.files" "$_wd/$_slug.files" + LC_ALL=C sort -o "$_wd/$_slug.dirs" "$_wd/$_slug.dirs" done cut -f1 "$_wd/assigned.map" | LC_ALL=C sort > "$_wd/assigned.paths" LC_ALL=C comm -23 "$_wd/all.files" "$_wd/assigned.paths" > "$_wd/00-base.files" + cut -f1 "$_wd/assigned-dirs.map" | LC_ALL=C sort > "$_wd/assigned-dirs.paths" + LC_ALL=C comm -23 "$_wd/empty.dirs" "$_wd/assigned-dirs.paths" >> "$_wd/00-base.dirs" + LC_ALL=C sort -o "$_wd/00-base.dirs" "$_wd/00-base.dirs" # 6. Completeness / disjointness. _total="$(wc -l < "$_wd/all.files")" @@ -231,8 +261,9 @@ bee_layer_classify() ( : > "$_wd/classify-report.txt" for _slug in $_slugs; do _n="$(wc -l < "$_wd/$_slug.files")" + _nd="$(wc -l < "$_wd/$_slug.dirs")" _sum=$((_sum + _n)) - printf '%-20s %8d files\n' "$_slug" "$_n" >> "$_wd/classify-report.txt" + printf '%-20s %8d files %6d empty dirs\n' "$_slug" "$_n" "$_nd" >> "$_wd/classify-report.txt" done printf '%-20s %8d files\n' "TOTAL" "$_sum" >> "$_wd/classify-report.txt" printf '%-20s %8d files\n' "tree" "$_total" >> "$_wd/classify-report.txt" @@ -249,6 +280,15 @@ bee_layer_classify() ( printf '%s\n' "$_dups" | head >&2 exit 1 fi + _dirtotal="$(wc -l < "$_wd/empty.dirs")" + _dirsum=0 + for _slug in $_slugs; do + _dirsum=$((_dirsum + $(wc -l < "$_wd/$_slug.dirs"))) + done + if [ "$_dirsum" -ne "$_dirtotal" ]; then + echo "bee_layer_classify: partition covers $_dirsum of $_dirtotal empty dirs" >&2 + exit 1 + fi # 7. merged-usr guard. for _link in bin sbin lib lib64; do @@ -278,6 +318,7 @@ bee_layer_build() ( for _slug in $_slugs; do _list="$_wd/$_slug.files" + _dirlist="$_wd/$_slug.dirs" [ -s "$_list" ] || { echo "bee_layer_build: empty file list for $_slug" >&2; exit 1; } _stage="$_wd/stage-$_slug" _sq="$_out/filesystem-v${_ver}-${_slug}.squashfs" @@ -286,8 +327,11 @@ bee_layer_build() ( # --files-from with -a recreates implied parent directories from the # source but copies only listed entries. --link-dest hardlinks unchanged # files from the source tree (same fs) so a 5 GB rootfs is not physically - # copied once per layer. - rsync -a --link-dest="$_root" --files-from="$_list" "$_root/" "$_stage/" + # copied once per layer. $_dirlist entries are directories that are + # empty all the way down (bee_layer_classify guarantees this), so + # -a's implicit recursion into them copies nothing extra. + cat "$_list" "$_dirlist" > "$_wd/$_slug.rsync-list" + rsync -a --link-dest="$_root" --files-from="$_wd/$_slug.rsync-list" "$_root/" "$_stage/" rm -f "$_sq" # shellcheck disable=SC2086 mksquashfs "$_stage" "$_sq" $BEE_LAYER_MKSQUASHFS_OPTS diff --git a/iso/builder/test-squashfs-layers.sh b/iso/builder/test-squashfs-layers.sh index b7d876a..e29f8a2 100644 --- a/iso/builder/test-squashfs-layers.sh +++ b/iso/builder/test-squashfs-layers.sh @@ -98,8 +98,15 @@ mk usr/local/bin/bee-dcgmproftester-staggered "#!stag" # unruled file under an nvidia dir stays in base. mk usr/share/doc/nvidia-random/README "notes" +# --- empty directories: no file/symlink anywhere in their subtree, so they +# never appear in all.files and must be classified/carried separately. +mkdir -p "$ROOT/var/log/nvidia-dcgm" # ruled -> 20-nvidia-platform +mkdir -p "$ROOT/var/lib/bee-empty-unruled" # unruled -> base +mkdir -p "$ROOT/var/lib/bee-empty-nested/still-empty" # nested empty branch -> base + fail() { echo "FAIL: $*" >&2; exit 1; } in_layer() { LC_ALL=C grep -qx "$2" "$WD/$1.files" || fail "expected $2 in layer $1"; } +in_dir_layer() { LC_ALL=C grep -qx "$2" "$WD/$1.dirs" || fail "expected empty dir $2 in layer $1"; } MONO="$T/filesystem-v14.99.squashfs" mksquashfs "$ROOT" "$MONO" -comp zstd -noappend -no-progress -no-xattrs >/dev/null @@ -146,11 +153,20 @@ in_layer 40-nvidia-dcgm-cuda usr/bin/dcgmproftester12 in_layer 40-nvidia-dcgm-cuda usr/local/bin/bee-dcgmproftester-staggered in_layer 00-base usr/share/doc/nvidia-random/README +in_dir_layer 20-nvidia-platform var/log/nvidia-dcgm +in_dir_layer 00-base var/lib/bee-empty-unruled +in_dir_layer 00-base var/lib/bee-empty-nested +in_dir_layer 00-base var/lib/bee-empty-nested/still-empty + OUT="$T/out" bee_layer_build "$R2" "$WD" "$OUT" 14.99 nvidia bee_layer_verify_each "$OUT" 14.99 nvidia bee_layer_merge "$OUT" 14.99 nvidia "$T/merged" +[ -d "$T/merged/var/log/nvidia-dcgm" ] || fail "empty dir var/log/nvidia-dcgm missing from merged rootfs" +[ -d "$T/merged/var/lib/bee-empty-unruled" ] || fail "empty dir var/lib/bee-empty-unruled missing from merged rootfs" +[ -d "$T/merged/var/lib/bee-empty-nested/still-empty" ] || fail "nested empty dir missing from merged rootfs" + MODLIVE="$T/live"; mkdir -p "$MODLIVE" cp "$OUT"/filesystem-v14.99-*.squashfs "$MODLIVE/" bee_layer_write_module_file "$MODLIVE" 14.99 nvidia