Mikhail ChusavitinandClaude Sonnet 5
b8c45d54c1
Booting via BMC virtual CD reads the ~2.8 GB filesystem squashfs sequentially during the live-boot toram copy; a mid-read drop of the redirected medium loses the whole copy and fails the boot (v14). Split the rootfs into self-contained semantic layers so a retry re-reads at most one ~500-700 MiB layer, not everything. This is a resilience / reduced-re-read mechanism, not a fix for the virtual-media instability. NVIDIA variants now ship 7 layers (00-base, 05-firmware, 08-desktop, 10-nvidia-driver, 20-nvidia-platform, 30-nvidia-cuda-libs, 40-nvidia-dcgm-cuda) plus an explicit live/filesystem.module that fixes their OverlayFS order; amd/nogpu keep a single squashfs. - lib/squashfs-layers.sh: deterministic classifier (dpkg file ownership plus explicit rules for build.sh-injected files, never a path substring), per-layer mksquashfs, 800 MiB hard ceiling, unsquashfs -s plus strict extraction of every layer, merged-rootfs bootability check. - build.sh: split the monolith after the full lb build, verify and merge, write the module file, delete the monolith only then; abort before ISO assembly on any failure. Runs the builder test suites up front. - fast-path: force a full build for a multi-layer medium; fast_path_repack_squashfs hard-refuses (it would drop layers). - iso-validation.sh: validate_iso_squashfs_layers (module vs layer set match, size ceiling, no lone giant squashfs) and validate_iso_media_integrity (xorriso -check_media). - bee-install: honour filesystem.module order, abort on any layer failure. - 9013-toram-retry: record the real rsync exit code (it printed a false rc=0) and correct the "resumes the tail" comment (rsync without --partial keeps only fully-copied layers). No unsafe partial resume. - tests: test-squashfs-layers.sh plus a multi-layer guard in test-build-libs.sh; both run at the top of every build. - docs: bible-local architecture and decision, iso/README, iso-build-rules. Verified by a full nvidia build: 7 layers 622/199/256/466/37/567/562 MiB, every validator passes, xorriso -check_media good, merged rootfs bootable. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
807 lines
29 KiB
Bash
Executable File
807 lines
29 KiB
Bash
Executable File
iso_list_files() {
|
|
iso_path="$1"
|
|
|
|
if command -v bsdtar >/dev/null 2>&1; then
|
|
bsdtar -tf "$iso_path"
|
|
return $?
|
|
fi
|
|
|
|
if command -v xorriso >/dev/null 2>&1; then
|
|
xorriso -indev "$iso_path" -find / -type f -print 2>/dev/null | sed 's#^/##'
|
|
return $?
|
|
fi
|
|
|
|
return 127
|
|
}
|
|
|
|
iso_extract_file() {
|
|
iso_path="$1"
|
|
iso_member="$2"
|
|
|
|
if command -v bsdtar >/dev/null 2>&1; then
|
|
bsdtar -xOf "$iso_path" "$iso_member"
|
|
return $?
|
|
fi
|
|
|
|
if command -v xorriso >/dev/null 2>&1; then
|
|
xorriso -osirrox on -indev "$iso_path" -cat "/$iso_member" 2>/dev/null
|
|
return $?
|
|
fi
|
|
|
|
return 127
|
|
}
|
|
|
|
iso_read_file_list() {
|
|
iso_path="$1"
|
|
out_path="$2"
|
|
|
|
iso_list_files "$iso_path" > "$out_path" || return 1
|
|
[ -s "$out_path" ] || return 1
|
|
return 0
|
|
}
|
|
|
|
iso_read_member() {
|
|
iso_path="$1"
|
|
iso_member="$2"
|
|
out_path="$3"
|
|
|
|
iso_extract_file "$iso_path" "$iso_member" > "$out_path" || return 1
|
|
[ -s "$out_path" ] || return 1
|
|
return 0
|
|
}
|
|
|
|
require_iso_reader() {
|
|
command -v bsdtar >/dev/null 2>&1 && return 0
|
|
command -v xorriso >/dev/null 2>&1 && return 0
|
|
memtest_fail "ISO reader is required for validation/debug (expected bsdtar or xorriso)" "${1:-}"
|
|
}
|
|
|
|
dump_memtest_debug() {
|
|
phase="$1"
|
|
lb_dir="${2:-}"
|
|
iso_path="${3:-}"
|
|
phase_slug="$(printf '%s' "${phase}" | tr ' /' '__')"
|
|
memtest_log="${LOG_DIR:-}/memtest-${phase_slug}.log"
|
|
|
|
(
|
|
echo "=== memtest debug: ${phase} ==="
|
|
|
|
echo "-- auto/config --"
|
|
if [ -f "${BUILDER_DIR}/auto/config" ]; then
|
|
grep -n -- '--memtest' "${BUILDER_DIR}/auto/config" || echo " (no --memtest line found)"
|
|
else
|
|
echo " (missing ${BUILDER_DIR}/auto/config)"
|
|
fi
|
|
|
|
echo "-- source bootloader templates --"
|
|
for cfg in \
|
|
"${BUILDER_DIR}/config/bootloaders/grub-efi/grub.cfg" \
|
|
"${BUILDER_DIR}/config/bootloaders/isolinux/live.cfg.in"; do
|
|
if [ -f "$cfg" ]; then
|
|
echo " file: $cfg"
|
|
grep -n 'Memory Test\|memtest' "$cfg" || echo " (no memtest lines)"
|
|
fi
|
|
done
|
|
|
|
echo "-- source binary hooks --"
|
|
for hook in \
|
|
"${BUILDER_DIR}/config/hooks/normal/9100-memtest.hook.binary"; do
|
|
if [ -f "$hook" ]; then
|
|
echo " hook: $hook"
|
|
else
|
|
echo " (missing $hook)"
|
|
fi
|
|
done
|
|
|
|
if [ -n "$lb_dir" ] && [ -d "$lb_dir" ]; then
|
|
echo "-- live-build workdir package lists --"
|
|
for pkg in \
|
|
"$lb_dir/config/package-lists/bee.list.chroot" \
|
|
"$lb_dir/config/package-lists/bee-gpu.list.chroot" \
|
|
"$lb_dir/config/package-lists/bee-nvidia.list.chroot"; do
|
|
if [ -f "$pkg" ]; then
|
|
echo " file: $pkg"
|
|
grep -n 'memtest' "$pkg" || echo " (no memtest lines)"
|
|
fi
|
|
done
|
|
|
|
echo "-- live-build chroot/boot --"
|
|
if [ -d "$lb_dir/chroot/boot" ]; then
|
|
find "$lb_dir/chroot/boot" -maxdepth 1 -name 'memtest*' -print | sed 's/^/ /' || true
|
|
else
|
|
echo " (missing $lb_dir/chroot/boot)"
|
|
fi
|
|
|
|
echo "-- live-build binary/boot --"
|
|
if [ -d "$lb_dir/binary/boot" ]; then
|
|
find "$lb_dir/binary/boot" -maxdepth 1 -name 'memtest*' -print | sed 's/^/ /' || true
|
|
else
|
|
echo " (missing $lb_dir/binary/boot)"
|
|
fi
|
|
|
|
echo "-- live-build binary grub cfg --"
|
|
if [ -f "$lb_dir/binary/boot/grub/grub.cfg" ]; then
|
|
grep -n 'Memory Test\|memtest' "$lb_dir/binary/boot/grub/grub.cfg" || echo " (no memtest lines)"
|
|
else
|
|
echo " (missing $lb_dir/binary/boot/grub/grub.cfg)"
|
|
fi
|
|
|
|
echo "-- live-build binary isolinux cfg --"
|
|
if [ -f "$lb_dir/binary/isolinux/live.cfg" ]; then
|
|
grep -n 'Memory Test\|memtest' "$lb_dir/binary/isolinux/live.cfg" || echo " (no memtest lines)"
|
|
else
|
|
echo " (missing $lb_dir/binary/isolinux/live.cfg)"
|
|
fi
|
|
|
|
echo "-- live-build package cache --"
|
|
if [ -d "$lb_dir/cache/packages.chroot" ]; then
|
|
find "$lb_dir/cache/packages.chroot" -maxdepth 1 -name 'memtest86+*.deb' -print | sed 's/^/ /' || true
|
|
else
|
|
echo " (missing $lb_dir/cache/packages.chroot)"
|
|
fi
|
|
fi
|
|
|
|
if [ -n "$iso_path" ] && [ -f "$iso_path" ]; then
|
|
iso_files="$(mktemp)"
|
|
iso_grub_cfg="$(mktemp)"
|
|
iso_isolinux_cfg="$(mktemp)"
|
|
|
|
echo "-- ISO memtest files --"
|
|
if iso_read_file_list "$iso_path" "$iso_files"; then
|
|
grep 'memtest' "$iso_files" | sed 's/^/ /' || echo " (no memtest files in ISO)"
|
|
else
|
|
echo " (failed to list ISO contents)"
|
|
fi
|
|
|
|
echo "-- ISO GRUB memtest lines --"
|
|
if iso_read_member "$iso_path" boot/grub/grub.cfg "$iso_grub_cfg"; then
|
|
grep -n 'Memory Test\|memtest' "$iso_grub_cfg" || echo " (no memtest lines in boot/grub/grub.cfg)"
|
|
else
|
|
echo " (failed to read boot/grub/grub.cfg from ISO)"
|
|
fi
|
|
|
|
echo "-- ISO isolinux memtest lines --"
|
|
if iso_read_member "$iso_path" isolinux/live.cfg "$iso_isolinux_cfg"; then
|
|
grep -n 'Memory Test\|memtest' "$iso_isolinux_cfg" || echo " (no memtest lines in isolinux/live.cfg)"
|
|
else
|
|
echo " (failed to read isolinux/live.cfg from ISO)"
|
|
fi
|
|
|
|
rm -f "$iso_files" "$iso_grub_cfg" "$iso_isolinux_cfg"
|
|
fi
|
|
|
|
echo "=== end memtest debug: ${phase} ==="
|
|
) | {
|
|
if [ -n "${LOG_DIR:-}" ] && [ -d "${LOG_DIR}" ]; then
|
|
tee "${memtest_log}"
|
|
else
|
|
cat
|
|
fi
|
|
}
|
|
}
|
|
|
|
memtest_fail() {
|
|
msg="$1"
|
|
iso_path="${2:-}"
|
|
level="WARNING"
|
|
if [ "${BEE_REQUIRE_MEMTEST:-0}" = "1" ]; then
|
|
level="ERROR"
|
|
fi
|
|
echo "${level}: ${msg}" >&2
|
|
dump_memtest_debug "failure" "${LB_DIR:-}" "$iso_path" >&2
|
|
if [ "${BEE_REQUIRE_MEMTEST:-0}" = "1" ]; then
|
|
exit 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
nvidia_runtime_fail() {
|
|
msg="$1"
|
|
echo "ERROR: ${msg}" >&2
|
|
exit 1
|
|
}
|
|
|
|
iso_memtest_present() {
|
|
iso_path="$1"
|
|
iso_files="$(mktemp)"
|
|
|
|
[ -f "$iso_path" ] || return 1
|
|
|
|
if command -v bsdtar >/dev/null 2>&1; then
|
|
:
|
|
elif command -v xorriso >/dev/null 2>&1; then
|
|
:
|
|
else
|
|
return 2
|
|
fi
|
|
|
|
iso_read_file_list "$iso_path" "$iso_files" || {
|
|
rm -f "$iso_files"
|
|
return 2
|
|
}
|
|
|
|
grep -q '^boot/memtest86+x64\.bin$' "$iso_files" || {
|
|
rm -f "$iso_files"
|
|
return 1
|
|
}
|
|
grep -q '^boot/memtest86+x64\.efi$' "$iso_files" || {
|
|
rm -f "$iso_files"
|
|
return 1
|
|
}
|
|
|
|
grub_cfg="$(mktemp)"
|
|
isolinux_cfg="$(mktemp)"
|
|
|
|
iso_read_member "$iso_path" boot/grub/grub.cfg "$grub_cfg" || {
|
|
rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg"
|
|
return 2
|
|
}
|
|
iso_read_member "$iso_path" isolinux/live.cfg "$isolinux_cfg" || {
|
|
rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg"
|
|
return 2
|
|
}
|
|
|
|
grep -q 'Memory Test (memtest86+)' "$grub_cfg" || {
|
|
rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg"
|
|
return 1
|
|
}
|
|
grep -q '/boot/memtest86+x64\.efi' "$grub_cfg" || {
|
|
rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg"
|
|
return 1
|
|
}
|
|
grep -q '/boot/memtest86+x64\.bin' "$grub_cfg" || {
|
|
rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg"
|
|
return 1
|
|
}
|
|
grep -q 'Memory Test (memtest86+)' "$isolinux_cfg" || {
|
|
rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg"
|
|
return 1
|
|
}
|
|
grep -q '/boot/memtest86+x64\.bin' "$isolinux_cfg" || {
|
|
rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg"
|
|
return 1
|
|
}
|
|
|
|
rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg"
|
|
return 0
|
|
}
|
|
|
|
validate_iso_memtest() {
|
|
iso_path="$1"
|
|
echo "=== validating memtest in ISO ==="
|
|
|
|
[ -f "$iso_path" ] || {
|
|
memtest_fail "ISO not found for validation: $iso_path" "$iso_path"
|
|
return 0
|
|
}
|
|
require_iso_reader "$iso_path" || return 0
|
|
|
|
iso_files="$(mktemp)"
|
|
iso_read_file_list "$iso_path" "$iso_files" || {
|
|
memtest_fail "failed to list ISO contents while validating memtest" "$iso_path"
|
|
rm -f "$iso_files"
|
|
return 0
|
|
}
|
|
|
|
grep -q '^boot/memtest86+x64\.bin$' "$iso_files" || {
|
|
memtest_fail "memtest BIOS binary missing in ISO: boot/memtest86+x64.bin" "$iso_path"
|
|
rm -f "$iso_files"
|
|
return 0
|
|
}
|
|
grep -q '^boot/memtest86+x64\.efi$' "$iso_files" || {
|
|
memtest_fail "memtest EFI binary missing in ISO: boot/memtest86+x64.efi" "$iso_path"
|
|
rm -f "$iso_files"
|
|
return 0
|
|
}
|
|
|
|
grub_cfg="$(mktemp)"
|
|
isolinux_cfg="$(mktemp)"
|
|
|
|
iso_read_member "$iso_path" boot/grub/grub.cfg "$grub_cfg" || {
|
|
memtest_fail "failed to read boot/grub/grub.cfg from ISO" "$iso_path"
|
|
rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg"
|
|
return 0
|
|
}
|
|
iso_read_member "$iso_path" isolinux/live.cfg "$isolinux_cfg" || {
|
|
memtest_fail "failed to read isolinux/live.cfg from ISO" "$iso_path"
|
|
rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg"
|
|
return 0
|
|
}
|
|
|
|
grep -q 'Memory Test (memtest86+)' "$grub_cfg" || {
|
|
memtest_fail "GRUB menu entry for memtest is missing" "$iso_path"
|
|
rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg"
|
|
return 0
|
|
}
|
|
grep -q '/boot/memtest86+x64\.efi' "$grub_cfg" || {
|
|
memtest_fail "GRUB memtest EFI path is missing" "$iso_path"
|
|
rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg"
|
|
return 0
|
|
}
|
|
grep -q '/boot/memtest86+x64\.bin' "$grub_cfg" || {
|
|
memtest_fail "GRUB memtest BIOS path is missing" "$iso_path"
|
|
rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg"
|
|
return 0
|
|
}
|
|
grep -q 'Memory Test (memtest86+)' "$isolinux_cfg" || {
|
|
memtest_fail "isolinux menu entry for memtest is missing" "$iso_path"
|
|
rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg"
|
|
return 0
|
|
}
|
|
grep -q '/boot/memtest86+x64\.bin' "$isolinux_cfg" || {
|
|
memtest_fail "isolinux memtest path is missing" "$iso_path"
|
|
rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg"
|
|
return 0
|
|
}
|
|
|
|
rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg"
|
|
echo "=== memtest validation OK ==="
|
|
}
|
|
|
|
validate_live_cmdline_params() {
|
|
cfg="$1"
|
|
command="$2"
|
|
bootloader="$3"
|
|
expected_label="$4"
|
|
|
|
awk -v command="$command" -v bootloader="$bootloader" -v expected_label="$expected_label" '
|
|
function has(token, i) {
|
|
for (i = 1; i <= NF; i++) {
|
|
if ($i == token) {
|
|
return 1
|
|
}
|
|
}
|
|
return 0
|
|
}
|
|
|
|
function reject(message) {
|
|
printf "ERROR: %s live entry at %s:%d: %s\n", bootloader, FILENAME, NR, message
|
|
bad = 1
|
|
}
|
|
|
|
$1 == command {
|
|
live_entries++
|
|
|
|
if (!has("boot=live")) {
|
|
reject("missing boot=live")
|
|
}
|
|
if (!has("udev.children_max=1")) {
|
|
reject("missing udev.children_max=1")
|
|
}
|
|
if (!has("intel_iommu=on")) {
|
|
reject("missing intel_iommu=on")
|
|
}
|
|
if (!has("iommu.passthrough=0")) {
|
|
reject("missing iommu.passthrough=0")
|
|
}
|
|
if (!has("efi=disable_early_pci_dma")) {
|
|
reject("missing efi=disable_early_pci_dma")
|
|
}
|
|
if (!has("live-media-label=" expected_label)) {
|
|
reject("missing expected live-media-label=" expected_label)
|
|
}
|
|
if (has("iommu=pt")) {
|
|
reject("contains forbidden iommu=pt")
|
|
}
|
|
|
|
if (has("pci=realloc")) {
|
|
failsafe_entries++
|
|
if (!has("iommu.strict=1")) {
|
|
reject("pci=realloc entry is missing iommu.strict=1")
|
|
}
|
|
} else if (has("iommu.strict=1")) {
|
|
reject("iommu.strict=1 is allowed only in the pci=realloc fail-safe entry")
|
|
}
|
|
}
|
|
|
|
END {
|
|
if (live_entries == 0) {
|
|
printf "ERROR: %s config has no live boot entries\n", bootloader
|
|
bad = 1
|
|
}
|
|
if (failsafe_entries != 1) {
|
|
printf "ERROR: %s config has %d pci=realloc fail-safe entries, expected 1\n", bootloader, failsafe_entries
|
|
bad = 1
|
|
}
|
|
exit bad ? 1 : 0
|
|
}
|
|
' "$cfg"
|
|
}
|
|
|
|
validate_iso_live_boot_entries() {
|
|
iso_path="$1"
|
|
echo "=== validating live boot entries in ISO ==="
|
|
|
|
[ -f "$iso_path" ] || {
|
|
echo "ERROR: ISO not found for live boot validation: $iso_path" >&2
|
|
exit 1
|
|
}
|
|
require_iso_reader "$iso_path" >/dev/null 2>&1 || {
|
|
echo "ERROR: ISO reader unavailable for live boot validation" >&2
|
|
exit 1
|
|
}
|
|
|
|
grub_cfg="$(mktemp)"
|
|
isolinux_cfg="$(mktemp)"
|
|
|
|
iso_read_member "$iso_path" boot/grub/grub.cfg "$grub_cfg" || {
|
|
echo "ERROR: failed to read boot/grub/grub.cfg from ISO" >&2
|
|
rm -f "$grub_cfg" "$isolinux_cfg"
|
|
exit 1
|
|
}
|
|
iso_read_member "$iso_path" isolinux/live.cfg "$isolinux_cfg" || {
|
|
echo "ERROR: failed to read isolinux/live.cfg from ISO" >&2
|
|
rm -f "$grub_cfg" "$isolinux_cfg"
|
|
exit 1
|
|
}
|
|
|
|
if grep -Eq '@[A-Z][A-Z_]*@' "$grub_cfg" "$isolinux_cfg"; then
|
|
echo "ERROR: unresolved live-build placeholders remain in ISO bootloader config" >&2
|
|
grep -En '@[A-Z][A-Z_]*@' "$grub_cfg" | sed 's/^/ GRUB: /' >&2 || true
|
|
grep -En '@[A-Z][A-Z_]*@' "$isolinux_cfg" | sed 's/^/ isolinux: /' >&2 || true
|
|
rm -f "$grub_cfg" "$isolinux_cfg"
|
|
exit 1
|
|
fi
|
|
|
|
if grep -q 'iommu=pt' "$grub_cfg" "$isolinux_cfg"; then
|
|
echo "ERROR: forbidden iommu=pt remains in ISO bootloader config" >&2
|
|
rm -f "$grub_cfg" "$isolinux_cfg"
|
|
exit 1
|
|
fi
|
|
|
|
if ! validate_live_cmdline_params "$grub_cfg" linux GRUB "${BEE_ISO_VOLUME}"; then
|
|
rm -f "$grub_cfg" "$isolinux_cfg"
|
|
exit 1
|
|
fi
|
|
if ! validate_live_cmdline_params "$isolinux_cfg" append isolinux "${BEE_ISO_VOLUME}"; then
|
|
rm -f "$grub_cfg" "$isolinux_cfg"
|
|
exit 1
|
|
fi
|
|
|
|
grep -q 'menuentry "EASY-BEE v' "$grub_cfg" || {
|
|
echo "ERROR: GRUB default EASY-BEE entry is missing" >&2
|
|
rm -f "$grub_cfg" "$isolinux_cfg"
|
|
exit 1
|
|
}
|
|
grep -Fq "menuentry \"EASY-BEE v${PROJECT_VERSION_EFFECTIVE}\"" "$grub_cfg" || {
|
|
echo "ERROR: GRUB version does not match ${PROJECT_VERSION_EFFECTIVE}" >&2
|
|
rm -f "$grub_cfg" "$isolinux_cfg"
|
|
exit 1
|
|
}
|
|
grep -Fq "menu label ^EASY-BEE v${PROJECT_VERSION_EFFECTIVE}" "$isolinux_cfg" || {
|
|
echo "ERROR: isolinux version does not match ${PROJECT_VERSION_EFFECTIVE}" >&2
|
|
rm -f "$grub_cfg" "$isolinux_cfg"
|
|
exit 1
|
|
}
|
|
|
|
if command -v xorriso >/dev/null 2>&1; then
|
|
iso_volume="$(xorriso -indev "$iso_path" -pvd_info 2>/dev/null | awk -F: '/Volume [Ii]d/ { sub(/^[[:space:]'\''"]+/, "", $2); sub(/[[:space:]'\''"]+$/, "", $2); print $2; exit }')"
|
|
if [ "$iso_volume" != "${BEE_ISO_VOLUME}" ]; then
|
|
echo "ERROR: ISO volume ID is ${iso_volume:-unknown}, expected ${BEE_ISO_VOLUME}" >&2
|
|
rm -f "$grub_cfg" "$isolinux_cfg"
|
|
exit 1
|
|
fi
|
|
fi
|
|
grep -q 'menuentry "EASY-BEE v.* -- load to RAM (toram)"' "$grub_cfg" || {
|
|
echo "ERROR: GRUB toram entry is missing" >&2
|
|
rm -f "$grub_cfg" "$isolinux_cfg"
|
|
exit 1
|
|
}
|
|
grep -q 'linux .*boot=live ' "$grub_cfg" || {
|
|
echo "ERROR: GRUB live entry is missing boot=live" >&2
|
|
rm -f "$grub_cfg" "$isolinux_cfg"
|
|
exit 1
|
|
}
|
|
grep -q 'linux .*boot=live .*toram ' "$grub_cfg" || {
|
|
echo "ERROR: GRUB toram entry is missing boot=live or toram" >&2
|
|
rm -f "$grub_cfg" "$isolinux_cfg"
|
|
exit 1
|
|
}
|
|
grep -q 'linux .*live-media-label=EASY_BEE_' "$grub_cfg" || {
|
|
echo "ERROR: GRUB live entry is missing live-media-label pinning" >&2
|
|
rm -f "$grub_cfg" "$isolinux_cfg"
|
|
exit 1
|
|
}
|
|
|
|
grep -q 'append .*boot=live ' "$isolinux_cfg" || {
|
|
echo "ERROR: isolinux live entry is missing boot=live" >&2
|
|
rm -f "$grub_cfg" "$isolinux_cfg"
|
|
exit 1
|
|
}
|
|
grep -q 'append .*boot=live .*toram ' "$isolinux_cfg" || {
|
|
echo "ERROR: isolinux toram entry is missing boot=live or toram" >&2
|
|
rm -f "$grub_cfg" "$isolinux_cfg"
|
|
exit 1
|
|
}
|
|
grep -q 'append .*live-media-label=EASY_BEE_' "$isolinux_cfg" || {
|
|
echo "ERROR: isolinux live entry is missing live-media-label pinning" >&2
|
|
rm -f "$grub_cfg" "$isolinux_cfg"
|
|
exit 1
|
|
}
|
|
|
|
rm -f "$grub_cfg" "$isolinux_cfg"
|
|
echo "=== live boot validation OK ==="
|
|
}
|
|
|
|
validate_iso_grub_assets() {
|
|
iso_path="$1"
|
|
echo "=== validating GRUB assets in ISO ==="
|
|
|
|
[ -f "$iso_path" ] || {
|
|
echo "ERROR: ISO not found for GRUB asset validation: $iso_path" >&2
|
|
exit 1
|
|
}
|
|
require_iso_reader "$iso_path" >/dev/null 2>&1 || {
|
|
echo "ERROR: ISO reader unavailable for GRUB asset validation" >&2
|
|
exit 1
|
|
}
|
|
|
|
iso_files="$(mktemp)"
|
|
iso_list_files "$iso_path" > "$iso_files" || {
|
|
echo "ERROR: failed to list ISO files for GRUB asset validation" >&2
|
|
rm -f "$iso_files"
|
|
exit 1
|
|
}
|
|
|
|
for required in \
|
|
boot/grub/config.cfg \
|
|
boot/grub/grub.cfg; do
|
|
grep -q "^${required}$" "$iso_files" || {
|
|
echo "ERROR: missing GRUB asset in ISO: ${required}" >&2
|
|
rm -f "$iso_files"
|
|
exit 1
|
|
}
|
|
done
|
|
|
|
rm -f "$iso_files"
|
|
echo "=== GRUB asset validation OK ==="
|
|
}
|
|
|
|
validate_iso_nvidia_runtime() {
|
|
iso_path="$1"
|
|
[ "$BEE_GPU_VENDOR" = "nvidia" ] || return 0
|
|
|
|
echo "=== validating NVIDIA runtime in ISO ==="
|
|
|
|
[ -f "$iso_path" ] || nvidia_runtime_fail "ISO not found for NVIDIA runtime validation: $iso_path"
|
|
require_iso_reader "$iso_path" >/dev/null 2>&1 || nvidia_runtime_fail "ISO reader unavailable for NVIDIA runtime validation"
|
|
command -v unsquashfs >/dev/null 2>&1 || nvidia_runtime_fail "unsquashfs is required for NVIDIA runtime validation"
|
|
|
|
squashfs_tmp="$(mktemp)"
|
|
squashfs_list="$(mktemp)"
|
|
iso_files="$(mktemp)"
|
|
dpkg_status_dir="$(mktemp -d)"
|
|
iso_list_files "$iso_path" > "$iso_files" || {
|
|
rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files"
|
|
rm -rf "$dpkg_status_dir"
|
|
nvidia_runtime_fail "failed to list ISO files for NVIDIA runtime validation"
|
|
}
|
|
grep '^live/.*\.squashfs$' "$iso_files" | while IFS= read -r squashfs_member; do
|
|
iso_read_member "$iso_path" "$squashfs_member" "$squashfs_tmp" || {
|
|
rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files"
|
|
rm -rf "$dpkg_status_dir"
|
|
nvidia_runtime_fail "failed to extract $squashfs_member from ISO"
|
|
}
|
|
unsquashfs -ll "$squashfs_tmp" >> "$squashfs_list" 2>/dev/null || {
|
|
rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files"
|
|
rm -rf "$dpkg_status_dir"
|
|
nvidia_runtime_fail "failed to inspect $squashfs_member from ISO"
|
|
}
|
|
# var/lib/dpkg/status lives in whichever squashfs layer has the base
|
|
# rootfs (not the usr/firmware split-off layers); harmless no-op on
|
|
# the others.
|
|
unsquashfs -d "${dpkg_status_dir}/extract" -f "$squashfs_tmp" var/lib/dpkg/status >/dev/null 2>&1 || true
|
|
: > "$squashfs_tmp"
|
|
done
|
|
|
|
grep -Eq 'usr/bin/dcgmi$' "$squashfs_list" || {
|
|
rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files"
|
|
rm -rf "$dpkg_status_dir"
|
|
nvidia_runtime_fail "dcgmi missing from final NVIDIA ISO"
|
|
}
|
|
grep -Eq 'usr/bin/nv-hostengine$' "$squashfs_list" || {
|
|
rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files"
|
|
rm -rf "$dpkg_status_dir"
|
|
nvidia_runtime_fail "nv-hostengine missing from final NVIDIA ISO"
|
|
}
|
|
grep -Eq 'usr/bin/dcgmproftester([0-9]+)?$' "$squashfs_list" || {
|
|
rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files"
|
|
rm -rf "$dpkg_status_dir"
|
|
nvidia_runtime_fail "dcgmproftester missing from final NVIDIA ISO"
|
|
}
|
|
|
|
# Cross-check the DCGM package version actually baked into the squashfs
|
|
# against VERSIONS. dcgmi/nv-hostengine/dcgmproftester presence alone
|
|
# doesn't catch a stale squashfs served by a mis-detected fast-path build
|
|
# (dcgmi stays present across DCGM versions); this does.
|
|
dpkg_status_file="${dpkg_status_dir}/extract/var/lib/dpkg/status"
|
|
if [ -f "$dpkg_status_file" ]; then
|
|
_installed_dcgm_version="$(awk '
|
|
/^Package: datacenter-gpu-manager-4-core$/ { in_pkg=1; next }
|
|
/^Package: / { in_pkg=0 }
|
|
in_pkg && /^Version: / { sub(/^Version: /, ""); print; exit }
|
|
' "$dpkg_status_file")"
|
|
if [ -z "$_installed_dcgm_version" ]; then
|
|
echo "=== WARNING: datacenter-gpu-manager-4-core not found in ISO dpkg status; skipping DCGM version check ==="
|
|
else
|
|
_installed_dcgm_no_epoch="${_installed_dcgm_version#*:}"
|
|
if [ "$_installed_dcgm_no_epoch" != "${DCGM_VERSION}" ]; then
|
|
rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files"
|
|
rm -rf "$dpkg_status_dir"
|
|
nvidia_runtime_fail "DCGM version mismatch: VERSIONS pins ${DCGM_VERSION} but ISO has ${_installed_dcgm_version} (stale squashfs; retry with --clean-build)"
|
|
fi
|
|
fi
|
|
else
|
|
echo "=== WARNING: could not read dpkg status from ISO; skipping DCGM version check ==="
|
|
fi
|
|
|
|
rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files"
|
|
rm -rf "$dpkg_status_dir"
|
|
echo "=== NVIDIA runtime validation OK ==="
|
|
}
|
|
|
|
rootfs_layout_fail() {
|
|
echo "ERROR: $1" >&2
|
|
exit 1
|
|
}
|
|
|
|
squashfs_layers_fail() {
|
|
echo "ERROR: $1" >&2
|
|
exit 1
|
|
}
|
|
|
|
# Validate the semantic squashfs layering in the final ISO:
|
|
# - NVIDIA variants must ship live/filesystem.module and >= 2 layers
|
|
# - every squashfs in live/ is listed in the module file and vice versa
|
|
# - no single layer exceeds the agreed compressed-size ceiling
|
|
# - a single giant squashfs is rejected for a multi-layer variant
|
|
validate_iso_squashfs_layers() {
|
|
iso_path="$1"
|
|
echo "=== validating squashfs layers in ISO ==="
|
|
|
|
[ -f "$iso_path" ] || squashfs_layers_fail "ISO not found for squashfs layer validation: $iso_path"
|
|
require_iso_reader "$iso_path" >/dev/null 2>&1 || squashfs_layers_fail "ISO reader unavailable for squashfs layer validation"
|
|
|
|
_files="$(mktemp)"
|
|
_module="$(mktemp)"
|
|
iso_list_files "$iso_path" > "$_files" || squashfs_layers_fail "failed to list ISO files"
|
|
|
|
_sq_in_iso="$(grep -E '^live/filesystem.*\.squashfs$' "$_files" | sed 's#^live/##' | LC_ALL=C sort)"
|
|
_sq_count="$(printf '%s\n' "$_sq_in_iso" | grep -c . || true)"
|
|
_has_module=0
|
|
grep -qx 'live/filesystem.module' "$_files" && _has_module=1
|
|
|
|
if [ "${BEE_MULTILAYER_SQUASHFS:-0}" != "1" ]; then
|
|
echo "=== single-layer variant: $_sq_count squashfs, module file present=$_has_module ==="
|
|
[ "$_has_module" = 0 ] || squashfs_layers_fail "single-layer variant unexpectedly ships filesystem.module"
|
|
rm -f "$_files" "$_module"
|
|
echo "=== squashfs layer validation OK (single layer) ==="
|
|
return 0
|
|
fi
|
|
|
|
[ "$_has_module" = 1 ] || squashfs_layers_fail "multi-layer variant is missing live/filesystem.module (monolith slipped through)"
|
|
iso_read_member "$iso_path" live/filesystem.module "$_module" || squashfs_layers_fail "failed to read live/filesystem.module from ISO"
|
|
|
|
_listed="$(grep -vE '^[[:space:]]*(#|$)' "$_module" | LC_ALL=C sort)"
|
|
_listed_count="$(printf '%s\n' "$_listed" | grep -c . || true)"
|
|
[ "$_listed_count" -ge 2 ] || squashfs_layers_fail "filesystem.module lists only $_listed_count layer(s), expected >= 2"
|
|
|
|
if [ "$(printf '%s\n' "$_listed")" != "$(printf '%s\n' "$_sq_in_iso")" ]; then
|
|
echo " module lists:" >&2; printf ' %s\n' $_listed >&2
|
|
echo " ISO carries:" >&2; printf ' %s\n' $_sq_in_iso >&2
|
|
squashfs_layers_fail "filesystem.module and the squashfs files in live/ do not match exactly"
|
|
fi
|
|
|
|
_limit_mib="${BEE_LAYER_MAX_MIB:-800}"
|
|
for _name in $_sq_in_iso; do
|
|
_bytes=""
|
|
if command -v xorriso >/dev/null 2>&1; then
|
|
_bytes="$(xorriso -indev "$iso_path" -lsl "/live/${_name}" 2>/dev/null \
|
|
| awk '$1 ~ /^-/ { print $5; exit }')"
|
|
fi
|
|
if ! printf '%s' "$_bytes" | grep -Eq '^[0-9]+$'; then
|
|
_tmp_sq="$(mktemp)"
|
|
iso_read_member "$iso_path" "live/${_name}" "$_tmp_sq" || squashfs_layers_fail "failed to extract live/${_name}"
|
|
_bytes="$(wc -c < "$_tmp_sq" | tr -d ' ')"
|
|
rm -f "$_tmp_sq"
|
|
fi
|
|
_mib=$(( _bytes / 1048576 ))
|
|
printf ' %-40s %5d MiB\n' "$_name" "$_mib"
|
|
[ "$_mib" -le "$_limit_mib" ] || squashfs_layers_fail "layer ${_name} is ${_mib} MiB, over the ${_limit_mib} MiB ceiling"
|
|
case "$_name" in
|
|
*-00-base.squashfs) : ;;
|
|
*) [ "$_mib" -ge 1 ] || squashfs_layers_fail "layer ${_name} is suspiciously empty" ;;
|
|
esac
|
|
done
|
|
|
|
if [ "$_sq_count" -eq 1 ]; then
|
|
squashfs_layers_fail "multi-layer variant shipped a single squashfs"
|
|
fi
|
|
|
|
rm -f "$_files" "$_module"
|
|
echo "=== squashfs layer validation OK ($_sq_count layers) ==="
|
|
}
|
|
|
|
# Read the whole raw ISO back and confirm every sector is readable. Catches a
|
|
# truncated or corrupt image before it ships - the split path rewrites the
|
|
# squashfs area and reassembles the ISO, so this is worth a full re-read.
|
|
validate_iso_media_integrity() {
|
|
iso_path="$1"
|
|
echo "=== validating ISO media integrity (xorriso -check_media) ==="
|
|
|
|
[ -f "$iso_path" ] || { echo "ERROR: ISO not found for media check: $iso_path" >&2; exit 1; }
|
|
if ! command -v xorriso >/dev/null 2>&1; then
|
|
echo "WARNING: xorriso not available, skipping -check_media" >&2
|
|
return 0
|
|
fi
|
|
|
|
_out="$(xorriso -indev "$iso_path" -check_media 2>&1)" || {
|
|
printf '%s\n' "$_out" >&2
|
|
echo "ERROR: xorriso -check_media failed for $iso_path" >&2
|
|
exit 1
|
|
}
|
|
printf '%s\n' "$_out" | grep -E 'Media region|Bad blocks|checked|md5' || true
|
|
if printf '%s\n' "$_out" | grep -Eiq 'bad block[^s]|[1-9][0-9]* bad blocks|damaged|not readable'; then
|
|
printf '%s\n' "$_out" >&2
|
|
echo "ERROR: xorriso -check_media reported unreadable/bad sectors in $iso_path" >&2
|
|
exit 1
|
|
fi
|
|
echo "=== ISO media integrity OK ==="
|
|
}
|
|
|
|
# Guard against a corrupted merged-usr layout in the live rootfs. On Debian
|
|
# bookworm /bin, /sbin, /lib and /lib64 are symlinks into /usr; if a build step
|
|
# stages a real directory of that name into the overlay, `rsync -a` replaces the
|
|
# symlink with a plain directory and orphans everything reachable only through
|
|
# it (/lib/systemd/systemd, /lib/x86_64-linux-gnu/*, ...). The image still packs
|
|
# and mounts fine but panics at boot: "run-init: can't execute '/sbin/init'".
|
|
validate_iso_rootfs_layout() {
|
|
iso_path="$1"
|
|
echo "=== validating rootfs merged-usr layout in ISO ==="
|
|
|
|
[ -f "$iso_path" ] || rootfs_layout_fail "ISO not found for rootfs layout validation: $iso_path"
|
|
require_iso_reader "$iso_path" >/dev/null 2>&1 || rootfs_layout_fail "ISO reader unavailable for rootfs layout validation"
|
|
command -v unsquashfs >/dev/null 2>&1 || rootfs_layout_fail "unsquashfs is required for rootfs layout validation"
|
|
|
|
_sq_tmp="$(mktemp)"
|
|
_sq_list="$(mktemp)"
|
|
_iso_files="$(mktemp)"
|
|
|
|
iso_list_files "$iso_path" > "$_iso_files" \
|
|
|| rootfs_layout_fail "failed to list ISO files for rootfs layout validation"
|
|
|
|
_saw_rootfs=0
|
|
while IFS= read -r _member; do
|
|
[ -n "$_member" ] || continue
|
|
iso_read_member "$iso_path" "$_member" "$_sq_tmp" \
|
|
|| rootfs_layout_fail "failed to extract $_member from ISO"
|
|
: > "$_sq_list"
|
|
unsquashfs -ll "$_sq_tmp" > "$_sq_list" 2>/dev/null \
|
|
|| rootfs_layout_fail "failed to inspect $_member from ISO"
|
|
|
|
# A plain-directory top-level bin/sbin/lib/lib64 is always wrong here.
|
|
_bad="$(grep -E '^d[rwxsStT-]{9} .* squashfs-root/(bin|sbin|lib|lib64)$' "$_sq_list" || true)"
|
|
if [ -n "$_bad" ]; then
|
|
echo "$_bad" >&2
|
|
rootfs_layout_fail "$_member: /bin|/sbin|/lib|/lib64 is a plain directory, not a merged-usr symlink (overlay clobbered it; stage under usr/lib instead)"
|
|
fi
|
|
|
|
# In the layer that carries the base rootfs, assert init is resolvable.
|
|
if grep -Eq ' squashfs-root/usr/sbin/init( ->|$)' "$_sq_list"; then
|
|
_saw_rootfs=1
|
|
grep -Eq ' squashfs-root/usr/lib/systemd/systemd$' "$_sq_list" \
|
|
|| rootfs_layout_fail "$_member: /usr/sbin/init present but /usr/lib/systemd/systemd missing"
|
|
grep -Eq ' squashfs-root/lib -> ' "$_sq_list" \
|
|
|| rootfs_layout_fail "$_member: /lib is not a symlink into /usr (merged-usr broken)"
|
|
fi
|
|
done <<EOF
|
|
$(grep -E '^live/.*\.squashfs$' "$_iso_files")
|
|
EOF
|
|
|
|
[ "$_saw_rootfs" = 1 ] \
|
|
|| rootfs_layout_fail "no squashfs layer with /usr/sbin/init found in ISO"
|
|
|
|
rm -f "$_sq_tmp" "$_sq_list" "$_iso_files"
|
|
echo "=== rootfs layout validation OK ==="
|
|
}
|