iso_list_files() { iso_path="$1" if command -v bsdtar >/dev/null 2>&1; then bsdtar -tf "$iso_path" return $? fi if command -v xorriso >/dev/null 2>&1; then xorriso -indev "$iso_path" -find / -type f -print 2>/dev/null | sed 's#^/##' return $? fi return 127 } iso_extract_file() { iso_path="$1" iso_member="$2" if command -v bsdtar >/dev/null 2>&1; then bsdtar -xOf "$iso_path" "$iso_member" return $? fi if command -v xorriso >/dev/null 2>&1; then xorriso -osirrox on -indev "$iso_path" -cat "/$iso_member" 2>/dev/null return $? fi return 127 } iso_read_file_list() { iso_path="$1" out_path="$2" iso_list_files "$iso_path" > "$out_path" || return 1 [ -s "$out_path" ] || return 1 return 0 } iso_read_member() { iso_path="$1" iso_member="$2" out_path="$3" iso_extract_file "$iso_path" "$iso_member" > "$out_path" || return 1 [ -s "$out_path" ] || return 1 return 0 } require_iso_reader() { command -v bsdtar >/dev/null 2>&1 && return 0 command -v xorriso >/dev/null 2>&1 && return 0 memtest_fail "ISO reader is required for validation/debug (expected bsdtar or xorriso)" "${1:-}" } dump_memtest_debug() { phase="$1" lb_dir="${2:-}" iso_path="${3:-}" phase_slug="$(printf '%s' "${phase}" | tr ' /' '__')" memtest_log="${LOG_DIR:-}/memtest-${phase_slug}.log" ( echo "=== memtest debug: ${phase} ===" echo "-- auto/config --" if [ -f "${BUILDER_DIR}/auto/config" ]; then grep -n -- '--memtest' "${BUILDER_DIR}/auto/config" || echo " (no --memtest line found)" else echo " (missing ${BUILDER_DIR}/auto/config)" fi echo "-- source bootloader templates --" for cfg in \ "${BUILDER_DIR}/config/bootloaders/grub-efi/grub.cfg" \ "${BUILDER_DIR}/config/bootloaders/isolinux/live.cfg.in"; do if [ -f "$cfg" ]; then echo " file: $cfg" grep -n 'Memory Test\|memtest' "$cfg" || echo " (no memtest lines)" fi done echo "-- source binary hooks --" for hook in \ "${BUILDER_DIR}/config/hooks/normal/9100-memtest.hook.binary"; do if [ -f "$hook" ]; then echo " hook: $hook" else echo " (missing $hook)" fi done if [ -n "$lb_dir" ] && [ -d "$lb_dir" ]; then echo "-- live-build workdir package lists --" for pkg in \ "$lb_dir/config/package-lists/bee.list.chroot" \ "$lb_dir/config/package-lists/bee-gpu.list.chroot" \ "$lb_dir/config/package-lists/bee-nvidia.list.chroot"; do if [ -f "$pkg" ]; then echo " file: $pkg" grep -n 'memtest' "$pkg" || echo " (no memtest lines)" fi done echo "-- live-build chroot/boot --" if [ -d "$lb_dir/chroot/boot" ]; then find "$lb_dir/chroot/boot" -maxdepth 1 -name 'memtest*' -print | sed 's/^/ /' || true else echo " (missing $lb_dir/chroot/boot)" fi echo "-- live-build binary/boot --" if [ -d "$lb_dir/binary/boot" ]; then find "$lb_dir/binary/boot" -maxdepth 1 -name 'memtest*' -print | sed 's/^/ /' || true else echo " (missing $lb_dir/binary/boot)" fi echo "-- live-build binary grub cfg --" if [ -f "$lb_dir/binary/boot/grub/grub.cfg" ]; then grep -n 'Memory Test\|memtest' "$lb_dir/binary/boot/grub/grub.cfg" || echo " (no memtest lines)" else echo " (missing $lb_dir/binary/boot/grub/grub.cfg)" fi echo "-- live-build binary isolinux cfg --" if [ -f "$lb_dir/binary/isolinux/live.cfg" ]; then grep -n 'Memory Test\|memtest' "$lb_dir/binary/isolinux/live.cfg" || echo " (no memtest lines)" else echo " (missing $lb_dir/binary/isolinux/live.cfg)" fi echo "-- live-build package cache --" if [ -d "$lb_dir/cache/packages.chroot" ]; then find "$lb_dir/cache/packages.chroot" -maxdepth 1 -name 'memtest86+*.deb' -print | sed 's/^/ /' || true else echo " (missing $lb_dir/cache/packages.chroot)" fi fi if [ -n "$iso_path" ] && [ -f "$iso_path" ]; then iso_files="$(mktemp)" iso_grub_cfg="$(mktemp)" iso_isolinux_cfg="$(mktemp)" echo "-- ISO memtest files --" if iso_read_file_list "$iso_path" "$iso_files"; then grep 'memtest' "$iso_files" | sed 's/^/ /' || echo " (no memtest files in ISO)" else echo " (failed to list ISO contents)" fi echo "-- ISO GRUB memtest lines --" if iso_read_member "$iso_path" boot/grub/grub.cfg "$iso_grub_cfg"; then grep -n 'Memory Test\|memtest' "$iso_grub_cfg" || echo " (no memtest lines in boot/grub/grub.cfg)" else echo " (failed to read boot/grub/grub.cfg from ISO)" fi echo "-- ISO isolinux memtest lines --" if iso_read_member "$iso_path" isolinux/live.cfg "$iso_isolinux_cfg"; then grep -n 'Memory Test\|memtest' "$iso_isolinux_cfg" || echo " (no memtest lines in isolinux/live.cfg)" else echo " (failed to read isolinux/live.cfg from ISO)" fi rm -f "$iso_files" "$iso_grub_cfg" "$iso_isolinux_cfg" fi echo "=== end memtest debug: ${phase} ===" ) | { if [ -n "${LOG_DIR:-}" ] && [ -d "${LOG_DIR}" ]; then tee "${memtest_log}" else cat fi } } memtest_fail() { msg="$1" iso_path="${2:-}" level="WARNING" if [ "${BEE_REQUIRE_MEMTEST:-0}" = "1" ]; then level="ERROR" fi echo "${level}: ${msg}" >&2 dump_memtest_debug "failure" "${LB_DIR:-}" "$iso_path" >&2 if [ "${BEE_REQUIRE_MEMTEST:-0}" = "1" ]; then exit 1 fi return 0 } nvidia_runtime_fail() { msg="$1" echo "ERROR: ${msg}" >&2 exit 1 } iso_memtest_present() { iso_path="$1" iso_files="$(mktemp)" [ -f "$iso_path" ] || return 1 if command -v bsdtar >/dev/null 2>&1; then : elif command -v xorriso >/dev/null 2>&1; then : else return 2 fi iso_read_file_list "$iso_path" "$iso_files" || { rm -f "$iso_files" return 2 } grep -q '^boot/memtest86+x64\.bin$' "$iso_files" || { rm -f "$iso_files" return 1 } grep -q '^boot/memtest86+x64\.efi$' "$iso_files" || { rm -f "$iso_files" return 1 } grub_cfg="$(mktemp)" isolinux_cfg="$(mktemp)" iso_read_member "$iso_path" boot/grub/grub.cfg "$grub_cfg" || { rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" return 2 } iso_read_member "$iso_path" isolinux/live.cfg "$isolinux_cfg" || { rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" return 2 } grep -q 'Memory Test (memtest86+)' "$grub_cfg" || { rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" return 1 } grep -q '/boot/memtest86+x64\.efi' "$grub_cfg" || { rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" return 1 } grep -q '/boot/memtest86+x64\.bin' "$grub_cfg" || { rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" return 1 } grep -q 'Memory Test (memtest86+)' "$isolinux_cfg" || { rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" return 1 } grep -q '/boot/memtest86+x64\.bin' "$isolinux_cfg" || { rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" return 1 } rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" return 0 } validate_iso_memtest() { iso_path="$1" echo "=== validating memtest in ISO ===" [ -f "$iso_path" ] || { memtest_fail "ISO not found for validation: $iso_path" "$iso_path" return 0 } require_iso_reader "$iso_path" || return 0 iso_files="$(mktemp)" iso_read_file_list "$iso_path" "$iso_files" || { memtest_fail "failed to list ISO contents while validating memtest" "$iso_path" rm -f "$iso_files" return 0 } grep -q '^boot/memtest86+x64\.bin$' "$iso_files" || { memtest_fail "memtest BIOS binary missing in ISO: boot/memtest86+x64.bin" "$iso_path" rm -f "$iso_files" return 0 } grep -q '^boot/memtest86+x64\.efi$' "$iso_files" || { memtest_fail "memtest EFI binary missing in ISO: boot/memtest86+x64.efi" "$iso_path" rm -f "$iso_files" return 0 } grub_cfg="$(mktemp)" isolinux_cfg="$(mktemp)" iso_read_member "$iso_path" boot/grub/grub.cfg "$grub_cfg" || { memtest_fail "failed to read boot/grub/grub.cfg from ISO" "$iso_path" rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" return 0 } iso_read_member "$iso_path" isolinux/live.cfg "$isolinux_cfg" || { memtest_fail "failed to read isolinux/live.cfg from ISO" "$iso_path" rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" return 0 } grep -q 'Memory Test (memtest86+)' "$grub_cfg" || { memtest_fail "GRUB menu entry for memtest is missing" "$iso_path" rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" return 0 } grep -q '/boot/memtest86+x64\.efi' "$grub_cfg" || { memtest_fail "GRUB memtest EFI path is missing" "$iso_path" rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" return 0 } grep -q '/boot/memtest86+x64\.bin' "$grub_cfg" || { memtest_fail "GRUB memtest BIOS path is missing" "$iso_path" rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" return 0 } grep -q 'Memory Test (memtest86+)' "$isolinux_cfg" || { memtest_fail "isolinux menu entry for memtest is missing" "$iso_path" rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" return 0 } grep -q '/boot/memtest86+x64\.bin' "$isolinux_cfg" || { memtest_fail "isolinux memtest path is missing" "$iso_path" rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" return 0 } rm -f "$iso_files" "$grub_cfg" "$isolinux_cfg" echo "=== memtest validation OK ===" } validate_live_cmdline_params() { cfg="$1" command="$2" bootloader="$3" expected_label="$4" awk -v command="$command" -v bootloader="$bootloader" -v expected_label="$expected_label" ' function has(token, i) { for (i = 1; i <= NF; i++) { if ($i == token) { return 1 } } return 0 } function reject(message) { printf "ERROR: %s live entry at %s:%d: %s\n", bootloader, FILENAME, NR, message bad = 1 } $1 == command { live_entries++ if (!has("boot=live")) { reject("missing boot=live") } if (!has("udev.children_max=1")) { reject("missing udev.children_max=1") } if (!has("intel_iommu=on")) { reject("missing intel_iommu=on") } if (!has("iommu.passthrough=0")) { reject("missing iommu.passthrough=0") } if (!has("efi=disable_early_pci_dma")) { reject("missing efi=disable_early_pci_dma") } if (!has("live-media-label=" expected_label)) { reject("missing expected live-media-label=" expected_label) } if (has("iommu=pt")) { reject("contains forbidden iommu=pt") } if (has("pci=realloc")) { failsafe_entries++ if (!has("iommu.strict=1")) { reject("pci=realloc entry is missing iommu.strict=1") } } else if (has("iommu.strict=1")) { reject("iommu.strict=1 is allowed only in the pci=realloc fail-safe entry") } } END { if (live_entries == 0) { printf "ERROR: %s config has no live boot entries\n", bootloader bad = 1 } if (failsafe_entries != 1) { printf "ERROR: %s config has %d pci=realloc fail-safe entries, expected 1\n", bootloader, failsafe_entries bad = 1 } exit bad ? 1 : 0 } ' "$cfg" } validate_iso_live_boot_entries() { iso_path="$1" echo "=== validating live boot entries in ISO ===" [ -f "$iso_path" ] || { echo "ERROR: ISO not found for live boot validation: $iso_path" >&2 exit 1 } require_iso_reader "$iso_path" >/dev/null 2>&1 || { echo "ERROR: ISO reader unavailable for live boot validation" >&2 exit 1 } grub_cfg="$(mktemp)" isolinux_cfg="$(mktemp)" iso_read_member "$iso_path" boot/grub/grub.cfg "$grub_cfg" || { echo "ERROR: failed to read boot/grub/grub.cfg from ISO" >&2 rm -f "$grub_cfg" "$isolinux_cfg" exit 1 } iso_read_member "$iso_path" isolinux/live.cfg "$isolinux_cfg" || { echo "ERROR: failed to read isolinux/live.cfg from ISO" >&2 rm -f "$grub_cfg" "$isolinux_cfg" exit 1 } if grep -Eq '@[A-Z][A-Z_]*@' "$grub_cfg" "$isolinux_cfg"; then echo "ERROR: unresolved live-build placeholders remain in ISO bootloader config" >&2 grep -En '@[A-Z][A-Z_]*@' "$grub_cfg" | sed 's/^/ GRUB: /' >&2 || true grep -En '@[A-Z][A-Z_]*@' "$isolinux_cfg" | sed 's/^/ isolinux: /' >&2 || true rm -f "$grub_cfg" "$isolinux_cfg" exit 1 fi if grep -q 'iommu=pt' "$grub_cfg" "$isolinux_cfg"; then echo "ERROR: forbidden iommu=pt remains in ISO bootloader config" >&2 rm -f "$grub_cfg" "$isolinux_cfg" exit 1 fi if ! validate_live_cmdline_params "$grub_cfg" linux GRUB "${BEE_ISO_VOLUME}"; then rm -f "$grub_cfg" "$isolinux_cfg" exit 1 fi if ! validate_live_cmdline_params "$isolinux_cfg" append isolinux "${BEE_ISO_VOLUME}"; then rm -f "$grub_cfg" "$isolinux_cfg" exit 1 fi grep -q 'menuentry "EASY-BEE v' "$grub_cfg" || { echo "ERROR: GRUB default EASY-BEE entry is missing" >&2 rm -f "$grub_cfg" "$isolinux_cfg" exit 1 } grep -Fq "menuentry \"EASY-BEE v${PROJECT_VERSION_EFFECTIVE}\"" "$grub_cfg" || { echo "ERROR: GRUB version does not match ${PROJECT_VERSION_EFFECTIVE}" >&2 rm -f "$grub_cfg" "$isolinux_cfg" exit 1 } grep -Fq "menu label ^EASY-BEE v${PROJECT_VERSION_EFFECTIVE}" "$isolinux_cfg" || { echo "ERROR: isolinux version does not match ${PROJECT_VERSION_EFFECTIVE}" >&2 rm -f "$grub_cfg" "$isolinux_cfg" exit 1 } if command -v xorriso >/dev/null 2>&1; then iso_volume="$(xorriso -indev "$iso_path" -pvd_info 2>/dev/null | awk -F: '/Volume [Ii]d/ { sub(/^[[:space:]'\''"]+/, "", $2); sub(/[[:space:]'\''"]+$/, "", $2); print $2; exit }')" if [ "$iso_volume" != "${BEE_ISO_VOLUME}" ]; then echo "ERROR: ISO volume ID is ${iso_volume:-unknown}, expected ${BEE_ISO_VOLUME}" >&2 rm -f "$grub_cfg" "$isolinux_cfg" exit 1 fi fi grep -q 'menuentry "EASY-BEE v.* -- load to RAM (toram)"' "$grub_cfg" || { echo "ERROR: GRUB toram entry is missing" >&2 rm -f "$grub_cfg" "$isolinux_cfg" exit 1 } grep -q 'linux .*boot=live ' "$grub_cfg" || { echo "ERROR: GRUB live entry is missing boot=live" >&2 rm -f "$grub_cfg" "$isolinux_cfg" exit 1 } grep -q 'linux .*boot=live .*toram ' "$grub_cfg" || { echo "ERROR: GRUB toram entry is missing boot=live or toram" >&2 rm -f "$grub_cfg" "$isolinux_cfg" exit 1 } grep -q 'linux .*live-media-label=EASY_BEE_' "$grub_cfg" || { echo "ERROR: GRUB live entry is missing live-media-label pinning" >&2 rm -f "$grub_cfg" "$isolinux_cfg" exit 1 } grep -q 'append .*boot=live ' "$isolinux_cfg" || { echo "ERROR: isolinux live entry is missing boot=live" >&2 rm -f "$grub_cfg" "$isolinux_cfg" exit 1 } grep -q 'append .*boot=live .*toram ' "$isolinux_cfg" || { echo "ERROR: isolinux toram entry is missing boot=live or toram" >&2 rm -f "$grub_cfg" "$isolinux_cfg" exit 1 } grep -q 'append .*live-media-label=EASY_BEE_' "$isolinux_cfg" || { echo "ERROR: isolinux live entry is missing live-media-label pinning" >&2 rm -f "$grub_cfg" "$isolinux_cfg" exit 1 } rm -f "$grub_cfg" "$isolinux_cfg" echo "=== live boot validation OK ===" } validate_iso_grub_assets() { iso_path="$1" echo "=== validating GRUB assets in ISO ===" [ -f "$iso_path" ] || { echo "ERROR: ISO not found for GRUB asset validation: $iso_path" >&2 exit 1 } require_iso_reader "$iso_path" >/dev/null 2>&1 || { echo "ERROR: ISO reader unavailable for GRUB asset validation" >&2 exit 1 } iso_files="$(mktemp)" iso_list_files "$iso_path" > "$iso_files" || { echo "ERROR: failed to list ISO files for GRUB asset validation" >&2 rm -f "$iso_files" exit 1 } for required in \ boot/grub/config.cfg \ boot/grub/grub.cfg; do grep -q "^${required}$" "$iso_files" || { echo "ERROR: missing GRUB asset in ISO: ${required}" >&2 rm -f "$iso_files" exit 1 } done rm -f "$iso_files" echo "=== GRUB asset validation OK ===" } validate_iso_nvidia_runtime() { iso_path="$1" [ "$BEE_GPU_VENDOR" = "nvidia" ] || return 0 echo "=== validating NVIDIA runtime in ISO ===" [ -f "$iso_path" ] || nvidia_runtime_fail "ISO not found for NVIDIA runtime validation: $iso_path" require_iso_reader "$iso_path" >/dev/null 2>&1 || nvidia_runtime_fail "ISO reader unavailable for NVIDIA runtime validation" command -v unsquashfs >/dev/null 2>&1 || nvidia_runtime_fail "unsquashfs is required for NVIDIA runtime validation" squashfs_tmp="$(mktemp)" squashfs_list="$(mktemp)" iso_files="$(mktemp)" dpkg_status_dir="$(mktemp -d)" iso_list_files "$iso_path" > "$iso_files" || { rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files" rm -rf "$dpkg_status_dir" nvidia_runtime_fail "failed to list ISO files for NVIDIA runtime validation" } grep '^live/.*\.squashfs$' "$iso_files" | while IFS= read -r squashfs_member; do iso_read_member "$iso_path" "$squashfs_member" "$squashfs_tmp" || { rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files" rm -rf "$dpkg_status_dir" nvidia_runtime_fail "failed to extract $squashfs_member from ISO" } unsquashfs -ll "$squashfs_tmp" >> "$squashfs_list" 2>/dev/null || { rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files" rm -rf "$dpkg_status_dir" nvidia_runtime_fail "failed to inspect $squashfs_member from ISO" } # var/lib/dpkg/status lives in whichever squashfs layer has the base # rootfs (not the usr/firmware split-off layers); harmless no-op on # the others. unsquashfs -d "${dpkg_status_dir}/extract" -f "$squashfs_tmp" var/lib/dpkg/status >/dev/null 2>&1 || true : > "$squashfs_tmp" done grep -Eq 'usr/bin/dcgmi$' "$squashfs_list" || { rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files" rm -rf "$dpkg_status_dir" nvidia_runtime_fail "dcgmi missing from final NVIDIA ISO" } grep -Eq 'usr/bin/nv-hostengine$' "$squashfs_list" || { rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files" rm -rf "$dpkg_status_dir" nvidia_runtime_fail "nv-hostengine missing from final NVIDIA ISO" } grep -Eq 'usr/bin/dcgmproftester([0-9]+)?$' "$squashfs_list" || { rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files" rm -rf "$dpkg_status_dir" nvidia_runtime_fail "dcgmproftester missing from final NVIDIA ISO" } # Cross-check the DCGM package version actually baked into the squashfs # against VERSIONS. dcgmi/nv-hostengine/dcgmproftester presence alone # doesn't catch a stale squashfs served by a mis-detected fast-path build # (dcgmi stays present across DCGM versions); this does. dpkg_status_file="${dpkg_status_dir}/extract/var/lib/dpkg/status" if [ -f "$dpkg_status_file" ]; then _installed_dcgm_version="$(awk ' /^Package: datacenter-gpu-manager-4-core$/ { in_pkg=1; next } /^Package: / { in_pkg=0 } in_pkg && /^Version: / { sub(/^Version: /, ""); print; exit } ' "$dpkg_status_file")" if [ -z "$_installed_dcgm_version" ]; then echo "=== WARNING: datacenter-gpu-manager-4-core not found in ISO dpkg status; skipping DCGM version check ===" else _installed_dcgm_no_epoch="${_installed_dcgm_version#*:}" if [ "$_installed_dcgm_no_epoch" != "${DCGM_VERSION}" ]; then rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files" rm -rf "$dpkg_status_dir" nvidia_runtime_fail "DCGM version mismatch: VERSIONS pins ${DCGM_VERSION} but ISO has ${_installed_dcgm_version} (stale squashfs; retry with --clean-build)" fi fi else echo "=== WARNING: could not read dpkg status from ISO; skipping DCGM version check ===" fi rm -f "$squashfs_tmp" "$squashfs_list" "$iso_files" rm -rf "$dpkg_status_dir" echo "=== NVIDIA runtime validation OK ===" }