diff --git a/.gitignore b/.gitignore index 637d097..c9989e7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ .env .DS_Store dist/ -iso/out/ build-cache/ audit/bee diff --git a/iso/builder/build-in-container.sh b/iso/builder/build-in-container.sh index 38a687c..8cc761e 100755 --- a/iso/builder/build-in-container.sh +++ b/iso/builder/build-in-container.sh @@ -8,7 +8,7 @@ BUILDER_DIR="${REPO_ROOT}/iso/builder" CONTAINER_TOOL="${CONTAINER_TOOL:-docker}" IMAGE_TAG="${BEE_BUILDER_IMAGE:-bee-iso-builder}" BUILDER_PLATFORM="${BEE_BUILDER_PLATFORM:-linux/amd64}" -CACHE_DIR="${BEE_BUILDER_CACHE_DIR:-${REPO_ROOT}/dist/container-cache}" +CACHE_DIR="${BEE_BUILDER_CACHE_DIR:-${REPO_ROOT}/dist/cache}" AUTH_KEYS="" CLEAN_CACHE=0 VARIANT="all" @@ -54,14 +54,14 @@ if [ "$CLEAN_CACHE" = "1" ]; then "${CACHE_DIR:?}/bee" \ "${CACHE_DIR:?}/lb-packages" echo "=== cleaning live-build work dirs ===" - rm -rf "${REPO_ROOT}/dist/live-build-work-nvidia" - rm -rf "${REPO_ROOT}/dist/live-build-work-nvidia-legacy" - rm -rf "${REPO_ROOT}/dist/live-build-work-amd" - rm -rf "${REPO_ROOT}/dist/live-build-work-nogpu" - rm -rf "${REPO_ROOT}/dist/overlay-stage-nvidia" - rm -rf "${REPO_ROOT}/dist/overlay-stage-nvidia-legacy" - rm -rf "${REPO_ROOT}/dist/overlay-stage-amd" - rm -rf "${REPO_ROOT}/dist/overlay-stage-nogpu" + rm -rf "${REPO_ROOT}/dist/cache/live-build-work-nvidia" + rm -rf "${REPO_ROOT}/dist/cache/live-build-work-nvidia-legacy" + rm -rf "${REPO_ROOT}/dist/cache/live-build-work-amd" + rm -rf "${REPO_ROOT}/dist/cache/live-build-work-nogpu" + rm -rf "${REPO_ROOT}/dist/cache/overlay-stage-nvidia" + rm -rf "${REPO_ROOT}/dist/cache/overlay-stage-nvidia-legacy" + rm -rf "${REPO_ROOT}/dist/cache/overlay-stage-amd" + rm -rf "${REPO_ROOT}/dist/cache/overlay-stage-nogpu" echo "=== caches cleared, proceeding with build ===" fi diff --git a/iso/builder/build.sh b/iso/builder/build.sh index abbe5ab..673ad1e 100755 --- a/iso/builder/build.sh +++ b/iso/builder/build.sh @@ -51,8 +51,8 @@ case "$BUILD_VARIANT" in ;; esac -BUILD_WORK_DIR="${DIST_DIR}/live-build-work-${BUILD_VARIANT}" -OVERLAY_STAGE_DIR="${DIST_DIR}/overlay-stage-${BUILD_VARIANT}" +BUILD_WORK_DIR="${DIST_DIR}/cache/live-build-work-${BUILD_VARIANT}" +OVERLAY_STAGE_DIR="${DIST_DIR}/cache/overlay-stage-${BUILD_VARIANT}" export BEE_GPU_VENDOR BEE_NVIDIA_MODULE_FLAVOR BUILD_VARIANT @@ -63,7 +63,7 @@ export PATH="$PATH:/usr/local/go/bin" # Allow git to read the bind-mounted repo (different UID inside container). git config --global safe.directory "${REPO_ROOT}" -mkdir -p "${DIST_DIR}" +mkdir -p "${DIST_DIR}/cache" "${DIST_DIR}/release" mkdir -p "${CACHE_ROOT}" : "${GOCACHE:=${CACHE_ROOT}/go-build}" : "${GOMODCACHE:=${CACHE_ROOT}/go-mod}" @@ -1105,7 +1105,7 @@ PROJECT_VERSION_EFFECTIVE="$(resolve_project_version)" SQUASHFS_FILENAME="filesystem-v${PROJECT_VERSION_EFFECTIVE}.squashfs" ISO_BASENAME="easy-bee-${BUILD_VARIANT}-v${PROJECT_VERSION_EFFECTIVE}-amd64" # Versioned output directory: dist/easy-bee-v4.1/ — all final artefacts live here. -OUT_DIR="${DIST_DIR}/easy-bee-v${PROJECT_VERSION_EFFECTIVE}" +OUT_DIR="${DIST_DIR}/release/easy-bee-v${PROJECT_VERSION_EFFECTIVE}" ISO_VERSION_LABEL_TOKEN="$(printf '%s' "${PROJECT_VERSION_EFFECTIVE}" | tr '[:lower:].-' '[:upper:]__')" mkdir -p "${OUT_DIR}" LOG_DIR="${OUT_DIR}/${ISO_BASENAME}.logs" @@ -1290,7 +1290,7 @@ run_step "sync git submodules" "05-git-submodules" \ # --- compile bee binary (static, Linux amd64) --- # Shared between variants — built once, reused on second pass. -BEE_BIN="${DIST_DIR}/bee-linux-amd64" +BEE_BIN="${DIST_DIR}/cache/bee-linux-amd64" NEED_BUILD=1 if [ -f "$BEE_BIN" ]; then NEWEST_SRC=$(find "${REPO_ROOT}/audit" -name '*.go' -newer "$BEE_BIN" | head -1) @@ -1321,16 +1321,16 @@ else fi # --- NVIDIA-only build steps --- -GPU_BURN_WORKER_BIN="${DIST_DIR}/bee-gpu-burn-worker-linux-amd64" +GPU_BURN_WORKER_BIN="${DIST_DIR}/cache/bee-gpu-burn-worker-linux-amd64" if [ "$BEE_GPU_VENDOR" = "nvidia" ]; then run_step "download cuBLAS/cuBLASLt/cudart ${NCCL_CUDA_VERSION} userspace" "20-cublas" \ sh "${BUILDER_DIR}/build-cublas.sh" \ "${CUBLAS_VERSION}" \ "${CUDA_USERSPACE_VERSION}" \ "${NCCL_CUDA_VERSION}" \ - "${DIST_DIR}" + "${DIST_DIR}/cache" - CUBLAS_CACHE="${DIST_DIR}/cublas-${CUBLAS_VERSION}+cuda${NCCL_CUDA_VERSION}" + CUBLAS_CACHE="${DIST_DIR}/cache/cublas-${CUBLAS_VERSION}+cuda${NCCL_CUDA_VERSION}" echo "=== bee-gpu-burn FP4 header probe ===" fp4_type_match="$(grep -Rsnm 1 'CUDA_R_4F_E2M1' "${CUBLAS_CACHE}/include" 2>/dev/null || true)" @@ -1456,7 +1456,7 @@ fi # --- copy bee binary into overlay --- mkdir -p "${OVERLAY_STAGE_DIR}/usr/local/bin" -cp "${DIST_DIR}/bee-linux-amd64" "${OVERLAY_STAGE_DIR}/usr/local/bin/bee" +cp "$BEE_BIN" "${OVERLAY_STAGE_DIR}/usr/local/bin/bee" chmod +x "${OVERLAY_STAGE_DIR}/usr/local/bin/bee" if [ "$BEE_GPU_VENDOR" = "nvidia" ] && [ -f "$GPU_BURN_WORKER_BIN" ]; then @@ -1486,10 +1486,10 @@ done # --- NVIDIA kernel modules and userspace libs --- if [ "$BEE_GPU_VENDOR" = "nvidia" ]; then run_step "build NVIDIA ${NVIDIA_DRIVER_VERSION} modules" "40-nvidia-module" \ - sh "${BUILDER_DIR}/build-nvidia-module.sh" "${NVIDIA_DRIVER_VERSION}" "${DIST_DIR}" "${DEBIAN_KERNEL_ABI}" "${BEE_NVIDIA_MODULE_FLAVOR}" + sh "${BUILDER_DIR}/build-nvidia-module.sh" "${NVIDIA_DRIVER_VERSION}" "${DIST_DIR}/cache" "${DEBIAN_KERNEL_ABI}" "${BEE_NVIDIA_MODULE_FLAVOR}" KVER="${DEBIAN_KERNEL_ABI}-amd64" - NVIDIA_CACHE="${DIST_DIR}/nvidia-${BEE_NVIDIA_MODULE_FLAVOR}-${NVIDIA_DRIVER_VERSION}-${KVER}" + NVIDIA_CACHE="${DIST_DIR}/cache/nvidia-${BEE_NVIDIA_MODULE_FLAVOR}-${NVIDIA_DRIVER_VERSION}-${KVER}" # Inject .ko files into overlay at /usr/local/lib/nvidia/ OVERLAY_KMOD_DIR="${OVERLAY_STAGE_DIR}/usr/local/lib/nvidia" @@ -1515,9 +1515,9 @@ if [ "$BEE_GPU_VENDOR" = "nvidia" ]; then # --- build / download NCCL --- run_step "download NCCL ${NCCL_VERSION}+cuda${NCCL_CUDA_VERSION}" "50-nccl" \ - sh "${BUILDER_DIR}/build-nccl.sh" "${NCCL_VERSION}" "${NCCL_CUDA_VERSION}" "${DIST_DIR}" "${NCCL_SHA256:-}" + sh "${BUILDER_DIR}/build-nccl.sh" "${NCCL_VERSION}" "${NCCL_CUDA_VERSION}" "${DIST_DIR}/cache" "${NCCL_SHA256:-}" - NCCL_CACHE="${DIST_DIR}/nccl-${NCCL_VERSION}+cuda${NCCL_CUDA_VERSION}" + NCCL_CACHE="${DIST_DIR}/cache/nccl-${NCCL_VERSION}+cuda${NCCL_CUDA_VERSION}" # Inject libnccl.so.* into overlay alongside other NVIDIA userspace libs cp "${NCCL_CACHE}/lib/"* "${OVERLAY_STAGE_DIR}/usr/lib/" @@ -1533,19 +1533,19 @@ if [ "$BEE_GPU_VENDOR" = "nvidia" ]; then "${NCCL_TESTS_VERSION}" \ "${NCCL_VERSION}" \ "${NCCL_CUDA_VERSION}" \ - "${DIST_DIR}" \ + "${DIST_DIR}/cache" \ "${NVCC_VERSION}" \ "${DEBIAN_VERSION}" - NCCL_TESTS_CACHE="${DIST_DIR}/nccl-tests-${NCCL_TESTS_VERSION}" + NCCL_TESTS_CACHE="${DIST_DIR}/cache/nccl-tests-${NCCL_TESTS_VERSION}" cp "${NCCL_TESTS_CACHE}/bin/all_reduce_perf" "${OVERLAY_STAGE_DIR}/usr/local/bin/all_reduce_perf" chmod +x "${OVERLAY_STAGE_DIR}/usr/local/bin/all_reduce_perf" cp "${NCCL_TESTS_CACHE}/lib/"* "${OVERLAY_STAGE_DIR}/usr/lib/" 2>/dev/null || true echo "=== all_reduce_perf injected ===" run_step "build john jumbo ${JOHN_JUMBO_COMMIT}" "70-john" \ - sh "${BUILDER_DIR}/build-john.sh" "${JOHN_JUMBO_COMMIT}" "${DIST_DIR}" - JOHN_CACHE="${DIST_DIR}/john-${JOHN_JUMBO_COMMIT}" + sh "${BUILDER_DIR}/build-john.sh" "${JOHN_JUMBO_COMMIT}" "${DIST_DIR}/cache" + JOHN_CACHE="${DIST_DIR}/cache/john-${JOHN_JUMBO_COMMIT}" mkdir -p "${OVERLAY_STAGE_DIR}/usr/local/lib/bee/john" rsync -a --delete "${JOHN_CACHE}/run/" "${OVERLAY_STAGE_DIR}/usr/local/lib/bee/john/run/" ln -sfn ../lib/bee/john/run/john "${OVERLAY_STAGE_DIR}/usr/local/bin/john" diff --git a/scripts/build.sh b/scripts/build.sh index a5133e3..dee6987 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -27,7 +27,7 @@ BUILDER_HOST="${BUILDER_HOST:-}" BUILDER_USER="${BUILDER_USER:-}" # Cache lives inside the repo under dist/ (gitignored). -CACHE_DIR="${REPO_ROOT}/dist/container-cache" +CACHE_DIR="${REPO_ROOT}/dist/cache" # Forward all arguments as-is to the underlying build script. EXTRA_ARGS="$*" @@ -79,7 +79,7 @@ screen -S bee-build -X quit 2>/dev/null || true echo "--- starting build in screen session (survives SSH disconnect) ---" echo "--- log: \$LOG ---" -screen -dmS bee-build sh -c "sh iso/builder/build-in-container.sh --cache-dir \$REPO/dist/container-cache ${EXTRA_ARGS} > \$LOG 2>&1; echo \$? > /tmp/bee-build-exit" +screen -dmS bee-build sh -c "sh iso/builder/build-in-container.sh --cache-dir \$REPO/dist/cache ${EXTRA_ARGS} > \$LOG 2>&1; echo \$? > /tmp/bee-build-exit" echo "--- streaming build log (Ctrl+C safe -- build continues on VM) ---" tail -n +1 -f "\$LOG" 2>/dev/null & @@ -98,7 +98,7 @@ ENDSSH echo "" echo "=== downloading ISO ===" - LOCAL_ISO_DIR="${REPO_ROOT}/iso/out" + LOCAL_ISO_DIR="${REPO_ROOT}/dist/release" mkdir -p "${LOCAL_ISO_DIR}" if command -v rsync >/dev/null 2>&1 && ssh -o StrictHostKeyChecking=no "${BUILDER_USER}@${BUILDER_HOST}" command -v rsync >/dev/null 2>&1; then rsync -az --progress \