355 lines
13 KiB
Bash
Executable File
355 lines
13 KiB
Bash
Executable File
#!/bin/sh
|
|
# build-hpl.sh — build HPL (High Performance LINPACK) for the bee LiveCD.
|
|
#
|
|
# Downloads HPL 2.3 from netlib, downloads OpenBLAS runtime from the Debian 12
|
|
# apt repo, and compiles xhpl using a minimal single-process MPI stub so that
|
|
# no MPI package is required inside the ISO.
|
|
#
|
|
# The resulting xhpl binary is a standard HPL binary whose output is compatible
|
|
# with the accepted HPL format (WR... Gflops lines).
|
|
#
|
|
# Output:
|
|
# $CACHE_DIR/bin/xhpl
|
|
# $CACHE_DIR/lib/libopenblas.so* (runtime, injected into ISO /usr/lib/)
|
|
|
|
set -e
|
|
|
|
HPL_VERSION="$1"
|
|
HPL_SHA256="$2"
|
|
DIST_DIR="$3"
|
|
|
|
[ -n "$HPL_VERSION" ] || { echo "usage: $0 <hpl-version> <sha256> <dist-dir>"; exit 1; }
|
|
[ -n "$HPL_SHA256" ] || { echo "usage: $0 <hpl-version> <sha256> <dist-dir>"; exit 1; }
|
|
[ -n "$DIST_DIR" ] || { echo "usage: $0 <hpl-version> <sha256> <dist-dir>"; exit 1; }
|
|
|
|
echo "=== HPL ${HPL_VERSION} ==="
|
|
|
|
CACHE_DIR="${DIST_DIR}/hpl-${HPL_VERSION}"
|
|
CACHE_ROOT="${BEE_CACHE_DIR:-${DIST_DIR}/cache}"
|
|
DOWNLOAD_CACHE_DIR="${CACHE_ROOT}/hpl-downloads"
|
|
HPL_SOURCE_META="${DOWNLOAD_CACHE_DIR}/hpl-${HPL_VERSION}.source"
|
|
|
|
cache_valid() {
|
|
[ -x "${CACHE_DIR}/bin/xhpl" ] || return 1
|
|
[ -L "${CACHE_DIR}/lib/libopenblas.so" ] || return 1
|
|
[ -L "${CACHE_DIR}/lib/libblas.so" ] || return 1
|
|
find "${CACHE_DIR}/lib" -maxdepth 1 -name 'libopenblas*.so*' -type f | grep -q .
|
|
}
|
|
|
|
if cache_valid; then
|
|
echo "=== HPL cached, skipping build ==="
|
|
echo "binary: ${CACHE_DIR}/bin/xhpl"
|
|
exit 0
|
|
fi
|
|
|
|
rm -rf "${CACHE_DIR}"
|
|
mkdir -p "${DOWNLOAD_CACHE_DIR}" "${CACHE_DIR}/bin" "${CACHE_DIR}/lib"
|
|
|
|
# ── download HPL source ────────────────────────────────────────────────────────
|
|
HPL_TAR="${DOWNLOAD_CACHE_DIR}/hpl-${HPL_VERSION}.tar.gz"
|
|
DEFAULT_HPL_URLS="
|
|
https://www.netlib.org/benchmark/hpl/hpl-${HPL_VERSION}.tar.gz
|
|
https://fossies.org/linux/privat/hpl-${HPL_VERSION}.tar.gz
|
|
"
|
|
HPL_GIT_URL="${HPL_GIT_URL:-https://github.com/icl-utk-edu/hpl.git}"
|
|
DEFAULT_HPL_GIT_REFS="v${HPL_VERSION} ${HPL_VERSION} main"
|
|
HPL_SOURCE_MODE="tarball"
|
|
|
|
download_to_file() {
|
|
url="$1"
|
|
out="$2"
|
|
|
|
if command -v curl >/dev/null 2>&1; then
|
|
curl -fL \
|
|
--connect-timeout 15 \
|
|
--max-time 180 \
|
|
--retry 2 \
|
|
--retry-delay 2 \
|
|
--output "${out}" \
|
|
"${url}"
|
|
return $?
|
|
fi
|
|
|
|
wget \
|
|
--show-progress \
|
|
--tries=2 \
|
|
--timeout=30 \
|
|
-O "${out}" \
|
|
"${url}"
|
|
}
|
|
|
|
download_hpl_tarball() {
|
|
out="$1"
|
|
tmp="${out}.part"
|
|
urls="${HPL_URLS:-$DEFAULT_HPL_URLS}"
|
|
|
|
rm -f "${tmp}"
|
|
for url in ${urls}; do
|
|
[ -n "${url}" ] || continue
|
|
echo "=== trying HPL source: ${url} ==="
|
|
if download_to_file "${url}" "${tmp}"; then
|
|
mv "${tmp}" "${out}"
|
|
printf 'mode=tarball\nurl=%s\n' "${url}" > "${HPL_SOURCE_META}"
|
|
return 0
|
|
fi
|
|
rm -f "${tmp}"
|
|
echo "=== failed: ${url} ==="
|
|
done
|
|
|
|
echo "ERROR: failed to download HPL ${HPL_VERSION} from all configured URLs" >&2
|
|
return 1
|
|
}
|
|
|
|
download_hpl_from_git_archive() {
|
|
out="$1"
|
|
refs="${HPL_GIT_REFS:-$DEFAULT_HPL_GIT_REFS}"
|
|
tmp_root="$(mktemp -d)"
|
|
repo_dir="${tmp_root}/repo"
|
|
archive_dir="${tmp_root}/hpl-${HPL_VERSION}"
|
|
archive_tmp="${out}.part"
|
|
|
|
for ref in ${refs}; do
|
|
[ -n "${ref}" ] || continue
|
|
echo "=== trying HPL git source: ${HPL_GIT_URL} ref ${ref} ==="
|
|
rm -rf "${repo_dir}" "${archive_dir}" "${archive_tmp}"
|
|
if git clone --depth 1 --branch "${ref}" "${HPL_GIT_URL}" "${repo_dir}"; then
|
|
resolved_commit="$(git -C "${repo_dir}" rev-parse HEAD)"
|
|
mv "${repo_dir}" "${archive_dir}"
|
|
tar czf "${archive_tmp}" -C "${tmp_root}" "hpl-${HPL_VERSION}"
|
|
mv "${archive_tmp}" "${out}"
|
|
printf 'mode=git\nurl=%s\nref=%s\ncommit=%s\n' "${HPL_GIT_URL}" "${ref}" "${resolved_commit}" > "${HPL_SOURCE_META}"
|
|
rm -rf "${tmp_root}"
|
|
HPL_SOURCE_MODE="git"
|
|
return 0
|
|
fi
|
|
echo "=== failed git ref: ${ref} ==="
|
|
done
|
|
|
|
rm -rf "${tmp_root}" "${archive_tmp}"
|
|
echo "ERROR: failed to obtain HPL ${HPL_VERSION} from all configured sources" >&2
|
|
echo " looked for cache: ${out}" >&2
|
|
echo " tarball mirrors: ${HPL_URLS:-$DEFAULT_HPL_URLS}" >&2
|
|
echo " git fallback: ${HPL_GIT_URL} refs ${refs}" >&2
|
|
echo " override mirrors with HPL_URLS=\"https://mirror1/...\"" >&2
|
|
echo " override git refs with HPL_GIT_REFS=\"v${HPL_VERSION} ${HPL_VERSION} main\"" >&2
|
|
return 1
|
|
}
|
|
|
|
if [ -f "${HPL_SOURCE_META}" ]; then
|
|
case "$(awk -F= '$1=="mode"{print $2}' "${HPL_SOURCE_META}" 2>/dev/null)" in
|
|
git) HPL_SOURCE_MODE="git" ;;
|
|
tarball) HPL_SOURCE_MODE="tarball" ;;
|
|
esac
|
|
fi
|
|
|
|
if [ ! -f "${HPL_TAR}" ]; then
|
|
echo "=== downloading HPL ${HPL_VERSION} ==="
|
|
download_hpl_tarball "${HPL_TAR}" || download_hpl_from_git_archive "${HPL_TAR}"
|
|
fi
|
|
|
|
if [ "${HPL_SOURCE_MODE}" = "tarball" ]; then
|
|
actual_sha="$(sha256sum "${HPL_TAR}" | awk '{print $1}')"
|
|
if [ "${actual_sha}" != "${HPL_SHA256}" ]; then
|
|
echo "ERROR: sha256 mismatch for hpl-${HPL_VERSION}.tar.gz" >&2
|
|
echo " expected: ${HPL_SHA256}" >&2
|
|
echo " actual: ${actual_sha}" >&2
|
|
rm -f "${HPL_TAR}"
|
|
exit 1
|
|
fi
|
|
echo "sha256 OK: hpl-${HPL_VERSION}.tar.gz"
|
|
else
|
|
echo "=== HPL source obtained from git fallback; skipping tarball sha256 check ==="
|
|
[ -f "${HPL_SOURCE_META}" ] && cat "${HPL_SOURCE_META}"
|
|
fi
|
|
|
|
# ── download OpenBLAS from Debian 12 apt repo ─────────────────────────────────
|
|
REPO_BASE="https://deb.debian.org/debian/pool/main/o/openblas"
|
|
PACKAGES_GZ="${DOWNLOAD_CACHE_DIR}/Packages.gz"
|
|
OPENBLAS_PKG="libopenblas0-openmp"
|
|
|
|
echo "=== fetching Debian 12 Packages.gz ==="
|
|
wget -q -O "${PACKAGES_GZ}" \
|
|
"https://deb.debian.org/debian/dists/bookworm/main/binary-amd64/Packages.gz"
|
|
|
|
lookup_deb() {
|
|
pkg="$1"
|
|
gzip -dc "${PACKAGES_GZ}" | awk -v pkg="$pkg" '
|
|
/^Package: / { cur=$2 }
|
|
/^Filename: / { file=$2 }
|
|
/^SHA256: / { sha=$2 }
|
|
/^$/ {
|
|
if (cur == pkg) { print file " " sha; found=1; exit }
|
|
cur=""; file=""; sha=""
|
|
}
|
|
END {
|
|
if (!found && cur == pkg) print file " " sha
|
|
}'
|
|
}
|
|
|
|
meta="$(lookup_deb "${OPENBLAS_PKG}")"
|
|
[ -n "$meta" ] || { echo "ERROR: ${OPENBLAS_PKG} not found in Packages.gz"; exit 1; }
|
|
repo_file="$(printf '%s' "$meta" | awk '{print $1}')"
|
|
repo_sha="$(printf '%s' "$meta" | awk '{print $2}')"
|
|
|
|
OPENBLAS_DEB="${DOWNLOAD_CACHE_DIR}/$(basename "${repo_file}")"
|
|
if [ -f "${OPENBLAS_DEB}" ]; then
|
|
actual="$(sha256sum "${OPENBLAS_DEB}" | awk '{print $1}')"
|
|
[ "$actual" = "$repo_sha" ] || rm -f "${OPENBLAS_DEB}"
|
|
fi
|
|
if [ ! -f "${OPENBLAS_DEB}" ]; then
|
|
echo "=== downloading ${OPENBLAS_PKG} ==="
|
|
wget --show-progress -O "${OPENBLAS_DEB}" "https://deb.debian.org/debian/${repo_file}"
|
|
actual="$(sha256sum "${OPENBLAS_DEB}" | awk '{print $1}')"
|
|
[ "$actual" = "$repo_sha" ] || { echo "ERROR: sha256 mismatch for ${OPENBLAS_PKG}"; rm -f "${OPENBLAS_DEB}"; exit 1; }
|
|
fi
|
|
|
|
# extract libopenblas shared libs
|
|
TMP_DEB=$(mktemp -d)
|
|
trap 'rm -rf "${TMP_DEB}" "${BUILD_TMP:-}"' EXIT INT TERM
|
|
(
|
|
cd "${TMP_DEB}"
|
|
ar x "${OPENBLAS_DEB}"
|
|
tar xf data.tar.*
|
|
)
|
|
find "${TMP_DEB}" \( -name 'libopenblas*.so*' \) \( -type f -o -type l \) \
|
|
-exec cp -a {} "${CACHE_DIR}/lib/" \;
|
|
echo "=== OpenBLAS libs: $(ls "${CACHE_DIR}/lib/" | wc -l) files ==="
|
|
|
|
# Debian runtime packages may ship the real ELF under a variant-specific name
|
|
# such as libopenblasp-r0.3.xx.so, while libopenblas.so.0 is only a symlink.
|
|
# Pick any real shared object we extracted, then synthesize the generic linker
|
|
# names HPL expects.
|
|
OPENBLAS_REAL_SO="$(find "${CACHE_DIR}/lib" -maxdepth 1 -name 'libopenblas*.so*' -type f | sort | head -1)"
|
|
[ -n "${OPENBLAS_REAL_SO}" ] || { echo "ERROR: libopenblas shared object not extracted"; ls -l "${CACHE_DIR}/lib"; exit 1; }
|
|
OPENBLAS_REAL_NAME="$(basename "${OPENBLAS_REAL_SO}")"
|
|
ln -sf "${OPENBLAS_REAL_NAME}" "${CACHE_DIR}/lib/libopenblas.so" 2>/dev/null || true
|
|
ln -sf "${OPENBLAS_REAL_NAME}" "${CACHE_DIR}/lib/libblas.so" 2>/dev/null || true
|
|
|
|
# ── build HPL ─────────────────────────────────────────────────────────────────
|
|
BUILD_TMP=$(mktemp -d)
|
|
|
|
cd "${BUILD_TMP}"
|
|
tar xf "${HPL_TAR}"
|
|
SRC_DIR="$(find . -maxdepth 1 -type d -name 'hpl-*' | head -1)"
|
|
[ -n "${SRC_DIR}" ] || { echo "ERROR: HPL source dir not found"; exit 1; }
|
|
cd "${SRC_DIR}"
|
|
|
|
# Write a minimal single-process MPI stub so we don't need an MPI package.
|
|
# HPL only needs these functions for single-process execution.
|
|
cat > "${BUILD_TMP}/mpi_stub.c" <<'MPISTUB'
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
#include <sys/time.h>
|
|
|
|
typedef int MPI_Comm;
|
|
typedef int MPI_Datatype;
|
|
typedef int MPI_Op;
|
|
typedef int MPI_Status;
|
|
typedef int MPI_Request;
|
|
|
|
#define MPI_COMM_WORLD 0
|
|
#define MPI_SUCCESS 0
|
|
#define MPI_DOUBLE 6
|
|
#define MPI_INT 5
|
|
#define MPI_SUM 0
|
|
#define MPI_MAX 1
|
|
#define MPI_MIN 2
|
|
#define MPI_BYTE 1
|
|
#define MPI_ANY_SOURCE -1
|
|
#define MPI_ANY_TAG -1
|
|
#define MPI_STATUS_IGNORE ((MPI_Status*)0)
|
|
|
|
int MPI_Init(int *argc, char ***argv) { (void)argc; (void)argv; return MPI_SUCCESS; }
|
|
int MPI_Finalize(void) { return MPI_SUCCESS; }
|
|
int MPI_Comm_rank(MPI_Comm c, int *rank) { (void)c; *rank = 0; return MPI_SUCCESS; }
|
|
int MPI_Comm_size(MPI_Comm c, int *size) { (void)c; *size = 1; return MPI_SUCCESS; }
|
|
int MPI_Bcast(void *b, int n, MPI_Datatype t, int r, MPI_Comm c)
|
|
{ (void)b;(void)n;(void)t;(void)r;(void)c; return MPI_SUCCESS; }
|
|
int MPI_Reduce(const void *s, void *r, int n, MPI_Datatype t, MPI_Op op, int root, MPI_Comm c) {
|
|
(void)op;(void)root;(void)c;
|
|
size_t sz = (t==MPI_DOUBLE)?sizeof(double):(t==MPI_INT)?sizeof(int):1;
|
|
memcpy(r, s, (size_t)n * sz);
|
|
return MPI_SUCCESS;
|
|
}
|
|
int MPI_Allreduce(const void *s, void *r, int n, MPI_Datatype t, MPI_Op op, MPI_Comm c)
|
|
{ return MPI_Reduce(s,r,n,t,op,0,c); }
|
|
int MPI_Send(const void *b, int n, MPI_Datatype t, int d, int tag, MPI_Comm c)
|
|
{ (void)b;(void)n;(void)t;(void)d;(void)tag;(void)c; return MPI_SUCCESS; }
|
|
int MPI_Recv(void *b, int n, MPI_Datatype t, int s, int tag, MPI_Comm c, MPI_Status *st)
|
|
{ (void)b;(void)n;(void)t;(void)s;(void)tag;(void)c;(void)st; return MPI_SUCCESS; }
|
|
int MPI_Sendrecv(const void *sb, int sn, MPI_Datatype st2, int dest, int stag,
|
|
void *rb, int rn, MPI_Datatype rt, int src, int rtag,
|
|
MPI_Comm c, MPI_Status *status)
|
|
{ (void)sb;(void)sn;(void)st2;(void)dest;(void)stag;
|
|
(void)rb;(void)rn;(void)rt;(void)src;(void)rtag;(void)c;(void)status;
|
|
return MPI_SUCCESS; }
|
|
int MPI_Irecv(void *b, int n, MPI_Datatype t, int s, int tag, MPI_Comm c, MPI_Request *req)
|
|
{ (void)b;(void)n;(void)t;(void)s;(void)tag;(void)c;(void)req; return MPI_SUCCESS; }
|
|
int MPI_Wait(MPI_Request *req, MPI_Status *st)
|
|
{ (void)req;(void)st; return MPI_SUCCESS; }
|
|
int MPI_Abort(MPI_Comm c, int code) { (void)c; exit(code); }
|
|
double MPI_Wtime(void) {
|
|
struct timeval tv;
|
|
gettimeofday(&tv, NULL);
|
|
return (double)tv.tv_sec + (double)tv.tv_usec * 1e-6;
|
|
}
|
|
MPISTUB
|
|
|
|
# Write Make.bee — HPL makefile configuration
|
|
cat > Make.bee <<MAKEFILE
|
|
SHELL = /bin/sh
|
|
CD = cd
|
|
CP = cp
|
|
LN_S = ln -s
|
|
MKDIR = mkdir -p
|
|
RM = /bin/rm -f
|
|
TOUCH = touch
|
|
ARCH = bee
|
|
|
|
# Directories
|
|
TOPdir = \$(shell pwd)
|
|
INCdir = \$(TOPdir)/include
|
|
BINdir = \$(TOPdir)/bin/\$(ARCH)
|
|
LIBdir = \$(TOPdir)/lib/\$(ARCH)
|
|
HPLlib = \$(LIBdir)/libhpl.a
|
|
|
|
# Compiler
|
|
CC = gcc
|
|
CCNOOPT = \$(HPL_DEFS)
|
|
CCFLAGS = \$(HPL_DEFS) -O3 -mtune=generic -funroll-loops -fomit-frame-pointer
|
|
|
|
# Linker
|
|
LINKER = gcc
|
|
LINKFLAGS = \$(CCFLAGS)
|
|
|
|
# MPI (single-process stub — no actual MPI needed)
|
|
MPdir =
|
|
MPinc = -I${BUILD_TMP}
|
|
MPlib = ${BUILD_TMP}/mpi_stub.o
|
|
|
|
# BLAS (OpenBLAS)
|
|
LAdir = ${CACHE_DIR}/lib
|
|
LAinc =
|
|
LAlib = -L\$(LAdir) -Wl,-rpath,/usr/lib -lopenblas
|
|
|
|
HPL_OPTS =
|
|
HPL_DEFS = \$(HPL_OPTS) -DHPL_CALL_CBLAS
|
|
MAKEFILE
|
|
echo "=== Make.bee written ==="
|
|
|
|
# compile MPI stub
|
|
gcc -O2 -c -o "${BUILD_TMP}/mpi_stub.o" "${BUILD_TMP}/mpi_stub.c"
|
|
|
|
# build HPL
|
|
echo "=== building HPL ${HPL_VERSION} ==="
|
|
make -j"$(nproc)" arch=bee 2>&1 | tail -20
|
|
|
|
XHPL_BIN="bin/bee/xhpl"
|
|
[ -x "${XHPL_BIN}" ] || { echo "ERROR: xhpl not found after build"; exit 1; }
|
|
|
|
cp "${XHPL_BIN}" "${CACHE_DIR}/bin/xhpl"
|
|
chmod +x "${CACHE_DIR}/bin/xhpl"
|
|
echo "=== HPL build complete ==="
|
|
echo "binary: ${CACHE_DIR}/bin/xhpl"
|
|
echo "libs: $(ls "${CACHE_DIR}/lib/")"
|