bee/iso/builder/smoketest.sh

#!/bin/sh
# smoketest.sh — run on a live ISO via SSH to verify all critical components.
#
# Usage:
#   ssh root@<ip> 'sh -s' < smoketest.sh
#   or: scp smoketest.sh root@<ip>:/var/tmp/ && ssh root@<ip> sh /var/tmp/smoketest.sh
#
# Exit code: 0 = all required checks passed, 1 = at least one required check failed.

PASS=0
FAIL=0
WARN=0

ok()   { echo "[  OK  ] $*"; PASS=$((PASS+1)); }
fail() { echo "[ FAIL ] $*"; FAIL=$((FAIL+1)); }
warn() { echo "[ WARN ] $*"; WARN=$((WARN+1)); }
info() { echo "[ INFO ] $*"; }

echo "========================================"
echo " bee live ISO smoketest"
echo " host:   $(uname -n)"
echo " kernel: $(uname -r)"
echo " date:   $(date -u)"
echo "========================================"
echo ""

KVER=$(uname -r)
info "kernel: $KVER"

# --- PATH & binaries ---
echo "-- PATH & binaries --"
for tool in dmidecode smartctl nvme ipmitool lspci audit; do
    if p=$(PATH="/usr/local/bin:$PATH" command -v "$tool" 2>/dev/null); then
        ok "$tool found: $p"
    else
        fail "$tool: NOT FOUND"
    fi
done

if p=$(PATH="/usr/local/bin:$PATH" command -v nvidia-smi 2>/dev/null); then
    ok "nvidia-smi found: $p"
else
    fail "nvidia-smi: NOT FOUND"
fi

echo ""
echo "-- NVIDIA modules --"
KO_DIR="/usr/local/lib/nvidia"
if [ -d "$KO_DIR" ]; then
    ko_count=$(ls "$KO_DIR"/*.ko 2>/dev/null | wc -l)
    ok "NVIDIA ko dir exists: $KO_DIR ($ko_count .ko files)"
else
    fail "NVIDIA ko dir missing: $KO_DIR"
fi

for mod in nvidia nvidia_modeset nvidia_uvm; do
    if /sbin/lsmod 2>/dev/null | grep -q "^$mod "; then
        ok "module loaded: $mod"
    else
        fail "module NOT loaded: $mod"
    fi
done

echo ""
echo "-- NVIDIA device nodes --"
for dev in nvidiactl nvidia0 nvidia-uvm; do
    if [ -e "/dev/$dev" ]; then
        ok "/dev/$dev exists"
    else
        fail "/dev/$dev missing — nvidia-smi will return NVML_ERROR_LIBRARY_NOT_FOUND"
    fi
done

echo ""
echo "-- nvidia-smi --"
if PATH="/usr/local/bin:$PATH" command -v nvidia-smi >/dev/null 2>&1; then
    if PATH="/usr/local/bin:$PATH" nvidia-smi -L 2>/dev/null | grep -q "GPU"; then
        gpu_count=$(PATH="/usr/local/bin:$PATH" nvidia-smi -L 2>/dev/null | grep -c "GPU")
        ok "nvidia-smi: $gpu_count GPU(s) found"
    else
        fail "nvidia-smi: runs but no GPUs detected"
    fi
else
    fail "nvidia-smi: not found in PATH"
fi

echo ""
echo "-- lib symlinks --"
for lib in libnvidia-ml libcuda; do
    if [ -f "/usr/lib/${lib}.so.1" ] || [ -L "/usr/lib/${lib}.so.1" ]; then
        ok "/usr/lib/${lib}.so.1 exists"
    else
        fail "/usr/lib/${lib}.so.1 MISSING (nvidia-smi will fail)"
    fi
done

echo ""
echo "-- systemd services --"
for svc in bee-nvidia bee-network bee-audit; do
    if systemctl is-active --quiet "$svc" 2>/dev/null; then
        ok "service active: $svc"
    else
        fail "service NOT active: $svc"
    fi
done

for svc in ssh bee-sshsetup; do
    if systemctl is-active --quiet "$svc" 2>/dev/null \
        || systemctl show "$svc" --property=ActiveState 2>/dev/null | grep -q "inactive\|exited"; then
        ok "service ok: $svc"
    else
        warn "service status unknown: $svc"
    fi
done

echo ""
echo "-- audit binary --"
AUDIT=/usr/local/bin/audit
if [ -x "$AUDIT" ]; then
    ok "audit binary: present"
    ver=$("$AUDIT" --version 2>/dev/null || "$AUDIT" version 2>/dev/null || echo "unknown")
    info "audit version: $ver"
else
    fail "audit binary: NOT FOUND at $AUDIT"
fi

echo ""
echo "-- audit last run --"
if [ -f /var/log/bee-audit.json ] && [ -s /var/log/bee-audit.json ]; then
    ok "audit: bee-audit.json present and non-empty"
    info "size: $(du -sh /var/log/bee-audit.json | cut -f1)"
else
    fail "audit: bee-audit.json missing or empty"
fi

if [ -f /var/log/bee-audit.log ]; then
    last_line=$(tail -1 /var/log/bee-audit.log)
    info "last log line: $last_line"
    if grep -q "audit output written" /var/log/bee-audit.log 2>/dev/null; then
        ok "audit: completed successfully"
    else
        warn "audit: 'audit output written' not found in log — may have failed"
    fi
    if grep -q "nvidia: enrichment skipped\|nvidia.*skipped\|enrichment skipped" /var/log/bee-audit.log 2>/dev/null; then
        reason=$(grep -E "nvidia.*skipped|enrichment skipped" /var/log/bee-audit.log | tail -1)
        fail "audit: nvidia enrichment skipped — $reason"
    else
        ok "audit: nvidia enrichment OK (no skip message)"
    fi
else
    warn "audit: no log found at /var/log/bee-audit.log"
fi

echo ""
echo "-- network --"
if ip route show default 2>/dev/null | grep -q "default"; then
    gw=$(ip route show default | awk '{print $3}' | head -1)
    ok "default route: $gw"
else
    fail "no default route"
fi
if ping -c1 -W3 1.1.1.1 >/dev/null 2>&1; then
    ok "internet: reachable (1.1.1.1)"
else
    warn "internet: unreachable (expected — live CD runs in isolated network segment)"
fi

echo ""
echo "-- /etc/profile.d/bee.sh PATH --"
if grep -q "/usr/local/bin" /etc/profile.d/bee.sh 2>/dev/null; then
    ok "/etc/profile.d/bee.sh exports /usr/local/bin"
else
    fail "/etc/profile.d/bee.sh does not add /usr/local/bin to PATH"
fi

echo ""
echo "========================================"
echo " Results: OK=$PASS  FAIL=$FAIL  WARN=$WARN"
echo "========================================"
[ "$FAIL" -eq 0 ] && exit 0 || exit 1