fix(iso): retry toram RAM copy with exponential backoff

On hosts where the boot medium is BMC/IPMI virtual media, the ~2.8 GB
squashfs read often drops mid-transfer: rsync dies with "Input/output
error (5)", every file "has vanished", and live-boot moves the partial
RAM copy over the medium, ending the boot with "No supported filesystem
images found at /live".

Add hook 9013-toram-retry, which patches live-boot's 9990-toram-todisk.sh
so the whole-medium copy runs in a retry loop: up to 8 attempts with a
geometrically growing pause (15s -> 900s cap), unmounting/waiting/remounting
the medium between tries and honouring rsync's exit code (upstream ignores
it). rsync resumes from where it stopped, so a retry needing only the tail
finishes fast.

Verified against live-boot 1:20230131+deb12u1: substitution applies,
sh -n is clean, hook is idempotent.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_018JA6Cj84yxUfk8N3ppw6mr

iso/builder/config/hooks/normal/9013-toram-retry.hook.chroot

@@ -0,0 +1,118 @@
+#!/bin/sh
+# 9013-toram-retry.hook.chroot — wrap live-boot's "copy whole medium to RAM"
+# rsync in a retry loop with exponential backoff.
+#
+# Why: on some hosts the boot medium (notably BMC / IPMI virtual CD redirection)
+# drops off the bus part-way through the ~2.8 GB squashfs read.  rsync then dies
+# with "Input/output error (5)" and every file "has vanished", live-boot moves
+# the partial RAM copy over the medium, and the boot ends with
+# "No supported filesystem images found at /live" -> BOOT FAILED.
+#
+# live-boot's own retry is just rsync's single in-run delta retry.  This patch
+# adds several full attempts with a geometrically growing pause between them
+# (15s, 30s, 60s, 120s, 240s, 480s, 900s), giving the virtual media time to
+# re-enumerate.  Between attempts the medium is unmounted, waited on, and
+# remounted.  rsync resumes (already-copied files are skipped), so a retry that
+# only needs the tail of the squashfs finishes quickly.
+set -e
+
+TORAM_SCRIPT="/usr/lib/live/boot/9990-toram-todisk.sh"
+
+if [ ! -f "${TORAM_SCRIPT}" ]; then
+    echo "9013-toram-retry: ${TORAM_SCRIPT} not found, skipping"
+    exit 0
+fi
+
+if grep -q "bee_rsync_retry" "${TORAM_SCRIPT}"; then
+    echo "9013-toram-retry: already patched, skipping"
+    exit 0
+fi
+
+RSYNC_LINE='rsync -a --progress ${copyfrom}/\* ${copyto} 1>/dev/console'
+if ! grep -qF 'rsync -a --progress ${copyfrom}/* ${copyto} 1>/dev/console' "${TORAM_SCRIPT}"; then
+    echo "9013-toram-retry: expected rsync line not found in ${TORAM_SCRIPT}" >&2
+    echo "9013-toram-retry: live-boot layout changed -- refusing to patch blindly" >&2
+    exit 1
+fi
+
+echo "9013-toram-retry: patching ${TORAM_SCRIPT}"
+
+# 1. Inject the retry helper right after the shebang.
+cat > /tmp/bee_rsync_retry.func <<'FUNC'
+
+# --- bee: retry the whole-medium RAM copy with exponential backoff ----------
+bee_rsync_retry ()
+{
+	_src="${1}"
+	_dst="${2}"
+	_dev=$(awk -v m="${_src}" '$2 == m {print $1; exit}' /proc/mounts)
+	_fst=$(awk -v m="${_src}" '$2 == m {print $3; exit}' /proc/mounts)
+	_delay=15
+	_maxdelay=900
+	_try=1
+	_maxtry=8
+
+	while : ; do
+		echo " * bee: toram copy attempt ${_try}/${_maxtry} (medium ${_dev:-?} ${_fst:-?})" 1>/dev/console
+
+		if rsync -a --progress --timeout=180 ${_src}/* ${_dst} 1>/dev/console
+		then
+			echo " * bee: toram copy completed on attempt ${_try}" 1>/dev/console
+			return 0
+		fi
+
+		echo " * bee: toram copy FAILED (rsync rc=$?) on attempt ${_try}" 1>/dev/console
+
+		if [ "${_try}" -ge "${_maxtry}" ]
+		then
+			echo " * bee: giving up after ${_try} attempts" 1>/dev/console
+			return 1
+		fi
+
+		echo " * bee: waiting ${_delay}s for the medium to re-settle..." 1>/dev/console
+		sleep "${_delay}"
+
+		# Recover the medium: unmount, wait for the device node, remount.
+		if [ -n "${_dev}" ]
+		then
+			umount "${_src}" 2>/dev/null || umount -l "${_src}" 2>/dev/null || true
+			_w=0
+			while [ ! -b "${_dev}" ] && [ "${_w}" -lt 120 ]
+			do
+				sleep 3
+				_w=$((_w + 3))
+			done
+			mount -r ${_fst:+-t ${_fst}} "${_dev}" "${_src}" 2>/dev/null \
+				|| mount -r "${_dev}" "${_src}" 2>/dev/null \
+				|| echo " * bee: remount of ${_dev} failed, will retry anyway" 1>/dev/console
+		fi
+
+		_try=$((_try + 1))
+		_delay=$((_delay * 2))
+		[ "${_delay}" -gt "${_maxdelay}" ] && _delay="${_maxdelay}"
+	done
+}
+FUNC
+
+sed -i "2r /tmp/bee_rsync_retry.func" "${TORAM_SCRIPT}"
+rm -f /tmp/bee_rsync_retry.func
+
+# 2. Route the whole-medium copy through the helper and honour its exit code
+#    (upstream ignores rsync's exit status and moves the partial copy anyway).
+sed -i \
+  's#\([[:space:]]*\)rsync -a --progress ${copyfrom}/[*] ${copyto} 1>/dev/console.*#\1bee_rsync_retry "${copyfrom}" "${copyto}" || { log_warning_msg "bee: toram copy failed after all retries"; rmdir "${copyto}" 2>/dev/null || true; return 1; }#' \
+  "${TORAM_SCRIPT}"
+
+if ! grep -q "bee_rsync_retry \"\${copyfrom}\"" "${TORAM_SCRIPT}"; then
+    echo "9013-toram-retry: call-site substitution failed" >&2
+    exit 1
+fi
+
+echo "9013-toram-retry: patch applied"
+grep -n "bee_rsync_retry" "${TORAM_SCRIPT}"
+
+# 3. Rebuild the initramfs so the patched script lands in initrd.img.
+KVER=$(ls /lib/modules | sort -V | tail -1)
+echo "9013-toram-retry: rebuilding initramfs for kernel ${KVER}"
+update-initramfs -u -k "${KVER}"
+echo "9013-toram-retry: done"