From b1ab866f5815ede18a40fa3906a928e1a287be3d Mon Sep 17 00:00:00 2001 From: Mikhail Chusavitin Date: Thu, 3 Sep 2026 10:47:15 +0300 Subject: [PATCH] fix(runtime): make Copy to RAM switch live backing --- audit/internal/platform/install_to_ram.go | 172 ++++++++---------- .../internal/platform/install_to_ram_linux.go | 4 +- .../platform/install_to_ram_linux_test.go | 12 ++ .../internal/platform/install_to_ram_test.go | 20 +- bible-local/architecture/api-surface.md | 2 +- ...9-03-runtime-copy-to-ram-loop-change-fd.md | 50 +++++ bible-local/decisions/README.md | 1 + 7 files changed, 150 insertions(+), 111 deletions(-) create mode 100644 audit/internal/platform/install_to_ram_linux_test.go create mode 100644 bible-local/decisions/2026-09-03-runtime-copy-to-ram-loop-change-fd.md diff --git a/audit/internal/platform/install_to_ram.go b/audit/internal/platform/install_to_ram.go index 4f1e87f..25bfd66 100644 --- a/audit/internal/platform/install_to_ram.go +++ b/audit/internal/platform/install_to_ram.go @@ -155,41 +155,33 @@ func (s *System) RunInstallToRAM(ctx context.Context, logFunc func(string)) (ret log("Already running from RAM — installation media can be safely disconnected.") return nil } + originalStatus := state.LiveBootSource squashfsFiles, sourceAvailable := ensureLiveMediumAvailable(log) - dstDir := installToRAMDir + loopsSwitched := false + defer func() { + if retErr == nil || loopsSwitched { + return + } + _ = os.RemoveAll(dstDir) + log("Removed incomplete RAM copy.") + }() - // If the source medium is unavailable, check whether a previous run already - // produced a complete copy in RAM. If so, skip the copy phase and proceed - // directly to the loop-rebind / bind-mount steps. + // A worker or virtual-CD disconnect may have interrupted the task after a + // complete squashfs copy was fsynced. That copy is useful, but it is not + // success by itself: the active loop device still has to accept it as its + // backing file and the live-medium mount must move to tmpfs below. if !sourceAvailable { copiedFiles, _ := filepath.Glob(filepath.Join(dstDir, "*.squashfs")) - if len(copiedFiles) > 0 { - log("Source medium not available, but a previous RAM copy was found — resuming from existing copy.") - // Proceed to rebind with the already-copied files. - for _, dst := range copiedFiles { - base := filepath.Base(dst) - // Re-associate the loop device that was originally backed by the - // source file (now gone); find it by the old source path pattern. - srcGuess := "/run/live/medium/live/" + base - loopDev, lerr := findLoopForFile(srcGuess) - if lerr != nil { - log(fmt.Sprintf("Loop device for %s not found (%v) — skipping re-association.", base, lerr)) - continue - } - if rerr := reassociateLoopDevice(loopDev, dst); rerr != nil { - log(fmt.Sprintf("Warning: could not re-associate %s → %s: %v", loopDev, dst, rerr)) - } else { - log(fmt.Sprintf("Loop device %s now backed by RAM copy.", loopDev)) - } - } - goto bindMedium + if len(copiedFiles) == 0 { + return fmt.Errorf("no squashfs files found in /run/live/medium/live/ and no prior RAM copy in %s — reconnect the installation medium and retry (or run bee-remount-medium as root)", dstDir) } - return fmt.Errorf("no squashfs files found in /run/live/medium/live/ and no prior RAM copy in %s — reconnect the installation medium and retry (or run bee-remount-medium as root)", dstDir) + log("Source medium not available, but a previous RAM copy was found — verifying and resuming it.") + squashfsFiles = copiedFiles } - { + if sourceAvailable { free := freeMemBytes() var needed int64 for _, sf := range squashfsFiles { @@ -204,72 +196,78 @@ func (s *System) RunInstallToRAM(ctx context.Context, logFunc func(string)) (ret return fmt.Errorf("insufficient RAM: need %s, available %s", humanBytes(needed+headroom), humanBytes(free)) } - } - if state.CopyPresent { - log("Removing stale partial RAM copy before retry...") - } - _ = os.RemoveAll(dstDir) - if err := os.MkdirAll(dstDir, 0755); err != nil { - return fmt.Errorf("create tmpfs dir: %v", err) - } - defer func() { - if retErr == nil { - return + if state.CopyPresent { + log("Removing stale partial RAM copy before retry...") } _ = os.RemoveAll(dstDir) - log("Removed incomplete RAM copy.") - }() + if err := os.MkdirAll(dstDir, 0755); err != nil { + return fmt.Errorf("create tmpfs dir: %v", err) + } - for _, sf := range squashfsFiles { + for _, sf := range squashfsFiles { + if err := ctx.Err(); err != nil { + return err + } + base := filepath.Base(sf) + dst := filepath.Join(dstDir, base) + log(fmt.Sprintf("Copying %s to RAM...", base)) + if err := copyFileLarge(ctx, sf, dst, log); err != nil { + return fmt.Errorf("copy %s: %v", base, err) + } + log(fmt.Sprintf("Copied %s.", base)) + } + + log("Copying remaining medium files...") + if err := cpDir(ctx, "/run/live/medium", dstDir, log); err != nil { + log(fmt.Sprintf("Warning: partial metadata copy: %v", err)) + } if err := ctx.Err(); err != nil { return err } + } + + switchedCount := 0 + for _, sf := range squashfsFiles { base := filepath.Base(sf) dst := filepath.Join(dstDir, base) - log(fmt.Sprintf("Copying %s to RAM...", base)) - if err := copyFileLarge(ctx, sf, dst, log); err != nil { - return fmt.Errorf("copy %s: %v", base, err) + oldBacking := sf + if !sourceAvailable { + oldBacking = filepath.Join("/run/live/medium/live", base) } - log(fmt.Sprintf("Copied %s.", base)) - - loopDev, err := findLoopForFile(sf) + loopDev, err := findLoopForFile(oldBacking) if err != nil { - log(fmt.Sprintf("Loop device for %s not found (%v) — skipping re-association.", base, err)) - continue + return fmt.Errorf("find active loop device for %s: %v", base, err) } if err := reassociateLoopDevice(loopDev, dst); err != nil { - log(fmt.Sprintf("Warning: could not re-associate %s → %s: %v", loopDev, dst, err)) - } else { - log(fmt.Sprintf("Loop device %s now backed by RAM copy.", loopDev)) + return fmt.Errorf("switch %s to RAM copy %s: %v", loopDev, dst, err) + } + loopsSwitched = true + newLoop, err := findLoopForFile(dst) + if err != nil || newLoop != loopDev { + return fmt.Errorf("verify %s RAM backing: active backing file did not change to %s", loopDev, dst) + } + switchedCount++ + log(fmt.Sprintf("Loop device %s now backed by RAM copy.", loopDev)) + } + + if strings.TrimSpace(mountSource("/run/live/medium")) != "" { + log("Unmounting original live medium...") + if err := umountLiveMedium(); err != nil { + return fmt.Errorf("unmount original /run/live/medium: %v", err) } } - -bindMedium: - log("Copying remaining medium files...") - if err := cpDir(ctx, "/run/live/medium", dstDir, log); err != nil { - log(fmt.Sprintf("Warning: partial copy: %v", err)) - } - if err := ctx.Err(); err != nil { - return err - } - - mediumRebound := false if err := bindMount(dstDir, "/run/live/medium"); err != nil { - log(fmt.Sprintf("Warning: rebind /run/live/medium → %s failed: %v", dstDir, err)) - } else { - mediumRebound = true + return fmt.Errorf("bind RAM copy on /run/live/medium: %v", err) } log("Verifying live medium now served from RAM...") status := s.LiveBootSource() - if err := verifyInstallToRAMStatus(status, dstDir, mediumRebound, log); err != nil { + if err := verifyInstallToRAMStatus(status, switchedCount); err != nil { return err } - if status.InRAM { - log(fmt.Sprintf("Verification passed: live medium now served from %s.", describeLiveBootSource(status))) - } - detachInstallMedium(status, log) + log(fmt.Sprintf("Verification passed: %d loop device(s) and live medium now served from RAM.", switchedCount)) + detachInstallMedium(originalStatus, log) log("Done. Squashfs files are in RAM. Installation media has been detached when possible.") return nil } @@ -324,12 +322,6 @@ func detachInstallMedium(status LiveBootSource, log func(string)) { } log("Detaching original installation medium...") - if err := umountLiveMedium(); err != nil { - log(fmt.Sprintf("Warning: could not unmount /run/live/medium: %v", err)) - } else { - log("Unmounted /run/live/medium.") - } - device := strings.TrimSpace(status.Device) if device == "" { device = strings.TrimSpace(status.Source) @@ -346,27 +338,14 @@ func detachInstallMedium(status LiveBootSource, log func(string)) { log(fmt.Sprintf("Ejected %s.", device)) } -func verifyInstallToRAMStatus(status LiveBootSource, dstDir string, mediumRebound bool, log func(string)) error { - if status.InRAM { - return nil +func verifyInstallToRAMStatus(status LiveBootSource, switchedLoops int) error { + if switchedLoops <= 0 { + return fmt.Errorf("install to RAM verification failed: no active squashfs loop device was switched to RAM") } - - // The live medium mount was not redirected to RAM. This is expected when - // booting from an ISO/CD-ROM: the squashfs loop device has a non-zero - // offset and LOOP_CHANGE_FD cannot be used; the bind mount also fails - // because the CD-ROM mount is in use. Check whether files were at least - // copied to the tmpfs directory — that is sufficient for safe disconnection - // once the kernel has paged in all actively-used data. - files, _ := filepath.Glob(filepath.Join(dstDir, "*.squashfs")) - if len(files) > 0 { - if !mediumRebound { - log(fmt.Sprintf("Note: squashfs copied to RAM (%s) but /run/live/medium still shows the original source.", dstDir)) - log("This is normal for CD-ROM boots. For a fully transparent RAM boot, add 'toram' to the kernel parameters.") - } - return nil + if !status.InRAM { + return fmt.Errorf("install to RAM verification failed: live medium still mounted from %s", describeLiveBootSource(status)) } - - return fmt.Errorf("install to RAM verification failed: live medium still mounted from %s and no squashfs found in %s", describeLiveBootSource(status), dstDir) + return nil } func describeLiveBootSource(status LiveBootSource) string { @@ -519,8 +498,5 @@ func reassociateLoopDevice(loopDev, newFile string) error { if off := loopDeviceOffset(loopDev); off > 0 { return fmt.Errorf("loop device has non-zero offset (%d bytes, typical for ISO/CD-ROM) — LOOP_CHANGE_FD not supported; use 'toram' kernel parameter for RAM boot", off) } - if err := exec.Command("losetup", "--replace", loopDev, newFile).Run(); err == nil { - return nil - } return loopChangeFD(loopDev, newFile) } diff --git a/audit/internal/platform/install_to_ram_linux.go b/audit/internal/platform/install_to_ram_linux.go index 4538447..4c09a98 100644 --- a/audit/internal/platform/install_to_ram_linux.go +++ b/audit/internal/platform/install_to_ram_linux.go @@ -7,7 +7,9 @@ import ( "syscall" ) -const ioctlLoopChangeFD = 0x4C08 +// LOOP_CHANGE_FD from . 0x4C08 is LOOP_SET_DIRECT_IO and was +// the reason every runtime Copy to RAM attempt ended in EINVAL. +const ioctlLoopChangeFD = 0x4C06 func loopChangeFD(loopDev, newFile string) error { lf, err := os.OpenFile(loopDev, os.O_RDWR, 0) diff --git a/audit/internal/platform/install_to_ram_linux_test.go b/audit/internal/platform/install_to_ram_linux_test.go new file mode 100644 index 0000000..29498c6 --- /dev/null +++ b/audit/internal/platform/install_to_ram_linux_test.go @@ -0,0 +1,12 @@ +//go:build linux + +package platform + +import "testing" + +func TestLoopChangeFDIOCTLMatchesLinuxABI(t *testing.T) { + const linuxLoopChangeFD = 0x4C06 + if ioctlLoopChangeFD != linuxLoopChangeFD { + t.Fatalf("ioctlLoopChangeFD=%#x want LOOP_CHANGE_FD %#x", ioctlLoopChangeFD, linuxLoopChangeFD) + } +} diff --git a/audit/internal/platform/install_to_ram_test.go b/audit/internal/platform/install_to_ram_test.go index acd2793..5787379 100644 --- a/audit/internal/platform/install_to_ram_test.go +++ b/audit/internal/platform/install_to_ram_test.go @@ -36,19 +36,21 @@ func TestInferLiveBootKind(t *testing.T) { func TestVerifyInstallToRAMStatus(t *testing.T) { t.Parallel() - dstDir := t.TempDir() - - if err := verifyInstallToRAMStatus(LiveBootSource{InRAM: true, Kind: "ram", Source: "tmpfs"}, dstDir, false, nil); err != nil { + if err := verifyInstallToRAMStatus(LiveBootSource{InRAM: true, Kind: "ram", Source: "tmpfs"}, 1); err != nil { t.Fatalf("expected success for RAM-backed status, got %v", err) } - err := verifyInstallToRAMStatus(LiveBootSource{InRAM: false, Kind: "usb", Device: "/dev/sdb1"}, dstDir, false, nil) + err := verifyInstallToRAMStatus(LiveBootSource{InRAM: false, Kind: "usb", Device: "/dev/sdb1"}, 1) if err == nil { t.Fatal("expected verification failure when media is still on USB") } - if got := err.Error(); got != "install to RAM verification failed: live medium still mounted from USB (/dev/sdb1) and no squashfs found in "+dstDir { + if got := err.Error(); got != "install to RAM verification failed: live medium still mounted from USB (/dev/sdb1)" { t.Fatalf("error=%q", got) } + + if err := verifyInstallToRAMStatus(LiveBootSource{InRAM: true, Kind: "ram", Source: "tmpfs"}, 0); err == nil { + t.Fatal("expected verification failure when no loop device was switched") + } } func TestDescribeLiveBootSource(t *testing.T) { @@ -214,10 +216,9 @@ func TestDetachInstallMedium(t *testing.T) { }) t.Run("success", func(t *testing.T) { - var umountCalled bool var ejected string umountLiveMedium = func() error { - umountCalled = true + t.Fatal("detachInstallMedium must not unmount the RAM bind mount") return nil } ejectDevice = func(device string) error { @@ -226,13 +227,10 @@ func TestDetachInstallMedium(t *testing.T) { } var logs []string detachInstallMedium(LiveBootSource{Kind: "cdrom", Device: "/dev/sr1"}, func(msg string) { logs = append(logs, msg) }) - if !umountCalled { - t.Fatal("expected umountLiveMedium to be called") - } if ejected != "/dev/sr1" { t.Fatalf("ejected=%q want /dev/sr1", ejected) } - if len(logs) < 3 { + if len(logs) < 2 { t.Fatalf("logs=%v", logs) } }) diff --git a/bible-local/architecture/api-surface.md b/bible-local/architecture/api-surface.md index 99f6a8a..bbae0fa 100644 --- a/bible-local/architecture/api-surface.md +++ b/bible-local/architecture/api-surface.md @@ -138,7 +138,7 @@ All SAT run endpoints enqueue an async task. Response: `{"task_id": "..."}`. | Method | Path | Description | |--------|------------------------------|---------------------------------------------------| | GET | `/api/system/ram-status` | toram boot state and ISO copy status | -| POST | `/api/system/install-to-ram` | Copy ISO to RAM (background task) | +| POST | `/api/system/install-to-ram` | Copy ISO to RAM and verify active loop backing | | GET | `/api/system/time` | Current host epoch, local wall-clock text, and timezone | | POST | `/api/system/time-sync` | Set host time and timezone from the browser | | POST | `/api/system/reboot` | Reboot the host | diff --git a/bible-local/decisions/2026-09-03-runtime-copy-to-ram-loop-change-fd.md b/bible-local/decisions/2026-09-03-runtime-copy-to-ram-loop-change-fd.md new file mode 100644 index 0000000..bfd3fa9 --- /dev/null +++ b/bible-local/decisions/2026-09-03-runtime-copy-to-ram-loop-change-fd.md @@ -0,0 +1,50 @@ +# Runtime Copy to RAM must switch the active loop backing file + +**Date:** 2026-09-03 +**Status:** active + +## Evidence + +On `bee@172.16.41.97`, the runtime `Copy to RAM` task showed two contradictory +results: + +- with the BMC virtual CD connected, the 2.7 GB squashfs copy reached 100%, + loop reassociation logged `invalid argument`, and the worker failed; +- without the ISO connected, a later task found the stale file in + `/dev/shm/bee-live`, logged the same reassociation error, but returned + success. + +After the reported success, `/sys/block/loop0/loop/backing_file` still named +`/run/live/medium/live/filesystem-*.squashfs`, `/run/live/medium` was backed by +`/dev/sr0`, and disconnecting virtual media produced repeated `Medium not +present` and I/O errors. The system was not running independently from RAM. + +The Linux ABI defines `LOOP_CHANGE_FD` as `0x4C06` and +`LOOP_SET_DIRECT_IO` as `0x4C08`. The application had named `0x4C08` as +`ioctlLoopChangeFD`, so every fallback reassociation called the wrong ioctl +and received `EINVAL`. + +## Decision + +- Use the correct `LOOP_CHANGE_FD` request number, `0x4C06`, and protect it + with a Linux-specific regression test. +- Copy all source data before changing active loop devices. +- Treat a missing loop device, failed reassociation, or failed post-change + backing-file lookup as a fatal task error. A copied file alone is never + proof that the running system uses it. +- After all squashfs loops point at their files in `/dev/shm/bee-live`, + unmount the original live-medium mount and bind the RAM directory at + `/run/live/medium`. +- Success requires both at least one switched loop and a tmpfs-backed + `/run/live/medium`. +- Eject the original device without unmounting the new RAM bind mount. +- Preserve a complete RAM copy after any failure that occurs after loop + reassociation begins, so retry never deletes a file backing a live loop. + +## Consequences + +The button can resume a complete copy left by an interrupted worker even if +the virtual CD has since disappeared, but it reports success only if the +kernel actually accepts that file as the live loop backing. Partial or stale +copies can no longer produce a false `done` result. Boot-time `toram` remains +a separate initramfs path and is not evidence that the runtime button worked. diff --git a/bible-local/decisions/README.md b/bible-local/decisions/README.md index b89a132..792bf8c 100644 --- a/bible-local/decisions/README.md +++ b/bible-local/decisions/README.md @@ -17,3 +17,4 @@ One file per decision, named `YYYY-MM-DD-short-topic.md`. | 2026-08-31 | "Run All" SAT planning happens on the backend, not the browser | active | | 2026-08-31 | Support bundle uses private staging and unique atomic output | active | | 2026-09-03 | PCIe link verdict comes only from the existing real-traffic GPU bandwidth SAT | active | +| 2026-09-03 | Runtime Copy to RAM succeeds only after active loop devices move to tmpfs | active |