fix(webui): repair broken scenario Run button onclick, dedupe build.sh overlay staging

- page_scenario.go: onclick built via JSON.stringify() embedded raw double
  quotes inside a double-quoted HTML attribute, truncating the attribute so
  the click handler never compiled; pass the name through an escaped
  data-scenario-name attribute instead.
- build.sh: overlay staging rsyncs (OVERLAY_DIR->stage, stage->includes.chroot)
  ran without --delete, so a scenario removed from the repo (a9924b0) stayed
  baked into every ISO built from the persistent stage cache since — the
  "second script" in the Scenario page's list.
- blackbox: rewritten around a deterministic local zip + incremental
  patch-the-changed-suffix onto removable media, instead of walking/copying
  ~90 files through a synchronous ntfs-3g FUSE mount every cycle. journalctl
  captures are now "--since last sync" (were "--since boot", growing with
  uptime) and metrics.db is excluded (was copied whole every cycle).
- scenario: nvbandwidth-acs-ab now escalates GPU count (same-socket pair,
  other socket's pair, one cross-socket pair, all GPUs) under each ACS state
  instead of always running all 6 GPUs at once, using a new `bee
  gpu-bandwidth-groups` subcommand that discovers socket layout from
  `nvidia-smi topo -m` at runtime — gpu_indices is host-specific, so this
  can't be baked into the scenario file.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
2026-07-29 18:11:24 +03:00
co-authored by Claude Sonnet 5
parent 1045fa9118
commit 8a91f0f783
12 changed files with 847 additions and 32 deletions
+63 -9
View File
@@ -572,6 +572,13 @@ func adjustFlushPeriod(current, duration time.Duration, success bool, fastCycles
return next
}
// syncCycle stages the current export tree on fast local storage (never
// touching the removable-media mountpoint for the expensive part), packs it
// into a single deterministic zip, and patches only the changed suffix of
// that zip onto the target device — see blackbox_archive.go for why: walking
// and rewriting ~90 small files through a synchronous FUSE mount (ntfs-3g
// -o sync) was taking ~2x the flush period, dominated by journalctl output
// that grows with uptime and got fully re-read/re-written every cycle.
func (w *blackboxWorker) syncCycle() error {
target, marker := w.snapshotTarget()
mountpoint, mountedByBee, err := ensureMountedTarget(target, marker.EnrollmentID)
@@ -580,24 +587,59 @@ func (w *blackboxWorker) syncCycle() error {
}
w.recordMountpoint(mountpoint, mountedByBee)
root := filepath.Join(mountpoint, w.runtime.bootFolder)
if err := os.MkdirAll(root, 0755); err != nil {
stageRoot := filepath.Join(w.runtime.exportDir, ".blackbox-stage", w.enrollmentID)
if err := os.RemoveAll(stageRoot); err != nil {
return err
}
if err := categorizeExportTree(w.runtime.exportDir, root); err != nil {
// includeMetricsDB=false: metrics.db is a live, growing SQLite file: past
// versions copied it whole every cycle, which alone could dominate a
// cycle's cost as it grew. Not carried onto the blackbox mirror.
if err := categorizeExportTree(w.runtime.exportDir, stageRoot, false); err != nil {
return err
}
if err := w.captureSnapshots(stageRoot, w.lastCaptureSince()); err != nil {
return err
}
// Same doc pair the support bundle ships at its root — a blackbox
// capture on removable media has no manifest.txt/support-bundle
// equivalent to explain its layout, so without this an agent handed
// only the media would have nothing pointing it at README.md.
if err := writeBundleDocs(root); err != nil {
if err := writeBundleDocs(stageRoot); err != nil {
return err
}
if err := w.captureSnapshots(root); err != nil {
cacheDir := filepath.Join(w.runtime.exportDir, ".blackbox-cache")
if err := os.MkdirAll(cacheDir, 0755); err != nil {
return err
}
return syncFilesystem(root)
cachedPath := filepath.Join(cacheDir, w.enrollmentID+".zip")
newZipPath := filepath.Join(cacheDir, w.enrollmentID+".zip.new")
if err := buildZipArchive(stageRoot, newZipPath); err != nil {
return err
}
targetPath := filepath.Join(mountpoint, w.runtime.bootFolder+" blackbox.zip")
if err := patchArchiveOnTarget(targetPath, newZipPath, cachedPath); err != nil {
return err
}
if err := os.Rename(newZipPath, cachedPath); err != nil {
return err
}
return nil
}
// lastCaptureSince is the "--since" boundary for this cycle's incremental
// journalctl captures: the previous successful sync, or boot time on the
// very first cycle. Using the last sync instead of always "--since boot"
// keeps each cycle's journalctl output bounded by the flush period instead
// of growing with total uptime.
func (w *blackboxWorker) lastCaptureSince() time.Time {
w.mu.Lock()
defer w.mu.Unlock()
if w.lastSyncAt.IsZero() {
return w.runtime.bootStarted
}
return w.lastSyncAt
}
func (w *blackboxWorker) cleanup() {
@@ -623,13 +665,25 @@ func (w *blackboxWorker) recordMountpoint(mountpoint string, mountedByBee bool)
w.mountedByBee = mountedByBee
}
func (w *blackboxWorker) captureSnapshots(root string) error {
if err := captureCommandAtomic(filepath.Join(root, "tasks", "_services", "combined.journal.log"), "journalctl", "--no-pager", "--since", w.runtime.bootStarted.Format(time.RFC3339)); err != nil {
// captureSnapshots writes this cycle's journalctl/dmesg/status snapshots
// into root (a local staging tree — see syncCycle). journalctl output is
// captured incrementally ("--since" the last successful cycle, not boot) and
// appended as a new timestamped entry under journal-increments/ each cycle,
// rather than overwriting one ever-growing file: re-dumping "--since boot"
// every cycle made each cycle's journalctl call (and the file it produced)
// grow with total uptime, independent of how much actually happened since
// the last sync.
func (w *blackboxWorker) captureSnapshots(root string, since time.Time) error {
cycleTS := blackboxNow().Format("20060102-150405.000")
sinceArg := since.Format(time.RFC3339)
incDir := filepath.Join(root, "tasks", "_services", "journal-increments")
if err := captureCommandAtomic(filepath.Join(incDir, "combined-"+cycleTS+".log"), "journalctl", "--no-pager", "--since", sinceArg); err != nil {
return err
}
for _, svc := range supportBundleServices {
dir := filepath.Join(root, serviceBundleDir(svc))
if err := captureCommandAtomic(filepath.Join(dir, svc+".journal.log"), "journalctl", "--no-pager", "-u", svc, "--since", w.runtime.bootStarted.Format(time.RFC3339)); err != nil {
if err := captureCommandAtomic(filepath.Join(incDir, svc+"-"+cycleTS+".log"), "journalctl", "--no-pager", "-u", svc, "--since", sinceArg); err != nil {
return err
}
if err := captureCommandAtomic(filepath.Join(dir, svc+".status.txt"), "systemctl", "status", svc, "--no-pager"); err != nil {
+183
View File
@@ -0,0 +1,183 @@
package app
import (
"archive/zip"
"bufio"
"io"
"io/fs"
"os"
"path/filepath"
)
// blackboxArchiveCompareChunk is the buffer size used when diffing the newly
// built local zip against the last one successfully written to removable
// media, to find how many leading bytes are unchanged.
const blackboxArchiveCompareChunk = 256 * 1024
// buildZipArchive walks root (a locally-staged, fast-storage copy of the
// blackbox tree — never the slow removable-media mountpoint) and writes a
// single deterministic zip to destPath: same input tree -> byte-identical
// output, so two cycles that changed nothing produce identical archives and
// patchArchiveOnTarget (below) can skip re-writing the unchanged prefix to
// the slow target. Determinism relies on fs.WalkDir's guaranteed lexical
// order and each entry's Modified time coming from the source file's mtime
// (stable across cycles for files nothing touched).
func buildZipArchive(root, destPath string) error {
if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil {
return err
}
f, err := os.OpenFile(destPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
if err != nil {
return err
}
zw := zip.NewWriter(f)
walkErr := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
return nil
}
rel, err := filepath.Rel(root, path)
if err != nil {
return err
}
info, err := d.Info()
if err != nil {
return err
}
header, err := zip.FileInfoHeader(info)
if err != nil {
return err
}
header.Name = filepath.ToSlash(rel)
header.Method = zip.Deflate
w, err := zw.CreateHeader(header)
if err != nil {
return err
}
src, err := os.Open(path)
if err != nil {
return err
}
defer src.Close()
_, err = io.Copy(w, src)
return err
})
if walkErr != nil {
_ = zw.Close()
_ = f.Close()
return walkErr
}
if err := zw.Close(); err != nil {
_ = f.Close()
return err
}
return f.Close()
}
// commonPrefixLen returns how many leading bytes two files share. Used to
// find how much of a freshly-rebuilt local zip is identical to the previous
// cycle's, so patchArchiveOnTarget only has to write the changed suffix to
// removable media.
func commonPrefixLen(pathA, pathB string) (int64, error) {
fa, err := os.Open(pathA)
if err != nil {
return 0, err
}
defer fa.Close()
fb, err := os.Open(pathB)
if err != nil {
return 0, err
}
defer fb.Close()
ra := bufio.NewReaderSize(fa, blackboxArchiveCompareChunk)
rb := bufio.NewReaderSize(fb, blackboxArchiveCompareChunk)
bufA := make([]byte, blackboxArchiveCompareChunk)
bufB := make([]byte, blackboxArchiveCompareChunk)
var total int64
for {
na, errA := io.ReadFull(ra, bufA)
nb, errB := io.ReadFull(rb, bufB)
n := na
if nb < n {
n = nb
}
for i := 0; i < n; i++ {
if bufA[i] != bufB[i] {
return total + int64(i), nil
}
}
total += int64(n)
if na != nb || isEOFLike(errA) || isEOFLike(errB) {
return total, nil
}
if errA != nil {
return total, errA
}
if errB != nil {
return total, errB
}
}
}
func isEOFLike(err error) bool {
return err == io.EOF || err == io.ErrUnexpectedEOF
}
// patchArchiveOnTarget makes targetPath (on removable media, possibly
// FUSE-mounted with synchronous writes) byte-identical to newLocalZipPath (on
// fast local storage), writing only the changed suffix instead of the whole
// file. cachedPath is our own local record of what we last wrote to
// targetPath; if targetPath's size doesn't match what cachedPath implies
// (first run, external tampering, a previous crash mid-write), it falls back
// to writing the whole archive rather than risk corrupting it with a wrong
// truncate point.
func patchArchiveOnTarget(targetPath, newLocalZipPath, cachedPath string) error {
newInfo, err := os.Stat(newLocalZipPath)
if err != nil {
return err
}
var prefixLen int64
if cachedInfo, err := os.Stat(cachedPath); err == nil {
if targetInfo, err := os.Stat(targetPath); err == nil && targetInfo.Size() == cachedInfo.Size() {
prefixLen, err = commonPrefixLen(cachedPath, newLocalZipPath)
if err != nil {
prefixLen = 0
}
}
}
if prefixLen > newInfo.Size() {
prefixLen = 0
}
target, err := os.OpenFile(targetPath, os.O_CREATE|os.O_RDWR, 0644)
if err != nil {
return err
}
defer target.Close()
if err := target.Truncate(prefixLen); err != nil {
return err
}
if _, err := target.Seek(prefixLen, io.SeekStart); err != nil {
return err
}
src, err := os.Open(newLocalZipPath)
if err != nil {
return err
}
defer src.Close()
if _, err := src.Seek(prefixLen, io.SeekStart); err != nil {
return err
}
if _, err := io.Copy(target, src); err != nil {
return err
}
return target.Sync()
}
+235
View File
@@ -0,0 +1,235 @@
package app
import (
"archive/zip"
"bytes"
"io"
"os"
"path/filepath"
"sort"
"testing"
"time"
)
func writeTestFile(t *testing.T, path string, content string, mtime time.Time) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
t.Fatalf("mkdir: %v", err)
}
if err := os.WriteFile(path, []byte(content), 0644); err != nil {
t.Fatalf("write: %v", err)
}
if err := os.Chtimes(path, mtime, mtime); err != nil {
t.Fatalf("chtimes: %v", err)
}
}
func readZipEntries(t *testing.T, path string) map[string]string {
t.Helper()
r, err := zip.OpenReader(path)
if err != nil {
t.Fatalf("open zip %s: %v", path, err)
}
defer r.Close()
out := map[string]string{}
for _, f := range r.File {
rc, err := f.Open()
if err != nil {
t.Fatalf("open entry %s: %v", f.Name, err)
}
data, err := io.ReadAll(rc)
rc.Close()
if err != nil {
t.Fatalf("read entry %s: %v", f.Name, err)
}
out[f.Name] = string(data)
}
return out
}
func TestBuildZipArchiveRoundTrip(t *testing.T) {
root := t.TempDir()
mtime := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
writeTestFile(t, filepath.Join(root, "a.txt"), "hello", mtime)
writeTestFile(t, filepath.Join(root, "sub", "b.txt"), "world", mtime)
dest := filepath.Join(t.TempDir(), "out.zip")
if err := buildZipArchive(root, dest); err != nil {
t.Fatalf("buildZipArchive: %v", err)
}
entries := readZipEntries(t, dest)
if entries["a.txt"] != "hello" || entries["sub/b.txt"] != "world" {
t.Fatalf("entries=%+v", entries)
}
}
func TestBuildZipArchiveDeterministicWhenUnchanged(t *testing.T) {
root := t.TempDir()
mtime := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
writeTestFile(t, filepath.Join(root, "a.txt"), "hello", mtime)
writeTestFile(t, filepath.Join(root, "b.txt"), "world", mtime)
dir := t.TempDir()
p1 := filepath.Join(dir, "one.zip")
p2 := filepath.Join(dir, "two.zip")
if err := buildZipArchive(root, p1); err != nil {
t.Fatalf("build 1: %v", err)
}
if err := buildZipArchive(root, p2); err != nil {
t.Fatalf("build 2: %v", err)
}
b1, _ := os.ReadFile(p1)
b2, _ := os.ReadFile(p2)
if !bytes.Equal(b1, b2) {
t.Fatalf("two builds of an unchanged tree produced different bytes (len %d vs %d)", len(b1), len(b2))
}
}
func TestPatchArchiveOnTargetIncrementalAndValid(t *testing.T) {
root := t.TempDir()
mtime := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
writeTestFile(t, filepath.Join(root, "static.txt"), "does not change", mtime)
writeTestFile(t, filepath.Join(root, "growing.txt"), "cycle-1 line\n", mtime)
work := t.TempDir()
cachedPath := filepath.Join(work, "cache.zip")
newPath := filepath.Join(work, "new.zip")
targetPath := filepath.Join(work, "target.zip")
// Cycle 1: cold start, no cache, no target yet.
if err := buildZipArchive(root, newPath); err != nil {
t.Fatalf("build cycle1: %v", err)
}
if err := patchArchiveOnTarget(targetPath, newPath, cachedPath); err != nil {
t.Fatalf("patch cycle1: %v", err)
}
if err := os.Rename(newPath, cachedPath); err != nil {
t.Fatalf("rename cycle1: %v", err)
}
want1, _ := os.ReadFile(cachedPath)
got1, _ := os.ReadFile(targetPath)
if !bytes.Equal(want1, got1) {
t.Fatalf("cycle1: target does not match freshly built zip")
}
entries1 := readZipEntries(t, targetPath)
if entries1["static.txt"] != "does not change" || entries1["growing.txt"] != "cycle-1 line\n" {
t.Fatalf("cycle1 entries=%+v", entries1)
}
// Cycle 2: only growing.txt changes (as it would with an appended log).
writeTestFile(t, filepath.Join(root, "growing.txt"), "cycle-1 line\ncycle-2 line\n", mtime.Add(time.Minute))
if err := buildZipArchive(root, newPath); err != nil {
t.Fatalf("build cycle2: %v", err)
}
prefixLen, err := commonPrefixLen(cachedPath, newPath)
if err != nil {
t.Fatalf("commonPrefixLen: %v", err)
}
newInfo, _ := os.Stat(newPath)
if prefixLen <= 0 {
t.Fatalf("expected a nonzero unchanged prefix (static.txt entry should be identical), got %d", prefixLen)
}
if prefixLen >= newInfo.Size() {
t.Fatalf("expected the changed growing.txt entry to make the archives diverge before EOF, prefixLen=%d size=%d", prefixLen, newInfo.Size())
}
if err := patchArchiveOnTarget(targetPath, newPath, cachedPath); err != nil {
t.Fatalf("patch cycle2: %v", err)
}
if err := os.Rename(newPath, cachedPath); err != nil {
t.Fatalf("rename cycle2: %v", err)
}
want2, _ := os.ReadFile(cachedPath)
got2, _ := os.ReadFile(targetPath)
if !bytes.Equal(want2, got2) {
t.Fatalf("cycle2: target does not match freshly built zip after incremental patch")
}
entries2 := readZipEntries(t, targetPath)
if entries2["growing.txt"] != "cycle-1 line\ncycle-2 line\n" {
t.Fatalf("cycle2 growing.txt=%q", entries2["growing.txt"])
}
if entries2["static.txt"] != "does not change" {
t.Fatalf("cycle2 static.txt=%q", entries2["static.txt"])
}
}
func TestPatchArchiveOnTargetFallsBackOnDrift(t *testing.T) {
root := t.TempDir()
mtime := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
writeTestFile(t, filepath.Join(root, "a.txt"), "hello", mtime)
work := t.TempDir()
cachedPath := filepath.Join(work, "cache.zip")
newPath := filepath.Join(work, "new.zip")
targetPath := filepath.Join(work, "target.zip")
if err := buildZipArchive(root, newPath); err != nil {
t.Fatalf("build: %v", err)
}
if err := patchArchiveOnTarget(targetPath, newPath, cachedPath); err != nil {
t.Fatalf("patch: %v", err)
}
if err := os.Rename(newPath, cachedPath); err != nil {
t.Fatalf("rename: %v", err)
}
// Simulate drift: something external truncated/replaced the target file
// so its size no longer matches what our local cache expects.
if err := os.WriteFile(targetPath, []byte("garbage"), 0644); err != nil {
t.Fatalf("simulate drift: %v", err)
}
writeTestFile(t, filepath.Join(root, "a.txt"), "hello again", mtime.Add(time.Minute))
if err := buildZipArchive(root, newPath); err != nil {
t.Fatalf("build2: %v", err)
}
if err := patchArchiveOnTarget(targetPath, newPath, cachedPath); err != nil {
t.Fatalf("patch2: %v", err)
}
want, _ := os.ReadFile(newPath)
got, _ := os.ReadFile(targetPath)
if !bytes.Equal(want, got) {
t.Fatalf("expected drift to trigger a full rewrite, target still corrupt/stale")
}
entries := readZipEntries(t, targetPath)
if entries["a.txt"] != "hello again" {
t.Fatalf("entries=%+v", entries)
}
}
func TestCommonPrefixLenSortedIndependent(t *testing.T) {
// Sanity check that WalkDir's lexical ordering means entry order in the
// zip doesn't depend on directory-read order — build twice from the same
// tree and expect the entry name list to already come out sorted.
root := t.TempDir()
mtime := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
names := []string{"z.txt", "a.txt", "m/one.txt", "m/two.txt"}
for _, n := range names {
writeTestFile(t, filepath.Join(root, n), n, mtime)
}
dest := filepath.Join(t.TempDir(), "out.zip")
if err := buildZipArchive(root, dest); err != nil {
t.Fatalf("build: %v", err)
}
r, err := zip.OpenReader(dest)
if err != nil {
t.Fatalf("open: %v", err)
}
defer r.Close()
var got []string
for _, f := range r.File {
got = append(got, f.Name)
}
sorted := append([]string(nil), got...)
sort.Strings(sorted)
for i := range got {
if got[i] != sorted[i] {
t.Fatalf("entries not in lexical order: %v", got)
}
}
}
+10 -5
View File
@@ -68,8 +68,11 @@ func keepForBundleCopy(rel string, _ os.FileInfo) bool {
// export/ here is expected (e.g. a SAT run's narrative disk report).
//
// Used by both BuildSupportBundle (on-demand tar.gz) and the blackbox USB
// mirror, so both artifacts share one shape.
func categorizeExportTree(srcExportDir, destRoot string) error {
// mirror, so both artifacts share one shape. includeMetricsDB is false for
// the blackbox mirror: metrics.db is a live, growing SQLite file and copying
// it whole every sync cycle was a major contributor to slow blackbox cycles
// (see blackbox_archive.go); the on-demand support bundle still wants it.
func categorizeExportTree(srcExportDir, destRoot string, includeMetricsDB bool) error {
exportDir := filepath.Join(destRoot, "export")
statusDir := filepath.Join(destRoot, "status")
tasksDir := filepath.Join(destRoot, "tasks")
@@ -134,9 +137,11 @@ func categorizeExportTree(srcExportDir, destRoot string) error {
return err
}
}
if _, err := os.Stat(DefaultMetricsDBPath); err == nil {
if err := copyPath(DefaultMetricsDBPath, filepath.Join(statusDir, "metrics.db")); err != nil {
return err
if includeMetricsDB {
if _, err := os.Stat(DefaultMetricsDBPath); err == nil {
if err := copyPath(DefaultMetricsDBPath, filepath.Join(statusDir, "metrics.db")); err != nil {
return err
}
}
}
+1 -1
View File
@@ -446,7 +446,7 @@ func BuildSupportBundle(exportDir string) (string, error) {
}
defer os.RemoveAll(stageRoot)
if err := categorizeExportTree(exportDir, stageRoot); err != nil {
if err := categorizeExportTree(exportDir, stageRoot, true); err != nil {
return "", err
}
if err := writeJournalDump(filepath.Join(stageRoot, "tasks", "_services", "combined.journal.log")); err != nil {