fix(webui): repair broken scenario Run button onclick, dedupe build.sh overlay staging
- page_scenario.go: onclick built via JSON.stringify() embedded raw double
quotes inside a double-quoted HTML attribute, truncating the attribute so
the click handler never compiled; pass the name through an escaped
data-scenario-name attribute instead.
- build.sh: overlay staging rsyncs (OVERLAY_DIR->stage, stage->includes.chroot)
ran without --delete, so a scenario removed from the repo (a9924b0) stayed
baked into every ISO built from the persistent stage cache since — the
"second script" in the Scenario page's list.
- blackbox: rewritten around a deterministic local zip + incremental
patch-the-changed-suffix onto removable media, instead of walking/copying
~90 files through a synchronous ntfs-3g FUSE mount every cycle. journalctl
captures are now "--since last sync" (were "--since boot", growing with
uptime) and metrics.db is excluded (was copied whole every cycle).
- scenario: nvbandwidth-acs-ab now escalates GPU count (same-socket pair,
other socket's pair, one cross-socket pair, all GPUs) under each ACS state
instead of always running all 6 GPUs at once, using a new `bee
gpu-bandwidth-groups` subcommand that discovers socket layout from
`nvidia-smi topo -m` at runtime — gpu_indices is host-specific, so this
can't be baked into the scenario file.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 5
@@ -8,6 +8,7 @@ import (
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/exec"
|
||||
"runtime/debug"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -88,6 +89,8 @@ func run(args []string, stdout, stderr io.Writer) (exitCode int) {
|
||||
return runBenchmark(args[1:], stdout, stderr)
|
||||
case "bee-worker":
|
||||
return runBeeWorker(args[1:], stdout, stderr)
|
||||
case "gpu-bandwidth-groups":
|
||||
return runGPUBandwidthGroups(args[1:], stdout, stderr)
|
||||
case "version", "--version", "-version":
|
||||
fmt.Fprintln(stdout, Version)
|
||||
return 0
|
||||
@@ -111,6 +114,7 @@ func printRootUsage(w io.Writer) {
|
||||
bee run <file.json|name> (bare name is looked up as scenarios/<name>.json on removable media)
|
||||
bee benchmark nvidia [--profile standard|stability|overnight]
|
||||
bee bee-worker --export-dir `+app.DefaultExportDir+` --task-id TASK-001
|
||||
bee gpu-bandwidth-groups [--stage <label>]
|
||||
bee version
|
||||
bee help [command]`)
|
||||
}
|
||||
@@ -139,6 +143,8 @@ func runHelp(args []string, stdout, stderr io.Writer) int {
|
||||
return runBenchmark([]string{"--help"}, stdout, stderr)
|
||||
case "bee-worker":
|
||||
return runBeeWorker([]string{"--help"}, stdout, stderr)
|
||||
case "gpu-bandwidth-groups":
|
||||
return runGPUBandwidthGroups([]string{"--help"}, stdout, stderr)
|
||||
case "version":
|
||||
fmt.Fprintln(stdout, "usage: bee version")
|
||||
return 0
|
||||
@@ -622,6 +628,72 @@ func runBenchmark(args []string, stdout, stderr io.Writer) int {
|
||||
return 0
|
||||
}
|
||||
|
||||
// runGPUBandwidthGroups discovers this host's GPU-to-socket layout from
|
||||
// "nvidia-smi topo -m" and prints a progressive multi-GPU test plan: a pair
|
||||
// within each socket, one cross-socket pair, then every GPU. gpu_indices in
|
||||
// a scenario JSON file is host-specific (see scenarios/README.md) — this
|
||||
// exists so a scenario can stage a GPU-count escalation ("same socket, other
|
||||
// socket, cross-socket, all") without baking any host's specific GPU indices
|
||||
// into the file at all.
|
||||
//
|
||||
// With --stage, prints just that stage's comma-joined GPU indices (for a
|
||||
// scenario job's cmd to substitute directly, e.g.
|
||||
// `dcgmi diag -r nvbandwidth -i "$(bee gpu-bandwidth-groups --stage all)"`)
|
||||
// and exits nonzero with nothing on stdout if that stage doesn't apply on
|
||||
// this host (e.g. only one socket present, so there's no "cross-socket"
|
||||
// stage) — callers should treat that as "skip this stage here", not a
|
||||
// hard failure.
|
||||
// Without --stage, lists every applicable stage as "<label>\t<gpu,indices>".
|
||||
func runGPUBandwidthGroups(args []string, stdout, stderr io.Writer) int {
|
||||
fs := flag.NewFlagSet("gpu-bandwidth-groups", flag.ContinueOnError)
|
||||
fs.SetOutput(stderr)
|
||||
stage := fs.String("stage", "", "print only this stage's GPU indices (same-socket-1, same-socket-2, ..., cross-socket, all)")
|
||||
fs.Usage = func() {
|
||||
fmt.Fprintln(stderr, "usage: bee gpu-bandwidth-groups [--stage <label>]")
|
||||
fs.PrintDefaults()
|
||||
}
|
||||
if err := fs.Parse(args); err != nil {
|
||||
if err == flag.ErrHelp {
|
||||
return 0
|
||||
}
|
||||
return 2
|
||||
}
|
||||
if fs.NArg() != 0 {
|
||||
fs.Usage()
|
||||
return 2
|
||||
}
|
||||
|
||||
raw, err := exec.Command("nvidia-smi", "topo", "-m").CombinedOutput()
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "bee gpu-bandwidth-groups: nvidia-smi topo -m: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
groups := platform.NvidiaProgressiveBandwidthGroups(platform.ParseNvidiaSocketGroups(string(raw)))
|
||||
|
||||
if *stage == "" {
|
||||
for _, g := range groups {
|
||||
fmt.Fprintf(stdout, "%s\t%s\n", g.Label, joinInts(g.GPUIndices))
|
||||
}
|
||||
return 0
|
||||
}
|
||||
for _, g := range groups {
|
||||
if g.Label == *stage {
|
||||
fmt.Fprintln(stdout, joinInts(g.GPUIndices))
|
||||
return 0
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(stderr, "bee gpu-bandwidth-groups: stage %q does not apply on this host\n", *stage)
|
||||
return 1
|
||||
}
|
||||
|
||||
func joinInts(vals []int) string {
|
||||
parts := make([]string, len(vals))
|
||||
for i, v := range vals {
|
||||
parts[i] = strconv.Itoa(v)
|
||||
}
|
||||
return strings.Join(parts, ",")
|
||||
}
|
||||
|
||||
func runBeeWorker(args []string, stdout, stderr io.Writer) int {
|
||||
fs := flag.NewFlagSet("bee-worker", flag.ContinueOnError)
|
||||
fs.SetOutput(stderr)
|
||||
|
||||
@@ -572,6 +572,13 @@ func adjustFlushPeriod(current, duration time.Duration, success bool, fastCycles
|
||||
return next
|
||||
}
|
||||
|
||||
// syncCycle stages the current export tree on fast local storage (never
|
||||
// touching the removable-media mountpoint for the expensive part), packs it
|
||||
// into a single deterministic zip, and patches only the changed suffix of
|
||||
// that zip onto the target device — see blackbox_archive.go for why: walking
|
||||
// and rewriting ~90 small files through a synchronous FUSE mount (ntfs-3g
|
||||
// -o sync) was taking ~2x the flush period, dominated by journalctl output
|
||||
// that grows with uptime and got fully re-read/re-written every cycle.
|
||||
func (w *blackboxWorker) syncCycle() error {
|
||||
target, marker := w.snapshotTarget()
|
||||
mountpoint, mountedByBee, err := ensureMountedTarget(target, marker.EnrollmentID)
|
||||
@@ -580,24 +587,59 @@ func (w *blackboxWorker) syncCycle() error {
|
||||
}
|
||||
w.recordMountpoint(mountpoint, mountedByBee)
|
||||
|
||||
root := filepath.Join(mountpoint, w.runtime.bootFolder)
|
||||
if err := os.MkdirAll(root, 0755); err != nil {
|
||||
stageRoot := filepath.Join(w.runtime.exportDir, ".blackbox-stage", w.enrollmentID)
|
||||
if err := os.RemoveAll(stageRoot); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := categorizeExportTree(w.runtime.exportDir, root); err != nil {
|
||||
// includeMetricsDB=false: metrics.db is a live, growing SQLite file: past
|
||||
// versions copied it whole every cycle, which alone could dominate a
|
||||
// cycle's cost as it grew. Not carried onto the blackbox mirror.
|
||||
if err := categorizeExportTree(w.runtime.exportDir, stageRoot, false); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := w.captureSnapshots(stageRoot, w.lastCaptureSince()); err != nil {
|
||||
return err
|
||||
}
|
||||
// Same doc pair the support bundle ships at its root — a blackbox
|
||||
// capture on removable media has no manifest.txt/support-bundle
|
||||
// equivalent to explain its layout, so without this an agent handed
|
||||
// only the media would have nothing pointing it at README.md.
|
||||
if err := writeBundleDocs(root); err != nil {
|
||||
if err := writeBundleDocs(stageRoot); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := w.captureSnapshots(root); err != nil {
|
||||
|
||||
cacheDir := filepath.Join(w.runtime.exportDir, ".blackbox-cache")
|
||||
if err := os.MkdirAll(cacheDir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
return syncFilesystem(root)
|
||||
cachedPath := filepath.Join(cacheDir, w.enrollmentID+".zip")
|
||||
newZipPath := filepath.Join(cacheDir, w.enrollmentID+".zip.new")
|
||||
if err := buildZipArchive(stageRoot, newZipPath); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
targetPath := filepath.Join(mountpoint, w.runtime.bootFolder+" blackbox.zip")
|
||||
if err := patchArchiveOnTarget(targetPath, newZipPath, cachedPath); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(newZipPath, cachedPath); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// lastCaptureSince is the "--since" boundary for this cycle's incremental
|
||||
// journalctl captures: the previous successful sync, or boot time on the
|
||||
// very first cycle. Using the last sync instead of always "--since boot"
|
||||
// keeps each cycle's journalctl output bounded by the flush period instead
|
||||
// of growing with total uptime.
|
||||
func (w *blackboxWorker) lastCaptureSince() time.Time {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.lastSyncAt.IsZero() {
|
||||
return w.runtime.bootStarted
|
||||
}
|
||||
return w.lastSyncAt
|
||||
}
|
||||
|
||||
func (w *blackboxWorker) cleanup() {
|
||||
@@ -623,13 +665,25 @@ func (w *blackboxWorker) recordMountpoint(mountpoint string, mountedByBee bool)
|
||||
w.mountedByBee = mountedByBee
|
||||
}
|
||||
|
||||
func (w *blackboxWorker) captureSnapshots(root string) error {
|
||||
if err := captureCommandAtomic(filepath.Join(root, "tasks", "_services", "combined.journal.log"), "journalctl", "--no-pager", "--since", w.runtime.bootStarted.Format(time.RFC3339)); err != nil {
|
||||
// captureSnapshots writes this cycle's journalctl/dmesg/status snapshots
|
||||
// into root (a local staging tree — see syncCycle). journalctl output is
|
||||
// captured incrementally ("--since" the last successful cycle, not boot) and
|
||||
// appended as a new timestamped entry under journal-increments/ each cycle,
|
||||
// rather than overwriting one ever-growing file: re-dumping "--since boot"
|
||||
// every cycle made each cycle's journalctl call (and the file it produced)
|
||||
// grow with total uptime, independent of how much actually happened since
|
||||
// the last sync.
|
||||
func (w *blackboxWorker) captureSnapshots(root string, since time.Time) error {
|
||||
cycleTS := blackboxNow().Format("20060102-150405.000")
|
||||
sinceArg := since.Format(time.RFC3339)
|
||||
incDir := filepath.Join(root, "tasks", "_services", "journal-increments")
|
||||
|
||||
if err := captureCommandAtomic(filepath.Join(incDir, "combined-"+cycleTS+".log"), "journalctl", "--no-pager", "--since", sinceArg); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, svc := range supportBundleServices {
|
||||
dir := filepath.Join(root, serviceBundleDir(svc))
|
||||
if err := captureCommandAtomic(filepath.Join(dir, svc+".journal.log"), "journalctl", "--no-pager", "-u", svc, "--since", w.runtime.bootStarted.Format(time.RFC3339)); err != nil {
|
||||
if err := captureCommandAtomic(filepath.Join(incDir, svc+"-"+cycleTS+".log"), "journalctl", "--no-pager", "-u", svc, "--since", sinceArg); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := captureCommandAtomic(filepath.Join(dir, svc+".status.txt"), "systemctl", "status", svc, "--no-pager"); err != nil {
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bufio"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// blackboxArchiveCompareChunk is the buffer size used when diffing the newly
|
||||
// built local zip against the last one successfully written to removable
|
||||
// media, to find how many leading bytes are unchanged.
|
||||
const blackboxArchiveCompareChunk = 256 * 1024
|
||||
|
||||
// buildZipArchive walks root (a locally-staged, fast-storage copy of the
|
||||
// blackbox tree — never the slow removable-media mountpoint) and writes a
|
||||
// single deterministic zip to destPath: same input tree -> byte-identical
|
||||
// output, so two cycles that changed nothing produce identical archives and
|
||||
// patchArchiveOnTarget (below) can skip re-writing the unchanged prefix to
|
||||
// the slow target. Determinism relies on fs.WalkDir's guaranteed lexical
|
||||
// order and each entry's Modified time coming from the source file's mtime
|
||||
// (stable across cycles for files nothing touched).
|
||||
func buildZipArchive(root, destPath string) error {
|
||||
if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
f, err := os.OpenFile(destPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
zw := zip.NewWriter(f)
|
||||
|
||||
walkErr := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
rel, err := filepath.Rel(root, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
info, err := d.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
header, err := zip.FileInfoHeader(info)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
header.Name = filepath.ToSlash(rel)
|
||||
header.Method = zip.Deflate
|
||||
w, err := zw.CreateHeader(header)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
src, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer src.Close()
|
||||
_, err = io.Copy(w, src)
|
||||
return err
|
||||
})
|
||||
if walkErr != nil {
|
||||
_ = zw.Close()
|
||||
_ = f.Close()
|
||||
return walkErr
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
_ = f.Close()
|
||||
return err
|
||||
}
|
||||
return f.Close()
|
||||
}
|
||||
|
||||
// commonPrefixLen returns how many leading bytes two files share. Used to
|
||||
// find how much of a freshly-rebuilt local zip is identical to the previous
|
||||
// cycle's, so patchArchiveOnTarget only has to write the changed suffix to
|
||||
// removable media.
|
||||
func commonPrefixLen(pathA, pathB string) (int64, error) {
|
||||
fa, err := os.Open(pathA)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer fa.Close()
|
||||
fb, err := os.Open(pathB)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer fb.Close()
|
||||
|
||||
ra := bufio.NewReaderSize(fa, blackboxArchiveCompareChunk)
|
||||
rb := bufio.NewReaderSize(fb, blackboxArchiveCompareChunk)
|
||||
bufA := make([]byte, blackboxArchiveCompareChunk)
|
||||
bufB := make([]byte, blackboxArchiveCompareChunk)
|
||||
|
||||
var total int64
|
||||
for {
|
||||
na, errA := io.ReadFull(ra, bufA)
|
||||
nb, errB := io.ReadFull(rb, bufB)
|
||||
n := na
|
||||
if nb < n {
|
||||
n = nb
|
||||
}
|
||||
for i := 0; i < n; i++ {
|
||||
if bufA[i] != bufB[i] {
|
||||
return total + int64(i), nil
|
||||
}
|
||||
}
|
||||
total += int64(n)
|
||||
if na != nb || isEOFLike(errA) || isEOFLike(errB) {
|
||||
return total, nil
|
||||
}
|
||||
if errA != nil {
|
||||
return total, errA
|
||||
}
|
||||
if errB != nil {
|
||||
return total, errB
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func isEOFLike(err error) bool {
|
||||
return err == io.EOF || err == io.ErrUnexpectedEOF
|
||||
}
|
||||
|
||||
// patchArchiveOnTarget makes targetPath (on removable media, possibly
|
||||
// FUSE-mounted with synchronous writes) byte-identical to newLocalZipPath (on
|
||||
// fast local storage), writing only the changed suffix instead of the whole
|
||||
// file. cachedPath is our own local record of what we last wrote to
|
||||
// targetPath; if targetPath's size doesn't match what cachedPath implies
|
||||
// (first run, external tampering, a previous crash mid-write), it falls back
|
||||
// to writing the whole archive rather than risk corrupting it with a wrong
|
||||
// truncate point.
|
||||
func patchArchiveOnTarget(targetPath, newLocalZipPath, cachedPath string) error {
|
||||
newInfo, err := os.Stat(newLocalZipPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var prefixLen int64
|
||||
if cachedInfo, err := os.Stat(cachedPath); err == nil {
|
||||
if targetInfo, err := os.Stat(targetPath); err == nil && targetInfo.Size() == cachedInfo.Size() {
|
||||
prefixLen, err = commonPrefixLen(cachedPath, newLocalZipPath)
|
||||
if err != nil {
|
||||
prefixLen = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
if prefixLen > newInfo.Size() {
|
||||
prefixLen = 0
|
||||
}
|
||||
|
||||
target, err := os.OpenFile(targetPath, os.O_CREATE|os.O_RDWR, 0644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer target.Close()
|
||||
|
||||
if err := target.Truncate(prefixLen); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := target.Seek(prefixLen, io.SeekStart); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
src, err := os.Open(newLocalZipPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer src.Close()
|
||||
if _, err := src.Seek(prefixLen, io.SeekStart); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(target, src); err != nil {
|
||||
return err
|
||||
}
|
||||
return target.Sync()
|
||||
}
|
||||
@@ -0,0 +1,235 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func writeTestFile(t *testing.T, path string, content string, mtime time.Time) {
|
||||
t.Helper()
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
|
||||
t.Fatalf("mkdir: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(content), 0644); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
if err := os.Chtimes(path, mtime, mtime); err != nil {
|
||||
t.Fatalf("chtimes: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func readZipEntries(t *testing.T, path string) map[string]string {
|
||||
t.Helper()
|
||||
r, err := zip.OpenReader(path)
|
||||
if err != nil {
|
||||
t.Fatalf("open zip %s: %v", path, err)
|
||||
}
|
||||
defer r.Close()
|
||||
out := map[string]string{}
|
||||
for _, f := range r.File {
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
t.Fatalf("open entry %s: %v", f.Name, err)
|
||||
}
|
||||
data, err := io.ReadAll(rc)
|
||||
rc.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("read entry %s: %v", f.Name, err)
|
||||
}
|
||||
out[f.Name] = string(data)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestBuildZipArchiveRoundTrip(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
mtime := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
writeTestFile(t, filepath.Join(root, "a.txt"), "hello", mtime)
|
||||
writeTestFile(t, filepath.Join(root, "sub", "b.txt"), "world", mtime)
|
||||
|
||||
dest := filepath.Join(t.TempDir(), "out.zip")
|
||||
if err := buildZipArchive(root, dest); err != nil {
|
||||
t.Fatalf("buildZipArchive: %v", err)
|
||||
}
|
||||
|
||||
entries := readZipEntries(t, dest)
|
||||
if entries["a.txt"] != "hello" || entries["sub/b.txt"] != "world" {
|
||||
t.Fatalf("entries=%+v", entries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildZipArchiveDeterministicWhenUnchanged(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
mtime := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
writeTestFile(t, filepath.Join(root, "a.txt"), "hello", mtime)
|
||||
writeTestFile(t, filepath.Join(root, "b.txt"), "world", mtime)
|
||||
|
||||
dir := t.TempDir()
|
||||
p1 := filepath.Join(dir, "one.zip")
|
||||
p2 := filepath.Join(dir, "two.zip")
|
||||
if err := buildZipArchive(root, p1); err != nil {
|
||||
t.Fatalf("build 1: %v", err)
|
||||
}
|
||||
if err := buildZipArchive(root, p2); err != nil {
|
||||
t.Fatalf("build 2: %v", err)
|
||||
}
|
||||
b1, _ := os.ReadFile(p1)
|
||||
b2, _ := os.ReadFile(p2)
|
||||
if !bytes.Equal(b1, b2) {
|
||||
t.Fatalf("two builds of an unchanged tree produced different bytes (len %d vs %d)", len(b1), len(b2))
|
||||
}
|
||||
}
|
||||
|
||||
func TestPatchArchiveOnTargetIncrementalAndValid(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
mtime := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
writeTestFile(t, filepath.Join(root, "static.txt"), "does not change", mtime)
|
||||
writeTestFile(t, filepath.Join(root, "growing.txt"), "cycle-1 line\n", mtime)
|
||||
|
||||
work := t.TempDir()
|
||||
cachedPath := filepath.Join(work, "cache.zip")
|
||||
newPath := filepath.Join(work, "new.zip")
|
||||
targetPath := filepath.Join(work, "target.zip")
|
||||
|
||||
// Cycle 1: cold start, no cache, no target yet.
|
||||
if err := buildZipArchive(root, newPath); err != nil {
|
||||
t.Fatalf("build cycle1: %v", err)
|
||||
}
|
||||
if err := patchArchiveOnTarget(targetPath, newPath, cachedPath); err != nil {
|
||||
t.Fatalf("patch cycle1: %v", err)
|
||||
}
|
||||
if err := os.Rename(newPath, cachedPath); err != nil {
|
||||
t.Fatalf("rename cycle1: %v", err)
|
||||
}
|
||||
|
||||
want1, _ := os.ReadFile(cachedPath)
|
||||
got1, _ := os.ReadFile(targetPath)
|
||||
if !bytes.Equal(want1, got1) {
|
||||
t.Fatalf("cycle1: target does not match freshly built zip")
|
||||
}
|
||||
entries1 := readZipEntries(t, targetPath)
|
||||
if entries1["static.txt"] != "does not change" || entries1["growing.txt"] != "cycle-1 line\n" {
|
||||
t.Fatalf("cycle1 entries=%+v", entries1)
|
||||
}
|
||||
|
||||
// Cycle 2: only growing.txt changes (as it would with an appended log).
|
||||
writeTestFile(t, filepath.Join(root, "growing.txt"), "cycle-1 line\ncycle-2 line\n", mtime.Add(time.Minute))
|
||||
|
||||
if err := buildZipArchive(root, newPath); err != nil {
|
||||
t.Fatalf("build cycle2: %v", err)
|
||||
}
|
||||
prefixLen, err := commonPrefixLen(cachedPath, newPath)
|
||||
if err != nil {
|
||||
t.Fatalf("commonPrefixLen: %v", err)
|
||||
}
|
||||
newInfo, _ := os.Stat(newPath)
|
||||
if prefixLen <= 0 {
|
||||
t.Fatalf("expected a nonzero unchanged prefix (static.txt entry should be identical), got %d", prefixLen)
|
||||
}
|
||||
if prefixLen >= newInfo.Size() {
|
||||
t.Fatalf("expected the changed growing.txt entry to make the archives diverge before EOF, prefixLen=%d size=%d", prefixLen, newInfo.Size())
|
||||
}
|
||||
|
||||
if err := patchArchiveOnTarget(targetPath, newPath, cachedPath); err != nil {
|
||||
t.Fatalf("patch cycle2: %v", err)
|
||||
}
|
||||
if err := os.Rename(newPath, cachedPath); err != nil {
|
||||
t.Fatalf("rename cycle2: %v", err)
|
||||
}
|
||||
|
||||
want2, _ := os.ReadFile(cachedPath)
|
||||
got2, _ := os.ReadFile(targetPath)
|
||||
if !bytes.Equal(want2, got2) {
|
||||
t.Fatalf("cycle2: target does not match freshly built zip after incremental patch")
|
||||
}
|
||||
entries2 := readZipEntries(t, targetPath)
|
||||
if entries2["growing.txt"] != "cycle-1 line\ncycle-2 line\n" {
|
||||
t.Fatalf("cycle2 growing.txt=%q", entries2["growing.txt"])
|
||||
}
|
||||
if entries2["static.txt"] != "does not change" {
|
||||
t.Fatalf("cycle2 static.txt=%q", entries2["static.txt"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestPatchArchiveOnTargetFallsBackOnDrift(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
mtime := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
writeTestFile(t, filepath.Join(root, "a.txt"), "hello", mtime)
|
||||
|
||||
work := t.TempDir()
|
||||
cachedPath := filepath.Join(work, "cache.zip")
|
||||
newPath := filepath.Join(work, "new.zip")
|
||||
targetPath := filepath.Join(work, "target.zip")
|
||||
|
||||
if err := buildZipArchive(root, newPath); err != nil {
|
||||
t.Fatalf("build: %v", err)
|
||||
}
|
||||
if err := patchArchiveOnTarget(targetPath, newPath, cachedPath); err != nil {
|
||||
t.Fatalf("patch: %v", err)
|
||||
}
|
||||
if err := os.Rename(newPath, cachedPath); err != nil {
|
||||
t.Fatalf("rename: %v", err)
|
||||
}
|
||||
|
||||
// Simulate drift: something external truncated/replaced the target file
|
||||
// so its size no longer matches what our local cache expects.
|
||||
if err := os.WriteFile(targetPath, []byte("garbage"), 0644); err != nil {
|
||||
t.Fatalf("simulate drift: %v", err)
|
||||
}
|
||||
|
||||
writeTestFile(t, filepath.Join(root, "a.txt"), "hello again", mtime.Add(time.Minute))
|
||||
if err := buildZipArchive(root, newPath); err != nil {
|
||||
t.Fatalf("build2: %v", err)
|
||||
}
|
||||
if err := patchArchiveOnTarget(targetPath, newPath, cachedPath); err != nil {
|
||||
t.Fatalf("patch2: %v", err)
|
||||
}
|
||||
|
||||
want, _ := os.ReadFile(newPath)
|
||||
got, _ := os.ReadFile(targetPath)
|
||||
if !bytes.Equal(want, got) {
|
||||
t.Fatalf("expected drift to trigger a full rewrite, target still corrupt/stale")
|
||||
}
|
||||
entries := readZipEntries(t, targetPath)
|
||||
if entries["a.txt"] != "hello again" {
|
||||
t.Fatalf("entries=%+v", entries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCommonPrefixLenSortedIndependent(t *testing.T) {
|
||||
// Sanity check that WalkDir's lexical ordering means entry order in the
|
||||
// zip doesn't depend on directory-read order — build twice from the same
|
||||
// tree and expect the entry name list to already come out sorted.
|
||||
root := t.TempDir()
|
||||
mtime := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
names := []string{"z.txt", "a.txt", "m/one.txt", "m/two.txt"}
|
||||
for _, n := range names {
|
||||
writeTestFile(t, filepath.Join(root, n), n, mtime)
|
||||
}
|
||||
dest := filepath.Join(t.TempDir(), "out.zip")
|
||||
if err := buildZipArchive(root, dest); err != nil {
|
||||
t.Fatalf("build: %v", err)
|
||||
}
|
||||
r, err := zip.OpenReader(dest)
|
||||
if err != nil {
|
||||
t.Fatalf("open: %v", err)
|
||||
}
|
||||
defer r.Close()
|
||||
var got []string
|
||||
for _, f := range r.File {
|
||||
got = append(got, f.Name)
|
||||
}
|
||||
sorted := append([]string(nil), got...)
|
||||
sort.Strings(sorted)
|
||||
for i := range got {
|
||||
if got[i] != sorted[i] {
|
||||
t.Fatalf("entries not in lexical order: %v", got)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -68,8 +68,11 @@ func keepForBundleCopy(rel string, _ os.FileInfo) bool {
|
||||
// export/ here is expected (e.g. a SAT run's narrative disk report).
|
||||
//
|
||||
// Used by both BuildSupportBundle (on-demand tar.gz) and the blackbox USB
|
||||
// mirror, so both artifacts share one shape.
|
||||
func categorizeExportTree(srcExportDir, destRoot string) error {
|
||||
// mirror, so both artifacts share one shape. includeMetricsDB is false for
|
||||
// the blackbox mirror: metrics.db is a live, growing SQLite file and copying
|
||||
// it whole every sync cycle was a major contributor to slow blackbox cycles
|
||||
// (see blackbox_archive.go); the on-demand support bundle still wants it.
|
||||
func categorizeExportTree(srcExportDir, destRoot string, includeMetricsDB bool) error {
|
||||
exportDir := filepath.Join(destRoot, "export")
|
||||
statusDir := filepath.Join(destRoot, "status")
|
||||
tasksDir := filepath.Join(destRoot, "tasks")
|
||||
@@ -134,11 +137,13 @@ func categorizeExportTree(srcExportDir, destRoot string) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if includeMetricsDB {
|
||||
if _, err := os.Stat(DefaultMetricsDBPath); err == nil {
|
||||
if err := copyPath(DefaultMetricsDBPath, filepath.Join(statusDir, "metrics.db")); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Remaining orchestration state -> tasks/_state/
|
||||
for _, name := range []string{"blackbox-state.json", "tasks-state.json"} {
|
||||
|
||||
@@ -446,7 +446,7 @@ func BuildSupportBundle(exportDir string) (string, error) {
|
||||
}
|
||||
defer os.RemoveAll(stageRoot)
|
||||
|
||||
if err := categorizeExportTree(exportDir, stageRoot); err != nil {
|
||||
if err := categorizeExportTree(exportDir, stageRoot, true); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := writeJournalDump(filepath.Join(stageRoot, "tasks", "_services", "combined.journal.log")); err != nil {
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// nvidiaCPUAffinityRe matches an nvidia-smi "topo -m" CPU Affinity cell, e.g.
|
||||
// "0-95,192-287" or a plain "0". Matched by shape rather than column
|
||||
// position: NIC count (and therefore column offsets) varies per host, but
|
||||
// this is the first token after the GPU/NIC relation cells (X/PIX/NODE/SYS)
|
||||
// that looks like a core range list, on every layout seen so far.
|
||||
var nvidiaCPUAffinityRe = regexp.MustCompile(`^[0-9]+(-[0-9]+)?(,[0-9]+(-[0-9]+)?)*$`)
|
||||
|
||||
// NvidiaSocketGroup is every GPU index sharing one CPU Affinity range in an
|
||||
// "nvidia-smi topo -m" matrix — a proxy for "these GPUs are on the same CPU
|
||||
// socket/NUMA node".
|
||||
type NvidiaSocketGroup struct {
|
||||
CPUAffinity string
|
||||
GPUIndices []int
|
||||
}
|
||||
|
||||
// ParseNvidiaSocketGroups groups GPU indices from an "nvidia-smi topo -m"
|
||||
// matrix by CPU Affinity, so a scenario can pick "a pair on one socket,
|
||||
// then the other, then one cross-socket pair" without gpu_indices hardcoded
|
||||
// per host — topology (which GPUs share a socket) differs machine to
|
||||
// machine, so a scenario file can't bake this in the way it can bake in "run
|
||||
// on all GPUs".
|
||||
func ParseNvidiaSocketGroups(raw string) []NvidiaSocketGroup {
|
||||
lines := strings.Split(nvidiaNVLinkANSIRe.ReplaceAllString(raw, ""), "\n")
|
||||
headerIdx := -1
|
||||
for i, line := range lines {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if strings.HasPrefix(trimmed, "GPU0") {
|
||||
headerIdx = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if headerIdx < 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
order := map[string]int{}
|
||||
groups := map[string][]int{}
|
||||
for _, line := range lines[headerIdx+1:] {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if !strings.HasPrefix(trimmed, "GPU") {
|
||||
continue
|
||||
}
|
||||
cells := strings.Fields(trimmed)
|
||||
if len(cells) < 2 {
|
||||
continue
|
||||
}
|
||||
rowGPU, err := strconv.Atoi(strings.TrimPrefix(cells[0], "GPU"))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
affinity := ""
|
||||
for _, cell := range cells[1:] {
|
||||
if nvidiaCPUAffinityRe.MatchString(cell) {
|
||||
affinity = cell
|
||||
break
|
||||
}
|
||||
}
|
||||
if affinity == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := order[affinity]; !ok {
|
||||
order[affinity] = len(order)
|
||||
}
|
||||
groups[affinity] = append(groups[affinity], rowGPU)
|
||||
}
|
||||
|
||||
out := make([]NvidiaSocketGroup, 0, len(groups))
|
||||
for affinity, indices := range groups {
|
||||
sort.Ints(indices)
|
||||
out = append(out, NvidiaSocketGroup{CPUAffinity: affinity, GPUIndices: indices})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
return out[i].GPUIndices[0] < out[j].GPUIndices[0]
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// NvidiaBandwidthTestGroup is one stage of a progressive multi-GPU bandwidth
|
||||
// test: a label and the GPU indices to run it against.
|
||||
type NvidiaBandwidthTestGroup struct {
|
||||
Label string
|
||||
GPUIndices []int
|
||||
}
|
||||
|
||||
// NvidiaProgressiveBandwidthGroups turns socket groups into an escalating
|
||||
// test plan: a pair within each socket that has one, then one cross-socket
|
||||
// pair (first two sockets' lowest-indexed GPU each), then every GPU. Lets a
|
||||
// scenario narrow down whether a failure needs the full GPU count or already
|
||||
// reproduces on a single cross-socket pair, instead of only ever testing
|
||||
// "all GPUs at once".
|
||||
func NvidiaProgressiveBandwidthGroups(socketGroups []NvidiaSocketGroup) []NvidiaBandwidthTestGroup {
|
||||
var out []NvidiaBandwidthTestGroup
|
||||
var allGPUs []int
|
||||
var crossSocketPair []int
|
||||
|
||||
for i, sg := range socketGroups {
|
||||
allGPUs = append(allGPUs, sg.GPUIndices...)
|
||||
if len(sg.GPUIndices) >= 2 {
|
||||
out = append(out, NvidiaBandwidthTestGroup{
|
||||
Label: "same-socket-" + strconv.Itoa(i+1),
|
||||
GPUIndices: []int{sg.GPUIndices[0], sg.GPUIndices[1]},
|
||||
})
|
||||
}
|
||||
if len(crossSocketPair) < 2 {
|
||||
crossSocketPair = append(crossSocketPair, sg.GPUIndices[0])
|
||||
}
|
||||
}
|
||||
if len(crossSocketPair) == 2 {
|
||||
out = append(out, NvidiaBandwidthTestGroup{Label: "cross-socket", GPUIndices: crossSocketPair})
|
||||
}
|
||||
if len(allGPUs) > 0 {
|
||||
sort.Ints(allGPUs)
|
||||
out = append(out, NvidiaBandwidthTestGroup{Label: "all", GPUIndices: allGPUs})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// realTopoTwoSocketSixGPU is a real "nvidia-smi topo -m" capture (CG480-S6053,
|
||||
// 6 GPUs across 2 sockets: GPU0-3 on one, GPU4-5 on the other), ANSI escapes
|
||||
// included as nvidia-smi actually emits them in the header row.
|
||||
const realTopoTwoSocketSixGPU = "\t\x1b[4mGPU0\tGPU1\tGPU2\tGPU3\tGPU4\tGPU5\tNIC0\tNIC1\tNIC2\tNIC3\tCPU Affinity\tNUMA Affinity\tGPU NUMA ID\x1b[0m\n" +
|
||||
"GPU0\t X \tPIX\tNODE\tNODE\tSYS\tSYS\tNODE\tNODE\tPIX\tPIX\t0-95,192-287\t0\t\tN/A\n" +
|
||||
"GPU1\tPIX\t X \tNODE\tNODE\tSYS\tSYS\tNODE\tNODE\tPIX\tPIX\t0-95,192-287\t0\t\tN/A\n" +
|
||||
"GPU2\tNODE\tNODE\t X \tPIX\tSYS\tSYS\tPIX\tPIX\tNODE\tNODE\t0-95,192-287\t0\t\tN/A\n" +
|
||||
"GPU3\tNODE\tNODE\tPIX\t X \tSYS\tSYS\tPIX\tPIX\tNODE\tNODE\t0-95,192-287\t0\t\tN/A\n" +
|
||||
"GPU4\tSYS\tSYS\tSYS\tSYS\t X \tPIX\tSYS\tSYS\tSYS\tSYS\t96-191,288-383\t1\t\tN/A\n" +
|
||||
"GPU5\tSYS\tSYS\tSYS\tSYS\tPIX\t X \tSYS\tSYS\tSYS\tSYS\t96-191,288-383\t1\t\tN/A\n"
|
||||
|
||||
// realTopoSingleSocketFourGPU simulates a single-socket host (all GPUs share
|
||||
// one CPU Affinity range) — no cross-socket stage should be produced.
|
||||
const realTopoSingleSocketFourGPU = "GPU0\tGPU1\tGPU2\tGPU3\tCPU Affinity\tNUMA Affinity\n" +
|
||||
"GPU0\t X \tPIX\tNODE\tNODE\t0-31\t0\n" +
|
||||
"GPU1\tPIX\t X \tNODE\tNODE\t0-31\t0\n" +
|
||||
"GPU2\tNODE\tNODE\t X \tPIX\t0-31\t0\n" +
|
||||
"GPU3\tNODE\tNODE\tPIX\t X \t0-31\t0\n"
|
||||
|
||||
func TestParseNvidiaSocketGroupsTwoSockets(t *testing.T) {
|
||||
groups := ParseNvidiaSocketGroups(realTopoTwoSocketSixGPU)
|
||||
if len(groups) != 2 {
|
||||
t.Fatalf("groups=%+v, want 2", groups)
|
||||
}
|
||||
if !reflect.DeepEqual(groups[0].GPUIndices, []int{0, 1, 2, 3}) {
|
||||
t.Fatalf("group0=%v, want [0 1 2 3]", groups[0].GPUIndices)
|
||||
}
|
||||
if !reflect.DeepEqual(groups[1].GPUIndices, []int{4, 5}) {
|
||||
t.Fatalf("group1=%v, want [4 5]", groups[1].GPUIndices)
|
||||
}
|
||||
if groups[0].CPUAffinity == groups[1].CPUAffinity {
|
||||
t.Fatalf("expected distinct CPU affinities, got %q for both", groups[0].CPUAffinity)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNvidiaProgressiveBandwidthGroupsTwoSockets(t *testing.T) {
|
||||
socketGroups := ParseNvidiaSocketGroups(realTopoTwoSocketSixGPU)
|
||||
stages := NvidiaProgressiveBandwidthGroups(socketGroups)
|
||||
|
||||
want := []NvidiaBandwidthTestGroup{
|
||||
{Label: "same-socket-1", GPUIndices: []int{0, 1}},
|
||||
{Label: "same-socket-2", GPUIndices: []int{4, 5}},
|
||||
{Label: "cross-socket", GPUIndices: []int{0, 4}},
|
||||
{Label: "all", GPUIndices: []int{0, 1, 2, 3, 4, 5}},
|
||||
}
|
||||
if len(stages) != len(want) {
|
||||
t.Fatalf("stages=%+v, want %+v", stages, want)
|
||||
}
|
||||
for i, w := range want {
|
||||
if stages[i].Label != w.Label || !reflect.DeepEqual(stages[i].GPUIndices, w.GPUIndices) {
|
||||
t.Fatalf("stage %d = %+v, want %+v", i, stages[i], w)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNvidiaProgressiveBandwidthGroupsSingleSocket(t *testing.T) {
|
||||
socketGroups := ParseNvidiaSocketGroups(realTopoSingleSocketFourGPU)
|
||||
stages := NvidiaProgressiveBandwidthGroups(socketGroups)
|
||||
|
||||
want := []NvidiaBandwidthTestGroup{
|
||||
{Label: "same-socket-1", GPUIndices: []int{0, 1}},
|
||||
{Label: "all", GPUIndices: []int{0, 1, 2, 3}},
|
||||
}
|
||||
if len(stages) != len(want) {
|
||||
t.Fatalf("stages=%+v, want %+v (no cross-socket stage on a single-socket host)", stages, want)
|
||||
}
|
||||
for i, w := range want {
|
||||
if stages[i].Label != w.Label || !reflect.DeepEqual(stages[i].GPUIndices, w.GPUIndices) {
|
||||
t.Fatalf("stage %d = %+v, want %+v", i, stages[i], w)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseNvidiaSocketGroupsEmptyOnGarbage(t *testing.T) {
|
||||
if got := ParseNvidiaSocketGroups("not a topology matrix"); len(got) != 0 {
|
||||
t.Fatalf("got %+v, want empty", got)
|
||||
}
|
||||
}
|
||||
@@ -50,7 +50,7 @@ function scenarioRefresh() {
|
||||
html += '<tr><td style="white-space:nowrap">' + escapeHTML(f.name) + '</td>'
|
||||
+ '<td style="font-size:12px;max-width:360px;color:var(--muted)">' + desc + '</td>'
|
||||
+ '<td style="color:var(--muted);font-size:12px;white-space:nowrap">' + escapeHTML(f.device) + '</td>'
|
||||
+ '<td style="white-space:nowrap"><button class="btn btn-primary" onclick="scenarioRun(' + JSON.stringify(f.name) + ', this)">▶ Run</button></td></tr>';
|
||||
+ '<td style="white-space:nowrap"><button class="btn btn-primary" data-scenario-name="' + escapeHTML(f.name) + '" onclick="scenarioRun(this.dataset.scenarioName, this)">▶ Run</button></td></tr>';
|
||||
}
|
||||
html += '</tbody></table>';
|
||||
list.innerHTML = html;
|
||||
|
||||
@@ -1460,7 +1460,7 @@ elif [ -d "${LB_PKG_CACHE}" ] && [ "$(ls -A "${LB_PKG_CACHE}" 2>/dev/null)" ]; t
|
||||
rsync -a "${LB_PKG_CACHE}/" "${BUILD_WORK_DIR}/cache/packages.chroot/"
|
||||
fi
|
||||
|
||||
rsync -a "${OVERLAY_DIR}/" "${OVERLAY_STAGE_DIR}/"
|
||||
rsync -a --delete "${OVERLAY_DIR}/" "${OVERLAY_STAGE_DIR}/"
|
||||
rm -f \
|
||||
"${OVERLAY_STAGE_DIR}/etc/bee-ssh-password-fallback" \
|
||||
"${OVERLAY_STAGE_DIR}/etc/bee-release" \
|
||||
@@ -1734,7 +1734,7 @@ fi
|
||||
LB_DIR="${BUILD_WORK_DIR}"
|
||||
LB_INCLUDES="${LB_DIR}/config/includes.chroot"
|
||||
mkdir -p "${LB_INCLUDES}"
|
||||
rsync -a "${OVERLAY_STAGE_DIR}/" "${LB_INCLUDES}/"
|
||||
rsync -a --delete "${OVERLAY_STAGE_DIR}/" "${LB_INCLUDES}/"
|
||||
|
||||
# Ensure SSH authorized_keys perms are correct (rsync may alter)
|
||||
if [ -f "${LB_INCLUDES}/root/.ssh/authorized_keys" ]; then
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "nvbandwidth-acs-ab",
|
||||
"description": "Two-phase A/B for the MSI CG480-S6053 cross-socket GPU P2P hard-reboot. Phase 1 disables PCIe ACS redirect at runtime via setpci (P2P goes device-direct instead of up to the root complex) then runs nvbandwidth across all 6 GPUs — the run expected to SURVIVE. Phase 2 restores ACS to the BIOS default and repeats — the run expected to REPRODUCE the reboot. Safe phase runs first so its full blackbox reaches the USB stick before the risky phase can reset the machine.",
|
||||
"description": "Progressive A/B for the MSI CG480-S6053 cross-socket GPU P2P hard-reboot. Each phase escalates GPU count instead of jumping straight to all GPUs: a pair on one socket, a pair on the other socket, one cross-socket pair, then every GPU — so a crash pinpoints which GPU count/topology it needs, not just whether the full run fails. GPU indices are discovered at runtime via `bee gpu-bandwidth-groups` (nvidia-smi topo -m), not hardcoded, since socket layout is host-specific. Phase 1 disables PCIe ACS redirect at runtime via setpci (P2P goes device-direct instead of up to the root complex) and is expected to SURVIVE all four stages. Phase 2 restores ACS to the BIOS default and repeats the same four stages — the run expected to REPRODUCE the reboot at some stage. Safe phase runs first so its full blackbox reaches the USB stick before the risky phase can reset the machine. A stage is skipped (not failed) if this host's topology doesn't have it, e.g. a single-socket host has no cross-socket stage.",
|
||||
"timeout_sec": 1800,
|
||||
"jobs": [
|
||||
{
|
||||
@@ -9,10 +9,24 @@
|
||||
"cmd": ["bash", "-c", ": > /run/bee-acs-orig; for bdf in $(lspci -D | awk '{print $1}'); do setpci -s $bdf ECAP_ACS.w >/dev/null 2>&1 || continue; v=$(setpci -s $bdf ECAP_ACS+0x6.w 2>/dev/null) || continue; echo $bdf $v >> /run/bee-acs-orig; setpci -s $bdf ECAP_ACS+0x6.w=0000 >/dev/null 2>&1 || true; done; echo ACS disabled on $(wc -l < /run/bee-acs-orig) bridges; echo -n 'bridges still ReqRedir+: '; lspci -vvv 2>/dev/null | grep -c 'ReqRedir+' || true"]
|
||||
},
|
||||
{
|
||||
"name": "nvbandwidth-acs-off",
|
||||
"name": "nvbandwidth-acs-off-same-socket-1",
|
||||
"type": "command",
|
||||
"cmd": ["dcgmi", "diag", "-r", "nvbandwidth", "-i", "{{gpus}}"],
|
||||
"gpu_indices": [0, 1, 2, 3, 4, 5]
|
||||
"cmd": ["bash", "-c", "gpus=$(bee gpu-bandwidth-groups --stage same-socket-1) || { echo 'skip: no same-socket-1 group on this host'; exit 0; }; dcgmi diag -r nvbandwidth -i \"$gpus\""]
|
||||
},
|
||||
{
|
||||
"name": "nvbandwidth-acs-off-same-socket-2",
|
||||
"type": "command",
|
||||
"cmd": ["bash", "-c", "gpus=$(bee gpu-bandwidth-groups --stage same-socket-2) || { echo 'skip: no same-socket-2 group on this host'; exit 0; }; dcgmi diag -r nvbandwidth -i \"$gpus\""]
|
||||
},
|
||||
{
|
||||
"name": "nvbandwidth-acs-off-cross-socket",
|
||||
"type": "command",
|
||||
"cmd": ["bash", "-c", "gpus=$(bee gpu-bandwidth-groups --stage cross-socket) || { echo 'skip: no cross-socket pair on this host'; exit 0; }; dcgmi diag -r nvbandwidth -i \"$gpus\""]
|
||||
},
|
||||
{
|
||||
"name": "nvbandwidth-acs-off-all",
|
||||
"type": "command",
|
||||
"cmd": ["bash", "-c", "gpus=$(bee gpu-bandwidth-groups --stage all) || { echo 'skip: no GPUs discovered on this host'; exit 0; }; dcgmi diag -r nvbandwidth -i \"$gpus\""]
|
||||
},
|
||||
{
|
||||
"name": "acs-restore",
|
||||
@@ -20,10 +34,24 @@
|
||||
"cmd": ["bash", "-c", "test -s /run/bee-acs-orig || { echo no saved ACS state; exit 0; }; while read -r bdf v; do setpci -s $bdf ECAP_ACS+0x6.w=$v >/dev/null 2>&1 || true; done < /run/bee-acs-orig; echo ACS restored on $(wc -l < /run/bee-acs-orig) bridges; echo -n 'bridges ReqRedir+ now: '; lspci -vvv 2>/dev/null | grep -c 'ReqRedir+' || true"]
|
||||
},
|
||||
{
|
||||
"name": "nvbandwidth-acs-on",
|
||||
"name": "nvbandwidth-acs-on-same-socket-1",
|
||||
"type": "command",
|
||||
"cmd": ["dcgmi", "diag", "-r", "nvbandwidth", "-i", "{{gpus}}"],
|
||||
"gpu_indices": [0, 1, 2, 3, 4, 5]
|
||||
"cmd": ["bash", "-c", "gpus=$(bee gpu-bandwidth-groups --stage same-socket-1) || { echo 'skip: no same-socket-1 group on this host'; exit 0; }; dcgmi diag -r nvbandwidth -i \"$gpus\""]
|
||||
},
|
||||
{
|
||||
"name": "nvbandwidth-acs-on-same-socket-2",
|
||||
"type": "command",
|
||||
"cmd": ["bash", "-c", "gpus=$(bee gpu-bandwidth-groups --stage same-socket-2) || { echo 'skip: no same-socket-2 group on this host'; exit 0; }; dcgmi diag -r nvbandwidth -i \"$gpus\""]
|
||||
},
|
||||
{
|
||||
"name": "nvbandwidth-acs-on-cross-socket",
|
||||
"type": "command",
|
||||
"cmd": ["bash", "-c", "gpus=$(bee gpu-bandwidth-groups --stage cross-socket) || { echo 'skip: no cross-socket pair on this host'; exit 0; }; dcgmi diag -r nvbandwidth -i \"$gpus\""]
|
||||
},
|
||||
{
|
||||
"name": "nvbandwidth-acs-on-all",
|
||||
"type": "command",
|
||||
"cmd": ["bash", "-c", "gpus=$(bee gpu-bandwidth-groups --stage all) || { echo 'skip: no GPUs discovered on this host'; exit 0; }; dcgmi diag -r nvbandwidth -i \"$gpus\""]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "nvbandwidth-acs-ab",
|
||||
"description": "Two-phase A/B for the MSI CG480-S6053 cross-socket GPU P2P hard-reboot. Phase 1 disables PCIe ACS redirect at runtime via setpci (P2P goes device-direct instead of up to the root complex) then runs nvbandwidth across all 6 GPUs — the run expected to SURVIVE. Phase 2 restores ACS to the BIOS default and repeats — the run expected to REPRODUCE the reboot. Safe phase runs first so its full blackbox reaches the USB stick before the risky phase can reset the machine.",
|
||||
"description": "Progressive A/B for the MSI CG480-S6053 cross-socket GPU P2P hard-reboot. Each phase escalates GPU count instead of jumping straight to all GPUs: a pair on one socket, a pair on the other socket, one cross-socket pair, then every GPU — so a crash pinpoints which GPU count/topology it needs, not just whether the full run fails. GPU indices are discovered at runtime via `bee gpu-bandwidth-groups` (nvidia-smi topo -m), not hardcoded, since socket layout is host-specific. Phase 1 disables PCIe ACS redirect at runtime via setpci (P2P goes device-direct instead of up to the root complex) and is expected to SURVIVE all four stages. Phase 2 restores ACS to the BIOS default and repeats the same four stages — the run expected to REPRODUCE the reboot at some stage. Safe phase runs first so its full blackbox reaches the USB stick before the risky phase can reset the machine. A stage is skipped (not failed) if this host's topology doesn't have it, e.g. a single-socket host has no cross-socket stage.",
|
||||
"timeout_sec": 1800,
|
||||
"jobs": [
|
||||
{
|
||||
@@ -9,10 +9,24 @@
|
||||
"cmd": ["bash", "-c", ": > /run/bee-acs-orig; for bdf in $(lspci -D | awk '{print $1}'); do setpci -s $bdf ECAP_ACS.w >/dev/null 2>&1 || continue; v=$(setpci -s $bdf ECAP_ACS+0x6.w 2>/dev/null) || continue; echo $bdf $v >> /run/bee-acs-orig; setpci -s $bdf ECAP_ACS+0x6.w=0000 >/dev/null 2>&1 || true; done; echo ACS disabled on $(wc -l < /run/bee-acs-orig) bridges; echo -n 'bridges still ReqRedir+: '; lspci -vvv 2>/dev/null | grep -c 'ReqRedir+' || true"]
|
||||
},
|
||||
{
|
||||
"name": "nvbandwidth-acs-off",
|
||||
"name": "nvbandwidth-acs-off-same-socket-1",
|
||||
"type": "command",
|
||||
"cmd": ["dcgmi", "diag", "-r", "nvbandwidth", "-i", "{{gpus}}"],
|
||||
"gpu_indices": [0, 1, 2, 3, 4, 5]
|
||||
"cmd": ["bash", "-c", "gpus=$(bee gpu-bandwidth-groups --stage same-socket-1) || { echo 'skip: no same-socket-1 group on this host'; exit 0; }; dcgmi diag -r nvbandwidth -i \"$gpus\""]
|
||||
},
|
||||
{
|
||||
"name": "nvbandwidth-acs-off-same-socket-2",
|
||||
"type": "command",
|
||||
"cmd": ["bash", "-c", "gpus=$(bee gpu-bandwidth-groups --stage same-socket-2) || { echo 'skip: no same-socket-2 group on this host'; exit 0; }; dcgmi diag -r nvbandwidth -i \"$gpus\""]
|
||||
},
|
||||
{
|
||||
"name": "nvbandwidth-acs-off-cross-socket",
|
||||
"type": "command",
|
||||
"cmd": ["bash", "-c", "gpus=$(bee gpu-bandwidth-groups --stage cross-socket) || { echo 'skip: no cross-socket pair on this host'; exit 0; }; dcgmi diag -r nvbandwidth -i \"$gpus\""]
|
||||
},
|
||||
{
|
||||
"name": "nvbandwidth-acs-off-all",
|
||||
"type": "command",
|
||||
"cmd": ["bash", "-c", "gpus=$(bee gpu-bandwidth-groups --stage all) || { echo 'skip: no GPUs discovered on this host'; exit 0; }; dcgmi diag -r nvbandwidth -i \"$gpus\""]
|
||||
},
|
||||
{
|
||||
"name": "acs-restore",
|
||||
@@ -20,10 +34,24 @@
|
||||
"cmd": ["bash", "-c", "test -s /run/bee-acs-orig || { echo no saved ACS state; exit 0; }; while read -r bdf v; do setpci -s $bdf ECAP_ACS+0x6.w=$v >/dev/null 2>&1 || true; done < /run/bee-acs-orig; echo ACS restored on $(wc -l < /run/bee-acs-orig) bridges; echo -n 'bridges ReqRedir+ now: '; lspci -vvv 2>/dev/null | grep -c 'ReqRedir+' || true"]
|
||||
},
|
||||
{
|
||||
"name": "nvbandwidth-acs-on",
|
||||
"name": "nvbandwidth-acs-on-same-socket-1",
|
||||
"type": "command",
|
||||
"cmd": ["dcgmi", "diag", "-r", "nvbandwidth", "-i", "{{gpus}}"],
|
||||
"gpu_indices": [0, 1, 2, 3, 4, 5]
|
||||
"cmd": ["bash", "-c", "gpus=$(bee gpu-bandwidth-groups --stage same-socket-1) || { echo 'skip: no same-socket-1 group on this host'; exit 0; }; dcgmi diag -r nvbandwidth -i \"$gpus\""]
|
||||
},
|
||||
{
|
||||
"name": "nvbandwidth-acs-on-same-socket-2",
|
||||
"type": "command",
|
||||
"cmd": ["bash", "-c", "gpus=$(bee gpu-bandwidth-groups --stage same-socket-2) || { echo 'skip: no same-socket-2 group on this host'; exit 0; }; dcgmi diag -r nvbandwidth -i \"$gpus\""]
|
||||
},
|
||||
{
|
||||
"name": "nvbandwidth-acs-on-cross-socket",
|
||||
"type": "command",
|
||||
"cmd": ["bash", "-c", "gpus=$(bee gpu-bandwidth-groups --stage cross-socket) || { echo 'skip: no cross-socket pair on this host'; exit 0; }; dcgmi diag -r nvbandwidth -i \"$gpus\""]
|
||||
},
|
||||
{
|
||||
"name": "nvbandwidth-acs-on-all",
|
||||
"type": "command",
|
||||
"cmd": ["bash", "-c", "gpus=$(bee gpu-bandwidth-groups --stage all) || { echo 'skip: no GPUs discovered on this host'; exit 0; }; dcgmi diag -r nvbandwidth -i \"$gpus\""]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user