diff --git a/audit/cmd/bee/main.go b/audit/cmd/bee/main.go index 23742d6..8915c3e 100644 --- a/audit/cmd/bee/main.go +++ b/audit/cmd/bee/main.go @@ -485,10 +485,12 @@ func runSAT(args []string, stdout, stderr io.Writer) int { const scenarioUsage = `usage: bee run A path (contains "/" or ends in ".json") is read directly from local disk. - A bare name is instead looked up as scenarios/.json on any mounted - removable media (e.g. the same USB stick already plugged in for - blackbox) — lets an air-gapped engineer author a scenario on another - machine and drop it there without needing a network path onto the host. + A bare name is instead resolved as scenarios/.json, checked first + against the scenarios shipped with this image (always available) and + then against any mounted removable media (e.g. the same USB stick + already plugged in for blackbox) — lets an air-gapped engineer author a + scenario on another machine and drop it there without needing a network + path onto the host. See ParseScenarioJSON in audit/internal/platform/scenario.go for the file format and a worked example (per-GPU/all-GPU load with concurrent IPMI/ @@ -525,7 +527,7 @@ func runScenario(args []string, stdout, stderr io.Writer) int { if strings.Contains(arg, "/") || strings.HasSuffix(arg, ".json") { data, err = os.ReadFile(arg) } else { - data, err = sys.ReadScenarioFromRemovableMedia(arg) + data, err = sys.ReadScenario(arg) } if err != nil { fmt.Fprintf(stderr, "bee run: %v\n", err) diff --git a/audit/internal/app/app.go b/audit/internal/app/app.go index 67a42c8..7a04082 100644 --- a/audit/internal/app/app.go +++ b/audit/internal/app/app.go @@ -75,6 +75,8 @@ type exportManager interface { ExportFileToTarget(src string, target platform.RemovableTarget) (string, error) ListScenarioFilesOnRemovableMedia() ([]platform.ScenarioFileOnRemovableMedia, error) ReadScenarioFromRemovableMedia(name string) ([]byte, error) + ListAvailableScenarios() ([]platform.ScenarioFileOnRemovableMedia, error) + ReadScenario(name string) ([]byte, error) } type toolManager interface { diff --git a/audit/internal/app/app_install.go b/audit/internal/app/app_install.go index c8e450b..f728ffe 100644 --- a/audit/internal/app/app_install.go +++ b/audit/internal/app/app_install.go @@ -28,6 +28,20 @@ func (a *App) ReadScenarioFromRemovableMedia(name string) ([]byte, error) { return a.exports.ReadScenarioFromRemovableMedia(name) } +// ListAvailableScenarios lists every scenario runnable via ReadScenario: +// shipped with the image plus anything found on removable media — see +// platform.System.ListAvailableScenarios. +func (a *App) ListAvailableScenarios() ([]platform.ScenarioFileOnRemovableMedia, error) { + return a.exports.ListAvailableScenarios() +} + +// ReadScenario resolves scenarios/.json, preferring the copy shipped +// with the image over one on removable media — see +// platform.System.ReadScenario. +func (a *App) ReadScenario(name string) ([]byte, error) { + return a.exports.ReadScenario(name) +} + func (a *App) ExportLatestAudit(target platform.RemovableTarget) (string, error) { if _, err := os.Stat(DefaultAuditJSONPath); err != nil { return "", err diff --git a/audit/internal/app/app_test.go b/audit/internal/app/app_test.go index f0b08e8..d9aeb07 100644 --- a/audit/internal/app/app_test.go +++ b/audit/internal/app/app_test.go @@ -99,6 +99,14 @@ func (f fakeExports) ReadScenarioFromRemovableMedia(name string) ([]byte, error) return nil, nil } +func (f fakeExports) ListAvailableScenarios() ([]platform.ScenarioFileOnRemovableMedia, error) { + return nil, nil +} + +func (f fakeExports) ReadScenario(name string) ([]byte, error) { + return nil, nil +} + type fakeRuntime struct { collectFn func(string) (schema.RuntimeHealth, error) dumpFn func(string) error diff --git a/audit/internal/platform/export.go b/audit/internal/platform/export.go index 9f35f3f..f253ce8 100644 --- a/audit/internal/platform/export.go +++ b/audit/internal/platform/export.go @@ -247,3 +247,72 @@ func (s *System) ListScenarioFilesOnRemovableMedia() ([]ScenarioFileOnRemovableM sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) return out, nil } + +// LocalScenariosDir is where scenario JSON files shipped with the ISO +// itself live (rsync'd from this repo's scenarios/ into the overlay at +// build time — see iso/builder/build.sh's "preparing staged overlay" step +// and iso/overlay/usr/share/bee/scenarios/). Part of the read-only +// squashfs, so a scenario here is always available with no USB stick +// needed. A package var (not a const) so tests can point it at a temp dir. +var LocalScenariosDir = "/usr/share/bee/scenarios" + +// ReadScenarioLocal reads scenarios/.json from LocalScenariosDir. +func (s *System) ReadScenarioLocal(name string) ([]byte, error) { + name = strings.TrimSpace(name) + if name == "" { + return nil, fmt.Errorf("scenario name is required") + } + return os.ReadFile(filepath.Join(LocalScenariosDir, name+".json")) +} + +// ListLocalScenarioFiles lists the *.json files under LocalScenariosDir. +func (s *System) ListLocalScenarioFiles() ([]ScenarioFileOnRemovableMedia, error) { + entries, err := os.ReadDir(LocalScenariosDir) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + var out []ScenarioFileOnRemovableMedia + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".json") { + continue + } + out = append(out, ScenarioFileOnRemovableMedia{ + Name: strings.TrimSuffix(e.Name(), ".json"), + Device: "local (shipped with image)", + }) + } + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + return out, nil +} + +// ReadScenario resolves scenarios/.json, checking LocalScenariosDir +// first (always available, no removable media required) and falling back +// to ReadScenarioFromRemovableMedia. Prefer this over calling either one +// directly unless you specifically need to restrict the lookup to one +// source (as the tests for each do). +func (s *System) ReadScenario(name string) ([]byte, error) { + if data, err := s.ReadScenarioLocal(name); err == nil { + return data, nil + } + return s.ReadScenarioFromRemovableMedia(name) +} + +// ListAvailableScenarios merges ListLocalScenarioFiles (first) with +// ListScenarioFilesOnRemovableMedia, for UIs that want to show everything +// runnable via ReadScenario in one list. +func (s *System) ListAvailableScenarios() ([]ScenarioFileOnRemovableMedia, error) { + local, err := s.ListLocalScenarioFiles() + if err != nil { + return nil, err + } + removable, err := s.ListScenarioFilesOnRemovableMedia() + if err != nil { + // A removable-media scan failure (e.g. no lsblk) shouldn't hide the + // scenarios that are always available regardless of media. + return local, nil + } + return append(local, removable...), nil +} diff --git a/audit/internal/platform/export_test.go b/audit/internal/platform/export_test.go index 122e823..2eb9239 100644 --- a/audit/internal/platform/export_test.go +++ b/audit/internal/platform/export_test.go @@ -157,3 +157,115 @@ func TestReadScenarioFromRemovableMediaNotFound(t *testing.T) { t.Fatal("expected error for missing scenario file") } } + +func withLocalScenariosDir(t *testing.T) string { + t.Helper() + dir := t.TempDir() + old := LocalScenariosDir + LocalScenariosDir = dir + t.Cleanup(func() { LocalScenariosDir = old }) + return dir +} + +func TestReadScenarioPrefersLocalOverRemovableMedia(t *testing.T) { + localDir := withLocalScenariosDir(t) + localData := []byte(`{"name":"local-copy"}`) + if err := os.WriteFile(filepath.Join(localDir, "power-watch.json"), localData, 0644); err != nil { + t.Fatalf("write local scenario: %v", err) + } + + // A removable target also has a same-named file with different content + // — ReadScenario must prefer the local (always-available) copy. + mountpoint := t.TempDir() + if err := os.MkdirAll(filepath.Join(mountpoint, "scenarios"), 0755); err != nil { + t.Fatalf("mkdir scenarios: %v", err) + } + if err := os.WriteFile(filepath.Join(mountpoint, "scenarios", "power-watch.json"), []byte(`{"name":"usb-copy"}`), 0644); err != nil { + t.Fatalf("write usb scenario: %v", err) + } + oldExec := exportExecCommand + lsblkOut := `NAME="sdb1" TYPE="part" PKNAME="sdb" RM="1" RO="0" FSTYPE="vfat" MOUNTPOINT="` + mountpoint + `" SIZE="29.8G" LABEL="USB" MODEL=""` + exportExecCommand = func(name string, args ...string) *exec.Cmd { + cmd := exec.Command("sh", "-c", "printf '%s\n' \"$LSBLK_OUT\"") + cmd.Env = append(os.Environ(), "LSBLK_OUT="+lsblkOut) + return cmd + } + t.Cleanup(func() { exportExecCommand = oldExec }) + + s := &System{} + got, err := s.ReadScenario("power-watch") + if err != nil { + t.Fatalf("ReadScenario error: %v", err) + } + if string(got) != string(localData) { + t.Fatalf("got=%q want local copy %q", got, localData) + } +} + +func TestReadScenarioFallsBackToRemovableMediaWhenNotLocal(t *testing.T) { + withLocalScenariosDir(t) // empty — nothing local + + mountpoint := t.TempDir() + if err := os.MkdirAll(filepath.Join(mountpoint, "scenarios"), 0755); err != nil { + t.Fatalf("mkdir scenarios: %v", err) + } + want := []byte(`{"name":"usb-only"}`) + if err := os.WriteFile(filepath.Join(mountpoint, "scenarios", "usb-only.json"), want, 0644); err != nil { + t.Fatalf("write usb scenario: %v", err) + } + oldExec := exportExecCommand + lsblkOut := `NAME="sdb1" TYPE="part" PKNAME="sdb" RM="1" RO="0" FSTYPE="vfat" MOUNTPOINT="` + mountpoint + `" SIZE="29.8G" LABEL="USB" MODEL=""` + exportExecCommand = func(name string, args ...string) *exec.Cmd { + cmd := exec.Command("sh", "-c", "printf '%s\n' \"$LSBLK_OUT\"") + cmd.Env = append(os.Environ(), "LSBLK_OUT="+lsblkOut) + return cmd + } + t.Cleanup(func() { exportExecCommand = oldExec }) + + s := &System{} + got, err := s.ReadScenario("usb-only") + if err != nil { + t.Fatalf("ReadScenario error: %v", err) + } + if string(got) != string(want) { + t.Fatalf("got=%q want=%q", got, want) + } +} + +func TestListAvailableScenariosMergesLocalAndRemovable(t *testing.T) { + localDir := withLocalScenariosDir(t) + if err := os.WriteFile(filepath.Join(localDir, "shipped.json"), []byte(`{}`), 0644); err != nil { + t.Fatalf("write local scenario: %v", err) + } + + mountpoint := t.TempDir() + if err := os.MkdirAll(filepath.Join(mountpoint, "scenarios"), 0755); err != nil { + t.Fatalf("mkdir scenarios: %v", err) + } + if err := os.WriteFile(filepath.Join(mountpoint, "scenarios", "from-usb.json"), []byte(`{}`), 0644); err != nil { + t.Fatalf("write usb scenario: %v", err) + } + oldExec := exportExecCommand + lsblkOut := `NAME="sdb1" TYPE="part" PKNAME="sdb" RM="1" RO="0" FSTYPE="vfat" MOUNTPOINT="` + mountpoint + `" SIZE="29.8G" LABEL="USB" MODEL=""` + exportExecCommand = func(name string, args ...string) *exec.Cmd { + cmd := exec.Command("sh", "-c", "printf '%s\n' \"$LSBLK_OUT\"") + cmd.Env = append(os.Environ(), "LSBLK_OUT="+lsblkOut) + return cmd + } + t.Cleanup(func() { exportExecCommand = oldExec }) + + s := &System{} + got, err := s.ListAvailableScenarios() + if err != nil { + t.Fatalf("ListAvailableScenarios error: %v", err) + } + if len(got) != 2 { + t.Fatalf("got=%v want 2 entries", got) + } + if got[0].Name != "shipped" || got[0].Device != "local (shipped with image)" { + t.Fatalf("got[0]=%+v want local shipped entry first", got[0]) + } + if got[1].Name != "from-usb" { + t.Fatalf("got[1]=%+v want from-usb", got[1]) + } +} diff --git a/audit/internal/webui/api.go b/audit/internal/webui/api.go index aea141c..e9a5dea 100644 --- a/audit/internal/webui/api.go +++ b/audit/internal/webui/api.go @@ -582,7 +582,7 @@ func (h *handler) handleAPIScenarioList(w http.ResponseWriter, _ *http.Request) writeError(w, http.StatusServiceUnavailable, "app not configured") return } - files, err := h.opts.App.ListScenarioFilesOnRemovableMedia() + files, err := h.opts.App.ListAvailableScenarios() if err != nil { writeError(w, http.StatusInternalServerError, err.Error()) return diff --git a/audit/internal/webui/page_scenario.go b/audit/internal/webui/page_scenario.go index 399a2fc..e80cd98 100644 --- a/audit/internal/webui/page_scenario.go +++ b/audit/internal/webui/page_scenario.go @@ -1,28 +1,31 @@ package webui -// renderScenario renders the "6. Scenario" page: scenarios/*.json files -// found on any mounted removable media (e.g. the blackbox USB stick), -// each runnable with one click. A run is enqueued as a normal Task -// (target "scenario") — progress/logs live in Tasks like every other SAT -// pack, no separate live-output UI needed here. +// renderScenario renders the "6. Scenario" page: every scenarios/*.json +// runnable via platform.System.ReadScenario — the ones shipped with the +// image (platform.LocalScenariosDir) plus anything found on mounted +// removable media (e.g. the blackbox USB stick) — each runnable with one +// click. A run is enqueued as a normal Task (target "scenario") — +// progress/logs live in Tasks like every other SAT pack, no separate +// live-output UI needed here. // // See audit/internal/platform/scenario.go for the scenario file format and -// scenarios/README.md for the drop-a-file-on-the-USB-stick workflow this -// page is built around. +// scenarios/README.md for how to add one (shipped in the image, or dropped +// on the USB stick for a one-off/air-gapped run). func renderScenario(opts HandlerOptions) string { return `

Scripted, ad-hoc test scenarios — commands plus background samplers (IPMI sensors, nvidia-smi, ...), described in a small JSON file instead of a - hardcoded test. Drop a <name>.json file under - scenarios/ on a removable drive (e.g. the same USB stick - already plugged in for blackbox) and it shows up below — no rebuild - needed. Runs are enqueued like any other task; watch progress in + hardcoded test. The list below includes scenarios shipped with this image + as well as any <name>.json dropped under + scenarios/ on a mounted removable drive (e.g. the same USB + stick already plugged in for blackbox) — no rebuild needed for the + latter. Runs are enqueued like any other task; watch progress in Tasks.

- Scenarios found on removable media + Available scenarios
@@ -38,7 +41,7 @@ function scenarioRefresh() { .then(r => r.json()) .then(files => { if (!files || !files.length) { - list.innerHTML = '

No scenarios/*.json found on any mounted removable media.

'; + list.innerHTML = '

No scenarios found — none shipped with this image and none under scenarios/ on mounted removable media.

'; return; } let html = ''; diff --git a/audit/internal/webui/tasks.go b/audit/internal/webui/tasks.go index a17fd99..bf153f7 100644 --- a/audit/internal/webui/tasks.go +++ b/audit/internal/webui/tasks.go @@ -995,7 +995,7 @@ func (q *taskQueue) runTask(t *Task, j *jobState, ctx context.Context) { break } var data []byte - data, err = a.ReadScenarioFromRemovableMedia(t.params.ScenarioName) + data, err = a.ReadScenario(t.params.ScenarioName) if err != nil { break } diff --git a/iso/overlay/usr/share/bee/scenarios/nvbandwidth-all-gpu-power-watch.json b/iso/overlay/usr/share/bee/scenarios/nvbandwidth-all-gpu-power-watch.json new file mode 100644 index 0000000..0633307 --- /dev/null +++ b/iso/overlay/usr/share/bee/scenarios/nvbandwidth-all-gpu-power-watch.json @@ -0,0 +1,24 @@ +{ + "name": "nvbandwidth-all-gpu-power-watch", + "timeout_sec": 1800, + "jobs": [ + { + "name": "ipmi-sensors", + "type": "sampler", + "interval_sec": 2, + "cmd": ["ipmitool", "sensor"] + }, + { + "name": "gpu-power", + "type": "sampler", + "interval_sec": 2, + "cmd": ["nvidia-smi", "--query-gpu=index,power.draw,temperature.gpu,clocks.sm", "--format=csv"] + }, + { + "name": "nvbandwidth-all", + "type": "command", + "cmd": ["dcgmi", "diag", "-r", "nvbandwidth", "-i", "{{gpus}}"], + "gpu_indices": [0, 1, 2, 3, 4, 5] + } + ] +} diff --git a/scenarios/README.md b/scenarios/README.md index eab1248..5d9c640 100644 --- a/scenarios/README.md +++ b/scenarios/README.md @@ -12,14 +12,19 @@ code. See `audit/internal/platform/scenario.go` (`ParseScenarioJSON`, bee run ``` -or, for a scenario file dropped under `scenarios/` on a mounted removable -drive (e.g. the same USB stick already plugged in for blackbox — useful on -an air-gapped host with no other way to get a file onto it): +or, for a bare name (no `/`, no `.json`): ``` -bee run # looks for scenarios/.json on any mounted removable media +bee run ``` +which resolves `scenarios/.json` by checking, in order: (1) the +scenarios shipped with the image itself — always available, no media +needed — then (2) any mounted removable drive (e.g. the same USB stick +already plugged in for blackbox), for scenarios authored on another +machine and dropped there without a network path onto the host. Same +resolution powers the "6. Scenario" page in the web UI. + `bee scenario run ` is the same command under a longer name. ## Files checked in here @@ -32,10 +37,20 @@ bee run # looks for scenarios/.json on any mounted removab **`gpu_indices` is host-specific** — update it to match the GPU indices `nvidia-smi -L` actually reports on the box under test before running. +This directory is the source of truth; `iso/builder/build.sh`'s "preparing +staged overlay" step rsyncs the whole overlay tree — including a checked-in +copy under `iso/overlay/usr/share/bee/scenarios/` — into the built image, so +anything meant to ship baked-in needs to exist in **both** places (this repo +doesn't auto-copy one into the other). `platform.LocalScenariosDir` +(`/usr/share/bee/scenarios` on the built host) is what `bee run `/the +web UI actually reads at runtime. + ## Adding more -Not every scenario needs to be checked in here. For a one-off test on a +Not every scenario needs to ship in the image. For a one-off test on a specific host (especially air-gapped), it's simpler to write the JSON file directly onto the blackbox USB stick under `scenarios/.json` and run -`bee run ` — no code change, no rebuild. Check a scenario in here only -when it's worth keeping around as a reusable/named test across hosts. +`bee run ` — no code change, no rebuild. Check a scenario into +`iso/overlay/usr/share/bee/scenarios/` only when it's worth keeping around +as a reusable/named test that should be available on every host without a +USB stick.
NameFound on