diff --git a/audit/cmd/bee/main.go b/audit/cmd/bee/main.go index 86f94af..23742d6 100644 --- a/audit/cmd/bee/main.go +++ b/audit/cmd/bee/main.go @@ -74,6 +74,16 @@ func run(args []string, stdout, stderr io.Writer) (exitCode int) { return runBlackbox(args[1:], stdout, stderr) case "sat": return runSAT(args[1:], stdout, stderr) + case "run": + return runScenario(args[1:], stdout, stderr) + case "scenario": + // "bee scenario run " is a longer alias for "bee run "; + // strip the "run" verb (if present) and delegate to the same code. + rest := args[1:] + if len(rest) > 0 && rest[0] == "run" { + rest = rest[1:] + } + return runScenario(rest, stdout, stderr) case "benchmark": return runBenchmark(args[1:], stdout, stderr) case "bee-worker": @@ -98,6 +108,7 @@ func printRootUsage(w io.Writer) { bee web --listen :80 [--audit-path `+app.DefaultAuditJSONPath+`] bee blackbox --export-dir `+app.DefaultExportDir+` [--state-file `+app.DefaultBlackboxStatePath+`] bee sat nvidia|memory|storage|cpu [--duration ] + bee run (bare name is looked up as scenarios/.json on removable media) bee benchmark nvidia [--profile standard|stability|overnight] bee bee-worker --export-dir `+app.DefaultExportDir+` --task-id TASK-001 bee version @@ -122,6 +133,8 @@ func runHelp(args []string, stdout, stderr io.Writer) int { return runBlackbox([]string{"--help"}, stdout, stdout) case "sat": return runSAT([]string{"--help"}, stdout, stderr) + case "run", "scenario": + return runScenario([]string{"--help"}, stdout, stderr) case "benchmark": return runBenchmark([]string{"--help"}, stdout, stderr) case "bee-worker": @@ -469,6 +482,77 @@ func runSAT(args []string, stdout, stderr io.Writer) int { return 0 } +const scenarioUsage = `usage: bee run + + A path (contains "/" or ends in ".json") is read directly from local disk. + A bare name is instead looked up as scenarios/.json on any mounted + removable media (e.g. the same USB stick already plugged in for + blackbox) — lets an air-gapped engineer author a scenario on another + machine and drop it there without needing a network path onto the host. + +See ParseScenarioJSON in audit/internal/platform/scenario.go for the file +format and a worked example (per-GPU/all-GPU load with concurrent IPMI/ +nvidia-smi sampling).` + +// runScenario implements both "bee run " and "bee scenario run " +// (the latter kept as a longer alias). args is just the scenario +// file/name — no leading verb. +func runScenario(args []string, stdout, stderr io.Writer) int { + if len(args) > 0 && (args[0] == "help" || args[0] == "--help" || args[0] == "-h") { + fmt.Fprintln(stdout, scenarioUsage) + return 0 + } + + fs := flag.NewFlagSet("run", flag.ContinueOnError) + fs.SetOutput(stderr) + if err := fs.Parse(args); err != nil { + if err == flag.ErrHelp { + return 0 + } + return 2 + } + if fs.NArg() != 1 { + fmt.Fprintln(stderr, scenarioUsage) + return 2 + } + arg := fs.Arg(0) + + sys := platform.New() + var ( + data []byte + err error + ) + if strings.Contains(arg, "/") || strings.HasSuffix(arg, ".json") { + data, err = os.ReadFile(arg) + } else { + data, err = sys.ReadScenarioFromRemovableMedia(arg) + } + if err != nil { + fmt.Fprintf(stderr, "bee run: %v\n", err) + return 1 + } + + spec, err := platform.ParseScenarioJSON(data) + if err != nil { + fmt.Fprintf(stderr, "bee run: %v\n", err) + return 1 + } + + // Wires the same blackbox kick/sync-bracket hooks the SAT job runner + // uses (see app.New()), so a scenario command job's evidence reaches + // removable media the same way a SAT job's does. + _ = app.New(sys) + + logLine := func(s string) { fmt.Fprintln(stderr, s) } + runDir, err := sys.RunScenario(context.Background(), "", spec, logLine) + if err != nil { + fmt.Fprintf(stderr, "bee run: %v\n", err) + return 1 + } + fmt.Fprintln(stdout, runDir) + return 0 +} + func runBenchmark(args []string, stdout, stderr io.Writer) int { if len(args) == 0 { fmt.Fprintln(stderr, "usage: bee benchmark nvidia [--profile standard|stability|overnight] [--devices 0,1] [--exclude 2,3] [--size-mb N] [--skip-nccl]") diff --git a/audit/cmd/bee/main_test.go b/audit/cmd/bee/main_test.go index e326f55..1625a42 100644 --- a/audit/cmd/bee/main_test.go +++ b/audit/cmd/bee/main_test.go @@ -99,6 +99,58 @@ func TestRunSATUsage(t *testing.T) { } } +func TestRunUsage(t *testing.T) { + t.Parallel() + + var stdout, stderr bytes.Buffer + rc := run([]string{"run"}, &stdout, &stderr) + if rc != 2 { + t.Fatalf("rc=%d want 2", rc) + } + if !strings.Contains(stderr.String(), "usage: bee run ") { + t.Fatalf("stderr missing run usage:\n%s", stderr.String()) + } +} + +func TestRunHelp(t *testing.T) { + t.Parallel() + + var stdout, stderr bytes.Buffer + rc := run([]string{"run", "--help"}, &stdout, &stderr) + if rc != 0 { + t.Fatalf("rc=%d want 0", rc) + } + if !strings.Contains(stdout.String(), "usage: bee run ") { + t.Fatalf("stdout missing run usage:\n%s", stdout.String()) + } +} + +func TestRunMissingScenarioFile(t *testing.T) { + t.Parallel() + + var stdout, stderr bytes.Buffer + rc := run([]string{"run", "/nonexistent/scenario.json"}, &stdout, &stderr) + if rc != 1 { + t.Fatalf("rc=%d want 1", rc) + } + if !strings.Contains(stderr.String(), "bee run:") { + t.Fatalf("stderr missing error prefix:\n%s", stderr.String()) + } +} + +func TestRunScenarioAliasAcceptsRunVerb(t *testing.T) { + t.Parallel() + + var stdout, stderr bytes.Buffer + rc := run([]string{"scenario", "run", "/nonexistent/scenario.json"}, &stdout, &stderr) + if rc != 1 { + t.Fatalf("rc=%d want 1", rc) + } + if !strings.Contains(stderr.String(), "bee run:") { + t.Fatalf("stderr missing error prefix:\n%s", stderr.String()) + } +} + func TestRunPreflightRejectsExtraArgs(t *testing.T) { t.Parallel() diff --git a/audit/internal/platform/export.go b/audit/internal/platform/export.go index 3030642..efb131f 100644 --- a/audit/internal/platform/export.go +++ b/audit/internal/platform/export.go @@ -151,3 +151,55 @@ func (s *System) ExportFileToTarget(src string, target RemovableTarget) (dst str return dst, nil } + +// mountRemovableTargetReadOnly mounts target for reading if it isn't +// already mounted (reusing its existing mountpoint otherwise), returning +// whether this call did the mounting so the caller knows whether to +// unmount afterward. +func mountRemovableTargetReadOnly(target RemovableTarget) (mountpoint string, mountedHere bool, err error) { + if mp := strings.TrimSpace(target.Mountpoint); mp != "" { + return mp, false, nil + } + mountpoint = filepath.Join("/tmp", "bee-scenario-"+filepath.Base(target.Device)) + if err := os.MkdirAll(mountpoint, 0755); err != nil { + return "", false, err + } + if raw, err := exportExecCommand("mount", target.Device, mountpoint).CombinedOutput(); err != nil { + _ = os.Remove(mountpoint) + return "", false, formatMountTargetError(target, string(raw), err) + } + return mountpoint, true, nil +} + +// ReadScenarioFromRemovableMedia mounts each removable target in turn +// (unmounting again afterward if it mounted it itself), looking for +// scenarios/.json, and returns the contents of the first one found. +// Lets an air-gapped engineer author a scenario JSON file on another +// machine, drop it under scenarios/ on the same flash drive already +// plugged in for blackbox, and run it on the host with no network path +// required — see RunScenario/ParseScenarioJSON. +func (s *System) ReadScenarioFromRemovableMedia(name string) ([]byte, error) { + name = strings.TrimSpace(name) + if name == "" { + return nil, fmt.Errorf("scenario name is required") + } + targets, err := s.ListRemovableTargets() + if err != nil { + return nil, err + } + for _, target := range targets { + mountpoint, mountedHere, mountErr := mountRemovableTargetReadOnly(target) + if mountErr != nil { + continue + } + data, readErr := os.ReadFile(filepath.Join(mountpoint, "scenarios", name+".json")) + if mountedHere { + _, _ = exportExecCommand("umount", mountpoint).CombinedOutput() + _ = os.Remove(mountpoint) + } + if readErr == nil { + return data, nil + } + } + return nil, fmt.Errorf("scenarios/%s.json not found on any removable media", name) +} diff --git a/audit/internal/platform/export_test.go b/audit/internal/platform/export_test.go index 54da24f..122e823 100644 --- a/audit/internal/platform/export_test.go +++ b/audit/internal/platform/export_test.go @@ -110,3 +110,50 @@ NAME="sdb1" TYPE="part" PKNAME="sdb" RM="1" RO="0" FSTYPE="vfat" MOUNTPOINT="/me t.Fatalf("device=%q want /dev/sdb1", got) } } + +func TestReadScenarioFromRemovableMediaFindsFileOnAlreadyMountedTarget(t *testing.T) { + mountpoint := t.TempDir() + if err := os.MkdirAll(filepath.Join(mountpoint, "scenarios"), 0755); err != nil { + t.Fatalf("mkdir scenarios: %v", err) + } + want := []byte(`{"name":"power-watch","jobs":[{"name":"a","type":"command","cmd":["true"]}]}`) + if err := os.WriteFile(filepath.Join(mountpoint, "scenarios", "power-watch.json"), want, 0644); err != nil { + t.Fatalf("write scenario file: %v", err) + } + + oldExec := exportExecCommand + lsblkOut := `NAME="sdb1" TYPE="part" PKNAME="sdb" RM="1" RO="0" FSTYPE="vfat" MOUNTPOINT="` + mountpoint + `" SIZE="29.8G" LABEL="USB" MODEL=""` + exportExecCommand = func(name string, args ...string) *exec.Cmd { + cmd := exec.Command("sh", "-c", "printf '%s\n' \"$LSBLK_OUT\"") + cmd.Env = append(os.Environ(), "LSBLK_OUT="+lsblkOut) + return cmd + } + t.Cleanup(func() { exportExecCommand = oldExec }) + + s := &System{} + got, err := s.ReadScenarioFromRemovableMedia("power-watch") + if err != nil { + t.Fatalf("ReadScenarioFromRemovableMedia error: %v", err) + } + if string(got) != string(want) { + t.Fatalf("got=%q want=%q", got, want) + } +} + +func TestReadScenarioFromRemovableMediaNotFound(t *testing.T) { + mountpoint := t.TempDir() // no scenarios/ dir at all + + oldExec := exportExecCommand + lsblkOut := `NAME="sdb1" TYPE="part" PKNAME="sdb" RM="1" RO="0" FSTYPE="vfat" MOUNTPOINT="` + mountpoint + `" SIZE="29.8G" LABEL="USB" MODEL=""` + exportExecCommand = func(name string, args ...string) *exec.Cmd { + cmd := exec.Command("sh", "-c", "printf '%s\n' \"$LSBLK_OUT\"") + cmd.Env = append(os.Environ(), "LSBLK_OUT="+lsblkOut) + return cmd + } + t.Cleanup(func() { exportExecCommand = oldExec }) + + s := &System{} + if _, err := s.ReadScenarioFromRemovableMedia("nope"); err == nil { + t.Fatal("expected error for missing scenario file") + } +} diff --git a/audit/internal/platform/scenario.go b/audit/internal/platform/scenario.go new file mode 100644 index 0000000..9b3a8bd --- /dev/null +++ b/audit/internal/platform/scenario.go @@ -0,0 +1,293 @@ +package platform + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +// ScenarioJob is one step of a ScenarioSpec: either a one-shot/blocking +// command ("command") or a periodic background sampler ("sampler") that +// keeps running until every command job in the scenario has finished (or +// the scenario's overall timeout elapses). +type ScenarioJob struct { + Name string `json:"name"` + Type string `json:"type"` // "command" | "sampler" + Cmd []string `json:"cmd"` + // GPUIndices, if non-empty, replaces the literal token "{{gpus}}" in Cmd + // with a comma-joined index list — lets a scenario file say + // ["dcgmi","diag","-r","nvbandwidth","-i","{{gpus}}"] once instead of + // hardcoding a specific host's GPU indices into the file. + GPUIndices []int `json:"gpu_indices,omitempty"` + // IntervalSec is the sampling period for a "sampler" job; ignored for + // "command" jobs. + IntervalSec int `json:"interval_sec,omitempty"` + // Parallel, for a "command" job, means it starts alongside whichever + // other parallel command jobs precede it instead of waiting for them to + // finish first. Sequential (default) command jobs run in file order, + // each waiting for any preceding parallel batch to finish first. + Parallel bool `json:"parallel,omitempty"` +} + +// ScenarioSpec is a user-authored test scenario: what to run, on which +// GPUs, and what to sample in the background while it runs. Parsed from +// plain JSON (see ParseScenarioJSON) so authoring one needs no tooling +// beyond a text editor and no new Go dependency for this codebase. +// +// Example — reproduce a hard reboot seen only when all GPUs run nvbandwidth +// together (not on any single socket alone), while watching PSU/IPMI +// sensors for a power-delivery correlation: +// +// { +// "name": "nvbandwidth-all-gpu-power-watch", +// "timeout_sec": 1800, +// "jobs": [ +// {"name": "ipmi-sensors", "type": "sampler", "interval_sec": 2, +// "cmd": ["ipmitool", "sensor"]}, +// {"name": "gpu-power", "type": "sampler", "interval_sec": 2, +// "cmd": ["nvidia-smi", "--query-gpu=index,power.draw,temperature.gpu,clocks.sm", "--format=csv"]}, +// {"name": "nvbandwidth-all", "type": "command", +// "cmd": ["dcgmi", "diag", "-r", "nvbandwidth", "-i", "{{gpus}}"], +// "gpu_indices": [0, 1, 2, 3, 4]} +// ] +// } +type ScenarioSpec struct { + Name string `json:"name"` + TimeoutSec int `json:"timeout_sec,omitempty"` + Jobs []ScenarioJob `json:"jobs"` +} + +// ParseScenarioJSON parses and validates a scenario file's contents. +func ParseScenarioJSON(data []byte) (ScenarioSpec, error) { + var spec ScenarioSpec + if err := json.Unmarshal(data, &spec); err != nil { + return ScenarioSpec{}, fmt.Errorf("parse scenario: %w", err) + } + if strings.TrimSpace(spec.Name) == "" { + return ScenarioSpec{}, fmt.Errorf(`scenario: "name" is required`) + } + if len(spec.Jobs) == 0 { + return ScenarioSpec{}, fmt.Errorf("scenario: at least one job is required") + } + seen := map[string]bool{} + for i, j := range spec.Jobs { + if strings.TrimSpace(j.Name) == "" { + return ScenarioSpec{}, fmt.Errorf("scenario: job %d: \"name\" is required", i) + } + if seen[j.Name] { + return ScenarioSpec{}, fmt.Errorf("scenario: duplicate job name %q", j.Name) + } + seen[j.Name] = true + if j.Type != "command" && j.Type != "sampler" { + return ScenarioSpec{}, fmt.Errorf("scenario: job %q: type must be \"command\" or \"sampler\", got %q", j.Name, j.Type) + } + if len(j.Cmd) == 0 { + return ScenarioSpec{}, fmt.Errorf("scenario: job %q: \"cmd\" is required", j.Name) + } + if j.Type == "sampler" && j.IntervalSec <= 0 { + return ScenarioSpec{}, fmt.Errorf("scenario: sampler job %q: \"interval_sec\" must be > 0", j.Name) + } + } + return spec, nil +} + +// RunScenario executes a ScenarioSpec: runs its "command" jobs (sequential +// unless marked Parallel, each blocking until it exits), while every +// "sampler" job runs concurrently in the background on its own interval +// until all command jobs have finished or TimeoutSec elapses. Every job's +// own output streams live to its own file under the returned run +// directory. +// +// Command jobs are wired through the same satJobBoundaryHook/ +// satSyncBracketHook seams the SAT job runner uses (see sat.go), so a +// scenario run gets the same durability treatment: a crash mid-command +// still leaves whatever printed up to that point, and — if blackbox is +// running — that evidence is requested (and, for the sync-bracket hook, +// waited on) to reach removable media before/after the command runs rather +// than only at the end of an adaptive flush period. +func (s *System) RunScenario(ctx context.Context, baseDir string, spec ScenarioSpec, logFunc func(string)) (string, error) { + if baseDir == "" { + baseDir = "/var/log/bee-sat" + } + ts := time.Now().UTC().Format("20060102-150405") + runDir := filepath.Join(baseDir, "scenario-"+sanitizeScenarioName(spec.Name)+"-"+ts) + if err := os.MkdirAll(runDir, 0755); err != nil { + return "", err + } + verboseLog := filepath.Join(runDir, "verbose.log") + + if spec.TimeoutSec > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, time.Duration(spec.TimeoutSec)*time.Second) + defer cancel() + } + + sampCtx, stopSamplers := context.WithCancel(ctx) + defer stopSamplers() + var samplerWG sync.WaitGroup + for _, job := range spec.Jobs { + if job.Type != "sampler" { + continue + } + job := job + samplerWG.Add(1) + go func() { + defer samplerWG.Done() + runScenarioSampler(sampCtx, runDir, verboseLog, job, logFunc) + }() + } + + var ( + mu sync.Mutex + summary strings.Builder + cmdWG sync.WaitGroup + ) + fmt.Fprintf(&summary, "scenario=%s\n", spec.Name) + fmt.Fprintf(&summary, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339)) + + runOne := func(job ScenarioJob) { + err := runScenarioCommand(ctx, runDir, verboseLog, job, logFunc) + mu.Lock() + if err != nil { + fmt.Fprintf(&summary, "%s_status=FAILED\n", job.Name) + fmt.Fprintf(&summary, "%s_error=%s\n", job.Name, err.Error()) + } else { + fmt.Fprintf(&summary, "%s_status=OK\n", job.Name) + } + mu.Unlock() + } + + for _, job := range spec.Jobs { + if job.Type != "command" { + continue + } + if job.Parallel { + cmdWG.Add(1) + go func(j ScenarioJob) { + defer cmdWG.Done() + runOne(j) + }(job) + continue + } + // A sequential job waits for any parallel batch launched ahead of it + // to finish first, so scenario-file order stays intuitive: parallel + // jobs run together, the next sequential job starts only once they + // (and any earlier sequential job) are done. + cmdWG.Wait() + runOne(job) + } + cmdWG.Wait() + + stopSamplers() + samplerWG.Wait() + + if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary.String()), 0644); err != nil { + return "", err + } + return runDir, nil +} + +func runScenarioCommand(ctx context.Context, runDir, verboseLog string, job ScenarioJob, logFunc func(string)) error { + cmd := substituteGPUIndices(job.Cmd, job.GPUIndices) + livePath := filepath.Join(runDir, sanitizeScenarioName(job.Name)+".log") + + if satSyncBracketHook != nil { + if err := satSyncBracketHook(job.Name, "before"); err != nil && logFunc != nil { + logFunc(fmt.Sprintf("%s: blackbox sync wait (before) did not complete cleanly: %v", job.Name, err)) + } + } + _, err := runSATCommandCtx(ctx, verboseLog, job.Name, cmd, nil, logFunc, livePath) + if satJobBoundaryHook != nil { + satJobBoundaryHook(job.Name) + } + if satSyncBracketHook != nil { + if syncErr := satSyncBracketHook(job.Name, "after"); syncErr != nil && logFunc != nil { + logFunc(fmt.Sprintf("%s: blackbox sync wait (after) did not complete cleanly: %v", job.Name, syncErr)) + } + } + return err +} + +// runScenarioSampler runs job.Cmd once immediately (so even a scenario that +// finishes very quickly gets a baseline reading) and then every +// job.IntervalSec until ctx is done, appending each timestamped sample to +// its own file under runDir. +func runScenarioSampler(ctx context.Context, runDir, verboseLog string, job ScenarioJob, logFunc func(string)) { + path := filepath.Join(runDir, sanitizeScenarioName(job.Name)+".log") + f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + if logFunc != nil { + logFunc(fmt.Sprintf("%s: could not open sampler output %s: %v", job.Name, path, err)) + } + return + } + defer f.Close() + + cmd := substituteGPUIndices(job.Cmd, job.GPUIndices) + sample := func() { + out, cmdErr := satExecCommand(cmd[0], cmd[1:]...).CombinedOutput() + ts := time.Now().UTC().Format(time.RFC3339Nano) + fmt.Fprintf(f, "=== %s ===\n", ts) + if cmdErr != nil { + fmt.Fprintf(f, "error: %v\n", cmdErr) + } + _, _ = f.Write(out) + if len(out) == 0 || out[len(out)-1] != '\n' { + _, _ = f.Write([]byte("\n")) + } + _ = f.Sync() + } + + sample() + ticker := time.NewTicker(time.Duration(job.IntervalSec) * time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + appendSATVerboseLog(verboseLog, fmt.Sprintf("[%s] sampler %s stopped", time.Now().UTC().Format(time.RFC3339), job.Name)) + return + case <-ticker.C: + sample() + } + } +} + +// substituteGPUIndices replaces the literal token "{{gpus}}" in each cmd +// argument with a comma-joined GPU index list. Returns cmd unchanged +// (same backing values, no allocation) when gpuIndices is empty. +func substituteGPUIndices(cmd []string, gpuIndices []int) []string { + if len(gpuIndices) == 0 { + return cmd + } + joined := joinIndexList(gpuIndices) + out := make([]string, len(cmd)) + for i, arg := range cmd { + out[i] = strings.ReplaceAll(arg, "{{gpus}}", joined) + } + return out +} + +// sanitizeScenarioName converts a scenario/job name into a safe filename +// component: lowercase alphanumerics, '-', and '_' pass through; anything +// else (spaces, punctuation) becomes '-'. +func sanitizeScenarioName(name string) string { + var b strings.Builder + for _, r := range strings.ToLower(strings.TrimSpace(name)) { + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '-', r == '_': + b.WriteRune(r) + default: + b.WriteRune('-') + } + } + s := b.String() + if s == "" { + return "scenario" + } + return s +} diff --git a/audit/internal/platform/scenario_test.go b/audit/internal/platform/scenario_test.go new file mode 100644 index 0000000..42d7b9f --- /dev/null +++ b/audit/internal/platform/scenario_test.go @@ -0,0 +1,206 @@ +package platform + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" +) + +func TestParseScenarioJSONValidatesRequiredFields(t *testing.T) { + cases := []struct { + name string + json string + wantErr string + }{ + {"missing name", `{"jobs":[{"name":"a","type":"command","cmd":["true"]}]}`, `"name" is required`}, + {"no jobs", `{"name":"x","jobs":[]}`, "at least one job is required"}, + {"job missing name", `{"name":"x","jobs":[{"type":"command","cmd":["true"]}]}`, `"name" is required`}, + {"duplicate job name", `{"name":"x","jobs":[{"name":"a","type":"command","cmd":["true"]},{"name":"a","type":"command","cmd":["true"]}]}`, "duplicate job name"}, + {"bad type", `{"name":"x","jobs":[{"name":"a","type":"bogus","cmd":["true"]}]}`, `type must be "command" or "sampler"`}, + {"missing cmd", `{"name":"x","jobs":[{"name":"a","type":"command"}]}`, `"cmd" is required`}, + {"sampler no interval", `{"name":"x","jobs":[{"name":"a","type":"sampler","cmd":["true"]}]}`, `"interval_sec" must be > 0`}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + _, err := ParseScenarioJSON([]byte(c.json)) + if err == nil || !strings.Contains(err.Error(), c.wantErr) { + t.Fatalf("err=%v want containing %q", err, c.wantErr) + } + }) + } +} + +func TestParseScenarioJSONValid(t *testing.T) { + spec, err := ParseScenarioJSON([]byte(`{ + "name": "power-watch", + "timeout_sec": 60, + "jobs": [ + {"name": "ipmi", "type": "sampler", "interval_sec": 2, "cmd": ["ipmitool", "sensor"]}, + {"name": "load", "type": "command", "cmd": ["dcgmi", "diag", "-r", "nvbandwidth", "-i", "{{gpus}}"], "gpu_indices": [0,1,2]} + ] + }`)) + if err != nil { + t.Fatalf("ParseScenarioJSON error: %v", err) + } + if spec.Name != "power-watch" || spec.TimeoutSec != 60 || len(spec.Jobs) != 2 { + t.Fatalf("spec=%+v", spec) + } +} + +func TestSubstituteGPUIndices(t *testing.T) { + cmd := []string{"dcgmi", "diag", "-r", "nvbandwidth", "-i", "{{gpus}}"} + got := substituteGPUIndices(cmd, []int{0, 2, 4}) + want := []string{"dcgmi", "diag", "-r", "nvbandwidth", "-i", "0,2,4"} + if len(got) != len(want) { + t.Fatalf("got=%v want=%v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("got=%v want=%v", got, want) + } + } + // No GPUIndices: cmd passes through untouched (no {{gpus}} to replace). + same := substituteGPUIndices(cmd, nil) + if same[5] != "{{gpus}}" { + t.Fatalf("expected token left alone when no indices given, got %v", same) + } +} + +func TestSanitizeScenarioName(t *testing.T) { + cases := map[string]string{ + "power-watch": "power-watch", + "Power Watch!!": "power-watch--", + "": "scenario", + "already_ok-123": "already_ok-123", + } + for in, want := range cases { + if got := sanitizeScenarioName(in); got != want { + t.Fatalf("sanitizeScenarioName(%q)=%q want %q", in, got, want) + } + } +} + +func TestRunScenarioRunsCommandsAndSamplersConcurrently(t *testing.T) { + old := satExecCommand + var samplerCalls int32 + satExecCommand = func(name string, args ...string) *exec.Cmd { + if name == "sample-cmd" { + atomic.AddInt32(&samplerCalls, 1) + return exec.Command("printf", "sampled\n") + } + return exec.Command(name, args...) + } + t.Cleanup(func() { satExecCommand = old }) + + dir := t.TempDir() + spec := ScenarioSpec{ + Name: "test-scenario", + TimeoutSec: 10, + Jobs: []ScenarioJob{ + {Name: "watch", Type: "sampler", Cmd: []string{"sample-cmd"}, IntervalSec: 1}, + {Name: "load", Type: "command", Cmd: []string{"printf", "loaded\n"}}, + }, + } + + s := &System{} + runDir, err := s.RunScenario(context.Background(), dir, spec, nil) + if err != nil { + t.Fatalf("RunScenario error: %v", err) + } + + if _, err := os.Stat(filepath.Join(runDir, "load.log")); err != nil { + t.Fatalf("load.log missing: %v", err) + } + loadOut, err := os.ReadFile(filepath.Join(runDir, "load.log")) + if err != nil || strings.TrimSpace(string(loadOut)) != "loaded" { + t.Fatalf("load.log=%q err=%v", loadOut, err) + } + + watchOut, err := os.ReadFile(filepath.Join(runDir, "watch.log")) + if err != nil { + t.Fatalf("watch.log missing: %v", err) + } + if !strings.Contains(string(watchOut), "sampled") { + t.Fatalf("watch.log=%q want it to contain a sample", watchOut) + } + if atomic.LoadInt32(&samplerCalls) < 1 { + t.Fatalf("sampler command was never invoked") + } + + summary, err := os.ReadFile(filepath.Join(runDir, "summary.txt")) + if err != nil { + t.Fatalf("summary.txt missing: %v", err) + } + if !strings.Contains(string(summary), "load_status=OK") { + t.Fatalf("summary=%q want load_status=OK", summary) + } +} + +func TestRunScenarioStopsSamplersWhenCommandsFinish(t *testing.T) { + old := satExecCommand + var samplerCalls int32 + satExecCommand = func(name string, args ...string) *exec.Cmd { + atomic.AddInt32(&samplerCalls, 1) + return exec.Command("true") + } + t.Cleanup(func() { satExecCommand = old }) + + dir := t.TempDir() + spec := ScenarioSpec{ + Name: "fast-scenario", + Jobs: []ScenarioJob{ + // Interval much longer than the scenario itself takes to run — + // if the sampler weren't stopped promptly when the command + // finishes, this test would need to wait out the interval. + {Name: "watch", Type: "sampler", Cmd: []string{"sample-cmd"}, IntervalSec: 3600}, + {Name: "load", Type: "command", Cmd: []string{"true"}}, + }, + } + + s := &System{} + start := time.Now() + if _, err := s.RunScenario(context.Background(), dir, spec, nil); err != nil { + t.Fatalf("RunScenario error: %v", err) + } + if elapsed := time.Since(start); elapsed > 5*time.Second { + t.Fatalf("RunScenario took %s — sampler wasn't stopped promptly", elapsed) + } + // Exactly one sample: the immediate baseline one, no second tick. + if got := atomic.LoadInt32(&samplerCalls); got != 1 { + t.Fatalf("samplerCalls=%d want 1 (baseline only)", got) + } +} + +func TestRunScenarioRecordsFailedCommandInSummary(t *testing.T) { + old := satExecCommand + satExecCommand = func(name string, args ...string) *exec.Cmd { + return exec.Command(name, args...) + } + t.Cleanup(func() { satExecCommand = old }) + + dir := t.TempDir() + spec := ScenarioSpec{ + Name: "failing-scenario", + Jobs: []ScenarioJob{ + {Name: "will-fail", Type: "command", Cmd: []string{"false"}}, + }, + } + + s := &System{} + runDir, err := s.RunScenario(context.Background(), dir, spec, nil) + if err != nil { + t.Fatalf("RunScenario error: %v", err) + } + summary, err := os.ReadFile(filepath.Join(runDir, "summary.txt")) + if err != nil { + t.Fatalf("summary.txt missing: %v", err) + } + if !strings.Contains(string(summary), "will-fail_status=FAILED") { + t.Fatalf("summary=%q want will-fail_status=FAILED", summary) + } +} diff --git a/audit/internal/webui/api.go b/audit/internal/webui/api.go index d072a49..1a789e0 100644 --- a/audit/internal/webui/api.go +++ b/audit/internal/webui/api.go @@ -1865,9 +1865,17 @@ func (h *handler) handleAPIComponentDetail(w http.ResponseWriter, r *http.Reques records = matchedRecords(all, exact, prefixes) } + fromInventory := false + if len(records) == 0 { + if fallback := inventoryFallbackRecords(compType, h.opts); len(fallback) > 0 { + records = fallback + fromInventory = true + } + } + w.Header().Set("Content-Type", "text/html; charset=utf-8") w.Header().Set("Cache-Control", "no-store") - fmt.Fprint(w, renderComponentDetail(title, records)) + fmt.Fprint(w, renderComponentDetail(title, records, fromInventory)) } func (h *handler) rollbackPendingNetworkChange() error { diff --git a/audit/internal/webui/page_topo.go b/audit/internal/webui/page_topo.go index cda9e35..8fa4f58 100644 --- a/audit/internal/webui/page_topo.go +++ b/audit/internal/webui/page_topo.go @@ -11,6 +11,7 @@ import ( "strconv" "strings" + "bee/audit/internal/app" "bee/audit/internal/schema" ) @@ -1221,3 +1222,126 @@ func errNoteSuffix(hasError bool) string { } return "" } + +// --------------------------------------------------------------------------- +// Inventory fallback for the component-detail modal +// +// handleAPIComponentDetail normally sources records from app.ComponentStatusDB, +// which only gains entries once something has actually written a status +// observation (SAT run, watchdog tick, ...). On a freshly booted host that +// hasn't run SAT yet, StatusDB can be entirely empty for a component type even +// though the /topo card for it already shows "N OK" — that card reads +// schema.HardwareComponentStatus.Status straight from the audit snapshot. +// inventoryFallbackRecords bridges that gap by building synthetic records +// from the same snapshot/classifiers the topology card uses, so the two +// views never disagree about how many devices exist or their status. +// --------------------------------------------------------------------------- + +// topoSeverityStatus renders classifyTopoSeverity's rank back into the status +// string vocabulary renderComponentDetail/chipLetterClass expect ("OK", +// "Warning", "Critical", "Unknown") — kept in lockstep with classifyTopoSeverity +// so a device the topo card counts as "OK" is never shown here as "Unknown". +func topoSeverityStatus(status *string) string { + switch classifyTopoSeverity(status) { + case 3: + return "Critical" + case 2: + return "Warning" + case 1: + return "OK" + default: + return "Unknown" + } +} + +// pcieDeviceKind classifies a PCIe device the same way renderTopoMainDiagram +// does, returning "" for devices that aren't GPU/NIC/RAID. +func pcieDeviceKind(dev schema.HardwarePCIeDevice) string { + switch { + case dev.DeviceClass != nil && isGPUDeviceClass(*dev.DeviceClass): + return "gpu" + case isNICDeviceClassDev(dev): + return "nic" + case dev.DeviceClass != nil && isRAIDControllerClass(*dev.DeviceClass): + return "raid" + default: + return "" + } +} + +// pcieDeviceKey builds a stable, human-readable component key for a PCIe +// device: ":" when a slot/BDF is known, else ":". +func pcieDeviceKey(kind string, index int, dev schema.HardwarePCIeDevice) string { + bdf := "" + if dev.Slot != nil { + bdf = normalizeTopoBDF(*dev.Slot) + } else if dev.BDF != nil { + bdf = normalizeTopoBDF(*dev.BDF) + } + if bdf != "" { + return kind + ":" + bdf + } + return fmt.Sprintf("%s:%d", kind, index) +} + +// inventoryFallbackRecords builds ComponentStatusRecord entries straight from +// the audit inventory (bee-audit.json) for the given component type, used +// when ComponentStatusDB has no matching records yet. Records carry only +// ComponentKey/Status — no LastCheckedAt/History — so renderComponentDetail +// renders them without a "checked at" timestamp or sparkline. +func inventoryFallbackRecords(compType string, opts HandlerOptions) []app.ComponentStatusRecord { + data, err := loadSnapshot(opts.AuditPath) + if err != nil { + return nil + } + var ingest schema.HardwareIngestRequest + if err := json.Unmarshal(data, &ingest); err != nil { + return nil + } + hw := ingest.Hardware + + var records []app.ComponentStatusRecord + switch compType { + case "cpu": + for i, cpu := range hw.CPUs { + key := fmt.Sprintf("cpu:%d", i) + if cpu.Socket != nil { + key = fmt.Sprintf("cpu:socket%d", *cpu.Socket) + } + records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(cpu.Status)}) + } + case "memory": + for i, m := range hw.Memory { + key := fmt.Sprintf("memory:%d", i) + if m.Slot != nil && strings.TrimSpace(*m.Slot) != "" { + key = "memory:" + strings.TrimSpace(*m.Slot) + } + records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(m.Status)}) + } + case "storage": + for i, s := range hw.Storage { + key := fmt.Sprintf("storage:%d", i) + if s.Slot != nil && strings.TrimSpace(*s.Slot) != "" { + key = "storage:" + strings.TrimSpace(*s.Slot) + } + records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(s.Status)}) + } + case "psu": + for i, p := range hw.PowerSupplies { + key := fmt.Sprintf("psu:%d", i) + if p.Slot != nil && strings.TrimSpace(*p.Slot) != "" { + key = "psu:" + strings.TrimSpace(*p.Slot) + } + records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(p.Status)}) + } + case "gpu", "nic", "raid": + for i, dev := range hw.PCIeDevices { + if pcieDeviceKind(dev) != compType { + continue + } + key := pcieDeviceKey(compType, i, dev) + records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(dev.Status)}) + } + } + return records +} diff --git a/audit/internal/webui/page_topo_test.go b/audit/internal/webui/page_topo_test.go index b4c1a62..fe16246 100644 --- a/audit/internal/webui/page_topo_test.go +++ b/audit/internal/webui/page_topo_test.go @@ -576,6 +576,64 @@ GPU 1: NVIDIA H100 80GB HBM3 (UUID: GPU-603fe750-0516-9db5-86ec-ea61af3fce35) } } +// TestComponentDetailFallsBackToInventoryWhenStatusDBEmpty covers the bug where +// a /topo card shows "3 OK" (from schema.HardwareComponentStatus.Status in the +// audit snapshot) but clicking it opens a modal saying "No status data recorded +// yet" (because ComponentStatusDB has no pcie:gpu:* entries — nothing has run a +// SAT test on this boot yet). The modal must show the same 3 devices/status the +// card does, not an empty state. +func TestComponentDetailFallsBackToInventoryWhenStatusDBEmpty(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "audit.json") + + okStatus := "OK" + warnStatus := "Warning" + deviceClass := "VideoController" + var gpus []schema.HardwarePCIeDevice + for i, st := range []*string{&okStatus, &okStatus, &warnStatus} { + slot := "0000:c" + strconv.Itoa(i) + ":00.0" + gpus = append(gpus, schema.HardwarePCIeDevice{ + HardwareComponentStatus: schema.HardwareComponentStatus{Status: st}, + DeviceClass: &deviceClass, + Slot: &slot, + }) + } + + ingest := schema.HardwareIngestRequest{ + Hardware: schema.HardwareSnapshot{PCIeDevices: gpus}, + } + data, err := json.Marshal(ingest) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0644); err != nil { + t.Fatal(err) + } + + // No HandlerOptions.App / StatusDB set — matches a host where nothing has + // written to ComponentStatusDB yet. + handler := NewHandler(HandlerOptions{AuditPath: path}) + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/components/gpu", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("status=%d body=%s", rec.Code, rec.Body.String()) + } + body := rec.Body.String() + + if strings.Contains(body, "No status data recorded yet") { + t.Fatalf("modal should not show empty state when inventory has GPUs: %s", body) + } + if strings.Count(body, "chip-ok") != 2 { + t.Fatalf("expected 2 OK chips from inventory fallback: %s", body) + } + if strings.Count(body, "chip-warn") != 1 { + t.Fatalf("expected 1 Warning chip from inventory fallback: %s", body) + } + if !strings.Contains(body, "No SAT-test history yet") { + t.Fatalf("expected fallback marker text: %s", body) + } +} + func TestParseTopoNVLinkErrors(t *testing.T) { input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140) Link 0: Replay Errors: 0 diff --git a/audit/internal/webui/pages.go b/audit/internal/webui/pages.go index 800fafc..aa9f113 100644 --- a/audit/internal/webui/pages.go +++ b/audit/internal/webui/pages.go @@ -1160,7 +1160,10 @@ func renderSparkline(history []app.ComponentStatusEntry) string { // renderComponentDetail renders a modal content fragment for one component type. // Called by handleAPIComponentDetail and displayed inside #component-detail-dialog. -func renderComponentDetail(title string, records []app.ComponentStatusRecord) string { +// fromInventory marks that records were synthesized from the audit inventory +// snapshot (no ComponentStatusDB history yet) rather than real SAT/watchdog +// observations — see inventoryFallbackRecords. +func renderComponentDetail(title string, records []app.ComponentStatusRecord, fromInventory bool) string { var b strings.Builder fmt.Fprintf(&b, `
`) fmt.Fprintf(&b, `
`) @@ -1174,6 +1177,10 @@ func renderComponentDetail(title string, records []app.ComponentStatusRecord) st return b.String() } + if fromInventory { + b.WriteString(`

No SAT-test history yet — showing latest inventory snapshot.

`) + } + sort.Slice(records, func(i, j int) bool { return records[i].ComponentKey < records[j].ComponentKey }) diff --git a/scenarios/README.md b/scenarios/README.md new file mode 100644 index 0000000..eab1248 --- /dev/null +++ b/scenarios/README.md @@ -0,0 +1,41 @@ +# bee test scenarios + +A scenario is a plain JSON file describing an ad-hoc diagnostic run: which +commands to execute (sequentially or in parallel) and what to sample in the +background while they run — without hardcoding a new test into bee's own +code. See `audit/internal/platform/scenario.go` (`ParseScenarioJSON`, +`ScenarioSpec`) for the full field reference. + +## Running one + +``` +bee run +``` + +or, for a scenario file dropped under `scenarios/` on a mounted removable +drive (e.g. the same USB stick already plugged in for blackbox — useful on +an air-gapped host with no other way to get a file onto it): + +``` +bee run # looks for scenarios/.json on any mounted removable media +``` + +`bee scenario run ` is the same command under a longer name. + +## Files checked in here + +- `nvbandwidth-all-gpu-power-watch.json` — the scenario that reproduced the + CG480-S6053 reboot: full `nvbandwidth` across all GPUs at once (the + per-GPU-socket passes alone never reproduced it), with `ipmitool sensor` + and `nvidia-smi` power/temp sampled every 2s in the background so a crash + mid-run still leaves telemetry to check for a power-delivery correlation. + **`gpu_indices` is host-specific** — update it to match the GPU indices + `nvidia-smi -L` actually reports on the box under test before running. + +## Adding more + +Not every scenario needs to be checked in here. For a one-off test on a +specific host (especially air-gapped), it's simpler to write the JSON file +directly onto the blackbox USB stick under `scenarios/.json` and run +`bee run ` — no code change, no rebuild. Check a scenario in here only +when it's worth keeping around as a reusable/named test across hosts. diff --git a/scenarios/nvbandwidth-all-gpu-power-watch.json b/scenarios/nvbandwidth-all-gpu-power-watch.json new file mode 100644 index 0000000..0633307 --- /dev/null +++ b/scenarios/nvbandwidth-all-gpu-power-watch.json @@ -0,0 +1,24 @@ +{ + "name": "nvbandwidth-all-gpu-power-watch", + "timeout_sec": 1800, + "jobs": [ + { + "name": "ipmi-sensors", + "type": "sampler", + "interval_sec": 2, + "cmd": ["ipmitool", "sensor"] + }, + { + "name": "gpu-power", + "type": "sampler", + "interval_sec": 2, + "cmd": ["nvidia-smi", "--query-gpu=index,power.draw,temperature.gpu,clocks.sm", "--format=csv"] + }, + { + "name": "nvbandwidth-all", + "type": "command", + "cmd": ["dcgmi", "diag", "-r", "nvbandwidth", "-i", "{{gpus}}"], + "gpu_indices": [0, 1, 2, 3, 4, 5] + } + ] +}