From 20cd317c8765bdb0b4d44fb3854aef5c72a871c4 Mon Sep 17 00:00:00 2001 From: Mikhail Chusavitin Date: Tue, 28 Jul 2026 17:51:45 +0300 Subject: [PATCH] platform/cmd/webui: add a scriptable test-scenario engine, load scenarios from blackbox USB, GPU status detail inventory fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Investigating the CG480-S6053 reboot needed a way to run an ad-hoc load (nvbandwidth across a specific GPU set) while sampling IPMI/nvidia-smi telemetry in the background — without hardcoding a one-off test into the SAT pack code for a single investigation. - audit/internal/platform/scenario.go: ScenarioSpec/ScenarioJob (JSON, no new dependency) + System.RunScenario. "command" jobs run sequential or parallel (per-job "parallel" flag); "sampler" jobs run concurrently in the background on their own interval until every command job finishes or the scenario's timeout elapses. "{{gpus}}" in a command's cmd is substituted from that job's gpu_indices. Command jobs are wired through the same satJobBoundaryHook/satSyncBracketHook seams the SAT job runner uses, so a scenario run gets the same durability treatment (evidence that a risky command started/finished reaches blackbox before a possible crash, not just whatever streamed to the RAM-backed export dir). - export.go: ReadScenarioFromRemovableMedia mounts each removable target looking for scenarios/.json — an air-gapped engineer can author a scenario elsewhere, drop it under scenarios/ on the same USB stick already plugged in for blackbox, and run it with no network path onto the host. - cmd/bee: new `bee run ` (bare name = looked up on removable media); `bee scenario run ` kept as a longer alias. - scenarios/nvbandwidth-all-gpu-power-watch.json: the scenario that reproduced the actual reboot (full nvbandwidth across all GPUs, which crashed, vs. clean per-socket passes), with IPMI sensor + GPU power/temp sampling for a power-delivery correlation check. Also: webui/page_topo.go — the /topo page's component-status-detail modal (GET /api/component-detail/{type}) showed "No status data recorded yet" for any component type ComponentStatusDB has no history for yet (e.g. GPU before a SAT run this boot), even though the topology card for the same component already showed "N OK" from the audit inventory snapshot. inventoryFallbackRecords now synthesizes records from that same inventory snapshot when StatusDB is empty, using the same device classifiers (isGPUDeviceClass etc.) and severity mapping (classifyTopoSeverity) the topology card itself uses, so the two views never disagree. Co-Authored-By: Claude Sonnet 5 --- audit/cmd/bee/main.go | 84 +++++ audit/cmd/bee/main_test.go | 52 ++++ audit/internal/platform/export.go | 52 ++++ audit/internal/platform/export_test.go | 47 +++ audit/internal/platform/scenario.go | 293 ++++++++++++++++++ audit/internal/platform/scenario_test.go | 206 ++++++++++++ audit/internal/webui/api.go | 10 +- audit/internal/webui/page_topo.go | 124 ++++++++ audit/internal/webui/page_topo_test.go | 58 ++++ audit/internal/webui/pages.go | 9 +- scenarios/README.md | 41 +++ .../nvbandwidth-all-gpu-power-watch.json | 24 ++ 12 files changed, 998 insertions(+), 2 deletions(-) create mode 100644 audit/internal/platform/scenario.go create mode 100644 audit/internal/platform/scenario_test.go create mode 100644 scenarios/README.md create mode 100644 scenarios/nvbandwidth-all-gpu-power-watch.json diff --git a/audit/cmd/bee/main.go b/audit/cmd/bee/main.go index 86f94af..23742d6 100644 --- a/audit/cmd/bee/main.go +++ b/audit/cmd/bee/main.go @@ -74,6 +74,16 @@ func run(args []string, stdout, stderr io.Writer) (exitCode int) { return runBlackbox(args[1:], stdout, stderr) case "sat": return runSAT(args[1:], stdout, stderr) + case "run": + return runScenario(args[1:], stdout, stderr) + case "scenario": + // "bee scenario run " is a longer alias for "bee run "; + // strip the "run" verb (if present) and delegate to the same code. + rest := args[1:] + if len(rest) > 0 && rest[0] == "run" { + rest = rest[1:] + } + return runScenario(rest, stdout, stderr) case "benchmark": return runBenchmark(args[1:], stdout, stderr) case "bee-worker": @@ -98,6 +108,7 @@ func printRootUsage(w io.Writer) { bee web --listen :80 [--audit-path `+app.DefaultAuditJSONPath+`] bee blackbox --export-dir `+app.DefaultExportDir+` [--state-file `+app.DefaultBlackboxStatePath+`] bee sat nvidia|memory|storage|cpu [--duration ] + bee run (bare name is looked up as scenarios/.json on removable media) bee benchmark nvidia [--profile standard|stability|overnight] bee bee-worker --export-dir `+app.DefaultExportDir+` --task-id TASK-001 bee version @@ -122,6 +133,8 @@ func runHelp(args []string, stdout, stderr io.Writer) int { return runBlackbox([]string{"--help"}, stdout, stdout) case "sat": return runSAT([]string{"--help"}, stdout, stderr) + case "run", "scenario": + return runScenario([]string{"--help"}, stdout, stderr) case "benchmark": return runBenchmark([]string{"--help"}, stdout, stderr) case "bee-worker": @@ -469,6 +482,77 @@ func runSAT(args []string, stdout, stderr io.Writer) int { return 0 } +const scenarioUsage = `usage: bee run + + A path (contains "/" or ends in ".json") is read directly from local disk. + A bare name is instead looked up as scenarios/.json on any mounted + removable media (e.g. the same USB stick already plugged in for + blackbox) — lets an air-gapped engineer author a scenario on another + machine and drop it there without needing a network path onto the host. + +See ParseScenarioJSON in audit/internal/platform/scenario.go for the file +format and a worked example (per-GPU/all-GPU load with concurrent IPMI/ +nvidia-smi sampling).` + +// runScenario implements both "bee run " and "bee scenario run " +// (the latter kept as a longer alias). args is just the scenario +// file/name — no leading verb. +func runScenario(args []string, stdout, stderr io.Writer) int { + if len(args) > 0 && (args[0] == "help" || args[0] == "--help" || args[0] == "-h") { + fmt.Fprintln(stdout, scenarioUsage) + return 0 + } + + fs := flag.NewFlagSet("run", flag.ContinueOnError) + fs.SetOutput(stderr) + if err := fs.Parse(args); err != nil { + if err == flag.ErrHelp { + return 0 + } + return 2 + } + if fs.NArg() != 1 { + fmt.Fprintln(stderr, scenarioUsage) + return 2 + } + arg := fs.Arg(0) + + sys := platform.New() + var ( + data []byte + err error + ) + if strings.Contains(arg, "/") || strings.HasSuffix(arg, ".json") { + data, err = os.ReadFile(arg) + } else { + data, err = sys.ReadScenarioFromRemovableMedia(arg) + } + if err != nil { + fmt.Fprintf(stderr, "bee run: %v\n", err) + return 1 + } + + spec, err := platform.ParseScenarioJSON(data) + if err != nil { + fmt.Fprintf(stderr, "bee run: %v\n", err) + return 1 + } + + // Wires the same blackbox kick/sync-bracket hooks the SAT job runner + // uses (see app.New()), so a scenario command job's evidence reaches + // removable media the same way a SAT job's does. + _ = app.New(sys) + + logLine := func(s string) { fmt.Fprintln(stderr, s) } + runDir, err := sys.RunScenario(context.Background(), "", spec, logLine) + if err != nil { + fmt.Fprintf(stderr, "bee run: %v\n", err) + return 1 + } + fmt.Fprintln(stdout, runDir) + return 0 +} + func runBenchmark(args []string, stdout, stderr io.Writer) int { if len(args) == 0 { fmt.Fprintln(stderr, "usage: bee benchmark nvidia [--profile standard|stability|overnight] [--devices 0,1] [--exclude 2,3] [--size-mb N] [--skip-nccl]") diff --git a/audit/cmd/bee/main_test.go b/audit/cmd/bee/main_test.go index e326f55..1625a42 100644 --- a/audit/cmd/bee/main_test.go +++ b/audit/cmd/bee/main_test.go @@ -99,6 +99,58 @@ func TestRunSATUsage(t *testing.T) { } } +func TestRunUsage(t *testing.T) { + t.Parallel() + + var stdout, stderr bytes.Buffer + rc := run([]string{"run"}, &stdout, &stderr) + if rc != 2 { + t.Fatalf("rc=%d want 2", rc) + } + if !strings.Contains(stderr.String(), "usage: bee run ") { + t.Fatalf("stderr missing run usage:\n%s", stderr.String()) + } +} + +func TestRunHelp(t *testing.T) { + t.Parallel() + + var stdout, stderr bytes.Buffer + rc := run([]string{"run", "--help"}, &stdout, &stderr) + if rc != 0 { + t.Fatalf("rc=%d want 0", rc) + } + if !strings.Contains(stdout.String(), "usage: bee run ") { + t.Fatalf("stdout missing run usage:\n%s", stdout.String()) + } +} + +func TestRunMissingScenarioFile(t *testing.T) { + t.Parallel() + + var stdout, stderr bytes.Buffer + rc := run([]string{"run", "/nonexistent/scenario.json"}, &stdout, &stderr) + if rc != 1 { + t.Fatalf("rc=%d want 1", rc) + } + if !strings.Contains(stderr.String(), "bee run:") { + t.Fatalf("stderr missing error prefix:\n%s", stderr.String()) + } +} + +func TestRunScenarioAliasAcceptsRunVerb(t *testing.T) { + t.Parallel() + + var stdout, stderr bytes.Buffer + rc := run([]string{"scenario", "run", "/nonexistent/scenario.json"}, &stdout, &stderr) + if rc != 1 { + t.Fatalf("rc=%d want 1", rc) + } + if !strings.Contains(stderr.String(), "bee run:") { + t.Fatalf("stderr missing error prefix:\n%s", stderr.String()) + } +} + func TestRunPreflightRejectsExtraArgs(t *testing.T) { t.Parallel() diff --git a/audit/internal/platform/export.go b/audit/internal/platform/export.go index 3030642..efb131f 100644 --- a/audit/internal/platform/export.go +++ b/audit/internal/platform/export.go @@ -151,3 +151,55 @@ func (s *System) ExportFileToTarget(src string, target RemovableTarget) (dst str return dst, nil } + +// mountRemovableTargetReadOnly mounts target for reading if it isn't +// already mounted (reusing its existing mountpoint otherwise), returning +// whether this call did the mounting so the caller knows whether to +// unmount afterward. +func mountRemovableTargetReadOnly(target RemovableTarget) (mountpoint string, mountedHere bool, err error) { + if mp := strings.TrimSpace(target.Mountpoint); mp != "" { + return mp, false, nil + } + mountpoint = filepath.Join("/tmp", "bee-scenario-"+filepath.Base(target.Device)) + if err := os.MkdirAll(mountpoint, 0755); err != nil { + return "", false, err + } + if raw, err := exportExecCommand("mount", target.Device, mountpoint).CombinedOutput(); err != nil { + _ = os.Remove(mountpoint) + return "", false, formatMountTargetError(target, string(raw), err) + } + return mountpoint, true, nil +} + +// ReadScenarioFromRemovableMedia mounts each removable target in turn +// (unmounting again afterward if it mounted it itself), looking for +// scenarios/.json, and returns the contents of the first one found. +// Lets an air-gapped engineer author a scenario JSON file on another +// machine, drop it under scenarios/ on the same flash drive already +// plugged in for blackbox, and run it on the host with no network path +// required — see RunScenario/ParseScenarioJSON. +func (s *System) ReadScenarioFromRemovableMedia(name string) ([]byte, error) { + name = strings.TrimSpace(name) + if name == "" { + return nil, fmt.Errorf("scenario name is required") + } + targets, err := s.ListRemovableTargets() + if err != nil { + return nil, err + } + for _, target := range targets { + mountpoint, mountedHere, mountErr := mountRemovableTargetReadOnly(target) + if mountErr != nil { + continue + } + data, readErr := os.ReadFile(filepath.Join(mountpoint, "scenarios", name+".json")) + if mountedHere { + _, _ = exportExecCommand("umount", mountpoint).CombinedOutput() + _ = os.Remove(mountpoint) + } + if readErr == nil { + return data, nil + } + } + return nil, fmt.Errorf("scenarios/%s.json not found on any removable media", name) +} diff --git a/audit/internal/platform/export_test.go b/audit/internal/platform/export_test.go index 54da24f..122e823 100644 --- a/audit/internal/platform/export_test.go +++ b/audit/internal/platform/export_test.go @@ -110,3 +110,50 @@ NAME="sdb1" TYPE="part" PKNAME="sdb" RM="1" RO="0" FSTYPE="vfat" MOUNTPOINT="/me t.Fatalf("device=%q want /dev/sdb1", got) } } + +func TestReadScenarioFromRemovableMediaFindsFileOnAlreadyMountedTarget(t *testing.T) { + mountpoint := t.TempDir() + if err := os.MkdirAll(filepath.Join(mountpoint, "scenarios"), 0755); err != nil { + t.Fatalf("mkdir scenarios: %v", err) + } + want := []byte(`{"name":"power-watch","jobs":[{"name":"a","type":"command","cmd":["true"]}]}`) + if err := os.WriteFile(filepath.Join(mountpoint, "scenarios", "power-watch.json"), want, 0644); err != nil { + t.Fatalf("write scenario file: %v", err) + } + + oldExec := exportExecCommand + lsblkOut := `NAME="sdb1" TYPE="part" PKNAME="sdb" RM="1" RO="0" FSTYPE="vfat" MOUNTPOINT="` + mountpoint + `" SIZE="29.8G" LABEL="USB" MODEL=""` + exportExecCommand = func(name string, args ...string) *exec.Cmd { + cmd := exec.Command("sh", "-c", "printf '%s\n' \"$LSBLK_OUT\"") + cmd.Env = append(os.Environ(), "LSBLK_OUT="+lsblkOut) + return cmd + } + t.Cleanup(func() { exportExecCommand = oldExec }) + + s := &System{} + got, err := s.ReadScenarioFromRemovableMedia("power-watch") + if err != nil { + t.Fatalf("ReadScenarioFromRemovableMedia error: %v", err) + } + if string(got) != string(want) { + t.Fatalf("got=%q want=%q", got, want) + } +} + +func TestReadScenarioFromRemovableMediaNotFound(t *testing.T) { + mountpoint := t.TempDir() // no scenarios/ dir at all + + oldExec := exportExecCommand + lsblkOut := `NAME="sdb1" TYPE="part" PKNAME="sdb" RM="1" RO="0" FSTYPE="vfat" MOUNTPOINT="` + mountpoint + `" SIZE="29.8G" LABEL="USB" MODEL=""` + exportExecCommand = func(name string, args ...string) *exec.Cmd { + cmd := exec.Command("sh", "-c", "printf '%s\n' \"$LSBLK_OUT\"") + cmd.Env = append(os.Environ(), "LSBLK_OUT="+lsblkOut) + return cmd + } + t.Cleanup(func() { exportExecCommand = oldExec }) + + s := &System{} + if _, err := s.ReadScenarioFromRemovableMedia("nope"); err == nil { + t.Fatal("expected error for missing scenario file") + } +} diff --git a/audit/internal/platform/scenario.go b/audit/internal/platform/scenario.go new file mode 100644 index 0000000..9b3a8bd --- /dev/null +++ b/audit/internal/platform/scenario.go @@ -0,0 +1,293 @@ +package platform + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +// ScenarioJob is one step of a ScenarioSpec: either a one-shot/blocking +// command ("command") or a periodic background sampler ("sampler") that +// keeps running until every command job in the scenario has finished (or +// the scenario's overall timeout elapses). +type ScenarioJob struct { + Name string `json:"name"` + Type string `json:"type"` // "command" | "sampler" + Cmd []string `json:"cmd"` + // GPUIndices, if non-empty, replaces the literal token "{{gpus}}" in Cmd + // with a comma-joined index list — lets a scenario file say + // ["dcgmi","diag","-r","nvbandwidth","-i","{{gpus}}"] once instead of + // hardcoding a specific host's GPU indices into the file. + GPUIndices []int `json:"gpu_indices,omitempty"` + // IntervalSec is the sampling period for a "sampler" job; ignored for + // "command" jobs. + IntervalSec int `json:"interval_sec,omitempty"` + // Parallel, for a "command" job, means it starts alongside whichever + // other parallel command jobs precede it instead of waiting for them to + // finish first. Sequential (default) command jobs run in file order, + // each waiting for any preceding parallel batch to finish first. + Parallel bool `json:"parallel,omitempty"` +} + +// ScenarioSpec is a user-authored test scenario: what to run, on which +// GPUs, and what to sample in the background while it runs. Parsed from +// plain JSON (see ParseScenarioJSON) so authoring one needs no tooling +// beyond a text editor and no new Go dependency for this codebase. +// +// Example — reproduce a hard reboot seen only when all GPUs run nvbandwidth +// together (not on any single socket alone), while watching PSU/IPMI +// sensors for a power-delivery correlation: +// +// { +// "name": "nvbandwidth-all-gpu-power-watch", +// "timeout_sec": 1800, +// "jobs": [ +// {"name": "ipmi-sensors", "type": "sampler", "interval_sec": 2, +// "cmd": ["ipmitool", "sensor"]}, +// {"name": "gpu-power", "type": "sampler", "interval_sec": 2, +// "cmd": ["nvidia-smi", "--query-gpu=index,power.draw,temperature.gpu,clocks.sm", "--format=csv"]}, +// {"name": "nvbandwidth-all", "type": "command", +// "cmd": ["dcgmi", "diag", "-r", "nvbandwidth", "-i", "{{gpus}}"], +// "gpu_indices": [0, 1, 2, 3, 4]} +// ] +// } +type ScenarioSpec struct { + Name string `json:"name"` + TimeoutSec int `json:"timeout_sec,omitempty"` + Jobs []ScenarioJob `json:"jobs"` +} + +// ParseScenarioJSON parses and validates a scenario file's contents. +func ParseScenarioJSON(data []byte) (ScenarioSpec, error) { + var spec ScenarioSpec + if err := json.Unmarshal(data, &spec); err != nil { + return ScenarioSpec{}, fmt.Errorf("parse scenario: %w", err) + } + if strings.TrimSpace(spec.Name) == "" { + return ScenarioSpec{}, fmt.Errorf(`scenario: "name" is required`) + } + if len(spec.Jobs) == 0 { + return ScenarioSpec{}, fmt.Errorf("scenario: at least one job is required") + } + seen := map[string]bool{} + for i, j := range spec.Jobs { + if strings.TrimSpace(j.Name) == "" { + return ScenarioSpec{}, fmt.Errorf("scenario: job %d: \"name\" is required", i) + } + if seen[j.Name] { + return ScenarioSpec{}, fmt.Errorf("scenario: duplicate job name %q", j.Name) + } + seen[j.Name] = true + if j.Type != "command" && j.Type != "sampler" { + return ScenarioSpec{}, fmt.Errorf("scenario: job %q: type must be \"command\" or \"sampler\", got %q", j.Name, j.Type) + } + if len(j.Cmd) == 0 { + return ScenarioSpec{}, fmt.Errorf("scenario: job %q: \"cmd\" is required", j.Name) + } + if j.Type == "sampler" && j.IntervalSec <= 0 { + return ScenarioSpec{}, fmt.Errorf("scenario: sampler job %q: \"interval_sec\" must be > 0", j.Name) + } + } + return spec, nil +} + +// RunScenario executes a ScenarioSpec: runs its "command" jobs (sequential +// unless marked Parallel, each blocking until it exits), while every +// "sampler" job runs concurrently in the background on its own interval +// until all command jobs have finished or TimeoutSec elapses. Every job's +// own output streams live to its own file under the returned run +// directory. +// +// Command jobs are wired through the same satJobBoundaryHook/ +// satSyncBracketHook seams the SAT job runner uses (see sat.go), so a +// scenario run gets the same durability treatment: a crash mid-command +// still leaves whatever printed up to that point, and — if blackbox is +// running — that evidence is requested (and, for the sync-bracket hook, +// waited on) to reach removable media before/after the command runs rather +// than only at the end of an adaptive flush period. +func (s *System) RunScenario(ctx context.Context, baseDir string, spec ScenarioSpec, logFunc func(string)) (string, error) { + if baseDir == "" { + baseDir = "/var/log/bee-sat" + } + ts := time.Now().UTC().Format("20060102-150405") + runDir := filepath.Join(baseDir, "scenario-"+sanitizeScenarioName(spec.Name)+"-"+ts) + if err := os.MkdirAll(runDir, 0755); err != nil { + return "", err + } + verboseLog := filepath.Join(runDir, "verbose.log") + + if spec.TimeoutSec > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, time.Duration(spec.TimeoutSec)*time.Second) + defer cancel() + } + + sampCtx, stopSamplers := context.WithCancel(ctx) + defer stopSamplers() + var samplerWG sync.WaitGroup + for _, job := range spec.Jobs { + if job.Type != "sampler" { + continue + } + job := job + samplerWG.Add(1) + go func() { + defer samplerWG.Done() + runScenarioSampler(sampCtx, runDir, verboseLog, job, logFunc) + }() + } + + var ( + mu sync.Mutex + summary strings.Builder + cmdWG sync.WaitGroup + ) + fmt.Fprintf(&summary, "scenario=%s\n", spec.Name) + fmt.Fprintf(&summary, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339)) + + runOne := func(job ScenarioJob) { + err := runScenarioCommand(ctx, runDir, verboseLog, job, logFunc) + mu.Lock() + if err != nil { + fmt.Fprintf(&summary, "%s_status=FAILED\n", job.Name) + fmt.Fprintf(&summary, "%s_error=%s\n", job.Name, err.Error()) + } else { + fmt.Fprintf(&summary, "%s_status=OK\n", job.Name) + } + mu.Unlock() + } + + for _, job := range spec.Jobs { + if job.Type != "command" { + continue + } + if job.Parallel { + cmdWG.Add(1) + go func(j ScenarioJob) { + defer cmdWG.Done() + runOne(j) + }(job) + continue + } + // A sequential job waits for any parallel batch launched ahead of it + // to finish first, so scenario-file order stays intuitive: parallel + // jobs run together, the next sequential job starts only once they + // (and any earlier sequential job) are done. + cmdWG.Wait() + runOne(job) + } + cmdWG.Wait() + + stopSamplers() + samplerWG.Wait() + + if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary.String()), 0644); err != nil { + return "", err + } + return runDir, nil +} + +func runScenarioCommand(ctx context.Context, runDir, verboseLog string, job ScenarioJob, logFunc func(string)) error { + cmd := substituteGPUIndices(job.Cmd, job.GPUIndices) + livePath := filepath.Join(runDir, sanitizeScenarioName(job.Name)+".log") + + if satSyncBracketHook != nil { + if err := satSyncBracketHook(job.Name, "before"); err != nil && logFunc != nil { + logFunc(fmt.Sprintf("%s: blackbox sync wait (before) did not complete cleanly: %v", job.Name, err)) + } + } + _, err := runSATCommandCtx(ctx, verboseLog, job.Name, cmd, nil, logFunc, livePath) + if satJobBoundaryHook != nil { + satJobBoundaryHook(job.Name) + } + if satSyncBracketHook != nil { + if syncErr := satSyncBracketHook(job.Name, "after"); syncErr != nil && logFunc != nil { + logFunc(fmt.Sprintf("%s: blackbox sync wait (after) did not complete cleanly: %v", job.Name, syncErr)) + } + } + return err +} + +// runScenarioSampler runs job.Cmd once immediately (so even a scenario that +// finishes very quickly gets a baseline reading) and then every +// job.IntervalSec until ctx is done, appending each timestamped sample to +// its own file under runDir. +func runScenarioSampler(ctx context.Context, runDir, verboseLog string, job ScenarioJob, logFunc func(string)) { + path := filepath.Join(runDir, sanitizeScenarioName(job.Name)+".log") + f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + if logFunc != nil { + logFunc(fmt.Sprintf("%s: could not open sampler output %s: %v", job.Name, path, err)) + } + return + } + defer f.Close() + + cmd := substituteGPUIndices(job.Cmd, job.GPUIndices) + sample := func() { + out, cmdErr := satExecCommand(cmd[0], cmd[1:]...).CombinedOutput() + ts := time.Now().UTC().Format(time.RFC3339Nano) + fmt.Fprintf(f, "=== %s ===\n", ts) + if cmdErr != nil { + fmt.Fprintf(f, "error: %v\n", cmdErr) + } + _, _ = f.Write(out) + if len(out) == 0 || out[len(out)-1] != '\n' { + _, _ = f.Write([]byte("\n")) + } + _ = f.Sync() + } + + sample() + ticker := time.NewTicker(time.Duration(job.IntervalSec) * time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + appendSATVerboseLog(verboseLog, fmt.Sprintf("[%s] sampler %s stopped", time.Now().UTC().Format(time.RFC3339), job.Name)) + return + case <-ticker.C: + sample() + } + } +} + +// substituteGPUIndices replaces the literal token "{{gpus}}" in each cmd +// argument with a comma-joined GPU index list. Returns cmd unchanged +// (same backing values, no allocation) when gpuIndices is empty. +func substituteGPUIndices(cmd []string, gpuIndices []int) []string { + if len(gpuIndices) == 0 { + return cmd + } + joined := joinIndexList(gpuIndices) + out := make([]string, len(cmd)) + for i, arg := range cmd { + out[i] = strings.ReplaceAll(arg, "{{gpus}}", joined) + } + return out +} + +// sanitizeScenarioName converts a scenario/job name into a safe filename +// component: lowercase alphanumerics, '-', and '_' pass through; anything +// else (spaces, punctuation) becomes '-'. +func sanitizeScenarioName(name string) string { + var b strings.Builder + for _, r := range strings.ToLower(strings.TrimSpace(name)) { + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '-', r == '_': + b.WriteRune(r) + default: + b.WriteRune('-') + } + } + s := b.String() + if s == "" { + return "scenario" + } + return s +} diff --git a/audit/internal/platform/scenario_test.go b/audit/internal/platform/scenario_test.go new file mode 100644 index 0000000..42d7b9f --- /dev/null +++ b/audit/internal/platform/scenario_test.go @@ -0,0 +1,206 @@ +package platform + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" +) + +func TestParseScenarioJSONValidatesRequiredFields(t *testing.T) { + cases := []struct { + name string + json string + wantErr string + }{ + {"missing name", `{"jobs":[{"name":"a","type":"command","cmd":["true"]}]}`, `"name" is required`}, + {"no jobs", `{"name":"x","jobs":[]}`, "at least one job is required"}, + {"job missing name", `{"name":"x","jobs":[{"type":"command","cmd":["true"]}]}`, `"name" is required`}, + {"duplicate job name", `{"name":"x","jobs":[{"name":"a","type":"command","cmd":["true"]},{"name":"a","type":"command","cmd":["true"]}]}`, "duplicate job name"}, + {"bad type", `{"name":"x","jobs":[{"name":"a","type":"bogus","cmd":["true"]}]}`, `type must be "command" or "sampler"`}, + {"missing cmd", `{"name":"x","jobs":[{"name":"a","type":"command"}]}`, `"cmd" is required`}, + {"sampler no interval", `{"name":"x","jobs":[{"name":"a","type":"sampler","cmd":["true"]}]}`, `"interval_sec" must be > 0`}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + _, err := ParseScenarioJSON([]byte(c.json)) + if err == nil || !strings.Contains(err.Error(), c.wantErr) { + t.Fatalf("err=%v want containing %q", err, c.wantErr) + } + }) + } +} + +func TestParseScenarioJSONValid(t *testing.T) { + spec, err := ParseScenarioJSON([]byte(`{ + "name": "power-watch", + "timeout_sec": 60, + "jobs": [ + {"name": "ipmi", "type": "sampler", "interval_sec": 2, "cmd": ["ipmitool", "sensor"]}, + {"name": "load", "type": "command", "cmd": ["dcgmi", "diag", "-r", "nvbandwidth", "-i", "{{gpus}}"], "gpu_indices": [0,1,2]} + ] + }`)) + if err != nil { + t.Fatalf("ParseScenarioJSON error: %v", err) + } + if spec.Name != "power-watch" || spec.TimeoutSec != 60 || len(spec.Jobs) != 2 { + t.Fatalf("spec=%+v", spec) + } +} + +func TestSubstituteGPUIndices(t *testing.T) { + cmd := []string{"dcgmi", "diag", "-r", "nvbandwidth", "-i", "{{gpus}}"} + got := substituteGPUIndices(cmd, []int{0, 2, 4}) + want := []string{"dcgmi", "diag", "-r", "nvbandwidth", "-i", "0,2,4"} + if len(got) != len(want) { + t.Fatalf("got=%v want=%v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("got=%v want=%v", got, want) + } + } + // No GPUIndices: cmd passes through untouched (no {{gpus}} to replace). + same := substituteGPUIndices(cmd, nil) + if same[5] != "{{gpus}}" { + t.Fatalf("expected token left alone when no indices given, got %v", same) + } +} + +func TestSanitizeScenarioName(t *testing.T) { + cases := map[string]string{ + "power-watch": "power-watch", + "Power Watch!!": "power-watch--", + "": "scenario", + "already_ok-123": "already_ok-123", + } + for in, want := range cases { + if got := sanitizeScenarioName(in); got != want { + t.Fatalf("sanitizeScenarioName(%q)=%q want %q", in, got, want) + } + } +} + +func TestRunScenarioRunsCommandsAndSamplersConcurrently(t *testing.T) { + old := satExecCommand + var samplerCalls int32 + satExecCommand = func(name string, args ...string) *exec.Cmd { + if name == "sample-cmd" { + atomic.AddInt32(&samplerCalls, 1) + return exec.Command("printf", "sampled\n") + } + return exec.Command(name, args...) + } + t.Cleanup(func() { satExecCommand = old }) + + dir := t.TempDir() + spec := ScenarioSpec{ + Name: "test-scenario", + TimeoutSec: 10, + Jobs: []ScenarioJob{ + {Name: "watch", Type: "sampler", Cmd: []string{"sample-cmd"}, IntervalSec: 1}, + {Name: "load", Type: "command", Cmd: []string{"printf", "loaded\n"}}, + }, + } + + s := &System{} + runDir, err := s.RunScenario(context.Background(), dir, spec, nil) + if err != nil { + t.Fatalf("RunScenario error: %v", err) + } + + if _, err := os.Stat(filepath.Join(runDir, "load.log")); err != nil { + t.Fatalf("load.log missing: %v", err) + } + loadOut, err := os.ReadFile(filepath.Join(runDir, "load.log")) + if err != nil || strings.TrimSpace(string(loadOut)) != "loaded" { + t.Fatalf("load.log=%q err=%v", loadOut, err) + } + + watchOut, err := os.ReadFile(filepath.Join(runDir, "watch.log")) + if err != nil { + t.Fatalf("watch.log missing: %v", err) + } + if !strings.Contains(string(watchOut), "sampled") { + t.Fatalf("watch.log=%q want it to contain a sample", watchOut) + } + if atomic.LoadInt32(&samplerCalls) < 1 { + t.Fatalf("sampler command was never invoked") + } + + summary, err := os.ReadFile(filepath.Join(runDir, "summary.txt")) + if err != nil { + t.Fatalf("summary.txt missing: %v", err) + } + if !strings.Contains(string(summary), "load_status=OK") { + t.Fatalf("summary=%q want load_status=OK", summary) + } +} + +func TestRunScenarioStopsSamplersWhenCommandsFinish(t *testing.T) { + old := satExecCommand + var samplerCalls int32 + satExecCommand = func(name string, args ...string) *exec.Cmd { + atomic.AddInt32(&samplerCalls, 1) + return exec.Command("true") + } + t.Cleanup(func() { satExecCommand = old }) + + dir := t.TempDir() + spec := ScenarioSpec{ + Name: "fast-scenario", + Jobs: []ScenarioJob{ + // Interval much longer than the scenario itself takes to run — + // if the sampler weren't stopped promptly when the command + // finishes, this test would need to wait out the interval. + {Name: "watch", Type: "sampler", Cmd: []string{"sample-cmd"}, IntervalSec: 3600}, + {Name: "load", Type: "command", Cmd: []string{"true"}}, + }, + } + + s := &System{} + start := time.Now() + if _, err := s.RunScenario(context.Background(), dir, spec, nil); err != nil { + t.Fatalf("RunScenario error: %v", err) + } + if elapsed := time.Since(start); elapsed > 5*time.Second { + t.Fatalf("RunScenario took %s — sampler wasn't stopped promptly", elapsed) + } + // Exactly one sample: the immediate baseline one, no second tick. + if got := atomic.LoadInt32(&samplerCalls); got != 1 { + t.Fatalf("samplerCalls=%d want 1 (baseline only)", got) + } +} + +func TestRunScenarioRecordsFailedCommandInSummary(t *testing.T) { + old := satExecCommand + satExecCommand = func(name string, args ...string) *exec.Cmd { + return exec.Command(name, args...) + } + t.Cleanup(func() { satExecCommand = old }) + + dir := t.TempDir() + spec := ScenarioSpec{ + Name: "failing-scenario", + Jobs: []ScenarioJob{ + {Name: "will-fail", Type: "command", Cmd: []string{"false"}}, + }, + } + + s := &System{} + runDir, err := s.RunScenario(context.Background(), dir, spec, nil) + if err != nil { + t.Fatalf("RunScenario error: %v", err) + } + summary, err := os.ReadFile(filepath.Join(runDir, "summary.txt")) + if err != nil { + t.Fatalf("summary.txt missing: %v", err) + } + if !strings.Contains(string(summary), "will-fail_status=FAILED") { + t.Fatalf("summary=%q want will-fail_status=FAILED", summary) + } +} diff --git a/audit/internal/webui/api.go b/audit/internal/webui/api.go index d072a49..1a789e0 100644 --- a/audit/internal/webui/api.go +++ b/audit/internal/webui/api.go @@ -1865,9 +1865,17 @@ func (h *handler) handleAPIComponentDetail(w http.ResponseWriter, r *http.Reques records = matchedRecords(all, exact, prefixes) } + fromInventory := false + if len(records) == 0 { + if fallback := inventoryFallbackRecords(compType, h.opts); len(fallback) > 0 { + records = fallback + fromInventory = true + } + } + w.Header().Set("Content-Type", "text/html; charset=utf-8") w.Header().Set("Cache-Control", "no-store") - fmt.Fprint(w, renderComponentDetail(title, records)) + fmt.Fprint(w, renderComponentDetail(title, records, fromInventory)) } func (h *handler) rollbackPendingNetworkChange() error { diff --git a/audit/internal/webui/page_topo.go b/audit/internal/webui/page_topo.go index cda9e35..8fa4f58 100644 --- a/audit/internal/webui/page_topo.go +++ b/audit/internal/webui/page_topo.go @@ -11,6 +11,7 @@ import ( "strconv" "strings" + "bee/audit/internal/app" "bee/audit/internal/schema" ) @@ -1221,3 +1222,126 @@ func errNoteSuffix(hasError bool) string { } return "" } + +// --------------------------------------------------------------------------- +// Inventory fallback for the component-detail modal +// +// handleAPIComponentDetail normally sources records from app.ComponentStatusDB, +// which only gains entries once something has actually written a status +// observation (SAT run, watchdog tick, ...). On a freshly booted host that +// hasn't run SAT yet, StatusDB can be entirely empty for a component type even +// though the /topo card for it already shows "N OK" — that card reads +// schema.HardwareComponentStatus.Status straight from the audit snapshot. +// inventoryFallbackRecords bridges that gap by building synthetic records +// from the same snapshot/classifiers the topology card uses, so the two +// views never disagree about how many devices exist or their status. +// --------------------------------------------------------------------------- + +// topoSeverityStatus renders classifyTopoSeverity's rank back into the status +// string vocabulary renderComponentDetail/chipLetterClass expect ("OK", +// "Warning", "Critical", "Unknown") — kept in lockstep with classifyTopoSeverity +// so a device the topo card counts as "OK" is never shown here as "Unknown". +func topoSeverityStatus(status *string) string { + switch classifyTopoSeverity(status) { + case 3: + return "Critical" + case 2: + return "Warning" + case 1: + return "OK" + default: + return "Unknown" + } +} + +// pcieDeviceKind classifies a PCIe device the same way renderTopoMainDiagram +// does, returning "" for devices that aren't GPU/NIC/RAID. +func pcieDeviceKind(dev schema.HardwarePCIeDevice) string { + switch { + case dev.DeviceClass != nil && isGPUDeviceClass(*dev.DeviceClass): + return "gpu" + case isNICDeviceClassDev(dev): + return "nic" + case dev.DeviceClass != nil && isRAIDControllerClass(*dev.DeviceClass): + return "raid" + default: + return "" + } +} + +// pcieDeviceKey builds a stable, human-readable component key for a PCIe +// device: ":" when a slot/BDF is known, else ":". +func pcieDeviceKey(kind string, index int, dev schema.HardwarePCIeDevice) string { + bdf := "" + if dev.Slot != nil { + bdf = normalizeTopoBDF(*dev.Slot) + } else if dev.BDF != nil { + bdf = normalizeTopoBDF(*dev.BDF) + } + if bdf != "" { + return kind + ":" + bdf + } + return fmt.Sprintf("%s:%d", kind, index) +} + +// inventoryFallbackRecords builds ComponentStatusRecord entries straight from +// the audit inventory (bee-audit.json) for the given component type, used +// when ComponentStatusDB has no matching records yet. Records carry only +// ComponentKey/Status — no LastCheckedAt/History — so renderComponentDetail +// renders them without a "checked at" timestamp or sparkline. +func inventoryFallbackRecords(compType string, opts HandlerOptions) []app.ComponentStatusRecord { + data, err := loadSnapshot(opts.AuditPath) + if err != nil { + return nil + } + var ingest schema.HardwareIngestRequest + if err := json.Unmarshal(data, &ingest); err != nil { + return nil + } + hw := ingest.Hardware + + var records []app.ComponentStatusRecord + switch compType { + case "cpu": + for i, cpu := range hw.CPUs { + key := fmt.Sprintf("cpu:%d", i) + if cpu.Socket != nil { + key = fmt.Sprintf("cpu:socket%d", *cpu.Socket) + } + records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(cpu.Status)}) + } + case "memory": + for i, m := range hw.Memory { + key := fmt.Sprintf("memory:%d", i) + if m.Slot != nil && strings.TrimSpace(*m.Slot) != "" { + key = "memory:" + strings.TrimSpace(*m.Slot) + } + records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(m.Status)}) + } + case "storage": + for i, s := range hw.Storage { + key := fmt.Sprintf("storage:%d", i) + if s.Slot != nil && strings.TrimSpace(*s.Slot) != "" { + key = "storage:" + strings.TrimSpace(*s.Slot) + } + records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(s.Status)}) + } + case "psu": + for i, p := range hw.PowerSupplies { + key := fmt.Sprintf("psu:%d", i) + if p.Slot != nil && strings.TrimSpace(*p.Slot) != "" { + key = "psu:" + strings.TrimSpace(*p.Slot) + } + records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(p.Status)}) + } + case "gpu", "nic", "raid": + for i, dev := range hw.PCIeDevices { + if pcieDeviceKind(dev) != compType { + continue + } + key := pcieDeviceKey(compType, i, dev) + records = append(records, app.ComponentStatusRecord{ComponentKey: key, Status: topoSeverityStatus(dev.Status)}) + } + } + return records +} diff --git a/audit/internal/webui/page_topo_test.go b/audit/internal/webui/page_topo_test.go index b4c1a62..fe16246 100644 --- a/audit/internal/webui/page_topo_test.go +++ b/audit/internal/webui/page_topo_test.go @@ -576,6 +576,64 @@ GPU 1: NVIDIA H100 80GB HBM3 (UUID: GPU-603fe750-0516-9db5-86ec-ea61af3fce35) } } +// TestComponentDetailFallsBackToInventoryWhenStatusDBEmpty covers the bug where +// a /topo card shows "3 OK" (from schema.HardwareComponentStatus.Status in the +// audit snapshot) but clicking it opens a modal saying "No status data recorded +// yet" (because ComponentStatusDB has no pcie:gpu:* entries — nothing has run a +// SAT test on this boot yet). The modal must show the same 3 devices/status the +// card does, not an empty state. +func TestComponentDetailFallsBackToInventoryWhenStatusDBEmpty(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "audit.json") + + okStatus := "OK" + warnStatus := "Warning" + deviceClass := "VideoController" + var gpus []schema.HardwarePCIeDevice + for i, st := range []*string{&okStatus, &okStatus, &warnStatus} { + slot := "0000:c" + strconv.Itoa(i) + ":00.0" + gpus = append(gpus, schema.HardwarePCIeDevice{ + HardwareComponentStatus: schema.HardwareComponentStatus{Status: st}, + DeviceClass: &deviceClass, + Slot: &slot, + }) + } + + ingest := schema.HardwareIngestRequest{ + Hardware: schema.HardwareSnapshot{PCIeDevices: gpus}, + } + data, err := json.Marshal(ingest) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0644); err != nil { + t.Fatal(err) + } + + // No HandlerOptions.App / StatusDB set — matches a host where nothing has + // written to ComponentStatusDB yet. + handler := NewHandler(HandlerOptions{AuditPath: path}) + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/api/components/gpu", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("status=%d body=%s", rec.Code, rec.Body.String()) + } + body := rec.Body.String() + + if strings.Contains(body, "No status data recorded yet") { + t.Fatalf("modal should not show empty state when inventory has GPUs: %s", body) + } + if strings.Count(body, "chip-ok") != 2 { + t.Fatalf("expected 2 OK chips from inventory fallback: %s", body) + } + if strings.Count(body, "chip-warn") != 1 { + t.Fatalf("expected 1 Warning chip from inventory fallback: %s", body) + } + if !strings.Contains(body, "No SAT-test history yet") { + t.Fatalf("expected fallback marker text: %s", body) + } +} + func TestParseTopoNVLinkErrors(t *testing.T) { input := `GPU 0: NVIDIA H100 80GB HBM3 (UUID: GPU-a59f6931-c099-8fba-a0b3-08469d86f140) Link 0: Replay Errors: 0 diff --git a/audit/internal/webui/pages.go b/audit/internal/webui/pages.go index 800fafc..aa9f113 100644 --- a/audit/internal/webui/pages.go +++ b/audit/internal/webui/pages.go @@ -1160,7 +1160,10 @@ func renderSparkline(history []app.ComponentStatusEntry) string { // renderComponentDetail renders a modal content fragment for one component type. // Called by handleAPIComponentDetail and displayed inside #component-detail-dialog. -func renderComponentDetail(title string, records []app.ComponentStatusRecord) string { +// fromInventory marks that records were synthesized from the audit inventory +// snapshot (no ComponentStatusDB history yet) rather than real SAT/watchdog +// observations — see inventoryFallbackRecords. +func renderComponentDetail(title string, records []app.ComponentStatusRecord, fromInventory bool) string { var b strings.Builder fmt.Fprintf(&b, `
`) fmt.Fprintf(&b, `
`) @@ -1174,6 +1177,10 @@ func renderComponentDetail(title string, records []app.ComponentStatusRecord) st return b.String() } + if fromInventory { + b.WriteString(`

No SAT-test history yet — showing latest inventory snapshot.

`) + } + sort.Slice(records, func(i, j int) bool { return records[i].ComponentKey < records[j].ComponentKey }) diff --git a/scenarios/README.md b/scenarios/README.md new file mode 100644 index 0000000..eab1248 --- /dev/null +++ b/scenarios/README.md @@ -0,0 +1,41 @@ +# bee test scenarios + +A scenario is a plain JSON file describing an ad-hoc diagnostic run: which +commands to execute (sequentially or in parallel) and what to sample in the +background while they run — without hardcoding a new test into bee's own +code. See `audit/internal/platform/scenario.go` (`ParseScenarioJSON`, +`ScenarioSpec`) for the full field reference. + +## Running one + +``` +bee run +``` + +or, for a scenario file dropped under `scenarios/` on a mounted removable +drive (e.g. the same USB stick already plugged in for blackbox — useful on +an air-gapped host with no other way to get a file onto it): + +``` +bee run # looks for scenarios/.json on any mounted removable media +``` + +`bee scenario run ` is the same command under a longer name. + +## Files checked in here + +- `nvbandwidth-all-gpu-power-watch.json` — the scenario that reproduced the + CG480-S6053 reboot: full `nvbandwidth` across all GPUs at once (the + per-GPU-socket passes alone never reproduced it), with `ipmitool sensor` + and `nvidia-smi` power/temp sampled every 2s in the background so a crash + mid-run still leaves telemetry to check for a power-delivery correlation. + **`gpu_indices` is host-specific** — update it to match the GPU indices + `nvidia-smi -L` actually reports on the box under test before running. + +## Adding more + +Not every scenario needs to be checked in here. For a one-off test on a +specific host (especially air-gapped), it's simpler to write the JSON file +directly onto the blackbox USB stick under `scenarios/.json` and run +`bee run ` — no code change, no rebuild. Check a scenario in here only +when it's worth keeping around as a reusable/named test across hosts. diff --git a/scenarios/nvbandwidth-all-gpu-power-watch.json b/scenarios/nvbandwidth-all-gpu-power-watch.json new file mode 100644 index 0000000..0633307 --- /dev/null +++ b/scenarios/nvbandwidth-all-gpu-power-watch.json @@ -0,0 +1,24 @@ +{ + "name": "nvbandwidth-all-gpu-power-watch", + "timeout_sec": 1800, + "jobs": [ + { + "name": "ipmi-sensors", + "type": "sampler", + "interval_sec": 2, + "cmd": ["ipmitool", "sensor"] + }, + { + "name": "gpu-power", + "type": "sampler", + "interval_sec": 2, + "cmd": ["nvidia-smi", "--query-gpu=index,power.draw,temperature.gpu,clocks.sm", "--format=csv"] + }, + { + "name": "nvbandwidth-all", + "type": "command", + "cmd": ["dcgmi", "diag", "-r", "nvbandwidth", "-i", "{{gpus}}"], + "gpu_indices": [0, 1, 2, 3, 4, 5] + } + ] +}