Mikhail ChusavitinandClaude Sonnet 5
20cd317c87
Investigating the CG480-S6053 reboot needed a way to run an ad-hoc load
(nvbandwidth across a specific GPU set) while sampling IPMI/nvidia-smi
telemetry in the background — without hardcoding a one-off test into the
SAT pack code for a single investigation.
- audit/internal/platform/scenario.go: ScenarioSpec/ScenarioJob (JSON,
no new dependency) + System.RunScenario. "command" jobs run sequential
or parallel (per-job "parallel" flag); "sampler" jobs run concurrently
in the background on their own interval until every command job
finishes or the scenario's timeout elapses. "{{gpus}}" in a command's
cmd is substituted from that job's gpu_indices. Command jobs are wired
through the same satJobBoundaryHook/satSyncBracketHook seams the SAT
job runner uses, so a scenario run gets the same durability treatment
(evidence that a risky command started/finished reaches blackbox before
a possible crash, not just whatever streamed to the RAM-backed export
dir).
- export.go: ReadScenarioFromRemovableMedia mounts each removable target
looking for scenarios/<name>.json — an air-gapped engineer can author a
scenario elsewhere, drop it under scenarios/ on the same USB stick
already plugged in for blackbox, and run it with no network path onto
the host.
- cmd/bee: new `bee run <file.json|name>` (bare name = looked up on
removable media); `bee scenario run <arg>` kept as a longer alias.
- scenarios/nvbandwidth-all-gpu-power-watch.json: the scenario that
reproduced the actual reboot (full nvbandwidth across all GPUs, which
crashed, vs. clean per-socket passes), with IPMI sensor + GPU power/temp
sampling for a power-delivery correlation check.
Also: webui/page_topo.go — the /topo page's component-status-detail modal
(GET /api/component-detail/{type}) showed "No status data recorded yet"
for any component type ComponentStatusDB has no history for yet (e.g. GPU
before a SAT run this boot), even though the topology card for the same
component already showed "N OK" from the audit inventory snapshot.
inventoryFallbackRecords now synthesizes records from that same inventory
snapshot when StatusDB is empty, using the same device classifiers
(isGPUDeviceClass etc.) and severity mapping (classifyTopoSeverity) the
topology card itself uses, so the two views never disagree.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
280 lines
7.0 KiB
Go
280 lines
7.0 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestRunRootHelp(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
var stdout, stderr bytes.Buffer
|
|
rc := run([]string{"help"}, &stdout, &stderr)
|
|
if rc != 0 {
|
|
t.Fatalf("rc=%d want 0", rc)
|
|
}
|
|
if !strings.Contains(stdout.String(), "bee commands:") {
|
|
t.Fatalf("stdout missing root usage:\n%s", stdout.String())
|
|
}
|
|
}
|
|
|
|
func TestRunNoArgsPrintsUsage(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
var stdout, stderr bytes.Buffer
|
|
rc := run(nil, &stdout, &stderr)
|
|
if rc != 2 {
|
|
t.Fatalf("rc=%d want 2", rc)
|
|
}
|
|
if !strings.Contains(stderr.String(), "bee commands:") {
|
|
t.Fatalf("stderr missing root usage:\n%s", stderr.String())
|
|
}
|
|
}
|
|
|
|
func TestRunUnknownCommand(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
var stdout, stderr bytes.Buffer
|
|
rc := run([]string{"wat"}, &stdout, &stderr)
|
|
if rc != 2 {
|
|
t.Fatalf("rc=%d want 2", rc)
|
|
}
|
|
if !strings.Contains(stderr.String(), `unknown command "wat"`) {
|
|
t.Fatalf("stderr missing unknown command message:\n%s", stderr.String())
|
|
}
|
|
}
|
|
|
|
func TestRunVersion(t *testing.T) {
|
|
old := Version
|
|
Version = "test-version"
|
|
t.Cleanup(func() { Version = old })
|
|
|
|
var stdout, stderr bytes.Buffer
|
|
rc := run([]string{"version"}, &stdout, &stderr)
|
|
if rc != 0 {
|
|
t.Fatalf("rc=%d want 0", rc)
|
|
}
|
|
if strings.TrimSpace(stdout.String()) != "test-version" {
|
|
t.Fatalf("stdout=%q want %q", strings.TrimSpace(stdout.String()), "test-version")
|
|
}
|
|
}
|
|
|
|
func TestBuildLabelUsesVersionAsIs(t *testing.T) {
|
|
old := Version
|
|
Version = "1.2.3"
|
|
t.Cleanup(func() { Version = old })
|
|
|
|
if got := buildLabel(); got != "1.2.3" {
|
|
t.Fatalf("buildLabel=%q want %q", got, "1.2.3")
|
|
}
|
|
}
|
|
|
|
func TestRunExportRequiresTarget(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
var stdout, stderr bytes.Buffer
|
|
rc := run([]string{"export"}, &stdout, &stderr)
|
|
if rc != 2 {
|
|
t.Fatalf("rc=%d want 2", rc)
|
|
}
|
|
if !strings.Contains(stderr.String(), "--target is required") {
|
|
t.Fatalf("stderr missing target error:\n%s", stderr.String())
|
|
}
|
|
if !strings.Contains(stderr.String(), "usage: bee export --target <device>") {
|
|
t.Fatalf("stderr missing export usage:\n%s", stderr.String())
|
|
}
|
|
}
|
|
|
|
func TestRunSATUsage(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
var stdout, stderr bytes.Buffer
|
|
rc := run([]string{"sat"}, &stdout, &stderr)
|
|
if rc != 2 {
|
|
t.Fatalf("rc=%d want 2", rc)
|
|
}
|
|
if !strings.Contains(stderr.String(), "usage: bee sat nvidia|memory|storage") {
|
|
t.Fatalf("stderr missing sat usage:\n%s", stderr.String())
|
|
}
|
|
}
|
|
|
|
func TestRunUsage(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
var stdout, stderr bytes.Buffer
|
|
rc := run([]string{"run"}, &stdout, &stderr)
|
|
if rc != 2 {
|
|
t.Fatalf("rc=%d want 2", rc)
|
|
}
|
|
if !strings.Contains(stderr.String(), "usage: bee run <file.json | scenario-name>") {
|
|
t.Fatalf("stderr missing run usage:\n%s", stderr.String())
|
|
}
|
|
}
|
|
|
|
func TestRunHelp(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
var stdout, stderr bytes.Buffer
|
|
rc := run([]string{"run", "--help"}, &stdout, &stderr)
|
|
if rc != 0 {
|
|
t.Fatalf("rc=%d want 0", rc)
|
|
}
|
|
if !strings.Contains(stdout.String(), "usage: bee run <file.json | scenario-name>") {
|
|
t.Fatalf("stdout missing run usage:\n%s", stdout.String())
|
|
}
|
|
}
|
|
|
|
func TestRunMissingScenarioFile(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
var stdout, stderr bytes.Buffer
|
|
rc := run([]string{"run", "/nonexistent/scenario.json"}, &stdout, &stderr)
|
|
if rc != 1 {
|
|
t.Fatalf("rc=%d want 1", rc)
|
|
}
|
|
if !strings.Contains(stderr.String(), "bee run:") {
|
|
t.Fatalf("stderr missing error prefix:\n%s", stderr.String())
|
|
}
|
|
}
|
|
|
|
func TestRunScenarioAliasAcceptsRunVerb(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
var stdout, stderr bytes.Buffer
|
|
rc := run([]string{"scenario", "run", "/nonexistent/scenario.json"}, &stdout, &stderr)
|
|
if rc != 1 {
|
|
t.Fatalf("rc=%d want 1", rc)
|
|
}
|
|
if !strings.Contains(stderr.String(), "bee run:") {
|
|
t.Fatalf("stderr missing error prefix:\n%s", stderr.String())
|
|
}
|
|
}
|
|
|
|
func TestRunPreflightRejectsExtraArgs(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
var stdout, stderr bytes.Buffer
|
|
rc := run([]string{"preflight", "extra"}, &stdout, &stderr)
|
|
if rc != 2 {
|
|
t.Fatalf("rc=%d want 2", rc)
|
|
}
|
|
if !strings.Contains(stderr.String(), "usage: bee preflight") {
|
|
t.Fatalf("stderr missing preflight usage:\n%s", stderr.String())
|
|
}
|
|
}
|
|
|
|
func TestRunSupportBundleRejectsExtraArgs(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
var stdout, stderr bytes.Buffer
|
|
rc := run([]string{"support-bundle", "extra"}, &stdout, &stderr)
|
|
if rc != 2 {
|
|
t.Fatalf("rc=%d want 2", rc)
|
|
}
|
|
if !strings.Contains(stderr.String(), "usage: bee support-bundle") {
|
|
t.Fatalf("stderr missing support-bundle usage:\n%s", stderr.String())
|
|
}
|
|
}
|
|
|
|
func TestRunHelpForSubcommand(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
var stdout, stderr bytes.Buffer
|
|
rc := run([]string{"help", "export"}, &stdout, &stderr)
|
|
if rc != 0 {
|
|
t.Fatalf("rc=%d want 0", rc)
|
|
}
|
|
if !strings.Contains(stdout.String(), "usage: bee export --target <device>") {
|
|
t.Fatalf("stdout missing export help:\n%s", stdout.String())
|
|
}
|
|
}
|
|
|
|
func TestRunHelpUnknownSubcommand(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
var stdout, stderr bytes.Buffer
|
|
rc := run([]string{"help", "wat"}, &stdout, &stderr)
|
|
if rc != 2 {
|
|
t.Fatalf("rc=%d want 2", rc)
|
|
}
|
|
if !strings.Contains(stderr.String(), `bee help: unknown command "wat"`) {
|
|
t.Fatalf("stderr missing help error:\n%s", stderr.String())
|
|
}
|
|
}
|
|
|
|
func TestRunSATUnknownTarget(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
var stdout, stderr bytes.Buffer
|
|
rc := run([]string{"sat", "amd"}, &stdout, &stderr)
|
|
if rc != 2 {
|
|
t.Fatalf("rc=%d want 2", rc)
|
|
}
|
|
if !strings.Contains(stderr.String(), `unknown target "amd"`) {
|
|
t.Fatalf("stderr missing sat target error:\n%s", stderr.String())
|
|
}
|
|
}
|
|
|
|
func TestRunSATHelp(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
var stdout, stderr bytes.Buffer
|
|
rc := run([]string{"sat", "--help"}, &stdout, &stderr)
|
|
if rc != 0 {
|
|
t.Fatalf("rc=%d want 0", rc)
|
|
}
|
|
if !strings.Contains(stdout.String(), "usage: bee sat nvidia|memory|storage|cpu") {
|
|
t.Fatalf("stdout missing sat help:\n%s", stdout.String())
|
|
}
|
|
}
|
|
|
|
func TestRunSATRejectsExtraArgs(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
var stdout, stderr bytes.Buffer
|
|
rc := run([]string{"sat", "memory", "extra"}, &stdout, &stderr)
|
|
if rc != 2 {
|
|
t.Fatalf("rc=%d want 2", rc)
|
|
}
|
|
if !strings.Contains(stderr.String(), "bee sat: unexpected arguments") {
|
|
t.Fatalf("stderr missing sat error:\n%s", stderr.String())
|
|
}
|
|
}
|
|
|
|
func TestRunAuditInvalidRuntime(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
var stdout, stderr bytes.Buffer
|
|
rc := run([]string{"audit", "--runtime", "bad"}, &stdout, &stderr)
|
|
if rc != 1 {
|
|
t.Fatalf("rc=%d want 1", rc)
|
|
}
|
|
}
|
|
|
|
func TestRunAuditRejectsExtraArgs(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
var stdout, stderr bytes.Buffer
|
|
rc := run([]string{"audit", "extra"}, &stdout, &stderr)
|
|
if rc != 2 {
|
|
t.Fatalf("rc=%d want 2", rc)
|
|
}
|
|
if !strings.Contains(stderr.String(), "usage: bee audit") {
|
|
t.Fatalf("stderr missing audit usage:\n%s", stderr.String())
|
|
}
|
|
}
|
|
|
|
func TestRunExportRejectsExtraArgs(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
var stdout, stderr bytes.Buffer
|
|
rc := run([]string{"export", "--target", "/dev/sdb1", "extra"}, &stdout, &stderr)
|
|
if rc != 2 {
|
|
t.Fatalf("rc=%d want 2", rc)
|
|
}
|
|
if !strings.Contains(stderr.String(), "usage: bee export --target <device>") {
|
|
t.Fatalf("stderr missing export usage:\n%s", stderr.String())
|
|
}
|
|
}
|