Investigating the CG480-S6053 reboot needed a way to run an ad-hoc load
(nvbandwidth across a specific GPU set) while sampling IPMI/nvidia-smi
telemetry in the background — without hardcoding a one-off test into the
SAT pack code for a single investigation.
- audit/internal/platform/scenario.go: ScenarioSpec/ScenarioJob (JSON,
no new dependency) + System.RunScenario. "command" jobs run sequential
or parallel (per-job "parallel" flag); "sampler" jobs run concurrently
in the background on their own interval until every command job
finishes or the scenario's timeout elapses. "{{gpus}}" in a command's
cmd is substituted from that job's gpu_indices. Command jobs are wired
through the same satJobBoundaryHook/satSyncBracketHook seams the SAT
job runner uses, so a scenario run gets the same durability treatment
(evidence that a risky command started/finished reaches blackbox before
a possible crash, not just whatever streamed to the RAM-backed export
dir).
- export.go: ReadScenarioFromRemovableMedia mounts each removable target
looking for scenarios/<name>.json — an air-gapped engineer can author a
scenario elsewhere, drop it under scenarios/ on the same USB stick
already plugged in for blackbox, and run it with no network path onto
the host.
- cmd/bee: new `bee run <file.json|name>` (bare name = looked up on
removable media); `bee scenario run <arg>` kept as a longer alias.
- scenarios/nvbandwidth-all-gpu-power-watch.json: the scenario that
reproduced the actual reboot (full nvbandwidth across all GPUs, which
crashed, vs. clean per-socket passes), with IPMI sensor + GPU power/temp
sampling for a power-delivery correlation check.
Also: webui/page_topo.go — the /topo page's component-status-detail modal
(GET /api/component-detail/{type}) showed "No status data recorded yet"
for any component type ComponentStatusDB has no history for yet (e.g. GPU
before a SAT run this boot), even though the topology card for the same
component already showed "N OK" from the audit inventory snapshot.
inventoryFallbackRecords now synthesizes records from that same inventory
snapshot when StatusDB is empty, using the same device classifiers
(isGPUDeviceClass etc.) and severity mapping (classifyTopoSeverity) the
topology card itself uses, so the two views never disagree.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
206 lines
5.9 KiB
Go
206 lines
5.9 KiB
Go
package platform
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
)
|
|
|
|
var exportExecCommand = exec.Command
|
|
|
|
func formatMountTargetError(target RemovableTarget, raw string, err error) error {
|
|
msg := strings.TrimSpace(raw)
|
|
fstype := strings.ToLower(strings.TrimSpace(target.FSType))
|
|
if fstype == "exfat" && strings.Contains(strings.ToLower(msg), "unknown filesystem type 'exfat'") {
|
|
return fmt.Errorf("mount %s: exFAT support is missing in this ISO build: %w", target.Device, err)
|
|
}
|
|
if msg == "" {
|
|
return err
|
|
}
|
|
return fmt.Errorf("%s: %w", msg, err)
|
|
}
|
|
|
|
func removableTargetReadOnly(fields map[string]string) bool {
|
|
if fields["RO"] == "1" {
|
|
return true
|
|
}
|
|
switch strings.ToLower(strings.TrimSpace(fields["FSTYPE"])) {
|
|
case "iso9660", "squashfs":
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func ensureWritableMountpoint(mountpoint string) error {
|
|
probe, err := os.CreateTemp(mountpoint, ".bee-write-test-*")
|
|
if err != nil {
|
|
return fmt.Errorf("target filesystem is not writable: %w", err)
|
|
}
|
|
name := probe.Name()
|
|
if closeErr := probe.Close(); closeErr != nil {
|
|
_ = os.Remove(name)
|
|
return closeErr
|
|
}
|
|
if err := os.Remove(name); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *System) ListRemovableTargets() ([]RemovableTarget, error) {
|
|
raw, err := exportExecCommand("lsblk", "-P", "-o", "NAME,TYPE,PKNAME,RM,RO,FSTYPE,MOUNTPOINT,SIZE,LABEL,MODEL").Output()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
var out []RemovableTarget
|
|
for _, line := range strings.Split(strings.TrimSpace(string(raw)), "\n") {
|
|
if strings.TrimSpace(line) == "" {
|
|
continue
|
|
}
|
|
fields := parseLSBLKPairs(line)
|
|
deviceType := fields["TYPE"]
|
|
if deviceType == "rom" || deviceType == "loop" {
|
|
continue
|
|
}
|
|
|
|
removable := fields["RM"] == "1"
|
|
if !removable {
|
|
if parent := fields["PKNAME"]; parent != "" {
|
|
if data, err := os.ReadFile(filepath.Join("/sys/class/block", parent, "removable")); err == nil {
|
|
removable = strings.TrimSpace(string(data)) == "1"
|
|
}
|
|
}
|
|
}
|
|
if !removable || fields["FSTYPE"] == "" || removableTargetReadOnly(fields) {
|
|
continue
|
|
}
|
|
|
|
out = append(out, RemovableTarget{
|
|
Device: "/dev/" + fields["NAME"],
|
|
FSType: fields["FSTYPE"],
|
|
Size: fields["SIZE"],
|
|
Label: fields["LABEL"],
|
|
Model: fields["MODEL"],
|
|
Mountpoint: fields["MOUNTPOINT"],
|
|
})
|
|
}
|
|
|
|
sort.Slice(out, func(i, j int) bool { return out[i].Device < out[j].Device })
|
|
return out, nil
|
|
}
|
|
|
|
func (s *System) ExportFileToTarget(src string, target RemovableTarget) (dst string, retErr error) {
|
|
if src == "" || target.Device == "" {
|
|
return "", fmt.Errorf("source and target are required")
|
|
}
|
|
if _, err := os.Stat(src); err != nil {
|
|
return "", err
|
|
}
|
|
|
|
mountpoint := strings.TrimSpace(target.Mountpoint)
|
|
mountedHere := false
|
|
mounted := mountpoint != ""
|
|
if mountpoint == "" {
|
|
mountpoint = filepath.Join("/tmp", "bee-export-"+filepath.Base(target.Device))
|
|
if err := os.MkdirAll(mountpoint, 0755); err != nil {
|
|
return "", err
|
|
}
|
|
if raw, err := exportExecCommand("mount", target.Device, mountpoint).CombinedOutput(); err != nil {
|
|
_ = os.Remove(mountpoint)
|
|
return "", formatMountTargetError(target, string(raw), err)
|
|
}
|
|
mountedHere = true
|
|
mounted = true
|
|
}
|
|
defer func() {
|
|
if !mounted {
|
|
return
|
|
}
|
|
_ = exportExecCommand("sync").Run()
|
|
if raw, err := exportExecCommand("umount", mountpoint).CombinedOutput(); err != nil && retErr == nil {
|
|
msg := strings.TrimSpace(string(raw))
|
|
if msg == "" {
|
|
retErr = err
|
|
} else {
|
|
retErr = fmt.Errorf("%s: %w", msg, err)
|
|
}
|
|
}
|
|
if mountedHere {
|
|
_ = os.Remove(mountpoint)
|
|
}
|
|
}()
|
|
|
|
if err := ensureWritableMountpoint(mountpoint); err != nil {
|
|
return "", err
|
|
}
|
|
|
|
filename := filepath.Base(src)
|
|
dst = filepath.Join(mountpoint, filename)
|
|
data, err := os.ReadFile(src)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if err := os.WriteFile(dst, data, 0644); err != nil {
|
|
return "", err
|
|
}
|
|
|
|
return dst, nil
|
|
}
|
|
|
|
// mountRemovableTargetReadOnly mounts target for reading if it isn't
|
|
// already mounted (reusing its existing mountpoint otherwise), returning
|
|
// whether this call did the mounting so the caller knows whether to
|
|
// unmount afterward.
|
|
func mountRemovableTargetReadOnly(target RemovableTarget) (mountpoint string, mountedHere bool, err error) {
|
|
if mp := strings.TrimSpace(target.Mountpoint); mp != "" {
|
|
return mp, false, nil
|
|
}
|
|
mountpoint = filepath.Join("/tmp", "bee-scenario-"+filepath.Base(target.Device))
|
|
if err := os.MkdirAll(mountpoint, 0755); err != nil {
|
|
return "", false, err
|
|
}
|
|
if raw, err := exportExecCommand("mount", target.Device, mountpoint).CombinedOutput(); err != nil {
|
|
_ = os.Remove(mountpoint)
|
|
return "", false, formatMountTargetError(target, string(raw), err)
|
|
}
|
|
return mountpoint, true, nil
|
|
}
|
|
|
|
// ReadScenarioFromRemovableMedia mounts each removable target in turn
|
|
// (unmounting again afterward if it mounted it itself), looking for
|
|
// scenarios/<name>.json, and returns the contents of the first one found.
|
|
// Lets an air-gapped engineer author a scenario JSON file on another
|
|
// machine, drop it under scenarios/ on the same flash drive already
|
|
// plugged in for blackbox, and run it on the host with no network path
|
|
// required — see RunScenario/ParseScenarioJSON.
|
|
func (s *System) ReadScenarioFromRemovableMedia(name string) ([]byte, error) {
|
|
name = strings.TrimSpace(name)
|
|
if name == "" {
|
|
return nil, fmt.Errorf("scenario name is required")
|
|
}
|
|
targets, err := s.ListRemovableTargets()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, target := range targets {
|
|
mountpoint, mountedHere, mountErr := mountRemovableTargetReadOnly(target)
|
|
if mountErr != nil {
|
|
continue
|
|
}
|
|
data, readErr := os.ReadFile(filepath.Join(mountpoint, "scenarios", name+".json"))
|
|
if mountedHere {
|
|
_, _ = exportExecCommand("umount", mountpoint).CombinedOutput()
|
|
_ = os.Remove(mountpoint)
|
|
}
|
|
if readErr == nil {
|
|
return data, nil
|
|
}
|
|
}
|
|
return nil, fmt.Errorf("scenarios/%s.json not found on any removable media", name)
|
|
}
|