refactor: modularize audit and harden build validation

audit/internal/platform/sat.go

@@ -1,10 +1,8 @@
 package platform
 
 import (
-	"archive/tar"
 	"bufio"
 	"bytes"
-	"compress/gzip"
 	"context"
 	"errors"
 	"fmt"
@@ -194,774 +192,6 @@ func streamExecOutput(cmd *exec.Cmd, logFunc func(string), livePath string) ([]b
 }
 
 // NvidiaGPU holds basic GPU info from nvidia-smi.
-type NvidiaGPU struct {
-	Index    int    `json:"index"`
-	Name     string `json:"name"`
-	MemoryMB int    `json:"memory_mb"`
-}
-
-type NvidiaGPUStatus struct {
-	Index        int    `json:"index"`
-	Name         string `json:"name"`
-	BDF          string `json:"bdf,omitempty"`
-	Serial       string `json:"serial,omitempty"`
-	Status       string `json:"status"`
-	RawLine      string `json:"raw_line,omitempty"`
-	NeedsReset   bool   `json:"needs_reset"`
-	ParseFailure bool   `json:"parse_failure,omitempty"`
-}
-
-type nvidiaGPUHealth struct {
-	Index        int
-	Name         string
-	NeedsReset   bool
-	RawLine      string
-	ParseFailure bool
-}
-
-type nvidiaGPUStatusFile struct {
-	Index      int
-	Name       string
-	RunStatus  string
-	Reason     string
-	Health     string
-	HealthRaw  string
-	Observed   bool
-	Selected   bool
-	FailingJob string
-}
-
-// AMDGPUInfo holds basic info about an AMD GPU from rocm-smi.
-type AMDGPUInfo struct {
-	Index int    `json:"index"`
-	Name  string `json:"name"`
-}
-
-// DetectGPUVendor returns "nvidia" if /dev/nvidia0 exists, "amd" if /dev/kfd exists, or "" otherwise.
-func (s *System) DetectGPUVendor() string {
-	if _, err := os.Stat("/dev/nvidia0"); err == nil {
-		return "nvidia"
-	}
-	if _, err := os.Stat("/dev/kfd"); err == nil {
-		return "amd"
-	}
-	if raw, err := exec.Command("lspci", "-nn").Output(); err == nil {
-		// Only match AMD GPU device classes [0300]=VGA, [0302]=3D controller, [0380]=Display.
-		// AMD CPUs also appear in lspci as "Advanced Micro Devices" (Root Complex, IOMMU, etc.)
-		// so matching vendor alone causes false positives on AMD CPU servers without GPUs.
-		for _, line := range strings.Split(strings.ToLower(string(raw)), "\n") {
-			if !strings.Contains(line, "advanced micro devices") && !strings.Contains(line, "amd/ati") {
-				continue
-			}
-			if strings.Contains(line, "[0300]") || strings.Contains(line, "[0302]") || strings.Contains(line, "[0380]") {
-				return "amd"
-			}
-		}
-	}
-	return ""
-}
-
-// ListAMDGPUs returns AMD GPUs visible to rocm-smi.
-func (s *System) ListAMDGPUs() ([]AMDGPUInfo, error) {
-	out, err := runROCmSMI("--showproductname", "--csv")
-	if err != nil {
-		return nil, fmt.Errorf("rocm-smi: %w", err)
-	}
-	var gpus []AMDGPUInfo
-	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
-		line = strings.TrimSpace(line)
-		if line == "" || strings.HasPrefix(strings.ToLower(line), "device") {
-			continue
-		}
-		parts := strings.SplitN(line, ",", 2)
-		name := ""
-		if len(parts) >= 2 {
-			name = strings.TrimSpace(parts[1])
-		}
-		idx := len(gpus)
-		gpus = append(gpus, AMDGPUInfo{Index: idx, Name: name})
-	}
-	return gpus, nil
-}
-
-// RunAMDAcceptancePack runs an AMD GPU diagnostic pack using rocm-smi.
-func (s *System) RunAMDAcceptancePack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
-	return runAcceptancePackCtx(ctx, baseDir, "gpu-amd", []satJob{
-		{name: "01-rocm-smi.log", cmd: []string{"rocm-smi"}},
-		{name: "02-rocm-smi-showallinfo.log", cmd: []string{"rocm-smi", "--showallinfo"}},
-		{name: "03-dmidecode-baseboard.log", cmd: []string{"dmidecode", "-t", "baseboard"}},
-		{name: "04-dmidecode-system.log", cmd: []string{"dmidecode", "-t", "system"}},
-	}, logFunc)
-}
-
-// RunAMDMemIntegrityPack runs the official RVS MEM module as a validate-style memory integrity test.
-func (s *System) RunAMDMemIntegrityPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
-	if err := ensureAMDRuntimeReady(); err != nil {
-		return "", err
-	}
-	cfgFile := "/tmp/bee-amd-mem.conf"
-	cfg := `actions:
-- name: mem_integrity
-  device: all
-  module: mem
-  parallel: true
-  duration: 60000
-  copy_matrix: false
-  target_stress: 90
-  matrix_size: 8640
-`
-	_ = os.WriteFile(cfgFile, []byte(cfg), 0644)
-	return runAcceptancePackCtx(ctx, baseDir, "gpu-amd-mem", []satJob{
-		{name: "01-rocm-smi.log", cmd: []string{"rocm-smi"}},
-		{name: "02-rvs-mem.log", cmd: []string{"rvs", "-c", cfgFile}},
-		{name: "03-rocm-smi-after.log", cmd: []string{"rocm-smi", "--showtemp", "--showpower", "--showmemuse", "--csv"}},
-	}, logFunc)
-}
-
-// RunAMDMemBandwidthPack runs AMD's memory/interconnect bandwidth-oriented tools.
-func (s *System) RunAMDMemBandwidthPack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
-	if err := ensureAMDRuntimeReady(); err != nil {
-		return "", err
-	}
-	cfgFile := "/tmp/bee-amd-babel.conf"
-	cfg := `actions:
-- name: babel_mem_bw
-  device: all
-  module: babel
-  parallel: true
-  copy_matrix: true
-  target_stress: 90
-  matrix_size: 134217728
-`
-	_ = os.WriteFile(cfgFile, []byte(cfg), 0644)
-	return runAcceptancePackCtx(ctx, baseDir, "gpu-amd-bandwidth", []satJob{
-		{name: "01-rocm-smi.log", cmd: []string{"rocm-smi"}},
-		{name: "02-rocm-bandwidth-test.log", cmd: []string{"rocm-bandwidth-test"}},
-		{name: "03-rvs-babel.log", cmd: []string{"rvs", "-c", cfgFile}},
-		{name: "04-rocm-smi-after.log", cmd: []string{"rocm-smi", "--showtemp", "--showpower", "--showmemuse", "--csv"}},
-	}, logFunc)
-}
-
-// RunAMDStressPack runs an AMD GPU burn-in pack.
-// Missing tools are reported as UNSUPPORTED, consistent with the existing SAT pattern.
-func (s *System) RunAMDStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
-	seconds := durationSec
-	if seconds <= 0 {
-		seconds = envInt("BEE_AMD_STRESS_SECONDS", 300)
-	}
-	if err := ensureAMDRuntimeReady(); err != nil {
-		return "", err
-	}
-	// Enable copy_matrix so the same GST run drives VRAM traffic in addition to compute.
-	rvsCfg := amdStressRVSConfig(seconds)
-	cfgFile := "/tmp/bee-amd-gst.conf"
-	_ = os.WriteFile(cfgFile, []byte(rvsCfg), 0644)
-
-	return runAcceptancePackCtx(ctx, baseDir, "gpu-amd-stress", amdStressJobs(seconds, cfgFile), logFunc)
-}
-
-func amdStressRVSConfig(seconds int) string {
-	return fmt.Sprintf(`actions:
-- name: gst_stress
-  device: all
-  module: gst
-  parallel: true
-  duration: %d
-  copy_matrix: false
-  target_stress: 90
-  matrix_size_a: 8640
-  matrix_size_b: 8640
-  matrix_size_c: 8640
-`, seconds*1000)
-}
-
-func amdStressJobs(seconds int, cfgFile string) []satJob {
-	return []satJob{
-		{name: "01-rocm-smi.log", cmd: []string{"rocm-smi"}},
-		{name: "02-rocm-bandwidth-test.log", cmd: []string{"rocm-bandwidth-test"}},
-		{name: fmt.Sprintf("03-rvs-gst-%ds.log", seconds), cmd: []string{"rvs", "-c", cfgFile}},
-		{name: fmt.Sprintf("04-rocm-smi-after.log"), cmd: []string{"rocm-smi", "--showtemp", "--showpower", "--csv"}},
-	}
-}
-
-// ListNvidiaGPUs returns GPUs visible to nvidia-smi.
-func (s *System) ListNvidiaGPUs() ([]NvidiaGPU, error) {
-	out, err := exec.Command("nvidia-smi",
-		"--query-gpu=index,name,memory.total",
-		"--format=csv,noheader,nounits").Output()
-	if err != nil {
-		return nil, fmt.Errorf("nvidia-smi: %w", err)
-	}
-	var gpus []NvidiaGPU
-	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
-		line = strings.TrimSpace(line)
-		if line == "" {
-			continue
-		}
-		parts := strings.SplitN(line, ", ", 3)
-		if len(parts) != 3 {
-			continue
-		}
-		idx, err := strconv.Atoi(strings.TrimSpace(parts[0]))
-		if err != nil {
-			continue
-		}
-		memMB, _ := strconv.Atoi(strings.TrimSpace(parts[2]))
-		gpus = append(gpus, NvidiaGPU{
-			Index:    idx,
-			Name:     strings.TrimSpace(parts[1]),
-			MemoryMB: memMB,
-		})
-	}
-	sort.Slice(gpus, func(i, j int) bool {
-		return gpus[i].Index < gpus[j].Index
-	})
-	return gpus, nil
-}
-
-func (s *System) ListNvidiaGPUStatuses() ([]NvidiaGPUStatus, error) {
-	out, err := satExecCommand(
-		"nvidia-smi",
-		"--query-gpu=index,name,pci.bus_id,serial,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total",
-		"--format=csv,noheader,nounits",
-	).Output()
-	if err != nil {
-		return nil, fmt.Errorf("nvidia-smi: %w", err)
-	}
-	var gpus []NvidiaGPUStatus
-	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
-		line = strings.TrimSpace(line)
-		if line == "" {
-			continue
-		}
-		parts := strings.Split(line, ",")
-		if len(parts) < 4 {
-			gpus = append(gpus, NvidiaGPUStatus{RawLine: line, Status: "UNKNOWN", ParseFailure: true})
-			continue
-		}
-		idx, err := strconv.Atoi(strings.TrimSpace(parts[0]))
-		if err != nil {
-			gpus = append(gpus, NvidiaGPUStatus{RawLine: line, Status: "UNKNOWN", ParseFailure: true})
-			continue
-		}
-		upper := strings.ToUpper(line)
-		needsReset := strings.Contains(upper, "GPU REQUIRES RESET")
-		status := "OK"
-		if needsReset {
-			status = "RESET_REQUIRED"
-		}
-		gpus = append(gpus, NvidiaGPUStatus{
-			Index:      idx,
-			Name:       strings.TrimSpace(parts[1]),
-			BDF:        normalizeNvidiaBusID(strings.TrimSpace(parts[2])),
-			Serial:     strings.TrimSpace(parts[3]),
-			Status:     status,
-			RawLine:    line,
-			NeedsReset: needsReset,
-		})
-	}
-	sort.Slice(gpus, func(i, j int) bool { return gpus[i].Index < gpus[j].Index })
-	return gpus, nil
-}
-
-func normalizeNvidiaBusID(v string) string {
-	v = strings.TrimSpace(strings.ToLower(v))
-	parts := strings.Split(v, ":")
-	if len(parts) == 3 && len(parts[0]) > 4 {
-		parts[0] = parts[0][len(parts[0])-4:]
-		return strings.Join(parts, ":")
-	}
-	return v
-}
-
-func (s *System) ResetNvidiaGPU(index int) (string, error) {
-	return resetNvidiaGPU(index)
-}
-
-// RunNCCLTests runs nccl-tests all_reduce_perf across the selected NVIDIA GPUs.
-// Measures collective communication bandwidth over NVLink/PCIe.
-func (s *System) RunNCCLTests(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) {
-	selected, err := resolveDCGMGPUIndices(gpuIndices)
-	if err != nil {
-		return "", err
-	}
-	gpuCount := len(selected)
-	if gpuCount < 1 {
-		gpuCount = 1
-	}
-	return runAcceptancePackCtx(ctx, baseDir, "nccl-tests", withNvidiaPersistenceMode(
-		satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
-		satJob{name: "02-all-reduce-perf.log", cmd: []string{
-			"all_reduce_perf", "-b", "512M", "-e", "4G", "-f", "2",
-			"-g", strconv.Itoa(gpuCount), "--iters", "20",
-		}, env: nvidiaVisibleDevicesEnv(selected), syncBracket: true},
-	), logFunc)
-}
-
-func (s *System) RunNvidiaOfficialComputePack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, staggerSec int, logFunc func(string)) (string, error) {
-	selected, err := resolveDCGMGPUIndices(gpuIndices)
-	if err != nil {
-		return "", err
-	}
-	var (
-		profCmd []string
-		profEnv []string
-	)
-	if len(selected) > 1 {
-		// For multiple GPUs, always spawn one dcgmproftester process per GPU via
-		// bee-dcgmproftester-staggered (stagger=0 means all start simultaneously).
-		// A single dcgmproftester process without -i only loads GPU 0 regardless
-		// of CUDA_VISIBLE_DEVICES.
-		stagger := staggerSec
-		if stagger < 0 {
-			stagger = 0
-		}
-		profCmd = []string{
-			"bee-dcgmproftester-staggered",
-			"--seconds", strconv.Itoa(normalizeNvidiaBurnDuration(durationSec)),
-			"--stagger-seconds", strconv.Itoa(stagger),
-			"--devices", joinIndexList(selected),
-		}
-	} else {
-		profCmd, err = resolveDCGMProfTesterCommand("--no-dcgm-validation", "-t", "1004", "-d", strconv.Itoa(normalizeNvidiaBurnDuration(durationSec)))
-		if err != nil {
-			return "", err
-		}
-		profEnv = nvidiaVisibleDevicesEnv(selected)
-	}
-	return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-compute", withNvidiaPersistenceMode(
-		satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
-		satJob{name: "02-dcgmi-version.log", cmd: []string{"dcgmi", "-v"}},
-		satJob{
-			name:        "03-dcgmproftester.log",
-			cmd:         profCmd,
-			env:         profEnv,
-			collectGPU:  true,
-			gpuIndices:  selected,
-			syncBracket: true,
-		},
-		satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
-	), logFunc)
-}
-
-func (s *System) RunNvidiaTargetedPowerPack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) {
-	selected, err := resolveDCGMGPUIndices(gpuIndices)
-	if err != nil {
-		return "", err
-	}
-	// Kill any lingering nvvs/dcgmi processes from a previous interrupted run
-	// before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34).
-	if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil {
-		for _, p := range killed {
-			logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name))
-		}
-	}
-	return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-targeted-power", withNvidiaPersistenceMode(
-		satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
-		satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
-		satJob{
-			name:        "03-dcgmi-targeted-power.log",
-			cmd:         nvidiaDCGMNamedDiagCommand("targeted_power", normalizeNvidiaBurnDuration(durationSec), selected),
-			collectGPU:  true,
-			gpuIndices:  selected,
-			syncBracket: true,
-		},
-		satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
-	), logFunc)
-}
-
-func (s *System) RunNvidiaPulseTestPack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) {
-	selected, err := resolveDCGMGPUIndices(gpuIndices)
-	if err != nil {
-		return "", err
-	}
-	// Kill any lingering nvvs/dcgmi processes from a previous interrupted run
-	// before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34).
-	if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil {
-		for _, p := range killed {
-			logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name))
-		}
-	}
-	return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-pulse", withNvidiaPersistenceMode(
-		satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
-		satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
-		satJob{
-			name:        "03-dcgmi-pulse-test.log",
-			cmd:         nvidiaDCGMNamedDiagCommand("pulse_test", normalizeNvidiaBurnDuration(durationSec), selected),
-			collectGPU:  true,
-			gpuIndices:  selected,
-			syncBracket: true,
-		},
-		satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
-	), logFunc)
-}
-
-func (s *System) RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpuIndices []int, logFunc func(string)) (string, error) {
-	selected, err := resolveDCGMGPUIndices(gpuIndices)
-	if err != nil {
-		return "", err
-	}
-	// Kill any lingering nvvs/dcgmi processes from a previous interrupted run
-	// before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34).
-	if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil {
-		for _, p := range killed {
-			logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name))
-		}
-	}
-	jobs := []satJob{
-		{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
-		{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
-	}
-
-	// On a system with GPUs on more than one CPU socket, run each socket's
-	// GPUs through nvbandwidth in isolation before the all-GPU pass. Without
-	// NVLink, cross-socket peer-to-peer traffic is a distinct fault domain
-	// from same-socket traffic; if the single-socket passes log clean and
-	// only the all-GPU pass doesn't complete, that isolates the cross-socket
-	// path as the trigger instead of leaving it conflated with a general
-	// GPU/PCIe fault. Systems with one socket (or no resolvable NUMA
-	// affinity) get a single group back and keep the original one-pass shape.
-	step := 3
-	socketGroups := gpuBandwidthSocketGroups(selected, logFunc)
-	if len(socketGroups) <= 1 {
-		jobs = append(jobs, satJob{
-			name:        fmt.Sprintf("%02d-dcgmi-nvbandwidth.log", step),
-			cmd:         nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, selected),
-			collectGPU:  true,
-			gpuIndices:  selected,
-			syncBracket: true,
-		})
-		step++
-	} else {
-		for i, group := range socketGroups {
-			jobs = append(jobs, satJob{
-				name:        fmt.Sprintf("%02d-dcgmi-nvbandwidth-socket%d.log", step, i),
-				cmd:         nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, group),
-				collectGPU:  true,
-				gpuIndices:  group,
-				syncBracket: true,
-			})
-			step++
-		}
-		jobs = append(jobs, satJob{
-			name:        fmt.Sprintf("%02d-dcgmi-nvbandwidth-all.log", step),
-			cmd:         nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, selected),
-			collectGPU:  true,
-			gpuIndices:  selected,
-			syncBracket: true,
-		})
-		step++
-	}
-	jobs = append(jobs, satJob{
-		name: fmt.Sprintf("%02d-nvidia-smi-after.log", step),
-		cmd:  []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"},
-	})
-
-	return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-bandwidth", withNvidiaPersistenceMode(jobs...), logFunc)
-}
-
-func (s *System) RunNvidiaAcceptancePack(baseDir string, logFunc func(string)) (string, error) {
-	return runAcceptancePackCtx(context.Background(), baseDir, "gpu-nvidia", nvidiaSATJobs(), logFunc)
-}
-
-// RunNvidiaAcceptancePackWithOptions runs the NVIDIA diagnostics via DCGM.
-// diagLevel: 1=quick, 2=medium, 3=targeted stress, 4=extended stress.
-// gpuIndices: specific GPU indices to test (empty = all GPUs).
-// ctx cancellation kills the running job.
-func (s *System) RunNvidiaAcceptancePackWithOptions(ctx context.Context, baseDir string, diagLevel int, gpuIndices []int, logFunc func(string)) (string, error) {
-	resolvedGPUIndices, err := resolveDCGMGPUIndices(gpuIndices)
-	if err != nil {
-		return "", err
-	}
-	return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia", nvidiaDCGMJobs(diagLevel, resolvedGPUIndices), logFunc)
-}
-
-func (s *System) RunNvidiaTargetedStressValidatePack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) {
-	selected, err := resolveDCGMGPUIndices(gpuIndices)
-	if err != nil {
-		return "", err
-	}
-	// Kill any lingering nvvs/dcgmi processes from a previous interrupted run
-	// before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34).
-	if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil {
-		for _, p := range killed {
-			logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name))
-		}
-	}
-	return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-targeted-stress", withNvidiaPersistenceMode(
-		satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
-		satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
-		satJob{
-			name:        "03-dcgmi-targeted-stress.log",
-			cmd:         nvidiaDCGMNamedDiagCommand("targeted_stress", normalizeNvidiaBurnDuration(durationSec), selected),
-			collectGPU:  true,
-			gpuIndices:  selected,
-			syncBracket: true,
-		},
-		satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
-	), logFunc)
-}
-
-func resolveDCGMGPUIndices(gpuIndices []int) ([]int, error) {
-	if len(gpuIndices) > 0 {
-		return dedupeSortedIndices(gpuIndices), nil
-	}
-	all, err := listNvidiaGPUIndices()
-	if err != nil {
-		return nil, err
-	}
-	if len(all) == 0 {
-		return nil, fmt.Errorf("nvidia-smi found no NVIDIA GPUs")
-	}
-	return all, nil
-}
-
-func memoryStressSizeArg() string {
-	if mb := envInt("BEE_VM_STRESS_SIZE_MB", 0); mb > 0 {
-		return fmt.Sprintf("%dM", mb)
-	}
-	availBytes := satFreeMemBytes()
-	if availBytes <= 0 {
-		return "80%"
-	}
-	availMB := availBytes / (1024 * 1024)
-	targetMB := (availMB * 2) / 3
-	if targetMB >= 256 {
-		targetMB = (targetMB / 256) * 256
-	}
-	if targetMB <= 0 {
-		return "80%"
-	}
-	return fmt.Sprintf("%dM", targetMB)
-}
-
-func (s *System) RunMemoryAcceptancePack(ctx context.Context, baseDir string, sizeMB, passes int, logFunc func(string)) (string, error) {
-	if sizeMB <= 0 {
-		sizeMB = 256
-	}
-	if passes <= 0 {
-		passes = 1
-	}
-	// Keep Validate Memory bounded to a quick diagnostic window. The timeout is
-	// intentionally conservative enough for healthy systems while avoiding the
-	// prior 30-80 minute hangs caused by memtester spinning on a bad subtest.
-	timeoutSec := sizeMB*passes*20/100 + 60
-	if timeoutSec < 180 {
-		timeoutSec = 180
-	}
-	if timeoutSec > 900 {
-		timeoutSec = 900
-	}
-	return runAcceptancePackCtx(ctx, baseDir, "memory", []satJob{
-		{name: "01-free-before.log", cmd: []string{"free", "-h"}},
-		{name: "02-memtester.log", cmd: []string{"timeout", fmt.Sprintf("%d", timeoutSec), "memtester", fmt.Sprintf("%dM", sizeMB), fmt.Sprintf("%d", passes)}, syncBracket: true},
-		{name: "03-free-after.log", cmd: []string{"free", "-h"}},
-	}, logFunc)
-}
-
-func (s *System) RunMemoryStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
-	seconds := durationSec
-	if seconds <= 0 {
-		seconds = envInt("BEE_VM_STRESS_SECONDS", 300)
-	}
-	// Base the default on current MemAvailable and keep headroom for the OS and
-	// concurrent stressors so mixed burn runs do not trip the OOM killer.
-	sizeArg := memoryStressSizeArg()
-	return runAcceptancePackCtx(ctx, baseDir, "memory-stress", []satJob{
-		{name: "01-free-before.log", cmd: []string{"free", "-h"}},
-		{name: "02-stress-ng-vm.log", cmd: []string{
-			"stress-ng", "--vm", "1",
-			"--vm-bytes", sizeArg,
-			"--vm-method", "all",
-			"--timeout", fmt.Sprintf("%d", seconds),
-			"--metrics-brief",
-		}, syncBracket: true},
-		{name: "03-free-after.log", cmd: []string{"free", "-h"}},
-	}, logFunc)
-}
-
-func (s *System) RunSATStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
-	seconds := durationSec
-	if seconds <= 0 {
-		seconds = envInt("BEE_SAT_STRESS_SECONDS", 300)
-	}
-	cmd := []string{"stressapptest", "-s", fmt.Sprintf("%d", seconds), "-W", "--cc_test"}
-	if mb := envInt("BEE_SAT_STRESS_MB", 0); mb > 0 {
-		cmd = append(cmd, "-M", fmt.Sprintf("%d", mb))
-	}
-	return runAcceptancePackCtx(ctx, baseDir, "sat-stress", []satJob{
-		{name: "01-free-before.log", cmd: []string{"free", "-h"}},
-		{name: "02-stressapptest.log", cmd: cmd},
-		{name: "03-free-after.log", cmd: []string{"free", "-h"}},
-	}, logFunc)
-}
-
-// cpuThermalThrottleSysDir is the sysfs root the throttle-check scripts glob
-// under. Overridden in tests so they can point at a fake directory tree
-// instead of the real /sys.
-var cpuThermalThrottleSysDir = "/sys/devices/system/cpu"
-
-// cpuThrottleSumScript is the shell fragment both before/after scripts use to
-// sum the kernel's cumulative-since-boot thermal throttle counters across
-// every CPU.
-func cpuThrottleSumScript() string {
-	return fmt.Sprintf(`
-sum=0
-for f in %[1]s/cpu*/thermal_throttle/core_throttle_count %[1]s/cpu*/thermal_throttle/package_throttle_count; do
-  [ -f "$f" ] || continue
-  v=$(cat "$f" 2>/dev/null)
-  case "$v" in ''|*[!0-9]*) continue ;; esac
-  sum=$((sum + v))
-done
-`, cpuThermalThrottleSysDir)
-}
-
-// cpuThrottleBeforeScript snapshots the throttle counter sum into a file in
-// {{run_dir}} so cpuThrottleCheckScript can later diff before/after despite
-// each satJob running as an independent process.
-func cpuThrottleBeforeScript() string {
-	return cpuThrottleSumScript() + `echo "$sum" | tee {{run_dir}}/.cpu-throttle-before` + "\n"
-}
-
-// cpuThrottleCheckScript compares the after-run throttle counter sum against
-// the snapshot cpuThrottleBeforeScript took, and fails (non-zero exit) if it
-// increased — i.e. the CPU actually hit thermal throttling during this
-// specific run, not just at some earlier point this boot. classifySATResult
-// maps a failed job here to SAT status FAILED, which ApplySATResultToDB
-// records as component status "Warning" for cpu:all — without this, the
-// "cpu" SAT pack only checks stress-ng's exit code, which is 0 whether or
-// not the CPU throttled while running it.
-func cpuThrottleCheckScript() string {
-	return `before=$(cat {{run_dir}}/.cpu-throttle-before 2>/dev/null)
-case "$before" in ''|*[!0-9]*) before=0 ;; esac
-` + cpuThrottleSumScript() + `after=$sum
-echo "throttle_count_before=$before"
-echo "throttle_count_after=$after"
-if [ "$after" -gt "$before" ]; then
-  echo "THROTTLE DETECTED: CPU package/core hit thermal throttling during this stress-ng run ($before -> $after)"
-  exit 1
-fi
-echo "no new thermal throttling detected during this run"
-`
-}
-
-func cpuSATJobs(durationSec int) []satJob {
-	return []satJob{
-		{name: "01-lscpu.log", cmd: []string{"lscpu"}},
-		{name: "02-sensors-before.log", cmd: []string{"sensors"}},
-		{name: "02-thermal-throttle-before.log", cmd: []string{"sh", "-c", cpuThrottleBeforeScript()}, informational: true},
-		{name: "03-stress-ng.log", cmd: []string{"stress-ng", "--cpu", "0", "--cpu-method", "all", "--timeout", fmt.Sprintf("%d", durationSec)}, syncBracket: true},
-		{name: "04-sensors-after.log", cmd: []string{"sensors"}},
-		{name: "05-thermal-throttle-check.log", cmd: []string{"sh", "-c", cpuThrottleCheckScript()}},
-	}
-}
-
-func (s *System) RunCPUAcceptancePack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
-	if durationSec <= 0 {
-		durationSec = 60
-	}
-	return runAcceptancePackCtx(ctx, baseDir, "cpu", cpuSATJobs(durationSec), logFunc)
-}
-
-func (s *System) RunStorageAcceptancePack(ctx context.Context, baseDir string, extended bool, logFunc func(string)) (string, error) {
-	if baseDir == "" {
-		baseDir = "/var/log/bee-sat"
-	}
-	ts := time.Now().UTC().Format("20060102-150405")
-	runDir := filepath.Join(baseDir, "storage-"+ts)
-	if err := os.MkdirAll(runDir, 0755); err != nil {
-		return "", err
-	}
-	verboseLog := filepath.Join(runDir, "verbose.log")
-
-	devices, err := listStorageDevices()
-	if err != nil {
-		return "", err
-	}
-	sort.Strings(devices)
-
-	var summary strings.Builder
-	stats := satStats{}
-	fmt.Fprintf(&summary, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339))
-	if len(devices) == 0 {
-		fmt.Fprintln(&summary, "devices=0")
-		stats.Unsupported++
-	} else {
-		fmt.Fprintf(&summary, "devices=%d\n", len(devices))
-	}
-
-	for index, devPath := range devices {
-		if ctx.Err() != nil {
-			break
-		}
-		prefix := fmt.Sprintf("%02d-%s", index+1, filepath.Base(devPath))
-		commands := storageSATCommands(devPath, extended)
-		deviceOutputs := make(map[string][]byte, len(commands))
-		for cmdIndex, job := range commands {
-			if ctx.Err() != nil {
-				break
-			}
-			name := fmt.Sprintf("%s-%02d-%s.log", prefix, cmdIndex+1, job.name)
-			livePath := filepath.Join(runDir, name)
-			runSyncBracketHook(job, "before", logFunc)
-			out, err := runSATCommandCtx(ctx, verboseLog, job.name, job.cmd, nil, logFunc, livePath)
-			deviceOutputs[job.name] = out
-			if writeErr := os.WriteFile(livePath, out, 0644); writeErr != nil {
-				return "", writeErr
-			}
-			if satJobBoundaryHook != nil {
-				satJobBoundaryHook(name)
-			}
-			// smartctl -t short only launches the self-test on the drive firmware and
-			// returns immediately ("Testing has begun"); unlike `nvme device-self-test
-			// --wait`, smartctl has no blocking mode, so we must poll the drive
-			// ourselves until the self-test actually finishes. Hold the "after" sync
-			// until that poll completes — the self-test itself, not just its launch,
-			// is the load worth having durable evidence of.
-			deferSyncBracketAfter := job.name == "smartctl-self-test-short" && err == nil
-			if !deferSyncBracketAfter {
-				runSyncBracketHook(job, "after", logFunc)
-			}
-			status, rc := classifySATResult(job.name, out, err)
-			// A zero smartctl exit status only proves the command ran.  If the
-			// drive did not return its overall-health verdict, it must not turn
-			// the storage SAT green.
-			if job.name == "smartctl-health" && status == "OK" && !hasSMARTOverallHealth(out) {
-				status = "UNSUPPORTED"
-			}
-			stats.Add(status)
-			key := filepath.Base(devPath) + "_" + strings.ReplaceAll(job.name, "-", "_")
-			fmt.Fprintf(&summary, "%s_rc=%d\n", key, rc)
-			fmt.Fprintf(&summary, "%s_status=%s\n", key, status)
-
-			if deferSyncBracketAfter {
-				statusName := "smartctl-self-test-status"
-				statusOut := waitForSmartctlSelfTest(ctx, verboseLog, devPath, logFunc)
-				deviceOutputs[statusName] = statusOut
-				statusFile := fmt.Sprintf("%s-%02d-%s.log", prefix, cmdIndex+2, statusName)
-				if writeErr := os.WriteFile(filepath.Join(runDir, statusFile), statusOut, 0644); writeErr != nil {
-					return "", writeErr
-				}
-				runSyncBracketHook(job, "after", logFunc)
-				sStatus, sRC := classifySATResult(statusName, statusOut, nil)
-				stats.Add(sStatus)
-				sKey := filepath.Base(devPath) + "_" + strings.ReplaceAll(statusName, "-", "_")
-				fmt.Fprintf(&summary, "%s_rc=%d\n", sKey, sRC)
-				fmt.Fprintf(&summary, "%s_status=%s\n", sKey, sStatus)
-			}
-		}
-		reportText := GenerateDiskReportText(index+1, devPath, deviceOutputs, time.Now().UTC())
-		reportName := "disk-" + prefix + "-report.txt"
-		_ = os.WriteFile(filepath.Join(runDir, reportName), []byte(reportText), 0644)
-	}
-
-	writeSATStats(&summary, stats)
-	if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary.String()), 0644); err != nil {
-		return "", err
-	}
-	return runDir, nil
-}
-
 type satJob struct {
 	name       string
 	cmd        []string
@@ -1574,41 +804,6 @@ func hasSMARTOverallHealth(out []byte) bool {
 	return len(m) > 1 && strings.TrimSpace(m[1]) != ""
 }
 
-func runSATCommand(verboseLog, name string, cmd []string, logFunc func(string)) ([]byte, error) {
-	start := time.Now().UTC()
-	resolvedCmd, err := resolveSATCommand(cmd)
-	appendSATVerboseLog(verboseLog,
-		fmt.Sprintf("[%s] start %s", start.Format(time.RFC3339), name),
-		"cmd: "+strings.Join(resolvedCmd, " "),
-	)
-	if logFunc != nil {
-		logFunc(fmt.Sprintf("=== %s ===", name))
-	}
-	if err != nil {
-		appendSATVerboseLog(verboseLog,
-			fmt.Sprintf("[%s] finish %s", time.Now().UTC().Format(time.RFC3339), name),
-			"rc: 1",
-			fmt.Sprintf("duration_ms: %d", time.Since(start).Milliseconds()),
-			"",
-		)
-		return []byte(err.Error() + "\n"), err
-	}
-
-	out, err := streamExecOutput(satExecCommand(resolvedCmd[0], resolvedCmd[1:]...), logFunc, "")
-
-	rc := 0
-	if err != nil {
-		rc = 1
-	}
-	appendSATVerboseLog(verboseLog,
-		fmt.Sprintf("[%s] finish %s", time.Now().UTC().Format(time.RFC3339), name),
-		fmt.Sprintf("rc: %d", rc),
-		fmt.Sprintf("duration_ms: %d", time.Since(start).Milliseconds()),
-		"",
-	)
-	return out, err
-}
-
 func runROCmSMI(args ...string) ([]byte, error) {
 	cmd, err := resolveROCmSMICommand(args...)
 	if err != nil {
@@ -1802,46 +997,3 @@ func envInt(name string, fallback int) int {
 	}
 	return value
 }
-
-func createTarGz(dst, srcDir string) error {
-	file, err := os.Create(dst)
-	if err != nil {
-		return err
-	}
-	defer file.Close()
-
-	gz := gzip.NewWriter(file)
-	defer gz.Close()
-
-	tw := tar.NewWriter(gz)
-	defer tw.Close()
-
-	base := filepath.Dir(srcDir)
-	return filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-		if info.IsDir() {
-			return nil
-		}
-		header, err := tar.FileInfoHeader(info, "")
-		if err != nil {
-			return err
-		}
-		rel, err := filepath.Rel(base, path)
-		if err != nil {
-			return err
-		}
-		header.Name = rel
-		if err := tw.WriteHeader(header); err != nil {
-			return err
-		}
-		file, err := os.Open(path)
-		if err != nil {
-			return err
-		}
-		defer file.Close()
-		_, err = io.Copy(tw, file)
-		return err
-	})
-}