809 lines
24 KiB
Go
809 lines
24 KiB
Go
package platform
|
|
|
|
import (
|
|
"archive/tar"
|
|
"bufio"
|
|
"bytes"
|
|
"compress/gzip"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
var (
|
|
satExecCommand = exec.Command
|
|
satLookPath = exec.LookPath
|
|
satGlob = filepath.Glob
|
|
satStat = os.Stat
|
|
|
|
rocmSMIExecutableGlobs = []string{
|
|
"/opt/rocm/bin/rocm-smi",
|
|
"/opt/rocm-*/bin/rocm-smi",
|
|
}
|
|
rocmSMIScriptGlobs = []string{
|
|
"/opt/rocm/libexec/rocm_smi/rocm_smi.py",
|
|
"/opt/rocm-*/libexec/rocm_smi/rocm_smi.py",
|
|
}
|
|
)
|
|
|
|
// streamExecOutput runs cmd and streams each output line to logFunc (if non-nil).
|
|
// Returns combined stdout+stderr as a byte slice.
|
|
func streamExecOutput(cmd *exec.Cmd, logFunc func(string)) ([]byte, error) {
|
|
pr, pw := io.Pipe()
|
|
cmd.Stdout = pw
|
|
cmd.Stderr = pw
|
|
|
|
var buf bytes.Buffer
|
|
var wg sync.WaitGroup
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
scanner := bufio.NewScanner(pr)
|
|
for scanner.Scan() {
|
|
line := scanner.Text()
|
|
buf.WriteString(line + "\n")
|
|
if logFunc != nil {
|
|
logFunc(line)
|
|
}
|
|
}
|
|
}()
|
|
|
|
err := cmd.Start()
|
|
if err != nil {
|
|
_ = pw.Close()
|
|
wg.Wait()
|
|
return nil, err
|
|
}
|
|
waitErr := cmd.Wait()
|
|
_ = pw.Close()
|
|
wg.Wait()
|
|
return buf.Bytes(), waitErr
|
|
}
|
|
|
|
// NvidiaGPU holds basic GPU info from nvidia-smi.
|
|
type NvidiaGPU struct {
|
|
Index int
|
|
Name string
|
|
MemoryMB int
|
|
}
|
|
|
|
// AMDGPUInfo holds basic info about an AMD GPU from rocm-smi.
|
|
type AMDGPUInfo struct {
|
|
Index int
|
|
Name string
|
|
}
|
|
|
|
// DetectGPUVendor returns "nvidia" if /dev/nvidia0 exists, "amd" if /dev/kfd exists, or "" otherwise.
|
|
func (s *System) DetectGPUVendor() string {
|
|
if _, err := os.Stat("/dev/nvidia0"); err == nil {
|
|
return "nvidia"
|
|
}
|
|
if _, err := os.Stat("/dev/kfd"); err == nil {
|
|
return "amd"
|
|
}
|
|
if raw, err := exec.Command("lspci", "-nn").Output(); err == nil {
|
|
text := strings.ToLower(string(raw))
|
|
if strings.Contains(text, "advanced micro devices") || strings.Contains(text, "amd/ati") {
|
|
return "amd"
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// ListAMDGPUs returns AMD GPUs visible to rocm-smi.
|
|
func (s *System) ListAMDGPUs() ([]AMDGPUInfo, error) {
|
|
out, err := runROCmSMI("--showproductname", "--csv")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("rocm-smi: %w", err)
|
|
}
|
|
var gpus []AMDGPUInfo
|
|
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
|
line = strings.TrimSpace(line)
|
|
if line == "" || strings.HasPrefix(strings.ToLower(line), "device") {
|
|
continue
|
|
}
|
|
parts := strings.SplitN(line, ",", 2)
|
|
name := ""
|
|
if len(parts) >= 2 {
|
|
name = strings.TrimSpace(parts[1])
|
|
}
|
|
idx := len(gpus)
|
|
gpus = append(gpus, AMDGPUInfo{Index: idx, Name: name})
|
|
}
|
|
return gpus, nil
|
|
}
|
|
|
|
// RunAMDAcceptancePack runs an AMD GPU diagnostic pack using rocm-smi.
|
|
func (s *System) RunAMDAcceptancePack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
|
|
return runAcceptancePackCtx(ctx, baseDir, "gpu-amd", []satJob{
|
|
{name: "01-rocm-smi.log", cmd: []string{"rocm-smi"}},
|
|
{name: "02-rocm-smi-showallinfo.log", cmd: []string{"rocm-smi", "--showallinfo"}},
|
|
{name: "03-dmidecode-baseboard.log", cmd: []string{"dmidecode", "-t", "baseboard"}},
|
|
{name: "04-dmidecode-system.log", cmd: []string{"dmidecode", "-t", "system"}},
|
|
}, logFunc)
|
|
}
|
|
|
|
// RunAMDStressPack runs an AMD GPU burn-in pack.
|
|
// Missing tools are reported as UNSUPPORTED, consistent with the existing SAT pattern.
|
|
func (s *System) RunAMDStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
|
|
seconds := durationSec
|
|
if seconds <= 0 {
|
|
seconds = envInt("BEE_AMD_STRESS_SECONDS", 300)
|
|
}
|
|
if err := ensureAMDRuntimeReady(); err != nil {
|
|
return "", err
|
|
}
|
|
return runAcceptancePackCtx(ctx, baseDir, "gpu-amd-stress", []satJob{
|
|
{name: "01-rocm-smi.log", cmd: []string{"rocm-smi"}},
|
|
{name: "02-rocm-bandwidth-test.log", cmd: []string{"rocm-bandwidth-test"}},
|
|
{name: fmt.Sprintf("03-rocm-smi-monitor-%ds.log", seconds), cmd: []string{
|
|
"bash", "-lc",
|
|
fmt.Sprintf("end=$((SECONDS+%d)); while [ \"$SECONDS\" -lt \"$end\" ]; do rocm-smi --showtemp --showpower --csv; sleep 1; done", seconds),
|
|
}},
|
|
}, logFunc)
|
|
}
|
|
|
|
// ListNvidiaGPUs returns GPUs visible to nvidia-smi.
|
|
func (s *System) ListNvidiaGPUs() ([]NvidiaGPU, error) {
|
|
out, err := exec.Command("nvidia-smi",
|
|
"--query-gpu=index,name,memory.total",
|
|
"--format=csv,noheader,nounits").Output()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("nvidia-smi: %w", err)
|
|
}
|
|
var gpus []NvidiaGPU
|
|
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
|
line = strings.TrimSpace(line)
|
|
if line == "" {
|
|
continue
|
|
}
|
|
parts := strings.SplitN(line, ", ", 3)
|
|
if len(parts) != 3 {
|
|
continue
|
|
}
|
|
idx, err := strconv.Atoi(strings.TrimSpace(parts[0]))
|
|
if err != nil {
|
|
continue
|
|
}
|
|
memMB, _ := strconv.Atoi(strings.TrimSpace(parts[2]))
|
|
gpus = append(gpus, NvidiaGPU{
|
|
Index: idx,
|
|
Name: strings.TrimSpace(parts[1]),
|
|
MemoryMB: memMB,
|
|
})
|
|
}
|
|
return gpus, nil
|
|
}
|
|
|
|
// RunNCCLTests runs nccl-tests all_reduce_perf across all NVIDIA GPUs.
|
|
// Measures collective communication bandwidth over NVLink/PCIe.
|
|
func (s *System) RunNCCLTests(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
|
|
// detect GPU count
|
|
out, _ := exec.Command("nvidia-smi", "--query-gpu=index", "--format=csv,noheader").Output()
|
|
gpuCount := len(strings.Split(strings.TrimSpace(string(out)), "\n"))
|
|
if gpuCount < 1 {
|
|
gpuCount = 1
|
|
}
|
|
return runAcceptancePackCtx(ctx, baseDir, "nccl-tests", []satJob{
|
|
{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
|
|
{name: "02-all-reduce-perf.log", cmd: []string{
|
|
"all_reduce_perf", "-b", "512M", "-e", "4G", "-f", "2",
|
|
"-g", strconv.Itoa(gpuCount), "--iters", "20",
|
|
}},
|
|
}, logFunc)
|
|
}
|
|
|
|
func (s *System) RunNvidiaAcceptancePack(baseDir string, logFunc func(string)) (string, error) {
|
|
return runAcceptancePackCtx(context.Background(), baseDir, "gpu-nvidia", nvidiaSATJobs(), logFunc)
|
|
}
|
|
|
|
// RunNvidiaAcceptancePackWithOptions runs the NVIDIA diagnostics via DCGM.
|
|
// diagLevel: 1=quick, 2=medium, 3=targeted stress, 4=extended stress.
|
|
// gpuIndices: specific GPU indices to test (empty = all GPUs).
|
|
// ctx cancellation kills the running job.
|
|
func (s *System) RunNvidiaAcceptancePackWithOptions(ctx context.Context, baseDir string, diagLevel int, gpuIndices []int, logFunc func(string)) (string, error) {
|
|
return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia", nvidiaDCGMJobs(diagLevel, gpuIndices), logFunc)
|
|
}
|
|
|
|
func (s *System) RunMemoryAcceptancePack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
|
|
sizeMB := envInt("BEE_MEMTESTER_SIZE_MB", 128)
|
|
passes := envInt("BEE_MEMTESTER_PASSES", 1)
|
|
return runAcceptancePackCtx(ctx, baseDir, "memory", []satJob{
|
|
{name: "01-free-before.log", cmd: []string{"free", "-h"}},
|
|
{name: "02-memtester.log", cmd: []string{"memtester", fmt.Sprintf("%dM", sizeMB), fmt.Sprintf("%d", passes)}},
|
|
{name: "03-free-after.log", cmd: []string{"free", "-h"}},
|
|
}, logFunc)
|
|
}
|
|
|
|
func (s *System) RunMemoryStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
|
|
seconds := durationSec
|
|
if seconds <= 0 {
|
|
seconds = envInt("BEE_VM_STRESS_SECONDS", 300)
|
|
}
|
|
// Use 80% of RAM by default; override with BEE_VM_STRESS_SIZE_MB.
|
|
sizeArg := "80%"
|
|
if mb := envInt("BEE_VM_STRESS_SIZE_MB", 0); mb > 0 {
|
|
sizeArg = fmt.Sprintf("%dM", mb)
|
|
}
|
|
return runAcceptancePackCtx(ctx, baseDir, "memory-stress", []satJob{
|
|
{name: "01-free-before.log", cmd: []string{"free", "-h"}},
|
|
{name: "02-stress-ng-vm.log", cmd: []string{
|
|
"stress-ng", "--vm", "1",
|
|
"--vm-bytes", sizeArg,
|
|
"--vm-method", "all",
|
|
"--timeout", fmt.Sprintf("%d", seconds),
|
|
"--metrics-brief",
|
|
}},
|
|
{name: "03-free-after.log", cmd: []string{"free", "-h"}},
|
|
}, logFunc)
|
|
}
|
|
|
|
func (s *System) RunSATStressPack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
|
|
seconds := durationSec
|
|
if seconds <= 0 {
|
|
seconds = envInt("BEE_SAT_STRESS_SECONDS", 300)
|
|
}
|
|
cmd := []string{"stressapptest", "-s", fmt.Sprintf("%d", seconds), "-W", "--cc_test"}
|
|
if mb := envInt("BEE_SAT_STRESS_MB", 0); mb > 0 {
|
|
cmd = append(cmd, "-M", fmt.Sprintf("%d", mb))
|
|
}
|
|
return runAcceptancePackCtx(ctx, baseDir, "sat-stress", []satJob{
|
|
{name: "01-free-before.log", cmd: []string{"free", "-h"}},
|
|
{name: "02-stressapptest.log", cmd: cmd},
|
|
{name: "03-free-after.log", cmd: []string{"free", "-h"}},
|
|
}, logFunc)
|
|
}
|
|
|
|
func (s *System) RunCPUAcceptancePack(ctx context.Context, baseDir string, durationSec int, logFunc func(string)) (string, error) {
|
|
if durationSec <= 0 {
|
|
durationSec = 60
|
|
}
|
|
return runAcceptancePackCtx(ctx, baseDir, "cpu", []satJob{
|
|
{name: "01-lscpu.log", cmd: []string{"lscpu"}},
|
|
{name: "02-sensors-before.log", cmd: []string{"sensors"}},
|
|
{name: "03-stress-ng.log", cmd: []string{"stress-ng", "--cpu", "0", "--cpu-method", "all", "--timeout", fmt.Sprintf("%d", durationSec)}},
|
|
{name: "04-sensors-after.log", cmd: []string{"sensors"}},
|
|
}, logFunc)
|
|
}
|
|
|
|
func (s *System) RunStorageAcceptancePack(ctx context.Context, baseDir string, logFunc func(string)) (string, error) {
|
|
if baseDir == "" {
|
|
baseDir = "/var/log/bee-sat"
|
|
}
|
|
ts := time.Now().UTC().Format("20060102-150405")
|
|
runDir := filepath.Join(baseDir, "storage-"+ts)
|
|
if err := os.MkdirAll(runDir, 0755); err != nil {
|
|
return "", err
|
|
}
|
|
verboseLog := filepath.Join(runDir, "verbose.log")
|
|
|
|
devices, err := listStorageDevices()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
sort.Strings(devices)
|
|
|
|
var summary strings.Builder
|
|
stats := satStats{}
|
|
fmt.Fprintf(&summary, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339))
|
|
if len(devices) == 0 {
|
|
fmt.Fprintln(&summary, "devices=0")
|
|
stats.Unsupported++
|
|
} else {
|
|
fmt.Fprintf(&summary, "devices=%d\n", len(devices))
|
|
}
|
|
|
|
for index, devPath := range devices {
|
|
if ctx.Err() != nil {
|
|
break
|
|
}
|
|
prefix := fmt.Sprintf("%02d-%s", index+1, filepath.Base(devPath))
|
|
commands := storageSATCommands(devPath)
|
|
for cmdIndex, job := range commands {
|
|
if ctx.Err() != nil {
|
|
break
|
|
}
|
|
name := fmt.Sprintf("%s-%02d-%s.log", prefix, cmdIndex+1, job.name)
|
|
out, err := runSATCommandCtx(ctx, verboseLog, job.name, job.cmd, nil, logFunc)
|
|
if writeErr := os.WriteFile(filepath.Join(runDir, name), out, 0644); writeErr != nil {
|
|
return "", writeErr
|
|
}
|
|
status, rc := classifySATResult(job.name, out, err)
|
|
stats.Add(status)
|
|
key := filepath.Base(devPath) + "_" + strings.ReplaceAll(job.name, "-", "_")
|
|
fmt.Fprintf(&summary, "%s_rc=%d\n", key, rc)
|
|
fmt.Fprintf(&summary, "%s_status=%s\n", key, status)
|
|
}
|
|
}
|
|
|
|
writeSATStats(&summary, stats)
|
|
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary.String()), 0644); err != nil {
|
|
return "", err
|
|
}
|
|
archive := filepath.Join(baseDir, "storage-"+ts+".tar.gz")
|
|
if err := createTarGz(archive, runDir); err != nil {
|
|
return "", err
|
|
}
|
|
return archive, nil
|
|
}
|
|
|
|
type satJob struct {
|
|
name string
|
|
cmd []string
|
|
env []string // extra env vars (appended to os.Environ)
|
|
collectGPU bool // collect GPU metrics via nvidia-smi while this job runs
|
|
gpuIndices []int // GPU indices to collect metrics for (empty = all)
|
|
}
|
|
|
|
type satStats struct {
|
|
OK int
|
|
Failed int
|
|
Unsupported int
|
|
}
|
|
|
|
func nvidiaSATJobs() []satJob {
|
|
seconds := envInt("BEE_GPU_STRESS_SECONDS", 5)
|
|
sizeMB := envInt("BEE_GPU_STRESS_SIZE_MB", 64)
|
|
return []satJob{
|
|
{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
|
|
{name: "02-dmidecode-baseboard.log", cmd: []string{"dmidecode", "-t", "baseboard"}},
|
|
{name: "03-dmidecode-system.log", cmd: []string{"dmidecode", "-t", "system"}},
|
|
{name: "04-nvidia-bug-report.log", cmd: []string{"nvidia-bug-report.sh", "--output-file", "{{run_dir}}/nvidia-bug-report.log"}},
|
|
{name: "05-bee-gpu-stress.log", cmd: []string{"bee-gpu-stress", "--seconds", fmt.Sprintf("%d", seconds), "--size-mb", fmt.Sprintf("%d", sizeMB)}},
|
|
}
|
|
}
|
|
|
|
func nvidiaDCGMJobs(diagLevel int, gpuIndices []int) []satJob {
|
|
if diagLevel < 1 || diagLevel > 4 {
|
|
diagLevel = 3
|
|
}
|
|
diagArgs := []string{"dcgmi", "diag", "-r", strconv.Itoa(diagLevel)}
|
|
if len(gpuIndices) > 0 {
|
|
ids := make([]string, len(gpuIndices))
|
|
for i, idx := range gpuIndices {
|
|
ids[i] = strconv.Itoa(idx)
|
|
}
|
|
diagArgs = append(diagArgs, "-i", strings.Join(ids, ","))
|
|
}
|
|
return []satJob{
|
|
{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
|
|
{name: "02-dmidecode-baseboard.log", cmd: []string{"dmidecode", "-t", "baseboard"}},
|
|
{name: "03-dmidecode-system.log", cmd: []string{"dmidecode", "-t", "system"}},
|
|
{name: "04-dcgmi-diag.log", cmd: diagArgs},
|
|
}
|
|
}
|
|
|
|
func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []satJob, logFunc func(string)) (string, error) {
|
|
if ctx == nil {
|
|
ctx = context.Background()
|
|
}
|
|
if baseDir == "" {
|
|
baseDir = "/var/log/bee-sat"
|
|
}
|
|
ts := time.Now().UTC().Format("20060102-150405")
|
|
runDir := filepath.Join(baseDir, prefix+"-"+ts)
|
|
if err := os.MkdirAll(runDir, 0755); err != nil {
|
|
return "", err
|
|
}
|
|
verboseLog := filepath.Join(runDir, "verbose.log")
|
|
|
|
var summary strings.Builder
|
|
stats := satStats{}
|
|
fmt.Fprintf(&summary, "run_at_utc=%s\n", time.Now().UTC().Format(time.RFC3339))
|
|
for _, job := range jobs {
|
|
if ctx.Err() != nil {
|
|
break
|
|
}
|
|
cmd := make([]string, 0, len(job.cmd))
|
|
for _, arg := range job.cmd {
|
|
cmd = append(cmd, strings.ReplaceAll(arg, "{{run_dir}}", runDir))
|
|
}
|
|
|
|
var out []byte
|
|
var err error
|
|
|
|
if job.collectGPU {
|
|
out, err = runSATCommandWithMetrics(ctx, verboseLog, job.name, cmd, job.env, job.gpuIndices, runDir, logFunc)
|
|
} else {
|
|
out, err = runSATCommandCtx(ctx, verboseLog, job.name, cmd, job.env, logFunc)
|
|
}
|
|
|
|
if writeErr := os.WriteFile(filepath.Join(runDir, job.name), out, 0644); writeErr != nil {
|
|
return "", writeErr
|
|
}
|
|
status, rc := classifySATResult(job.name, out, err)
|
|
stats.Add(status)
|
|
key := strings.TrimSuffix(strings.TrimPrefix(job.name, "0"), ".log")
|
|
fmt.Fprintf(&summary, "%s_rc=%d\n", key, rc)
|
|
fmt.Fprintf(&summary, "%s_status=%s\n", key, status)
|
|
}
|
|
writeSATStats(&summary, stats)
|
|
if err := os.WriteFile(filepath.Join(runDir, "summary.txt"), []byte(summary.String()), 0644); err != nil {
|
|
return "", err
|
|
}
|
|
|
|
archive := filepath.Join(baseDir, prefix+"-"+ts+".tar.gz")
|
|
if err := createTarGz(archive, runDir); err != nil {
|
|
return "", err
|
|
}
|
|
return archive, nil
|
|
}
|
|
|
|
func runSATCommandCtx(ctx context.Context, verboseLog, name string, cmd []string, env []string, logFunc func(string)) ([]byte, error) {
|
|
start := time.Now().UTC()
|
|
resolvedCmd, err := resolveSATCommand(cmd)
|
|
appendSATVerboseLog(verboseLog,
|
|
fmt.Sprintf("[%s] start %s", start.Format(time.RFC3339), name),
|
|
"cmd: "+strings.Join(resolvedCmd, " "),
|
|
)
|
|
if logFunc != nil {
|
|
logFunc(fmt.Sprintf("=== %s ===", name))
|
|
}
|
|
if err != nil {
|
|
appendSATVerboseLog(verboseLog,
|
|
fmt.Sprintf("[%s] finish %s", time.Now().UTC().Format(time.RFC3339), name),
|
|
"rc: 1",
|
|
fmt.Sprintf("duration_ms: %d", time.Since(start).Milliseconds()),
|
|
"",
|
|
)
|
|
return []byte(err.Error() + "\n"), err
|
|
}
|
|
|
|
c := exec.CommandContext(ctx, resolvedCmd[0], resolvedCmd[1:]...)
|
|
if len(env) > 0 {
|
|
c.Env = append(os.Environ(), env...)
|
|
}
|
|
out, err := streamExecOutput(c, logFunc)
|
|
|
|
rc := 0
|
|
if err != nil {
|
|
rc = 1
|
|
}
|
|
appendSATVerboseLog(verboseLog,
|
|
fmt.Sprintf("[%s] finish %s", time.Now().UTC().Format(time.RFC3339), name),
|
|
fmt.Sprintf("rc: %d", rc),
|
|
fmt.Sprintf("duration_ms: %d", time.Since(start).Milliseconds()),
|
|
"",
|
|
)
|
|
return out, err
|
|
}
|
|
|
|
func listStorageDevices() ([]string, error) {
|
|
out, err := satExecCommand("lsblk", "-dn", "-o", "NAME,TYPE,TRAN").Output()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return parseStorageDevices(string(out)), nil
|
|
}
|
|
|
|
func storageSATCommands(devPath string) []satJob {
|
|
if strings.Contains(filepath.Base(devPath), "nvme") {
|
|
return []satJob{
|
|
{name: "nvme-id-ctrl", cmd: []string{"nvme", "id-ctrl", devPath, "-o", "json"}},
|
|
{name: "nvme-smart-log", cmd: []string{"nvme", "smart-log", devPath, "-o", "json"}},
|
|
{name: "nvme-device-self-test", cmd: []string{"nvme", "device-self-test", devPath, "-s", "1", "--wait"}},
|
|
}
|
|
}
|
|
return []satJob{
|
|
{name: "smartctl-health", cmd: []string{"smartctl", "-H", "-A", devPath}},
|
|
{name: "smartctl-self-test-short", cmd: []string{"smartctl", "-t", "short", devPath}},
|
|
}
|
|
}
|
|
|
|
func (s *satStats) Add(status string) {
|
|
switch status {
|
|
case "OK":
|
|
s.OK++
|
|
case "UNSUPPORTED":
|
|
s.Unsupported++
|
|
default:
|
|
s.Failed++
|
|
}
|
|
}
|
|
|
|
func (s satStats) Overall() string {
|
|
if s.Failed > 0 {
|
|
return "FAILED"
|
|
}
|
|
if s.Unsupported > 0 {
|
|
return "PARTIAL"
|
|
}
|
|
return "OK"
|
|
}
|
|
|
|
func writeSATStats(summary *strings.Builder, stats satStats) {
|
|
fmt.Fprintf(summary, "overall_status=%s\n", stats.Overall())
|
|
fmt.Fprintf(summary, "job_ok=%d\n", stats.OK)
|
|
fmt.Fprintf(summary, "job_failed=%d\n", stats.Failed)
|
|
fmt.Fprintf(summary, "job_unsupported=%d\n", stats.Unsupported)
|
|
}
|
|
|
|
func classifySATResult(name string, out []byte, err error) (string, int) {
|
|
rc := 0
|
|
if err != nil {
|
|
rc = 1
|
|
}
|
|
if err == nil {
|
|
return "OK", rc
|
|
}
|
|
|
|
text := strings.ToLower(string(out))
|
|
// No output at all means the tool failed to start (mlock limit, binary missing,
|
|
// etc.) — we cannot say anything about hardware health → UNSUPPORTED.
|
|
if len(strings.TrimSpace(text)) == 0 {
|
|
return "UNSUPPORTED", rc
|
|
}
|
|
if strings.Contains(text, "unsupported") ||
|
|
strings.Contains(text, "not supported") ||
|
|
strings.Contains(text, "invalid opcode") ||
|
|
strings.Contains(text, "unknown command") ||
|
|
strings.Contains(text, "not implemented") ||
|
|
strings.Contains(text, "not available") ||
|
|
strings.Contains(text, "cuda_error_system_not_ready") ||
|
|
strings.Contains(text, "no such device") ||
|
|
// nvidia-smi on a machine with no NVIDIA GPU
|
|
strings.Contains(text, "couldn't communicate with the nvidia driver") ||
|
|
strings.Contains(text, "no nvidia gpu") ||
|
|
(strings.Contains(name, "self-test") && strings.Contains(text, "aborted")) {
|
|
return "UNSUPPORTED", rc
|
|
}
|
|
return "FAILED", rc
|
|
}
|
|
|
|
func runSATCommand(verboseLog, name string, cmd []string, logFunc func(string)) ([]byte, error) {
|
|
start := time.Now().UTC()
|
|
resolvedCmd, err := resolveSATCommand(cmd)
|
|
appendSATVerboseLog(verboseLog,
|
|
fmt.Sprintf("[%s] start %s", start.Format(time.RFC3339), name),
|
|
"cmd: "+strings.Join(resolvedCmd, " "),
|
|
)
|
|
if logFunc != nil {
|
|
logFunc(fmt.Sprintf("=== %s ===", name))
|
|
}
|
|
if err != nil {
|
|
appendSATVerboseLog(verboseLog,
|
|
fmt.Sprintf("[%s] finish %s", time.Now().UTC().Format(time.RFC3339), name),
|
|
"rc: 1",
|
|
fmt.Sprintf("duration_ms: %d", time.Since(start).Milliseconds()),
|
|
"",
|
|
)
|
|
return []byte(err.Error() + "\n"), err
|
|
}
|
|
|
|
out, err := streamExecOutput(satExecCommand(resolvedCmd[0], resolvedCmd[1:]...), logFunc)
|
|
|
|
rc := 0
|
|
if err != nil {
|
|
rc = 1
|
|
}
|
|
appendSATVerboseLog(verboseLog,
|
|
fmt.Sprintf("[%s] finish %s", time.Now().UTC().Format(time.RFC3339), name),
|
|
fmt.Sprintf("rc: %d", rc),
|
|
fmt.Sprintf("duration_ms: %d", time.Since(start).Milliseconds()),
|
|
"",
|
|
)
|
|
return out, err
|
|
}
|
|
|
|
func runROCmSMI(args ...string) ([]byte, error) {
|
|
cmd, err := resolveROCmSMICommand(args...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return satExecCommand(cmd[0], cmd[1:]...).CombinedOutput()
|
|
}
|
|
|
|
func resolveSATCommand(cmd []string) ([]string, error) {
|
|
if len(cmd) == 0 {
|
|
return nil, errors.New("empty SAT command")
|
|
}
|
|
if cmd[0] != "rocm-smi" {
|
|
return cmd, nil
|
|
}
|
|
return resolveROCmSMICommand(cmd[1:]...)
|
|
}
|
|
|
|
func resolveROCmSMICommand(args ...string) ([]string, error) {
|
|
if path, err := satLookPath("rocm-smi"); err == nil {
|
|
return append([]string{path}, args...), nil
|
|
}
|
|
|
|
for _, path := range rocmSMIExecutableCandidates() {
|
|
return append([]string{path}, args...), nil
|
|
}
|
|
|
|
pythonPath, pyErr := satLookPath("python3")
|
|
if pyErr == nil {
|
|
for _, script := range rocmSMIScriptCandidates() {
|
|
cmd := []string{pythonPath, script}
|
|
cmd = append(cmd, args...)
|
|
return cmd, nil
|
|
}
|
|
}
|
|
|
|
return nil, errors.New("rocm-smi not found in PATH or under /opt/rocm")
|
|
}
|
|
|
|
func ensureAMDRuntimeReady() error {
|
|
if _, err := os.Stat("/dev/kfd"); err == nil {
|
|
return nil
|
|
}
|
|
if raw, err := os.ReadFile("/sys/module/amdgpu/initstate"); err == nil {
|
|
state := strings.TrimSpace(string(raw))
|
|
if strings.EqualFold(state, "live") {
|
|
return nil
|
|
}
|
|
return fmt.Errorf("AMD driver is present but not initialized: amdgpu initstate=%q", state)
|
|
}
|
|
return errors.New("AMD GPUs are present but the runtime is not initialized: /dev/kfd is missing and amdgpu is not loaded")
|
|
}
|
|
|
|
func rocmSMIExecutableCandidates() []string {
|
|
return expandExistingPaths(rocmSMIExecutableGlobs)
|
|
}
|
|
|
|
func rocmSMIScriptCandidates() []string {
|
|
return expandExistingPaths(rocmSMIScriptGlobs)
|
|
}
|
|
|
|
func expandExistingPaths(patterns []string) []string {
|
|
seen := make(map[string]struct{})
|
|
var paths []string
|
|
for _, pattern := range patterns {
|
|
matches, err := satGlob(pattern)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
sort.Strings(matches)
|
|
for _, match := range matches {
|
|
if _, err := satStat(match); err != nil {
|
|
continue
|
|
}
|
|
if _, ok := seen[match]; ok {
|
|
continue
|
|
}
|
|
seen[match] = struct{}{}
|
|
paths = append(paths, match)
|
|
}
|
|
}
|
|
return paths
|
|
}
|
|
|
|
func parseStorageDevices(raw string) []string {
|
|
var devices []string
|
|
for _, line := range strings.Split(strings.TrimSpace(raw), "\n") {
|
|
fields := strings.Fields(strings.TrimSpace(line))
|
|
if len(fields) < 2 || fields[1] != "disk" {
|
|
continue
|
|
}
|
|
if len(fields) >= 3 && strings.EqualFold(fields[2], "usb") {
|
|
continue
|
|
}
|
|
devices = append(devices, "/dev/"+fields[0])
|
|
}
|
|
return devices
|
|
}
|
|
|
|
// runSATCommandWithMetrics runs a command while collecting GPU metrics in the background.
|
|
// On completion it writes gpu-metrics.csv and gpu-metrics.html into runDir.
|
|
func runSATCommandWithMetrics(ctx context.Context, verboseLog, name string, cmd []string, env []string, gpuIndices []int, runDir string, logFunc func(string)) ([]byte, error) {
|
|
stopCh := make(chan struct{})
|
|
doneCh := make(chan struct{})
|
|
var metricRows []GPUMetricRow
|
|
start := time.Now()
|
|
|
|
go func() {
|
|
defer close(doneCh)
|
|
ticker := time.NewTicker(time.Second)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-stopCh:
|
|
return
|
|
case <-ticker.C:
|
|
samples, err := sampleGPUMetrics(gpuIndices)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
elapsed := time.Since(start).Seconds()
|
|
for i := range samples {
|
|
samples[i].ElapsedSec = elapsed
|
|
}
|
|
metricRows = append(metricRows, samples...)
|
|
}
|
|
}
|
|
}()
|
|
|
|
out, err := runSATCommandCtx(ctx, verboseLog, name, cmd, env, logFunc)
|
|
|
|
close(stopCh)
|
|
<-doneCh
|
|
|
|
if len(metricRows) > 0 {
|
|
_ = WriteGPUMetricsCSV(filepath.Join(runDir, "gpu-metrics.csv"), metricRows)
|
|
_ = WriteGPUMetricsHTML(filepath.Join(runDir, "gpu-metrics.html"), metricRows)
|
|
chart := RenderGPUTerminalChart(metricRows)
|
|
_ = os.WriteFile(filepath.Join(runDir, "gpu-metrics-term.txt"), []byte(chart), 0644)
|
|
}
|
|
|
|
return out, err
|
|
}
|
|
|
|
func appendSATVerboseLog(path string, lines ...string) {
|
|
if path == "" {
|
|
return
|
|
}
|
|
f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
|
|
if err != nil {
|
|
return
|
|
}
|
|
defer f.Close()
|
|
for _, line := range lines {
|
|
_, _ = io.WriteString(f, line+"\n")
|
|
}
|
|
}
|
|
|
|
func envInt(name string, fallback int) int {
|
|
raw := strings.TrimSpace(os.Getenv(name))
|
|
if raw == "" {
|
|
return fallback
|
|
}
|
|
value, err := strconv.Atoi(raw)
|
|
if err != nil || value <= 0 {
|
|
return fallback
|
|
}
|
|
return value
|
|
}
|
|
|
|
func createTarGz(dst, srcDir string) error {
|
|
file, err := os.Create(dst)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer file.Close()
|
|
|
|
gz := gzip.NewWriter(file)
|
|
defer gz.Close()
|
|
|
|
tw := tar.NewWriter(gz)
|
|
defer tw.Close()
|
|
|
|
base := filepath.Dir(srcDir)
|
|
return filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if info.IsDir() {
|
|
return nil
|
|
}
|
|
header, err := tar.FileInfoHeader(info, "")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
rel, err := filepath.Rel(base, path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
header.Name = rel
|
|
if err := tw.WriteHeader(header); err != nil {
|
|
return err
|
|
}
|
|
file, err := os.Open(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer file.Close()
|
|
_, err = io.Copy(tw, file)
|
|
return err
|
|
})
|
|
}
|