reanimator/bee
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

refactor: harden diagnostics and consolidate runtime code

Browse Source
This commit is contained in:
Mikhail Chusavitin
2026-09-01 13:01:28 +03:00
parent ac4bc0b2b7
commit 0a6ca8ba0f
49 changed files with 1441 additions and 837 deletions
Download Patch File Download Diff File Expand all files Collapse all files
audit/internal/platform/sat_gpu_packs.go
+36 -62
View File
@@ -324,17 +324,30 @@ func (s *System) RunNCCLTests(ctx context.Context, baseDir string, gpuIndices []
if err != nil {
return "", err
}
return runAcceptancePackCtx(ctx, baseDir, "nccl-tests", ncclSATJobs(selected), logFunc)
}
func ncclSATJobs(selected []int) []satJob {
gpuCount := len(selected)
if gpuCount < 1 {
gpuCount = 1
}
return runAcceptancePackCtx(ctx, baseDir, "nccl-tests", withNvidiaPersistenceMode(
ncclEnv := append(nvidiaVisibleDevicesEnv(selected),
"NCCL_DEBUG=INFO",
"NCCL_DEBUG_SUBSYS=INIT,GRAPH",
)
return withNvidiaPersistenceMode(
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
satJob{name: "01-nvidia-smi-topo-m.log", cmd: []string{"nvidia-smi", "topo", "-m"}, informational: true},
satJob{name: "01-nvidia-smi-nvlink-s.log", cmd: []string{"nvidia-smi", "nvlink", "-s"}, informational: true},
satJob{name: "02-all-reduce-perf.log", cmd: []string{
"all_reduce_perf", "-b", "512M", "-e", "4G", "-f", "2",
"-g", strconv.Itoa(gpuCount), "--iters", "20",
}, env: nvidiaVisibleDevicesEnv(selected), syncBracket: true},
), logFunc)
}, env: ncclEnv, gpuIndices: selected, syncBracket: true,
validate: func(runDir string, out []byte) (string, string) {
return validateNCCLAllReduceOutput(runDir, out, selected)
}},
)
}
func (s *System) RunNvidiaOfficialComputePack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, staggerSec int, logFunc func(string)) (string, error) {
@@ -384,23 +397,27 @@ func (s *System) RunNvidiaOfficialComputePack(ctx context.Context, baseDir strin
}
func (s *System) RunNvidiaTargetedPowerPack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) {
return s.runNvidiaNamedDiagPack(ctx, baseDir, durationSec, gpuIndices,
"gpu-nvidia-targeted-power", "targeted_power", "03-dcgmi-targeted-power.log", logFunc)
}
func (s *System) RunNvidiaPulseTestPack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) {
return s.runNvidiaNamedDiagPack(ctx, baseDir, durationSec, gpuIndices,
"gpu-nvidia-pulse", "pulse_test", "03-dcgmi-pulse-test.log", logFunc)
}
func (s *System) runNvidiaNamedDiagPack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, packName, diagName, logName string, logFunc func(string)) (string, error) {
selected, err := resolveDCGMGPUIndices(gpuIndices)
if err != nil {
return "", err
}
// Kill any lingering nvvs/dcgmi processes from a previous interrupted run
// before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34).
if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil {
for _, p := range killed {
logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name))
}
}
return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-targeted-power", withNvidiaPersistenceMode(
killStaleNvidiaTestWorkers(logFunc)
return runAcceptancePackCtx(ctx, baseDir, packName, withNvidiaPersistenceMode(
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
satJob{
name: "03-dcgmi-targeted-power.log",
cmd: nvidiaDCGMNamedDiagCommand("targeted_power", normalizeNvidiaBurnDuration(durationSec), selected),
name: logName,
cmd: nvidiaDCGMNamedDiagCommand(diagName, normalizeNvidiaBurnDuration(durationSec), selected),
collectGPU: true,
gpuIndices: selected,
syncBracket: true,
@@ -409,30 +426,14 @@ func (s *System) RunNvidiaTargetedPowerPack(ctx context.Context, baseDir string,
), logFunc)
}
func (s *System) RunNvidiaPulseTestPack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) {
selected, err := resolveDCGMGPUIndices(gpuIndices)
if err != nil {
return "", err
}
// Kill any lingering nvvs/dcgmi processes from a previous interrupted run
// before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34).
// Kill any lingering nvvs/dcgmi processes from a previous interrupted run
// before starting; otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34).
func killStaleNvidiaTestWorkers(logFunc func(string)) {
if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil {
for _, p := range killed {
logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name))
}
}
return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-pulse", withNvidiaPersistenceMode(
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
satJob{
name: "03-dcgmi-pulse-test.log",
cmd: nvidiaDCGMNamedDiagCommand("pulse_test", normalizeNvidiaBurnDuration(durationSec), selected),
collectGPU: true,
gpuIndices: selected,
syncBracket: true,
},
satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
), logFunc)
}
// RunNvidiaBandwidthPack runs `dcgmi diag -r nvbandwidth`. The only thing
@@ -449,13 +450,7 @@ func (s *System) RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpu
if err != nil {
return "", err
}
// Kill any lingering nvvs/dcgmi processes from a previous interrupted run
// before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34).
if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil {
for _, p := range killed {
logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name))
}
}
killStaleNvidiaTestWorkers(logFunc)
jobs := []satJob{
{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
@@ -528,29 +523,8 @@ func (s *System) RunNvidiaAcceptancePackWithOptions(ctx context.Context, baseDir
}
func (s *System) RunNvidiaTargetedStressValidatePack(ctx context.Context, baseDir string, durationSec int, gpuIndices []int, logFunc func(string)) (string, error) {
selected, err := resolveDCGMGPUIndices(gpuIndices)
if err != nil {
return "", err
}
// Kill any lingering nvvs/dcgmi processes from a previous interrupted run
// before starting — otherwise dcgmi diag fails with DCGM_ST_IN_USE (-34).
if killed := KillTestWorkers(); len(killed) > 0 && logFunc != nil {
for _, p := range killed {
logFunc(fmt.Sprintf("pre-flight: killed stale worker pid=%d name=%s", p.PID, p.Name))
}
}
return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-targeted-stress", withNvidiaPersistenceMode(
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
satJob{
name: "03-dcgmi-targeted-stress.log",
cmd: nvidiaDCGMNamedDiagCommand("targeted_stress", normalizeNvidiaBurnDuration(durationSec), selected),
collectGPU: true,
gpuIndices: selected,
syncBracket: true,
},
satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
), logFunc)
return s.runNvidiaNamedDiagPack(ctx, baseDir, durationSec, gpuIndices,
"gpu-nvidia-targeted-stress", "targeted_stress", "03-dcgmi-targeted-stress.log", logFunc)
}
func resolveDCGMGPUIndices(gpuIndices []int) ([]int, error) {