diff --git a/audit/internal/platform/sat.go b/audit/internal/platform/sat.go index 4b026d9..94c02a6 100644 --- a/audit/internal/platform/sat.go +++ b/audit/internal/platform/sat.go @@ -493,13 +493,14 @@ func (s *System) RunNvidiaTargetedPowerPack(ctx context.Context, baseDir string, } return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-targeted-power", withNvidiaPersistenceMode( satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}}, + satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}}, satJob{ - name: "02-dcgmi-targeted-power.log", + name: "03-dcgmi-targeted-power.log", cmd: nvidiaDCGMNamedDiagCommand("targeted_power", normalizeNvidiaBurnDuration(durationSec), selected), collectGPU: true, gpuIndices: selected, }, - satJob{name: "03-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}}, + satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}}, ), logFunc) } @@ -517,13 +518,14 @@ func (s *System) RunNvidiaPulseTestPack(ctx context.Context, baseDir string, dur } return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-pulse", withNvidiaPersistenceMode( satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}}, + satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}}, satJob{ - name: "02-dcgmi-pulse-test.log", + name: "03-dcgmi-pulse-test.log", cmd: nvidiaDCGMNamedDiagCommand("pulse_test", normalizeNvidiaBurnDuration(durationSec), selected), collectGPU: true, gpuIndices: selected, }, - satJob{name: "03-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}}, + satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}}, ), logFunc) } @@ -541,13 +543,14 @@ func (s *System) RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpu } return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-bandwidth", withNvidiaPersistenceMode( satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}}, + satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}}, satJob{ - name: "02-dcgmi-nvbandwidth.log", + name: "03-dcgmi-nvbandwidth.log", cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, selected), collectGPU: true, gpuIndices: selected, }, - satJob{name: "03-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}}, + satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}}, ), logFunc) } @@ -581,13 +584,14 @@ func (s *System) RunNvidiaTargetedStressValidatePack(ctx context.Context, baseDi } return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-targeted-stress", withNvidiaPersistenceMode( satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}}, + satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}}, satJob{ - name: "02-dcgmi-targeted-stress.log", + name: "03-dcgmi-targeted-stress.log", cmd: nvidiaDCGMNamedDiagCommand("targeted_stress", normalizeNvidiaBurnDuration(durationSec), selected), collectGPU: true, gpuIndices: selected, }, - satJob{name: "03-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}}, + satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}}, ), logFunc) } @@ -815,7 +819,7 @@ func nvidiaDCGMJobs(diagLevel int, gpuIndices []int) []satJob { if diagLevel < 1 || diagLevel > 4 { diagLevel = 3 } - diagArgs := []string{"dcgmi", "diag", "-r", strconv.Itoa(diagLevel)} + diagArgs := append([]string{"dcgmi", "diag", "-r", strconv.Itoa(diagLevel)}, nvidiaDCGMDiagDebugArgs("dcgmi-diag")...) if len(gpuIndices) > 0 { ids := make([]string, len(gpuIndices)) for i, idx := range gpuIndices { @@ -827,12 +831,21 @@ func nvidiaDCGMJobs(diagLevel int, gpuIndices []int) []satJob { satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}}, satJob{name: "02-dmidecode-baseboard.log", cmd: []string{"dmidecode", "-t", "baseboard"}}, satJob{name: "03-dmidecode-system.log", cmd: []string{"dmidecode", "-t", "system"}}, - satJob{name: "04-dcgmi-diag.log", cmd: diagArgs, gpuIndices: gpuIndices}, + satJob{name: "04-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}}, + satJob{name: "05-dcgmi-diag.log", cmd: diagArgs, gpuIndices: gpuIndices}, ) } +// nvidiaDCGMDiagDebugArgs returns flags that make dcgmi diag emit its full +// internal nvvs debug log (e.g. the real reason behind a terse "Detected +// driver major version 0" failure) into the SAT run dir instead of the +// default /var/log/nvidia-dcgm/nvvs.log, which may not be captured otherwise. +func nvidiaDCGMDiagDebugArgs(logPrefix string) []string { + return []string{"-v", "-d", "DEBUG", "--debugLogFile", "{{run_dir}}/" + logPrefix + "-debug.log"} +} + func nvidiaDCGMNamedDiagCommand(name string, durationSec int, gpuIndices []int) []string { - args := []string{"dcgmi", "diag", "-r", name} + args := append([]string{"dcgmi", "diag", "-r", name}, nvidiaDCGMDiagDebugArgs("dcgmi-"+strings.ReplaceAll(name, "_", "-"))...) if durationSec > 0 { args = append(args, "-p", fmt.Sprintf("%s.test_duration=%d", name, durationSec)) } diff --git a/audit/internal/platform/sat_test.go b/audit/internal/platform/sat_test.go index ca75fe2..701b771 100644 --- a/audit/internal/platform/sat_test.go +++ b/audit/internal/platform/sat_test.go @@ -132,14 +132,15 @@ func TestNvidiaSATJobsUseBuiltinBurnDefaults(t *testing.T) { func TestNvidiaDCGMJobsEnablePersistenceModeBeforeDiag(t *testing.T) { jobs := nvidiaDCGMJobs(3, []int{2, 0}) - if len(jobs) != 5 { - t.Fatalf("jobs=%d want 5", len(jobs)) + if len(jobs) != 6 { + t.Fatalf("jobs=%d want 6", len(jobs)) } if got := strings.Join(jobs[0].cmd, " "); got != "nvidia-smi -pm 1" { t.Fatalf("preflight=%q want %q", got, "nvidia-smi -pm 1") } - if got := strings.Join(jobs[4].cmd, " "); got != "dcgmi diag -r 3 -i 2,0" { - t.Fatalf("diag=%q want %q", got, "dcgmi diag -r 3 -i 2,0") + want := "dcgmi diag -r 3 -v -d DEBUG --debugLogFile {{run_dir}}/dcgmi-diag-debug.log -i 2,0" + if got := strings.Join(jobs[5].cmd, " "); got != want { + t.Fatalf("diag=%q want %q", got, want) } } @@ -338,7 +339,7 @@ func TestResolveDCGMProfTesterCommandUsesVersionedBinary(t *testing.T) { func TestNvidiaDCGMNamedDiagCommandUsesDurationAndSelection(t *testing.T) { cmd := nvidiaDCGMNamedDiagCommand("targeted_power", 900, []int{3, 1}) - want := []string{"dcgmi", "diag", "-r", "targeted_power", "-p", "targeted_power.test_duration=900", "-i", "3,1"} + want := []string{"dcgmi", "diag", "-r", "targeted_power", "-v", "-d", "DEBUG", "--debugLogFile", "{{run_dir}}/dcgmi-targeted-power-debug.log", "-p", "targeted_power.test_duration=900", "-i", "3,1"} if len(cmd) != len(want) { t.Fatalf("cmd len=%d want %d (%v)", len(cmd), len(want), cmd) } @@ -351,7 +352,7 @@ func TestNvidiaDCGMNamedDiagCommandUsesDurationAndSelection(t *testing.T) { func TestNvidiaDCGMNamedDiagCommandSkipsDurationForNVBandwidth(t *testing.T) { cmd := nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, []int{2, 0}) - want := []string{"dcgmi", "diag", "-r", "nvbandwidth", "-i", "2,0"} + want := []string{"dcgmi", "diag", "-r", "nvbandwidth", "-v", "-d", "DEBUG", "--debugLogFile", "{{run_dir}}/dcgmi-nvbandwidth-debug.log", "-i", "2,0"} if len(cmd) != len(want) { t.Fatalf("cmd len=%d want %d (%v)", len(cmd), len(want), cmd) }