sat: collect dcgmi discovery + full nvvs debug log for all DCGM diags

dcgmi diag's failures only surface a terse wrapper message ("Detected
driver major version 0 is not between the required versions 346 and
2000"), which doesn't say why. Reading DCGM's own nvvs source
(NvidiaValidationSuite.cpp::CheckDriverVersion) shows this fires when
dcgmSystem.GetDeviceAttributes(0, ...) comes back empty — the version
string is never actually malformed. Diagnosing further requires info
bee never collected: whether dcgmi discovery even sees GPU 0/1
correctly, and nvvs's own internal debug log (normally
/var/log/nvidia-dcgm/nvvs.log, which bee never captured).

Add a "dcgmi discovery -l" job before every DCGM diag invocation
(check-gpu-dcgm-l2, nvbandwidth, targeted_power, pulse_test,
targeted_stress), and route dcgmi diag through
"-v -d DEBUG --debugLogFile {{run_dir}}/..." so the full nvvs debug
log lands in the SAT run dir and gets picked up by the support bundle
automatically, instead of needing a live SSH session on hardware we
usually can't get a second look at.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

audit/internal/platform/sat.go

@@ -493,13 +493,14 @@ func (s *System) RunNvidiaTargetedPowerPack(ctx context.Context, baseDir string,
 	}
 	return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-targeted-power", withNvidiaPersistenceMode(
 		satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
+		satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}},
 		satJob{
-			name:       "02-dcgmi-targeted-power.log",
+			name:       "03-dcgmi-targeted-power.log",
 			cmd:        nvidiaDCGMNamedDiagCommand("targeted_power", normalizeNvidiaBurnDuration(durationSec), selected),
 			collectGPU: true,
 			gpuIndices: selected,
 		},
-		satJob{name: "03-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
+		satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
 	), logFunc)
 }
 
@@ -517,13 +518,14 @@ func (s *System) RunNvidiaPulseTestPack(ctx context.Context, baseDir string, dur
 	}
 	return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-pulse", withNvidiaPersistenceMode(
 		satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
+		satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}},
 		satJob{
-			name:       "02-dcgmi-pulse-test.log",
+			name:       "03-dcgmi-pulse-test.log",
 			cmd:        nvidiaDCGMNamedDiagCommand("pulse_test", normalizeNvidiaBurnDuration(durationSec), selected),
 			collectGPU: true,
 			gpuIndices: selected,
 		},
-		satJob{name: "03-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
+		satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
 	), logFunc)
 }
 
@@ -541,13 +543,14 @@ func (s *System) RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpu
 	}
 	return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-bandwidth", withNvidiaPersistenceMode(
 		satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
+		satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}},
 		satJob{
-			name:       "02-dcgmi-nvbandwidth.log",
+			name:       "03-dcgmi-nvbandwidth.log",
 			cmd:        nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, selected),
 			collectGPU: true,
 			gpuIndices: selected,
 		},
-		satJob{name: "03-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
+		satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
 	), logFunc)
 }
 
@@ -581,13 +584,14 @@ func (s *System) RunNvidiaTargetedStressValidatePack(ctx context.Context, baseDi
 	}
 	return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-targeted-stress", withNvidiaPersistenceMode(
 		satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
+		satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}},
 		satJob{
-			name:       "02-dcgmi-targeted-stress.log",
+			name:       "03-dcgmi-targeted-stress.log",
 			cmd:        nvidiaDCGMNamedDiagCommand("targeted_stress", normalizeNvidiaBurnDuration(durationSec), selected),
 			collectGPU: true,
 			gpuIndices: selected,
 		},
-		satJob{name: "03-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
+		satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
 	), logFunc)
 }
 
@@ -815,7 +819,7 @@ func nvidiaDCGMJobs(diagLevel int, gpuIndices []int) []satJob {
 	if diagLevel < 1 || diagLevel > 4 {
 		diagLevel = 3
 	}
-	diagArgs := []string{"dcgmi", "diag", "-r", strconv.Itoa(diagLevel)}
+	diagArgs := append([]string{"dcgmi", "diag", "-r", strconv.Itoa(diagLevel)}, nvidiaDCGMDiagDebugArgs("dcgmi-diag")...)
 	if len(gpuIndices) > 0 {
 		ids := make([]string, len(gpuIndices))
 		for i, idx := range gpuIndices {
@@ -827,12 +831,21 @@ func nvidiaDCGMJobs(diagLevel int, gpuIndices []int) []satJob {
 		satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
 		satJob{name: "02-dmidecode-baseboard.log", cmd: []string{"dmidecode", "-t", "baseboard"}},
 		satJob{name: "03-dmidecode-system.log", cmd: []string{"dmidecode", "-t", "system"}},
-		satJob{name: "04-dcgmi-diag.log", cmd: diagArgs, gpuIndices: gpuIndices},
+		satJob{name: "04-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}},
+		satJob{name: "05-dcgmi-diag.log", cmd: diagArgs, gpuIndices: gpuIndices},
 	)
 }
 
+// nvidiaDCGMDiagDebugArgs returns flags that make dcgmi diag emit its full
+// internal nvvs debug log (e.g. the real reason behind a terse "Detected
+// driver major version 0" failure) into the SAT run dir instead of the
+// default /var/log/nvidia-dcgm/nvvs.log, which may not be captured otherwise.
+func nvidiaDCGMDiagDebugArgs(logPrefix string) []string {
+	return []string{"-v", "-d", "DEBUG", "--debugLogFile", "{{run_dir}}/" + logPrefix + "-debug.log"}
+}
+
 func nvidiaDCGMNamedDiagCommand(name string, durationSec int, gpuIndices []int) []string {
-	args := []string{"dcgmi", "diag", "-r", name}
+	args := append([]string{"dcgmi", "diag", "-r", name}, nvidiaDCGMDiagDebugArgs("dcgmi-"+strings.ReplaceAll(name, "_", "-"))...)
 	if durationSec > 0 {
 		args = append(args, "-p", fmt.Sprintf("%s.test_duration=%d", name, durationSec))
 	}

audit/internal/platform/sat_test.go

@@ -132,14 +132,15 @@ func TestNvidiaSATJobsUseBuiltinBurnDefaults(t *testing.T) {
 
 func TestNvidiaDCGMJobsEnablePersistenceModeBeforeDiag(t *testing.T) {
 	jobs := nvidiaDCGMJobs(3, []int{2, 0})
-	if len(jobs) != 5 {
+	if len(jobs) != 6 {
-		t.Fatalf("jobs=%d want 5", len(jobs))
+		t.Fatalf("jobs=%d want 6", len(jobs))
 	}
 	if got := strings.Join(jobs[0].cmd, " "); got != "nvidia-smi -pm 1" {
 		t.Fatalf("preflight=%q want %q", got, "nvidia-smi -pm 1")
 	}
-	if got := strings.Join(jobs[4].cmd, " "); got != "dcgmi diag -r 3 -i 2,0" {
+	want := "dcgmi diag -r 3 -v -d DEBUG --debugLogFile {{run_dir}}/dcgmi-diag-debug.log -i 2,0"
-		t.Fatalf("diag=%q want %q", got, "dcgmi diag -r 3 -i 2,0")
+	if got := strings.Join(jobs[5].cmd, " "); got != want {
+		t.Fatalf("diag=%q want %q", got, want)
 	}
 }
 
@@ -338,7 +339,7 @@ func TestResolveDCGMProfTesterCommandUsesVersionedBinary(t *testing.T) {
 
 func TestNvidiaDCGMNamedDiagCommandUsesDurationAndSelection(t *testing.T) {
 	cmd := nvidiaDCGMNamedDiagCommand("targeted_power", 900, []int{3, 1})
-	want := []string{"dcgmi", "diag", "-r", "targeted_power", "-p", "targeted_power.test_duration=900", "-i", "3,1"}
+	want := []string{"dcgmi", "diag", "-r", "targeted_power", "-v", "-d", "DEBUG", "--debugLogFile", "{{run_dir}}/dcgmi-targeted-power-debug.log", "-p", "targeted_power.test_duration=900", "-i", "3,1"}
 	if len(cmd) != len(want) {
 		t.Fatalf("cmd len=%d want %d (%v)", len(cmd), len(want), cmd)
 	}
@@ -351,7 +352,7 @@ func TestNvidiaDCGMNamedDiagCommandUsesDurationAndSelection(t *testing.T) {
 
 func TestNvidiaDCGMNamedDiagCommandSkipsDurationForNVBandwidth(t *testing.T) {
 	cmd := nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, []int{2, 0})
-	want := []string{"dcgmi", "diag", "-r", "nvbandwidth", "-i", "2,0"}
+	want := []string{"dcgmi", "diag", "-r", "nvbandwidth", "-v", "-d", "DEBUG", "--debugLogFile", "{{run_dir}}/dcgmi-nvbandwidth-debug.log", "-i", "2,0"}
 	if len(cmd) != len(want) {
 		t.Fatalf("cmd len=%d want %d (%v)", len(cmd), len(want), cmd)
 	}