reanimator/bee
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

sat: collect dcgmi discovery + full nvvs debug log for all DCGM diags

Browse Source 
dcgmi diag's failures only surface a terse wrapper message ("Detected
driver major version 0 is not between the required versions 346 and
2000"), which doesn't say why. Reading DCGM's own nvvs source
(NvidiaValidationSuite.cpp::CheckDriverVersion) shows this fires when
dcgmSystem.GetDeviceAttributes(0, ...) comes back empty — the version
string is never actually malformed. Diagnosing further requires info
bee never collected: whether dcgmi discovery even sees GPU 0/1
correctly, and nvvs's own internal debug log (normally
/var/log/nvidia-dcgm/nvvs.log, which bee never captured).

Add a "dcgmi discovery -l" job before every DCGM diag invocation
(check-gpu-dcgm-l2, nvbandwidth, targeted_power, pulse_test,
targeted_stress), and route dcgmi diag through
"-v -d DEBUG --debugLogFile {{run_dir}}/..." so the full nvvs debug
log lands in the SAT run dir and gets picked up by the support bundle
automatically, instead of needing a live SSH session on hardware we
usually can't get a second look at.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-07-06 16:52:55 +03:00
parent 10b45bc267
commit a98721b038
2 changed files with 31 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
audit/internal/platform/sat.go
+24 -11
View File
@@ -493,13 +493,14 @@ func (s *System) RunNvidiaTargetedPowerPack(ctx context.Context, baseDir string,
}
return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-targeted-power", withNvidiaPersistenceMode(
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}},
satJob{
name: "02-dcgmi-targeted-power.log",
name: "03-dcgmi-targeted-power.log",
cmd: nvidiaDCGMNamedDiagCommand("targeted_power", normalizeNvidiaBurnDuration(durationSec), selected),
collectGPU: true,
gpuIndices: selected,
},
satJob{name: "03-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
), logFunc)
}
@@ -517,13 +518,14 @@ func (s *System) RunNvidiaPulseTestPack(ctx context.Context, baseDir string, dur
}
return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-pulse", withNvidiaPersistenceMode(
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}},
satJob{
name: "02-dcgmi-pulse-test.log",
name: "03-dcgmi-pulse-test.log",
cmd: nvidiaDCGMNamedDiagCommand("pulse_test", normalizeNvidiaBurnDuration(durationSec), selected),
collectGPU: true,
gpuIndices: selected,
},
satJob{name: "03-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
), logFunc)
}
@@ -541,13 +543,14 @@ func (s *System) RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpu
}
return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-bandwidth", withNvidiaPersistenceMode(
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}},
satJob{
name: "02-dcgmi-nvbandwidth.log",
name: "03-dcgmi-nvbandwidth.log",
cmd: nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, selected),
collectGPU: true,
gpuIndices: selected,
},
satJob{name: "03-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
), logFunc)
}
@@ -581,13 +584,14 @@ func (s *System) RunNvidiaTargetedStressValidatePack(ctx context.Context, baseDi
}
return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-targeted-stress", withNvidiaPersistenceMode(
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
satJob{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}},
satJob{
name: "02-dcgmi-targeted-stress.log",
name: "03-dcgmi-targeted-stress.log",
cmd: nvidiaDCGMNamedDiagCommand("targeted_stress", normalizeNvidiaBurnDuration(durationSec), selected),
collectGPU: true,
gpuIndices: selected,
},
satJob{name: "03-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
satJob{name: "04-nvidia-smi-after.log", cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"}},
), logFunc)
}
@@ -815,7 +819,7 @@ func nvidiaDCGMJobs(diagLevel int, gpuIndices []int) []satJob {
if diagLevel < 1 || diagLevel > 4 {
diagLevel = 3
}
diagArgs := []string{"dcgmi", "diag", "-r", strconv.Itoa(diagLevel)}
diagArgs := append([]string{"dcgmi", "diag", "-r", strconv.Itoa(diagLevel)}, nvidiaDCGMDiagDebugArgs("dcgmi-diag")...)
if len(gpuIndices) > 0 {
ids := make([]string, len(gpuIndices))
for i, idx := range gpuIndices {
@@ -827,12 +831,21 @@ func nvidiaDCGMJobs(diagLevel int, gpuIndices []int) []satJob {
satJob{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
satJob{name: "02-dmidecode-baseboard.log", cmd: []string{"dmidecode", "-t", "baseboard"}},
satJob{name: "03-dmidecode-system.log", cmd: []string{"dmidecode", "-t", "system"}},
satJob{name: "04-dcgmi-diag.log", cmd: diagArgs, gpuIndices: gpuIndices},
satJob{name: "04-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}},
satJob{name: "05-dcgmi-diag.log", cmd: diagArgs, gpuIndices: gpuIndices},
)
}
// nvidiaDCGMDiagDebugArgs returns flags that make dcgmi diag emit its full
// internal nvvs debug log (e.g. the real reason behind a terse "Detected
// driver major version 0" failure) into the SAT run dir instead of the
// default /var/log/nvidia-dcgm/nvvs.log, which may not be captured otherwise.
func nvidiaDCGMDiagDebugArgs(logPrefix string) []string {
return []string{"-v", "-d", "DEBUG", "--debugLogFile", "{{run_dir}}/" + logPrefix + "-debug.log"}
}
func nvidiaDCGMNamedDiagCommand(name string, durationSec int, gpuIndices []int) []string {
args := []string{"dcgmi", "diag", "-r", name}
args := append([]string{"dcgmi", "diag", "-r", name}, nvidiaDCGMDiagDebugArgs("dcgmi-"+strings.ReplaceAll(name, "_", "-"))...)
if durationSec > 0 {
args = append(args, "-p", fmt.Sprintf("%s.test_duration=%d", name, durationSec))
}