reanimator/bee
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

sat: collect dcgmi discovery + full nvvs debug log for all DCGM diags

Browse Source 
dcgmi diag's failures only surface a terse wrapper message ("Detected
driver major version 0 is not between the required versions 346 and
2000"), which doesn't say why. Reading DCGM's own nvvs source
(NvidiaValidationSuite.cpp::CheckDriverVersion) shows this fires when
dcgmSystem.GetDeviceAttributes(0, ...) comes back empty — the version
string is never actually malformed. Diagnosing further requires info
bee never collected: whether dcgmi discovery even sees GPU 0/1
correctly, and nvvs's own internal debug log (normally
/var/log/nvidia-dcgm/nvvs.log, which bee never captured).

Add a "dcgmi discovery -l" job before every DCGM diag invocation
(check-gpu-dcgm-l2, nvbandwidth, targeted_power, pulse_test,
targeted_stress), and route dcgmi diag through
"-v -d DEBUG --debugLogFile {{run_dir}}/..." so the full nvvs debug
log lands in the SAT run dir and gets picked up by the support bundle
automatically, instead of needing a live SSH session on hardware we
usually can't get a second look at.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-07-06 16:52:55 +03:00
parent 10b45bc267
commit a98721b038
2 changed files with 31 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
audit/internal/platform/sat_test.go
+7 -6
View File
@@ -132,14 +132,15 @@ func TestNvidiaSATJobsUseBuiltinBurnDefaults(t *testing.T) {
func TestNvidiaDCGMJobsEnablePersistenceModeBeforeDiag(t *testing.T) {
jobs := nvidiaDCGMJobs(3, []int{2, 0})
if len(jobs) != 5 {
t.Fatalf("jobs=%d want 5", len(jobs))
if len(jobs) != 6 {
t.Fatalf("jobs=%d want 6", len(jobs))
}
if got := strings.Join(jobs[0].cmd, " "); got != "nvidia-smi -pm 1" {
t.Fatalf("preflight=%q want %q", got, "nvidia-smi -pm 1")
}
if got := strings.Join(jobs[4].cmd, " "); got != "dcgmi diag -r 3 -i 2,0" {
t.Fatalf("diag=%q want %q", got, "dcgmi diag -r 3 -i 2,0")
want := "dcgmi diag -r 3 -v -d DEBUG --debugLogFile {{run_dir}}/dcgmi-diag-debug.log -i 2,0"
if got := strings.Join(jobs[5].cmd, " "); got != want {
t.Fatalf("diag=%q want %q", got, want)
}
}
@@ -338,7 +339,7 @@ func TestResolveDCGMProfTesterCommandUsesVersionedBinary(t *testing.T) {
func TestNvidiaDCGMNamedDiagCommandUsesDurationAndSelection(t *testing.T) {
cmd := nvidiaDCGMNamedDiagCommand("targeted_power", 900, []int{3, 1})
want := []string{"dcgmi", "diag", "-r", "targeted_power", "-p", "targeted_power.test_duration=900", "-i", "3,1"}
want := []string{"dcgmi", "diag", "-r", "targeted_power", "-v", "-d", "DEBUG", "--debugLogFile", "{{run_dir}}/dcgmi-targeted-power-debug.log", "-p", "targeted_power.test_duration=900", "-i", "3,1"}
if len(cmd) != len(want) {
t.Fatalf("cmd len=%d want %d (%v)", len(cmd), len(want), cmd)
}
@@ -351,7 +352,7 @@ func TestNvidiaDCGMNamedDiagCommandUsesDurationAndSelection(t *testing.T) {
func TestNvidiaDCGMNamedDiagCommandSkipsDurationForNVBandwidth(t *testing.T) {
cmd := nvidiaDCGMNamedDiagCommand("nvbandwidth", 0, []int{2, 0})
want := []string{"dcgmi", "diag", "-r", "nvbandwidth", "-i", "2,0"}
want := []string{"dcgmi", "diag", "-r", "nvbandwidth", "-v", "-d", "DEBUG", "--debugLogFile", "{{run_dir}}/dcgmi-nvbandwidth-debug.log", "-i", "2,0"}
if len(cmd) != len(want) {
t.Fatalf("cmd len=%d want %d (%v)", len(cmd), len(want), cmd)
}