fix(pcie): verify GPU links under real bandwidth load

This commit is contained in:
Mikhail Chusavitin
2026-09-03 10:23:39 +03:00
parent 642e68631d
commit 347bc8310a
27 changed files with 384 additions and 902 deletions
+14 -7
View File
@@ -451,6 +451,10 @@ func (s *System) RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpu
return "", err
}
killStaleNvidiaTestWorkers(logFunc)
linkFindings, err := captureNvidiaPCIeLinkBaseline(selected)
if err != nil {
return "", err
}
jobs := []satJob{
{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
@@ -476,6 +480,7 @@ func (s *System) RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpu
collectGPU: true,
gpuIndices: selected,
syncBracket: true,
afterRun: func() { sampleNvidiaPCIeLinkAfterLoad(linkFindings) },
})
step++
} else {
@@ -495,15 +500,17 @@ func (s *System) RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpu
collectGPU: true,
gpuIndices: selected,
syncBracket: true,
afterRun: func() { sampleNvidiaPCIeLinkAfterLoad(linkFindings) },
})
step++
}
jobs = append(jobs, satJob{
name: fmt.Sprintf("%02d-nvidia-smi-after.log", step),
cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"},
})
return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-bandwidth", withNvidiaPersistenceMode(jobs...), logFunc)
runDir, err := runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-bandwidth", withNvidiaPersistenceMode(jobs...), logFunc)
if err != nil {
return "", err
}
if err := finishNvidiaPCIeLinkCheck(runDir, linkFindings); err != nil {
return "", err
}
return runDir, nil
}
func (s *System) RunNvidiaAcceptancePack(baseDir string, logFunc func(string)) (string, error) {