fix(pcie): verify GPU links under real bandwidth load
This commit is contained in:
@@ -451,6 +451,10 @@ func (s *System) RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpu
|
||||
return "", err
|
||||
}
|
||||
killStaleNvidiaTestWorkers(logFunc)
|
||||
linkFindings, err := captureNvidiaPCIeLinkBaseline(selected)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
jobs := []satJob{
|
||||
{name: "01-nvidia-smi-q.log", cmd: []string{"nvidia-smi", "-q"}},
|
||||
{name: "02-dcgmi-discovery.log", cmd: []string{"dcgmi", "discovery", "-l"}, informational: true, retries: 2},
|
||||
@@ -476,6 +480,7 @@ func (s *System) RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpu
|
||||
collectGPU: true,
|
||||
gpuIndices: selected,
|
||||
syncBracket: true,
|
||||
afterRun: func() { sampleNvidiaPCIeLinkAfterLoad(linkFindings) },
|
||||
})
|
||||
step++
|
||||
} else {
|
||||
@@ -495,15 +500,17 @@ func (s *System) RunNvidiaBandwidthPack(ctx context.Context, baseDir string, gpu
|
||||
collectGPU: true,
|
||||
gpuIndices: selected,
|
||||
syncBracket: true,
|
||||
afterRun: func() { sampleNvidiaPCIeLinkAfterLoad(linkFindings) },
|
||||
})
|
||||
step++
|
||||
}
|
||||
jobs = append(jobs, satJob{
|
||||
name: fmt.Sprintf("%02d-nvidia-smi-after.log", step),
|
||||
cmd: []string{"nvidia-smi", "--query-gpu=index,name,temperature.gpu,power.draw,utilization.gpu,memory.used,memory.total", "--format=csv,noheader,nounits"},
|
||||
})
|
||||
|
||||
return runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-bandwidth", withNvidiaPersistenceMode(jobs...), logFunc)
|
||||
runDir, err := runAcceptancePackCtx(ctx, baseDir, "gpu-nvidia-bandwidth", withNvidiaPersistenceMode(jobs...), logFunc)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := finishNvidiaPCIeLinkCheck(runDir, linkFindings); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return runDir, nil
|
||||
}
|
||||
|
||||
func (s *System) RunNvidiaAcceptancePack(baseDir string, logFunc func(string)) (string, error) {
|
||||
|
||||
Reference in New Issue
Block a user