diff --git a/audit/internal/platform/benchmark.go b/audit/internal/platform/benchmark.go index 372e3f6..f308727 100644 --- a/audit/internal/platform/benchmark.go +++ b/audit/internal/platform/benchmark.go @@ -679,7 +679,10 @@ func runBenchmarkInterconnect(ctx context.Context, verboseLog, runDir string, gp "-g", strconv.Itoa(len(gpuIndices)), "--iters", strconv.Itoa(maxInt(20, spec.NCCLSec/10)), } - env := []string{"CUDA_VISIBLE_DEVICES=" + joinIndexList(gpuIndices)} + env := []string{ + "CUDA_DEVICE_ORDER=PCI_BUS_ID", + "CUDA_VISIBLE_DEVICES=" + joinIndexList(gpuIndices), + } logFunc(fmt.Sprintf("NCCL interconnect: gpus=%s", joinIndexList(gpuIndices))) out, err := runSATCommandCtx(ctx, verboseLog, "nccl-all-reduce.log", cmd, env, logFunc) _ = os.WriteFile(filepath.Join(runDir, "nccl-all-reduce.log"), out, 0644) diff --git a/audit/internal/platform/sat.go b/audit/internal/platform/sat.go index 0a41525..29c4037 100644 --- a/audit/internal/platform/sat.go +++ b/audit/internal/platform/sat.go @@ -630,7 +630,10 @@ func nvidiaVisibleDevicesEnv(gpuIndices []int) []string { if len(gpuIndices) == 0 { return nil } - return []string{"CUDA_VISIBLE_DEVICES=" + joinIndexList(gpuIndices)} + return []string{ + "CUDA_DEVICE_ORDER=PCI_BUS_ID", + "CUDA_VISIBLE_DEVICES=" + joinIndexList(gpuIndices), + } } func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []satJob, logFunc func(string)) (string, error) { diff --git a/audit/internal/platform/sat_test.go b/audit/internal/platform/sat_test.go index 5aab064..7f0475a 100644 --- a/audit/internal/platform/sat_test.go +++ b/audit/internal/platform/sat_test.go @@ -253,11 +253,14 @@ func TestNvidiaDCGMNamedDiagCommandUsesDurationAndSelection(t *testing.T) { func TestNvidiaVisibleDevicesEnvUsesSelectedGPUs(t *testing.T) { env := nvidiaVisibleDevicesEnv([]int{0, 2, 4}) - if len(env) != 1 { - t.Fatalf("env len=%d want 1 (%v)", len(env), env) + if len(env) != 2 { + t.Fatalf("env len=%d want 2 (%v)", len(env), env) } - if env[0] != "CUDA_VISIBLE_DEVICES=0,2,4" { - t.Fatalf("env[0]=%q want CUDA_VISIBLE_DEVICES=0,2,4", env[0]) + if env[0] != "CUDA_DEVICE_ORDER=PCI_BUS_ID" { + t.Fatalf("env[0]=%q want CUDA_DEVICE_ORDER=PCI_BUS_ID", env[0]) + } + if env[1] != "CUDA_VISIBLE_DEVICES=0,2,4" { + t.Fatalf("env[1]=%q want CUDA_VISIBLE_DEVICES=0,2,4", env[1]) } } diff --git a/iso/overlay/usr/local/bin/bee-gpu-burn b/iso/overlay/usr/local/bin/bee-gpu-burn index 00e0b18..899e4af 100644 --- a/iso/overlay/usr/local/bin/bee-gpu-burn +++ b/iso/overlay/usr/local/bin/bee-gpu-burn @@ -62,6 +62,8 @@ done echo "loader=bee-gpu-burn" echo "selected_gpus=${FINAL}" +export CUDA_DEVICE_ORDER="PCI_BUS_ID" + TMP_DIR=$(mktemp -d) trap 'rm -rf "${TMP_DIR}"' EXIT INT TERM @@ -78,7 +80,8 @@ for id in $(echo "${FINAL}" | tr ',' ' '); do fi fi echo "starting gpu ${id} size=${gpu_size_mb}MB" - "${WORKER}" --device "${id}" --seconds "${SECONDS}" --size-mb "${gpu_size_mb}" >"${log}" 2>&1 & + CUDA_VISIBLE_DEVICES="${id}" \ + "${WORKER}" --device 0 --seconds "${SECONDS}" --size-mb "${gpu_size_mb}" >"${log}" 2>&1 & pid=$! WORKERS="${WORKERS} ${pid}:${id}:${log}" done diff --git a/iso/overlay/usr/local/bin/bee-john-gpu-stress b/iso/overlay/usr/local/bin/bee-john-gpu-stress index f3803bb..24ac6a5 100644 --- a/iso/overlay/usr/local/bin/bee-john-gpu-stress +++ b/iso/overlay/usr/local/bin/bee-john-gpu-stress @@ -152,14 +152,19 @@ done [ -n "${FINAL}" ] || { echo "no NVIDIA GPUs selected after filters" >&2; exit 1; } +export CUDA_DEVICE_ORDER="PCI_BUS_ID" +export CUDA_VISIBLE_DEVICES="${FINAL}" + JOHN_DEVICES="" +local_id=1 for id in $(echo "${FINAL}" | tr ',' ' '); do - opencl_id=$((id + 1)) + opencl_id="${local_id}" if [ -z "${JOHN_DEVICES}" ]; then JOHN_DEVICES="${opencl_id}" else JOHN_DEVICES="${JOHN_DEVICES},${opencl_id}" fi + local_id=$((local_id + 1)) done echo "loader=john" diff --git a/iso/overlay/usr/local/bin/bee-nccl-gpu-stress b/iso/overlay/usr/local/bin/bee-nccl-gpu-stress index 92ca022..fd412cb 100644 --- a/iso/overlay/usr/local/bin/bee-nccl-gpu-stress +++ b/iso/overlay/usr/local/bin/bee-nccl-gpu-stress @@ -70,6 +70,8 @@ echo "gpu_count=${GPU_COUNT}" echo "range=${MIN_BYTES}..${MAX_BYTES}" echo "iters=${ITERS}" +export CUDA_DEVICE_ORDER="PCI_BUS_ID" + deadline=$(( $(date +%s) + SECONDS )) round=0