reanimator/bee
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Stabilize NVIDIA GPU device mapping across loaders

Browse Source
This commit is contained in:
Mikhail Chusavitin
2026-04-06 12:22:04 +03:00
parent 981315e6fd
commit 35f4c53887
6 changed files with 27 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
audit/internal/platform/benchmark.go
View File

@@ -679,7 +679,10 @@ func runBenchmarkInterconnect(ctx context.Context, verboseLog, runDir string, gp
"-g", strconv.Itoa(len(gpuIndices)), "-g", strconv.Itoa(len(gpuIndices)),
"--iters", strconv.Itoa(maxInt(20, spec.NCCLSec/10)), "--iters", strconv.Itoa(maxInt(20, spec.NCCLSec/10)),
} }
env := []string{"CUDA_VISIBLE_DEVICES=" + joinIndexList(gpuIndices)} env := []string{
"CUDA_DEVICE_ORDER=PCI_BUS_ID",
"CUDA_VISIBLE_DEVICES=" + joinIndexList(gpuIndices),
}
logFunc(fmt.Sprintf("NCCL interconnect: gpus=%s", joinIndexList(gpuIndices))) logFunc(fmt.Sprintf("NCCL interconnect: gpus=%s", joinIndexList(gpuIndices)))
out, err := runSATCommandCtx(ctx, verboseLog, "nccl-all-reduce.log", cmd, env, logFunc) out, err := runSATCommandCtx(ctx, verboseLog, "nccl-all-reduce.log", cmd, env, logFunc)
_ = os.WriteFile(filepath.Join(runDir, "nccl-all-reduce.log"), out, 0644) _ = os.WriteFile(filepath.Join(runDir, "nccl-all-reduce.log"), out, 0644)

5
audit/internal/platform/sat.go
View File

@@ -630,7 +630,10 @@ func nvidiaVisibleDevicesEnv(gpuIndices []int) []string {
if len(gpuIndices) == 0 { if len(gpuIndices) == 0 {
return nil return nil
} }
return []string{"CUDA_VISIBLE_DEVICES=" + joinIndexList(gpuIndices)} return []string{
"CUDA_DEVICE_ORDER=PCI_BUS_ID",
"CUDA_VISIBLE_DEVICES=" + joinIndexList(gpuIndices),
}
} }
func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []satJob, logFunc func(string)) (string, error) { func runAcceptancePackCtx(ctx context.Context, baseDir, prefix string, jobs []satJob, logFunc func(string)) (string, error) {

11
audit/internal/platform/sat_test.go
View File

@@ -253,11 +253,14 @@ func TestNvidiaDCGMNamedDiagCommandUsesDurationAndSelection(t *testing.T) {
func TestNvidiaVisibleDevicesEnvUsesSelectedGPUs(t *testing.T) { func TestNvidiaVisibleDevicesEnvUsesSelectedGPUs(t *testing.T) {
env := nvidiaVisibleDevicesEnv([]int{0, 2, 4}) env := nvidiaVisibleDevicesEnv([]int{0, 2, 4})
if len(env) != 1 { if len(env) != 2 {
t.Fatalf("env len=%d want 1 (%v)", len(env), env) t.Fatalf("env len=%d want 2 (%v)", len(env), env)
} }
if env[0] != "CUDA_VISIBLE_DEVICES=0,2,4" { if env[0] != "CUDA_DEVICE_ORDER=PCI_BUS_ID" {
t.Fatalf("env[0]=%q want CUDA_VISIBLE_DEVICES=0,2,4", env[0]) t.Fatalf("env[0]=%q want CUDA_DEVICE_ORDER=PCI_BUS_ID", env[0])
}
if env[1] != "CUDA_VISIBLE_DEVICES=0,2,4" {
t.Fatalf("env[1]=%q want CUDA_VISIBLE_DEVICES=0,2,4", env[1])
} }
} }

5
iso/overlay/usr/local/bin/bee-gpu-burn
View File

@@ -62,6 +62,8 @@ done
echo "loader=bee-gpu-burn" echo "loader=bee-gpu-burn"
echo "selected_gpus=${FINAL}" echo "selected_gpus=${FINAL}"
export CUDA_DEVICE_ORDER="PCI_BUS_ID"
TMP_DIR=$(mktemp -d) TMP_DIR=$(mktemp -d)
trap 'rm -rf "${TMP_DIR}"' EXIT INT TERM trap 'rm -rf "${TMP_DIR}"' EXIT INT TERM
@@ -78,7 +80,8 @@ for id in $(echo "${FINAL}" | tr ',' ' '); do
fi fi
fi fi
echo "starting gpu ${id} size=${gpu_size_mb}MB" echo "starting gpu ${id} size=${gpu_size_mb}MB"
"${WORKER}" --device "${id}" --seconds "${SECONDS}" --size-mb "${gpu_size_mb}" >"${log}" 2>&1 & CUDA_VISIBLE_DEVICES="${id}" \
"${WORKER}" --device 0 --seconds "${SECONDS}" --size-mb "${gpu_size_mb}" >"${log}" 2>&1 &
pid=$! pid=$!
WORKERS="${WORKERS} ${pid}:${id}:${log}" WORKERS="${WORKERS} ${pid}:${id}:${log}"
done done

7
iso/overlay/usr/local/bin/bee-john-gpu-stress
View File

@@ -152,14 +152,19 @@ done
[ -n "${FINAL}" ] || { echo "no NVIDIA GPUs selected after filters" >&2; exit 1; } [ -n "${FINAL}" ] || { echo "no NVIDIA GPUs selected after filters" >&2; exit 1; }
export CUDA_DEVICE_ORDER="PCI_BUS_ID"
export CUDA_VISIBLE_DEVICES="${FINAL}"
JOHN_DEVICES="" JOHN_DEVICES=""
local_id=1
for id in $(echo "${FINAL}" | tr ',' ' '); do for id in $(echo "${FINAL}" | tr ',' ' '); do
opencl_id=$((id + 1)) opencl_id="${local_id}"
if [ -z "${JOHN_DEVICES}" ]; then if [ -z "${JOHN_DEVICES}" ]; then
JOHN_DEVICES="${opencl_id}" JOHN_DEVICES="${opencl_id}"
else else
JOHN_DEVICES="${JOHN_DEVICES},${opencl_id}" JOHN_DEVICES="${JOHN_DEVICES},${opencl_id}"
fi fi
local_id=$((local_id + 1))
done done
echo "loader=john" echo "loader=john"

2
iso/overlay/usr/local/bin/bee-nccl-gpu-stress
View File

@@ -70,6 +70,8 @@ echo "gpu_count=${GPU_COUNT}"
echo "range=${MIN_BYTES}..${MAX_BYTES}" echo "range=${MIN_BYTES}..${MAX_BYTES}"
echo "iters=${ITERS}" echo "iters=${ITERS}"
export CUDA_DEVICE_ORDER="PCI_BUS_ID"
deadline=$(( $(date +%s) + SECONDS )) deadline=$(( $(date +%s) + SECONDS ))
round=0 round=0