gpu topology: detect cross-NUMA-only paths between GPU pairs

Flags GPUs that reach one or more peers only via a SYS-class PCIe hop
(crossing the CPU/NUMA-node boundary) in nvidia-smi topo -m. On servers
where GPUs are only bridged pairwise via NVLink bridge (no switched
NVLink fabric), this is the exact path that traffic between different
bridge pairs has to cross, and can cut multi-GPU throughput by 2x+ for
workloads spanning more than one pair.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Mikhail Chusavitin
2026-07-03 12:22:04 +03:00
parent abaeaea13f
commit 9db651c75a
3 changed files with 175 additions and 21 deletions
@@ -80,6 +80,42 @@ func TestParseNVIDIATopologyMatrixEmpty(t *testing.T) {
}
}
func TestParseCrossNUMAPeersDetectsSYS(t *testing.T) {
t.Parallel()
// 4-GPU box, two NVLink-bridged pairs (GPU0-GPU1, GPU2-GPU3); the pairs
// themselves only reach each other via SYS (cross-NUMA PCIe hop) — the
// exact topology of a server using pairwise NVLink bridge cards instead
// of a switched NVLink fabric.
input := ` GPU0 GPU1 GPU2 GPU3
GPU0 X NV4 SYS SYS
GPU1 NV4 X SYS SYS
GPU2 SYS SYS X NV4
GPU3 SYS SYS NV4 X
`
peers := parseCrossNUMAPeers(input)
if len(peers[0]) != 2 || peers[0][0] != 2 || peers[0][1] != 3 {
t.Fatalf("peers[0]=%v want [2 3]", peers[0])
}
if len(peers[2]) != 2 {
t.Fatalf("peers[2]=%v want 2 entries", peers[2])
}
}
func TestParseCrossNUMAPeersNoSYS(t *testing.T) {
t.Parallel()
// Full NVSwitch fabric: every GPU pair connects via NVLink, no SYS hops.
input := ` GPU0 GPU1
GPU0 X NV18
GPU1 NV18 X
`
if peers := parseCrossNUMAPeers(input); peers != nil {
t.Fatalf("peers=%v want nil (no SYS pairs)", peers)
}
}
func TestApplyPCIeLinkSpeedWarningNVLinkBridgeEscalates(t *testing.T) {
t.Parallel()