Skip to content

Commit 90baf02

Browse files
authored
[Common] Update NCCL submodule to have the fix for MAX_SUPPORTED_TOKENS_PER_RANK (#3150)
* nccl with relax num_dispatch_tokens%64!=0 Signed-off-by: Phuong Nguyen <phuonguyen@nvidia.com> * Skip EP tests/examples on nodes without NVLink Signed-off-by: Phuong Nguyen <phuonguyen@nvidia.com> --------- Signed-off-by: Phuong Nguyen <phuonguyen@nvidia.com>
1 parent 4cd244e commit 90baf02

5 files changed

Lines changed: 23 additions & 3 deletions

File tree

3rdparty/nccl

Submodule nccl updated 477 files

examples/jax/ep/bench/run_ep_bench.sh

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,13 @@ NUM_GPUS=$(nvidia-smi -L 2>/dev/null | wc -l)
4747
if [ "${NUM_GPUS}" -lt 4 ]; then
4848
echo "EP bench requires >=4 GPUs (found ${NUM_GPUS}); SKIPPING."; exit 0
4949
fi
50+
51+
# NCCL EP requires active NVLink P2P among ranks on the node.
52+
if ! nvidia-smi nvlink --status 2>/dev/null | grep -qE 'Link [0-9]+:.*GB/s'; then
53+
echo "NVLink not detected on this platform — EP bench requires NVLink; SKIPPING."
54+
exit 0
55+
fi
56+
5057
NUM=4
5158
COORD="${COORD:-127.0.0.1:23457}"
5259
TIMEOUT_S="${TIMEOUT_S:-1800}"

tests/cpp_distributed/run_test_ep.sh

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,12 @@ if (( MIN_SM > 0 && MIN_SM < 90 )); then
3535
exit 0
3636
fi
3737

38+
# NCCL EP requires active NVLink P2P among ranks on the node.
39+
if ! nvidia-smi nvlink --status 2>/dev/null | grep -qE 'Link [0-9]+:.*GB/s'; then
40+
echo "NVLink not detected on this platform; SKIPPING."
41+
exit 0
42+
fi
43+
3844
TEST_BIN="${BUILD_DIR}/test_ep"
3945
if [[ ! -x "${TEST_BIN}" ]]; then
4046
echo "ERROR: binary not found: ${TEST_BIN}"

tests/jax/multi_process_launch_ep.sh

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,13 @@ if [ "${NUM_RUNS}" -lt 4 ]; then
3232
echo "NCCL EP requires at least 4 GPUs (found ${NUM_RUNS}); SKIPPING."
3333
exit 0
3434
fi
35+
36+
# NCCL EP requires active NVLink P2P among ranks on the node.
37+
if ! nvidia-smi nvlink --status 2>/dev/null | grep -qE 'Link [0-9]+:.*GB/s'; then
38+
echo "NVLink not detected on this platform — EP test requires NVLink; SKIPPING."
39+
exit 0
40+
fi
41+
3542
# Default test mesh is (2, 2); use exactly 4 ranks even on larger boxes.
3643
NUM_RUNS="${NVTE_TEST_EP_NUM_RANKS:-4}"
3744

tests/pytorch/distributed/run_test_ep.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,10 @@ if [ "${DETECTED_GPUS}" -lt 4 ]; then
1818
exit 0
1919
fi
2020

21-
# NCCL EP requires NVLink/NVSwitch between GPUs.
21+
# NCCL EP requires active NVLink P2P among ranks on the node.
2222
# On PCIe-only nodes (no NVLink) it falls back to the network
2323
# transport and deadlocks, so skip cleanly there.
24-
if ! nvidia-smi topo -m 2>/dev/null | grep -qE "\bNV[0-9]+\b"; then
24+
if ! nvidia-smi nvlink --status 2>/dev/null | grep -qE 'Link [0-9]+:.*GB/s'; then
2525
echo "No NVLink between GPUs (PCIe-only fabric); NCCL EP is unsupported here. SKIPPING."
2626
exit 0
2727
fi

0 commit comments

Comments
 (0)