Adding the script to build and run the rccl-tests for PTS

.jenkins/common.groovy

@@ -14,7 +14,7 @@ def runCompileCommand(platform, project, jobName)
                 ${auxiliary.exitIfNotSuccess()}
                 cd ${project.paths.project_build_prefix}
                 cmake \
-                    -DCMAKE_CXX_COMPILER=/opt/rocm/hip/bin/hipcc \
+                    -DCMAKE_CXX_COMPILER=/opt/rocm/bin/hipcc \
                     -S . -B build
                 make -C build -j\$(nproc)
                 ${auxiliary.exitIfNotSuccess()}
@@ -30,9 +30,9 @@ def runTestCommand (platform, project)
     def command = """#!/usr/bin/env bash
                 set -x
                 cd ${project.paths.project_build_prefix}
-		python3 -m pip install --upgrade pytest
-		python3 -m pytest --version
-		python3 -m pytest -k "not MPI and not host and not fine" --verbose --junitxml=./testreport.xml
+                python3 -m pip install --upgrade pytest
+                python3 -m pytest --version
+                python3 -m pytest -k "not MPI and not host and not fine" --verbose --junitxml=./testreport.xml
             """
 
    platform.runCommand(this, command)

CMakeLists.txt

@@ -1,6 +1,29 @@
 # ########################################################################
 # Copyright 2022 Advanced Micro Devices, Inc.
 # ########################################################################
+#Adding pthread flag for linking
+set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread")
+macro(check_mpi mpi_compiler mpi_lib_a mpi_lib_so mpi_bin_dir mpi_base_lib_dir mpi_inc_dir)
+    find_program(MPI_MPICXX ${mpi_compiler} PATHS ${mpi_bin_dir} NO_DEFAULT_PATH)
+    if (MPI_MPICXX)
+        message ("-- ${mpi_compiler} found @ ${MPI_MPICXX}")
+        find_file(MPI_H mpi.h PATHS ${mpi_inc_dir} NO_DEFAULT_PATH)
+        message ("-- mpi.h is in ${MPI_H}")
+        find_file(MPI_LIB NAMES ${mpi_lib_so} ${mpi_lib_a} PATHS ${mpi_base_lib_dir} PATH_SUFFIXES lib lib64 lib/x86_64-linux-gnu NO_DEFAULT_PATH)
+        message ("-- libmpi is ${MPI_LIB}")
+	if (NOT MPI_H OR NOT MPI_LIB)
+	    set (MPI_MPICXX "MPI_MPICXX-NOTFOUND")
+	    set (MPI_H "MPI_H-NOTFOUND")
+	    set (MPI_LIB "MPI_LIB-NOTFOUND")
+	else()
+            add_definitions(-DMPI_SUPPORT)
+            include_directories(${mpi_inc_dir})
+            link_libraries(${MPI_LIB})
+	endif()
+    else()
+        message ("-- ${mpi_compiler} not found")
+    endif()
+endmacro()
 
 cmake_minimum_required(VERSION 3.16.3 FATAL_ERROR)
 
@@ -30,22 +53,52 @@ include(ROCMCheckTargetIds)
 include(ROCMClients)
 
 # Build variables
-option(USE_MPI "Build RCCL-tests with MPI support. Requires the MPI path to be set.")
-set(MPI_PATH "" CACHE PATH "Path to MPI installation")
+option(NO_MPI "Build RCCL-tests without MPI support.")
+option(MPI_PATH "Use MPI in the specified directory.")
 ## Get default GPU targets using rocm_check_target_ids
 rocm_check_target_ids(
     DEFAULT_AMDGPU_TARGETS
     TARGETS "gfx803;gfx900:xnack-;gfx906:xnack-;gfx908:xnack-;gfx90a:xnack-;gfx90a:xnack+;gfx1030"
 )
 set(AMDGPU_TARGETS "${DEFAULT_AMDGPU_TARGETS}" CACHE STRING "List of specific machine types for these tests to target.")
 
-# Find the MPI package if we're using MPI
-if (USE_MPI)
-    if(NOT MPI_PATH STREQUAL "")
-        set(MPI_HOME "${MPI_PATH}")
+if (NOT NO_MPI)
+    # CHECK for MPI Path first. User requested this directory explicitely
+    if (MPI_PATH)
+        set(mpi_spec_bin_dir "${MPI_PATH}/bin")
+	set(mpi_spec_inc_dir "${MPI_PATH}/include")
+        check_mpi(mpicxx libmpi.a libmpi.so ${mpi_spec_bin_dir} ${MPI_PATH} ${mpi_spec_inc_dir})
+	if (NOT MPI_MPICXX)
+            # Since the user explicitely requested this directory, abort if something went wrong.
+	    MESSAGE(FATAL_ERROR "Could not find MPI in ${MPI_PATH}")
+        endif()
+    endif()
+
+    # Check for MPICH Ubuntu installation
+    if (NOT MPI_MPICXX)
+        check_mpi(mpicxx.mpich libmpich.a libmpich.so /usr/bin /usr /usr/include/x86_64-linux-gnu/mpich)
+    endif()
+
+    # Check for Open MPI Ubuntu installation
+    if (NOT MPI_MPICXX)
+        check_mpi(mpicxx.openmpi libmpi.a libmpi.so /usr/bin  /usr/lib/x86_64-linux-gnu/openmpi /usr/lib/x86_64-linux-gnu/openmpi/include)
+    endif()
+
+    # Check for MPICH RHEL installation
+    if (NOT MPI_MPICXX)
+        check_mpi(mpicxx libmpich.a libmpich.so /usr/lib64/mpich/bin /usr/lib64/mpich /usr/include/mpich-x86_64)
+    endif()
+
+    # Check for Open MPI RHEL installation
+    if (NOT MPI_MPICXX)
+        check_mpi(mpicxx libmpi.a libmpi.so /usr/lib64/openmpi/bin /usr/lib64/openmpi /usr/include/openmpi-x64_64)
     endif()
-    find_package(MPI REQUIRED MODULE)
-    add_definitions(-DOMPI_SKIP_MPICXX -DMPI_SUPPORT)
+
+    if (NOT MPI_MPICXX)
+        message ("-- no MPI library found")
+    endif()
+else()
+    message ("-- MPI support explicitely disabled")
 endif()
 
 set(ROCM_USE_DEV_COMPONENT OFF)  # This repo doesn't have a dev component
@@ -55,7 +108,7 @@ add_subdirectory(src)
 
 # Create ROCm standard packages
 rocm_create_package(
-    NAME rccl-separate-tests
+    NAME rccl-tests
     DESCRIPTION "Tests for the ROCm Communication Collectives Library"
     MAINTAINER "RCCL Maintainer <[email protected]>"
 )

Makefile

@@ -4,6 +4,9 @@
 # See LICENCE.txt for license information
 #
 
+BUILDDIR ?= build
+override BUILDDIR := $(abspath $(BUILDDIR))
+
 .PHONY : all clean
 
 default : src.build
@@ -14,7 +17,7 @@ all:   ${TARGETS:%=%.build}
 clean: ${TARGETS:%=%.clean}
 
 %.build:
-	${MAKE} -C $* build
+	${MAKE} -C $* build BUILDDIR=${BUILDDIR}
 
 %.clean:
-	${MAKE} -C $* clean
+	${MAKE} -C $* clean BUILDDIR=${BUILDDIR}

README.md

@@ -18,6 +18,23 @@ RCCL tests rely on MPI to work on multiple processes, hence multiple nodes. If y
 $ make MPI=1 MPI_HOME=/path/to/mpi HIP_HOME=/path/to/hip RCCL_HOME=/path/to/rccl
 ```
 
+RCCL tests can also be built using cmake. A typical sequence will be:
+
+```shell
+$ mkdir build
+$ cd build
+$ CXX=/opt/rocm/bin/hipcc cmake -DCMAKE_PREFIX_PATH=/path/to/rccl ..
+$ make
+```
+
+When using the cmake build procedure, please make sure that RCCL has also been built using cmake (i.e. not using the install.sh script), since cmake will check
+for cmake target and config files that are created during the RCCL build.
+
+Using the cmake method also has the advantage that the build is automatically checking for MPI installations, i.e. it is not necessary to explicitly request
+MPI builds. A user can request to use a particular MPI library by using the MPI_PATH variable. MPI support can be explicitely disabled by adding the -DNO_MPI=1
+flag to the cmake command line.
+
+
 ## Usage
 
 RCCL tests can run on multiple processes, multiple threads, and multiple HIP devices per thread. The number of process is managed by MPI and is therefore not passed to the tests as argument. The total number of ranks (=HIP devices) will be equal to (number of processes)\*(number of threads)\*(number of GPUs per thread).

scripts/rccl_tests_build_run_PTS.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+echo "This script is for building and running the rccl-tests as well as Unit tests"
+echo "Please ensure that the following environment variables are pointing to correct directions!"
+
+########## Set the appropriate directories ##########
+export _HIP_HOME=/opt/rocm/hip
+export _MPI_HOME=/path/to/mpi/build
+export _RCCL_HOME=/opt/rocm/rccl/build
+
+export LD_LIBRARY_PATH=$_MPI_HOME/lib:$LD_LIBRARY_PATH
+export PATH=$_MPI_HOME/bin/:$PATH
+echo "HIP_HOME=$_HIP_HOME"
+echo "MPI_HOME=$_MPI_HOME"
+echo "RCCL_HOME=$_RCCL_HOME"
+
+echo "########## Print the system information ##########"
+sudo dmidecode | grep "Product Name"
+rocm-smi --showtopo
+
+########## Set the number of GPUs ##########
+ngpus=8
+set -x
+########## Build the RCCL-tests benchmark ##########
+echo "Do you want to run tests on multiple nodes?"
+read -p '(y/n) ' RESPONSE
+if [ "$RESPONSE" = "y" ]; then
+
+        ########## MPI Installation check ##########
+        MPI_Installed=$(which mpicc)
+
+        if [ -z "$MPI_Installed" ]; then
+                echo "MPI is not installed! Install MPI and set the PATH environment variable to include PATH=/path/to/MPI-install/bin/:$PATH";
+                exit
+        else
+                cd ..
+                rm -rf rccl-tests
+                git clone https://github.com/ROCmSoftwarePlatform/rccl-tests.git
+                cd rccl-tests
+                make MPI=1 MPI_HOME=$_MPI_HOME HIP_HOME=$_HIP_HOME NCCL_HOME=$_RCCL_HOME
+        fi
+else
+        cd ..
+        rm -rf rccl-tests
+        git clone https://github.com/ROCmSoftwarePlatform/rccl-tests.git
+        cd rccl-tests
+        make HIP_HOME=$_HIP_HOME NCCL_HOME=$_RCCL_HOME
+fi       
+
+########## Run the RCCL-tests benchmark ##########
+cd build
+echo "Allreduce Test"
+./all_reduce_perf -b 8 -e 1G -f 2 -g $ngpus
+echo "Broadcast Test"
+./broadcast_perf -b 8 -e 1G -f 2 -g $ngpus
+echo "Reduce Test"
+./reduce_perf -b 8 -e 1G -f 2 -g $ngpus
+echo "Reduce_scatter Test"
+./reduce_scatter_perf -b 8 -e 1G -f 2 -g $ngpus
+echo "Allgather Test"
+./all_gather_perf -b 8 -e 1G -f 2 -g $ngpus
+echo "Send_Recv Test"
+./sendrecv_perf -b 8 -e 1G -f 2 -g $ngpus
+echo "Scatter Test"
+./scatter_perf -b 8 -e 1G -f 2 -g $ngpus
+echo "Gather Test"
+./gather_perf -b 8 -e 1G -f 2 -g $ngpus
+echo "Alltoall Test"
+./alltoall_perf -b 8 -e 1G -f 2 -g $ngpus
+echo "Alltoallv Test"
+./alltoallv_perf -b 8 -e 1G -f 2 -g $ngpus
+
+
+

src/CMakeLists.txt

@@ -3,8 +3,8 @@
 # ########################################################################
 
 # Compile common object library
-set_property(SOURCE common.cu PROPERTY LANGUAGE CXX)
-add_library(rccl_common OBJECT common.cu)
+set_property(SOURCE common.cu timer.cc ../verifiable/verifiable.cu PROPERTY LANGUAGE CXX)
+add_library(rccl_common OBJECT common.cu timer.cc ../verifiable/verifiable.cu)
 if(USE_MPI)
     target_link_libraries(rccl_common roc::rccl MPI::MPI_CXX)
 else()

src/Makefile

@@ -1,6 +1,6 @@
 #
-# Copyright (c) 2015-2021, NVIDIA CORPORATION. All rights reserved.
-# Modifications are Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved.
+# Copyright (c) 2015-2022, NVIDIA CORPORATION. All rights reserved.
+# Modifications are Copyright (c) 2019-2022 Advanced Micro Devices, Inc. All rights reserved.
 #
 # See LICENSE.txt for license information
 #
@@ -12,22 +12,21 @@ VERBOSE ?= 0
 DEBUG ?= 0
 NCCL_HOME ?= ""
 
-HIPCC = $(ROCM_PATH)/hip/bin/hipcc
+HIPCC = $(ROCM_PATH)/bin/hipcc
 CXX = $(HIPCC)
 
 HIPCUFLAGS := -std=c++14
 LDFLAGS    :=
 HIPLDFLAGS :=
 
 ifneq ($(NCCL_HOME), "")
-HIPCUFLAGS += -I$(NCCL_HOME) -I$(NCCL_HOME)/rccl/include
-HIPLDFLAGS   += -Wl,-rpath,$(NCCL_HOME) -L$(NCCL_HOME)
+HIPCUFLAGS += -I$(NCCL_HOME)/ -I$(NCCL_HOME)/include
+HIPLDFLAGS   += -Wl,-rpath,$(NCCL_HOME) -L$(NCCL_HOME) -L$(NCCL_HOME)/lib
 endif
 HIPCUFLAGS += -I$(ROCM_PATH)/include
-HIPCUFLAGS += -I$(ROCM_PATH)/include/rccl
-HIPCUFLAGS += -I$(ROCM_PATH)/hip/include/hip
+HIPCUFLAGS += -I$(ROCM_PATH)/include/hip
 LDFLAGS    += -L$(ROCM_PATH)/lib -lhsa-runtime64 -lrt
-HIPLDFLAGS += $(CUSTOM_RCCL_LIB) -L$(ROCM_PATH)/lib -lhsa-runtime64 -lrt
+HIPLDFLAGS += $(CUSTOM_RCCL_LIB) -L$(ROCM_PATH)/lib -lhsa-runtime64 -lrt -pthread
 
 ifeq ($(DEBUG), 0)
 HIPCUFLAGS += -O3
@@ -65,15 +64,23 @@ build: ${BIN_FILES}
 clean:
 	rm -rf ${DST_DIR}
 
-${DST_DIR}/%.o: %.cu common.h
+TEST_VERIFIABLE_SRCDIR := ../verifiable
+TEST_VERIFIABLE_BUILDDIR := $(BUILDDIR)/verifiable
+include ../verifiable/verifiable.mk
+
+${DST_DIR}/%.o: %.cu common.h $(TEST_VERIFIABLE_HDRS)
 	@printf "Compiling  %-35s > %s\n" $< $@
 	@mkdir -p ${DST_DIR}
 	echo "$(HIPCC) -o $@ $(HIPCUFLAGS) -c $<"
 	$(HIPCC) -o $@ $(HIPCUFLAGS) -c $<
 
-${DST_DIR}/%_perf:${DST_DIR}/%.o ${DST_DIR}/common.o
+${DST_DIR}/timer.o: timer.cc timer.h
+	@printf "Compiling  %-35s > %s\n" $< $@
+	@mkdir -p ${DST_DIR}
+	$(CXX) $(CXXFLAGS) -o $@ -c timer.cc
+
+${DST_DIR}/%_perf:${DST_DIR}/%.o ${DST_DIR}/common.o ${DST_DIR}/timer.o $(TEST_VERIFIABLE_OBJS)
 	@printf "Linking  %-35s > %s\n" $< $@
 	@mkdir -p ${DST_DIR}
 	echo "$(HIPCC) -o $@ $(HIPCUFLAGS) $^ ${HIPLDFLAGS}"
 	$(HIPCC) -o $@ $(HIPCUFLAGS) $^ ${HIPLDFLAGS}
-

src/all_gather.cu

@@ -1,31 +1,22 @@
 /*************************************************************************
- * Copyright (c) 2016-2019, NVIDIA CORPORATION. All rights reserved.
- * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved.
+ * Copyright (c) 2016-2022, NVIDIA CORPORATION. All rights reserved.
+ * Modifications Copyright (c) 2019-2022 Advanced Micro Devices, Inc. All rights reserved.
  *
  * See LICENSE.txt for license information
  ************************************************************************/
 
 #include <hip/hip_runtime.h>
 #include "common.h"
 
-void print_header() {
-  PRINT("# %10s  %12s  %8s            out-of-place                       in-place          \n", "", "", "");
-  PRINT("# %10s  %12s  %8s  %7s  %6s  %6s  %5s  %7s  %6s  %6s  %5s\n", "size", "count", "type",
-        "time", "algbw", "busbw", "error", "time", "algbw", "busbw", "error");
-  PRINT("# %10s  %12s  %8s  %7s  %6s  %6s  %5s  %7s  %6s  %6s  %5s\n", "(B)", "(elements)", "",
-        "(us)", "(GB/s)", "(GB/s)", "", "(us)", "(GB/s)", "(GB/s)", "");
-}
-
-void print_line_header (size_t size, size_t count, const char *typeName, const char *opName, int root) {
-  PRINT("%12li  %12li  %8s", size, count, typeName);
-}
+#define ALIGN 4
 
 void AllGatherGetCollByteCount(size_t *sendcount, size_t *recvcount, size_t *paramcount, size_t *sendInplaceOffset, size_t *recvInplaceOffset, size_t count, int nranks) {
-  *sendcount = count/nranks;
-  *recvcount = (count/nranks)*nranks;
-  *sendInplaceOffset = count/nranks;
+  size_t base = (count/(ALIGN*nranks))*ALIGN;
+  *sendcount = base;
+  *recvcount = base*nranks;
+  *sendInplaceOffset = base;
   *recvInplaceOffset = 0;
-  *paramcount = *sendcount;
+  *paramcount = base;
 }
 
 testResult_t AllGatherInitData(struct threadArgs* args, ncclDataType_t type, ncclRedOp_t op, int root, int rep, int in_place) {
@@ -35,18 +26,15 @@ testResult_t AllGatherInitData(struct threadArgs* args, ncclDataType_t type, ncc
 
   int k=0;
   for (int i=0; i<args->nGpus; i++) {
-    int gpuid = args->localRank*args->nThreads*args->nGpus + args->thread*args->nGpus + i;
-    if (args->enable_multiranks)
-      gpuid = gpuid % args->localNumDevices;
-    HIPCHECK(hipSetDevice(gpuid));
+    HIPCHECK(hipSetDevice(args->gpus[i]));
 
     for (int l=0; l<args->nRanks; l++) {
       int rank = ((args->proc*args->nThreads + args->thread)*args->nGpus*args->nRanks + i*args->nRanks + l);
       HIPCHECK(hipMemset(args->recvbuffs[k], 0, args->expectedBytes));
       void* data = in_place ? ((char*)args->recvbuffs[k])+rank*args->sendBytes : args->sendbuffs[k];
-      TESTCHECK(InitData(data, sendcount, type, rep, rank));
+      TESTCHECK(InitData(data, sendcount, 0, type, ncclSum, 33*rep + rank, 1, 0));
       for (int j=0; j<nranks; j++) {
-	TESTCHECK(InitData(((char*)args->expected[k])+args->sendBytes*j, sendcount, type, rep, j));
+	TESTCHECK(InitData(((char*)args->expected[k])+args->sendBytes*j, sendcount, 0, type, ncclSum, 33*rep + j, 1, 0));
       }
       k++;
     }
@@ -98,7 +86,7 @@ testResult_t AllGatherRunTest(struct threadArgs* args, int root, ncclDataType_t
   }
 
   for (int i=0; i<type_count; i++) {
-    TESTCHECK(TimeTest(args, run_types[i], run_typenames[i], (ncclRedOp_t)0, "", -1));
+    TESTCHECK(TimeTest(args, run_types[i], run_typenames[i], (ncclRedOp_t)0, "none", -1));
   }
   return testSuccess;
 }