diff --git a/README.md b/README.md index 03e20dbc..480f6fbd 100644 --- a/README.md +++ b/README.md @@ -184,6 +184,49 @@ horovodrun -np 32 -H :8,:8,:8,:8 * ImageNet Dataset * Horovod +## Installation + +The first thing to do is install [OpenMPI](http://www.open-mpi.org), see the [OpenMPI installation guide](https://edu.itp.phys.ethz.ch/hs12/programming_techniques/openmpi.pdf). + +We need to install NCC as well: + +We also may need to set the following environment variables to fit your CUDA environment: + +```bash + export CUDA=/usr/local/cuda + export HOROVOD_CUDA_HOME=/usr/local/cuda + export CUDACXX=/usr/local/cuda/bin/nvcc + export CUDA_INCLUDE_DIRS=/usr/local/cuda + export LD_LIBRARY_PATH=$CUDA/lib64:$LD_LIBRARY_PATH + export PATH=$CUDA/bin:$PATH + + export HOROVOD_WITH_PYTORCH=1 + export HOROVOD_NCCL_INCLUDE=/usr/include + export HOROVOD_GPU=CUDA + export HOROVOD_WITHOUT_TENSORFLOW=1 + export HOROVOD_WITHOUT_MXNET=1 + export HOROVOD_WITHOUT_GLOO=1 + export HOROVOD_GPU_OPERATIONS=NCCL +``` + +Next, go ahead and run: + +```bash + conda env create -f conda-gpu.yml --force +``` + +This enables Horovod with GPU support. + +Next activate the environment: + +```bash + conda activate ofa-gpu + pip install . +``` + +You can now run ofa with GPU support. + + ## Related work on automated and efficient deep learning: [ProxylessNAS: Direct Neural Architecture Search on Target Task and Hardware](https://arxiv.org/pdf/1812.00332.pdf) (ICLR’19) diff --git a/conda-cpu.yml b/conda-cpu.yml new file mode 100644 index 00000000..2268f55c --- /dev/null +++ b/conda-cpu.yml @@ -0,0 +1,20 @@ +name: ofa-cpu + +channels: + - pytorch +dependencies: + - python==3.7 + - pip + - matplotlib + - opencv + - pip: + - numpy + - torch==1.8.1 + - torchaudio==0.8.1 + - torchvision==0.9.1 + - torchsummary + - filelock + - PyYaml + - tqdm + - thop + - -e . diff --git a/conda-gpu.yml b/conda-gpu.yml new file mode 100644 index 00000000..b15983d5 --- /dev/null +++ b/conda-gpu.yml @@ -0,0 +1,36 @@ +name: ofa-gpu + +channels: + - pytorch + - conda-forge + - defaults +dependencies: + - python==3.7 + - ccache + - cmake + - cudatoolkit=10.1 + - cudnn=7.6.5 + - cxx-compiler + - gcc_linux-64 + - gxx_linux-64 + - jupyterlab + - mpi4py # installs cuda-aware openmpi + - nccl + - nvcc_linux-64=10.1 + - openmpi + - pip + - matplotlib + - opencv + - pip: + - numpy + - torch==1.8.1 + - torchaudio==0.8.1 + - torchvision==0.9.1 + - torchsummary + - filelock + - PyYaml + - tqdm + - thop + - horovod==0.19.* + - -e . + - --no-binary=horovod diff --git a/ofa/utils/my_dataloader/my_random_resize_crop.py b/ofa/utils/my_dataloader/my_random_resize_crop.py index 1b484756..330dc13a 100644 --- a/ofa/utils/my_dataloader/my_random_resize_crop.py +++ b/ofa/utils/my_dataloader/my_random_resize_crop.py @@ -6,6 +6,7 @@ import torchvision.transforms.functional as F import torchvision.transforms as transforms +from torchvision.transforms import InterpolationMode __all__ = ['MyRandomResizedCrop', 'MyResizeRandomCrop', 'MyResize'] @@ -69,7 +70,11 @@ def sample_image_size(batch_id=None): MyRandomResizedCrop.ACTIVE_SIZE = random.choices(candidate_sizes, weights=relative_probs)[0] def __repr__(self): - interpolate_str = _pil_interpolation_to_str[self.interpolation] + if isinstance(self.interpolation, InterpolationMode): + interpolate_str = self.interpolation.value + else: + interpolate_str = _pil_interpolation_to_str[self.interpolation] + format_string = self.__class__.__name__ + '(size={0}'.format(MyRandomResizedCrop.IMAGE_SIZE_LIST) if MyRandomResizedCrop.CONTINUOUS: format_string += '@continuous' @@ -113,9 +118,14 @@ def __call__(self, img): return F.crop(img, i, j, h, w) def __repr__(self): + if isinstance(self.interpolation, InterpolationMode): + interpolate_str = self.interpolation.value + else: + interpolate_str = _pil_interpolation_to_str[self.interpolation] + return 'MyResizeRandomCrop(size=%s%s, interpolation=%s, use_padding=%s, fill=%s)' % ( MyRandomResizedCrop.IMAGE_SIZE_LIST, '@continuous' if MyRandomResizedCrop.CONTINUOUS else '', - _pil_interpolation_to_str[self.interpolation], self.use_padding, self.fill, + interpolate_str, self.use_padding, self.fill, ) @@ -130,7 +140,12 @@ def __call__(self, img): return img def __repr__(self): + if isinstance(self.interpolation, InterpolationMode): + interpolate_str = self.interpolation.value + else: + interpolate_str = _pil_interpolation_to_str[self.interpolation] + return 'MyResize(size=%s%s, interpolation=%s)' % ( MyRandomResizedCrop.IMAGE_SIZE_LIST, '@continuous' if MyRandomResizedCrop.CONTINUOUS else '', - _pil_interpolation_to_str[self.interpolation] - ) + interpolate_str + ) \ No newline at end of file