reinstall horovod in multi-gpu gha

Author: edknv

examples/usecases/multi-gpu/install_distributed_embeddings.sh

@@ -28,7 +28,9 @@ commands =
 ; Runs in: Github Actions
 ; Runs GPU-based tests.
 allowlist_externals =
+    git
     horovodrun
+    make
 deps =
     -rrequirements/test.txt
 passenv =
@@ -37,11 +39,22 @@ setenv =
     TF_GPU_ALLOCATOR=cuda_malloc_async
 sitepackages=true
 commands =
-    # Install Merlin packages
+    # Install Merlin packages.
     python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/core.git@{posargs:main}
     python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/dataloader.git@{posargs:main}
     python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/nvtabular.git@{posargs:main}
-    # Run multi-gpu tests marked with `horovod` marker
+    # Reinstall Horovod using `pip install --no-cache-dir` to build with the new version.
+    {envdir}/env/bin/python -m pip install horovod --no-cache-dir
+    {envdir}/env/bin/horovodrun --check-build
+    # Install distributed-embeddings.
+    git clone https://github.com/NVIDIA-Merlin/distributed-embeddings.git
+    cd distributed-embeddings
+    git submodule update --init --recursive
+    make pip_pkg
+    python -m pip install --force-reinstall artifacts/*.whl
+    python setup.py install
+    python -c "import distributed_embeddings"
+    # Run multi-gpu tests marked with `horovod` marker.
     horovodrun -np 2 sh examples/usecases/multi-gpu/hvd_wrapper.sh pytest -m horovod -rxs tests/unit
 
 [testenv:py38-horovod-cpu]
@@ -58,8 +71,6 @@ commands =
     # Install horovod and check build
     {envdir}/env/bin/python -m pip install horovod --no-cache-dir
     {envdir}/env/bin/horovodrun --check-build
-    # Install distributed embeddings and check build
-    bash examples/usecases/multi-gpu/install_distributed_embeddings.sh {envtmpdir}
     # Install Merlin packages
     {envdir}/env/bin/python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/core.git
     {envdir}/env/bin/python -m pip install --upgrade git+https://github.com/NVIDIA-Merlin/dataloader.git