Update package versions, small enhancements

Author: tlkh

build.sh

@@ -7,27 +7,27 @@ cd src
 echo -e "\nBuilding images\n"
 
 docker build . -f base.Dockerfile -t nvaitc/ai-lab:$TAG-base
-docker build . -f tf.Dockerfile -t nvaitc/ai-lab:$TAG-tf
+#docker build . -f tf.Dockerfile -t nvaitc/ai-lab:$TAG-tf
 docker build . -f tf2.Dockerfile -t nvaitc/ai-lab:$TAG-tf2
 docker build . -f full.Dockerfile -t nvaitc/ai-lab:$TAG
 docker build . -f vnc.Dockerfile -t nvaitc/ai-lab:$TAG-vnc
 
 docker build . -f base.Dockerfile -t nvcr.io/nvidian/sae/ai-lab:$TAG-base
-docker build . -f tf.Dockerfile -t nvcr.io/nvidian/sae/ai-lab:$TAG-tf
+#docker build . -f tf.Dockerfile -t nvcr.io/nvidian/sae/ai-lab:$TAG-tf
 docker build . -f tf2.Dockerfile -t nvcr.io/nvidian/sae/ai-lab:$TAG-tf2
 docker build . -f full.Dockerfile -t nvcr.io/nvidian/sae/ai-lab:$TAG
 docker build . -f vnc.Dockerfile -t nvcr.io/nvidian/sae/ai-lab:$TAG-vnc
 
 echo -e "\nPushing images\n"
 
 docker push nvaitc/ai-lab:$TAG-base
-docker push nvaitc/ai-lab:$TAG-tf
+#docker push nvaitc/ai-lab:$TAG-tf
 docker push nvaitc/ai-lab:$TAG-tf2
 docker push nvaitc/ai-lab:$TAG
 docker push nvaitc/ai-lab:$TAG-vnc
 
 docker push nvcr.io/nvidian/sae/ai-lab:$TAG-base
-docker push nvcr.io/nvidian/sae/ai-lab:$TAG-tf
+#docker push nvcr.io/nvidian/sae/ai-lab:$TAG-tf
 docker push nvcr.io/nvidian/sae/ai-lab:$TAG-tf2
 docker push nvcr.io/nvidian/sae/ai-lab:$TAG
 docker push nvcr.io/nvidian/sae/ai-lab:$TAG-vnc

src/base-batch.Dockerfile

@@ -117,7 +117,7 @@ RUN cd /tmp/ && \
     conda install --quiet --yes \
       -c nvidia -c numba -c pytorch -c conda-forge -c rapidsai -c defaults \
       'python=3.6' \
-      'numpy=1.16.1' \
+      'numpy' \
       'pandas' \
       'cudatoolkit=10.1' \
       'pytorch' \
@@ -187,7 +187,7 @@ RUN conda install \
       -c rapidsai/label/cuda10.1 \
       -c numba -c conda-forge -c defaults \
       'python=3.6' \
-      'numpy=1.16.1' \
+      'numpy' \
       'dask' \
       'cudf' \
       'cuml' \

src/base.Dockerfile

@@ -122,21 +122,25 @@ RUN cd /tmp/ && \
     conda install --quiet --yes \
       -c nvidia -c numba -c pytorch -c conda-forge -c rapidsai -c defaults \
       'python=3.6' \
-      'numpy=1.16.1' \
+      'numpy' \
       'pandas' \
       'cudatoolkit=10.1' \
       'tk' \
       'tini' \
       'blas=*=openblas' && \
     conda install --quiet --yes \
-      'notebook=5.7.*' \
+      'notebook=6.0.*' \
       'jupyterhub=1.0.*' \
       'jupyterlab=1.*' \
       'widgetsnbextension' \
       'jupyter_contrib_nbextensions' \
       'ipywidgets=7.5.*' && \
     pip install --no-cache-dir -r $HOME/requirements.txt && \
     rm $HOME/requirements.txt && \
+    git clone --depth 1 https://github.com/huggingface/neuralcoref && \
+    cd neuralcoref && \
+    pip install --no-cache-dir -r requirements.txt && \
+    pip install --no-cache-dir . && \
     cd $HOME && \
     pip uninstall opencv-python opencv-contrib-python -y && \
     pip install --no-cache-dir opencv-contrib-python && \
@@ -145,22 +149,25 @@ RUN cd /tmp/ && \
     jupyter notebook --generate-config && \
     jupyter nbextension enable --py widgetsnbextension --sys-prefix && \
     jupyter contrib nbextension install --sys-prefix && \
+    echo "Installing System Monitor" && \
+     pip --no-cache-dir install nbresuse && \
+     jupyter labextension install jupyterlab-topbar-extension jupyterlab-system-monitor && \
     echo "Installing @jupyter-widgets/jupyterlab-manager" && \
-    jupyter labextension install @jupyter-widgets/jupyterlab-manager && \
+     jupyter labextension install @jupyter-widgets/jupyterlab-manager && \
     echo "Installing @jupyterlab/toc" && \
-    jupyter labextension install @jupyterlab/toc && \
+     jupyter labextension install @jupyterlab/toc && \
     echo "Installing @jupyterlab/git" && \
-    jupyter labextension install @jupyterlab/git && \
-    pip install --no-cache-dir --upgrade jupyterlab-git && \
-    jupyter serverextension enable --py --sys-prefix jupyterlab_git && \
+     jupyter labextension install @jupyterlab/git && \
+     pip install --no-cache-dir --upgrade jupyterlab-git && \
+     jupyter serverextension enable --py --sys-prefix jupyterlab_git && \
     jupyter labextension install jupyterlab_bokeh && \
     echo "Installing jupyterlab-server-proxy" && \
-    cd /tmp/ && \
-    git clone --depth 1  https://github.com/qrtt1/jupyter_tensorboard && \
-    git clone --depth 1 https://github.com/jupyterhub/jupyter-server-proxy && \
-    cd /tmp/jupyter-server-proxy/jupyterlab-server-proxy && \
-    npm install && npm run build && jupyter labextension link . && \
-    npm run build && jupyter lab build && \
+     cd /tmp/ && \
+     git clone --depth 1  https://github.com/qrtt1/jupyter_tensorboard && \
+     git clone --depth 1 https://github.com/jupyterhub/jupyter-server-proxy && \
+     cd /tmp/jupyter-server-proxy/jupyterlab-server-proxy && \
+     npm install && npm run build && jupyter labextension link . && \
+     npm run build && jupyter lab build && \
     cd /tmp/jupyter_tensorboard && \
     pip --no-cache-dir install . && \
     jupyter labextension install jupyterlab_tensorboard && \
@@ -227,7 +234,8 @@ COPY ld.so.conf /etc/
 ENV NB_PASSWD="" \
     SUDO_PASSWD=volta
 
-RUN mkdir /results/ && \
+RUN ldconfig && \
+    mkdir /results/ && \
     chmod -R 777 /results/ && \
     echo "${SUDO_PASSWD}\n${SUDO_PASSWD}\n" | (passwd $NB_USER)
 

src/batch-tf2.Dockerfile

@@ -12,8 +12,8 @@ USER root
 ENV DEBIAN_FRONTEND=noninteractive \
     TF_FORCE_GPU_ALLOW_GROWTH=true
 
-ENV TENSORFLOW_URL=https://github.com/tlkh/getcuda/releases/download/0c/tensorflow-2.0.0-cp36-cp36m-linux_x86_64.whl \
-    TENSORFLOW_FILENAME=tensorflow-2.0.0-cp36-cp36m-linux_x86_64.whl
+ENV TENSORFLOW_URL=https://github.com/tlkh/getcuda/releases/download/0e/tensorflow-2.1.0-cp36-cp36m-linux_x86_64.whl \
+    TENSORFLOW_FILENAME=tensorflow-2.1.0-cp36-cp36m-linux_x86_64.whl
 
 RUN cd /tmp/ && \
     wget -O ${TENSORFLOW_FILENAME} ${TENSORFLOW_URL} && \

src/full.Dockerfile

@@ -14,14 +14,12 @@ USER $NB_UID
 
 RUN conda install -c pytorch --quiet --yes \
       'python=3.6' \
-      'numpy=1.16.1' \
       'pytorch' \
       'torchvision' \
       'cudatoolkit=10.1' && \
     pip install --no-cache-dir torchtext && \
     conda install -c pytorch -c fastai --quiet --yes \
       'python=3.6' \
-      'numpy=1.16.1' \
       'fastai' \
       'dataclasses' && \
     pip uninstall pillow -y && \
@@ -75,19 +73,12 @@ RUN apt-get update && \
 USER $NB_UID
 
 RUN conda install \
-      -c nvidia/label/cuda10.1 \
-      -c rapidsai/label/cuda10.1 \
+      -c nvidia \
+      -c rapidsai \
       -c numba -c conda-forge -c defaults \
       'python=3.6' \
-      'numpy=1.16.1' \
-      'dask' \
-      'cudf' \
-      'cuml' \
-      'cugraph' \
-      'dask-cuda' \
-      'dask-cudf' \
-      'dask-cuml' \
-      'nvstrings' && \
+      'rapids=0.11' \
+      'cudatoolkit=10.1' && \
     conda install \
       -c rapidsai/label/xgboost \
       'xgboost' \
@@ -122,6 +113,7 @@ RUN apt-get update && \
       cuda-libraries-dev-$CUDA_PKG_VERSION \
       cuda-minimal-build-$CUDA_PKG_VERSION \
       libnccl-dev=$NCCL_VERSION-1+cuda10.1 && \
+    pip uninstall horovod -y && \
     pip install --no-cache-dir horovod && \
     apt-get remove -yq \
       cuda-nvml-dev-$CUDA_PKG_VERSION \

src/tf2.Dockerfile

@@ -30,13 +30,12 @@ RUN apt-get update && \
 
 USER $NB_UID
 
-ENV TENSORFLOW_URL=https://github.com/tlkh/getcuda/releases/download/0c/tensorflow-2.0.0-cp36-cp36m-linux_x86_64.whl \
-    TENSORFLOW_FILENAME=tensorflow-2.0.0-cp36-cp36m-linux_x86_64.whl
+ENV TENSORFLOW_URL=https://github.com/tlkh/getcuda/releases/download/0e/tensorflow-2.1.0-cp36-cp36m-linux_x86_64.whl \
+    TENSORFLOW_FILENAME=tensorflow-2.1.0-cp36-cp36m-linux_x86_64.whl
 
 RUN cd $HOME/ && \
     conda install -c pytorch --quiet --yes \
       'python=3.6' \
-      'numpy=1.16.1' \
       'pytorch' \
       'cudatoolkit=10.1' && \
     echo -c "Downloading ${TENSORFLOW_FILENAME} from ${TENSORFLOW_URL}" && \

src/vnc.Dockerfile

@@ -52,6 +52,7 @@ RUN apt-get update && \
     xkb-data \
     x11proto-gl-dev \
     # install Nsight profiling tools 
+    libqt5x11extras5 \
     openjdk-8-jre \
     cuda-visual-tools-10-0 \
     cuda-nsight-systems-10-1 \
@@ -95,12 +96,12 @@ RUN apt-get update && \
     add-apt-repository ppa:ubuntu-desktop/ubuntu-make -y && \
     apt-get update && \
     apt-get install --no-upgrade -yq \
-    xvfb libosmesa6-dev mesa-utils \
+    xvfb libosmesa6-dev mesa-utils libgles2-mesa \
     mesa-common-dev libgl1-mesa-dev freeglut3-dev libglu1-mesa-dev \
     novnc supervisor xinit ubuntu-make \
     xubuntu-desktop idle3 && \
     apt-get purge -yq \
-    libreoffice* thunderbird* pidgin* sgt-puzzles* \
+    libreoffice* thunderbird* pidgin* sgt-puzzles* xscreensaver \
     gnome* blueman* bluez* unity* cups* totem* xfce4-dict* \
     empathy* evolution* rhythmbox* shotwell* xfburn* \
     account-plugin-* example-content* duplicity* \
@@ -135,14 +136,16 @@ WORKDIR /home/$NB_USER
 USER $NB_USER
 
 RUN pip install --no-cache-dir \
-    keras-rl pyopengl \
-    gym[atari] ray[rllib] roboschool \
+    pyopengl gym[atari] \
     jupyter-vscode-server jedi pysc2 \
-    python-language-server[yapf] \
-    setuptools wheel && \
+    python-language-server[yapf] && \
     pip uninstall opencv-python opencv-python-headless opencv-contrib-python -y && \
     pip install --no-cache-dir opencv-contrib-python -U && \ 
-    rm -rf /tmp/* && \
+    cd /tmp/ && \
+    git clone --depth 1 https://github.com/tlkh/keras-rl2.git && \
+    cd keras-rl2 && \
+    pip install --no-cache-dir . && \
+    rm -rf /tmp/* && cd && \
     rm -rf $HOME/.cache && \
     rm -rf $HOME/.node-gyp && \
     fix-permissions $CONDA_DIR && \