Merge pull request #4923 from Le-Zheng/migrate-colab

[bigdl-2.0]Migrate colab orca quickstart

python/orca/dev/colab-notebook/ipynb2py.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+## Usage ################################
+# ./ipynb2py <file-name without extension>
+# Example:
+# ipynb2py notebooks/neural_networks/rnn
+#########################################
+if [ $# -ne "1" ]; then
+    echo "Usage: ./nb2script <file-name without extension>"
+else
+    cp $1.ipynb $1.tmp.ipynb
+    sed -i 's/%%/#/' $1.tmp.ipynb
+    sed -i 's/%pylab/#/' $1.tmp.ipynb
+
+    jupyter nbconvert $1.tmp.ipynb --to python
+
+    mv $1.tmp.py $1.py
+    sed -i '1i# -*- coding: utf-8 -*-' $1.py
+    sed -i '#!/usr/bin/python' $1.py
+    rm $1.tmp.ipynb
+fi

python/orca/dev/colab-notebook/run-notebook-pytorch.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+# orca test
+echo "orca test start"
+
+dir=${ANALYTICS_ZOO_HOME}/python/orca/colab-notebook/quickstart
+pytorchFiles=("pytorch_lenet_mnist_data_creator_func" "pytorch_lenet_mnist" "pytorch_distributed_lenet_mnist" "autoestimator_pytorch_lenet_mnist" "autoxgboost_regressor_sklearn_boston")
+index=1
+
+set -e
+
+for f in "${pytorchFiles[@]}"
+do
+
+	filename="${dir}/${f}"
+	echo "#${index} start example for ${f}"
+	#timer
+	start=$(date "+%s")
+
+	${ANALYTICS_ZOO_HOME}/python/orca/dev/colab-notebook/ipynb2py.sh ${filename}
+	sed -i "s/get_ipython()/#/g"  ${filename}.py
+	sed -i "s/import os/#import os/g" ${filename}.py
+	sed -i "s/import sys/#import sys/g" ${filename}.py
+	sed -i 's/^[^#].*environ*/#&/g' ${filename}.py
+	sed -i 's/^[^#].*__future__ */#&/g' ${filename}.py
+	sed -i "s/_ = (sys.path/#_ = (sys.path/g" ${filename}.py
+	sed -i "s/.append/#.append/g" ${filename}.py
+	sed -i 's/^[^#].*site-packages*/#&/g' ${filename}.py
+	sed -i 's/version_info/#version_info/g' ${filename}.py
+	sed -i 's/python_version/#python_version/g' ${filename}.py
+	sed -i 's/epochs=30/epochs=1/g' ${filename}.py
+
+	python ${filename}.py
+
+	now=$(date "+%s")
+	time=$((now-start))
+	echo "Complete #${index} with time ${time} seconds"	
+	index=$((index+1))
+done
+

python/orca/dev/colab-notebook/run-notebook-tf1.sh

@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+clear_up() {
+  echo "Clearing up environment. Uninstalling analytics-zoo"
+  pip uninstall -y bigdl-dllib
+  pip uninstall -y bigdl-orca
+  pip uninstall -y pyspark
+}
+
+set -e
+
+echo "#1 start test for tf_lenet_mnist.ipynb "
+#replace '!pip install --pre' to '#pip install --pre', here we test pr with built whl package. In nightly-build job, we test only use "ipython notebook" for pre-release Analytics Zoo
+start=$(date "+%s")
+${ANALYTICS_ZOO_HOME}/python/orca/dev/colab-notebook/ipynb2py.sh ${ANALYTICS_ZOO_HOME}/python/orca/colab-notebook/quickstart/tf_lenet_mnist
+sed -i '/get_ipython/s/^/#/' ${ANALYTICS_ZOO_HOME}/python/orca/colab-notebook/quickstart/tf_lenet_mnist.py
+python ${ANALYTICS_ZOO_HOME}/python/orca/colab-notebook/quickstart/tf_lenet_mnist.py
+
+exit_status=$?
+if [ $exit_status -ne 0 ]; then
+  clear_up
+  echo "tf_lenet_mnist failed"
+  exit $exit_status
+fi
+
+now=$(date "+%s")
+time1=$((now - start))
+
+echo "#2 start test for keras_lenet_mnist.ipynb "
+start=$(date "+%s")
+${ANALYTICS_ZOO_HOME}/python/orca/dev/colab-notebook/ipynb2py.sh ${ANALYTICS_ZOO_HOME}/python/orca/colab-notebook/quickstart/keras_lenet_mnist
+sed -i '/get_ipython/s/^/#/' ${ANALYTICS_ZOO_HOME}/python/orca/colab-notebook/quickstart/keras_lenet_mnist.py
+python ${ANALYTICS_ZOO_HOME}/python/orca/colab-notebook/quickstart/keras_lenet_mnist.py
+
+exit_status=$?
+if [ $exit_status -ne 0 ]; then
+  clear_up
+  echo "keras_lenet_mnist failed"
+  exit $exit_status
+fi
+
+now=$(date "+%s")
+time2=$((now - start))
+
+echo "#3 start test for ncf_xshards_pandas "
+${ANALYTICS_ZOO_HOME}/python/orca/dev/colab-notebook/ipynb2py.sh ${ANALYTICS_ZOO_HOME}/python/orca/colab-notebook/quickstart/ncf_xshards_pandas
+sed -i '/get_ipython/s/^/#/' ${ANALYTICS_ZOO_HOME}/python/orca/colab-notebook/quickstart/ncf_xshards_pandas.py
+start=$(date "+%s")
+python ${ANALYTICS_ZOO_HOME}/python/orca/colab-notebook/quickstart/ncf_xshards_pandas.py
+
+exit_status=$?
+if [ $exit_status -ne 0 ]; then
+  clear_up
+  echo "ncf_xshards_pandas failed"
+  exit $exit_status
+fi
+
+now=$(date "+%s")
+time3=$((now - start))
+
+echo "#4 start test for basic_text_classification"
+#${ANALYTICS_ZOO_HOME}/python/orca/dev/colab-notebook/ipynb2py.sh ${ANALYTICS_ZOO_HOME}/python/orca/colab-notebook/examples/basic_text_classification
+#sed -i '/get_ipython/s/^/#/' ${ANALYTICS_ZOO_HOME}/python/orca/colab-notebook/examples/basic_text_classification.py
+#start=$(date "+%s")
+#python ${ANALYTICS_ZOO_HOME}/python/orca/colab-notebook/examples/basic_text_classification.py
+
+exit_status=$?
+if [ $exit_status -ne 0 ]; then
+  clear_up
+  echo "basic_text_classification failed"
+  exit $exit_status
+fi
+
+now=$(date "+%s")
+time4=$((now - start))
+
+echo "#1 tf_lenet_mnist time used: $time1 seconds"
+echo "#2 keras_lenet_mnist time used: $time2 seconds"
+echo "#3 ncf_xshards_pandas time used: $time3 seconds"
+echo "#4 basic_text_classification time used: $time4 seconds"

python/orca/dev/colab-notebook/run-notebook-tf2.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+clear_up() {
+  echo "Clearing up environment. Uninstalling analytics-zoo"
+  pip uninstall -y bigdl-dllib
+  pip uninstall -y bigdl-orca
+  pip uninstall -y pyspark
+}
+
+set -e
+
+echo "#1 start test for tf2_lenet_mnist.ipynb"
+#replace '!pip install --pre' to '#pip install --pre', here we test pr with built whl package. In nightly-build job, we test only use "ipython notebook" for pre-release Analytics Zoo
+start=$(date "+%s")
+${ANALYTICS_ZOO_HOME}/python/orca/dev/colab-notebook/ipynb2py.sh ${ANALYTICS_ZOO_HOME}/python/orca/colab-notebook/quickstart/tf2_keras_lenet_mnist
+sed -i '/get_ipython/s/^/#/' $ANALYTICS_ZOO_HOME/python/orca/colab-notebook/quickstart/tf2_keras_lenet_mnist.py
+python ${ANALYTICS_ZOO_HOME}/python/orca/colab-notebook/quickstart/tf2_keras_lenet_mnist.py
+
+exit_status=$?
+if [ $exit_status -ne 0 ]; then
+  clear_up
+  echo "tf2_lenet_mnist failed"
+  exit $exit_status
+fi
+
+now=$(date "+%s")
+time1=$((now - start))
+
+echo "#2 start test for ncf_dataframe.ipynb"
+start=$(date "+%s")
+${ANALYTICS_ZOO_HOME}/python/orca/dev/colab-notebook/ipynb2py.sh ${ANALYTICS_ZOO_HOME}/python/orca/colab-notebook/quickstart/ncf_dataframe
+sed -i '/get_ipython/s/^/#/' ${ANALYTICS_ZOO_HOME}/python/orca/colab-notebook/quickstart/ncf_dataframe.py
+python ${ANALYTICS_ZOO_HOME}/python/orca/colab-notebook/quickstart/ncf_dataframe.py
+
+exit_status=$?
+if [ $exit_status -ne 0 ]; then
+  clear_up
+  echo "ncf_dataframe failed"
+  exit $exit_status
+fi
+
+now=$(date "+%s")
+time2=$((now - start))
+
+echo "#1 tf2_keras_lenet_mnist time used: $time1 seconds"
+echo "#2 ncf_dataframe time used: $time2 seconds"