Skip to content

Commit 2b70433

Browse files
authored
Merge pull request #4923 from Le-Zheng/migrate-colab
[bigdl-2.0]Migrate colab orca quickstart
2 parents a04c53c + f47734a commit 2b70433

14 files changed

+4794
-0
lines changed

python/orca/colab-notebook/quickstart/autoestimator_pytorch_lenet_mnist.ipynb

Lines changed: 572 additions & 0 deletions
Large diffs are not rendered by default.

python/orca/colab-notebook/quickstart/autoxgboost_regressor_sklearn_boston.ipynb

Lines changed: 517 additions & 0 deletions
Large diffs are not rendered by default.

python/orca/colab-notebook/quickstart/keras_lenet_mnist.ipynb

Lines changed: 377 additions & 0 deletions
Large diffs are not rendered by default.

python/orca/colab-notebook/quickstart/ncf_dataframe.ipynb

Lines changed: 409 additions & 0 deletions
Large diffs are not rendered by default.

python/orca/colab-notebook/quickstart/ncf_xshards_pandas.ipynb

Lines changed: 477 additions & 0 deletions
Large diffs are not rendered by default.

python/orca/colab-notebook/quickstart/pytorch_distributed_lenet_mnist.ipynb

Lines changed: 500 additions & 0 deletions
Large diffs are not rendered by default.

python/orca/colab-notebook/quickstart/pytorch_lenet_mnist.ipynb

Lines changed: 493 additions & 0 deletions
Large diffs are not rendered by default.

python/orca/colab-notebook/quickstart/pytorch_lenet_mnist_data_creator_func.ipynb

Lines changed: 485 additions & 0 deletions
Large diffs are not rendered by default.

python/orca/colab-notebook/quickstart/tf2_keras_lenet_mnist.ipynb

Lines changed: 381 additions & 0 deletions
Large diffs are not rendered by default.

python/orca/colab-notebook/quickstart/tf_lenet_mnist.ipynb

Lines changed: 398 additions & 0 deletions
Large diffs are not rendered by default.
Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#!/bin/bash
2+
3+
## Usage ################################
4+
# ./ipynb2py <file-name without extension>
5+
# Example:
6+
# ipynb2py notebooks/neural_networks/rnn
7+
#########################################
8+
if [ $# -ne "1" ]; then
9+
echo "Usage: ./nb2script <file-name without extension>"
10+
else
11+
cp $1.ipynb $1.tmp.ipynb
12+
sed -i 's/%%/#/' $1.tmp.ipynb
13+
sed -i 's/%pylab/#/' $1.tmp.ipynb
14+
15+
jupyter nbconvert $1.tmp.ipynb --to python
16+
17+
mv $1.tmp.py $1.py
18+
sed -i '1i# -*- coding: utf-8 -*-' $1.py
19+
sed -i '#!/usr/bin/python' $1.py
20+
rm $1.tmp.ipynb
21+
fi
Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
#!/bin/bash
2+
3+
# orca test
4+
echo "orca test start"
5+
6+
dir=${ANALYTICS_ZOO_HOME}/python/orca/colab-notebook/quickstart
7+
pytorchFiles=("pytorch_lenet_mnist_data_creator_func" "pytorch_lenet_mnist" "pytorch_distributed_lenet_mnist" "autoestimator_pytorch_lenet_mnist" "autoxgboost_regressor_sklearn_boston")
8+
index=1
9+
10+
set -e
11+
12+
for f in "${pytorchFiles[@]}"
13+
do
14+
15+
filename="${dir}/${f}"
16+
echo "#${index} start example for ${f}"
17+
#timer
18+
start=$(date "+%s")
19+
20+
${ANALYTICS_ZOO_HOME}/python/orca/dev/colab-notebook/ipynb2py.sh ${filename}
21+
sed -i "s/get_ipython()/#/g" ${filename}.py
22+
sed -i "s/import os/#import os/g" ${filename}.py
23+
sed -i "s/import sys/#import sys/g" ${filename}.py
24+
sed -i 's/^[^#].*environ*/#&/g' ${filename}.py
25+
sed -i 's/^[^#].*__future__ */#&/g' ${filename}.py
26+
sed -i "s/_ = (sys.path/#_ = (sys.path/g" ${filename}.py
27+
sed -i "s/.append/#.append/g" ${filename}.py
28+
sed -i 's/^[^#].*site-packages*/#&/g' ${filename}.py
29+
sed -i 's/version_info/#version_info/g' ${filename}.py
30+
sed -i 's/python_version/#python_version/g' ${filename}.py
31+
sed -i 's/epochs=30/epochs=1/g' ${filename}.py
32+
33+
python ${filename}.py
34+
35+
now=$(date "+%s")
36+
time=$((now-start))
37+
echo "Complete #${index} with time ${time} seconds"
38+
index=$((index+1))
39+
done
40+
Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
#!/usr/bin/env bash
2+
clear_up() {
3+
echo "Clearing up environment. Uninstalling analytics-zoo"
4+
pip uninstall -y bigdl-dllib
5+
pip uninstall -y bigdl-orca
6+
pip uninstall -y pyspark
7+
}
8+
9+
set -e
10+
11+
echo "#1 start test for tf_lenet_mnist.ipynb "
12+
#replace '!pip install --pre' to '#pip install --pre', here we test pr with built whl package. In nightly-build job, we test only use "ipython notebook" for pre-release Analytics Zoo
13+
start=$(date "+%s")
14+
${ANALYTICS_ZOO_HOME}/python/orca/dev/colab-notebook/ipynb2py.sh ${ANALYTICS_ZOO_HOME}/python/orca/colab-notebook/quickstart/tf_lenet_mnist
15+
sed -i '/get_ipython/s/^/#/' ${ANALYTICS_ZOO_HOME}/python/orca/colab-notebook/quickstart/tf_lenet_mnist.py
16+
python ${ANALYTICS_ZOO_HOME}/python/orca/colab-notebook/quickstart/tf_lenet_mnist.py
17+
18+
exit_status=$?
19+
if [ $exit_status -ne 0 ]; then
20+
clear_up
21+
echo "tf_lenet_mnist failed"
22+
exit $exit_status
23+
fi
24+
25+
now=$(date "+%s")
26+
time1=$((now - start))
27+
28+
echo "#2 start test for keras_lenet_mnist.ipynb "
29+
start=$(date "+%s")
30+
${ANALYTICS_ZOO_HOME}/python/orca/dev/colab-notebook/ipynb2py.sh ${ANALYTICS_ZOO_HOME}/python/orca/colab-notebook/quickstart/keras_lenet_mnist
31+
sed -i '/get_ipython/s/^/#/' ${ANALYTICS_ZOO_HOME}/python/orca/colab-notebook/quickstart/keras_lenet_mnist.py
32+
python ${ANALYTICS_ZOO_HOME}/python/orca/colab-notebook/quickstart/keras_lenet_mnist.py
33+
34+
exit_status=$?
35+
if [ $exit_status -ne 0 ]; then
36+
clear_up
37+
echo "keras_lenet_mnist failed"
38+
exit $exit_status
39+
fi
40+
41+
now=$(date "+%s")
42+
time2=$((now - start))
43+
44+
echo "#3 start test for ncf_xshards_pandas "
45+
${ANALYTICS_ZOO_HOME}/python/orca/dev/colab-notebook/ipynb2py.sh ${ANALYTICS_ZOO_HOME}/python/orca/colab-notebook/quickstart/ncf_xshards_pandas
46+
sed -i '/get_ipython/s/^/#/' ${ANALYTICS_ZOO_HOME}/python/orca/colab-notebook/quickstart/ncf_xshards_pandas.py
47+
start=$(date "+%s")
48+
python ${ANALYTICS_ZOO_HOME}/python/orca/colab-notebook/quickstart/ncf_xshards_pandas.py
49+
50+
exit_status=$?
51+
if [ $exit_status -ne 0 ]; then
52+
clear_up
53+
echo "ncf_xshards_pandas failed"
54+
exit $exit_status
55+
fi
56+
57+
now=$(date "+%s")
58+
time3=$((now - start))
59+
60+
echo "#4 start test for basic_text_classification"
61+
#${ANALYTICS_ZOO_HOME}/python/orca/dev/colab-notebook/ipynb2py.sh ${ANALYTICS_ZOO_HOME}/python/orca/colab-notebook/examples/basic_text_classification
62+
#sed -i '/get_ipython/s/^/#/' ${ANALYTICS_ZOO_HOME}/python/orca/colab-notebook/examples/basic_text_classification.py
63+
#start=$(date "+%s")
64+
#python ${ANALYTICS_ZOO_HOME}/python/orca/colab-notebook/examples/basic_text_classification.py
65+
66+
exit_status=$?
67+
if [ $exit_status -ne 0 ]; then
68+
clear_up
69+
echo "basic_text_classification failed"
70+
exit $exit_status
71+
fi
72+
73+
now=$(date "+%s")
74+
time4=$((now - start))
75+
76+
echo "#1 tf_lenet_mnist time used: $time1 seconds"
77+
echo "#2 keras_lenet_mnist time used: $time2 seconds"
78+
echo "#3 ncf_xshards_pandas time used: $time3 seconds"
79+
echo "#4 basic_text_classification time used: $time4 seconds"
Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
#!/usr/bin/env bash
2+
clear_up() {
3+
echo "Clearing up environment. Uninstalling analytics-zoo"
4+
pip uninstall -y bigdl-dllib
5+
pip uninstall -y bigdl-orca
6+
pip uninstall -y pyspark
7+
}
8+
9+
set -e
10+
11+
echo "#1 start test for tf2_lenet_mnist.ipynb"
12+
#replace '!pip install --pre' to '#pip install --pre', here we test pr with built whl package. In nightly-build job, we test only use "ipython notebook" for pre-release Analytics Zoo
13+
start=$(date "+%s")
14+
${ANALYTICS_ZOO_HOME}/python/orca/dev/colab-notebook/ipynb2py.sh ${ANALYTICS_ZOO_HOME}/python/orca/colab-notebook/quickstart/tf2_keras_lenet_mnist
15+
sed -i '/get_ipython/s/^/#/' $ANALYTICS_ZOO_HOME/python/orca/colab-notebook/quickstart/tf2_keras_lenet_mnist.py
16+
python ${ANALYTICS_ZOO_HOME}/python/orca/colab-notebook/quickstart/tf2_keras_lenet_mnist.py
17+
18+
exit_status=$?
19+
if [ $exit_status -ne 0 ]; then
20+
clear_up
21+
echo "tf2_lenet_mnist failed"
22+
exit $exit_status
23+
fi
24+
25+
now=$(date "+%s")
26+
time1=$((now - start))
27+
28+
echo "#2 start test for ncf_dataframe.ipynb"
29+
start=$(date "+%s")
30+
${ANALYTICS_ZOO_HOME}/python/orca/dev/colab-notebook/ipynb2py.sh ${ANALYTICS_ZOO_HOME}/python/orca/colab-notebook/quickstart/ncf_dataframe
31+
sed -i '/get_ipython/s/^/#/' ${ANALYTICS_ZOO_HOME}/python/orca/colab-notebook/quickstart/ncf_dataframe.py
32+
python ${ANALYTICS_ZOO_HOME}/python/orca/colab-notebook/quickstart/ncf_dataframe.py
33+
34+
exit_status=$?
35+
if [ $exit_status -ne 0 ]; then
36+
clear_up
37+
echo "ncf_dataframe failed"
38+
exit $exit_status
39+
fi
40+
41+
now=$(date "+%s")
42+
time2=$((now - start))
43+
44+
echo "#1 tf2_keras_lenet_mnist time used: $time1 seconds"
45+
echo "#2 ncf_dataframe time used: $time2 seconds"

0 commit comments

Comments
 (0)