Skip to content

Commit 471116d

Browse files
hkvisionhkvision
authored
Add pip setup and release script for friesian (#4935)
* add pip release * add more arguments * add default script * move to test
1 parent d41e8a7 commit 471116d

File tree

5 files changed

+291
-2
lines changed

5 files changed

+291
-2
lines changed

‎python/friesian/dev/release/release.sh‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
#!/usr/bin/env bash
2+
3+
#
4+
# Copyright 2016 The BigDL Authors.
5+
#
6+
# Licensed under the Apache License, Version 2.0 (the "License");
7+
# you may not use this file except in compliance with the License.
8+
# You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
#
18+
19+
set -e
20+
RUN_SCRIPT_DIR=$(cd $(dirname $0) ; pwd)
21+
echo $RUN_SCRIPT_DIR
22+
BIGDL_DIR="$(cd ${RUN_SCRIPT_DIR}/../../../..; pwd)"
23+
echo $BIGDL_DIR
24+
BIGDL_PYTHON_DIR="$(cd ${BIGDL_DIR}/python/friesian/src; pwd)"
25+
echo $BIGDL_PYTHON_DIR
26+
27+
if (( $# < 4)); then
28+
echo "Usage: release.sh platform version quick_build upload mvn_parameters"
29+
echo "Usage example: bash release.sh linux default false true"
30+
echo "Usage example: bash release.sh linux 0.14.0.dev1 true true"
31+
echo "If needed, you can also add other profiles such as: -Dspark.version=2.4.6 -P spark_2.x"
32+
exit -1
33+
fi
34+
35+
platform=$1
36+
version=$2
37+
quick=$3 # Whether to rebuild the jar; quick=true means not rebuilding the jar
38+
upload=$4 # Whether to upload the whl to pypi
39+
profiles=${*:5}
40+
41+
if [ "${version}" != "default" ]; then
42+
echo "User specified version: ${version}"
43+
echo $version > $BIGDL_DIR/python/version.txt
44+
fi
45+
46+
bigdl_version=$(cat $BIGDL_DIR/python/version.txt | head -1)
47+
echo "The effective version is: ${bigdl_version}"
48+
49+
cd ${BIGDL_DIR}/scala
50+
if [ "$platform" == "mac" ]; then
51+
echo "Building bigdl for mac system"
52+
dist_profile="-P mac $profiles"
53+
verbose_pname="macosx_10_11_x86_64"
54+
elif [ "$platform" == "linux" ]; then
55+
echo "Building bigdl for linux system"
56+
dist_profile="-P linux $profiles"
57+
verbose_pname="manylinux1_x86_64"
58+
else
59+
echo "Unsupported platform"
60+
fi
61+
62+
bigdl_build_command="bash make-dist.sh ${dist_profile}"
63+
if [ "$quick" == "true" ]; then
64+
echo "Skip disting BigDL"
65+
else
66+
echo "Dist BigDL: $bigdl_build_command"
67+
cd ${BIGDL_DIR}/scala
68+
$bigdl_build_command
69+
fi
70+
71+
cd $BIGDL_PYTHON_DIR
72+
sdist_command="python setup.py sdist"
73+
echo "packing source code: ${sdist_command}"
74+
$sdist_command
75+
76+
if [ -d "${BIGDL_DIR}/python/friesian/src/build" ]; then
77+
rm -r ${BIGDL_DIR}/python/friesian/src/build
78+
fi
79+
80+
if [ -d "${BIGDL_DIR}/python/friesian/src/dist" ]; then
81+
rm -r ${BIGDL_DIR}/python/friesian/src/dist
82+
fi
83+
84+
wheel_command="python setup.py bdist_wheel --plat-name ${verbose_pname}"
85+
echo "Packing python distribution: $wheel_command"
86+
${wheel_command}
87+
88+
if [ ${upload} == true ]; then
89+
upload_command="twine upload python/friesian/src/dist/bigdl_friesian-${bigdl_version}-py3-none-${verbose_pname}.whl"
90+
echo "Please manually upload with this command: $upload_command"
91+
$upload_command
92+
fi

‎python/friesian/dev/release/release_default_linux_spark246.sh‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/usr/bin/env bash
2+
3+
#
4+
# Copyright 2016 The BigDL Authors.
5+
#
6+
# Licensed under the Apache License, Version 2.0 (the "License");
7+
# you may not use this file except in compliance with the License.
8+
# You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
#
18+
19+
# This is the default script with maven parameters to release bigdl-friesian built on top of
20+
# Spark 2.4.6 for linux.
21+
# Note that if the maven parameters to build bigdl-friesian need to be changed,
22+
# make sure to change this file accordingly.
23+
# If you want to customize the release, please use release.sh and specify maven parameters instead.
24+
25+
set -e
26+
RUN_SCRIPT_DIR=$(cd $(dirname $0) ; pwd)
27+
echo $RUN_SCRIPT_DIR
28+
29+
if (( $# < 1)); then
30+
echo "Usage: release_default_linux_spark246.sh version"
31+
echo "Usage example: bash release_default_linux_spark246.sh default"
32+
echo "Usage example: bash release_default_linux_spark246.sh 0.14.0.dev1"
33+
exit -1
34+
fi
35+
36+
version=$1
37+
38+
# TODO: change upload to true after uploading to pypi is enabled
39+
bash ${RUN_SCRIPT_DIR}/release.sh linux ${version} true false -Dspark.version=2.4.6 -P spark_2.x

‎python/friesian/dev/release/release_default_mac_spark246.sh‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/usr/bin/env bash
2+
3+
#
4+
# Copyright 2016 The BigDL Authors.
5+
#
6+
# Licensed under the Apache License, Version 2.0 (the "License");
7+
# you may not use this file except in compliance with the License.
8+
# You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
#
18+
19+
# This is the default script with maven parameters to release bigdl-friesian built on top of
20+
# Spark 2.4.6 for mac.
21+
# Note that if the maven parameters to build bigdl-friesian need to be changed,
22+
# make sure to change this file accordingly.
23+
# If you want to customize the release, please use release.sh and specify maven parameters instead.
24+
25+
set -e
26+
RUN_SCRIPT_DIR=$(cd $(dirname $0) ; pwd)
27+
echo $RUN_SCRIPT_DIR
28+
29+
if (( $# < 1)); then
30+
echo "Usage: release_default_mac_spark246.sh version"
31+
echo "Usage example: bash release_default_mac_spark246.sh default"
32+
echo "Usage example: bash release_default_mac_spark246.sh 0.14.0.dev1"
33+
exit -1
34+
fi
35+
36+
version=$1
37+
38+
# TODO: change upload to true after uploading to pypi is enabled
39+
bash ${RUN_SCRIPT_DIR}/release.sh mac ${version} true false -Dspark.version=2.4.6 -P spark_2.x

‎python/friesian/dev/run-pytests-friesian‎ renamed to ‎python/friesian/dev/test/run-pytests-friesian‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,12 @@
1616
# limitations under the License.
1717
#
1818

19-
. `dirname $0`/prepare_env.sh
19+
. `dirname $0`/../prepare_env.sh
2020

2121
cd "`dirname $0`"
2222

2323
echo "Running Friesian feature tests"
24-
python -m pytest -v ../test/bigdl/friesian/feature/
24+
python -m pytest -v ../../test/bigdl/friesian/feature/
2525
exit_status=$?
2626
if [ $exit_status -ne 0 ];
2727
then

‎python/friesian/src/setup.py‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
#!/usr/bin/env python
2+
3+
#
4+
# Copyright 2016 The BigDL Authors.
5+
#
6+
# Licensed under the Apache License, Version 2.0 (the "License");
7+
# you may not use this file except in compliance with the License.
8+
# You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
#
18+
19+
import os
20+
import sys
21+
from shutil import copytree, rmtree
22+
import fnmatch
23+
from setuptools import setup
24+
25+
TEMP_PATH = "bigdl/share/friesian"
26+
bigdl_home = os.path.abspath(__file__ + "/../../../..")
27+
exclude_patterns = ["*__pycache__*", "*ipynb_checkpoints*"]
28+
29+
VERSION = open(os.path.join(bigdl_home, 'python/version.txt'), 'r').read().strip()
30+
31+
building_error_msg = """
32+
If you are packing python API from BigDL source, you must build BigDL first
33+
and run sdist.
34+
To build BigDL with maven you can run:
35+
cd $BigDL_HOME
36+
./make-dist.sh
37+
Building the source dist is done in the Python directory:
38+
cd python
39+
python setup.py sdist
40+
pip install dist/*.tar.gz"""
41+
42+
43+
def build_from_source():
44+
code_path = bigdl_home + "/python/friesian/src/bigdl/friesian/__init__.py"
45+
print("Checking: %s to see if build from source" % code_path)
46+
if os.path.exists(code_path):
47+
return True
48+
return False
49+
50+
51+
def init_env():
52+
if build_from_source():
53+
print("Start to build distributed package")
54+
print("HOME OF BIGDL: " + bigdl_home)
55+
dist_source = bigdl_home + "/dist"
56+
if not os.path.exists(dist_source):
57+
print(building_error_msg)
58+
sys.exit(-1)
59+
if os.path.exists(TEMP_PATH):
60+
rmtree(TEMP_PATH)
61+
copytree(dist_source, TEMP_PATH)
62+
else:
63+
print("Do nothing for release installation")
64+
65+
66+
def get_bigdl_packages():
67+
bigdl_python_home = os.path.abspath(__file__ + "/..")
68+
bigdl_packages = ['bigdl.share.friesian']
69+
source_dir = os.path.join(bigdl_python_home, "bigdl")
70+
for dirpath, dirs, files in os.walk(source_dir):
71+
package = dirpath.split(bigdl_python_home)[1].replace('/', '.')
72+
if any(fnmatch.fnmatchcase(package, pat=pattern)
73+
for pattern in exclude_patterns):
74+
print("excluding", package)
75+
else:
76+
bigdl_packages.append(package)
77+
print("including", package)
78+
return bigdl_packages
79+
80+
81+
def setup_package():
82+
metadata = dict(
83+
name='bigdl-friesian',
84+
version=VERSION,
85+
description='Large-scale End-to-End Recommendation Solution on Big Data',
86+
author='BigDL Authors',
87+
author_email='[email protected]',
88+
license='Apache License, Version 2.0',
89+
url='https://github.com/intel-analytics/analytics-zoo',
90+
packages=get_bigdl_packages(),
91+
# TODO: add bigdl-orca as dependency and change to bigdl-orca[ray] in extras_require
92+
install_requires=['pyspark==2.4.6', 'conda-pack==0.3.1',
93+
'packaging', 'filelock'],
94+
extras_require={'train': ['ray==1.2.0', 'psutil', 'aiohttp==3.7.0', 'aioredis==1.1.0',
95+
'setproctitle', 'hiredis==1.1.0', 'async-timeout==3.0.1']},
96+
dependency_links=['https://d3kbcqa49mib13.cloudfront.net/spark-2.0.0-bin-hadoop2.7.tgz'],
97+
include_package_data=True,
98+
package_data={"bigdl.share.friesian": ['lib/bigdl-friesian*.jar']},
99+
classifiers=[
100+
'License :: OSI Approved :: Apache Software License',
101+
'Programming Language :: Python :: 3',
102+
'Programming Language :: Python :: 3.6',
103+
'Programming Language :: Python :: 3.7',
104+
'Programming Language :: Python :: Implementation :: CPython'],
105+
platforms=['mac', 'linux']
106+
)
107+
108+
setup(**metadata)
109+
110+
111+
if __name__ == '__main__':
112+
try:
113+
init_env()
114+
setup_package()
115+
except Exception as e:
116+
raise e
117+
finally:
118+
if build_from_source() and os.path.exists(TEMP_PATH):
119+
rmtree(TEMP_PATH)

0 commit comments

Comments
 (0)