Skip to content

Commit be0bdb5

Browse files
philo-hephilo-he
authored
[GLUTEN-9615][INFRA] Automate release process with build script and GHA workflow (#10807)
Use GHA to build Gluten jars for supported Spark versions, which can simplify the release process and ensure all required build options are enabled, avoiding potential omissions in manual builds.
1 parent 14cde7e commit be0bdb5

3 files changed

Lines changed: 171 additions & 0 deletions

File tree

‎.github/workflows/build_release.yml‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
name: Build Release
17+
18+
# Only triggered when new tag like v1.5.0 is created.
19+
on:
20+
push:
21+
tags:
22+
- 'v*'
23+
24+
jobs:
25+
build-velox-backend-release:
26+
runs-on: ubuntu-latest
27+
steps:
28+
- uses: actions/checkout@v4
29+
- name: Build
30+
run: |
31+
docker pull apache/gluten:vcpkg-centos-7
32+
docker run -v $GITHUB_WORKSPACE:/workspace -w /work apache/gluten:vcpkg-centos-7 bash -c "
33+
set -e
34+
cd /workspace
35+
bash dev/release/build-release.sh
36+
"
37+
- name: Upload packages
38+
if: ${{ success() }}
39+
uses: actions/upload-artifact@v4
40+
with:
41+
name: ${{ github.job }}-packages
42+
path: ${{ github.workspace }}/package/target/gluten-velox-bundle-*.jar

‎dev/release/build-release.sh‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
#!/bin/bash
2+
3+
# Licensed to the Apache Software Foundation (ASF) under one or more
4+
# contributor license agreements. See the NOTICE file distributed with
5+
# this work for additional information regarding copyright ownership.
6+
# The ASF licenses this file to You under the Apache License, Version 2.0
7+
# (the "License"); you may not use this file except in compliance with
8+
# the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
18+
set -eu
19+
20+
source /opt/rh/devtoolset-11/enable
21+
source /opt/rh/rh-git227/enable
22+
23+
CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd)
24+
GLUTEN_HOME=${CURRENT_DIR}/../../
25+
cd ${GLUTEN_HOME}
26+
27+
# Enable static build with support for S3, GCS, HDFS, and ABFS.
28+
./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF --build_tests=OFF --build_benchmarks=OFF \
29+
--build_examples=OFF --enable_s3=ON --enable_gcs=ON --enable_hdfs=ON --enable_abfs=ON
30+
31+
JAVA_VERSION=$("java" -version 2>&1 | awk -F '"' '/version/ {print $2}')
32+
33+
if [[ $JAVA_VERSION == 1.8* ]]; then
34+
echo "Java 8 is being used."
35+
else
36+
echo "Error: Java 8 is required. Current version is $JAVA_VERSION."
37+
exit 1
38+
fi
39+
40+
# Build Gluten for Spark 3.2 and 3.3 with Java 8. All feature modules are enabled.
41+
for spark_version in 3.2 3.3
42+
do
43+
mvn clean install -Pbackends-velox -Pspark-${spark_version} -Pceleborn,uniffle \
44+
-Piceberg,delta,hudi,paimon -DskipTests
45+
done
46+
47+
sudo curl -Lo /etc/yum.repos.d/corretto.repo https://yum.corretto.aws/corretto.repo
48+
sudo yum install -y java-17-amazon-corretto-devel
49+
export JAVA_HOME=/usr/lib/jvm/java-17-amazon-corretto
50+
export PATH=$JAVA_HOME/bin:$PATH
51+
52+
JAVA_VERSION=$("java" -version 2>&1 | awk -F '"' '/version/ {print $2}')
53+
if [[ $JAVA_VERSION == 17* ]]; then
54+
echo "Java 17 is being used."
55+
else
56+
echo "Error: Java 17 is required. Current version is $JAVA_VERSION."
57+
exit 1
58+
fi
59+
60+
# Build Gluten for Spark 3.4 and 3.5 with Java 17. The version of Iceberg being used requires Java 11 or higher.
61+
# All feature modules are enabled.
62+
for spark_version in 3.4 3.5
63+
do
64+
mvn clean install -Pjava-17 -Pbackends-velox -Pspark-${spark_version} -Pceleborn,uniffle \
65+
-Piceberg,delta,hudi,paimon -DskipTests
66+
done

‎dev/release/package-release.sh‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
#!/bin/bash
2+
3+
# Licensed to the Apache Software Foundation (ASF) under one or more
4+
# contributor license agreements. See the NOTICE file distributed with
5+
# this work for additional information regarding copyright ownership.
6+
# The ASF licenses this file to You under the Apache License, Version 2.0
7+
# (the "License"); you may not use this file except in compliance with
8+
# the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
18+
# Note: Manually create $GLUTEN_HOME/release/ and place the release JARs inside.
19+
# Also set TAG manually (e.g., v1.5.0).
20+
21+
set -eu
22+
23+
TAG=""
24+
25+
if [[ -z "$TAG" ]]; then
26+
echo "TAG is not set. Please set TAG=vX.Y.Z before running."
27+
exit 1
28+
fi
29+
30+
VERSION=${TAG#v}
31+
32+
CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd)
33+
GLUTEN_HOME=${CURRENT_DIR}/../../
34+
if [ ! -d "$GLUTEN_HOME/release/" ]; then
35+
echo "Release directory does not exist."
36+
fi
37+
pushd $GLUTEN_HOME/release/
38+
39+
SPARK_VERSIONS="3.2 3.3 3.4 3.5"
40+
41+
for v in $SPARK_VERSIONS; do
42+
JAR="gluten-velox-bundle-spark${v}_2.12-linux_amd64-${VERSION}.jar"
43+
if [[ ! -f "$JAR" ]]; then
44+
echo "Missing Gluten release JAR under $GLUTEN_HOME/release/ for Spark $v: $JAR"
45+
exit 1
46+
fi
47+
echo "Packaging for Spark $v..."
48+
tar -czf apache-gluten-$VERSION-incubating-bin-spark-${v}.tar.gz \
49+
${GLUTEN_HOME}/DISCLAIMER \
50+
$JAR
51+
done
52+
53+
SRC_ZIP="${TAG}.zip"
54+
SRC_DIR="incubator-gluten-${VERSION}"
55+
56+
echo "Packaging source code..."
57+
wget https://github.com/apache/incubator-gluten/archive/refs/tags/${SRC_ZIP}
58+
unzip -q ${SRC_ZIP}
59+
tar -czf apache-gluten-$VERSION-incubating-src.tar.gz ${SRC_DIR}
60+
rm -r ${SRC_ZIP} ${SRC_DIR}
61+
62+
popd
63+
echo "Finished packaging release binaries and source code."

0 commit comments

Comments
 (0)