diff --git a/.github/workflows/velox_backend_arm.yml b/.github/workflows/velox_backend_arm.yml index a70bf836f06..e5628926fe3 100644 --- a/.github/workflows/velox_backend_arm.yml +++ b/.github/workflows/velox_backend_arm.yml @@ -39,10 +39,15 @@ on: - 'cpp/**' - 'dev/**' - 'build/mvn' + - 'build.gradle.kts' + - 'settings.gradle.kts' + - 'gradle.properties' + - 'gradle/**' + - '*.gradle.kts' env: ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - MVN_CMD: 'build/mvn -ntp' + GRADLE_CMD: './gradlew --no-daemon' WGET_CMD: 'wget -nv' CCACHE_DIR: "${{ github.workspace }}/.ccache" @@ -123,9 +128,7 @@ jobs: run: | echo "JAVA_HOME: $JAVA_HOME" cd $GITHUB_WORKSPACE/ - $MVN_CMD clean install -P${{ matrix.spark }} -P${{ matrix.java }} -Pbackends-velox -DskipTests - cd $GITHUB_WORKSPACE/tools/gluten-it - $GITHUB_WORKSPACE/$MVN_CMD clean install -P${{ matrix.spark }} -P${{ matrix.java }} + bash dev/ci-gradle-build.sh -P${{ matrix.spark }} -P${{ matrix.java }} -Pbackends-velox -PglutenIt=true -DskipTests - name: Run TPC-H / TPC-DS run: | echo "JAVA_HOME: $JAVA_HOME" @@ -143,7 +146,7 @@ jobs: --extra-conf=spark.gluten.ras.enabled=true \ && GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ - --extra-conf=spark.gluten.ras.enabled=true + --extra-conf=spark.gluten.ras.enabled=true cpp-test-udf-test: runs-on: ubuntu-24.04-arm @@ -167,13 +170,13 @@ jobs: cd ./cpp/build && ctest -V - name: Run CPP benchmark test run: | - $MVN_CMD test -Pspark-3.5 -Pbackends-velox -pl backends-velox -am \ - -DtagsToInclude="org.apache.gluten.tags.GenerateExample" -Dtest=none -DfailIfNoTests=false -Dexec.skip - # This test depends on files generated by the above mvn test. + bash dev/ci-gradle-build.sh -Pspark-3.5 -Pbackends-velox -pl -am \ + -DtagsToInclude=org.apache.gluten.tags.GenerateExample -Dtest=none -DfailIfNoTests=false -Dexec.skip + # This test depends on files generated by the above gradle test. ./cpp/build/velox/benchmarks/generic_benchmark --with-shuffle --partitioning hash --threads 1 --iterations 1 \ - --conf $(realpath backends-velox/generated-native-benchmark/conf_12_0_*.ini) \ - --plan $(realpath backends-velox/generated-native-benchmark/plan_12_0_*.json) \ - --data $(realpath backends-velox/generated-native-benchmark/data_12_0_*_0.parquet),$(realpath backends-velox/generated-native-benchmark/data_12_0_*_1.parquet) + --conf $(realpath backends-velox/build/generated-native-benchmark/conf_12_0_*.ini) \ + --plan $(realpath backends-velox/build/generated-native-benchmark/plan_12_0_*.json) \ + --data $(realpath backends-velox/build/generated-native-benchmark/data_12_0_*_0.parquet),$(realpath backends-velox/build/generated-native-benchmark/data_12_0_*_1.parquet) # - name: Run UDF test # run: | # # Depends on --build_example=ON. @@ -182,16 +185,21 @@ jobs: # export JAVA_HOME=/usr/lib/jvm/java-17-openjdk # export PATH=$JAVA_HOME/bin:$PATH # java -version - # $MVN_CMD test -Pspark-3.5 -Pbackends-velox -Pjava-17 -Piceberg -Pdelta -Ppaimon -DtagsToExclude=None \ + # bash dev/ci-gradle-build.sh -Pspark-3.5 -Pbackends-velox -Pjava-17 -Piceberg -Pdelta -Ppaimon \ + # -DtagsToExclude=None \ # -DtagsToInclude=org.apache.gluten.tags.UDFTest - name: Upload test report uses: actions/upload-artifact@v4 with: name: ${{ github.job }}-report - path: '**/surefire-reports/TEST-*.xml' + path: | + **/surefire-reports/TEST-*.xml + **/build/test-results/test/TEST-*.xml - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 with: name: ${{ github.job }}-test-log - path: "**/target/*.log" + path: | + **/target/*.log + **/build/reports/tests/test/** diff --git a/.github/workflows/velox_backend_enhanced.yml b/.github/workflows/velox_backend_enhanced.yml index 2337993db3b..460d161fdfa 100644 --- a/.github/workflows/velox_backend_enhanced.yml +++ b/.github/workflows/velox_backend_enhanced.yml @@ -38,10 +38,15 @@ on: - 'cpp/**' - 'dev/**' - 'build/mvn' + - 'build.gradle.kts' + - 'settings.gradle.kts' + - 'gradle.properties' + - 'gradle/**' + - '*.gradle.kts' env: ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - MVN_CMD: 'build/mvn -ntp' + GRADLE_CMD: './gradlew --no-daemon' WGET_CMD: 'wget -nv' CCACHE_DIR: "${{ github.workspace }}/.ccache" @@ -127,7 +132,7 @@ jobs: java -version export SPARK_HOME=/opt/shims/spark34/spark_home/ ls -l $SPARK_HOME - $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Piceberg-test -Pdelta -Phudi \ + bash dev/ci-gradle-build.sh -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Piceberg-test -Pdelta -Phudi \ -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTest \ -DargLine="-Dspark.test.home=$SPARK_HOME" - name: Upload test report @@ -135,7 +140,9 @@ jobs: uses: actions/upload-artifact@v4 with: name: ${{ github.job }}-report - path: '**/surefire-reports/TEST-*.xml' + path: | + **/surefire-reports/TEST-*.xml + **/build/test-results/test/TEST-*.xml - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 @@ -143,6 +150,7 @@ jobs: name: ${{ github.job }}-test-log path: | **/target/*.log + **/build/reports/tests/test/** **/gluten-ut/**/hs_err_*.log **/gluten-ut/**/core.* - name: Upload golden files @@ -185,7 +193,7 @@ jobs: java -version export SPARK_HOME=/opt/shims/spark35/spark_home/ ls -l $SPARK_HOME - $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi \ + bash dev/ci-gradle-build.sh -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi \ -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTest \ -DargLine="-Dspark.test.home=$SPARK_HOME" - name: Upload test report @@ -193,7 +201,9 @@ jobs: uses: actions/upload-artifact@v4 with: name: ${{ github.job }}-report - path: '**/surefire-reports/TEST-*.xml' + path: | + **/surefire-reports/TEST-*.xml + **/build/test-results/test/TEST-*.xml - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 @@ -201,6 +211,7 @@ jobs: name: ${{ github.job }}-test-log path: | **/target/*.log + **/build/reports/tests/test/** **/gluten-ut/**/hs_err_*.log **/gluten-ut/**/core.* - name: Upload golden files @@ -237,7 +248,7 @@ jobs: export JAVA_HOME=/usr/lib/jvm/java-17-openjdk export PATH=$JAVA_HOME/bin:$PATH java -version - $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ + bash dev/ci-gradle-build.sh -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \ -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest - name: Upload test report @@ -245,7 +256,9 @@ jobs: uses: actions/upload-artifact@v4 with: name: ${{ github.job }}-report - path: '**/surefire-reports/TEST-*.xml' + path: | + **/surefire-reports/TEST-*.xml + **/build/test-results/test/TEST-*.xml - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 @@ -253,5 +266,6 @@ jobs: name: ${{ github.job }}-test-log path: | **/target/*.log + **/build/reports/tests/test/** **/gluten-ut/**/hs_err_*.log **/gluten-ut/**/core.* diff --git a/.github/workflows/velox_backend_x86.yml b/.github/workflows/velox_backend_x86.yml index 545fe71ab39..b550e1489c0 100644 --- a/.github/workflows/velox_backend_x86.yml +++ b/.github/workflows/velox_backend_x86.yml @@ -40,10 +40,16 @@ on: - 'cpp/**' - 'dev/**' - 'build/mvn' + - 'build.gradle.kts' + - 'settings.gradle.kts' + - 'gradle.properties' + - 'gradle/**' + - '*.gradle.kts' env: ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true MVN_CMD: 'build/mvn -ntp' + GRADLE_CMD: './gradlew --no-daemon' WGET_CMD: 'wget -nv' CCACHE_DIR: "${{ github.workspace }}/.ccache" # spark.sql.ansi.enabled defaults to false. @@ -190,14 +196,13 @@ jobs: echo "JAVA_HOME: $JAVA_HOME" case "${{ matrix.spark }}" in spark-4.0|spark-4.1) - $MVN_CMD clean install -P${{ matrix.spark }} -P${{ matrix.java }} -Pscala-2.13 -Pbackends-velox -DskipTests + bash dev/ci-gradle-build.sh -P${{ matrix.spark }} -P${{ matrix.java }} -Pscala-2.13 -Pbackends-velox -PglutenIt=true -DskipTests ;; *) - $MVN_CMD clean install -P${{ matrix.spark }} -P${{ matrix.java }} -Pbackends-velox -DskipTests + bash dev/ci-gradle-build.sh -P${{ matrix.spark }} -P${{ matrix.java }} -Pbackends-velox -PglutenIt=true -DskipTests ;; esac cd $GITHUB_WORKSPACE/tools/gluten-it - $GITHUB_WORKSPACE/$MVN_CMD clean install -P${{ matrix.spark }} -P${{ matrix.java }} GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ --local --preset=velox --benchmark-type=h --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 \ && GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh queries-compare \ @@ -281,14 +286,12 @@ jobs: cd $GITHUB_WORKSPACE/ case "${{ matrix.spark }}" in spark-4.0|spark-4.1) - $MVN_CMD clean install -P${{ matrix.spark }} -P${{ matrix.java }} -Pscala-2.13 -Pbackends-velox -DskipTests + bash dev/ci-gradle-build.sh -P${{ matrix.spark }} -P${{ matrix.java }} -Pscala-2.13 -Pbackends-velox -PglutenIt=true -DskipTests ;; *) - $MVN_CMD clean install -P${{ matrix.spark }} -P${{ matrix.java }} -Pbackends-velox -DskipTests + bash dev/ci-gradle-build.sh -P${{ matrix.spark }} -P${{ matrix.java }} -Pbackends-velox -PglutenIt=true -DskipTests ;; esac - cd $GITHUB_WORKSPACE/tools/gluten-it - $GITHUB_WORKSPACE/$MVN_CMD clean install -P${{ matrix.spark }} -P${{ matrix.java }} - name: Run TPC-H / TPC-DS run: | echo "JAVA_HOME: $JAVA_HOME" @@ -347,13 +350,11 @@ jobs: # Set environment variables export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk - # Build gluten - build/mvn -ntp clean install -P${{ matrix.spark }} -P${{ matrix.java }} -Pbackends-velox -DskipTests - # Build gluten-it - cd /work/tools/gluten-it - /work/build/mvn -ntp clean install -P${{ matrix.spark }} -P${{ matrix.java }} + # Build gluten and gluten-it + bash dev/ci-gradle-build.sh -P${{ matrix.spark }} -P${{ matrix.java }} -Pbackends-velox -PglutenIt=true -DskipTests # Run TPC-H / TPC-DS + cd /work/tools/gluten-it GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ --local --preset=velox --benchmark-type=h --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh queries-compare \ @@ -409,10 +410,9 @@ jobs: echo "JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64" >> $GITHUB_ENV - name: Build for Spark ${{ matrix.spark }} run: | - cd $GITHUB_WORKSPACE/ - $MVN_CMD clean install -P${{ matrix.spark }} -Pbackends-velox -DskipTests + cd $GITHUB_WORKSPACE/ + bash dev/ci-gradle-build.sh -P${{ matrix.spark }} -Pbackends-velox -PglutenIt=true -DskipTests cd $GITHUB_WORKSPACE/tools/gluten-it - $GITHUB_WORKSPACE/$MVN_CMD clean install -P${{ matrix.spark }} GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh data-gen-only --local --benchmark-type=ds -s=30.0 --threads=12 - name: TPC-DS SF30.0 Parquet local spark3.3 Q67/Q95 low memory, memory isolation off run: | @@ -524,10 +524,9 @@ jobs: echo "JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64" >> $GITHUB_ENV - name: Build for Spark ${{ matrix.spark }} run: | - cd $GITHUB_WORKSPACE/ - $MVN_CMD clean install -P${{ matrix.spark }} -Pbackends-velox -DskipTests + cd $GITHUB_WORKSPACE/ + bash dev/ci-gradle-build.sh -P${{ matrix.spark }} -Pbackends-velox -PglutenIt=true -DskipTests cd $GITHUB_WORKSPACE/tools/gluten-it - $GITHUB_WORKSPACE/$MVN_CMD clean install -P${{ matrix.spark }} GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh data-gen-only --local --benchmark-type=ds -s=30.0 --threads=12 - name: TPC-DS SF30.0 Parquet local spark3.3 random kill tasks run: | @@ -578,12 +577,11 @@ jobs: - name: Build for Spark ${{ matrix.spark }} run: | cd $GITHUB_WORKSPACE/ && \ - $MVN_CMD clean install -P${{ matrix.spark }} -Pbackends-velox -Puniffle -DskipTests + bash dev/ci-gradle-build.sh -P${{ matrix.spark }} -Pbackends-velox -Puniffle -PglutenIt=true -DskipTests - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.3 with uniffle-${{ matrix.uniffle }} run: | export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk && \ cd $GITHUB_WORKSPACE/tools/gluten-it && \ - $GITHUB_WORKSPACE/$MVN_CMD clean install -P${{ matrix.spark }} -Puniffle && \ GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \ --local --preset=velox-with-uniffle --benchmark-type=h --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1 @@ -622,16 +620,15 @@ jobs: - name: Build for Spark ${{ matrix.spark }} run: | cd $GITHUB_WORKSPACE/ - $MVN_CMD clean install -P${{ matrix.spark }} -Pbackends-velox -Pceleborn -DskipTests - - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.3 with ${{ matrix.celeborn }} - run: | EXTRA_PROFILE="" if [ "${{ matrix.celeborn }}" = "celeborn-0.5.4" ]; then EXTRA_PROFILE="-Pceleborn-0.5" elif [ "${{ matrix.celeborn }}" = "celeborn-0.6.1" ]; then EXTRA_PROFILE="-Pceleborn-0.6" fi - echo "EXTRA_PROFILE: ${EXTRA_PROFILE}" + bash dev/ci-gradle-build.sh -P${{ matrix.spark }} -Pbackends-velox -Pceleborn ${EXTRA_PROFILE} -PglutenIt=true -DskipTests + - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.3 with ${{ matrix.celeborn }} + run: | if [ ! -e "/opt/apache-${{ matrix.celeborn }}-bin.tgz" ]; then echo "WARNING: please pre-install your required package in docker image since the downloading is throttled by this site." wget -nv https://www.apache.org/dyn/closer.lua/celeborn/${{ matrix.celeborn }}/apache-${{ matrix.celeborn }}-bin.tgz?action=download -O /opt/apache-${{ matrix.celeborn }}-bin.tgz @@ -642,7 +639,7 @@ jobs: bash -c "echo -e 'CELEBORN_MASTER_MEMORY=8g\nCELEBORN_WORKER_MEMORY=8g\nCELEBORN_WORKER_OFFHEAP_MEMORY=16g' > ./conf/celeborn-env.sh" && \ bash -c "echo -e 'celeborn.worker.commitFiles.threads 32\nceleborn.worker.sortPartition.threads 16' > ./conf/celeborn-defaults.conf" && \ bash ./sbin/start-master.sh && bash ./sbin/start-worker.sh && \ - cd $GITHUB_WORKSPACE/tools/gluten-it && $GITHUB_WORKSPACE/$MVN_CMD clean install -Pspark-3.3 -Pceleborn ${EXTRA_PROFILE} && \ + cd $GITHUB_WORKSPACE/tools/gluten-it && \ GLUTEN_IT_JVM_ARGS=-Xmx16G sbin/gluten-it.sh queries-compare \ --extra-conf=spark.celeborn.client.spark.shuffle.writer=${{ matrix.writer }} \ --extra-conf=spark.sql.shuffle.partitions=16 \ @@ -696,7 +693,7 @@ jobs: export JAVA_HOME=/usr/lib/jvm/java-17-openjdk export PATH=$JAVA_HOME/bin:$PATH java -version - $MVN_CMD clean test -Pspark-3.3 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ + bash dev/ci-gradle-build.sh -Pspark-3.3 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ -DargLine="-Dspark.test.home=/opt/shims/spark33/spark_home/" \ -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest - name: Upload test report @@ -704,7 +701,9 @@ jobs: uses: actions/upload-artifact@v4 with: name: ${{ github.job }}-report - path: '**/surefire-reports/TEST-*.xml' + path: | + **/surefire-reports/TEST-*.xml + **/build/test-results/test/TEST-*.xml - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 @@ -712,6 +711,7 @@ jobs: name: ${{ github.job }}-test-log path: | **/target/*.log + **/build/reports/tests/test/** **/gluten-ut/**/hs_err_*.log **/gluten-ut/**/core.* - name: Upload golden files @@ -745,7 +745,7 @@ jobs: export JAVA_HOME=/usr/lib/jvm/java-17-openjdk export PATH=$JAVA_HOME/bin:$PATH java -version - $MVN_CMD clean test -Pspark-3.3 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ + bash dev/ci-gradle-build.sh -Pspark-3.3 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ -DargLine="-Dspark.test.home=/opt/shims/spark33/spark_home/" \ -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest - name: Upload test report @@ -753,7 +753,9 @@ jobs: uses: actions/upload-artifact@v4 with: name: ${{ github.job }}-report - path: '**/surefire-reports/TEST-*.xml' + path: | + **/surefire-reports/TEST-*.xml + **/build/test-results/test/TEST-*.xml - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 @@ -761,6 +763,7 @@ jobs: name: ${{ github.job }}-test-log path: | **/target/*.log + **/build/reports/tests/test/** **/gluten-ut/**/hs_err_*.log **/gluten-ut/**/core.* @@ -797,7 +800,7 @@ jobs: java -version export SPARK_HOME=/opt/shims/spark34/spark_home/ ls -l $SPARK_HOME - $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Piceberg-test -Pdelta -Phudi -Ppaimon -Pspark-ut \ + bash dev/ci-gradle-build.sh -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Piceberg-test -Pdelta -Phudi -Ppaimon -Pspark-ut \ -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest \ -DargLine="-Dspark.test.home=$SPARK_HOME" - name: Upload test report @@ -805,7 +808,9 @@ jobs: uses: actions/upload-artifact@v4 with: name: ${{ github.job }}-report - path: '**/surefire-reports/TEST-*.xml' + path: | + **/surefire-reports/TEST-*.xml + **/build/test-results/test/TEST-*.xml - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 @@ -813,6 +818,7 @@ jobs: name: ${{ github.job }}-test-log path: | **/target/*.log + **/build/reports/tests/test/** **/gluten-ut/**/hs_err_*.log **/gluten-ut/**/core.* - name: Upload golden files @@ -847,7 +853,7 @@ jobs: java -version export SPARK_HOME=/opt/shims/spark34/spark_home/ ls -l $SPARK_HOME - $MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ + bash dev/ci-gradle-build.sh -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest \ -DargLine="-Dspark.test.home=$SPARK_HOME" - name: Upload test report @@ -855,7 +861,9 @@ jobs: uses: actions/upload-artifact@v4 with: name: ${{ github.job }}-report - path: '**/surefire-reports/TEST-*.xml' + path: | + **/surefire-reports/TEST-*.xml + **/build/test-results/test/TEST-*.xml - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 @@ -863,6 +871,7 @@ jobs: name: ${{ github.job }}-test-log path: | **/target/*.log + **/build/reports/tests/test/** **/gluten-ut/**/hs_err_*.log **/gluten-ut/**/core.* @@ -899,7 +908,7 @@ jobs: export JAVA_HOME=/usr/lib/jvm/java-17-openjdk export PATH=$JAVA_HOME/bin:$PATH java -version - $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ + bash dev/ci-gradle-build.sh -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \ -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest - name: Upload test report @@ -907,7 +916,9 @@ jobs: uses: actions/upload-artifact@v4 with: name: ${{ github.job }}-report - path: '**/surefire-reports/TEST-*.xml' + path: | + **/surefire-reports/TEST-*.xml + **/build/test-results/test/TEST-*.xml - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 @@ -915,6 +926,7 @@ jobs: name: ${{ github.job }}-test-log path: | **/target/*.log + **/build/reports/tests/test/** **/gluten-ut/**/hs_err_*.log **/gluten-ut/**/core.* - name: Upload golden files @@ -957,7 +969,7 @@ jobs: export JAVA_HOME=/usr/lib/jvm/java-17-openjdk export PATH=$JAVA_HOME/bin:$PATH java -version - $MVN_CMD clean test -Pspark-3.5 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Piceberg \ + bash dev/ci-gradle-build.sh -Pspark-3.5 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Piceberg \ -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=/opt/shims/spark35-scala-2.13/spark_home/" \ -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest - name: Upload test report @@ -965,7 +977,9 @@ jobs: uses: actions/upload-artifact@v4 with: name: ${{ github.job }}-report - path: '**/surefire-reports/TEST-*.xml' + path: | + **/surefire-reports/TEST-*.xml + **/build/test-results/test/TEST-*.xml - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 @@ -973,6 +987,7 @@ jobs: name: ${{ github.job }}-test-log path: | **/target/*.log + **/build/reports/tests/test/** **/gluten-ut/**/hs_err_*.log **/gluten-ut/**/core.* @@ -1001,7 +1016,7 @@ jobs: export JAVA_HOME=/usr/lib/jvm/java-17-openjdk export PATH=$JAVA_HOME/bin:$PATH java -version - $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ + bash dev/ci-gradle-build.sh -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \ -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \ -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest - name: Upload test report @@ -1009,7 +1024,9 @@ jobs: uses: actions/upload-artifact@v4 with: name: ${{ github.job }}-report - path: '**/surefire-reports/TEST-*.xml' + path: | + **/surefire-reports/TEST-*.xml + **/build/test-results/test/TEST-*.xml - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 @@ -1017,6 +1034,7 @@ jobs: name: ${{ github.job }}-test-log path: | **/target/*.log + **/build/reports/tests/test/** **/gluten-ut/**/hs_err_*.log **/gluten-ut/**/core.* @@ -1053,14 +1071,16 @@ jobs: export JAVA_HOME=/usr/lib/jvm/java-17-openjdk export PATH=$JAVA_HOME/bin:$PATH java -version - $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Ppaimon -Pspark-ut \ + bash dev/ci-gradle-build.sh -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Ppaimon -Pspark-ut \ -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/ -Dspark.gluten.ras.enabled=true" \ -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest - name: Upload test report uses: actions/upload-artifact@v4 with: name: ${{ github.job }}-report - path: '**/surefire-reports/TEST-*.xml' + path: | + **/surefire-reports/TEST-*.xml + **/build/test-results/test/TEST-*.xml - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 @@ -1068,6 +1088,7 @@ jobs: name: ${{ github.job }}-test-log path: | **/target/*.log + **/build/reports/tests/test/** **/gluten-ut/**/hs_err_*.log **/gluten-ut/**/core.* @@ -1096,14 +1117,16 @@ jobs: export JAVA_HOME=/usr/lib/jvm/java-17-openjdk export PATH=$JAVA_HOME/bin:$PATH java -version - $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Ppaimon -Pspark-ut \ + bash dev/ci-gradle-build.sh -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Ppaimon -Pspark-ut \ -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/ -Dspark.gluten.ras.enabled=true" \ -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest - name: Upload test report uses: actions/upload-artifact@v4 with: name: ${{ github.job }}-report - path: '**/surefire-reports/TEST-*.xml' + path: | + **/surefire-reports/TEST-*.xml + **/build/test-results/test/TEST-*.xml - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 @@ -1111,6 +1134,7 @@ jobs: name: ${{ github.job }}-test-log path: | **/target/*.log + **/build/reports/tests/test/** **/gluten-ut/**/hs_err_*.log **/gluten-ut/**/core.* @@ -1147,14 +1171,16 @@ jobs: export JAVA_HOME=/usr/lib/jvm/java-17-openjdk export PATH=$JAVA_HOME/bin:$PATH java -version - $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Ppaimon -Pspark-ut \ + bash dev/ci-gradle-build.sh -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Ppaimon -Pspark-ut \ -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/ -Dspark.gluten.sql.columnar.forceShuffledHashJoin=false" \ -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest - name: Upload test report uses: actions/upload-artifact@v4 with: name: ${{ github.job }}-report - path: '**/surefire-reports/TEST-*.xml' + path: | + **/surefire-reports/TEST-*.xml + **/build/test-results/test/TEST-*.xml - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 @@ -1162,6 +1188,7 @@ jobs: name: ${{ github.job }}-test-log path: | **/target/*.log + **/build/reports/tests/test/** **/gluten-ut/**/hs_err_*.log **/gluten-ut/**/core.* @@ -1190,14 +1217,16 @@ jobs: export JAVA_HOME=/usr/lib/jvm/java-17-openjdk export PATH=$JAVA_HOME/bin:$PATH java -version - $MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Ppaimon -Pspark-ut \ + bash dev/ci-gradle-build.sh -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Ppaimon -Pspark-ut \ -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/ -Dspark.gluten.sql.columnar.forceShuffledHashJoin=false" \ -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest - name: Upload test report uses: actions/upload-artifact@v4 with: name: ${{ github.job }}-report - path: '**/surefire-reports/TEST-*.xml' + path: | + **/surefire-reports/TEST-*.xml + **/build/test-results/test/TEST-*.xml - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 @@ -1205,6 +1234,7 @@ jobs: name: ${{ github.job }}-test-log path: | **/target/*.log + **/build/reports/tests/test/** **/gluten-ut/**/hs_err_*.log **/gluten-ut/**/core.* @@ -1230,13 +1260,13 @@ jobs: cd ./cpp/build && ctest -V - name: Run CPP benchmark test run: | - $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -pl backends-velox -am \ - -DtagsToInclude="org.apache.gluten.tags.GenerateExample" -Dtest=none -DfailIfNoTests=false -Dexec.skip - # This test depends on files generated by the above mvn test. + bash dev/ci-gradle-build.sh -Pspark-3.5 -Pbackends-velox -pl -am \ + -DtagsToInclude=org.apache.gluten.tags.GenerateExample -Dtest=none -DfailIfNoTests=false -Dexec.skip + # This test depends on files generated by the above gradle test. ./cpp/build/velox/benchmarks/generic_benchmark --with-shuffle --partitioning hash --threads 1 --iterations 1 \ - --conf $(realpath backends-velox/generated-native-benchmark/conf_12_0_*.ini) \ - --plan $(realpath backends-velox/generated-native-benchmark/plan_12_0_*.json) \ - --data $(realpath backends-velox/generated-native-benchmark/data_12_0_*_0.parquet),$(realpath backends-velox/generated-native-benchmark/data_12_0_*_1.parquet) + --conf $(realpath backends-velox/build/generated-native-benchmark/conf_12_0_*.ini) \ + --plan $(realpath backends-velox/build/generated-native-benchmark/plan_12_0_*.json) \ + --data $(realpath backends-velox/build/generated-native-benchmark/data_12_0_*_0.parquet),$(realpath backends-velox/build/generated-native-benchmark/data_12_0_*_1.parquet) - name: Run UDF test run: | yum install -y java-17-openjdk-devel @@ -1244,7 +1274,8 @@ jobs: export PATH=$JAVA_HOME/bin:$PATH java -version # Depends on --build_example=ON. - $MVN_CMD test -Pspark-3.5 -Pbackends-velox -Pjava-17 -Piceberg -Pdelta -Ppaimon -DtagsToExclude=org.apache.gluten.tags.EnhancedFeaturesTest \ + bash dev/ci-gradle-build.sh -Pspark-3.5 -Pbackends-velox -Pjava-17 -Piceberg -Pdelta -Ppaimon \ + -DtagsToExclude=org.apache.gluten.tags.EnhancedFeaturesTest \ -DtagsToInclude=org.apache.gluten.tags.UDFTest \ -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" - name: Run CPP benchmark test for Spark 4.0 @@ -1253,18 +1284,20 @@ jobs: yum install -y java-17-openjdk-devel export JAVA_HOME=/usr/lib/jvm/java-17-openjdk export PATH=$JAVA_HOME/bin:$PATH - $MVN_CMD clean test -Pspark-4.0 -Pjava-17 -Pscala-2.13 -Pbackends-velox -pl backends-velox -am \ - -DtagsToInclude="org.apache.gluten.tags.GenerateExample" -Dtest=none -DfailIfNoTests=false -Dexec.skip - # This test depends on files generated by the above mvn test. + bash dev/ci-gradle-build.sh -Pspark-4.0 -Pjava-17 -Pscala-2.13 -Pbackends-velox -pl -am \ + -DtagsToInclude=org.apache.gluten.tags.GenerateExample -Dtest=none -DfailIfNoTests=false -Dexec.skip + # This test depends on files generated by the above gradle test. ./cpp/build/velox/benchmarks/generic_benchmark --with-shuffle --partitioning hash --threads 1 --iterations 1 \ - --conf $(realpath backends-velox/generated-native-benchmark/conf_12_0_*.ini) \ - --plan $(realpath backends-velox/generated-native-benchmark/plan_12_0_*.json) \ - --data $(realpath backends-velox/generated-native-benchmark/data_12_0_*_0.parquet),$(realpath backends-velox/generated-native-benchmark/data_12_0_*_1.parquet) + --conf $(realpath backends-velox/build/generated-native-benchmark/conf_12_0_*.ini) \ + --plan $(realpath backends-velox/build/generated-native-benchmark/plan_12_0_*.json) \ + --data $(realpath backends-velox/build/generated-native-benchmark/data_12_0_*_0.parquet),$(realpath backends-velox/build/generated-native-benchmark/data_12_0_*_1.parquet) - name: Upload test report uses: actions/upload-artifact@v4 with: name: ${{ github.job }}-report - path: '**/surefire-reports/TEST-*.xml' + path: | + **/surefire-reports/TEST-*.xml + **/build/test-results/test/TEST-*.xml - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 @@ -1310,7 +1343,7 @@ jobs: cd /work bash dev/builddeps-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF --build_tests=ON --build_benchmarks=ON --enable_gpu=ON # TODO: re-enable tests with more disk space rm -rf ep/build-velox/build/velox_ep - build/mvn clean package -Pbackends-velox -Pspark-3.4 -DskipTests + bash dev/ci-gradle-build.sh -Pbackends-velox -Pspark-3.4 -DskipTests ccache -s " @@ -1352,7 +1385,7 @@ jobs: export JAVA_HOME=/usr/lib/jvm/java-17-openjdk export PATH=$JAVA_HOME/bin:$PATH java -version - $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox \ + bash dev/ci-gradle-build.sh -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox \ -Pspark-ut -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/" \ -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest - name: Upload test report @@ -1360,7 +1393,9 @@ jobs: uses: actions/upload-artifact@v4 with: name: ${{ github.job }}-report - path: '**/surefire-reports/TEST-*.xml' + path: | + **/surefire-reports/TEST-*.xml + **/build/test-results/test/TEST-*.xml - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 @@ -1368,6 +1403,7 @@ jobs: name: ${{ github.job }}-test-log path: | **/target/*.log + **/build/reports/tests/test/** **/gluten-ut/**/hs_err_*.log **/gluten-ut/**/core.* @@ -1401,7 +1437,7 @@ jobs: export JAVA_HOME=/usr/lib/jvm/java-17-openjdk export PATH=$JAVA_HOME/bin:$PATH java -version - $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pspark-ut \ + bash dev/ci-gradle-build.sh -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pspark-ut \ -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/" \ -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest - name: Upload test report @@ -1409,7 +1445,9 @@ jobs: uses: actions/upload-artifact@v4 with: name: ${{ github.job }}-report - path: '**/surefire-reports/TEST-*.xml' + path: | + **/surefire-reports/TEST-*.xml + **/build/test-results/test/TEST-*.xml - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 @@ -1417,6 +1455,7 @@ jobs: name: ${{ github.job }}-test-log path: | **/target/*.log + **/build/reports/tests/test/** **/gluten-ut/**/hs_err_*.log **/gluten-ut/**/core.* @@ -1460,7 +1499,7 @@ jobs: export JAVA_HOME=/usr/lib/jvm/java-17-openjdk export PATH=$JAVA_HOME/bin:$PATH java -version - $MVN_CMD clean test -Pspark-4.1 -Pscala-2.13 -Pjava-17 -Pbackends-velox \ + bash dev/ci-gradle-build.sh -Pspark-4.1 -Pscala-2.13 -Pjava-17 -Pbackends-velox \ -Pspark-ut -DargLine="-Dspark.test.home=/opt/shims/spark41/spark_home/" \ -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest - name: Upload test report @@ -1468,7 +1507,9 @@ jobs: uses: actions/upload-artifact@v4 with: name: ${{ github.job }}-report - path: '**/surefire-reports/TEST-*.xml' + path: | + **/surefire-reports/TEST-*.xml + **/build/test-results/test/TEST-*.xml - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 @@ -1476,6 +1517,7 @@ jobs: name: ${{ github.job }}-test-log path: | **/target/*.log + **/build/reports/tests/test/** **/gluten-ut/**/hs_err_*.log **/gluten-ut/**/core.* @@ -1509,7 +1551,7 @@ jobs: export JAVA_HOME=/usr/lib/jvm/java-17-openjdk export PATH=$JAVA_HOME/bin:$PATH java -version - $MVN_CMD clean test -Pspark-4.1 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pspark-ut \ + bash dev/ci-gradle-build.sh -Pspark-4.1 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pspark-ut \ -DargLine="-Dspark.test.home=/opt/shims/spark41/spark_home/" \ -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest - name: Upload test report @@ -1517,7 +1559,9 @@ jobs: uses: actions/upload-artifact@v4 with: name: ${{ github.job }}-report - path: '**/surefire-reports/TEST-*.xml' + path: | + **/surefire-reports/TEST-*.xml + **/build/test-results/test/TEST-*.xml - name: Upload unit tests log files if: ${{ !success() }} uses: actions/upload-artifact@v4 @@ -1525,5 +1569,6 @@ jobs: name: ${{ github.job }}-test-log path: | **/target/*.log + **/build/reports/tests/test/** **/gluten-ut/**/hs_err_*.log **/gluten-ut/**/core.* diff --git a/.gitignore b/.gitignore index a679fc3002e..16f82903942 100644 --- a/.gitignore +++ b/.gitignore @@ -51,6 +51,8 @@ scalastyle-output.xml # Generated C++ benchmark data backends-velox/generated-native-benchmark/ backends-velox/spark-warehouse/ +# Symlink created by Gradle build for test resource path compatibility +backends-velox/tools # Generated C++ benchmark executables cpp/**/benchmarks/**/* !cpp/**/benchmarks/**/ @@ -78,6 +80,11 @@ metastore_db/ .github/copilot-instructions.md .github/copilot-setup-steps.yml +# Gradle +.gradle/ +gradlew +gradlew.bat + # Dev script build cache .run-scala-test-cache/ diff --git a/backends-clickhouse/build.gradle.kts b/backends-clickhouse/build.gradle.kts new file mode 100644 index 00000000000..883a4957051 --- /dev/null +++ b/backends-clickhouse/build.gradle.kts @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("gluten.scala-library") + id("gluten.protobuf") + id("gluten.scalatest") + id("gluten.spotless") + antlr +} + +val scalaBinaryVersion: String by project +val protobufVersion: String by project +val effectiveSparkFullVersion: String by rootProject.extra +val effectiveHadoopVersion: String by rootProject.extra +val effectiveFasterxmlVersion: String by rootProject.extra +val effectiveAntlr4Version: String by rootProject.extra + +// C++ build directories for ClickHouse +val cppChBuildDir = file("../cpp-ch/build") +val cppChReleasesDir = file("$cppChBuildDir/releases") + +dependencies { + antlr("org.antlr:antlr4:$effectiveAntlr4Version") + + implementation(project(":gluten-substrait")) + + implementation("com.google.protobuf:protobuf-java:$protobufVersion") + + compileOnly("org.apache.hadoop:hadoop-client:$effectiveHadoopVersion") + + implementation("org.scala-lang.modules:scala-collection-compat_$scalaBinaryVersion:2.11.0") + + compileOnly("com.fasterxml.jackson.core:jackson-databind:$effectiveFasterxmlVersion") + compileOnly("com.fasterxml.jackson.core:jackson-annotations:$effectiveFasterxmlVersion") + compileOnly("com.fasterxml.jackson.core:jackson-core:$effectiveFasterxmlVersion") + compileOnly("com.fasterxml.jackson.module:jackson-module-scala_$scalaBinaryVersion:$effectiveFasterxmlVersion") + + testImplementation("org.scalacheck:scalacheck_$scalaBinaryVersion:1.17.0") + testImplementation("org.mockito:mockito-core:2.23.4") + testImplementation("org.scalatestplus:scalatestplus-mockito_$scalaBinaryVersion:1.0.0-M2") + testImplementation("org.scalatestplus:scalatestplus-scalacheck_$scalaBinaryVersion:3.1.0.0-RC2") +} + +// ANTLR4 grammar configuration +tasks.generateGrammarSource { + arguments = arguments + listOf("-visitor") + outputDirectory = file("${layout.buildDirectory.get()}/generated-src/antlr/main/org/apache/gluten/sql/parser") +} + +// Ensure ANTLR sources are on the compile classpath +sourceSets { + main { + java { + srcDir("${layout.buildDirectory.get()}/generated-src/antlr/main") + } + proto { + srcDir("src/main/resources/org/apache/gluten/proto") + } + } +} + +// Include native libraries in the JAR +val platform = rootProject.extra.get("platform") as String +val arch = rootProject.extra.get("arch") as String + +tasks.processResources { + from(cppChReleasesDir) { + into("$platform/$arch") + } +} diff --git a/backends-velox/build.gradle.kts b/backends-velox/build.gradle.kts new file mode 100644 index 00000000000..c5bf052c6fa --- /dev/null +++ b/backends-velox/build.gradle.kts @@ -0,0 +1,308 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("gluten.scala-library") + id("gluten.protobuf") + id("gluten.spotless") + id("gluten.scalatest") +} + +val scalaBinaryVersion: String by project +val protobufVersion: String by project +val effectiveSparkFullVersion: String by rootProject.extra +val effectiveSparkPlainVersion: String by rootProject.extra +val effectiveHadoopVersion: String by rootProject.extra +val effectiveArrowVersion: String by rootProject.extra +val effectiveFasterxmlVersion: String by rootProject.extra + +// C++ build directories +val cppBuildDir = file("../cpp/build") +val cppReleasesDir = file("$cppBuildDir/releases") + +dependencies { + implementation(project(":gluten-substrait")) + implementation(project(":gluten-arrow")) + + testImplementation(project(":gluten-substrait", "testArtifacts")) + testImplementation(project(":gluten-ras-common", "testArtifacts")) + + implementation("com.google.protobuf:protobuf-java:$protobufVersion") + + compileOnly("org.apache.spark:spark-network-common_$scalaBinaryVersion:$effectiveSparkFullVersion") + + compileOnly("org.apache.hadoop:hadoop-client:$effectiveHadoopVersion") + + implementation("org.scala-lang.modules:scala-collection-compat_$scalaBinaryVersion:2.11.0") + + compileOnly("com.fasterxml.jackson.core:jackson-databind:$effectiveFasterxmlVersion") + compileOnly("com.fasterxml.jackson.core:jackson-annotations:$effectiveFasterxmlVersion") + compileOnly("com.fasterxml.jackson.core:jackson-core:$effectiveFasterxmlVersion") + compileOnly("com.fasterxml.jackson.module:jackson-module-scala_$scalaBinaryVersion:$effectiveFasterxmlVersion") + + compileOnly("commons-io:commons-io:2.14.0") + + implementation("com.google.jimfs:jimfs:1.3.0") + + testImplementation("org.scalacheck:scalacheck_$scalaBinaryVersion:1.17.0") + testImplementation("org.mockito:mockito-core:2.23.4") { + exclude(group = "net.bytebuddy", module = "byte-buddy") + } + testImplementation("net.bytebuddy:byte-buddy:1.9.3") + testImplementation("org.scalatestplus:scalatestplus-mockito_$scalaBinaryVersion:1.0.0-M2") + testImplementation("org.scalatestplus:scalatestplus-scalacheck_$scalaBinaryVersion:3.1.0.0-RC2") + testImplementation("com.github.javafaker:javafaker:1.0.2") + testImplementation("com.vladsch.flexmark:flexmark-all:0.62.2") + + testImplementation("org.apache.spark:spark-tags_$scalaBinaryVersion:$effectiveSparkFullVersion:tests") +} + +sourceSets { + main { + proto { + srcDir("src/main/resources/org/apache/gluten/proto") + } + } +} + +// Celeborn: add src-celeborn source dirs and dependencies when enabled +if (providers.gradleProperty("celeborn").getOrElse("false").toBoolean()) { + val celebornVersion: String by project + val sparkMajorVersion = if (effectiveSparkFullVersion.startsWith("4.")) "4" else "3" + sourceSets { + main { + scala { + srcDir("src-celeborn/main/scala") + srcDir("src-celeborn/main/java") + srcDir("src-celeborn-spark$effectiveSparkPlainVersion/main/scala") + srcDir("src-celeborn-spark$effectiveSparkPlainVersion/main/java") + } + resources { + srcDir("src-celeborn/main/resources") + srcDir("src-celeborn-spark$effectiveSparkPlainVersion/main/resources") + } + } + } + dependencies { + implementation(project(":gluten-celeborn")) + compileOnly("org.apache.celeborn:celeborn-client-spark-${sparkMajorVersion}-shaded_$scalaBinaryVersion:$celebornVersion") + } +} + +// Uniffle: add src-uniffle source dirs and dependencies when enabled +if (providers.gradleProperty("uniffle").getOrElse("false").toBoolean()) { + val uniffleVersion: String by project + val sparkMajorVersion = if (effectiveSparkFullVersion.startsWith("4.")) "4" else "3" + sourceSets { + main { + scala { + srcDir("src-uniffle/main/scala") + srcDir("src-uniffle/main/java") + srcDir("src-uniffle-spark$effectiveSparkPlainVersion/main/scala") + srcDir("src-uniffle-spark$effectiveSparkPlainVersion/main/java") + } + resources { + srcDir("src-uniffle/main/resources") + srcDir("src-uniffle-spark$effectiveSparkPlainVersion/main/resources") + } + } + } + dependencies { + implementation(project(":gluten-uniffle")) + compileOnly("org.apache.uniffle:rss-client-spark${sparkMajorVersion}-shaded:$uniffleVersion") + } +} + +// ============================================================ +// Native C++ build tasks +// ============================================================ + +val glutenDir = rootProject.projectDir +val veloxHome = + providers.gradleProperty("veloxHome") + .getOrElse("$glutenDir/ep/build-velox/build/velox_ep") +val buildType = providers.gradleProperty("nativeBuildType").getOrElse("Release") +val numThreads = + providers.gradleProperty("nativeThreads") + .getOrElse(Runtime.getRuntime().availableProcessors().toString()) + +// Feature flags for native build (match dev/builddeps-veloxbe.sh defaults) +val enableHdfs = providers.gradleProperty("enableHdfs").getOrElse("OFF") +val enableS3 = providers.gradleProperty("enableS3").getOrElse("OFF") +val enableGcs = providers.gradleProperty("enableGcs").getOrElse("OFF") +val enableAbfs = providers.gradleProperty("enableAbfs").getOrElse("OFF") +val enableQat = providers.gradleProperty("enableQat").getOrElse("OFF") +val enableGpu = providers.gradleProperty("enableGpu").getOrElse("OFF") +val buildTests = providers.gradleProperty("nativeBuildTests").getOrElse("OFF") +val buildBenchmarks = providers.gradleProperty("nativeBuildBenchmarks").getOrElse("OFF") + +// Task 1: Fetch Velox source code +val getVelox by tasks.registering(Exec::class) { + group = "native" + description = "Fetch Velox source code" + + workingDir = file("$glutenDir/ep/build-velox/src") + + commandLine( + "bash", + "./get-velox.sh", + "--velox_home=$veloxHome", + "--run_setup_script=OFF", + ) + + // Skip if velox source already present with correct commit + onlyIf { + !file("$veloxHome/CMakeLists.txt").exists() + } +} + +// Task 2: Build Velox library +val buildVelox by tasks.registering(Exec::class) { + group = "native" + description = "Build the Velox C++ library" + dependsOn(getVelox) + + workingDir = file("$glutenDir/ep/build-velox/src") + + commandLine( + "bash", "./build-velox.sh", + "--build_type=$buildType", + "--velox_home=$veloxHome", + "--enable_hdfs=$enableHdfs", + "--enable_s3=$enableS3", + "--enable_gcs=$enableGcs", + "--enable_abfs=$enableAbfs", + "--enable_gpu=$enableGpu", + "--build_test_utils=$buildTests", + "--num_threads=$numThreads", + ) + + // Skip if Velox is already built + onlyIf { + val buildDir = if (buildType == "Debug") "debug" else "release" + !file("$veloxHome/_build/$buildDir/lib/libvelox.a").exists() + } +} + +// Task 3: Configure CMake for Gluten C++ +val configureNative by tasks.registering(Exec::class) { + group = "native" + description = "Configure CMake for Gluten C++ build" + dependsOn(buildVelox) + + workingDir = cppBuildDir + + doFirst { + cppBuildDir.mkdirs() + } + + commandLine( + "cmake", + "-DBUILD_VELOX_BACKEND=ON", + "-DCMAKE_BUILD_TYPE=$buildType", + "-DVELOX_HOME=$veloxHome", + "-DBUILD_TESTS=$buildTests", + "-DBUILD_BENCHMARKS=$buildBenchmarks", + "-DENABLE_HDFS=$enableHdfs", + "-DENABLE_S3=$enableS3", + "-DENABLE_GCS=$enableGcs", + "-DENABLE_ABFS=$enableAbfs", + "-DENABLE_QAT=$enableQat", + "-DENABLE_GPU=$enableGpu", + "-DCMAKE_EXPORT_COMPILE_COMMANDS=ON", + "..", + ) + + // Skip if CMake is already configured + onlyIf { + !file("$cppBuildDir/CMakeCache.txt").exists() + } +} + +// Task 4: Build Gluten C++ (libgluten.so + libvelox.so) +val buildNative by tasks.registering(Exec::class) { + group = "native" + description = "Build native C++ libraries (libgluten.so + libvelox.so)" + dependsOn(configureNative) + + workingDir = file("../cpp") + + commandLine( + "cmake", + "--build", + "build", + "--parallel", + numThreads, + ) + + outputs.dir(cppReleasesDir) +} + +// Convenience task: full native build pipeline +val buildNativeAll by tasks.registering { + group = "native" + description = "Full native build: fetch Velox + build Velox + configure + build Gluten C++" + dependsOn(buildNative) +} + +// Include native libraries in the JAR +val platform = rootProject.extra.get("platform") as String +val arch = rootProject.extra.get("arch") as String + +tasks.processResources { + from(cppReleasesDir) { + into("$platform/$arch") + } +} + +// Bridge path differences between Maven and Gradle test output layouts. +// Test code uses getClass.getResource("/").getPath + "../../../../tools/..." to locate +// shared resources. Maven's classpath root is target/test-classes/ (2 levels deep), +// so ../../../../ goes to project root. Gradle's is build/classes/scala/test/ (4 levels +// deep), so ../../../../ only reaches the module root. Create a symlink so the traversal +// resolves correctly from Gradle's deeper output directory. +// Note: build/src symlink is handled by gluten.scala-conventions plugin. +val createTestPathSymlinks by tasks.registering { + val toolsLink = file("tools") + outputs.upToDateWhen { toolsLink.exists() } + doLast { + // backends-velox/tools -> ../tools (for ../../../../tools/... from build/classes/scala/test/) + if (!toolsLink.exists()) { + exec { commandLine("ln", "-s", "../tools", toolsLink.absolutePath) } + } + } +} + +// Configure ScalaTest +tasks.withType().configureEach { + dependsOn(createTestPathSymlinks) + systemProperty("velox.udf.lib.path", "$cppBuildDir/velox/udf/examples/libmyudf.so,$cppBuildDir/velox/udf/examples/libmyudaf.so") + + // Reorder test classpath so that project (shim) classes appear before external JARs. + // Spark 3.3's native write path requires the shim's FileFormatWriter to shadow Spark's + // own version. Maven achieves this via dependency declaration order (compile-scoped shim + // before provided-scoped Spark). Gradle's compileOnly→testImplementation inheritance + // places Spark JARs first. Fix by partitioning the classpath: project outputs first, + // then external JARs. + val rootPath = rootProject.projectDir.absolutePath + doFirst { + val (projectFiles, externalFiles) = classpath.partition { + it.absolutePath.startsWith(rootPath) + } + classpath = files(projectFiles, externalFiles) + } +} diff --git a/build.gradle.kts b/build.gradle.kts new file mode 100644 index 00000000000..b03976bc863 --- /dev/null +++ b/build.gradle.kts @@ -0,0 +1,276 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + base + id("gluten.spotless") +} + +group = "org.apache.gluten" +version = providers.gradleProperty("glutenVersion").getOrElse("1.6.0-SNAPSHOT") + +val sparkVersion: String by project +val sparkFullVersion: String by project +val sparkPlainVersion: String by project +val javaVersion: String by project +val backend: String by project + +// Spark 3.x defaults to Scala 2.12, Spark 4.x defaults to Scala 2.13. +// An explicit -PscalaBinaryVersion on the command line always wins because +// Gradle command-line properties override gradle.properties values. +val defaultScalaBinaryVersion = if (sparkVersion.startsWith("3.")) "2.12" else "2.13" +val scalaBinaryVersion: String = + providers.gradleProperty("scalaBinaryVersion").getOrElse(defaultScalaBinaryVersion) +// Derive full Scala version from binary version +val scalaVersion: String = + when (scalaBinaryVersion) { + "2.12" -> "2.12.18" + "2.13" -> providers.gradleProperty("scalaVersion").getOrElse("2.13.17") + else -> providers.gradleProperty("scalaVersion").getOrElse("2.13.17") + } + +// Version properties for dependencies +val arrowVersion: String by project +val hadoopVersion: String by project +val caffeineVersion: String = + if ((javaVersion.toIntOrNull() ?: 17) >= 11) { + providers.gradleProperty("caffeineVersion").getOrElse("3.1.8") + } else { + "2.9.3" + } + +// Computed properties based on Spark version +val sparkProperties = + when (sparkVersion) { + "3.3" -> + mapOf( + "sparkFullVersion" to "3.3.1", + "sparkPlainVersion" to "33", + "deltaVersion" to "2.3.0", + "deltaBinaryVersion" to "23", + "deltaPackageName" to "delta-core", + "icebergVersion" to "1.5.0", + "icebergBinaryVersion" to "5", + "antlr4Version" to "4.8", + "hudiVersion" to "0.15.0", + "arrowVersion" to "15.0.0-gluten", + "fasterxmlVersion" to "2.15.0", + ) + "3.4" -> + mapOf( + "sparkFullVersion" to "3.4.4", + "sparkPlainVersion" to "34", + "deltaVersion" to "2.4.0", + "deltaBinaryVersion" to "24", + "deltaPackageName" to "delta-core", + "icebergVersion" to "1.10.0", + "icebergBinaryVersion" to "10", + "antlr4Version" to "4.9.3", + "hudiVersion" to "0.15.0", + "arrowVersion" to "15.0.0-gluten", + "fasterxmlVersion" to "2.15.0", + ) + "3.5" -> + mapOf( + "sparkFullVersion" to "3.5.5", + "sparkPlainVersion" to "35", + "deltaVersion" to "3.3.2", + "deltaBinaryVersion" to "33", + "deltaPackageName" to "delta-spark", + "icebergVersion" to "1.10.0", + "icebergBinaryVersion" to "10", + "antlr4Version" to "4.9.3", + "hudiVersion" to "0.15.0", + "hadoopVersion" to "3.3.4", + "arrowVersion" to "15.0.0-gluten", + "fasterxmlVersion" to "2.15.1", + ) + "4.0" -> + mapOf( + "sparkFullVersion" to "4.0.1", + "sparkPlainVersion" to "40", + "deltaVersion" to "4.0.0", + "deltaBinaryVersion" to "40", + "deltaPackageName" to "delta-spark", + "icebergVersion" to "1.10.0", + "icebergBinaryVersion" to "10", + "antlr4Version" to "4.13.1", + "hudiVersion" to "1.1.0", + "paimonVersion" to "1.3.0", + "hadoopVersion" to "3.4.1", + "arrowVersion" to "18.1.0", + "fasterxmlVersion" to "2.18.2", + ) + // "4.1" and default + else -> + mapOf( + "sparkFullVersion" to "4.1.1", + "sparkPlainVersion" to "41", + "deltaVersion" to "4.0.0", + "deltaBinaryVersion" to "40", + "deltaPackageName" to "delta-spark", + "icebergVersion" to "1.10.0", + "icebergBinaryVersion" to "10", + "antlr4Version" to "4.13.1", + "hudiVersion" to "1.1.0", + "paimonVersion" to "1.3.0", + "hadoopVersion" to "3.4.1", + "arrowVersion" to "18.1.0", + "fasterxmlVersion" to "2.18.2", + ) + } + +// Make computed properties available to subprojects +extra["effectiveSparkFullVersion"] = sparkProperties["sparkFullVersion"] ?: sparkFullVersion +extra["effectiveSparkPlainVersion"] = sparkProperties["sparkPlainVersion"] ?: sparkPlainVersion +extra["effectiveHadoopVersion"] = sparkProperties["hadoopVersion"] ?: hadoopVersion +extra["effectiveArrowVersion"] = sparkProperties["arrowVersion"] ?: arrowVersion +extra["effectiveDeltaVersion"] = sparkProperties["deltaVersion"] +extra["effectiveDeltaBinaryVersion"] = sparkProperties["deltaBinaryVersion"] +extra["effectiveDeltaPackageName"] = sparkProperties["deltaPackageName"] +extra["effectiveIcebergVersion"] = sparkProperties["icebergVersion"] +extra["effectiveIcebergBinaryVersion"] = sparkProperties["icebergBinaryVersion"] +extra["effectiveHudiVersion"] = sparkProperties["hudiVersion"] +extra["effectivePaimonVersion"] = sparkProperties["paimonVersion"] + +val antlr4Version: String by project +extra["effectiveAntlr4Version"] = sparkProperties["antlr4Version"] ?: antlr4Version + +val fasterxmlVersion: String by project +extra["effectiveFasterxmlVersion"] = sparkProperties["fasterxmlVersion"] ?: fasterxmlVersion + +// Detect OS and architecture +val osName = System.getProperty("os.name").lowercase() +val osArch = System.getProperty("os.arch") + +val platform = + when { + osName.contains("linux") -> "linux" + osName.contains("mac") || osName.contains("darwin") -> "darwin" + osName.contains("windows") -> "windows" + else -> "unknown" + } + +val arch = + when (osArch) { + "amd64", "x86_64" -> "amd64" + "aarch64", "arm64" -> "aarch64" + else -> osArch + } + +extra["platform"] = platform +extra["arch"] = arch + +// Validate version compatibility +if (sparkVersion in listOf("4.0", "4.1")) { + val javaVersionInt = javaVersion.toIntOrNull() ?: 17 + require(javaVersionInt >= 17) { + "Spark $sparkVersion requires Java 17+, but javaVersion=$javaVersion" + } + require(scalaBinaryVersion == "2.13") { + "Spark $sparkVersion requires Scala 2.13, but scalaBinaryVersion=$scalaBinaryVersion" + } +} + +allprojects { + group = rootProject.group + version = rootProject.version + // Override computed properties for all projects so convention plugins pick up the correct values + extra["scalaBinaryVersion"] = scalaBinaryVersion + extra["scalaVersion"] = scalaVersion + extra["caffeineVersion"] = caffeineVersion +} + +val protobufVersion: String by project + +subprojects { + // jackson-bom:2.13.4.1 (pulled by Spark 3.3's jackson-databind:2.13.4.1) does not + // exist on Maven Central. Substitute it with 2.13.5 which is the closest available version. + configurations.all { + resolutionStrategy.eachDependency { + if (requested.group == "com.fasterxml.jackson" && requested.name == "jackson-bom" && requested.version == "2.13.4.1") { + useVersion("2.13.5") + because("jackson-bom:2.13.4.1 does not exist on Maven Central") + } + if (requested.group == "com.google.protobuf" && requested.name == "protobuf-java") { + useVersion(protobufVersion) + because("Align protobuf-java version across all transitive dependencies") + } + } + } + + // Configure test tasks + tasks.withType().configureEach { + maxHeapSize = "4g" + + jvmArgs( + "-XX:+IgnoreUnrecognizedVMOptions", + "--add-opens=java.base/java.lang=ALL-UNNAMED", + "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED", + "--add-opens=java.base/java.lang.reflect=ALL-UNNAMED", + "--add-opens=java.base/java.io=ALL-UNNAMED", + "--add-opens=java.base/java.net=ALL-UNNAMED", + "--add-opens=java.base/java.nio=ALL-UNNAMED", + "--add-opens=java.base/java.util=ALL-UNNAMED", + "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED", + "--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED", + "--add-opens=java.base/jdk.internal.ref=ALL-UNNAMED", + "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED", + "--add-opens=java.base/sun.nio.cs=ALL-UNNAMED", + "--add-opens=java.base/sun.security.action=ALL-UNNAMED", + "--add-opens=java.base/sun.util.calendar=ALL-UNNAMED", + "-Djdk.reflect.useDirectMethodHandle=false", + "-Dio.netty.tryReflectionSetAccessible=true", + ) + + testLogging { + events("passed", "skipped", "failed") + showStandardStreams = false + exceptionFormat = org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL + } + } +} + +// Task to print build configuration +tasks.register("printConfig") { + group = "help" + description = "Print the current build configuration" + doLast { + println( + """ + |=== Gluten Build Configuration === + |Version: ${project.version} + |Scala: $scalaVersion (binary: $scalaBinaryVersion) + |Spark: $sparkVersion (full: ${project.extra["effectiveSparkFullVersion"]}) + |Java: $javaVersion + |Backend: $backend + |Platform: $platform + |Architecture: $arch + |Jackson/FasterXML: ${project.extra["effectiveFasterxmlVersion"]} + | + |Optional modules: + | delta: ${providers.gradleProperty("delta").getOrElse("false")} + | iceberg: ${providers.gradleProperty("iceberg").getOrElse("false")} + | hudi: ${providers.gradleProperty("hudi").getOrElse("false")} + | paimon: ${providers.gradleProperty("paimon").getOrElse("false")} + | celeborn: ${providers.gradleProperty("celeborn").getOrElse("false")} + | uniffle: ${providers.gradleProperty("uniffle").getOrElse("false")} + | kafka: ${providers.gradleProperty("kafka").getOrElse("false")} + """.trimMargin(), + ) + } +} diff --git a/dev/checkstyle.xml b/dev/checkstyle.xml index 39b7b158cef..cdd924d32ec 100644 --- a/dev/checkstyle.xml +++ b/dev/checkstyle.xml @@ -49,7 +49,7 @@ - + diff --git a/dev/ci-gradle-build.sh b/dev/ci-gradle-build.sh new file mode 100755 index 00000000000..31848802c44 --- /dev/null +++ b/dev/ci-gradle-build.sh @@ -0,0 +1,116 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Translates Maven-style profile/property arguments to Gradle properties and runs a Gradle build. +# This allows CI workflows to use the same arguments for both Maven and Gradle builds. +# +# Usage (build only): +# bash dev/ci-gradle-build.sh -Pspark-3.5 -Pjava-8 -Pbackends-velox -DskipTests +# +# Usage (build + test with tag filtering): +# bash dev/ci-gradle-build.sh -Pspark-3.5 -Pjava-17 -Pbackends-velox -Pspark-ut \ +# -DtagsToExclude=org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTest \ +# -DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" + +set -euo pipefail + +GRADLE_ARGS="" +SKIP_TESTS="" +GRADLE_TASK="build" +EXTRA_JVM_ARGS="" + +for arg in "$@"; do + case "$arg" in + # Spark/Java/Scala version profiles + -Pspark-ut) GRADLE_ARGS="$GRADLE_ARGS -Pspark-ut=true" ;; + -Pspark-*) GRADLE_ARGS="$GRADLE_ARGS -PsparkVersion=${arg#-Pspark-}" ;; + -Pjava-*) GRADLE_ARGS="$GRADLE_ARGS -PjavaVersion=${arg#-Pjava-}" ;; + -Pscala-*) GRADLE_ARGS="$GRADLE_ARGS -PscalaBinaryVersion=${arg#-Pscala-}" ;; + -Pbackends-*) GRADLE_ARGS="$GRADLE_ARGS -Pbackend=${arg#-Pbackends-}" ;; + # Hadoop version profile + -Phadoop-*) GRADLE_ARGS="$GRADLE_ARGS -PhadoopVersion=${arg#-Phadoop-}" ;; + # Optional modules + -Pdelta) GRADLE_ARGS="$GRADLE_ARGS -Pdelta=true" ;; + -Piceberg) GRADLE_ARGS="$GRADLE_ARGS -Piceberg=true" ;; + -Phudi) GRADLE_ARGS="$GRADLE_ARGS -Phudi=true" ;; + -Ppaimon) GRADLE_ARGS="$GRADLE_ARGS -Ppaimon=true" ;; + -Pceleborn) GRADLE_ARGS="$GRADLE_ARGS -Pceleborn=true" ;; + -Pceleborn-0.5) GRADLE_ARGS="$GRADLE_ARGS -Pceleborn=true -PcelebornVersion=0.5.4" ;; + -Pceleborn-0.6) GRADLE_ARGS="$GRADLE_ARGS -Pceleborn=true -PcelebornVersion=0.6.1" ;; + -Puniffle) GRADLE_ARGS="$GRADLE_ARGS -Puniffle=true" ;; + -Pkafka) GRADLE_ARGS="$GRADLE_ARGS -Pkafka=true" ;; + -PglutenIt*) GRADLE_ARGS="$GRADLE_ARGS $arg" ;; + # Test control + -DskipTests) SKIP_TESTS="-x test" ;; + # ScalaTest tag filtering + -DtagsToInclude=*) GRADLE_ARGS="$GRADLE_ARGS -PtagsToInclude=${arg#-DtagsToInclude=}" ;; + -DtagsToExclude=*) GRADLE_ARGS="$GRADLE_ARGS -PtagsToExclude=${arg#-DtagsToExclude=}" ;; + # Spark test home and extra JVM args (extracted from -DargLine) + -DargLine=*) + ARGLINE="${arg#-DargLine=}" + # Extract spark.test.home from argLine + if [[ "$ARGLINE" =~ -Dspark\.test\.home=([^ \"]+) ]]; then + GRADLE_ARGS="$GRADLE_ARGS -PsparkTestHome=${BASH_REMATCH[1]}" + fi + # Collect other -D system properties for test JVM + for prop in $ARGLINE; do + case "$prop" in + -Dspark.test.home=*) ;; # already handled above + -D*) + if [ -z "$EXTRA_JVM_ARGS" ]; then + EXTRA_JVM_ARGS="$prop" + else + EXTRA_JVM_ARGS="$EXTRA_JVM_ARGS,$prop" + fi + ;; + esac + done + ;; + # Ignore Maven-specific flags that have no Gradle equivalent + -Piceberg-test|-ntp|-pl|-am) ;; + -Dmaven.*|-Dtest=*|-DfailIfNoTests=*|-Dexec.skip) ;; + *) + echo "Warning: ignoring unknown argument: $arg" >&2 + ;; + esac +done + +# Pass extra JVM args if any were collected from -DargLine +if [ -n "$EXTRA_JVM_ARGS" ]; then + GRADLE_ARGS="$GRADLE_ARGS -PtestJvmArgs=$EXTRA_JVM_ARGS" +fi + +echo "Gradle command: ./gradlew $GRADLE_TASK $SKIP_TESTS $GRADLE_ARGS --no-daemon" + +# Pre-download Gradle distribution with retry to handle transient network failures. +# This only retries the download, not the actual build/test execution. +MAX_RETRIES=5 +RETRY_DELAY=10 +for i in $(seq 1 $MAX_RETRIES); do + if ./gradlew --version --no-daemon > /dev/null 2>&1; then + break + fi + if [ "$i" -eq "$MAX_RETRIES" ]; then + echo "Failed to download Gradle distribution after $MAX_RETRIES attempts" >&2 + exit 1 + fi + echo "Gradle distribution download failed (attempt $i/$MAX_RETRIES), retrying in ${RETRY_DELAY}s..." >&2 + sleep $RETRY_DELAY + RETRY_DELAY=$((RETRY_DELAY * 2)) +done + +# shellcheck disable=SC2086 +./gradlew $GRADLE_TASK $SKIP_TESTS $GRADLE_ARGS --no-daemon diff --git a/gluten-arrow/build.gradle.kts b/gluten-arrow/build.gradle.kts new file mode 100644 index 00000000000..00d641c8948 --- /dev/null +++ b/gluten-arrow/build.gradle.kts @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("gluten.scala-library") + id("gluten.scalatest") + id("gluten.spotless") +} + +val scalaBinaryVersion: String by project +val effectiveSparkFullVersion: String by rootProject.extra +val effectiveSparkPlainVersion: String by rootProject.extra +val effectiveArrowVersion: String by rootProject.extra + +// Allow switching between arrow-memory-unsafe (default) and arrow-memory-netty +// via -ParrowMemoryArtifact=arrow-memory-netty +val arrowMemoryArtifact = providers.gradleProperty("arrowMemoryArtifact").getOrElse("arrow-memory-unsafe") + +dependencies { + api(project(":gluten-substrait")) + implementation(project(":shims-common")) + implementation(project(":shims-spark$effectiveSparkPlainVersion")) + + api("org.apache.arrow:arrow-vector:$effectiveArrowVersion") + api("org.apache.arrow:arrow-memory-core:$effectiveArrowVersion") + implementation("org.apache.arrow:$arrowMemoryArtifact:$effectiveArrowVersion") + api("org.apache.arrow:arrow-c-data:$effectiveArrowVersion") + api("org.apache.arrow:arrow-dataset:$effectiveArrowVersion") + + compileOnly("org.apache.spark:spark-network-common_$scalaBinaryVersion:$effectiveSparkFullVersion") + + testImplementation(project(":gluten-core", "testArtifacts")) +} diff --git a/gluten-celeborn/build.gradle.kts b/gluten-celeborn/build.gradle.kts new file mode 100644 index 00000000000..224ad120457 --- /dev/null +++ b/gluten-celeborn/build.gradle.kts @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("gluten.scala-library") + id("gluten.spotless") +} + +// The Celeborn module's Java source has empty catch blocks that violate the main project's +// checkstyle rules. The Maven build doesn't run checkstyle here. +checkstyle { + isIgnoreFailures = true +} + +val scalaBinaryVersion: String by project +val effectiveSparkPlainVersion: String by rootProject.extra +val celebornVersion: String by project + +// Add Celeborn-specific source directories +sourceSets { + main { + scala { + srcDir("src-celeborn/main/scala") + srcDir("src-celeborn/main/java") + srcDir("src-celeborn-spark$effectiveSparkPlainVersion/main/scala") + srcDir("src-celeborn-spark$effectiveSparkPlainVersion/main/java") + } + resources { + srcDir("src-celeborn/main/resources") + srcDir("src-celeborn-spark$effectiveSparkPlainVersion/main/resources") + } + } + test { + scala { + srcDir("src-celeborn/test/scala") + srcDir("src-celeborn/test/java") + srcDir("src-celeborn-spark$effectiveSparkPlainVersion/test/scala") + srcDir("src-celeborn-spark$effectiveSparkPlainVersion/test/java") + } + resources { + srcDir("src-celeborn/test/resources") + srcDir("src-celeborn-spark$effectiveSparkPlainVersion/test/resources") + } + } +} + +dependencies { + implementation(project(":gluten-substrait")) + + compileOnly( + "org.apache.celeborn:celeborn-client-spark-${effectiveSparkPlainVersion.take(1)}-shaded_$scalaBinaryVersion:$celebornVersion", + ) +} diff --git a/gluten-core/build.gradle.kts b/gluten-core/build.gradle.kts new file mode 100644 index 00000000000..037b09692b4 --- /dev/null +++ b/gluten-core/build.gradle.kts @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.time.LocalDateTime + +plugins { + id("gluten.scala-library") + id("gluten.protobuf") + id("gluten.scalatest") + id("gluten.spotless") +} + +val scalaBinaryVersion: String by project +val scalaVersion: String by project +val protobufVersion: String by project +val effectiveSparkFullVersion: String by rootProject.extra +val effectiveHadoopVersion: String by rootProject.extra + +val backend: String by project + +dependencies { + api(project(":gluten-ras-common")) + + compileOnly("org.apache.spark:spark-kvstore_$scalaBinaryVersion:$effectiveSparkFullVersion") + compileOnly("org.apache.spark:spark-network-common_$scalaBinaryVersion:$effectiveSparkFullVersion") + compileOnly("org.apache.spark:spark-network-shuffle_$scalaBinaryVersion:$effectiveSparkFullVersion") + + compileOnly("org.apache.hadoop:hadoop-client:$effectiveHadoopVersion") + + compileOnly("com.google.protobuf:protobuf-java:$protobufVersion") + + testImplementation("org.scalacheck:scalacheck_$scalaBinaryVersion:1.17.0") + testImplementation("org.mockito:mockito-core:2.23.4") + testImplementation("org.scalatestplus:scalatestplus-mockito_$scalaBinaryVersion:1.0.0-M2") + testImplementation("org.scalatestplus:scalatestplus-scalacheck_$scalaBinaryVersion:3.1.0.0-RC2") +} + +sourceSets { + main { + proto { + srcDir("src/main/resources/org/apache/gluten/proto") + } + } +} + +// Generate build info +val generateBuildInfo by tasks.registering { + group = "build" + description = "Generate Gluten build info properties file" + + // Declare inputs so Gradle re-runs this task when build properties change + inputs.property("sparkVersion", effectiveSparkFullVersion) + inputs.property("scalaVersion", scalaVersion) + inputs.property("hadoopVersion", effectiveHadoopVersion) + inputs.property("backend", backend) + inputs.property("projectVersion", project.version) + + val outputDir = layout.buildDirectory.dir("generated-resources") + outputs.dir(outputDir) + + doLast { + val propsFile = outputDir.get().file("gluten-build-info.properties").asFile + propsFile.parentFile.mkdirs() + + fun gitOutput(vararg args: String): String = + try { + ProcessBuilder(*args) + .directory(projectDir) + .redirectErrorStream(true) + .start() + .inputStream.bufferedReader().readText().trim() + } catch (e: Exception) { + "unknown" + } + + propsFile.writeText( + """ + |gluten_version=${project.version} + |backend_type=$backend + |branch=${gitOutput("git", "rev-parse", "--abbrev-ref", "HEAD")} + |revision=${gitOutput("git", "rev-parse", "HEAD")} + |revision_time=${gitOutput("git", "show", "-s", "--format=%ci", "HEAD")} + |url=${gitOutput("git", "config", "--get", "remote.origin.url")} + |java_version=${System.getProperty("java.version")} + |scala_version=$scalaVersion + |spark_version=$effectiveSparkFullVersion + |hadoop_version=$effectiveHadoopVersion + |build_user=${System.getProperty("user.name")} + |date=${LocalDateTime.now()} + """.trimMargin(), + ) + } +} + +tasks.processResources { + dependsOn(generateBuildInfo) + from(layout.buildDirectory.dir("generated-resources")) +} diff --git a/gluten-delta/build.gradle.kts b/gluten-delta/build.gradle.kts new file mode 100644 index 00000000000..b907e9456c2 --- /dev/null +++ b/gluten-delta/build.gradle.kts @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("gluten.scala-library") + id("gluten.scalatest") + id("gluten.spotless") +} + +val scalaBinaryVersion: String by project +val sparkVersion: String by project +val effectiveSparkPlainVersion: String by rootProject.extra +val effectiveDeltaVersion: String? by rootProject.extra +val effectiveDeltaBinaryVersion: String? by rootProject.extra +val effectiveDeltaPackageName: String? by rootProject.extra + +val deltaVersion = effectiveDeltaVersion ?: "4.0.0" +val deltaBinaryVersion = effectiveDeltaBinaryVersion ?: "40" +val deltaPackageName = effectiveDeltaPackageName ?: "delta-spark" + +// Add Delta-specific source directories +sourceSets { + main { + scala { + srcDir("src-delta/main/scala") + srcDir("src-delta/main/java") + srcDir("src-delta-spark$effectiveSparkPlainVersion/main/scala") + srcDir("src-delta-spark$effectiveSparkPlainVersion/main/java") + srcDir("src-delta$deltaBinaryVersion/main/scala") + srcDir("src-delta$deltaBinaryVersion/main/java") + srcDir("src-delta$deltaBinaryVersion-spark$effectiveSparkPlainVersion/main/scala") + srcDir("src-delta$deltaBinaryVersion-spark$effectiveSparkPlainVersion/main/java") + } + resources { + srcDir("src-delta/main/resources") + srcDir("src-delta-spark$effectiveSparkPlainVersion/main/resources") + srcDir("src-delta$deltaBinaryVersion/main/resources") + srcDir("src-delta$deltaBinaryVersion-spark$effectiveSparkPlainVersion/main/resources") + } + } + test { + scala { + srcDir("src-delta/test/scala") + srcDir("src-delta/test/java") + srcDir("src-delta-spark$effectiveSparkPlainVersion/test/scala") + srcDir("src-delta-spark$effectiveSparkPlainVersion/test/java") + srcDir("src-delta$deltaBinaryVersion/test/scala") + srcDir("src-delta$deltaBinaryVersion/test/java") + srcDir("src-delta$deltaBinaryVersion-spark$effectiveSparkPlainVersion/test/scala") + srcDir("src-delta$deltaBinaryVersion-spark$effectiveSparkPlainVersion/test/java") + } + resources { + srcDir("src-delta/test/resources") + srcDir("src-delta-spark$effectiveSparkPlainVersion/test/resources") + srcDir("src-delta$deltaBinaryVersion/test/resources") + srcDir("src-delta$deltaBinaryVersion-spark$effectiveSparkPlainVersion/test/resources") + } + } +} + +dependencies { + implementation(project(":gluten-substrait")) + + compileOnly("io.delta:${deltaPackageName}_$scalaBinaryVersion:$deltaVersion") { + exclude(group = "org.antlr") + exclude(group = "org.scala-lang", module = "scala-library") + } + + testImplementation(project(":backends-velox", "testArtifacts")) + testImplementation(project(":gluten-substrait", "testArtifacts")) + testImplementation("io.delta:${deltaPackageName}_$scalaBinaryVersion:$deltaVersion") { + exclude(group = "org.antlr") + exclude(group = "org.scala-lang", module = "scala-library") + } +} diff --git a/gluten-hudi/build.gradle.kts b/gluten-hudi/build.gradle.kts new file mode 100644 index 00000000000..0c9c064a3b6 --- /dev/null +++ b/gluten-hudi/build.gradle.kts @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("gluten.scala-library") + id("gluten.scalatest") + id("gluten.spotless") +} + +val scalaBinaryVersion: String by project +val sparkVersion: String by project +val effectiveSparkPlainVersion: String by rootProject.extra +val effectiveHudiVersion: String? by rootProject.extra + +val hudiVersion = effectiveHudiVersion ?: "0.15.0" + +// Add Hudi-specific source directories +sourceSets { + main { + scala { + srcDir("src-hudi/main/scala") + srcDir("src-hudi/main/java") + srcDir("src-hudi-spark$effectiveSparkPlainVersion/main/scala") + srcDir("src-hudi-spark$effectiveSparkPlainVersion/main/java") + } + resources { + srcDir("src-hudi/main/resources") + srcDir("src-hudi-spark$effectiveSparkPlainVersion/main/resources") + } + } + test { + scala { + srcDir("src-hudi/test/scala") + srcDir("src-hudi/test/java") + srcDir("src-hudi-spark$effectiveSparkPlainVersion/test/scala") + srcDir("src-hudi-spark$effectiveSparkPlainVersion/test/java") + } + resources { + srcDir("src-hudi/test/resources") + srcDir("src-hudi-spark$effectiveSparkPlainVersion/test/resources") + } + } +} + +dependencies { + implementation(project(":gluten-substrait")) + + compileOnly( + "org.apache.hudi:hudi-spark$sparkVersion-bundle_$scalaBinaryVersion:$hudiVersion", + ) + + testImplementation(project(":backends-velox", "testArtifacts")) + testImplementation(project(":gluten-substrait", "testArtifacts")) + testImplementation( + "org.apache.hudi:hudi-spark$sparkVersion-bundle_$scalaBinaryVersion:$hudiVersion", + ) +} diff --git a/gluten-iceberg/build.gradle.kts b/gluten-iceberg/build.gradle.kts new file mode 100644 index 00000000000..7e4c5caf899 --- /dev/null +++ b/gluten-iceberg/build.gradle.kts @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("gluten.scala-library") + id("gluten.scalatest") + id("gluten.spotless") +} + +val scalaBinaryVersion: String by project +val effectiveSparkPlainVersion: String by rootProject.extra +val effectiveIcebergVersion: String? by rootProject.extra +val effectiveIcebergBinaryVersion: String? by rootProject.extra + +val icebergVersion = effectiveIcebergVersion ?: "1.10.0" +val icebergBinaryVersion = effectiveIcebergBinaryVersion ?: "10" + +// Add Iceberg-specific source directories +sourceSets { + main { + scala { + srcDir("src-iceberg/main/scala") + srcDir("src-iceberg/main/java") + srcDir("src-iceberg$icebergBinaryVersion/main/scala") + srcDir("src-iceberg$icebergBinaryVersion/main/java") + srcDir("src-iceberg-spark$effectiveSparkPlainVersion/main/scala") + srcDir("src-iceberg-spark$effectiveSparkPlainVersion/main/java") + } + resources { + srcDir("src-iceberg/main/resources") + } + } + test { + scala { + srcDir("src-iceberg/test/scala") + srcDir("src-iceberg/test/java") + srcDir("src-iceberg$icebergBinaryVersion/test/scala") + srcDir("src-iceberg$icebergBinaryVersion/test/java") + } + resources { + srcDir("src-iceberg/test/resources") + srcDir("src-iceberg$icebergBinaryVersion/test/resources") + } + } +} + +dependencies { + implementation(project(":gluten-substrait")) + + compileOnly( + "org.apache.iceberg:iceberg-spark-runtime-${effectiveSparkPlainVersion.take( + 1, + )}.${effectiveSparkPlainVersion.drop(1)}_$scalaBinaryVersion:$icebergVersion", + ) + + testImplementation(project(":backends-velox", "testArtifacts")) + testImplementation(project(":gluten-substrait", "testArtifacts")) + testImplementation( + "org.apache.iceberg:iceberg-spark-runtime-${effectiveSparkPlainVersion.take( + 1, + )}.${effectiveSparkPlainVersion.drop(1)}_$scalaBinaryVersion:$icebergVersion", + ) +} diff --git a/gluten-kafka/build.gradle.kts b/gluten-kafka/build.gradle.kts new file mode 100644 index 00000000000..90688d0f6f0 --- /dev/null +++ b/gluten-kafka/build.gradle.kts @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("gluten.scala-library") + id("gluten.spotless") +} + +val scalaBinaryVersion: String by project +val protobufVersion: String by project +val effectiveSparkFullVersion: String by rootProject.extra +val effectiveHadoopVersion: String by rootProject.extra + +dependencies { + compileOnly(project(":gluten-substrait")) + + compileOnly("org.apache.spark:spark-sql-kafka-0-10_$scalaBinaryVersion:$effectiveSparkFullVersion") + + testImplementation(project(":gluten-substrait", "testArtifacts")) + testImplementation("org.apache.spark:spark-hive_$scalaBinaryVersion:$effectiveSparkFullVersion") + testImplementation("org.apache.hadoop:hadoop-client:$effectiveHadoopVersion") + testImplementation("com.google.protobuf:protobuf-java:$protobufVersion") +} diff --git a/gluten-paimon/build.gradle.kts b/gluten-paimon/build.gradle.kts new file mode 100644 index 00000000000..fa7ba654426 --- /dev/null +++ b/gluten-paimon/build.gradle.kts @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("gluten.scala-library") + id("gluten.scalatest") + id("gluten.spotless") +} + +val sparkVersion: String by project +val effectivePaimonVersion: String? by rootProject.extra + +val paimonVersion = effectivePaimonVersion ?: "1.3.0" + +// Add Paimon-specific source directories +sourceSets { + main { + scala { + srcDir("src-paimon/main/scala") + srcDir("src-paimon/main/java") + } + resources { + srcDir("src-paimon/main/resources") + } + } + test { + scala { + srcDir("src-paimon/test/scala") + srcDir("src-paimon/test/java") + } + resources { + srcDir("src-paimon/test/resources") + } + } +} + +dependencies { + implementation(project(":gluten-substrait")) + + compileOnly("org.apache.paimon:paimon-spark-${sparkVersion}:$paimonVersion") + + testImplementation(project(":backends-velox", "testArtifacts")) + testImplementation(project(":gluten-substrait", "testArtifacts")) + testImplementation("org.apache.paimon:paimon-spark-${sparkVersion}:$paimonVersion") +} diff --git a/gluten-ras/common/build.gradle.kts b/gluten-ras/common/build.gradle.kts new file mode 100644 index 00000000000..905de50e203 --- /dev/null +++ b/gluten-ras/common/build.gradle.kts @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("gluten.scala-library") + id("gluten.scalatest") + id("gluten.spotless") +} + +val scalaBinaryVersion: String by project +val caffeineVersion: String by project + +dependencies { + implementation("com.github.ben-manes.caffeine:caffeine:$caffeineVersion") + + testImplementation("org.scalatest:scalatest_$scalaBinaryVersion:3.2.16") + testImplementation("junit:junit:4.13.1") +} diff --git a/gluten-ras/planner/build.gradle.kts b/gluten-ras/planner/build.gradle.kts new file mode 100644 index 00000000000..e0719384bfd --- /dev/null +++ b/gluten-ras/planner/build.gradle.kts @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("gluten.scala-conventions") + id("gluten.scalatest") + id("gluten.spotless") +} + +val scalaBinaryVersion: String by project + +dependencies { + testImplementation("org.scalatest:scalatest_$scalaBinaryVersion:3.2.16") + testImplementation("junit:junit:4.13.1") +} diff --git a/gluten-substrait/build.gradle.kts b/gluten-substrait/build.gradle.kts new file mode 100644 index 00000000000..2e537f6c162 --- /dev/null +++ b/gluten-substrait/build.gradle.kts @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("gluten.scala-library") + id("gluten.protobuf") + id("gluten.scalatest") + id("gluten.spotless") +} + +val scalaBinaryVersion: String by project +val protobufVersion: String by project +val sparkVersion: String by project +val effectiveSparkFullVersion: String by rootProject.extra +val effectiveSparkPlainVersion: String by rootProject.extra +val effectiveHadoopVersion: String by rootProject.extra +val effectiveFasterxmlVersion: String by rootProject.extra + +dependencies { + api(project(":gluten-core")) + api(project(":shims-common")) + api(project(":shims-spark$effectiveSparkPlainVersion")) + implementation(project(":gluten-ui")) + + testImplementation(project(":gluten-core", "testArtifacts")) + + compileOnly("org.apache.spark:spark-network-common_$scalaBinaryVersion:$effectiveSparkFullVersion") + compileOnly("org.apache.spark:spark-launcher_$scalaBinaryVersion:$effectiveSparkFullVersion") + compileOnly("org.apache.spark:spark-kvstore_$scalaBinaryVersion:$effectiveSparkFullVersion") + + compileOnly("org.apache.hadoop:hadoop-client:$effectiveHadoopVersion") + + compileOnly("com.google.protobuf:protobuf-java:$protobufVersion") + implementation("com.google.protobuf:protobuf-java-util:$protobufVersion") + + compileOnly("commons-io:commons-io:2.14.0") + + compileOnly("com.fasterxml.jackson.core:jackson-databind:$effectiveFasterxmlVersion") + compileOnly("com.fasterxml.jackson.core:jackson-annotations:$effectiveFasterxmlVersion") + compileOnly("com.fasterxml.jackson.core:jackson-core:$effectiveFasterxmlVersion") + compileOnly("com.fasterxml.jackson.module:jackson-module-scala_$scalaBinaryVersion:$effectiveFasterxmlVersion") + + testImplementation("org.scalacheck:scalacheck_$scalaBinaryVersion:1.17.0") + testImplementation("org.mockito:mockito-core:2.23.4") + testImplementation("org.scalatestplus:scalatestplus-mockito_$scalaBinaryVersion:1.0.0-M2") + testImplementation("org.scalatestplus:scalatestplus-scalacheck_$scalaBinaryVersion:3.1.0.0-RC2") + testImplementation("com.vladsch.flexmark:flexmark-all:0.62.2") + testImplementation("com.github.javafaker:javafaker:1.0.2") +} + +sourceSets { + main { + proto { + srcDir("src/main/resources/substrait/proto") + } + } +} diff --git a/gluten-ui/build.gradle.kts b/gluten-ui/build.gradle.kts new file mode 100644 index 00000000000..7a120a8a6df --- /dev/null +++ b/gluten-ui/build.gradle.kts @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("gluten.scala-library") + id("gluten.spotless") +} + +val scalaBinaryVersion: String by project +val effectiveSparkFullVersion: String by rootProject.extra +val effectiveSparkPlainVersion: String by rootProject.extra +val effectiveFasterxmlVersion: String by rootProject.extra + +dependencies { + compileOnly(project(":shims-common")) + implementation(project(":shims-spark$effectiveSparkPlainVersion")) + + compileOnly("org.apache.spark:spark-kvstore_$scalaBinaryVersion:$effectiveSparkFullVersion") + + compileOnly("javax.servlet:javax.servlet-api:3.1.0") + + compileOnly("com.fasterxml.jackson.core:jackson-databind:$effectiveFasterxmlVersion") + compileOnly("com.fasterxml.jackson.core:jackson-annotations:$effectiveFasterxmlVersion") + compileOnly("com.fasterxml.jackson.core:jackson-core:$effectiveFasterxmlVersion") + + compileOnly("org.scala-lang.modules:scala-xml_$scalaBinaryVersion:2.2.0") +} diff --git a/gluten-uniffle/build.gradle.kts b/gluten-uniffle/build.gradle.kts new file mode 100644 index 00000000000..31ce1c3ea90 --- /dev/null +++ b/gluten-uniffle/build.gradle.kts @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("gluten.scala-library") + id("gluten.spotless") +} + +val effectiveSparkPlainVersion: String by rootProject.extra +val uniffleVersion: String by project + +// Add Uniffle-specific source directories +sourceSets { + main { + scala { + srcDir("src-uniffle/main/scala") + srcDir("src-uniffle/main/java") + srcDir("src-uniffle-spark$effectiveSparkPlainVersion/main/scala") + srcDir("src-uniffle-spark$effectiveSparkPlainVersion/main/java") + } + resources { + srcDir("src-uniffle/main/resources") + srcDir("src-uniffle-spark$effectiveSparkPlainVersion/main/resources") + } + } + test { + scala { + srcDir("src-uniffle/test/scala") + srcDir("src-uniffle/test/java") + srcDir("src-uniffle-spark$effectiveSparkPlainVersion/test/scala") + srcDir("src-uniffle-spark$effectiveSparkPlainVersion/test/java") + } + resources { + srcDir("src-uniffle/test/resources") + srcDir("src-uniffle-spark$effectiveSparkPlainVersion/test/resources") + } + } +} + +dependencies { + implementation(project(":gluten-substrait")) + + compileOnly("org.apache.uniffle:rss-client-spark${effectiveSparkPlainVersion.take(1)}-shaded:$uniffleVersion") +} diff --git a/gluten-ut/common/build.gradle.kts b/gluten-ut/common/build.gradle.kts new file mode 100644 index 00000000000..1d5643a86e3 --- /dev/null +++ b/gluten-ut/common/build.gradle.kts @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("gluten.scala-conventions") + id("gluten.spotless") +} + +val scalaBinaryVersion: String by project +val effectiveSparkFullVersion: String by rootProject.extra + +dependencies { + implementation(project(":gluten-substrait")) + + compileOnly("org.apache.spark:spark-sql_$scalaBinaryVersion:$effectiveSparkFullVersion") + compileOnly("org.apache.spark:spark-core_$scalaBinaryVersion:$effectiveSparkFullVersion") + compileOnly("org.apache.spark:spark-catalyst_$scalaBinaryVersion:$effectiveSparkFullVersion") + compileOnly("org.apache.spark:spark-hive_$scalaBinaryVersion:$effectiveSparkFullVersion") + + implementation("org.scalatest:scalatest_$scalaBinaryVersion:3.2.16") + implementation("junit:junit:4.13.1") + implementation("com.vladsch.flexmark:flexmark-all:0.62.2") + implementation("org.scalacheck:scalacheck_$scalaBinaryVersion:1.17.0") + implementation("org.scalatestplus:scalatestplus-scalacheck_$scalaBinaryVersion:3.1.0.0-RC2") + + implementation("org.apache.spark:spark-core_$scalaBinaryVersion:$effectiveSparkFullVersion:tests") + implementation("org.apache.spark:spark-sql_$scalaBinaryVersion:$effectiveSparkFullVersion:tests") + implementation("org.apache.spark:spark-catalyst_$scalaBinaryVersion:$effectiveSparkFullVersion:tests") + implementation("org.apache.spark:spark-hive_$scalaBinaryVersion:$effectiveSparkFullVersion:tests") +} + +// Create test JAR so downstream UT modules can access test utilities +// (e.g. BackendTestUtils, GlutenTestsBaseTrait) +val testJar by tasks.registering(Jar::class) { + archiveClassifier.set("tests") + from(sourceSets.test.get().output) +} + +artifacts { + add("archives", testJar) +} + +configurations { + create("testArtifacts") { + extendsFrom(configurations.testRuntimeClasspath.get()) + } +} + +artifacts { + add("testArtifacts", testJar) +} diff --git a/gluten-ut/spark33/build.gradle.kts b/gluten-ut/spark33/build.gradle.kts new file mode 100644 index 00000000000..6155f871920 --- /dev/null +++ b/gluten-ut/spark33/build.gradle.kts @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("gluten.scala-conventions") + id("gluten.scalatest") + id("gluten.spotless") +} + +val scalaBinaryVersion: String by project +val backend: String by project + +// Spark 3.3 specific versions +val sparkFullVersion = "3.3.1" + +dependencies { + implementation(project(":gluten-ut-common")) + testImplementation(project(":gluten-ut-common", "testArtifacts")) + implementation(project(":gluten-ut-test")) + + if (backend == "velox") { + implementation(project(":backends-velox")) + testImplementation(project(":backends-velox", "testArtifacts")) + } + + compileOnly("org.apache.spark:spark-sql_$scalaBinaryVersion:$sparkFullVersion") + compileOnly("org.apache.spark:spark-core_$scalaBinaryVersion:$sparkFullVersion") + compileOnly("org.apache.spark:spark-catalyst_$scalaBinaryVersion:$sparkFullVersion") + compileOnly("org.apache.spark:spark-hive_$scalaBinaryVersion:$sparkFullVersion") + + testImplementation("org.scalatest:scalatest_$scalaBinaryVersion:3.2.16") + testImplementation("junit:junit:4.13.1") + testImplementation("com.vladsch.flexmark:flexmark-all:0.62.2") + + testImplementation("org.apache.spark:spark-core_$scalaBinaryVersion:$sparkFullVersion:tests") + testImplementation("org.apache.spark:spark-sql_$scalaBinaryVersion:$sparkFullVersion:tests") + testImplementation("org.apache.spark:spark-catalyst_$scalaBinaryVersion:$sparkFullVersion:tests") + testImplementation("org.apache.spark:spark-hive_$scalaBinaryVersion:$sparkFullVersion:tests") +} + +// Add backend-specific test source directory +sourceSets { + test { + scala { + srcDir("src/test/backends-$backend") + } + } +} diff --git a/gluten-ut/spark34/build.gradle.kts b/gluten-ut/spark34/build.gradle.kts new file mode 100644 index 00000000000..e6170655ea7 --- /dev/null +++ b/gluten-ut/spark34/build.gradle.kts @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("gluten.scala-conventions") + id("gluten.scalatest") + id("gluten.spotless") +} + +val scalaBinaryVersion: String by project +val backend: String by project + +// Spark 3.4 specific versions +val sparkFullVersion = "3.4.4" + +dependencies { + implementation(project(":gluten-ut-common")) + testImplementation(project(":gluten-ut-common", "testArtifacts")) + implementation(project(":gluten-ut-test")) + + if (backend == "velox") { + implementation(project(":backends-velox")) + testImplementation(project(":backends-velox", "testArtifacts")) + } + + compileOnly("org.apache.spark:spark-sql_$scalaBinaryVersion:$sparkFullVersion") + compileOnly("org.apache.spark:spark-core_$scalaBinaryVersion:$sparkFullVersion") + compileOnly("org.apache.spark:spark-catalyst_$scalaBinaryVersion:$sparkFullVersion") + compileOnly("org.apache.spark:spark-hive_$scalaBinaryVersion:$sparkFullVersion") + + testImplementation("org.scalatest:scalatest_$scalaBinaryVersion:3.2.16") + testImplementation("junit:junit:4.13.1") + testImplementation("com.vladsch.flexmark:flexmark-all:0.62.2") + + testImplementation("org.apache.spark:spark-core_$scalaBinaryVersion:$sparkFullVersion:tests") + testImplementation("org.apache.spark:spark-sql_$scalaBinaryVersion:$sparkFullVersion:tests") + testImplementation("org.apache.spark:spark-catalyst_$scalaBinaryVersion:$sparkFullVersion:tests") + testImplementation("org.apache.spark:spark-hive_$scalaBinaryVersion:$sparkFullVersion:tests") +} diff --git a/gluten-ut/spark35/build.gradle.kts b/gluten-ut/spark35/build.gradle.kts new file mode 100644 index 00000000000..9a1e5804161 --- /dev/null +++ b/gluten-ut/spark35/build.gradle.kts @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("gluten.scala-conventions") + id("gluten.scalatest") + id("gluten.spotless") +} + +val scalaBinaryVersion: String by project +val backend: String by project + +// Spark 3.5 specific versions +val sparkFullVersion = "3.5.5" + +dependencies { + implementation(project(":gluten-ut-common")) + testImplementation(project(":gluten-ut-common", "testArtifacts")) + implementation(project(":gluten-ut-test")) + + if (backend == "velox") { + implementation(project(":backends-velox")) + testImplementation(project(":backends-velox", "testArtifacts")) + } + + compileOnly("org.apache.spark:spark-sql_$scalaBinaryVersion:$sparkFullVersion") + compileOnly("org.apache.spark:spark-core_$scalaBinaryVersion:$sparkFullVersion") + compileOnly("org.apache.spark:spark-catalyst_$scalaBinaryVersion:$sparkFullVersion") + compileOnly("org.apache.spark:spark-hive_$scalaBinaryVersion:$sparkFullVersion") + + testImplementation("org.scalatest:scalatest_$scalaBinaryVersion:3.2.16") + testImplementation("junit:junit:4.13.1") + testImplementation("com.vladsch.flexmark:flexmark-all:0.62.2") + + testImplementation("org.apache.spark:spark-core_$scalaBinaryVersion:$sparkFullVersion:tests") + testImplementation("org.apache.spark:spark-sql_$scalaBinaryVersion:$sparkFullVersion:tests") + testImplementation("org.apache.spark:spark-catalyst_$scalaBinaryVersion:$sparkFullVersion:tests") + testImplementation("org.apache.spark:spark-hive_$scalaBinaryVersion:$sparkFullVersion:tests") +} + +// Add backend-specific test source directory +sourceSets { + test { + scala { + srcDir("src/test/backends-$backend") + } + } +} diff --git a/gluten-ut/spark40/build.gradle.kts b/gluten-ut/spark40/build.gradle.kts new file mode 100644 index 00000000000..03b0262d2e9 --- /dev/null +++ b/gluten-ut/spark40/build.gradle.kts @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("gluten.scala-conventions") + id("gluten.scalatest") + id("gluten.spotless") +} + +// Spark 4.0 requires Scala 2.13 +val scalaBinaryVersion = "2.13" +val backend: String by project + +// Spark 4.0 specific versions +val sparkFullVersion = "4.0.1" + +dependencies { + implementation(project(":gluten-ut-common")) + testImplementation(project(":gluten-ut-common", "testArtifacts")) + implementation(project(":gluten-ut-test")) + + if (backend == "velox") { + implementation(project(":backends-velox")) + testImplementation(project(":backends-velox", "testArtifacts")) + } + + compileOnly("org.apache.spark:spark-sql_$scalaBinaryVersion:$sparkFullVersion") + compileOnly("org.apache.spark:spark-core_$scalaBinaryVersion:$sparkFullVersion") + compileOnly("org.apache.spark:spark-catalyst_$scalaBinaryVersion:$sparkFullVersion") + compileOnly("org.apache.spark:spark-hive_$scalaBinaryVersion:$sparkFullVersion") + + testImplementation("org.scalatest:scalatest_$scalaBinaryVersion:3.2.16") + testImplementation("junit:junit:4.13.1") + testImplementation("com.vladsch.flexmark:flexmark-all:0.62.2") + + testImplementation("org.apache.spark:spark-core_$scalaBinaryVersion:$sparkFullVersion:tests") + testImplementation("org.apache.spark:spark-sql_$scalaBinaryVersion:$sparkFullVersion:tests") + testImplementation("org.apache.spark:spark-catalyst_$scalaBinaryVersion:$sparkFullVersion:tests") + testImplementation("org.apache.spark:spark-hive_$scalaBinaryVersion:$sparkFullVersion:tests") +} + +// Add backend-specific test source directory +sourceSets { + test { + scala { + srcDir("src/test/backends-$backend") + } + } +} + +// Spark 4.0 requires Java 17+ +java { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 +} diff --git a/gluten-ut/spark41/build.gradle.kts b/gluten-ut/spark41/build.gradle.kts new file mode 100644 index 00000000000..e708750a25c --- /dev/null +++ b/gluten-ut/spark41/build.gradle.kts @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("gluten.scala-conventions") + id("gluten.scalatest") + id("gluten.spotless") +} + +// Spark 4.1 requires Scala 2.13 +val scalaBinaryVersion = "2.13" +val backend: String by project + +// Spark 4.1 specific versions +val sparkFullVersion = "4.1.1" + +dependencies { + implementation(project(":gluten-ut-common")) + testImplementation(project(":gluten-ut-common", "testArtifacts")) + implementation(project(":gluten-ut-test")) + + if (backend == "velox") { + implementation(project(":backends-velox")) + testImplementation(project(":backends-velox", "testArtifacts")) + } + + compileOnly("org.apache.spark:spark-sql_$scalaBinaryVersion:$sparkFullVersion") + compileOnly("org.apache.spark:spark-core_$scalaBinaryVersion:$sparkFullVersion") + compileOnly("org.apache.spark:spark-catalyst_$scalaBinaryVersion:$sparkFullVersion") + compileOnly("org.apache.spark:spark-hive_$scalaBinaryVersion:$sparkFullVersion") + + testImplementation("org.scalatest:scalatest_$scalaBinaryVersion:3.2.16") + testImplementation("junit:junit:4.13.1") + testImplementation("com.vladsch.flexmark:flexmark-all:0.62.2") + + testImplementation("org.apache.spark:spark-core_$scalaBinaryVersion:$sparkFullVersion:tests") + testImplementation("org.apache.spark:spark-sql_$scalaBinaryVersion:$sparkFullVersion:tests") + testImplementation("org.apache.spark:spark-catalyst_$scalaBinaryVersion:$sparkFullVersion:tests") + testImplementation("org.apache.spark:spark-hive_$scalaBinaryVersion:$sparkFullVersion:tests") +} + +// Add backend-specific test source directory +sourceSets { + test { + scala { + srcDir("src/test/backends-$backend") + } + } +} + +// Spark 4.1 requires Java 17+ +java { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 +} diff --git a/gluten-ut/test/build.gradle.kts b/gluten-ut/test/build.gradle.kts new file mode 100644 index 00000000000..80741606d7c --- /dev/null +++ b/gluten-ut/test/build.gradle.kts @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("gluten.scala-conventions") + id("gluten.scalatest") + id("gluten.spotless") +} + +val scalaBinaryVersion: String by project +val effectiveSparkFullVersion: String by rootProject.extra +val backend: String by project + +dependencies { + implementation(project(":gluten-ut-common")) + testImplementation(project(":gluten-ut-common", "testArtifacts")) + + if (backend == "velox") { + implementation(project(":backends-velox")) + testImplementation(project(":backends-velox", "testArtifacts")) + } else if (backend == "clickhouse") { + implementation(project(":backends-clickhouse")) + } + + compileOnly("org.apache.spark:spark-sql_$scalaBinaryVersion:$effectiveSparkFullVersion") + compileOnly("org.apache.spark:spark-core_$scalaBinaryVersion:$effectiveSparkFullVersion") + compileOnly("org.apache.spark:spark-catalyst_$scalaBinaryVersion:$effectiveSparkFullVersion") + compileOnly("org.apache.spark:spark-hive_$scalaBinaryVersion:$effectiveSparkFullVersion") + + testImplementation("org.scalatest:scalatest_$scalaBinaryVersion:3.2.16") + testImplementation("junit:junit:4.13.1") + testImplementation("com.vladsch.flexmark:flexmark-all:0.62.2") + + testImplementation("org.apache.spark:spark-core_$scalaBinaryVersion:$effectiveSparkFullVersion:tests") + testImplementation("org.apache.spark:spark-sql_$scalaBinaryVersion:$effectiveSparkFullVersion:tests") + testImplementation("org.apache.spark:spark-catalyst_$scalaBinaryVersion:$effectiveSparkFullVersion:tests") + testImplementation("org.apache.spark:spark-hive_$scalaBinaryVersion:$effectiveSparkFullVersion:tests") +} diff --git a/gradle.properties b/gradle.properties new file mode 100644 index 00000000000..5d90f6aeb9f --- /dev/null +++ b/gradle.properties @@ -0,0 +1,91 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Gradle settings +org.gradle.jvmargs=-Xmx4g -XX:MaxMetaspaceSize=1g +org.gradle.parallel=true +org.gradle.caching=true +org.gradle.configureondemand=true + +# Project version +glutenVersion=1.6.0-SNAPSHOT + +# Default Scala version (derived from scalaBinaryVersion in build.gradle.kts) +# Do NOT set scalaVersion here — it is computed from scalaBinaryVersion. +# scalaVersion is derived as: 2.12 → 2.12.18, 2.13 → 2.13.17 + +# Default Spark version (matches Maven default) +sparkVersion=3.5 +sparkFullVersion=3.5.5 +sparkPlainVersion=35 + +# Default Java version +javaVersion=8 + +# Default backend +backend=velox + +# Optional modules (disabled by default) +delta=false +iceberg=false +hudi=false +paimon=false +celeborn=false +uniffle=false + +# Dependency versions +arrowVersion=15.0.0-gluten +arrowGlutenVersion=15.0.0-gluten +protobufVersion=3.23.4 +guavaVersion=33.4.0-jre +hadoopVersion=3.3.4 +# caffeineVersion is computed in build.gradle.kts based on javaVersion: +# Java 8 → 2.9.3, Java 11+ → 3.1.8 +antlr4Version=4.9.3 + +# Delta versions (Spark version dependent) +deltaVersion=3.3.2 +deltaBinaryVersion=33 +deltaPackageName=delta-spark + +# Iceberg version +icebergVersion=1.10.0 +icebergBinaryVersion=10 + +# Paimon version +paimonVersion=1.2.0 + +# Hudi version +hudiVersion=0.15.0 + +# Celeborn version +celebornVersion=0.6.1 + +# Uniffle version +uniffleVersion=0.10.0 + +# Test versions +junitVersion=4.13.1 +junit5Version=5.11.4 +scalatestVersion=3.2.16 +fasterxmlVersion=2.15.1 + +# Spotless versions +spotlessVersion=6.25.0 +googleJavaFormatVersion=1.17.0 +scalafmtVersion=3.8.3 + +# Shading package name +glutenShadePackageName=org.apache.gluten.shaded diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml new file mode 100644 index 00000000000..72ad3a66e33 --- /dev/null +++ b/gradle/libs.versions.toml @@ -0,0 +1,176 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[versions] +# Core versions (these can be overridden via gradle.properties or -P flags) +scala212 = "2.12.15" +scala213 = "2.13.17" + +# Spark versions +spark33 = "3.3.1" +spark34 = "3.4.4" +spark35 = "3.5.5" +spark40 = "4.0.1" +spark41 = "4.1.1" + +# Arrow +arrow = "18.1.0" +arrowGluten = "18.1.0" + +# Protobuf and serialization +protobuf = "3.23.4" +flatbuffers = "1.12.0" + +# Google libraries +guava = "33.4.0-jre" +gson = "2.10.1" + +# Apache libraries +hadoop = "3.4.1" +antlr4 = "4.13.1" +commonsIo = "2.14.0" +commonsLang3 = "3.17.0" + +# Caching +caffeine = "3.1.8" + +# Logging +slf4j = "2.0.16" +log4j = "2.24.3" + +# Data lake connectors +delta40 = "4.0.0" +delta33 = "3.3.2" +delta24 = "2.4.0" +delta23 = "2.3.0" +iceberg = "1.10.0" +paimon = "1.3.0" +hudi = "1.1.0" +celeborn = "0.6.1" +uniffle = "0.10.0" + +# Testing +junit4 = "4.13.1" +junit5 = "5.11.4" +scalatest = "3.2.16" +scalacheck = "1.17.0" +mockito = "2.23.4" +bytebuddy = "1.9.3" +flexmark = "0.62.2" +javafaker = "1.0.2" + +# Jackson/Fasterxml +jackson = "2.18.2" + +# Plugins +scalaPlugin = "1.2.0" +spotless = "6.25.0" +shadow = "8.1.1" +protobufPlugin = "0.9.4" +antlrPlugin = "3.0.0-alpha.1" +googleJavaFormat = "1.17.0" +scalafmt = "3.8.3" +wartremover = "3.2.0" + +# XML +scalaXml = "2.1.0" + +# Servlet +javaxServlet = "3.1.0" + +[libraries] +# Scala libraries +scala-library212 = { module = "org.scala-lang:scala-library", version.ref = "scala212" } +scala-library213 = { module = "org.scala-lang:scala-library", version.ref = "scala213" } +scala-xml = { module = "org.scala-lang.modules:scala-xml_2.13", version.ref = "scalaXml" } +scala-collection-compat212 = { module = "org.scala-lang.modules:scala-collection-compat_2.12", version = "2.11.0" } +scala-collection-compat213 = { module = "org.scala-lang.modules:scala-collection-compat_2.13", version = "2.11.0" } + +# Arrow +arrow-vector = { module = "org.apache.arrow:arrow-vector", version.ref = "arrow" } +arrow-memory-unsafe = { module = "org.apache.arrow:arrow-memory-unsafe", version.ref = "arrow" } +arrow-memory-netty = { module = "org.apache.arrow:arrow-memory-netty", version.ref = "arrow" } + +# Protobuf +protobuf-java = { module = "com.google.protobuf:protobuf-java", version.ref = "protobuf" } +flatbuffers-java = { module = "com.google.flatbuffers:flatbuffers-java", version.ref = "flatbuffers" } + +# Google libraries +guava = { module = "com.google.guava:guava", version.ref = "guava" } +gson = { module = "com.google.code.gson:gson", version.ref = "gson" } +jimfs = { module = "com.google.jimfs:jimfs", version = "1.3.0" } + +# Apache commons +commons-io = { module = "commons-io:commons-io", version.ref = "commonsIo" } +commons-lang3 = { module = "org.apache.commons:commons-lang3", version.ref = "commonsLang3" } + +# Caching +caffeine = { module = "com.github.ben-manes.caffeine:caffeine", version.ref = "caffeine" } + +# Logging +slf4j-api = { module = "org.slf4j:slf4j-api", version.ref = "slf4j" } +slf4j-log4j12 = { module = "org.slf4j:slf4j-log4j12", version = "1.7.30" } +log4j-core = { module = "org.apache.logging.log4j:log4j-core", version.ref = "log4j" } +log4j-slf4j2-impl = { module = "org.apache.logging.log4j:log4j-slf4j2-impl", version.ref = "log4j" } + +# Jackson +jackson-core = { module = "com.fasterxml.jackson.core:jackson-core", version.ref = "jackson" } +jackson-databind = { module = "com.fasterxml.jackson.core:jackson-databind", version.ref = "jackson" } +jackson-annotations = { module = "com.fasterxml.jackson.core:jackson-annotations", version.ref = "jackson" } +jackson-datatype-jsr310 = { module = "com.fasterxml.jackson.datatype:jackson-datatype-jsr310", version.ref = "jackson" } +jackson-datatype-guava = { module = "com.fasterxml.jackson.datatype:jackson-datatype-guava", version.ref = "jackson" } +jackson-module-scala212 = { module = "com.fasterxml.jackson.module:jackson-module-scala_2.12", version.ref = "jackson" } +jackson-module-scala213 = { module = "com.fasterxml.jackson.module:jackson-module-scala_2.13", version.ref = "jackson" } + +# Testing +junit4 = { module = "junit:junit", version.ref = "junit4" } +junit5-jupiter = { module = "org.junit.jupiter:junit-jupiter", version.ref = "junit5" } +scalatest212 = { module = "org.scalatest:scalatest_2.12", version.ref = "scalatest" } +scalatest213 = { module = "org.scalatest:scalatest_2.13", version.ref = "scalatest" } +scalacheck212 = { module = "org.scalacheck:scalacheck_2.12", version.ref = "scalacheck" } +scalacheck213 = { module = "org.scalacheck:scalacheck_2.13", version.ref = "scalacheck" } +mockito-core = { module = "org.mockito:mockito-core", version.ref = "mockito" } +byte-buddy = { module = "net.bytebuddy:byte-buddy", version.ref = "bytebuddy" } +flexmark-all = { module = "com.vladsch.flexmark:flexmark-all", version.ref = "flexmark" } +javafaker = { module = "com.github.javafaker:javafaker", version.ref = "javafaker" } +scalatestplus-mockito212 = { module = "org.scalatestplus:scalatestplus-mockito_2.12", version = "1.0.0-M2" } +scalatestplus-mockito213 = { module = "org.scalatestplus:scalatestplus-mockito_2.13", version = "1.0.0-M2" } +scalatestplus-scalacheck212 = { module = "org.scalatestplus:scalatestplus-scalacheck_2.12", version = "3.1.0.0-RC2" } +scalatestplus-scalacheck213 = { module = "org.scalatestplus:scalatestplus-scalacheck_2.13", version = "3.1.0.0-RC2" } + +# ANTLR +antlr4 = { module = "org.antlr:antlr4", version.ref = "antlr4" } +antlr4-runtime = { module = "org.antlr:antlr4-runtime", version.ref = "antlr4" } + +# WartRemover for Scala +wartremover212 = { module = "org.wartremover:wartremover_2.12", version = "3.0.6" } +wartremover213 = { module = "org.wartremover:wartremover_2.13", version.ref = "wartremover" } +scalawarts212 = { module = "io.github.zhztheplayer.scalawarts:scalawarts_2.12", version = "0.1.2" } +scalawarts213 = { module = "io.github.zhztheplayer.scalawarts:scalawarts_2.13", version = "0.1.2" } + +# Servlet +javax-servlet = { module = "javax.servlet:javax.servlet-api", version.ref = "javaxServlet" } + +[bundles] +jackson = ["jackson-core", "jackson-databind", "jackson-annotations"] +testing-scala212 = ["scalatest212", "scalacheck212", "junit4", "mockito-core"] +testing-scala213 = ["scalatest213", "scalacheck213", "junit4", "mockito-core"] + +[plugins] +scala = { id = "com.github.maiflai.scalatest", version = "0.32" } +spotless = { id = "com.diffplug.spotless", version.ref = "spotless" } +shadow = { id = "com.github.johnrengelman.shadow", version.ref = "shadow" } +protobuf = { id = "com.google.protobuf", version.ref = "protobufPlugin" } +antlr = { id = "org.gradle.antlr", version = "8.4" } diff --git a/gradle/plugins/build.gradle.kts b/gradle/plugins/build.gradle.kts new file mode 100644 index 00000000000..93c5866b466 --- /dev/null +++ b/gradle/plugins/build.gradle.kts @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + `kotlin-dsl` +} + +repositories { + gradlePluginPortal() + mavenCentral() +} + +dependencies { + implementation("com.gradleup.shadow:shadow-gradle-plugin:8.3.0") + implementation("com.google.protobuf:protobuf-gradle-plugin:0.9.4") + implementation("com.github.maiflai:gradle-scalatest:0.32") + if (JavaVersion.current().isJava11Compatible) { + implementation("com.diffplug.spotless:spotless-plugin-gradle:6.25.0") + } +} + +// When Spotless is not on the classpath (JDK < 11), exclude GlutenSpotless.kt +// from compilation to avoid unresolved type references. +if (!JavaVersion.current().isJava11Compatible) { + sourceSets { + main { + kotlin { + exclude("GlutenSpotless.kt") + } + } + } +} diff --git a/gradle/plugins/settings.gradle.kts b/gradle/plugins/settings.gradle.kts new file mode 100644 index 00000000000..6080ac61aa4 --- /dev/null +++ b/gradle/plugins/settings.gradle.kts @@ -0,0 +1,18 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +rootProject.name = "gluten-plugins" diff --git a/gradle/plugins/src/main/kotlin/AbstractClassExcludeSpec.kt b/gradle/plugins/src/main/kotlin/AbstractClassExcludeSpec.kt new file mode 100644 index 00000000000..1e502d4dd54 --- /dev/null +++ b/gradle/plugins/src/main/kotlin/AbstractClassExcludeSpec.kt @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.gradle.api.file.FileTreeElement +import org.gradle.api.specs.Spec +import java.io.DataInputStream +import java.lang.reflect.Modifier + +/** + * Gradle [Spec] that excludes abstract classes and interfaces from test scanning. + * + * The ScalaTest JUnit 5 engine fails with [InstantiationException] when Gradle sends + * abstract test base classes or Scala traits to it. This spec reads the `access_flags` + * from the class file header to detect and exclude them before they reach the engine. + */ +class AbstractClassExcludeSpec : Spec { + override fun isSatisfiedBy(element: FileTreeElement): Boolean { + val f = element.file + if (!f.name.endsWith(".class") || !f.exists()) return false + return try { + DataInputStream(f.inputStream()).use { din -> + din.skipBytes(8) // magic(4) + version(4) + // Skip constant pool + val cpCount = din.readUnsignedShort() + var i = 1 + while (i < cpCount) { + when (din.readUnsignedByte()) { + 1 -> din.skipBytes(din.readUnsignedShort()) // Utf8 + 3, 4 -> din.skipBytes(4) // Integer, Float + 5, 6 -> { din.skipBytes(8); i++ } // Long, Double (2 pool slots) + 7, 8, 16, 19, 20 -> din.skipBytes(2) + 9, 10, 11, 12, 17, 18 -> din.skipBytes(4) + 15 -> din.skipBytes(3) // MethodHandle + } + i++ + } + val accessFlags = din.readUnsignedShort() + Modifier.isAbstract(accessFlags) || Modifier.isInterface(accessFlags) + } + } catch (_: Exception) { + false + } + } +} diff --git a/gradle/plugins/src/main/kotlin/GlutenSpotless.kt b/gradle/plugins/src/main/kotlin/GlutenSpotless.kt new file mode 100644 index 00000000000..87a2b978f52 --- /dev/null +++ b/gradle/plugins/src/main/kotlin/GlutenSpotless.kt @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.diffplug.gradle.spotless.SpotlessExtension +import org.gradle.api.JavaVersion +import org.gradle.api.Project + +/** + * Configures the Spotless extension to match the Maven spotless-maven-plugin configuration + * in pom.xml. This file is excluded from compilation on JDK < 11 (see build.gradle.kts). + */ +object GlutenSpotless { + + // spotless:off + private val LICENSE_HEADER = buildString { + append("/*\n") + append(" * Licensed to the Apache Software Foundation (ASF) under one or more\n") + append(" * contributor license agreements. See the NOTICE file distributed with\n") + append(" * this work for additional information regarding copyright ownership.\n") + append(" * The ASF licenses this file to You under the Apache License, Version 2.0\n") + append(" * (the \"License\"); you may not use this file except in compliance with\n") + append(" * the License. You may obtain a copy of the License at\n") + append(" *\n") + append(" * http://www.apache.org/licenses/LICENSE-2.0\n") + append(" *\n") + append(" * Unless required by applicable law or agreed to in writing, software\n") + append(" * distributed under the License is distributed on an \"AS IS\" BASIS,\n") + append(" * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n") + append(" * See the License for the specific language governing permissions and\n") + append(" * limitations under the License.\n") + append(" */") + } + // spotless:on + + fun apply(project: Project) { + project.pluginManager.apply("com.diffplug.spotless") + + // Use google-java-format 1.17.0 for all JDK 11+ builds. Spotless enforces + // minimum versions per JDK (1.8 for JDK 11, 1.10.0 for JDK 17), and 1.17.0 + // satisfies all of them while producing consistent formatting across JDKs. + val googleJavaFormatVersion = "1.17.0" + + val spotless = project.extensions.getByType(SpotlessExtension::class.java) + spotless.java { + toggleOffOn() + googleJavaFormat(googleJavaFormatVersion) + importOrder("org.apache.gluten", "io.substrait.spark", "", "javax", "java", "scala", "\\#") + removeUnusedImports() + licenseHeader(LICENSE_HEADER, "package") + target( + "src/main/java/**/*.java", + "src/test/java/**/*.java", + "src-*/main/java/**/*.java", + "src-*/test/java/**/*.java", + ) + } + if (project.hasProperty("scalaBinaryVersion")) { + val scalaBinaryVersion = project.property("scalaBinaryVersion") as String + spotless.scala { + toggleOffOn() + scalafmt("3.8.3") + .configFile(project.rootProject.file(".scalafmt.conf")) + .scalaMajorVersion(scalaBinaryVersion) + licenseHeader(LICENSE_HEADER, "package") + target( + "src/main/scala/**/*.scala", + "src/test/scala/**/*.scala", + "src-*/main/scala/**/*.scala", + "src-*/test/scala/**/*.scala", + ) + } + } + } +} diff --git a/gradle/plugins/src/main/kotlin/ScalaTestPerSuiteAction.kt b/gradle/plugins/src/main/kotlin/ScalaTestPerSuiteAction.kt new file mode 100644 index 00000000000..9c4c0b3a911 --- /dev/null +++ b/gradle/plugins/src/main/kotlin/ScalaTestPerSuiteAction.kt @@ -0,0 +1,376 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.gradle.api.Action +import org.gradle.api.GradleException +import org.gradle.api.tasks.testing.Test +import org.gradle.api.tasks.util.PatternSet +import java.io.DataInputStream +import java.io.File +import java.lang.reflect.Modifier +import java.util.concurrent.Executors +import java.util.concurrent.Future +import java.util.concurrent.atomic.AtomicInteger + +/** + * Custom [Action] for Gradle's [Test] task that replaces the gradle-scalatest plugin's + * default ScalaTestAction. Instead of running all suites in a single JVM via + * `Runner -R `, this action invokes `org.scalatest.tools.Runner -s ` + * in a separate `javaexec` for each test suite class. + * + * This achieves per-suite JVM isolation without depending on JUnit 5 Platform, + * preventing Spark's JVM-global state (SparkContext, daemon threads) from leaking + * between suites. Suites run in parallel up to [Test.getMaxParallelForks] concurrent + * JVMs. + * + * Tag filtering is read from the gradle-scalatest plugin's `tags` extension + * (a [PatternSet] registered on each Test task). `tags.includes` map to ScalaTest + * Runner's `-n` flags, `tags.excludes` map to `-l` flags. Exclude tags are also + * used for class-level pre-filtering via constant pool scanning to avoid spawning + * JVMs for excluded suites. + * + * Test filter patterns from `--tests` (Gradle's `filter.includePatterns` and + * `filter.commandLineIncludePatterns`) are translated to ScalaTest's `-q` (suite + * name match) or `-z` (substring match) flags, consistent with the plugin's behavior. + */ +class ScalaTestPerSuiteAction : Action { + + override fun execute(task: Test) { + // Read tags from the plugin's extension + val tags = task.extensions.findByName("tags") as? PatternSet + val tagsToInclude = tags?.includes ?: emptySet() + val tagsToExclude = tags?.excludes ?: emptySet() + + val suites = discoverSuites(task, tagsToExclude) + if (suites.isEmpty()) { + task.logger.lifecycle("No test suites found in ${task.path}") + return + } + + task.logger.lifecycle("Discovered ${suites.size} test suite(s) in ${task.path}") + + val reportsDir = File(task.reports.junitXml.outputLocation.asFile.get().absolutePath) + reportsDir.mkdirs() + + if (task.reports.html.required.get()) { + val htmlReportsDir = File(task.reports.html.outputLocation.asFile.get().absolutePath) + htmlReportsDir.mkdirs() + } + + // Collect test filter patterns from --tests flag + val filterPatterns = collectFilterPatterns(task) + + val parallelism = task.maxParallelForks.coerceAtLeast(1) + val failedSuites = java.util.Collections.synchronizedList(mutableListOf()) + val passedCount = AtomicInteger(0) + + if (parallelism == 1) { + // Sequential execution + for (suite in suites) { + runSuite(task, suite, reportsDir, tagsToInclude, tagsToExclude, filterPatterns, + failedSuites, passedCount) + } + } else { + // Parallel execution with bounded thread pool + task.logger.lifecycle("Running suites with parallelism=$parallelism") + val executor = Executors.newFixedThreadPool(parallelism) + try { + val futures = suites.map { suite -> + executor.submit { + runSuite(task, suite, reportsDir, tagsToInclude, tagsToExclude, + filterPatterns, failedSuites, passedCount) + } + } + futures.forEach { it.get() } + } finally { + executor.shutdown() + } + } + + task.logger.lifecycle( + "\nScalaTest results: ${passedCount.get()} passed, ${failedSuites.size} failed, " + + "${suites.size} total" + ) + + if (failedSuites.isNotEmpty()) { + val failureMessage = "ScalaTest failures in ${task.path}:\n" + + failedSuites.joinToString("\n") { " - $it" } + if (!task.ignoreFailures) { + throw GradleException(failureMessage) + } else { + task.logger.warn(failureMessage) + } + } + } + + companion object { + /** Matches class names that look like test suites (same heuristic as the plugin). */ + private val MAYBE_TEST = Regex("Spec|Test|Suite") + } + + private fun runSuite( + task: Test, + suite: String, + reportsDir: File, + tagsToInclude: Set, + tagsToExclude: Set, + filterPatterns: List, + failedSuites: MutableList, + passedCount: AtomicInteger + ) { + task.logger.lifecycle("Running suite: $suite") + + val runnerArgs = buildRunnerArgs( + task, suite, reportsDir, tagsToInclude, tagsToExclude, filterPatterns + ) + + try { + val result = task.project.javaexec { + mainClass.set("org.scalatest.tools.Runner") + classpath = task.classpath + args = runnerArgs + jvmArgs = task.allJvmArgs + environment = task.environment + isIgnoreExitValue = true + } + + if (result.exitValue != 0) { + failedSuites.add(suite) + task.logger.error("FAILED: $suite (exit code ${result.exitValue})") + } else { + passedCount.incrementAndGet() + task.logger.lifecycle("PASSED: $suite") + } + } catch (e: Exception) { + failedSuites.add(suite) + task.logger.error("ERROR: $suite - ${e.message}") + } + } + + /** + * Collects test filter patterns from Gradle's `--tests` flag. + * Both `commandLineIncludePatterns` and `includePatterns` are checked, + * matching the plugin's `ScalaTestAction.getArgs()` behavior. + */ + private fun collectFilterPatterns(task: Test): List { + val patterns = mutableListOf() + try { + // commandLineIncludePatterns is available in newer Gradle versions + val cmdPatterns = task.filter.javaClass + .getMethod("getCommandLineIncludePatterns") + .invoke(task.filter) as? Set<*> + cmdPatterns?.filterIsInstance()?.let { patterns.addAll(it) } + } catch (_: Exception) { + // Method not available in this Gradle version + } + patterns.addAll(task.filter.includePatterns) + return patterns + } + + private fun buildRunnerArgs( + task: Test, + suite: String, + reportsDir: File, + tagsToInclude: Set, + tagsToExclude: Set, + filterPatterns: List + ): List { + val args = mutableListOf() + + // Single suite mode + args.add("-s") + args.add(suite) + + // Output with durations + args.add("-oD") + + // JUnit XML reporter for CI integration + if (task.reports.junitXml.required.get()) { + args.add("-u") + args.add(reportsDir.absolutePath) + } + + // HTML reporter + if (task.reports.html.required.get()) { + args.add("-h") + args.add(task.reports.html.outputLocation.asFile.get().absolutePath) + } + + // Per-test tag filtering via ScalaTest Runner's native flags + tagsToInclude.forEach { tag -> + args.add("-n") + args.add(tag) + } + tagsToExclude.forEach { tag -> + args.add("-l") + args.add(tag) + } + + // --tests filter patterns: -q for suite names, -z for substrings + // (same logic as ScalaTestAction.getArgs) + filterPatterns.forEach { pattern -> + if (MAYBE_TEST.containsMatchIn(pattern)) { + args.add("-q") + } else { + args.add("-z") + } + args.add(pattern) + } + + return args + } + + /** + * Discovers concrete ScalaTest suite classes from the test classes directories. + * Excludes abstract classes, interfaces, Scala companion objects ($), inner classes, + * and classes with excluded tag annotations. + * + * Note: Only exclusion is done at the class level (annotation descriptors are visible + * in the constant pool). Inclusion filtering is deferred to ScalaTest Runner's `-n` flag + * because per-test `Tag` objects (e.g., `test("name", MyTag)`) don't produce annotation + * descriptors on the suite class — they're only visible at runtime. + */ + private fun discoverSuites(task: Test, tagsToExclude: Set): List { + val excludeDescriptors = tagsToExclude.map { "L${it.replace('.', '/')};" }.toSet() + + // Build a map of all class files for superclass chain resolution + val classFileMap = mutableMapOf() + task.testClassesDirs.files.forEach { classesDir -> + if (!classesDir.exists()) return@forEach + classesDir.walkTopDown() + .filter { it.isFile && it.name.endsWith(".class") } + .forEach { classFile -> + val internalName = classFile.relativeTo(classesDir).path + .removeSuffix(".class") + classFileMap[internalName] = classFile + } + } + + val suites = mutableListOf() + + classFileMap.forEach { (internalName, classFile) -> + // Skip $ classes (companions, inner classes) + if (internalName.substringAfterLast('/').contains("$")) return@forEach + + val info = readClassInfo(classFile) ?: return@forEach + + // Skip abstract classes and interfaces + if (Modifier.isAbstract(info.accessFlags) || Modifier.isInterface(info.accessFlags)) { + return@forEach + } + + // Check if this class is a ScalaTest suite by walking the superclass chain + if (!isScalaTestSuite(internalName, classFileMap)) return@forEach + + // Tag-based exclusion (class-level annotations only) + if (excludeDescriptors.isNotEmpty() && + excludeDescriptors.any { it in info.utf8Entries } + ) { + return@forEach + } + + val fqcn = internalName.replace('/', '.') + suites.add(fqcn) + } + + return suites.sorted() + } + + private data class ClassInfo( + val accessFlags: Int, + val superClass: String?, + val utf8Entries: Set + ) + + private val classInfoCache = mutableMapOf() + + /** + * Reads class file metadata: access flags, superclass name, and constant pool UTF-8 entries. + */ + private fun readClassInfo(classFile: File): ClassInfo? { + return try { + DataInputStream(classFile.inputStream()).use { din -> + din.skipBytes(8) // magic(4) + version(4) + + // Read constant pool + val utf8Entries = mutableSetOf() + val classRefs = mutableMapOf() // class index -> name index + val utf8ByIndex = mutableMapOf() + val cpCount = din.readUnsignedShort() + var i = 1 + while (i < cpCount) { + when (din.readUnsignedByte()) { + 1 -> { // Utf8 + val len = din.readUnsignedShort() + val bytes = ByteArray(len) + din.readFully(bytes) + val str = String(bytes) + utf8Entries.add(str) + utf8ByIndex[i] = str + } + 3, 4 -> din.skipBytes(4) // Integer, Float + 5, 6 -> { din.skipBytes(8); i++ } // Long, Double (2 pool slots) + 7 -> { // Class ref + val nameIndex = din.readUnsignedShort() + classRefs[i] = nameIndex + } + 8, 16, 19, 20 -> din.skipBytes(2) + 9, 10, 11, 12, 17, 18 -> din.skipBytes(4) + 15 -> din.skipBytes(3) // MethodHandle + } + i++ + } + + val accessFlags = din.readUnsignedShort() + din.readUnsignedShort() // this_class + val superClassIndex = din.readUnsignedShort() + val superClass = if (superClassIndex != 0) { + val nameIndex = classRefs[superClassIndex] + if (nameIndex != null) utf8ByIndex[nameIndex] else null + } else null + + ClassInfo(accessFlags, superClass, utf8Entries) + } + } catch (_: Exception) { + null + } + } + + /** + * Checks whether a class is a ScalaTest suite by walking the superclass chain. + * A class is a ScalaTest suite if it (or any of its superclasses in the test classes + * directories) references `org/scalatest/` in its constant pool. + */ + private fun isScalaTestSuite(internalName: String, classFileMap: Map): Boolean { + val visited = mutableSetOf() + var current: String? = internalName + while (current != null && current !in visited) { + visited.add(current) + + val classFile = classFileMap[current] ?: return false + val info = classInfoCache.getOrPut(current) { readClassInfo(classFile) } ?: return false + + // Check if any constant pool entry references ScalaTest + if (info.utf8Entries.any { it.startsWith("org/scalatest/") }) { + return true + } + + current = info.superClass + } + return false + } +} diff --git a/gradle/plugins/src/main/kotlin/TagExcludeSpec.kt b/gradle/plugins/src/main/kotlin/TagExcludeSpec.kt new file mode 100644 index 00000000000..a6fa969b230 --- /dev/null +++ b/gradle/plugins/src/main/kotlin/TagExcludeSpec.kt @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.gradle.api.file.FileTreeElement +import org.gradle.api.specs.Spec +import java.io.DataInputStream + +/** + * Gradle [Spec] that excludes test classes based on ScalaTest tag annotations. + * + * The ScalaTest JUnit 5 engine does not support JUnit Platform's includeTags/excludeTags + * filtering. This spec implements tag filtering at the Gradle level by scanning the class + * file constant pool for annotation type descriptors (e.g., `Lorg/apache/gluten/tags/UDFTest;`). + * + * @param tagsToExclude fully qualified annotation class names to exclude + * (e.g., `org.apache.gluten.tags.UDFTest`) + * @param tagsToInclude if non-empty, only classes with at least one of these tags will run; + * classes without any of these tags are excluded + */ +class TagExcludeSpec( + private val tagsToExclude: Set, + private val tagsToInclude: Set +) : Spec { + + // Convert class names to JVM type descriptors for constant pool matching. + // e.g., "org.apache.gluten.tags.UDFTest" -> "Lorg/apache/gluten/tags/UDFTest;" + private val excludeDescriptors = tagsToExclude.map { "L${it.replace('.', '/')};" }.toSet() + private val includeDescriptors = tagsToInclude.map { "L${it.replace('.', '/')};" }.toSet() + + override fun isSatisfiedBy(element: FileTreeElement): Boolean { + val f = element.file + if (!f.name.endsWith(".class") || !f.exists()) return false + if (excludeDescriptors.isEmpty() && includeDescriptors.isEmpty()) return false + return try { + val utf8Entries = readConstantPoolUtf8Entries(f) + // Exclude if any excluded tag is found + if (excludeDescriptors.any { it in utf8Entries }) return true + // Exclude if include filter is active and no included tag is found + if (includeDescriptors.isNotEmpty() && includeDescriptors.none { it in utf8Entries }) return true + false + } catch (_: Exception) { + false + } + } + + private fun readConstantPoolUtf8Entries(f: java.io.File): Set { + val entries = mutableSetOf() + DataInputStream(f.inputStream()).use { din -> + din.skipBytes(8) // magic(4) + version(4) + val cpCount = din.readUnsignedShort() + var i = 1 + while (i < cpCount) { + when (din.readUnsignedByte()) { + 1 -> { // Utf8 + val len = din.readUnsignedShort() + val bytes = ByteArray(len) + din.readFully(bytes) + entries.add(String(bytes)) + } + 3, 4 -> din.skipBytes(4) // Integer, Float + 5, 6 -> { din.skipBytes(8); i++ } // Long, Double (2 pool slots) + 7, 8, 16, 19, 20 -> din.skipBytes(2) + 9, 10, 11, 12, 17, 18 -> din.skipBytes(4) + 15 -> din.skipBytes(3) // MethodHandle + } + i++ + } + } + return entries + } +} diff --git a/gradle/plugins/src/main/kotlin/gluten.java-conventions.gradle.kts b/gradle/plugins/src/main/kotlin/gluten.java-conventions.gradle.kts new file mode 100644 index 00000000000..d2e98641538 --- /dev/null +++ b/gradle/plugins/src/main/kotlin/gluten.java-conventions.gradle.kts @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Convention plugin for Java compilation settings. + * Configures Java source/target compatibility and compiler options. + */ + +plugins { + java + `maven-publish` + checkstyle +} + +val javaVersion: String by project + +val javaVersionInt = javaVersion.toIntOrNull() ?: 17 + +java { + sourceCompatibility = JavaVersion.toVersion(javaVersionInt) + targetCompatibility = JavaVersion.toVersion(javaVersionInt) + + withSourcesJar() +} + +checkstyle { + toolVersion = "8.29" + configFile = rootProject.file("dev/checkstyle.xml") + // Set configDirectory to dev/ so Gradle auto-sets ${config_loc} to the dev/ directory. + // The checkstyle.xml uses "${config_loc}/checkstyle-suppressions.xml" to locate suppressions. + // Maven sets config_loc via in pom.xml. + configDirectory.set(rootProject.file("dev")) + isIgnoreFailures = false +} + +// Exclude generated sources from checkstyle. Maven's checkstyle plugin explicitly limits +// sourceDirectories to src/main/java and src/main/scala. Gradle's checkstyle scans the +// entire source set, which includes protobuf/ANTLR generated files under build/. +tasks.withType().configureEach { + // Only check hand-written sources under src/, not generated code under build/. + source = source.filter { file -> + !file.path.contains("/build/") + }.asFileTree +} + +// Exclude logging implementations that leak in via Spark/Hadoop transitive deps. +// Maven does this via global in the parent POM. +configurations.all { + exclude(group = "org.slf4j", module = "slf4j-log4j12") + exclude(group = "log4j", module = "log4j") +} + +// When the running JDK is newer than targetCompatibility (e.g. JDK 21 with target 17), +// Gradle and its plugins (Java, Scala, etc.) stamp outgoing variants with the running +// JDK version instead of the configured target. This causes variant resolution failures +// between projects (e.g. shims-common@21 vs shims-spark41@17). +// Override TargetJvmVersion on ALL configurations — including legacy ones like "archives" +// and "default", and plugin-added ones like "incrementalScalaAnalysisElements" — so +// inter-project dependencies resolve correctly regardless of the build JDK. +afterEvaluate { + configurations.configureEach { + attributes { + attribute( + TargetJvmVersion.TARGET_JVM_VERSION_ATTRIBUTE, + javaVersionInt + ) + } + } +} + +tasks.withType().configureEach { + options.encoding = "UTF-8" + options.compilerArgs.addAll(listOf("-Xlint:all,-serial,-path")) + options.isFork = true + options.forkOptions.jvmArgs = (options.forkOptions.jvmArgs ?: mutableListOf()).apply { + add("-Dfile.encoding=UTF-8") + } +} + +tasks.withType().configureEach { + options.encoding = "UTF-8" + isFailOnError = false +} + +// Make compileOnly dependencies available for test compilation +// This is needed because Spark dependencies are declared as compileOnly +// but are required for test compilation +configurations { + testImplementation { + extendsFrom(configurations.compileOnly.get()) + } +} + +// Maven publishing — equivalent to "mvn install" +// Usage: ./gradlew publishToMavenLocal +// Shims modules use a "spark-sql-columnar-" prefix in Maven to match the Maven build. +val mavenArtifactId = if (project.name.startsWith("shims-")) { + "spark-sql-columnar-${project.name}" +} else { + project.name +} + +publishing { + publications { + create("maven") { + from(components["java"]) + artifactId = mavenArtifactId + } + } +} diff --git a/gradle/plugins/src/main/kotlin/gluten.native.gradle.kts b/gradle/plugins/src/main/kotlin/gluten.native.gradle.kts new file mode 100644 index 00000000000..a26361e59e4 --- /dev/null +++ b/gradle/plugins/src/main/kotlin/gluten.native.gradle.kts @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Convention plugin for native C++ build integration. + * Provides CMake build tasks for the Velox backend. + */ + +plugins { + base +} + +val cppBuildDir = project.file("../cpp/build") +val cppReleasesDir = file("$cppBuildDir/releases") + +// Task to build native libraries using CMake +val buildNative by tasks.registering(Exec::class) { + group = "build" + description = "Build native C++ libraries using CMake" + + workingDir = project.file("../cpp") + + commandLine( + "cmake", + "--build", "build", + "--parallel", Runtime.getRuntime().availableProcessors().toString() + ) + + outputs.dir(cppReleasesDir) +} + +// Task to configure CMake +val configureNative by tasks.registering(Exec::class) { + group = "build" + description = "Configure native C++ build using CMake" + + workingDir = project.file("../cpp") + + doFirst { + project.file("../cpp/build").mkdirs() + } + + commandLine( + "cmake", + "-B", "build", + "-DCMAKE_BUILD_TYPE=Release" + ) +} + +// Task to clean native build +val cleanNative by tasks.registering(Delete::class) { + group = "build" + description = "Clean native C++ build artifacts" + + delete(cppBuildDir) +} + +// Make native resources available to the JAR +tasks.withType().configureEach { + dependsOn(buildNative) + + val platform = rootProject.extra.get("platform") as String + val arch = rootProject.extra.get("arch") as String + + from(cppReleasesDir) { + into("$platform/$arch") + } +} diff --git a/gradle/plugins/src/main/kotlin/gluten.protobuf.gradle.kts b/gradle/plugins/src/main/kotlin/gluten.protobuf.gradle.kts new file mode 100644 index 00000000000..5aac4f8c5c4 --- /dev/null +++ b/gradle/plugins/src/main/kotlin/gluten.protobuf.gradle.kts @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.google.protobuf.gradle.* +import java.time.Duration + +/** + * Convention plugin for Protobuf compilation. + * Configures protobuf-gradle-plugin for generating Java sources from .proto files. + */ + +plugins { + id("com.google.protobuf") + java +} + +val protobufVersion: String by project + +dependencies { + implementation("com.google.protobuf:protobuf-java:$protobufVersion") +} + +protobuf { + protoc { + artifact = "com.google.protobuf:protoc:$protobufVersion" + } +} + +// The protobuf plugin's extractInclude*Proto tasks scan the entire compile classpath +// for .proto files (via the compileProtoPath configuration that extends compileOnly + +// implementation). In Gluten this means scanning 600+ transitive Spark/Hadoop JARs, +// none of which contain .proto files. This causes CI stalls when dependency resolution +// blocks on network or lock contention. +// +// Fix: remove the extendsFrom relationships on compileProtoPath / testCompileProtoPath +// so only explicitly declared protobuf dependencies are scanned. +afterEvaluate { + listOf("compileProtoPath", "testCompileProtoPath").forEach { name -> + configurations.findByName(name)?.let { protoPath -> + protoPath.setExtendsFrom(emptyList()) + // Re-add only protobuf-java so its bundled .proto files are available as imports. + protoPath.dependencies.add( + project.dependencies.create("com.google.protobuf:protobuf-java:$protobufVersion") + ) + } + } +} + +tasks.withType().configureEach { + @Suppress("UnstableApiUsage") + timeout = Duration.ofMinutes(5) +} + +// Configure source sets to include generated protobuf sources +sourceSets { + main { + java { + srcDir("build/generated/source/proto/main/java") + } + } + test { + java { + srcDir("build/generated/source/proto/test/java") + } + } +} diff --git a/gradle/plugins/src/main/kotlin/gluten.scala-conventions.gradle.kts b/gradle/plugins/src/main/kotlin/gluten.scala-conventions.gradle.kts new file mode 100644 index 00000000000..dccd5f563ce --- /dev/null +++ b/gradle/plugins/src/main/kotlin/gluten.scala-conventions.gradle.kts @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Convention plugin for Scala compilation settings. + * Configures Scala compiler with WartRemover and other options. + */ + +plugins { + scala + id("gluten.java-conventions") +} + +val scalaVersion: String by project +val scalaBinaryVersion: String by project + +dependencies { + // Scala library + implementation("org.scala-lang:scala-library:$scalaVersion") + + // WartRemover compiler plugin + val wartremoverVersion = if (scalaBinaryVersion == "2.13") "3.2.0" else "3.0.6" + scalaCompilerPlugins("org.wartremover:wartremover_$scalaBinaryVersion:$wartremoverVersion") +} + +tasks.withType().configureEach { + scalaCompileOptions.apply { + encoding = "UTF-8" + + // Common compiler arguments + val commonArgs = mutableListOf( + "-deprecation", + "-feature", + "-unchecked" + ) + + // WartRemover configuration + commonArgs.add("-P:wartremover:traverser:io.github.zhztheplayer.scalawarts.InheritFromCaseClass") + + if (scalaBinaryVersion == "2.13") { + // Scala 2.13 specific arguments + commonArgs.addAll(listOf( + "-explaintypes", + "-Wconf:cat=deprecation:wv,any:e", + "-Wunused:imports", + "-Wconf:cat=scaladoc:wv", + "-Wconf:cat=other-nullary-override:wv", + "-Wconf:msg=^(?=.*?method|value|type|object|trait|inheritance)(?=.*?deprecated)(?=.*?since 2.13).+$:s", + "-Wconf:msg=^(?=.*?Widening conversion from)(?=.*?is deprecated because it loses precision).+$:s", + "-Wconf:msg=Auto-application to \\`\\(\\)\\` is deprecated:s", + "-Wconf:msg=method with a single empty parameter list overrides method without any parameter list:s", + "-Wconf:msg=method without a parameter list overrides a method with a single empty one:s", + "-Wconf:cat=deprecation&msg=procedure syntax is deprecated:e", + "-Wconf:cat=unchecked&msg=outer reference:s", + "-Wconf:cat=unchecked&msg=eliminated by erasure:s", + "-Wconf:msg=^(?=.*?a value of type)(?=.*?cannot also be).+$:s", + "-Wconf:cat=deprecation&msg=symbol literal is deprecated:s" + )) + } else { + // Scala 2.12 specific arguments + commonArgs.addAll(listOf( + "-Wconf:msg=While parsing annotations in:silent,any:e", + "-Ywarn-unused:imports", + "-Wconf:cat=deprecation:wv,any:e" + )) + } + + additionalParameters = commonArgs + + forkOptions.apply { + memoryMaximumSize = "4g" + jvmArgs = (jvmArgs ?: mutableListOf()).apply { + add("-Xss4m") + } + } + } + // Set encoding for the forked javac process that compiles Java sources + // during mixed Scala/Java compilation (zinc forks javac separately) + options.encoding = "UTF-8" + options.isFork = true + options.forkOptions.jvmArgs = (options.forkOptions.jvmArgs ?: mutableListOf()).apply { + add("-Dfile.encoding=UTF-8") + } +} + +// Configure mixed Scala/Java compilation +// When using mixed sources, Scala compiler handles both .scala and .java files +// to ensure proper compilation order and cross-referencing +sourceSets { + main { + scala { + srcDirs("src/main/scala", "src/main/java") + } + // Remove Java sources from Java source set to avoid double compilation + java { + setSrcDirs(emptyList()) + } + } + test { + scala { + srcDirs("src/test/scala", "src/test/java") + } + java { + setSrcDirs(emptyList()) + } + } +} + +// Merge test resources into the Scala test classes output directory. +// Maven puts both compiled classes and resources in target/test-classes/, so +// getClass.getResource("/").getPath + "some-resource" works. Gradle separates them +// (build/classes/scala/test/ vs build/resources/test/), breaking that pattern. +// Redirect processTestResources output to match Maven's layout. +sourceSets.test.get().output.setResourcesDir( + sourceSets.test.get().scala.classesDirectory.get().asFile +) + +// Bridge path differences between Maven and Gradle test output layouts. +// Test code uses getClass.getResource("/").getPath + "../../../src/test/resources/..." +// to locate source-tree resources. Maven's classpath root is target/test-classes/ +// (2 levels deep), so ../../../ reaches the project root. Gradle's is +// build/classes/scala/test/ (4 levels deep), so ../../../ only reaches build/. +// Create a symlink build/src -> ../src so the traversal resolves correctly. +val createBuildSrcSymlink by tasks.registering { + val buildSrcLink = file("build/src") + outputs.upToDateWhen { buildSrcLink.exists() } + doLast { + buildSrcLink.parentFile.mkdirs() + if (!buildSrcLink.exists()) { + exec { commandLine("ln", "-s", "../src", buildSrcLink.absolutePath) } + } + } +} +tasks.withType().configureEach { + dependsOn(createBuildSrcSymlink) +} + +// Handle potential duplicate class files +tasks.withType().configureEach { + duplicatesStrategy = DuplicatesStrategy.EXCLUDE +} diff --git a/gradle/plugins/src/main/kotlin/gluten.scala-library.gradle.kts b/gradle/plugins/src/main/kotlin/gluten.scala-library.gradle.kts new file mode 100644 index 00000000000..dcc225f900f --- /dev/null +++ b/gradle/plugins/src/main/kotlin/gluten.scala-library.gradle.kts @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Convention plugin for Scala library modules. + * Adds Spark version-specific source sets, common Spark/test dependencies, + * and test JAR publishing. + */ + +plugins { + id("gluten.scala-conventions") + `java-library` +} + +val scalaBinaryVersion: String by project +val sparkVersion: String by project +val sparkPlainVersion = when (sparkVersion) { + "3.3" -> "33" + "3.4" -> "34" + "3.5" -> "35" + "4.0" -> "40" + "4.1" -> "41" + else -> "41" +} + +val effectiveSparkFullVersion: String by rootProject.extra + +dependencies { + compileOnly("org.apache.spark:spark-sql_$scalaBinaryVersion:$effectiveSparkFullVersion") + compileOnly("org.apache.spark:spark-core_$scalaBinaryVersion:$effectiveSparkFullVersion") + compileOnly("org.apache.spark:spark-catalyst_$scalaBinaryVersion:$effectiveSparkFullVersion") + compileOnly("org.apache.spark:spark-hive_$scalaBinaryVersion:$effectiveSparkFullVersion") + + testImplementation("org.scalatest:scalatest_$scalaBinaryVersion:3.2.16") + testImplementation("junit:junit:4.13.1") + + testImplementation("org.apache.spark:spark-core_$scalaBinaryVersion:$effectiveSparkFullVersion:tests") + testImplementation("org.apache.spark:spark-sql_$scalaBinaryVersion:$effectiveSparkFullVersion:tests") + testImplementation("org.apache.spark:spark-catalyst_$scalaBinaryVersion:$effectiveSparkFullVersion:tests") + testImplementation("org.apache.spark:spark-hive_$scalaBinaryVersion:$effectiveSparkFullVersion:tests") +} + +if (effectiveSparkFullVersion.startsWith("4")) { + dependencies { + testImplementation("org.apache.spark:spark-common-utils_$scalaBinaryVersion:$effectiveSparkFullVersion") + } +} + +// Add Spark version-specific source directories if they exist +sourceSets { + main { + scala { + srcDir("src-spark${sparkPlainVersion}/main/scala") + srcDir("src-spark${sparkPlainVersion}/main/java") + } + resources { + srcDir("src-spark${sparkPlainVersion}/main/resources") + } + } + test { + scala { + srcDir("src-spark${sparkPlainVersion}/test/scala") + srcDir("src-spark${sparkPlainVersion}/test/java") + } + resources { + srcDir("src-spark${sparkPlainVersion}/test/resources") + } + } +} + +// Create test JAR +val testJar by tasks.registering(Jar::class) { + archiveClassifier.set("tests") + from(sourceSets.test.get().output) +} + +artifacts { + add("archives", testJar) +} + +configurations { + create("testArtifacts") { + extendsFrom(configurations.testRuntimeClasspath.get()) + } +} + +artifacts { + add("testArtifacts", testJar) +} diff --git a/gradle/plugins/src/main/kotlin/gluten.scalatest.gradle.kts b/gradle/plugins/src/main/kotlin/gluten.scalatest.gradle.kts new file mode 100644 index 00000000000..99529895e9f --- /dev/null +++ b/gradle/plugins/src/main/kotlin/gluten.scalatest.gradle.kts @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Convention plugin for ScalaTest testing. + * + * Applies the gradle-scalatest plugin to get ScalaTest's Runner on the classpath and + * configures common test settings (JVM args, tag filtering, etc.). + * + * Test execution mode depends on the module: + * - **gluten-ut modules**: Per-suite JVM isolation via [ScalaTestPerSuiteAction], which + * invokes `Runner -s ` in a separate `javaexec` per test suite class. This + * prevents Spark's JVM-global state (SparkContext, daemon threads) from leaking + * between suites. + * - **All other modules**: Default gradle-scalatest plugin behavior (`Runner -R `), + * running all suites in a single JVM. This matches Maven's scalatest-maven-plugin + * behavior, where suites may share JVM state (e.g., native library loading). + */ + +plugins { + java + id("com.github.maiflai.scalatest") +} + +val scalaBinaryVersion: String by project + +dependencies { + // Flexmark is required by ScalaTest for HTML report generation + testRuntimeOnly("com.vladsch.flexmark:flexmark-all:0.62.2") +} + +// ScalaTest tag filtering via Gradle properties: +// -PtagsToInclude=org.apache.spark.tags.ExtendedSQLTest +// -PtagsToExclude=org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTest +// Spark test home: +// -PsparkTestHome=/opt/shims/spark35/spark_home/ +val tagsToInclude: String? = providers.gradleProperty("tagsToInclude").orNull +val tagsToExclude: String? = providers.gradleProperty("tagsToExclude").orNull +val sparkTestHome: String? = providers.gradleProperty("sparkTestHome").orNull +val testJvmArgs: String? = providers.gradleProperty("testJvmArgs").orNull + +tasks.withType().configureEach { + maxParallelForks = 1 + + // Increase memory for tests + maxHeapSize = "4g" + + // Disable assertions to match Maven Surefire behavior. + // Spark's UnsafeRow has debug assertions (e.g. sizeInBytes % 8 == 0 in setTotalSize) + // that are not meant to be enforced at runtime. Gradle enables assertions by default + // while Maven does not, causing spurious test failures. + enableAssertions = false + + // Populate the gradle-scalatest plugin's `tags` extension from Gradle properties. + // The plugin registers a PatternSet named "tags" on each Test task. + // ScalaTestPerSuiteAction reads this extension to build -n/-l Runner flags. + val tags = extensions.findByName("tags") as? org.gradle.api.tasks.util.PatternSet + if (tags != null) { + tagsToInclude?.split(",")?.map { it.trim() }?.let { tags.include(it) } + tagsToExclude?.split(",")?.map { it.trim() }?.let { tags.exclude(it) } + } + + // Spark test home directory + if (sparkTestHome != null) { + systemProperty("spark.test.home", sparkTestHome!!) + } + + // Extra JVM args from CI (comma-separated -D flags) + testJvmArgs?.split(",")?.forEach { arg -> + if (arg.startsWith("-D") && arg.contains("=")) { + val key = arg.substringAfter("-D").substringBefore("=") + val value = arg.substringAfter("=") + systemProperty(key, value) + } + } + + // JVM arguments for Spark tests + jvmArgs( + "-XX:+IgnoreUnrecognizedVMOptions", + "--add-opens=java.base/java.lang=ALL-UNNAMED", + "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED", + "--add-opens=java.base/java.lang.reflect=ALL-UNNAMED", + "--add-opens=java.base/java.io=ALL-UNNAMED", + "--add-opens=java.base/java.net=ALL-UNNAMED", + "--add-opens=java.base/java.nio=ALL-UNNAMED", + "--add-opens=java.base/java.util=ALL-UNNAMED", + "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED", + "--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED", + "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED", + "--add-opens=java.base/sun.nio.cs=ALL-UNNAMED", + "--add-opens=java.base/sun.security.action=ALL-UNNAMED", + "--add-opens=java.base/sun.util.calendar=ALL-UNNAMED", + "-Djdk.reflect.useDirectMethodHandle=false" + ) +} + +// Replace the gradle-scalatest plugin's action with per-suite forking for gluten-ut modules. +// Other modules keep the plugin's default single-JVM behavior (Runner -R ), matching Maven. +// This must happen in afterEvaluate because the plugin sets its action during project evaluation. +val usePerSuiteForking = project.path.startsWith(":gluten-ut") +if (usePerSuiteForking) { + tasks.withType().configureEach { + // Run up to 3 suite JVMs in parallel (GitHub Actions runner: 4 vCPUs, 16 GB RAM). + // Each suite JVM uses ~4 GB heap, so 3 parallel forks use ~12 GB leaving headroom. + maxParallelForks = 3 + } + afterEvaluate { + tasks.withType().configureEach { + val customAction = ScalaTestPerSuiteAction() + actions.clear() + doLast { + customAction.execute(this as Test) + } + } + } +} diff --git a/gradle/plugins/src/main/kotlin/gluten.shading.gradle.kts b/gradle/plugins/src/main/kotlin/gluten.shading.gradle.kts new file mode 100644 index 00000000000..4f93f2c39d3 --- /dev/null +++ b/gradle/plugins/src/main/kotlin/gluten.shading.gradle.kts @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar + +/** + * Convention plugin for shading dependencies. + * Configures the Shadow plugin with Gluten-specific relocations. + */ + +plugins { + id("com.gradleup.shadow") +} + +val glutenShadePackageName: String by project + +tasks.withType().configureEach { + // Relocate protobuf + relocate("com.google.protobuf", "$glutenShadePackageName.com.google.protobuf") + + // Relocate guava (excluding jimfs) + relocate("com.google.common", "$glutenShadePackageName.com.google.common") { + exclude("com.google.common.jimfs.**") + } + + // Relocate other Google libraries + relocate("com.google.thirdparty", "$glutenShadePackageName.com.google.thirdparty") + relocate("com.google.errorprone.annotations", "$glutenShadePackageName.com.google.errorprone.annotations") + relocate("com.google.j2objc", "$glutenShadePackageName.com.google.j2objc") + relocate("com.google.gson", "$glutenShadePackageName.com.google.gson") + + // Relocate Arrow (excluding C bindings and dataset) + relocate("org.apache.arrow", "$glutenShadePackageName.org.apache.arrow") { + exclude("org.apache.arrow.c.*") + exclude("org.apache.arrow.c.jni.*") + exclude("org.apache.arrow.dataset.**") + } + + // Relocate FlatBuffers + relocate("com.google.flatbuffers", "$glutenShadePackageName.com.google.flatbuffers") + + // Exclude signatures + exclude("META-INF/*.SF") + exclude("META-INF/*.DSA") + exclude("META-INF/*.RSA") + exclude("META-INF/DEPENDENCIES") + exclude("META-INF/LICENSE.txt") + exclude("META-INF/NOTICE.txt") + exclude("LICENSE.txt") + exclude("NOTICE.txt") + + // Merge service files + mergeServiceFiles() + + // Archive settings + archiveClassifier.set("") +} diff --git a/gradle/plugins/src/main/kotlin/gluten.spotless.gradle.kts b/gradle/plugins/src/main/kotlin/gluten.spotless.gradle.kts new file mode 100644 index 00000000000..0e9e1f5d89d --- /dev/null +++ b/gradle/plugins/src/main/kotlin/gluten.spotless.gradle.kts @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Convention plugin for code formatting via Spotless. + * + * Skipped when the build JDK is below 11 because spotless-plugin-gradle requires Java 11+. + * For Java 11+ it configures: + * - Java: google-java-format, import ordering, license headers + * - Scala: scalafmt with .scalafmt.conf, license headers + * + * This mirrors the Maven spotless-maven-plugin configuration in pom.xml. + */ + +// No-op when JDK < 11. The actual configuration is in GlutenSpotless.kt, +// which is only compiled when spotless-plugin-gradle is on the classpath (JDK 11+). +// We use reflection to avoid a compile-time dependency on the GlutenSpotless class. +if (JavaVersion.current().isJava11Compatible) { + val clazz = Class.forName("GlutenSpotless") + val method = clazz.getMethod("apply", org.gradle.api.Project::class.java) + method.invoke(clazz.kotlin.objectInstance, project) +} diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 00000000000..d64cd491770 Binary files /dev/null and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000000..1af9e0930b8 --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,7 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip +networkTimeout=10000 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew new file mode 100755 index 00000000000..2afc9b761d6 --- /dev/null +++ b/gradlew @@ -0,0 +1,204 @@ +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX://shell features: +# * functions; +# * expansions «$var», «## «## «%% * «# +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#)}; t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # temporary args, so each arg winds up back in the position where + # temporary to the end of the arguments. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, GRADLE_OPTS, and GRADLE_USER_OPTS environment variables. +# * The arguments passed to this script. + +# Collect all arguments for the java command. +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xeli.GradleUserHomeLookupOverride" is set to "true". +# This is necessary for Android Studio's Gradle JDK selection dialog. + +# Use "xargs" to parse quoted args. +# +# With -n://to call xargs with -r to avoid spawning +# an extra process (no-run-if-empty). +# +# In anticipation of Gradle Wrapper 9.0 where args are passed via files instead of command line. +# check environment variable to skip this. +# https://github.com/gradle/gradle/issues/26956 +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+@./:528%=,]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 00000000000..9c39d8d7645 --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,81 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows://NT://cmd://parser://conventions +setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows://NT://cmd://parser://conventions +endlocal + +:omega diff --git a/package/build.gradle.kts b/package/build.gradle.kts new file mode 100644 index 00000000000..dedccfd5761 --- /dev/null +++ b/package/build.gradle.kts @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar + +plugins { + id("gluten.java-conventions") + id("gluten.shading") + id("gluten.spotless") +} + +val scalaBinaryVersion: String by project +val sparkVersion: String by project +val effectiveSparkFullVersion: String by rootProject.extra +val backend: String by project +val platform = rootProject.extra.get("platform") as String +val arch = rootProject.extra.get("arch") as String + +// Dynamic dependencies based on backend selection +dependencies { + compileOnly("org.apache.spark:spark-sql_$scalaBinaryVersion:$effectiveSparkFullVersion") + compileOnly("org.apache.spark:spark-core_$scalaBinaryVersion:$effectiveSparkFullVersion") + compileOnly("org.apache.spark:spark-catalyst_$scalaBinaryVersion:$effectiveSparkFullVersion") + compileOnly("org.apache.spark:spark-hive_$scalaBinaryVersion:$effectiveSparkFullVersion") + + // Backend dependency + if (backend == "velox") { + implementation(project(":backends-velox")) + } else if (backend == "clickhouse") { + implementation(project(":backends-clickhouse")) + } + + // Optional module dependencies + mapOf( + "delta" to ":gluten-delta", + "iceberg" to ":gluten-iceberg", + "hudi" to ":gluten-hudi", + "paimon" to ":gluten-paimon", + "celeborn" to ":gluten-celeborn", + "uniffle" to ":gluten-uniffle", + "kafka" to ":gluten-kafka", + ).forEach { (property, module) -> + if (providers.gradleProperty(property).getOrElse("false").toBoolean()) { + implementation(project(module)) + } + } +} + +// Include build info from gluten-core +tasks.processResources { + dependsOn(":gluten-core:generateBuildInfo") + from("${project(":gluten-core").layout.buildDirectory.get()}/generated-resources") { + include("gluten-build-info.properties") + } +} + +// The gluten.shading plugin provides relocations, META-INF exclusions, and mergeServiceFiles. +// Here we add package-specific configuration on top. +tasks.withType().configureEach { + exclude("gluten-build-info.properties") // Excluded from gluten-core, included from resources + + dependencies { + exclude(dependency("com.google.code.findbugs:jsr305")) + } + + // Output naming - match Maven convention + archiveBaseName.set("gluten-$backend-bundle-spark${sparkVersion}_$scalaBinaryVersion-${platform}_$arch") + archiveVersion.set(project.version.toString()) +} + +// Copy the shadow JAR with the Maven-compatible name +tasks.register("copyBundleJar") { + dependsOn(tasks.shadowJar) + from(tasks.shadowJar.get().archiveFile) + into(layout.buildDirectory.dir("libs")) + rename { _ -> + "gluten-$backend-bundle-spark${sparkVersion}_$scalaBinaryVersion-${platform}_$arch-${project.version}.jar" + } +} + +tasks.build { + dependsOn("copyBundleJar") +} + +// This is an assembly module (fat JAR); skip Maven publishing. +tasks.withType().configureEach { enabled = false } +tasks.withType().configureEach { enabled = false } diff --git a/pom.xml b/pom.xml index 46638817f0f..ce02f47f617 100644 --- a/pom.xml +++ b/pom.xml @@ -673,6 +673,7 @@ ${project.basedir}/src/test/java dev/checkstyle.xml + config_loc=${maven.multiModuleProjectDirectory}/dev ${project.basedir}/target/checkstyle-output.xml UTF-8 UTF-8 diff --git a/settings.gradle.kts b/settings.gradle.kts new file mode 100644 index 00000000000..dbe2a7b159e --- /dev/null +++ b/settings.gradle.kts @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +pluginManagement { + includeBuild("gradle/plugins") + repositories { + gradlePluginPortal() + mavenCentral() + maven { + url = uri("https://maven-central.storage-download.googleapis.com/maven2/") + } + } +} + +dependencyResolutionManagement { + repositoriesMode.set(RepositoriesMode.PREFER_SETTINGS) + repositories { + // mavenLocal is needed for locally-published Gluten artifacts (e.g. custom Arrow builds). + // Scope it to Gluten/Arrow groups to prevent partial-cache issues where mavenLocal has + // a POM but not a classifier JAR (e.g. spark-hive:tests), which blocks fallthrough to + // Maven Central. + mavenLocal { + content { + includeGroup("org.apache.gluten") + includeGroup("org.apache.arrow") + } + } + mavenCentral() + maven { + url = uri("https://maven-central.storage-download.googleapis.com/maven2/") + } + } +} + +rootProject.name = "gluten" + +// Core modules +include("gluten-core") +include("gluten-substrait") +include("gluten-ui") + +// RAS modules +include("gluten-ras-common") +project(":gluten-ras-common").projectDir = file("gluten-ras/common") + +include("gluten-ras-planner") +project(":gluten-ras-planner").projectDir = file("gluten-ras/planner") + +// Shims - common is always included +include("shims-common") +project(":shims-common").projectDir = file("shims/common") + +// Version-specific shims - included based on sparkVersion property +val sparkVersion = providers.gradleProperty("sparkVersion").getOrElse("4.1") +val sparkPlainVersion = + when (sparkVersion) { + "3.3" -> "33" + "3.4" -> "34" + "3.5" -> "35" + "4.0" -> "40" + "4.1" -> "41" + else -> "41" + } + +include("shims-spark$sparkPlainVersion") +project(":shims-spark$sparkPlainVersion").projectDir = file("shims/spark$sparkPlainVersion") + +// Backend selection +val backend = providers.gradleProperty("backend").getOrElse("velox") + +if (backend == "velox") { + include("gluten-arrow") + include("backends-velox") +} + +if (backend == "clickhouse") { + include("backends-clickhouse") +} + +// Optional feature modules +mapOf( + "delta" to "gluten-delta", + "iceberg" to "gluten-iceberg", + "hudi" to "gluten-hudi", + "paimon" to "gluten-paimon", + "celeborn" to "gluten-celeborn", + "uniffle" to "gluten-uniffle", + "kafka" to "gluten-kafka", +).forEach { (property, module) -> + if (providers.gradleProperty(property).getOrElse("false").toBoolean()) { + include(module) + } +} + +// Package module (shadow JAR assembly) +include("gluten-package") +project(":gluten-package").projectDir = file("package") + +// Unit test modules (opt-in: -Pspark-ut=true, matches Maven's -Pspark-ut profile) +if (providers.gradleProperty("spark-ut").getOrElse("false").toBoolean()) { + include("gluten-ut-common") + project(":gluten-ut-common").projectDir = file("gluten-ut/common") + + include("gluten-ut-test") + project(":gluten-ut-test").projectDir = file("gluten-ut/test") + + // Spark version-specific UT modules + include("gluten-ut-spark$sparkPlainVersion") + project(":gluten-ut-spark$sparkPlainVersion").projectDir = file("gluten-ut/spark$sparkPlainVersion") +} + +// Integration test modules (opt-in: -PglutenIt=true) +if (providers.gradleProperty("glutenIt").getOrElse("false").toBoolean()) { + include("gluten-it-3rd") + project(":gluten-it-3rd").projectDir = file("tools/gluten-it/3rd") + + include("gluten-it-common") + project(":gluten-it-common").projectDir = file("tools/gluten-it/common") + + include("gluten-it-package") + project(":gluten-it-package").projectDir = file("tools/gluten-it/package") +} diff --git a/shims/common/build.gradle.kts b/shims/common/build.gradle.kts new file mode 100644 index 00000000000..79ce02984a0 --- /dev/null +++ b/shims/common/build.gradle.kts @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("gluten.scala-conventions") + id("gluten.spotless") +} + +val scalaBinaryVersion: String by project +val effectiveSparkFullVersion: String by rootProject.extra +val effectiveHadoopVersion: String by rootProject.extra + +dependencies { + compileOnly("org.apache.spark:spark-sql_$scalaBinaryVersion:$effectiveSparkFullVersion") + compileOnly("org.apache.spark:spark-core_$scalaBinaryVersion:$effectiveSparkFullVersion") + compileOnly("org.apache.spark:spark-catalyst_$scalaBinaryVersion:$effectiveSparkFullVersion") + compileOnly("org.apache.spark:spark-hive_$scalaBinaryVersion:$effectiveSparkFullVersion") + + compileOnly("org.apache.hadoop:hadoop-client:$effectiveHadoopVersion") + + testImplementation("org.scalatest:scalatest_$scalaBinaryVersion:3.2.16") + testImplementation("org.slf4j:slf4j-log4j12:1.7.30") +} diff --git a/shims/spark33/build.gradle.kts b/shims/spark33/build.gradle.kts new file mode 100644 index 00000000000..192705846d4 --- /dev/null +++ b/shims/spark33/build.gradle.kts @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("gluten.scala-conventions") + id("gluten.spotless") +} + +val scalaBinaryVersion: String by project + +// Spark 3.3 specific versions +val sparkFullVersion = "3.3.1" +val hadoopVersion = "2.7.4" + +dependencies { + implementation(project(":shims-common")) + + compileOnly("org.apache.spark:spark-sql_$scalaBinaryVersion:$sparkFullVersion") + compileOnly("org.apache.spark:spark-core_$scalaBinaryVersion:$sparkFullVersion") + compileOnly("org.apache.spark:spark-catalyst_$scalaBinaryVersion:$sparkFullVersion") + compileOnly("org.apache.spark:spark-hive_$scalaBinaryVersion:$sparkFullVersion") + + compileOnly("org.apache.hadoop:hadoop-client:$hadoopVersion") + + testImplementation("org.scalatest:scalatest_$scalaBinaryVersion:3.2.16") + testImplementation("org.slf4j:slf4j-log4j12:1.7.30") +} diff --git a/shims/spark34/build.gradle.kts b/shims/spark34/build.gradle.kts new file mode 100644 index 00000000000..2bee0ebf87a --- /dev/null +++ b/shims/spark34/build.gradle.kts @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("gluten.scala-conventions") + id("gluten.spotless") +} + +val scalaBinaryVersion: String by project + +// Spark 3.4 specific versions +val sparkFullVersion = "3.4.4" +val hadoopVersion = "3.3.4" + +dependencies { + implementation(project(":shims-common")) + + compileOnly("org.apache.spark:spark-sql_$scalaBinaryVersion:$sparkFullVersion") + compileOnly("org.apache.spark:spark-core_$scalaBinaryVersion:$sparkFullVersion") + compileOnly("org.apache.spark:spark-catalyst_$scalaBinaryVersion:$sparkFullVersion") + compileOnly("org.apache.spark:spark-hive_$scalaBinaryVersion:$sparkFullVersion") + + compileOnly("org.apache.hadoop:hadoop-client:$hadoopVersion") + + testImplementation("org.scalatest:scalatest_$scalaBinaryVersion:3.2.16") + testImplementation("org.slf4j:slf4j-log4j12:1.7.30") +} diff --git a/shims/spark35/build.gradle.kts b/shims/spark35/build.gradle.kts new file mode 100644 index 00000000000..8c20f7c15a2 --- /dev/null +++ b/shims/spark35/build.gradle.kts @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("gluten.scala-conventions") + id("gluten.spotless") +} + +val scalaBinaryVersion: String by project + +// Spark 3.5 specific versions +val sparkFullVersion = "3.5.5" +val hadoopVersion = "3.3.4" + +dependencies { + implementation(project(":shims-common")) + + compileOnly("org.apache.spark:spark-sql_$scalaBinaryVersion:$sparkFullVersion") + compileOnly("org.apache.spark:spark-core_$scalaBinaryVersion:$sparkFullVersion") + compileOnly("org.apache.spark:spark-catalyst_$scalaBinaryVersion:$sparkFullVersion") + compileOnly("org.apache.spark:spark-hive_$scalaBinaryVersion:$sparkFullVersion") + + compileOnly("org.apache.hadoop:hadoop-client:$hadoopVersion") + + testImplementation("org.scalatest:scalatest_$scalaBinaryVersion:3.2.16") + testImplementation("org.slf4j:slf4j-log4j12:1.7.30") +} diff --git a/shims/spark40/build.gradle.kts b/shims/spark40/build.gradle.kts new file mode 100644 index 00000000000..e912b0f7a3e --- /dev/null +++ b/shims/spark40/build.gradle.kts @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("gluten.scala-conventions") + id("gluten.spotless") +} + +// Spark 4.0 requires Scala 2.13 +val scalaBinaryVersion = "2.13" + +// Spark 4.0 specific versions +val sparkFullVersion = "4.0.1" +val hadoopVersion = "3.4.1" + +dependencies { + implementation(project(":shims-common")) + + compileOnly("org.apache.spark:spark-sql_$scalaBinaryVersion:$sparkFullVersion") + compileOnly("org.apache.spark:spark-core_$scalaBinaryVersion:$sparkFullVersion") + compileOnly("org.apache.spark:spark-catalyst_$scalaBinaryVersion:$sparkFullVersion") + compileOnly("org.apache.spark:spark-hive_$scalaBinaryVersion:$sparkFullVersion") + + compileOnly("org.apache.hadoop:hadoop-client:$hadoopVersion") + + testImplementation("org.scalatest:scalatest_$scalaBinaryVersion:3.2.16") +} + +// Spark 4.0 requires Java 17+ +java { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 +} diff --git a/shims/spark41/build.gradle.kts b/shims/spark41/build.gradle.kts new file mode 100644 index 00000000000..ce46dbfe154 --- /dev/null +++ b/shims/spark41/build.gradle.kts @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("gluten.scala-conventions") + id("gluten.spotless") +} + +// Spark 4.1 requires Scala 2.13 +val scalaBinaryVersion = "2.13" + +// Spark 4.1 specific versions +val sparkFullVersion = "4.1.1" +val hadoopVersion = "3.4.1" + +dependencies { + implementation(project(":shims-common")) + + compileOnly("org.apache.spark:spark-sql_$scalaBinaryVersion:$sparkFullVersion") + compileOnly("org.apache.spark:spark-core_$scalaBinaryVersion:$sparkFullVersion") + compileOnly("org.apache.spark:spark-catalyst_$scalaBinaryVersion:$sparkFullVersion") + compileOnly("org.apache.spark:spark-hive_$scalaBinaryVersion:$sparkFullVersion") + + compileOnly("org.apache.hadoop:hadoop-client:$hadoopVersion") + + testImplementation("org.scalatest:scalatest_$scalaBinaryVersion:3.2.16") +} + +// Spark 4.1 requires Java 17+ +java { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 +} diff --git a/tools/gluten-it/3rd/build.gradle.kts b/tools/gluten-it/3rd/build.gradle.kts new file mode 100644 index 00000000000..1a8a80b7625 --- /dev/null +++ b/tools/gluten-it/3rd/build.gradle.kts @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("gluten.java-conventions") + id("com.gradleup.shadow") +} + +dependencies { + implementation("io.trino.tpcds:tpcds:1.4") + implementation("io.trino.tpch:tpch:1.1") +} + +tasks.shadowJar { + archiveClassifier.set("") + relocate("com.google.common", "org.apache.gluten.integration.shaded.com.google.common") + + // Exclude signatures + exclude("META-INF/*.SF") + exclude("META-INF/*.DSA") + exclude("META-INF/*.RSA") +} + +// Use the shadow JAR as the module's primary artifact +tasks.jar { enabled = false } +configurations { + runtimeElements { outgoing { artifact(tasks.shadowJar) } } + apiElements { outgoing { artifact(tasks.shadowJar) } } +} diff --git a/tools/gluten-it/common/build.gradle.kts b/tools/gluten-it/common/build.gradle.kts new file mode 100644 index 00000000000..678b0cca9d1 --- /dev/null +++ b/tools/gluten-it/common/build.gradle.kts @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("gluten.scala-conventions") + id("gluten.spotless") +} + +val scalaBinaryVersion: String by project +val effectiveSparkFullVersion: String by rootProject.extra + +// The Maven build of this module does not enable strict Scala warnings, so the source +// code has unused imports, deprecated procedure syntax, etc. Suppress the strict flags +// that the convention plugin enables. +tasks.withType().configureEach { + scalaCompileOptions.additionalParameters = + scalaCompileOptions.additionalParameters?.filter { + it != "-Ywarn-unused:imports" && it != "-Wunused:imports" && !it.startsWith("-Wconf:") + } +} + +// The IT module's Java source code does not conform to the main project's checkstyle +// rules (line length, log4j imports). The Maven build doesn't run checkstyle here. +checkstyle { + isIgnoreFailures = true +} + +dependencies { + // Shaded 3rd-party benchmark libs (exclude transitive deps since they're shaded) + implementation(project(":gluten-it-3rd")) { + isTransitive = false + } + + // Gluten package (runtime — provides the bundle JAR) + runtimeOnly(project(":gluten-package")) + + // Spark + implementation("org.apache.spark:spark-core_$scalaBinaryVersion:$effectiveSparkFullVersion") { + exclude(group = "org.apache.avro", module = "avro-mapred") + exclude(group = "org.apache.avro", module = "avro") + } + implementation("org.apache.spark:spark-sql_$scalaBinaryVersion:$effectiveSparkFullVersion") { + exclude(group = "com.google.protobuf", module = "protobuf-java") + } + implementation("org.apache.spark:spark-repl_$scalaBinaryVersion:$effectiveSparkFullVersion") + implementation("org.apache.spark:spark-catalyst_$scalaBinaryVersion:$effectiveSparkFullVersion") + implementation("org.apache.spark:spark-hive_$scalaBinaryVersion:$effectiveSparkFullVersion") { + exclude(group = "jline", module = "jline") + } + + // Spark test JARs (compile scope in Maven — needed at compile time) + implementation("org.apache.spark:spark-core_$scalaBinaryVersion:$effectiveSparkFullVersion:tests") + implementation("org.apache.spark:spark-catalyst_$scalaBinaryVersion:$effectiveSparkFullVersion:tests") + implementation("org.apache.spark:spark-sql_$scalaBinaryVersion:$effectiveSparkFullVersion:tests") + + // Scala collection compat + implementation("org.scala-lang.modules:scala-collection-compat_$scalaBinaryVersion:2.12.0") + + // SLF4J simple logger + implementation("org.slf4j:slf4j-simple:1.7.30") + + // CLI framework + implementation("info.picocli:picocli:4.6.3") + + // Optional: Celeborn + if (providers.gradleProperty("celeborn").getOrElse("false").toBoolean()) { + val celebornVersion: String by project + val sparkMajorVersion = if (effectiveSparkFullVersion.startsWith("4.")) "4" else "3" + runtimeOnly("org.apache.celeborn:celeborn-client-spark-${sparkMajorVersion}-shaded_$scalaBinaryVersion:$celebornVersion") + } + + // Optional: Uniffle + if (providers.gradleProperty("uniffle").getOrElse("false").toBoolean()) { + val uniffleVersion: String by project + val sparkMajorVersion = if (effectiveSparkFullVersion.startsWith("4.")) "4" else "3" + runtimeOnly("org.apache.uniffle:rss-client-spark${sparkMajorVersion}-shaded:$uniffleVersion") + } + + // Optional: Delta + if (providers.gradleProperty("delta").getOrElse("false").toBoolean()) { + val effectiveDeltaPackageName: String? by rootProject.extra + val effectiveDeltaVersion: String? by rootProject.extra + val deltaPackage = effectiveDeltaPackageName ?: "delta-spark" + val deltaVer = effectiveDeltaVersion ?: "4.0.0" + runtimeOnly("io.delta:${deltaPackage}_$scalaBinaryVersion:$deltaVer") { + exclude(group = "org.antlr") + exclude(group = "org.scala-lang", module = "scala-library") + } + } +} diff --git a/tools/gluten-it/package/build.gradle.kts b/tools/gluten-it/package/build.gradle.kts new file mode 100644 index 00000000000..7863ba64b63 --- /dev/null +++ b/tools/gluten-it/package/build.gradle.kts @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("gluten.java-conventions") +} + +dependencies { + runtimeOnly(project(":gluten-it-common")) +} + +// Copy all runtime dependencies to build/lib/ (mirrors Maven's copy-dependencies) +val copyDeps by tasks.registering(Copy::class) { + from(configurations.runtimeClasspath) + from(tasks.jar) + into(layout.buildDirectory.dir("lib")) +} + +tasks.build { + dependsOn(copyDeps) +} + +// Skip publishing for assembly module +tasks.withType().configureEach { enabled = false } +tasks.withType().configureEach { enabled = false } diff --git a/tools/gluten-it/sbin/gluten-it.sh b/tools/gluten-it/sbin/gluten-it.sh index dd4c479d7e1..d5bfba923c9 100755 --- a/tools/gluten-it/sbin/gluten-it.sh +++ b/tools/gluten-it/sbin/gluten-it.sh @@ -20,7 +20,13 @@ BASEDIR=$(dirname $0) LIB_DIR=$BASEDIR/../package/target/lib if [[ ! -d $LIB_DIR ]]; then - echo "Lib directory not found at $LIB_DIR. Please build gluten-it first. For example: mvn clean install" + # Fallback to Gradle build output layout + LIB_DIR=$BASEDIR/../package/build/lib +fi +if [[ ! -d $LIB_DIR ]]; then + echo "Lib directory not found. Please build gluten-it first. For example:" + echo " Maven: mvn clean install" + echo " Gradle: ./gradlew :gluten-it-package:build -PglutenIt=true" exit 1 fi @@ -30,7 +36,7 @@ SPARK_JVM_OPTIONS=$($JAVA_HOME/bin/java -cp $JAR_PATH org.apache.gluten.integrat EMBEDDED_SPARK_HOME=$BASEDIR/../spark-home -mkdir -p $EMBEDDED_SPARK_HOME && ln -snf ../package/target/lib $EMBEDDED_SPARK_HOME/jars +mkdir -p $EMBEDDED_SPARK_HOME && ln -snf $(realpath $LIB_DIR) $EMBEDDED_SPARK_HOME/jars # We temporarily disallow setting these two variables by caller. SPARK_HOME=""