Spark 344

.github/workflows/util/install_spark_resources.sh

@@ -50,13 +50,13 @@ case "$1" in
 3.4)
     # Spark-3.4
     cd ${INSTALL_DIR} && \
-    wget -nv https://archive.apache.org/dist/spark/spark-3.4.3/spark-3.4.3-bin-hadoop3.tgz && \
-    tar --strip-components=1 -xf spark-3.4.3-bin-hadoop3.tgz spark-3.4.3-bin-hadoop3/jars/ && \
-    rm -rf spark-3.4.3-bin-hadoop3.tgz && \
+    wget -nv https://archive.apache.org/dist/spark/spark-3.4.4/spark-3.4.4-bin-hadoop3.tgz && \
+    tar --strip-components=1 -xf spark-3.4.4-bin-hadoop3.tgz spark-3.4.4-bin-hadoop3/jars/ && \
+    rm -rf spark-3.4.4-bin-hadoop3.tgz && \
     mkdir -p ${INSTALL_DIR}/shims/spark34/spark_home/assembly/target/scala-2.12 && \
     mv jars ${INSTALL_DIR}/shims/spark34/spark_home/assembly/target/scala-2.12 && \
-    wget -nv https://github.com/apache/spark/archive/refs/tags/v3.4.3.tar.gz && \
-    tar --strip-components=1 -xf v3.4.3.tar.gz spark-3.4.3/sql/core/src/test/resources/  && \
+    wget -nv https://github.com/apache/spark/archive/refs/tags/v3.4.4.tar.gz && \
+    tar --strip-components=1 -xf v3.4.4.tar.gz spark-3.4.4/sql/core/src/test/resources/  && \
     mkdir -p shims/spark34/spark_home/ && \
     mv sql shims/spark34/spark_home/
     ;;

.github/workflows/velox_backend.yml

@@ -752,9 +752,9 @@ jobs:
           dnf module -y install python39 && \
           alternatives --set python3 /usr/bin/python3.9 && \
           pip3 install setuptools && \
-          pip3 install pyspark==3.4.3 cython && \
+          pip3 install pyspark==3.4.4 cython && \
           pip3 install pandas pyarrow
-      - name: Build and Run unit test for Spark 3.4.3 (other tests)
+      - name: Build and Run unit test for Spark 3.4.4 (other tests)
         run: |
           cd $GITHUB_WORKSPACE/
           export SPARK_SCALA_VERSION=2.12
@@ -791,7 +791,7 @@ jobs:
         with:
           name: arrow-jars-centos-7-${{github.sha}}
           path: /root/.m2/repository/org/apache/arrow/
-      - name: Build and Run unit test for Spark 3.4.3 (slow tests)
+      - name: Build and Run unit test for Spark 3.4.4 (slow tests)
         run: |
           cd $GITHUB_WORKSPACE/
           $MVN_CMD clean test -Pspark-3.4 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut -Phudi \

docs/get-started/Velox.md

@@ -9,7 +9,7 @@ parent: Getting-Started
 
 | Type  | Version                      |
 |-------|------------------------------|
-| Spark | 3.2.2, 3.3.1, 3.4.3, 3.5.1   |
+| Spark | 3.2.2, 3.3.1, 3.4.4, 3.5.1   |
 | OS    | Ubuntu20.04/22.04, Centos7/8 |
 | jdk   | openjdk8/jdk17               |
 | scala | 2.12                         |
@@ -19,7 +19,7 @@ parent: Getting-Started
 Currently, Gluten+Velox backend is only tested on **Ubuntu20.04/Ubuntu22.04/Centos7/Centos8**.
 Other kinds of OS support are still in progress. The long term goal is to support several common OS and conda env deployment.
 
-Currently, the officially supported Spark versions are 3.2.2, 3.3.1, 3.4.3 and 3.5.1.
+Currently, the officially supported Spark versions are 3.2.2, 3.3.1, 3.4.4 and 3.5.1.
 
 We need to set up the `JAVA_HOME` env. Currently, Gluten supports **java 8** and **java 17**.
 
@@ -434,7 +434,7 @@ After the set-up, you can now build Gluten with HBM. Below command is used to en
 ```bash
 cd /path/to/gluten
 
-## The script builds four jars for spark 3.2.2, 3.3.1, 3.4.3 and 3.5.1.
+## The script builds four jars for spark 3.2.2, 3.3.1, 3.4.4 and 3.5.1.
 ./dev/buildbundle-veloxbe.sh --enable_hbm=ON
 ```
 
@@ -520,7 +520,7 @@ exit
 ```bash
 cd /path/to/gluten
 
-## The script builds four jars for spark 3.2.2, 3.3.1, 3.4.3 and 3.5.1.
+## The script builds four jars for spark 3.2.2, 3.3.1, 3.4.4 and 3.5.1.
 ./dev/buildbundle-veloxbe.sh --enable_qat=ON
 ```
 
@@ -616,7 +616,7 @@ After the set-up, you can now build Gluten with QAT. Below command is used to en
 ```bash
 cd /path/to/gluten
 
-## The script builds four jars for spark 3.2.2, 3.3.1, 3.4.3 and 3.5.1.
+## The script builds four jars for spark 3.2.2, 3.3.1, 3.4.4 and 3.5.1.
 ./dev/buildbundle-veloxbe.sh --enable_iaa=ON
 ```
 

docs/get-started/build-guide.md

@@ -73,5 +73,5 @@ It's name pattern is `gluten-<backend_type>-bundle-spark<spark.bundle.version>_<
 |---------------|----------------------|----------------------|
 | 3.2.2         | 3.2                  | 2.12                 |
 | 3.3.1         | 3.3                  | 2.12                 |
-| 3.4.3         | 3.4                  | 2.12                 |
+| 3.4.4         | 3.4                  | 2.12                 |
 | 3.5.1         | 3.5                  | 2.12                 |

pom.xml

@@ -59,9 +59,9 @@
     <scala.version>2.12.15</scala.version>
     <spark.major.version>3</spark.major.version>
     <sparkbundle.version>3.4</sparkbundle.version>
-    <spark.version>3.4.3</spark.version>
+    <spark.version>3.4.4</spark.version>
     <sparkshim.artifactId>spark-sql-columnar-shims-spark34</sparkshim.artifactId>
-    <iceberg.version>1.5.0</iceberg.version>
+    <iceberg.version>1.7.1</iceberg.version>
     <delta.package.name>delta-core</delta.package.name>
     <delta.version>2.4.0</delta.version>
     <delta.binary.version>24</delta.binary.version>
@@ -322,8 +322,8 @@
       <properties>
         <sparkbundle.version>3.4</sparkbundle.version>
         <sparkshim.artifactId>spark-sql-columnar-shims-spark34</sparkshim.artifactId>
-        <spark.version>3.4.3</spark.version>
-        <iceberg.version>1.5.0</iceberg.version>
+        <spark.version>3.4.4</spark.version>
+        <iceberg.version>1.7.1</iceberg.version>
         <delta.package.name>delta-core</delta.package.name>
         <delta.version>2.4.0</delta.version>
         <delta.binary.version>24</delta.binary.version>

shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/SparkShimProvider.scala

@@ -20,7 +20,7 @@ import org.apache.gluten.sql.shims.{SparkShimDescriptor, SparkShims}
 import org.apache.gluten.sql.shims.spark34.SparkShimProvider.DESCRIPTOR
 
 object SparkShimProvider {
-  val DESCRIPTOR = SparkShimDescriptor(3, 4, 3)
+  val DESCRIPTOR = SparkShimDescriptor(3, 4, 4)
 }
 
 class SparkShimProvider extends org.apache.gluten.sql.shims.SparkShimProvider {

shims/spark34/src/main/scala/org/apache/spark/shuffle/SparkSortShuffleWriterUtil.scala

@@ -27,6 +27,6 @@ object SparkSortShuffleWriterUtil {
       context: TaskContext,
       writeMetrics: ShuffleWriteMetricsReporter,
       shuffleExecutorComponents: ShuffleExecutorComponents): ShuffleWriter[K, V] = {
-    new SortShuffleWriter(handle, mapId, context, shuffleExecutorComponents)
+    new SortShuffleWriter(handle, mapId, context, writeMetrics, shuffleExecutorComponents)
   }
 }

tools/gluten-it/pom.xml

@@ -18,7 +18,7 @@
     <maven.compiler.source>${java.version}</maven.compiler.source>
     <maven.compiler.target>${java.version}</maven.compiler.target>
     <scala.library.version>2.12.17</scala.library.version>
-    <spark.version>3.4.3</spark.version>
+    <spark.version>3.4.4</spark.version>
     <scala.binary.version>2.12</scala.binary.version>
     <spark.major.version>3</spark.major.version>
     <celeborn.version>0.3.2-incubating</celeborn.version>
@@ -163,7 +163,7 @@
     <profile>
       <id>spark-3.4</id>
       <properties>
-        <spark.version>3.4.3</spark.version>
+        <spark.version>3.4.4</spark.version>
         <scala.library.version>2.12.17</scala.library.version>
       </properties>
     </profile>