[Spark][Infra] Drop support for Spark 3.5 and formally pin to released Spark 4.0.1

.github/workflows/iceberg_test.yaml

@@ -1,5 +1,5 @@
 name: "Delta Iceberg Latest"
-on: [push, pull_request]
+on: [] # [push, pull_request]
 jobs:
   test:
     name: "DIL: Scala ${{ matrix.scala }}"
@@ -25,7 +25,7 @@ jobs:
         uses: actions/setup-java@v3
         with:
           distribution: "zulu"
-          java-version: "11"
+          java-version: "17"
       - name: Cache Scala, SBT
         uses: actions/cache@v3
         with:

.github/workflows/kernel_test.yaml

@@ -37,11 +37,12 @@ jobs:
           echo "Runner arch: ${{ runner.arch }}"
       - name: Checkout code
         uses: actions/checkout@v4
+      # Run unit tests with JDK 17. These unit tests depend on Spark, and Spark 4.0+ is JDK 17.
       - name: install java
         uses: actions/setup-java@v4
         with:
           distribution: "zulu"
-          java-version: "11"
+          java-version: "17"
       - name: Cache SBT and dependencies
         id: cache-sbt
         uses: actions/cache@v4
@@ -59,7 +60,7 @@ jobs:
           else
             echo "❌ Cache MISS - will download dependencies"
           fi
-      - name: Run tests
+      - name: Run unit tests
         run: |
           python run-tests.py --group kernel --coverage --shard ${{ matrix.shard }}
 
@@ -68,6 +69,7 @@ jobs:
     runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v3
+      # Run integration tests with JDK 11, as they have no Spark dependency
       - name: install java
         uses: actions/setup-java@v3
         with:

.github/workflows/kernel_unitycatalog_test.yaml

@@ -22,7 +22,7 @@ jobs:
         uses: actions/setup-java@v3
         with:
           distribution: "zulu"
-          java-version: "11"
+          java-version: "17"
         if: steps.git-diff.outputs.diff
       - name: Run Unity tests with coverage
         run: |

.github/workflows/spark_examples_test.yaml

@@ -24,7 +24,7 @@ jobs:
         uses: actions/setup-java@v3
         with:
           distribution: "zulu"
-          java-version: "11"
+          java-version: "17"
       - name: Cache Scala, SBT
         uses: actions/cache@v3
         with:

.github/workflows/spark_python_test.yaml

@@ -25,7 +25,7 @@ jobs:
         uses: actions/setup-java@v3
         with:
           distribution: "zulu"
-          java-version: "11"
+          java-version: "17"
       - name: Cache Scala, SBT
         uses: actions/cache@v3
         with:
@@ -53,33 +53,36 @@ jobs:
           export PATH="~/.pyenv/bin:$PATH"
           eval "$(pyenv init -)"
           eval "$(pyenv virtualenv-init -)"
-          pyenv install 3.8.18
-          pyenv global system 3.8.18
-          pipenv --python 3.8 install
+          pyenv install 3.9
+          pyenv global system 3.9
+          pipenv --python 3.9 install
           # Update the pip version to 24.0. By default `pyenv.run` installs the latest pip version
           # available. From version 24.1, `pip` doesn't allow installing python packages
           # with version string containing `-`. In Delta-Spark case, the pypi package generated has
           # `-SNAPSHOT` in version (e.g. `3.3.0-SNAPSHOT`) as the version is picked up from
           # the`version.sbt` file.
           pipenv run pip install pip==24.0 setuptools==69.5.1 wheel==0.43.0
-          # Install PySpark without bundled Scala 2.12 JARs - read more in the future note below
-          pipenv run pip install pyspark==3.5.3 --no-deps
-          pipenv run pip install py4j==0.10.9.7
-          pipenv run pip install flake8==3.5.0 pypandoc==1.3.3
-          pipenv run pip install black==23.9.1
+          pipenv run pip install pyspark==4.0.1
+          pipenv run pip install flake8==3.9.0
+          pipenv run pip install black==23.12.1
           pipenv run pip install importlib_metadata==3.10.0
-          # The mypy versions 0.982 and 1.8.0 have conflicting rules (cannot get style checks to
-          # pass for both versions on the same file) so we upgrade this to match Spark 4.0
           pipenv run pip install mypy==1.8.0
           pipenv run pip install mypy-protobuf==3.3.0
           pipenv run pip install cryptography==37.0.4
           pipenv run pip install twine==4.0.1
           pipenv run pip install wheel==0.33.4
           pipenv run pip install setuptools==41.1.0
           pipenv run pip install pydocstyle==3.0.0
-          pipenv run pip install pandas==1.1.3
-          pipenv run pip install pyarrow==8.0.0
-          pipenv run pip install numpy==1.20.3
+          pipenv run pip install pandas==2.2.0
+          pipenv run pip install pyarrow==11.0.0
+          pipenv run pip install pypandoc==1.3.3
+          pipenv run pip install numpy==1.22.4
+          pipenv run pip install grpcio==1.67.0
+          pipenv run pip install grpcio-status==1.67.0
+          pipenv run pip install googleapis-common-protos==1.65.0
+          pipenv run pip install protobuf==5.29.1
+          pipenv run pip install googleapis-common-protos-stubs==2.2.0
+          pipenv run pip install grpc-stubs==1.24.11
         if: steps.git-diff.outputs.diff
       - name: Run Python tests
         # when changing TEST_PARALLELISM_COUNT make sure to also change it in spark_master_test.yaml

.github/workflows/spark_test.yaml

@@ -29,7 +29,7 @@ jobs:
         uses: actions/setup-java@v3
         with:
           distribution: "zulu"
-          java-version: "11"
+          java-version: "17"
       - name: Cache Scala, SBT
         uses: actions/cache@v3
         with:
@@ -57,29 +57,36 @@ jobs:
           export PATH="~/.pyenv/bin:$PATH"
           eval "$(pyenv init -)"
           eval "$(pyenv virtualenv-init -)"
-          pyenv install 3.8.18
-          pyenv global system 3.8.18
-          pipenv --python 3.8 install
+          pyenv install 3.9
+          pyenv global system 3.9
+          pipenv --python 3.9 install
           # Update the pip version to 24.0. By default `pyenv.run` installs the latest pip version
           # available. From version 24.1, `pip` doesn't allow installing python packages
           # with version string containing `-`. In Delta-Spark case, the pypi package generated has
           # `-SNAPSHOT` in version (e.g. `3.3.0-SNAPSHOT`) as the version is picked up from
           # the`version.sbt` file.
           pipenv run pip install pip==24.0 setuptools==69.5.1 wheel==0.43.0
-          pipenv run pip install pyspark==3.5.3
-          pipenv run pip install flake8==3.5.0 pypandoc==1.3.3
-          pipenv run pip install black==23.9.1
+          pipenv run pip install pyspark==4.0.1
+          pipenv run pip install flake8==3.9.0
+          pipenv run pip install black==23.12.1
           pipenv run pip install importlib_metadata==3.10.0
-          pipenv run pip install mypy==0.982
+          pipenv run pip install mypy==1.8.0
           pipenv run pip install mypy-protobuf==3.3.0
           pipenv run pip install cryptography==37.0.4
           pipenv run pip install twine==4.0.1
           pipenv run pip install wheel==0.33.4
           pipenv run pip install setuptools==41.1.0
           pipenv run pip install pydocstyle==3.0.0
-          pipenv run pip install pandas==1.1.3
-          pipenv run pip install pyarrow==8.0.0
-          pipenv run pip install numpy==1.20.3
+          pipenv run pip install pandas==2.2.0
+          pipenv run pip install pyarrow==11.0.0
+          pipenv run pip install pypandoc==1.3.3
+          pipenv run pip install numpy==1.22.4
+          pipenv run pip install grpcio==1.67.0
+          pipenv run pip install grpcio-status==1.67.0
+          pipenv run pip install googleapis-common-protos==1.65.0
+          pipenv run pip install protobuf==5.29.1
+          pipenv run pip install googleapis-common-protos-stubs==2.2.0
+          pipenv run pip install grpc-stubs==1.24.11
         if: steps.git-diff.outputs.diff
       - name: Scala structured logging style check
         run: |

.github/workflows/unidoc.yaml

@@ -13,7 +13,7 @@
           uses: actions/setup-java@v3
           with:
             distribution: "zulu"
-            java-version: "11"
+            java-version: "17"
         - uses: actions/checkout@v3
         - name: generate unidoc
           run: build/sbt "++ ${{ matrix.scala }}" unidoc

build.sbt

@@ -66,7 +66,7 @@ val sparkVersion = settingKey[String]("Spark version")
 
 // Dependent library versions
 val defaultSparkVersion = SparkVersionSpec.DEFAULT.fullVersion // Spark version to use for testing in non-delta-spark related modules
-val hadoopVersion = "3.3.4"
+val hadoopVersion = "3.4.0"
 val scalaTestVersion = "3.2.15"
 val scalaTestVersionForConnectors = "3.0.8"
 val parquet4sVersion = "1.9.4"
@@ -257,7 +257,7 @@ lazy val connectClient = (project in file("spark-connect/client"))
         // Create a symlink for the log4j properties
         val confDir = distributionDir / "conf"
         IO.createDirectory(confDir)
-        val log4jProps = (spark / Test / resourceDirectory).value / "log4j2_spark_master.properties"
+        val log4jProps = (spark / Test / resourceDirectory).value / "log4j2.properties"
         val linkedLog4jProps = confDir / "log4j2.properties"
         Files.createSymbolicLink(linkedLog4jProps.toPath, log4jProps.toPath)
       }
@@ -705,6 +705,8 @@ lazy val contribs = (project in file("contribs"))
     Compile / compile := ((Compile / compile) dependsOn createTargetClassesDir).value
   ).configureUnidoc()
 
+/*
+TODO: compilation broken for Spark 4.0
 lazy val sharing = (project in file("sharing"))
   .dependsOn(spark % "compile->compile;test->test;provided->provided")
   .disablePlugins(JavaFormatterPlugin, ScalafmtPlugin)
@@ -715,22 +717,6 @@ lazy val sharing = (project in file("sharing"))
     releaseSettings,
     CrossSparkVersions.sparkDependentSettings(sparkVersion),
     Test / javaOptions ++= Seq("-ea"),
-    Compile / compile := runTaskOnlyOnSparkMaster(
-      task = Compile / compile,
-      taskName = "compile",
-      projectName = "delta-sharing-spark",
-      emptyValue = Analysis.empty.asInstanceOf[CompileAnalysis]
-    ).value,
-    Test / test := runTaskOnlyOnSparkMaster(
-      task = Test / test,
-      taskName = "test",
-      projectName = "delta-sharing-spark",
-      emptyValue = ()).value,
-    publish := runTaskOnlyOnSparkMaster(
-      task = publish,
-      taskName = "publish",
-      projectName = "delta-sharing-spark",
-      emptyValue = ()).value,
     libraryDependencies ++= Seq(
       "org.apache.spark" %% "spark-sql" % sparkVersion.value % "provided",
 
@@ -747,6 +733,7 @@ lazy val sharing = (project in file("sharing"))
       "org.apache.spark" %% "spark-hive" % sparkVersion.value % "test" classifier "tests",
     )
   ).configureUnidoc()
+*/
 
 lazy val kernelApi = (project in file("kernel/kernel-api"))
   .enablePlugins(ScalafmtPlugin)
@@ -898,7 +885,7 @@ lazy val kernelDefaults = (project in file("kernel/kernel-defaults"))
       // such as warm runs, cold runs, defining benchmark parameter variables etc.
       "org.openjdk.jmh" % "jmh-core" % "1.37" % "test",
       "org.openjdk.jmh" % "jmh-generator-annprocess" % "1.37" % "test",
-      "io.delta" %% "delta-spark" % "3.3.2" % "test",
+      "io.delta" %% "delta-spark" % "4.0.0" % "test",
 
       "org.apache.spark" %% "spark-hive" % defaultSparkVersion % "test" classifier "tests",
       "org.apache.spark" %% "spark-sql" % defaultSparkVersion % "test" classifier "tests",
@@ -1010,6 +997,8 @@ lazy val storageS3DynamoDB = (project in file("storage-s3-dynamodb"))
     )
   ).configureUnidoc()
 
+/*
+TODO: readd delta-iceberg on Spark 4.0+
 val icebergSparkRuntimeArtifactName = {
  val (expMaj, expMin, _) = getMajorMinorPatch(defaultSparkVersion)
  s"iceberg-spark-runtime-$expMaj.$expMin"
@@ -1165,6 +1154,7 @@ lazy val icebergShaded = (project in file("icebergShaded"))
     assembly / assemblyMergeStrategy := updateMergeStrategy((assembly / assemblyMergeStrategy).value),
     assemblyPackageScala / assembleArtifact := false,
   )
+*/
 
 lazy val hudi = (project in file("hudi"))
   .dependsOn(spark % "compile->compile;test->test;provided->provided")
@@ -1265,14 +1255,16 @@ val createTargetClassesDir = taskKey[Unit]("create target classes dir")
 
 // Don't use these groups for any other projects
 lazy val sparkGroup = project
-  .aggregate(spark, sparkV1, sparkV1Filtered, sparkV2, contribs, storage, storageS3DynamoDB, sharing, hudi)
+  // TODO: add sharing back after fixing compilation
+  .aggregate(spark, sparkV1, sparkV1Filtered, sparkV2, contribs, storage, storageS3DynamoDB, hudi)
   .settings(
     // crossScalaVersions must be set to Nil on the aggregating project
     crossScalaVersions := Nil,
     publishArtifact := false,
     publish / skip := false,
   )
 
+/*
 lazy val icebergGroup = project
   .aggregate(iceberg, testDeltaIcebergJar)
   .settings(
@@ -1281,6 +1273,7 @@ lazy val icebergGroup = project
     publishArtifact := false,
     publish / skip := false,
   )
+*/
 
 lazy val kernelGroup = project
   .aggregate(kernelApi, kernelDefaults, kernelBenchmarks)

examples/scala/build.sbt

@@ -42,8 +42,10 @@ def getMajorMinor(version: String): (Int, Int) = {
   }
 }
 val lookupSparkVersion: PartialFunction[(Int, Int), String] = {
-  // version 4.0.0-preview1
-  case (major, minor) if major >= 4 => "4.0.0-preview1"
+  // TODO: how to run integration tests for multiple Spark versions
+  case (major, minor) if major >= 4 && minor >= 1 => "4.0.1"
+  // version 4.0.0
+  case (major, minor) if major >= 4 => "4.0.0"
   // versions 3.3.x+
   case (major, minor) if major >= 3 && minor >=3 => "3.5.3"
   // versions 3.0.0 to 3.2.x

kernel/kernel-api/src/main/java/io/delta/kernel/internal/checkpoints/Checkpointer.java

@@ -91,6 +91,11 @@ public static void checkpoint(Engine engine, Clock clock, SnapshotImpl snapshot)
       numberOfAddFiles = checkpointDataIter.getNumberOfAddActions();
     } catch (FileAlreadyExistsException faee) {
       throw new CheckpointAlreadyExistsException(version);
+    } catch (IOException io) {
+      if (io.getCause() instanceof FileAlreadyExistsException) {
+        throw new CheckpointAlreadyExistsException(version);
+      }
+      throw io;
     }
 
     final CheckpointMetaData checkpointMetaData =