Spark 4.0

.github/workflows/dev-ci.yml

@@ -11,11 +11,14 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      - name: Set up JDK 8
+      - name: Set up JDK 8 and 17
         uses: actions/setup-java@v4
         with:
-          java-version: '8'
+          java-version: |
+            8
+            17
           distribution: 'temurin'
+          check-latest: false
       - name: Cache Maven packages
         uses: actions/cache@v4
         with:
@@ -35,6 +38,9 @@ jobs:
       - name: Build for Spark 3.5
         run: |
           mvn -B -e install -P3.5 -DskipTests
+      - name: Build for Spark 4.0
+        run: |
+          mvn -B -e install -P4.0 -DskipTests
       - name: Get short commit hash
         id: vars
         run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
@@ -47,6 +53,12 @@ jobs:
             fi
             cp "$file" shaded-jars/
           done
+          for file in spark-4.*/target/spark-4.*.jar; do
+            if [[ "$file" == *"lib"* || "$file" == *"original"* ]]; then
+              continue
+            fi
+            cp "$file" shaded-jars/
+          done
       - name: Upload Shaded JARs
         uses: actions/upload-artifact@v4
         with:

pom.xml

@@ -86,6 +86,17 @@
         <module>spark-3.5-spanner</module>
       </modules>
     </profile>
+    <profile>
+      <id>4.0</id>
+      <activation><activeByDefault>false</activeByDefault></activation>
+      <modules>
+        <module>spark-3.2-spanner-lib</module>
+        <module>spark-3.3-spanner-lib</module>
+        <module>spark-3.5-spanner-lib</module>
+        <module>spark-4.0-spanner-lib</module>
+        <module>spark-4.0-spanner</module>
+      </modules>
+    </profile>
  </profiles>
 
 </project>

spark-3.1-spanner-lib/pom.xml

@@ -57,7 +57,6 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-jar-plugin</artifactId>
-        <version>3.3.0</version>
         <configuration>
           <excludes>
             <exclude>**/org.apache.spark.sql.sources.DataSourceRegister</exclude>

spark-3.1-spanner-lib/src/test/java/com/google/cloud/spark/spanner/SpannerDataWriterTestBase.java

@@ -45,6 +45,7 @@
 import java.util.concurrent.Future;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
+import org.apache.spark.sql.Encoder;
 import org.apache.spark.sql.Row;
 import org.apache.spark.sql.RowFactory;
 import org.apache.spark.sql.catalyst.InternalRow;
@@ -69,10 +70,11 @@ public abstract class SpannerDataWriterTestBase {
 
   private final ScheduledExecutorService scheduledExecutor =
       TestingExecutors.sameThreadScheduledExecutor();
-  private StructType schema;
+  protected StructType schema;
   private Map<String, String> properties;
   private BatchClientWithCloser batchClientWithCloser;
-  private ExpressionEncoder.Serializer<Row> serializer;
+  protected ExpressionEncoder.Serializer<Row> serializer;
+  protected Encoder<Row> encoder;
   @Mock private ExecutorService mockExecutor;
   @Mock private ScheduledExecutorService mockScheduledExecutor;
 
@@ -89,20 +91,23 @@ public void setUp() {
               DataTypes.createStructField("string_col", DataTypes.StringType, true),
             });
 
-    ExpressionEncoder<Row> encoder = getEncoder(schema);
-    serializer = encoder.createSerializer();
+    encoder = getEncoder(schema);
+    localSetup();
     properties = new HashMap<>();
     properties.put("table", "testTable");
     properties.put("mutationsPerBatch", "2"); // Use a small batch size for tests
   }
 
+  /** Provides Spark version specific set up. */
+  protected abstract void localSetup();
+
   /**
    * Gets an ExpressionEncoder, the implementation of which depends on the version of Spark used.
    *
    * @param struct used to determine the Encoder.
-   * @return The ExpressionEncoder.
+   * @return The Encoder.
    */
-  protected abstract ExpressionEncoder<Row> getEncoder(StructType struct);
+  protected abstract Encoder<Row> getEncoder(StructType struct);
 
   private SpannerDataWriter createWriter(Map<String, String> props) {
     return new SpannerDataWriter(
@@ -453,7 +458,7 @@ public void testMutationTypeLowerCaseIsHonored() throws IOException {
     testMutationTypeIsHonored("mutationtype");
   }
 
-  private InternalRow CreateInternalRow(long i) {
+  protected InternalRow CreateInternalRow(long i) {
     return serializer.apply(RowFactory.create(i, "row" + i));
   }
 }

spark-3.1-spanner-lib/src/test/java/com/google/cloud/spark/spanner/Spark31SpannerDataWriterTest.java

@@ -14,6 +14,7 @@
 
 package com.google.cloud.spark.spanner;
 
+import org.apache.spark.sql.Encoder;
 import org.apache.spark.sql.Row;
 import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder;
 import org.apache.spark.sql.catalyst.encoders.RowEncoder;
@@ -22,7 +23,12 @@
 public class Spark31SpannerDataWriterTest extends SpannerDataWriterTestBase {
 
   @Override
-  protected ExpressionEncoder<Row> getEncoder(StructType schema) {
+  protected void localSetup() {
+    serializer = ((ExpressionEncoder<Row>) encoder).createSerializer();
+  }
+
+  @Override
+  protected Encoder<Row> getEncoder(StructType schema) {
     return RowEncoder.apply(schema);
   }
 }

spark-3.1-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/SparkSpannerIntegrationTestBase.java

@@ -34,6 +34,7 @@ public void setUpSpark() {
             .appName("SparkSpannerIntegrationTest")
             .config("spark.ui.enabled", "false")
             .config("spark.default.parallelism", 20)
+            .config("spark.sql.legacy.timeParserPolicy", "LEGACY")
             .getOrCreate();
 
     if (catalogProps.get("emulatorHost") != null) {

spark-3.2-spanner-lib/src/test/java/com/google/cloud/spark/spanner/Spark32SpannerDataWriterTest.java

@@ -14,6 +14,7 @@
 
 package com.google.cloud.spark.spanner;
 
+import org.apache.spark.sql.Encoder;
 import org.apache.spark.sql.Row;
 import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder;
 import org.apache.spark.sql.catalyst.encoders.RowEncoder;
@@ -22,7 +23,12 @@
 public class Spark32SpannerDataWriterTest extends SpannerDataWriterTestBase {
 
   @Override
-  protected ExpressionEncoder<Row> getEncoder(StructType schema) {
+  protected void localSetup() {
+    serializer = ((ExpressionEncoder<Row>) encoder).createSerializer();
+  }
+
+  @Override
+  protected Encoder<Row> getEncoder(StructType schema) {
     return RowEncoder.apply(schema);
   }
 }

spark-3.3-spanner-lib/src/test/java/com/google/cloud/spark/spanner/Spark33SpannerDataWriterTest.java

@@ -14,6 +14,7 @@
 
 package com.google.cloud.spark.spanner;
 
+import org.apache.spark.sql.Encoder;
 import org.apache.spark.sql.Row;
 import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder;
 import org.apache.spark.sql.catalyst.encoders.RowEncoder;
@@ -22,7 +23,12 @@
 public class Spark33SpannerDataWriterTest extends SpannerDataWriterTestBase {
 
   @Override
-  protected ExpressionEncoder<Row> getEncoder(StructType schema) {
+  protected void localSetup() {
+    serializer = ((ExpressionEncoder<Row>) encoder).createSerializer();
+  }
+
+  @Override
+  protected Encoder<Row> getEncoder(StructType schema) {
     return RowEncoder.apply(schema);
   }
 }

spark-3.5-spanner-lib/src/test/java/com/google/cloud/spark/spanner/Spark35SpannerDataWriterTest.java

@@ -14,6 +14,7 @@
 
 package com.google.cloud.spark.spanner;
 
+import org.apache.spark.sql.Encoder;
 import org.apache.spark.sql.Encoders;
 import org.apache.spark.sql.Row;
 import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder;
@@ -22,7 +23,12 @@
 public class Spark35SpannerDataWriterTest extends SpannerDataWriterTestBase {
 
   @Override
-  protected ExpressionEncoder<Row> getEncoder(StructType schema) {
-    return (ExpressionEncoder<Row>) Encoders.row(schema);
+  protected void localSetup() {
+    serializer = ((ExpressionEncoder<Row>) encoder).createSerializer();
+  }
+
+  @Override
+  protected Encoder<Row> getEncoder(StructType schema) {
+    return Encoders.row(schema);
   }
 }

spark-4.0-spanner-lib/pom.xml

@@ -0,0 +1,110 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>com.google.cloud.spark.spanner</groupId>
+    <artifactId>spark-spanner-lib-parent</artifactId>
+    <version>${revision}</version>
+    <relativePath>../spark-spanner-lib-parent</relativePath>
+  </parent>
+
+  <artifactId>spark-4.0-spanner-lib</artifactId>
+  <version>${revision}</version>
+  <name>Connector code for spanner DataSource v2 for Spark 4.0</name>
+  <properties>
+    <spark.version>4.0.1</spark.version>
+    <scala.version>2.13</scala.version>
+    <shade.skip>true</shade.skip>
+    <toolchain.jdk.version>[17,18)</toolchain.jdk.version>
+    <maven.compiler.release>17</maven.compiler.release>
+  </properties>
+  <licenses>
+    <license>
+      <name>Apache License, Version 2.0</name>
+      <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+      <distribution>repo</distribution>
+    </license>
+  </licenses>
+  <dependencies>
+    <dependency>
+      <groupId>com.google.cloud</groupId>
+      <artifactId>google-cloud-dataproc</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.google.cloud</groupId>
+      <artifactId>google-cloud-storage</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>${project.groupId}</groupId>
+      <artifactId>spark-3.1-spanner-lib</artifactId>
+      <version>${project.version}</version>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>${project.groupId}</groupId>
+      <artifactId>spark-3.5-spanner-lib</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+  </dependencies>
+  <profiles>
+    <profile>
+      <id>integration</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-failsafe-plugin</artifactId>
+            <configuration>
+              <forkCount>10</forkCount>
+              <reuseForks>false</reuseForks>
+              <includes>
+                <include>**/*IntegrationTest.java</include>
+              </includes>
+              <argLine>
+                --add-opens=java.base/java.lang=ALL-UNNAMED
+                --add-opens=java.base/java.lang.invoke=ALL-UNNAMED
+                --add-opens=java.base/java.lang.reflect=ALL-UNNAMED
+                --add-opens=java.base/java.io=ALL-UNNAMED
+                --add-opens=java.base/java.net=ALL-UNNAMED
+                --add-opens=java.base/java.nio=ALL-UNNAMED
+                --add-opens=java.base/java.util=ALL-UNNAMED
+                --add-opens=java.base/java.util.concurrent=ALL-UNNAMED
+                --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED
+                --add-opens=java.base/sun.nio.ch=ALL-UNNAMED
+                --add-opens=java.base/sun.nio.cs=ALL-UNNAMED
+                --add-opens=java.base/sun.security.action=ALL-UNNAMED
+                --add-opens=java.base/sun.util.calendar=ALL-UNNAMED
+                --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED
+                @{argLine}
+              </argLine>
+              <jdkToolchain>
+                <version>${toolchain.jdk.version}</version>
+              </jdkToolchain>
+            </configuration>
+            <executions>
+              <execution>
+                <id>integration-test</id>
+                <goals>
+                  <goal>integration-test</goal>
+                </goals>
+              </execution>
+              <execution>
+                <id>verify</id>
+                <goals>
+                  <goal>verify</goal>
+                </goals>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+  </profiles>
+</project>

spark-4.0-spanner-lib/src/main/java/com/google/cloud/spark/spanner/DefaultSource.java

@@ -0,0 +1,17 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.cloud.spark.spanner;
+
+public class DefaultSource extends Spark40SpannerTableProvider {}

spark-4.0-spanner-lib/src/main/java/com/google/cloud/spark/spanner/Spark40SpannerTableProvider.java

@@ -0,0 +1,17 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.cloud.spark.spanner;
+
+public class Spark40SpannerTableProvider extends SparkSpannerTableProviderBase {}