GoogleCloudDataproc · stevelordbq · Dec 18, 2025 · Dec 24, 2025 · Dec 24, 2025 · Jan 5, 2026
diff --git a/.github/workflows/dev-ci.yml b/.github/workflows/dev-ci.yml
@@ -32,6 +32,9 @@ jobs:
       - name: Build for Spark 3.3
         run: |
           mvn -B -e install -P3.3 -DskipTests
+      - name: Build for Spark 3.5
+        run: |
+          mvn -B -e install -P3.5 -DskipTests
       - name: Get short commit hash
         id: vars
         run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT

diff --git a/README-template.md b/README-template.md
@@ -38,14 +38,20 @@ If you run a Spark job on the Dataproc cluster, you'll have to assign correspond
 You can find the released jar file from the Releases tag on right of the github page. The name pattern is spark-3.1-spanner-x.x.x.jar. The 3.1 indicates the driver depends on the Spark 3.1 and x.x.x is the Spark Spanner connector version. The alternative way is to use `gs://spark-lib/spanner/spark-3.1-spanner-${next-release-tag}.jar` directly.
 
 ### Connector to Spark Compatibility Matrix
-| Connector \ Spark                     | 2.3     | 2.4<br>(Scala 2.11) | 2.4<br>(Scala 2.12) | 3.0     | 3.1     | 3.2     | 3.3     | 3.4     | 3.5     |
-|---------------------------------------|---------|---------------------|---------------------|---------|---------|---------|---------|---------|---------|
-| spark-3.1-spanner                     |         |                     |                     |         | &check; | &check; | &check; | &check; | &check; |
+| Connector \ Spark | 2.3     | 2.4<br>(Scala 2.11) | 2.4<br>(Scala 2.12) | 3.0     | 3.1     | 3.2     | 3.3     | 3.4     | 3.5     |
+|-------------------|---------|---------------------|---------------------|---------|---------|---------|---------|---------|---------|
+| spark-3.1-spanner |         |                     |                     |         | &check; | &check; | &check; | &check; | &check; |
+| spark-3.2-spanner |         |                     |                     |         |         | &check; | &check; | &check; | &check; |
+| spark-3.3-spanner |         |                     |                     |         |         |         | &check; | &check; | &check; |
+| spark-3.5-spanner |         |                     |                     |         |         |         |         |         | &check; |
 
 ### Connector to Dataproc Image Compatibility Matrix
-| Connector \ Dataproc Image            | 1.3     | 1.4     | 1.5     | 2.0     | 2.1     | 2.2     | Serverless<br>Image 1.0 | Serverless<br>Image 2.0 | Serverless<br>Image 2.1 |
-|---------------------------------------|---------|---------|---------|---------|---------|---------|-------------------------|-------------------------|-------------------------|
-| spark-3.1-spanner                     |         |         |         | &check; | &check; | &check; | &check;                 | &check;                 | &check;                 |
+| Connector \ Dataproc Image | 1.3     | 1.4     | 1.5     | 2.0     | 2.1     | 2.2     | Serverless<br>Image 1.1 | Serverless<br>Image 1.2 | Serverless<br>Image 2.0 | Serverless<br>Image 2.1 | Serverless<br>Image 2.2 |
+|----------------------------|---------|---------|---------|---------|---------|---------|-------------------------|-------------------------|-------------------------|-------------------------|-------------------------|
+| spark-3.1-spanner          |         |         |         | &check; | &check; | &check; | &check;                 |                         | &check;                 | &check;                 |                         |
+| spark-3.2-spanner          |         |         |         | &check; | &check; | &check; | &check;                 |                         | &check;                 | &check;                 |                         |
+| spark-3.3-spanner          |         |         |         | &check; | &check; | &check; | &check;                 |                         | &check;                 | &check;                 |                         |
+| spark-3.5-spanner          |         |         |         |         |         | &check; |                         | &check;                 |                         |                         | &check;                 |
 
 ### Maven / Ivy Package
 

diff --git a/README.md b/README.md
@@ -32,14 +32,20 @@ If you run a Spark job on the Dataproc cluster, you'll have to assign correspond
 You can find the released jar file from the Releases tag on right of the github page. The name pattern is spark-3.1-spanner-x.x.x.jar. The 3.1 indicates the driver depends on the Spark 3.1 and x.x.x is the Spark Spanner connector version. The alternative way is to use `gs://spark-lib/spanner/spark-3.1-spanner-1.1.1.jar` directly.
 
 ### Connector to Spark Compatibility Matrix
-| Connector \ Spark                     | 2.3     | 2.4<br>(Scala 2.11) | 2.4<br>(Scala 2.12) | 3.0     | 3.1     | 3.2     | 3.3     | 3.4     | 3.5     |
-|---------------------------------------|---------|---------------------|---------------------|---------|---------|---------|---------|---------|---------|
-| spark-3.1-spanner                     |         |                     |                     |         | &check; | &check; | &check; | &check; | &check; |
+| Connector \ Spark | 2.3     | 2.4<br>(Scala 2.11) | 2.4<br>(Scala 2.12) | 3.0     | 3.1     | 3.2     | 3.3     | 3.4     | 3.5     |
+|-------------------|---------|---------------------|---------------------|---------|---------|---------|---------|---------|---------|
+| spark-3.1-spanner |         |                     |                     |         | &check; | &check; | &check; | &check; | &check; |
+| spark-3.2-spanner |         |                     |                     |         |         | &check; | &check; | &check; | &check; |
+| spark-3.3-spanner |         |                     |                     |         |         |         | &check; | &check; | &check; |
+| spark-3.5-spanner |         |                     |                     |         |         |         |         |         | &check; |
 
 ### Connector to Dataproc Image Compatibility Matrix
-| Connector \ Dataproc Image            | 1.3     | 1.4     | 1.5     | 2.0     | 2.1     | 2.2     | Serverless<br>Image 1.0 | Serverless<br>Image 2.0 | Serverless<br>Image 2.1 |
-|---------------------------------------|---------|---------|---------|---------|---------|---------|-------------------------|-------------------------|-------------------------|
-| spark-3.1-spanner                     |         |         |         | &check; | &check; | &check; | &check;                 | &check;                 | &check;                 |
+| Connector \ Dataproc Image | 1.3     | 1.4     | 1.5     | 2.0     | 2.1     | 2.2     | Serverless<br>Image 1.1 | Serverless<br>Image 1.2 | Serverless<br>Image 2.0 | Serverless<br>Image 2.2 |
+|----------------------------|---------|---------|---------|---------|---------|---------|-------------------------|-------------------------|-------------------------|-------------------------|
+| spark-3.1-spanner          |         |         |         | &check; | &check; | &check; | &check;                 |                         | &check;                 |                         |
+| spark-3.2-spanner          |         |         |         | &check; | &check; | &check; | &check;                 |                         | &check;                 |                         |
+| spark-3.3-spanner          |         |         |         | &check; | &check; | &check; | &check;                 |                         | &check;                 |                         |
+| spark-3.5-spanner          |         |         |         |         |         | &check; |                         | &check;                 |                         | &check;                 |
 
 ### Maven / Ivy Package
 

diff --git a/pom.xml b/pom.xml
@@ -1,17 +1,13 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0"
-    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
-  <parent>
-    <groupId>com.google.cloud.spark.spanner</groupId>
-    <artifactId>spark-spanner-parent</artifactId>
-    <version>${revision}</version>
-    <relativePath>spark-spanner-parent</relativePath>
-  </parent>
-
+  <groupId>com.google.cloud.spark.spanner</groupId>
   <artifactId>spark-spanner-reactor</artifactId>
+  <version>${revision}</version>
   <packaging>pom</packaging>
   <name>Spark Spanner Connector Reactor</name>
+  <url>https://github.com/GoogleCloudDataproc/spark-spanner-connector</url>
 
   <licenses>
     <license>
@@ -44,39 +40,158 @@
       https://github.com/GoogleCloudDataproc/spark-spanner-connector/issues
     </url>
   </issueManagement>
+  <properties>
+    <scala.version>2.12</scala.version>
+    <revision>0.0.1-SNAPSHOT</revision>
+  </properties>
 
   <modules>
     <module>spark-spanner-parent</module>
     <module>spark-spanner-lib-parent</module>
-    <module>spark-3.1-spanner-lib</module>
+    <module>spanner-core</module>
   </modules>
 
   <profiles>
     <profile>
       <id>3.1</id>
-      <activation><activeByDefault>false</activeByDefault></activation>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <properties>
+        <spark.version>3.1.1</spark.version>
+        <shade.skip>false</shade.skip>
+      </properties>
+      <dependencyManagement>
+        <dependencies>
+          <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-sql_${scala.version}</artifactId>
+            <version>${spark.version}</version>
+            <scope>provided</scope>
+          </dependency>
+        </dependencies>
+      </dependencyManagement>
       <modules>
+        <module>spanner-spark-common</module>
         <module>spark-3.1-spanner</module>
+        <module>spark-3.1-spanner-lib</module>
       </modules>
     </profile>
     <profile>
       <id>3.2</id>
-      <activation><activeByDefault>false</activeByDefault></activation>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <properties>
+        <spark.version>3.2.0</spark.version>
+        <shade.skip>false</shade.skip>
+      </properties>
+      <dependencyManagement>
+        <dependencies>
+          <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-sql_${scala.version}</artifactId>
+            <version>${spark.version}</version>
+            <scope>provided</scope>
+          </dependency>
+        </dependencies>
+      </dependencyManagement>
       <modules>
+        <module>spanner-spark-common</module>
         <module>spark-3.2-spanner-lib</module>
         <module>spark-3.2-spanner</module>
       </modules>
     </profile>
     <profile>
       <id>3.3</id>
-      <activation><activeByDefault>false</activeByDefault></activation>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <properties>
+        <spark.version>3.3.4</spark.version>
+        <shade.skip>false</shade.skip>
+      </properties>
+      <dependencyManagement>
+        <dependencies>
+          <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-sql_${scala.version}</artifactId>
+            <version>${spark.version}</version>
+            <scope>provided</scope>
+          </dependency>
+        </dependencies>
+      </dependencyManagement>
       <modules>
-        <module>spark-3.2-spanner-lib</module>
-        <module>spark-3.2-spanner</module>
+        <module>spanner-spark-common</module>
         <module>spark-3.3-spanner-lib</module>
         <module>spark-3.3-spanner</module>
       </modules>
     </profile>
+    <profile>
+      <id>3.5</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <properties>
+        <spark.version>3.5.7</spark.version>
+        <shade.skip>false</shade.skip>
+      </properties>
+      <dependencyManagement>
+        <dependencies>
+          <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-sql_${scala.version}</artifactId>
+            <version>${spark.version}</version>
+            <scope>provided</scope>
+          </dependency>
+        </dependencies>
+      </dependencyManagement>
+      <modules>
+        <module>spanner-spark-common</module>
+        <module>spark-3.5-spanner-lib</module>
+        <module>spark-3.5-spanner</module>
+      </modules>
+    </profile>
   </profiles>
-
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>com.diffplug.spotless</groupId>
+        <artifactId>spotless-maven-plugin</artifactId>
+        <version>2.30.0</version>
+        <configuration>
+          <formats>
+            <format>
+              <includes>
+                <include>*.md</include>
+                <include>.gitignore</include>
+                <include>pom.xml</include>
+                <include>*/pom.xml</include>
+                <include>*/*/pom.xml</include>
+              </includes>
+              <trimTrailingWhitespace/>
+              <endWithNewline/>
+            </format>
+          </formats>
+          <java>
+            <toggleOffOn>
+              <off>fmt:off</off>
+              <on>fmt:on</on>
+            </toggleOffOn>
+            <googleJavaFormat>
+              <version>1.7</version>
+            </googleJavaFormat>
+          </java>
+        </configuration>
+        <executions>
+          <execution>
+            <goals>
+              <goal>check</goal>
+            </goals>
+            <phase>compile</phase>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
 </project>
diff --git a/spanner-core/pom.xml b/spanner-core/pom.xml
@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+      <groupId>com.google.cloud.spark.spanner</groupId>
+      <artifactId>spark-spanner-lib-parent</artifactId>
+      <version>${revision}</version>
+      <relativePath>../spark-spanner-lib-parent</relativePath>
+  </parent>
+
+  <artifactId>spanner-core</artifactId>
+  <name>Spanner core library</name>
+  <description>This module contains Spanner code and no Spark dependencies.</description>
+
+  <dependencies>
+        <dependency>
+            <groupId>com.google.cloud</groupId>
+            <artifactId>google-cloud-spanner</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.google.cloud</groupId>
+            <artifactId>google-cloud-dataproc</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>com.google.cloud</groupId>
+            <artifactId>google-cloud-storage</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>com.google.code.gson</groupId>
+            <artifactId>gson</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+        </dependency>
+    </dependencies>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jar-plugin</artifactId>
+                <version>3.3.0</version>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>test-jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>
diff --git a/.../spark/spanner/BatchClientWithCloser.java → .../spark/spanner/BatchClientWithCloser.java b/.../spark/spanner/BatchClientWithCloser.java → .../spark/spanner/BatchClientWithCloser.java
diff --git a/.../spark/spanner/InputPartitionContext.java → .../spark/spanner/InputPartitionContext.java b/.../spark/spanner/InputPartitionContext.java → .../spark/spanner/InputPartitionContext.java
diff --git a/.../spanner/InputPartitionReaderContext.java → .../spanner/InputPartitionReaderContext.java b/.../spanner/InputPartitionReaderContext.java → .../spanner/InputPartitionReaderContext.java
diff --git a/...rk/spanner/SpannerConnectorException.java → ...rk/spanner/SpannerConnectorException.java b/...rk/spanner/SpannerConnectorException.java → ...rk/spanner/SpannerConnectorException.java
diff --git a/...cloud/spark/spanner/SpannerErrorCode.java → ...cloud/spark/spanner/SpannerErrorCode.java b/...cloud/spark/spanner/SpannerErrorCode.java → ...cloud/spark/spanner/SpannerErrorCode.java
diff --git a/...park/spanner/graph/query/SelectField.java → ...park/spanner/graph/query/SelectField.java b/...park/spanner/graph/query/SelectField.java → ...park/spanner/graph/query/SelectField.java
diff --git a/.../google/cloud/spark/spanner/TestData.java → .../google/cloud/spark/spanner/TestData.java b/.../google/cloud/spark/spanner/TestData.java → .../google/cloud/spark/spanner/TestData.java
diff --git a/...ner/acceptance/AcceptanceTestContext.java → ...ner/acceptance/AcceptanceTestContext.java b/...ner/acceptance/AcceptanceTestContext.java → ...ner/acceptance/AcceptanceTestContext.java
diff --git a/...anner/acceptance/AcceptanceTestUtils.java → ...anner/acceptance/AcceptanceTestUtils.java b/...anner/acceptance/AcceptanceTestUtils.java → ...anner/acceptance/AcceptanceTestUtils.java
diff --git a/...cceptance/DataprocAcceptanceTestBase.java → ...cceptance/DataprocAcceptanceTestBase.java b/...cceptance/DataprocAcceptanceTestBase.java → ...cceptance/DataprocAcceptanceTestBase.java
@@ -47,7 +47,6 @@
 
 public class DataprocAcceptanceTestBase {
 
-  public static final String CONNECTOR_JAR_DIRECTORY = "../spark-3.1-spanner/target";
   public static final String REGION = "us-central1";
   public static final String DATAPROC_ENDPOINT = REGION + "-dataproc.googleapis.com:443";
   public static final String PROJECT_ID =
@@ -164,6 +163,7 @@ Job waitForJobCompletion(
 
   protected static AcceptanceTestContext setup(
       String dataprocImageVersion,
+      String connectorJarDirectory,
       String connectorJarPrefix,
       List<ClusterProperty> clusterProperties)
       throws Exception {
@@ -186,7 +186,7 @@ protected static AcceptanceTestContext setup(
     String testBaseGcsDir = AcceptanceTestUtils.createTestBaseGcsDir(testId);
     String connectorJarUri = testBaseGcsDir + "/connector.jar";
     AcceptanceTestUtils.uploadConnectorJar(
-        CONNECTOR_JAR_DIRECTORY, connectorJarPrefix, connectorJarUri);
+        connectorJarDirectory, connectorJarPrefix, connectorJarUri);
 
     String clusterName =
         createClusterIfNeeded(dataprocImageVersion, testId, properties, connectorJarUri);
@@ -282,6 +282,9 @@ protected static String createClusterIfNeeded(
       String connectorJarUri)
       throws Exception {
     String clusterName = generateClusterName(testId);
+    logger.info(
+        "createClusterIfNeeded clusterName: %s, testId: %s, dataprocImageVersion: %s, connectorJarUri: %s, PROJECT_ID: %s, REGION: %s%n",
+        clusterName, testId, dataprocImageVersion, connectorJarUri, PROJECT_ID, REGION);
     cluster(
         client ->
             client

diff --git a/...DataprocServerlessAcceptanceTestBase.java → ...DataprocServerlessAcceptanceTestBase.java b/...DataprocServerlessAcceptanceTestBase.java → ...DataprocServerlessAcceptanceTestBase.java
@@ -55,7 +55,6 @@
  * requireOsLogin disabled, otherwise an org policy violation error will be thrown.
  */
 public class DataprocServerlessAcceptanceTestBase {
-  public static final String CONNECTOR_JAR_DIRECTORY = "../spark-3.1-spanner/target";
   public static final String REGION = "us-central1";
   public static final String DATAPROC_ENDPOINT = REGION + "-dataproc.googleapis.com:443";
   public static final String PROJECT_ID =
@@ -96,17 +95,20 @@ public class DataprocServerlessAcceptanceTestBase {
           testId, generateClusterName(testId), testBaseGcsDir, connectorJarUri);
 
   private final String s8sImageVersion;
+  private final String connectorJarDirectory;
   private final String connectorJarPrefix;
 
-  public DataprocServerlessAcceptanceTestBase(String connectorJarPrefix, String s8sImageVersion) {
+  public DataprocServerlessAcceptanceTestBase(
+      String connectorJarDirectory, String connectorJarPrefix, String s8sImageVersion) {
+    this.connectorJarDirectory = connectorJarDirectory;
     this.connectorJarPrefix = connectorJarPrefix;
     this.s8sImageVersion = s8sImageVersion;
   }
 
   @Before
   public void createBatchControllerClient() throws Exception {
     AcceptanceTestUtils.uploadConnectorJar(
-        CONNECTOR_JAR_DIRECTORY, connectorJarPrefix, context.connectorJarUri);
+        connectorJarDirectory, connectorJarPrefix, context.connectorJarUri);
     createSpannerDataset();
 
     batchController =

diff --git a/...t/resources/acceptance/read_test_table.py → ...t/resources/acceptance/read_test_table.py b/...t/resources/acceptance/read_test_table.py → ...t/resources/acceptance/read_test_table.py
diff --git a/...lib/src/test/resources/db/insert_data.sql → ...ore/src/test/resources/db/insert_data.sql b/...lib/src/test/resources/db/insert_data.sql → ...ore/src/test/resources/db/insert_data.sql
diff --git a/...c/test/resources/db/insert_data_graph.sql → ...c/test/resources/db/insert_data_graph.sql b/...c/test/resources/db/insert_data_graph.sql → ...c/test/resources/db/insert_data_graph.sql
diff --git a/.../src/test/resources/db/insert_data_pg.sql → .../src/test/resources/db/insert_data_pg.sql b/.../src/test/resources/db/insert_data_pg.sql → .../src/test/resources/db/insert_data_pg.sql
diff --git a/...ib/src/test/resources/db/populate_ddl.sql → ...re/src/test/resources/db/populate_ddl.sql b/...ib/src/test/resources/db/populate_ddl.sql → ...re/src/test/resources/db/populate_ddl.sql
diff --git a/.../test/resources/db/populate_ddl_graph.sql → .../test/resources/db/populate_ddl_graph.sql b/.../test/resources/db/populate_ddl_graph.sql → .../test/resources/db/populate_ddl_graph.sql
diff --git a/...src/test/resources/db/populate_ddl_pg.sql → ...src/test/resources/db/populate_ddl_pg.sql b/...src/test/resources/db/populate_ddl_pg.sql → ...src/test/resources/db/populate_ddl_pg.sql
diff --git a/.../src/test/resources/db/shakespeare_bq.csv → .../src/test/resources/db/shakespeare_bq.csv b/.../src/test/resources/db/shakespeare_bq.csv → .../src/test/resources/db/shakespeare_bq.csv
diff --git a/spanner-core/src/test/resources/spark-spanner-connector.properties b/spanner-core/src/test/resources/spark-spanner-connector.properties
@@ -0,0 +1 @@
+connector.version=test