Pom refactor - Add Spark 3.5 connector

README.md

@@ -32,14 +32,20 @@ If you run a Spark job on the Dataproc cluster, you'll have to assign correspond
 You can find the released jar file from the Releases tag on right of the github page. The name pattern is spark-3.1-spanner-x.x.x.jar. The 3.1 indicates the driver depends on the Spark 3.1 and x.x.x is the Spark Spanner connector version. The alternative way is to use `gs://spark-lib/spanner/spark-3.1-spanner-1.1.1.jar` directly.
 
 ### Connector to Spark Compatibility Matrix
-| Connector \ Spark                     | 2.3     | 2.4<br>(Scala 2.11) | 2.4<br>(Scala 2.12) | 3.0     | 3.1     | 3.2     | 3.3     | 3.4     | 3.5     |
-|---------------------------------------|---------|---------------------|---------------------|---------|---------|---------|---------|---------|---------|
-| spark-3.1-spanner                     |         |                     |                     |         | &check; | &check; | &check; | &check; | &check; |
+| Connector \ Spark | 2.3     | 2.4<br>(Scala 2.11) | 2.4<br>(Scala 2.12) | 3.0     | 3.1     | 3.2     | 3.3     | 3.4     | 3.5     |
+|-------------------|---------|---------------------|---------------------|---------|---------|---------|---------|---------|---------|
+| spark-3.1-spanner |         |                     |                     |         | &check; | &check; | &check; | &check; | &check; |
+| spark-3.2-spanner |         |                     |                     |         |         | &check; | &check; | &check; | &check; |
+| spark-3.3-spanner |         |                     |                     |         |         |         | &check; | &check; | &check; |
+| spark-3.5-spanner |         |                     |                     |         |         |         |         |         | &check; |
 
 ### Connector to Dataproc Image Compatibility Matrix
-| Connector \ Dataproc Image            | 1.3     | 1.4     | 1.5     | 2.0     | 2.1     | 2.2     | Serverless<br>Image 1.0 | Serverless<br>Image 2.0 | Serverless<br>Image 2.1 |
-|---------------------------------------|---------|---------|---------|---------|---------|---------|-------------------------|-------------------------|-------------------------|
-| spark-3.1-spanner                     |         |         |         | &check; | &check; | &check; | &check;                 | &check;                 | &check;                 |
+| Connector \ Dataproc Image | 1.3     | 1.4     | 1.5     | 2.0     | 2.1     | 2.2     | Serverless<br>Image 1.1 | Serverless<br>Image 1.2 | Serverless<br>Image 2.0 | Serverless<br>Image 2.2 |
+|----------------------------|---------|---------|---------|---------|---------|---------|-------------------------|-------------------------|-------------------------|-------------------------|
+| spark-3.1-spanner          |         |         |         | &check; | &check; | &check; | &check;                 |                         | &check;                 |                         |
+| spark-3.2-spanner          |         |         |         | &check; | &check; | &check; | &check;                 |                         | &check;                 |                         |
+| spark-3.3-spanner          |         |         |         | &check; | &check; | &check; | &check;                 |                         | &check;                 |                         |
+| spark-3.5-spanner          |         |         |         |         |         | &check; |                         | &check;                 |                         | &check;                 |
 
 ### Maven / Ivy Package
 

pom.xml

@@ -1,17 +1,13 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0"
-    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
-  <parent>
-    <groupId>com.google.cloud.spark.spanner</groupId>
-    <artifactId>spark-spanner-parent</artifactId>
-    <version>${revision}</version>
-    <relativePath>spark-spanner-parent</relativePath>
-  </parent>
-
+  <groupId>com.google.cloud.spark.spanner</groupId>
   <artifactId>spark-spanner-reactor</artifactId>
+  <version>${revision}</version>
   <packaging>pom</packaging>
   <name>Spark Spanner Connector Reactor</name>
+  <url>https://github.com/GoogleCloudDataproc/spark-spanner-connector</url>
 
   <licenses>
     <license>
@@ -44,39 +40,154 @@
       https://github.com/GoogleCloudDataproc/spark-spanner-connector/issues
     </url>
   </issueManagement>
+  <properties>
+    <scala.version>2.12</scala.version>
+  </properties>
 
   <modules>
     <module>spark-spanner-parent</module>
     <module>spark-spanner-lib-parent</module>
-    <module>spark-3.1-spanner-lib</module>
+    <module>spanner-core</module>
   </modules>
 
   <profiles>
     <profile>
       <id>3.1</id>
-      <activation><activeByDefault>false</activeByDefault></activation>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <properties>
+        <spark.version>3.1.1</spark.version>
+        <shade.skip>false</shade.skip>
+      </properties>
+      <dependencyManagement>
+        <dependencies>
+          <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-sql_${scala.version}</artifactId>
+            <version>${spark.version}</version>
+            <scope>provided</scope>
+          </dependency>
+        </dependencies>
+      </dependencyManagement>
       <modules>
+        <module>spanner-spark-common</module>
         <module>spark-3.1-spanner</module>
+        <module>spark-3.1-spanner-lib</module>
       </modules>
     </profile>
     <profile>
       <id>3.2</id>
-      <activation><activeByDefault>false</activeByDefault></activation>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <properties>
+        <spark.version>3.2.0</spark.version>
+        <shade.skip>false</shade.skip>
+      </properties>
+      <dependencyManagement>
+        <dependencies>
+          <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-sql_${scala.version}</artifactId>
+            <version>${spark.version}</version>
+            <scope>provided</scope>
+          </dependency>
+        </dependencies>
+      </dependencyManagement>
       <modules>
+        <module>spanner-spark-common</module>
         <module>spark-3.2-spanner-lib</module>
         <module>spark-3.2-spanner</module>
       </modules>
     </profile>
     <profile>
       <id>3.3</id>
-      <activation><activeByDefault>false</activeByDefault></activation>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <properties>
+        <spark.version>3.3.4</spark.version>
+        <shade.skip>false</shade.skip>
+      </properties>
+      <dependencyManagement>
+        <dependencies>
+          <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-sql_${scala.version}</artifactId>
+            <version>${spark.version}</version>
+            <scope>provided</scope>
+          </dependency>
+        </dependencies>
+      </dependencyManagement>
       <modules>
-        <module>spark-3.2-spanner-lib</module>
-        <module>spark-3.2-spanner</module>
+        <module>spanner-spark-common</module>
         <module>spark-3.3-spanner-lib</module>
         <module>spark-3.3-spanner</module>
       </modules>
     </profile>
+    <profile>
+      <id>3.5</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <properties>
+        <spark.version>3.5.7</spark.version>
+        <shade.skip>false</shade.skip>
+      </properties>
+      <dependencyManagement>
+        <dependencies>
+          <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-sql_${scala.version}</artifactId>
+            <version>${spark.version}</version>
+            <scope>provided</scope>
+          </dependency>
+        </dependencies>
+      </dependencyManagement>
+      <modules>
+        <module>spanner-spark-common</module>
+        <module>spark-3.5-spanner-lib</module>
+        <module>spark-3.5-spanner</module>
+      </modules>
+    </profile>
   </profiles>
-
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>com.diffplug.spotless</groupId>
+        <artifactId>spotless-maven-plugin</artifactId>
+        <version>2.30.0</version>
+        <configuration>
+          <formats>
+            <format>
+              <includes>
+                <include>*.md</include>
+                <include>.gitignore</include>
+              </includes>
+              <trimTrailingWhitespace/>
+              <endWithNewline/>
+            </format>
+          </formats>
+          <java>
+            <toggleOffOn>
+              <off>fmt:off</off>
+              <on>fmt:on</on>
+            </toggleOffOn>
+            <googleJavaFormat>
+              <version>1.7</version>
+            </googleJavaFormat>
+          </java>
+        </configuration>
+        <executions>
+          <execution>
+            <goals>
+              <goal>check</goal>
+            </goals>
+            <phase>compile</phase>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
 </project>

spanner-core/pom.xml

@@ -0,0 +1,63 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+      <groupId>com.google.cloud.spark.spanner</groupId>
+      <artifactId>spark-spanner-lib-parent</artifactId>
+      <version>${revision}</version>
+      <relativePath>../spark-spanner-lib-parent</relativePath>
+  </parent>
+
+  <artifactId>spanner-core</artifactId>
+  <name>Spanner core library</name>
+  <description>This module contains Spanner code and no Spark dependencies.</description>
+
+  <dependencies>
+        <dependency>
+            <groupId>com.google.cloud</groupId>
+            <artifactId>google-cloud-spanner</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.google.cloud</groupId>
+            <artifactId>google-cloud-dataproc</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>com.google.cloud</groupId>
+            <artifactId>google-cloud-storage</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>com.google.code.gson</groupId>
+            <artifactId>gson</artifactId>
+            <version>2.10.1</version>
+        </dependency>
+
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+        </dependency>
+    </dependencies>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jar-plugin</artifactId>
+                <version>3.3.0</version>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>test-jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

spark-3.1-spanner-lib/src/test/java/com/google/cloud/spark/spanner/acceptance/DataprocAcceptanceTestBase.java → spanner-core/src/test/java/com/google/cloud/spark/spanner/acceptance/DataprocAcceptanceTestBase.java

@@ -47,7 +47,6 @@
 
 public class DataprocAcceptanceTestBase {
 
-  public static final String CONNECTOR_JAR_DIRECTORY = "../spark-3.1-spanner/target";
   public static final String REGION = "us-central1";
   public static final String DATAPROC_ENDPOINT = REGION + "-dataproc.googleapis.com:443";
   public static final String PROJECT_ID =
@@ -164,6 +163,7 @@ Job waitForJobCompletion(
 
   protected static AcceptanceTestContext setup(
       String dataprocImageVersion,
+      String connectorJarDirectory,
       String connectorJarPrefix,
       List<ClusterProperty> clusterProperties)
       throws Exception {
@@ -186,7 +186,7 @@ protected static AcceptanceTestContext setup(
     String testBaseGcsDir = AcceptanceTestUtils.createTestBaseGcsDir(testId);
     String connectorJarUri = testBaseGcsDir + "/connector.jar";
     AcceptanceTestUtils.uploadConnectorJar(
-        CONNECTOR_JAR_DIRECTORY, connectorJarPrefix, connectorJarUri);
+        connectorJarDirectory, connectorJarPrefix, connectorJarUri);
 
     String clusterName =
         createClusterIfNeeded(dataprocImageVersion, testId, properties, connectorJarUri);
@@ -282,6 +282,9 @@ protected static String createClusterIfNeeded(
       String connectorJarUri)
       throws Exception {
     String clusterName = generateClusterName(testId);
+    System.out.printf(
+        "createClusterIfNeeded clusterName: %s, testId: %s, dataprocImageVersion: %s, connectorJarUri: %s, PROJECT_ID: %s, REGION: %s%n",
+        clusterName, testId, dataprocImageVersion, connectorJarUri, PROJECT_ID, REGION);
     cluster(
         client ->
             client

spark-3.1-spanner-lib/src/test/java/com/google/cloud/spark/spanner/acceptance/DataprocServerlessAcceptanceTestBase.java → spanner-core/src/test/java/com/google/cloud/spark/spanner/acceptance/DataprocServerlessAcceptanceTestBase.java

@@ -55,7 +55,6 @@
  * requireOsLogin disabled, otherwise an org policy violation error will be thrown.
  */
 public class DataprocServerlessAcceptanceTestBase {
-  public static final String CONNECTOR_JAR_DIRECTORY = "../spark-3.1-spanner/target";
   public static final String REGION = "us-central1";
   public static final String DATAPROC_ENDPOINT = REGION + "-dataproc.googleapis.com:443";
   public static final String PROJECT_ID =
@@ -96,17 +95,20 @@ public class DataprocServerlessAcceptanceTestBase {
           testId, generateClusterName(testId), testBaseGcsDir, connectorJarUri);
 
   private final String s8sImageVersion;
+  private final String connectorJarDirectory;
   private final String connectorJarPrefix;
 
-  public DataprocServerlessAcceptanceTestBase(String connectorJarPrefix, String s8sImageVersion) {
+  public DataprocServerlessAcceptanceTestBase(
+      String connectorJarDirectory, String connectorJarPrefix, String s8sImageVersion) {
+    this.connectorJarDirectory = connectorJarDirectory;
     this.connectorJarPrefix = connectorJarPrefix;
     this.s8sImageVersion = s8sImageVersion;
   }
 
   @Before
   public void createBatchControllerClient() throws Exception {
     AcceptanceTestUtils.uploadConnectorJar(
-        CONNECTOR_JAR_DIRECTORY, connectorJarPrefix, context.connectorJarUri);
+        connectorJarDirectory, connectorJarPrefix, context.connectorJarUri);
     createSpannerDataset();
 
     batchController =

spanner-core/src/test/resources/spark-spanner-connector.properties

@@ -0,0 +1 @@
+connector.version=test