From b49bb97630e4100132307047cfba52ded981b850 Mon Sep 17 00:00:00 2001 From: stevelordbq Date: Tue, 14 Apr 2026 14:43:05 -0700 Subject: [PATCH 01/11] Spark 4.0 initial version --- pom.xml | 11 ++++ spark-4.0-spanner-lib/pom.xml | 63 +++++++++++++++++++ .../cloud/spark/spanner/DefaultSource.java | 17 +++++ .../spanner/Spark40SpannerTableProvider.java | 17 +++++ .../spanner/Spark40SpannerDataWriterTest.java | 28 +++++++++ ...taprocServerlessImage30AcceptanceTest.java | 45 +++++++++++++ .../Spark40DataprocImage30AcceptanceTest.java | 44 +++++++++++++ ...rk40GraphErrorHandlingIntegrationTest.java | 18 ++++++ .../Spark40GraphReadIntegrationTest.java | 17 +++++ ...unctionsAndExpressionsIntegrationTest.java | 22 +++++++ .../Spark40ReadPgIntegrationTest.java | 21 +++++++ ...park40SchemaValidationIntegrationTest.java | 63 +++++++++++++++++++ ...PartitionReaderContextIntegrationTest.java | 18 ++++++ ...rk40SpannerScanBuilderIntegrationTest.java | 18 ++++++ .../Spark40SpannerTableIntegrationTest.java | 17 +++++ ...40SpannerTableProviderIntegrationTest.java | 25 ++++++++ .../Spark40WriteIntegrationTest.java | 21 +++++++ ...pache.spark.sql.sources.DataSourceRegister | 1 + spark-4.0-spanner/pom.xml | 50 +++++++++++++++ ...pache.spark.sql.sources.DataSourceRegister | 1 + 20 files changed, 517 insertions(+) create mode 100644 spark-4.0-spanner-lib/pom.xml create mode 100644 spark-4.0-spanner-lib/src/main/java/com/google/cloud/spark/spanner/DefaultSource.java create mode 100644 spark-4.0-spanner-lib/src/main/java/com/google/cloud/spark/spanner/Spark40SpannerTableProvider.java create mode 100644 spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/Spark40SpannerDataWriterTest.java create mode 100644 spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/acceptance/DataprocServerlessImage30AcceptanceTest.java create mode 100644 spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/acceptance/Spark40DataprocImage30AcceptanceTest.java create mode 100644 spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/graph/Spark40GraphErrorHandlingIntegrationTest.java create mode 100644 spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/graph/Spark40GraphReadIntegrationTest.java create mode 100644 spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/Spark40FunctionsAndExpressionsIntegrationTest.java create mode 100644 spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/Spark40ReadPgIntegrationTest.java create mode 100644 spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/Spark40SchemaValidationIntegrationTest.java create mode 100644 spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/Spark40SpannerInputPartitionReaderContextIntegrationTest.java create mode 100644 spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/Spark40SpannerScanBuilderIntegrationTest.java create mode 100644 spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/Spark40SpannerTableIntegrationTest.java create mode 100644 spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/Spark40SpannerTableProviderIntegrationTest.java create mode 100644 spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/Spark40WriteIntegrationTest.java create mode 100644 spark-4.0-spanner-lib/src/test/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister create mode 100644 spark-4.0-spanner/pom.xml create mode 100644 spark-4.0-spanner/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister diff --git a/pom.xml b/pom.xml index a327c96c..c5102df6 100644 --- a/pom.xml +++ b/pom.xml @@ -86,6 +86,17 @@ spark-3.5-spanner + + 4.0 + false + + spark-3.2-spanner-lib + spark-3.3-spanner-lib + spark-3.5-spanner-lib + spark-4.0-spanner-lib + spark-4.0-spanner + + diff --git a/spark-4.0-spanner-lib/pom.xml b/spark-4.0-spanner-lib/pom.xml new file mode 100644 index 00000000..2e66a5be --- /dev/null +++ b/spark-4.0-spanner-lib/pom.xml @@ -0,0 +1,63 @@ + + 4.0.0 + + com.google.cloud.spark.spanner + spark-spanner-lib-parent + ${revision} + ../spark-spanner-lib-parent + + + spark-4.0-spanner-lib + ${revision} + Connector code for spanner DataSource v2 for Spark 4.0 + + 4.0.1 + 2.13 + true + + + + Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + com.google.cloud + google-cloud-dataproc + test + + + com.google.cloud + google-cloud-storage + test + + + ${project.groupId} + spark-3.1-spanner-lib + ${project.version} + test-jar + test + + + ${project.groupId} + spark-3.5-spanner-lib + ${project.version} + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.11.0 + + 17 + + + + + diff --git a/spark-4.0-spanner-lib/src/main/java/com/google/cloud/spark/spanner/DefaultSource.java b/spark-4.0-spanner-lib/src/main/java/com/google/cloud/spark/spanner/DefaultSource.java new file mode 100644 index 00000000..3bb4c60e --- /dev/null +++ b/spark-4.0-spanner-lib/src/main/java/com/google/cloud/spark/spanner/DefaultSource.java @@ -0,0 +1,17 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.cloud.spark.spanner; + +public class DefaultSource extends Spark40SpannerTableProvider {} diff --git a/spark-4.0-spanner-lib/src/main/java/com/google/cloud/spark/spanner/Spark40SpannerTableProvider.java b/spark-4.0-spanner-lib/src/main/java/com/google/cloud/spark/spanner/Spark40SpannerTableProvider.java new file mode 100644 index 00000000..91acd246 --- /dev/null +++ b/spark-4.0-spanner-lib/src/main/java/com/google/cloud/spark/spanner/Spark40SpannerTableProvider.java @@ -0,0 +1,17 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.cloud.spark.spanner; + +public class Spark40SpannerTableProvider extends SparkSpannerTableProviderBase {} diff --git a/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/Spark40SpannerDataWriterTest.java b/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/Spark40SpannerDataWriterTest.java new file mode 100644 index 00000000..17b6fff2 --- /dev/null +++ b/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/Spark40SpannerDataWriterTest.java @@ -0,0 +1,28 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.cloud.spark.spanner; + +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder; +import org.apache.spark.sql.types.StructType; + +public class Spark40SpannerDataWriterTest extends SpannerDataWriterTestBase { + + @Override + protected ExpressionEncoder getEncoder(StructType schema) { + return (ExpressionEncoder) Encoders.row(schema); + } +} diff --git a/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/acceptance/DataprocServerlessImage30AcceptanceTest.java b/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/acceptance/DataprocServerlessImage30AcceptanceTest.java new file mode 100644 index 00000000..c7c0a103 --- /dev/null +++ b/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/acceptance/DataprocServerlessImage30AcceptanceTest.java @@ -0,0 +1,45 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.cloud.spark.spanner.acceptance; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * The acceptance test on the Dataproc Serverless. The test have to be running on the project with + * requireOsLogin disabled, otherwise an org policy violation error will be thrown. + */ +@RunWith(JUnit4.class) +public final class DataprocServerlessImage30AcceptanceTest + extends DataprocServerlessAcceptanceTestBase { + + public static final String CONNECTOR_JAR_DIRECTORY = "../spark-4.0-spanner/target"; + + @BeforeClass + public static void setUp() throws Exception { + setup(CONNECTOR_JAR_DIRECTORY, "spark-4.0-spanner"); + } + + @AfterClass + public static void cleanUp() throws Exception { + teardown(); + } + + public DataprocServerlessImage30AcceptanceTest() { + super("3.0"); + } +} diff --git a/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/acceptance/Spark40DataprocImage30AcceptanceTest.java b/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/acceptance/Spark40DataprocImage30AcceptanceTest.java new file mode 100644 index 00000000..9f4eca94 --- /dev/null +++ b/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/acceptance/Spark40DataprocImage30AcceptanceTest.java @@ -0,0 +1,44 @@ +// Copyright 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.cloud.spark.spanner.acceptance; + +import com.google.common.collect.ImmutableList; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public final class Spark40DataprocImage30AcceptanceTest extends DataprocAcceptanceTestBase { + + private static AcceptanceTestContext context; + public static final String CONNECTOR_JAR_DIRECTORY = "../spark-4.0-spanner/target"; + + public Spark40DataprocImage30AcceptanceTest() { + super(context); + } + + @BeforeClass + public static void setup() throws Exception { + context = + DataprocAcceptanceTestBase.setup( + "3.0-debian12", CONNECTOR_JAR_DIRECTORY, "spark-4.0-spanner", ImmutableList.of()); + } + + @AfterClass + public static void tearDown() throws Exception { + DataprocAcceptanceTestBase.tearDown(context); + } +} diff --git a/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/graph/Spark40GraphErrorHandlingIntegrationTest.java b/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/graph/Spark40GraphErrorHandlingIntegrationTest.java new file mode 100644 index 00000000..8abfbb45 --- /dev/null +++ b/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/graph/Spark40GraphErrorHandlingIntegrationTest.java @@ -0,0 +1,18 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.cloud.spark.spanner.graph; + +public class Spark40GraphErrorHandlingIntegrationTest + extends GraphErrorHandlingIntegrationTestBase {} diff --git a/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/graph/Spark40GraphReadIntegrationTest.java b/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/graph/Spark40GraphReadIntegrationTest.java new file mode 100644 index 00000000..bf0e3f6f --- /dev/null +++ b/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/graph/Spark40GraphReadIntegrationTest.java @@ -0,0 +1,17 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.cloud.spark.spanner.graph; + +public class Spark40GraphReadIntegrationTest extends GraphReadIntegrationTest {} diff --git a/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/Spark40FunctionsAndExpressionsIntegrationTest.java b/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/Spark40FunctionsAndExpressionsIntegrationTest.java new file mode 100644 index 00000000..dd852cb0 --- /dev/null +++ b/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/Spark40FunctionsAndExpressionsIntegrationTest.java @@ -0,0 +1,22 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.cloud.spark.spanner.integration; + +public class Spark40FunctionsAndExpressionsIntegrationTest + extends FunctionsAndExpressionsIntegrationTestBase { + public Spark40FunctionsAndExpressionsIntegrationTest(boolean isPostgreSql) { + super(isPostgreSql); + } +} diff --git a/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/Spark40ReadPgIntegrationTest.java b/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/Spark40ReadPgIntegrationTest.java new file mode 100644 index 00000000..a19a985c --- /dev/null +++ b/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/Spark40ReadPgIntegrationTest.java @@ -0,0 +1,21 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.cloud.spark.spanner.integration; + +import org.junit.Ignore; + +@Ignore( + "Existing test that fails on timezone differences. Previously ignored as an integration test.") +public class Spark40ReadPgIntegrationTest extends ReadPgIntegrationTestBase {} diff --git a/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/Spark40SchemaValidationIntegrationTest.java b/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/Spark40SchemaValidationIntegrationTest.java new file mode 100644 index 00000000..34223b5c --- /dev/null +++ b/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/Spark40SchemaValidationIntegrationTest.java @@ -0,0 +1,63 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.cloud.spark.spanner.integration; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import org.apache.spark.sql.AnalysisException; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.RowFactory; +import org.apache.spark.sql.SaveMode; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; +import org.junit.Test; + +public class Spark40SchemaValidationIntegrationTest extends SchemaValidationIntegrationTestBase { + public Spark40SchemaValidationIntegrationTest(boolean usePostgreSql) { + super(usePostgreSql); + } + + @Override + @Test + public void testPartialWriteFailsWithoutOption() { + StructType partialSchema = + new StructType( + new StructField[] {DataTypes.createStructField("id", DataTypes.LongType, false)}); + List rows = Collections.singletonList(RowFactory.create(1L)); + Dataset df = spark.createDataFrame(rows, partialSchema); + + Map props = connectionProperties(usePostgreSql); + props.put("table", SCHEMA_VALIDATION_TABLE_NAME); + // "enablePartialRowUpdates" is NOT set + + // Expect Spark's AnalysisException, not our custom SpannerConnectorException + AnalysisException e = + assertThrows( + AnalysisException.class, + () -> df.write().format("cloud-spanner").options(props).mode(SaveMode.Append).save()); + + String message = e.getMessage(); + assertThat(message).contains("Cannot write incompatible data for the table"); + assertThat(message).contains(SCHEMA_VALIDATION_TABLE_NAME); + assertThat(message).contains("Cannot find data for the output column"); + assertThat(message).contains("name"); + } +} diff --git a/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/Spark40SpannerInputPartitionReaderContextIntegrationTest.java b/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/Spark40SpannerInputPartitionReaderContextIntegrationTest.java new file mode 100644 index 00000000..0667dc6c --- /dev/null +++ b/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/Spark40SpannerInputPartitionReaderContextIntegrationTest.java @@ -0,0 +1,18 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.cloud.spark.spanner.integration; + +public class Spark40SpannerInputPartitionReaderContextIntegrationTest + extends SpannerInputPartitionReaderContextIntegrationTestBase {} diff --git a/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/Spark40SpannerScanBuilderIntegrationTest.java b/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/Spark40SpannerScanBuilderIntegrationTest.java new file mode 100644 index 00000000..1c2aaa98 --- /dev/null +++ b/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/Spark40SpannerScanBuilderIntegrationTest.java @@ -0,0 +1,18 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.cloud.spark.spanner.integration; + +public class Spark40SpannerScanBuilderIntegrationTest + extends SpannerScanBuilderIntegrationTestBase {} diff --git a/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/Spark40SpannerTableIntegrationTest.java b/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/Spark40SpannerTableIntegrationTest.java new file mode 100644 index 00000000..e866345d --- /dev/null +++ b/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/Spark40SpannerTableIntegrationTest.java @@ -0,0 +1,17 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.cloud.spark.spanner.integration; + +public class Spark40SpannerTableIntegrationTest extends SpannerTableIntegrationTestBase {} diff --git a/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/Spark40SpannerTableProviderIntegrationTest.java b/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/Spark40SpannerTableProviderIntegrationTest.java new file mode 100644 index 00000000..f31bd583 --- /dev/null +++ b/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/Spark40SpannerTableProviderIntegrationTest.java @@ -0,0 +1,25 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.cloud.spark.spanner.integration; + +import com.google.cloud.spark.spanner.Spark35SpannerTableProvider; + +public class Spark40SpannerTableProviderIntegrationTest + extends SparkSpannerTableProviderIntegrationTestBase { + + protected Spark40SpannerTableProvider getInstance() { + return new Spark40SpannerTableProvider(); + } +} diff --git a/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/Spark40WriteIntegrationTest.java b/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/Spark40WriteIntegrationTest.java new file mode 100644 index 00000000..33f08637 --- /dev/null +++ b/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/Spark40WriteIntegrationTest.java @@ -0,0 +1,21 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.cloud.spark.spanner.integration; + +public class Spark40WriteIntegrationTest extends WriteIntegrationTest { + public Spark40WriteIntegrationTest(boolean usePostgresSql) { + super(usePostgresSql); + } +} diff --git a/spark-4.0-spanner-lib/src/test/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister b/spark-4.0-spanner-lib/src/test/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister new file mode 100644 index 00000000..518cfc41 --- /dev/null +++ b/spark-4.0-spanner-lib/src/test/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister @@ -0,0 +1 @@ +com.google.cloud.spark.spanner.Spark40SpannerTableProvider diff --git a/spark-4.0-spanner/pom.xml b/spark-4.0-spanner/pom.xml new file mode 100644 index 00000000..7f3b4fca --- /dev/null +++ b/spark-4.0-spanner/pom.xml @@ -0,0 +1,50 @@ + + 4.0.0 + + com.google.cloud.spark.spanner + spark-spanner-parent + ${revision} + ../spark-spanner-parent + + + spark-4.0-spanner + ${revision} + spanner DataSource v2 for Spark 4.0 + + 4.0.1 + 2.13 + false + + + + Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + ${project.groupId} + spark-4.0-spanner-lib + ${project.version} + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.11.0 + + 17 + + + + org.apache.maven.plugins + maven-shade-plugin + + + + diff --git a/spark-4.0-spanner/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister b/spark-4.0-spanner/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister new file mode 100644 index 00000000..518cfc41 --- /dev/null +++ b/spark-4.0-spanner/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister @@ -0,0 +1 @@ +com.google.cloud.spark.spanner.Spark40SpannerTableProvider From d88e2337ba09fb0e03c0c867aa0aaeb742e85953 Mon Sep 17 00:00:00 2001 From: stevelordbq Date: Tue, 14 Apr 2026 14:49:49 -0700 Subject: [PATCH 02/11] Spark 4.0 invalid import --- .../integration/Spark40SpannerTableProviderIntegrationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/Spark40SpannerTableProviderIntegrationTest.java b/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/Spark40SpannerTableProviderIntegrationTest.java index f31bd583..a29c4343 100644 --- a/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/Spark40SpannerTableProviderIntegrationTest.java +++ b/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/Spark40SpannerTableProviderIntegrationTest.java @@ -14,7 +14,7 @@ package com.google.cloud.spark.spanner.integration; -import com.google.cloud.spark.spanner.Spark35SpannerTableProvider; +import com.google.cloud.spark.spanner.Spark40SpannerTableProvider; public class Spark40SpannerTableProviderIntegrationTest extends SparkSpannerTableProviderIntegrationTestBase { From d7dd0a50cb785334fe096678511579bff60a83ad Mon Sep 17 00:00:00 2001 From: stevelordbq Date: Thu, 16 Apr 2026 23:00:18 -0700 Subject: [PATCH 03/11] Spark 4.0 integration tests extended --- spark-3.1-spanner-lib/pom.xml | 11 ++++++++++ .../spanner/SpannerDataWriterTestBase.java | 22 +++++++++++++------ .../spanner/Spark31SpannerDataWriterTest.java | 8 ++++++- .../SparkSpannerIntegrationTestBase.java | 1 + spark-3.1-spanner/pom.xml | 11 ++++++++++ spark-3.2-spanner-lib/pom.xml | 15 +++++++++++++ .../spanner/Spark32SpannerDataWriterTest.java | 8 ++++++- spark-3.2-spanner/pom.xml | 11 ++++++++++ spark-3.3-spanner-lib/pom.xml | 15 +++++++++++++ .../spanner/Spark33SpannerDataWriterTest.java | 8 ++++++- spark-3.3-spanner/pom.xml | 11 ++++++++++ spark-3.5-spanner-lib/pom.xml | 15 +++++++++++++ .../spanner/Spark35SpannerDataWriterTest.java | 10 +++++++-- spark-3.5-spanner/pom.xml | 11 ++++++++++ spark-4.0-spanner-lib/pom.xml | 11 ++++++++++ .../spanner/Spark40SpannerDataWriterTest.java | 20 ++++++++++++++--- spark-4.0-spanner/pom.xml | 11 ++++++++++ spark-spanner-parent/pom.xml | 18 ++++++++++++++- 18 files changed, 201 insertions(+), 16 deletions(-) diff --git a/spark-3.1-spanner-lib/pom.xml b/spark-3.1-spanner-lib/pom.xml index 8c434a0a..bb936f88 100644 --- a/spark-3.1-spanner-lib/pom.xml +++ b/spark-3.1-spanner-lib/pom.xml @@ -71,6 +71,17 @@ + + org.apache.maven.plugins + maven-toolchains-plugin + + + + 8 + + + + diff --git a/spark-3.1-spanner-lib/src/test/java/com/google/cloud/spark/spanner/SpannerDataWriterTestBase.java b/spark-3.1-spanner-lib/src/test/java/com/google/cloud/spark/spanner/SpannerDataWriterTestBase.java index 607dab3a..cbef2a2c 100644 --- a/spark-3.1-spanner-lib/src/test/java/com/google/cloud/spark/spanner/SpannerDataWriterTestBase.java +++ b/spark-3.1-spanner-lib/src/test/java/com/google/cloud/spark/spanner/SpannerDataWriterTestBase.java @@ -45,6 +45,7 @@ import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import org.apache.spark.sql.Encoder; import org.apache.spark.sql.Row; import org.apache.spark.sql.RowFactory; import org.apache.spark.sql.catalyst.InternalRow; @@ -69,10 +70,11 @@ public abstract class SpannerDataWriterTestBase { private final ScheduledExecutorService scheduledExecutor = TestingExecutors.sameThreadScheduledExecutor(); - private StructType schema; + protected StructType schema; private Map properties; private BatchClientWithCloser batchClientWithCloser; - private ExpressionEncoder.Serializer serializer; + protected ExpressionEncoder.Serializer serializer; + protected Encoder encoder; @Mock private ExecutorService mockExecutor; @Mock private ScheduledExecutorService mockScheduledExecutor; @@ -89,20 +91,23 @@ public void setUp() { DataTypes.createStructField("string_col", DataTypes.StringType, true), }); - ExpressionEncoder encoder = getEncoder(schema); - serializer = encoder.createSerializer(); + encoder = getEncoder(schema); + localSetup(); properties = new HashMap<>(); properties.put("table", "testTable"); properties.put("mutationsPerBatch", "2"); // Use a small batch size for tests } + /** Provides Spark version specific set up. */ + protected abstract void localSetup(); + /** * Gets an ExpressionEncoder, the implementation of which depends on the version of Spark used. * * @param struct used to determine the Encoder. - * @return The ExpressionEncoder. + * @return The Encoder. */ - protected abstract ExpressionEncoder getEncoder(StructType struct); + protected abstract Encoder getEncoder(StructType struct); private SpannerDataWriter createWriter(Map props) { return new SpannerDataWriter( @@ -453,7 +458,10 @@ public void testMutationTypeLowerCaseIsHonored() throws IOException { testMutationTypeIsHonored("mutationtype"); } - private InternalRow CreateInternalRow(long i) { + protected InternalRow CreateInternalRow(long i) { + // ExpressionEncoder exprEncoder = (ExpressionEncoder) encoder; + // ExpressionEncoder.Serializer serializer = exprEncoder.createSerializer(); + // return serializer.apply(RowFactory.create(i, "row" + i)); return serializer.apply(RowFactory.create(i, "row" + i)); } } diff --git a/spark-3.1-spanner-lib/src/test/java/com/google/cloud/spark/spanner/Spark31SpannerDataWriterTest.java b/spark-3.1-spanner-lib/src/test/java/com/google/cloud/spark/spanner/Spark31SpannerDataWriterTest.java index ec4cc202..6ab6781a 100644 --- a/spark-3.1-spanner-lib/src/test/java/com/google/cloud/spark/spanner/Spark31SpannerDataWriterTest.java +++ b/spark-3.1-spanner-lib/src/test/java/com/google/cloud/spark/spanner/Spark31SpannerDataWriterTest.java @@ -14,6 +14,7 @@ package com.google.cloud.spark.spanner; +import org.apache.spark.sql.Encoder; import org.apache.spark.sql.Row; import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder; import org.apache.spark.sql.catalyst.encoders.RowEncoder; @@ -22,7 +23,12 @@ public class Spark31SpannerDataWriterTest extends SpannerDataWriterTestBase { @Override - protected ExpressionEncoder getEncoder(StructType schema) { + protected void localSetup() { + serializer = ((ExpressionEncoder) encoder).createSerializer(); + } + + @Override + protected Encoder getEncoder(StructType schema) { return RowEncoder.apply(schema); } } diff --git a/spark-3.1-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/SparkSpannerIntegrationTestBase.java b/spark-3.1-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/SparkSpannerIntegrationTestBase.java index 44cb6135..726a1c10 100644 --- a/spark-3.1-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/SparkSpannerIntegrationTestBase.java +++ b/spark-3.1-spanner-lib/src/test/java/com/google/cloud/spark/spanner/integration/SparkSpannerIntegrationTestBase.java @@ -34,6 +34,7 @@ public void setUpSpark() { .appName("SparkSpannerIntegrationTest") .config("spark.ui.enabled", "false") .config("spark.default.parallelism", 20) + .config("spark.sql.legacy.timeParserPolicy", "LEGACY") .getOrCreate(); if (catalogProps.get("emulatorHost") != null) { diff --git a/spark-3.1-spanner/pom.xml b/spark-3.1-spanner/pom.xml index 77378b57..beb5c695 100644 --- a/spark-3.1-spanner/pom.xml +++ b/spark-3.1-spanner/pom.xml @@ -43,6 +43,17 @@ org.apache.maven.plugins maven-shade-plugin + + org.apache.maven.plugins + maven-toolchains-plugin + + + + 8 + + + + diff --git a/spark-3.2-spanner-lib/pom.xml b/spark-3.2-spanner-lib/pom.xml index 3583b2de..901b84b1 100644 --- a/spark-3.2-spanner-lib/pom.xml +++ b/spark-3.2-spanner-lib/pom.xml @@ -47,4 +47,19 @@ test + + + + org.apache.maven.plugins + maven-toolchains-plugin + + + + 8 + + + + + + diff --git a/spark-3.2-spanner-lib/src/test/java/com/google/cloud/spark/spanner/Spark32SpannerDataWriterTest.java b/spark-3.2-spanner-lib/src/test/java/com/google/cloud/spark/spanner/Spark32SpannerDataWriterTest.java index dcf3ac3d..e0612e8b 100644 --- a/spark-3.2-spanner-lib/src/test/java/com/google/cloud/spark/spanner/Spark32SpannerDataWriterTest.java +++ b/spark-3.2-spanner-lib/src/test/java/com/google/cloud/spark/spanner/Spark32SpannerDataWriterTest.java @@ -14,6 +14,7 @@ package com.google.cloud.spark.spanner; +import org.apache.spark.sql.Encoder; import org.apache.spark.sql.Row; import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder; import org.apache.spark.sql.catalyst.encoders.RowEncoder; @@ -22,7 +23,12 @@ public class Spark32SpannerDataWriterTest extends SpannerDataWriterTestBase { @Override - protected ExpressionEncoder getEncoder(StructType schema) { + protected void localSetup() { + serializer = ((ExpressionEncoder) encoder).createSerializer(); + } + + @Override + protected Encoder getEncoder(StructType schema) { return RowEncoder.apply(schema); } } diff --git a/spark-3.2-spanner/pom.xml b/spark-3.2-spanner/pom.xml index 7ee8b89e..f88393b3 100644 --- a/spark-3.2-spanner/pom.xml +++ b/spark-3.2-spanner/pom.xml @@ -36,6 +36,17 @@ org.apache.maven.plugins maven-shade-plugin + + org.apache.maven.plugins + maven-toolchains-plugin + + + + 8 + + + + diff --git a/spark-3.3-spanner-lib/pom.xml b/spark-3.3-spanner-lib/pom.xml index 2153a512..9b9f4791 100644 --- a/spark-3.3-spanner-lib/pom.xml +++ b/spark-3.3-spanner-lib/pom.xml @@ -47,4 +47,19 @@ ${project.version} + + + + org.apache.maven.plugins + maven-toolchains-plugin + + + + 8 + + + + + + diff --git a/spark-3.3-spanner-lib/src/test/java/com/google/cloud/spark/spanner/Spark33SpannerDataWriterTest.java b/spark-3.3-spanner-lib/src/test/java/com/google/cloud/spark/spanner/Spark33SpannerDataWriterTest.java index 06fdaa84..b12d8e0b 100644 --- a/spark-3.3-spanner-lib/src/test/java/com/google/cloud/spark/spanner/Spark33SpannerDataWriterTest.java +++ b/spark-3.3-spanner-lib/src/test/java/com/google/cloud/spark/spanner/Spark33SpannerDataWriterTest.java @@ -14,6 +14,7 @@ package com.google.cloud.spark.spanner; +import org.apache.spark.sql.Encoder; import org.apache.spark.sql.Row; import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder; import org.apache.spark.sql.catalyst.encoders.RowEncoder; @@ -22,7 +23,12 @@ public class Spark33SpannerDataWriterTest extends SpannerDataWriterTestBase { @Override - protected ExpressionEncoder getEncoder(StructType schema) { + protected void localSetup() { + serializer = ((ExpressionEncoder) encoder).createSerializer(); + } + + @Override + protected Encoder getEncoder(StructType schema) { return RowEncoder.apply(schema); } } diff --git a/spark-3.3-spanner/pom.xml b/spark-3.3-spanner/pom.xml index 0c138099..ccf29559 100644 --- a/spark-3.3-spanner/pom.xml +++ b/spark-3.3-spanner/pom.xml @@ -36,6 +36,17 @@ org.apache.maven.plugins maven-shade-plugin + + org.apache.maven.plugins + maven-toolchains-plugin + + + + 8 + + + + diff --git a/spark-3.5-spanner-lib/pom.xml b/spark-3.5-spanner-lib/pom.xml index 08e878e1..3b0114c3 100644 --- a/spark-3.5-spanner-lib/pom.xml +++ b/spark-3.5-spanner-lib/pom.xml @@ -47,4 +47,19 @@ ${project.version} + + + + org.apache.maven.plugins + maven-toolchains-plugin + + + + 8 + + + + + + diff --git a/spark-3.5-spanner-lib/src/test/java/com/google/cloud/spark/spanner/Spark35SpannerDataWriterTest.java b/spark-3.5-spanner-lib/src/test/java/com/google/cloud/spark/spanner/Spark35SpannerDataWriterTest.java index 0a1e5e56..2ace64c0 100644 --- a/spark-3.5-spanner-lib/src/test/java/com/google/cloud/spark/spanner/Spark35SpannerDataWriterTest.java +++ b/spark-3.5-spanner-lib/src/test/java/com/google/cloud/spark/spanner/Spark35SpannerDataWriterTest.java @@ -14,6 +14,7 @@ package com.google.cloud.spark.spanner; +import org.apache.spark.sql.Encoder; import org.apache.spark.sql.Encoders; import org.apache.spark.sql.Row; import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder; @@ -22,7 +23,12 @@ public class Spark35SpannerDataWriterTest extends SpannerDataWriterTestBase { @Override - protected ExpressionEncoder getEncoder(StructType schema) { - return (ExpressionEncoder) Encoders.row(schema); + protected void localSetup() { + serializer = ((ExpressionEncoder) encoder).createSerializer(); + } + + @Override + protected Encoder getEncoder(StructType schema) { + return Encoders.row(schema); } } diff --git a/spark-3.5-spanner/pom.xml b/spark-3.5-spanner/pom.xml index beb159cd..f4b2d378 100644 --- a/spark-3.5-spanner/pom.xml +++ b/spark-3.5-spanner/pom.xml @@ -36,6 +36,17 @@ org.apache.maven.plugins maven-shade-plugin + + org.apache.maven.plugins + maven-toolchains-plugin + + + + 8 + + + + diff --git a/spark-4.0-spanner-lib/pom.xml b/spark-4.0-spanner-lib/pom.xml index 2e66a5be..cd0a1f93 100644 --- a/spark-4.0-spanner-lib/pom.xml +++ b/spark-4.0-spanner-lib/pom.xml @@ -58,6 +58,17 @@ 17 + + org.apache.maven.plugins + maven-toolchains-plugin + + + + 17 + + + + diff --git a/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/Spark40SpannerDataWriterTest.java b/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/Spark40SpannerDataWriterTest.java index 17b6fff2..cdcc372e 100644 --- a/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/Spark40SpannerDataWriterTest.java +++ b/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/Spark40SpannerDataWriterTest.java @@ -14,15 +14,29 @@ package com.google.cloud.spark.spanner; +import org.apache.spark.sql.Encoder; import org.apache.spark.sql.Encoders; import org.apache.spark.sql.Row; -import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder; +import org.apache.spark.sql.RowFactory; +import org.apache.spark.sql.catalyst.CatalystTypeConverters; +import org.apache.spark.sql.catalyst.InternalRow; import org.apache.spark.sql.types.StructType; public class Spark40SpannerDataWriterTest extends SpannerDataWriterTestBase { @Override - protected ExpressionEncoder getEncoder(StructType schema) { - return (ExpressionEncoder) Encoders.row(schema); + protected void localSetup() {} + + @Override + protected Encoder getEncoder(StructType schema) { + // In Spark 4.0, this returns an AgnosticEncoder which implements Encoder + return Encoders.row(schema); + } + + @Override + protected InternalRow CreateInternalRow(long i) { + Row row = RowFactory.create(i, "row" + i); + + return (InternalRow) CatalystTypeConverters.createToCatalystConverter(schema).apply(row); } } diff --git a/spark-4.0-spanner/pom.xml b/spark-4.0-spanner/pom.xml index 7f3b4fca..91d0cf49 100644 --- a/spark-4.0-spanner/pom.xml +++ b/spark-4.0-spanner/pom.xml @@ -45,6 +45,17 @@ org.apache.maven.plugins maven-shade-plugin + + org.apache.maven.plugins + maven-toolchains-plugin + + + + 17 + + + + diff --git a/spark-spanner-parent/pom.xml b/spark-spanner-parent/pom.xml index 7b3c4bd9..21aa5460 100644 --- a/spark-spanner-parent/pom.xml +++ b/spark-spanner-parent/pom.xml @@ -69,6 +69,8 @@ true false true + + @@ -517,6 +519,18 @@ ${deploy.skip} + + org.apache.maven.plugins + maven-toolchains-plugin + 3.2.0 + + + + toolchain + + + + @@ -533,7 +547,9 @@ org.apache.maven.plugins maven-surefire-plugin - ${argLine} + + ${argLine} + **/*AcceptanceTest.java **/*IntegrationTest.java From 073889fb4772a6c920eac449bdb804d668555df3 Mon Sep 17 00:00:00 2001 From: stevelordbq Date: Fri, 17 Apr 2026 18:59:05 -0700 Subject: [PATCH 04/11] Spark 4.0 integration tests working --- spark-4.0-spanner-lib/pom.xml | 42 +++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/spark-4.0-spanner-lib/pom.xml b/spark-4.0-spanner-lib/pom.xml index cd0a1f93..a3aeaaa4 100644 --- a/spark-4.0-spanner-lib/pom.xml +++ b/spark-4.0-spanner-lib/pom.xml @@ -69,6 +69,48 @@ + + org.apache.maven.plugins + maven-failsafe-plugin + + 10 + false + + **/*IntegrationTest.java + + + --add-opens=java.base/java.lang=ALL-UNNAMED + --add-opens=java.base/java.lang.invoke=ALL-UNNAMED + --add-opens=java.base/java.lang.reflect=ALL-UNNAMED + --add-opens=java.base/java.io=ALL-UNNAMED + --add-opens=java.base/java.net=ALL-UNNAMED + --add-opens=java.base/java.nio=ALL-UNNAMED + --add-opens=java.base/java.util=ALL-UNNAMED + --add-opens=java.base/java.util.concurrent=ALL-UNNAMED + --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED + --add-opens=java.base/sun.nio.ch=ALL-UNNAMED + --add-opens=java.base/sun.nio.cs=ALL-UNNAMED + --add-opens=java.base/sun.security.action=ALL-UNNAMED + --add-opens=java.base/sun.util.calendar=ALL-UNNAMED + --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED + @{argLine} + + + + + integration-test + + integration-test + + + + verify + + verify + + + + From f2908b57daf4dbf50a887208fe3fdb25b3f8a63b Mon Sep 17 00:00:00 2001 From: stevelordbq Date: Mon, 20 Apr 2026 23:00:48 -0700 Subject: [PATCH 05/11] Spark 4.0 consolidate maven-compiler-plugin --- spark-4.0-spanner-lib/pom.xml | 9 +-------- spark-4.0-spanner/pom.xml | 9 +-------- spark-spanner-parent/pom.xml | 19 ++++++++++++------- 3 files changed, 14 insertions(+), 23 deletions(-) diff --git a/spark-4.0-spanner-lib/pom.xml b/spark-4.0-spanner-lib/pom.xml index a3aeaaa4..e28d2a84 100644 --- a/spark-4.0-spanner-lib/pom.xml +++ b/spark-4.0-spanner-lib/pom.xml @@ -16,6 +16,7 @@ 4.0.1 2.13 true + 17 @@ -50,14 +51,6 @@ - - org.apache.maven.plugins - maven-compiler-plugin - 3.11.0 - - 17 - - org.apache.maven.plugins maven-toolchains-plugin diff --git a/spark-4.0-spanner/pom.xml b/spark-4.0-spanner/pom.xml index 91d0cf49..78543488 100644 --- a/spark-4.0-spanner/pom.xml +++ b/spark-4.0-spanner/pom.xml @@ -16,6 +16,7 @@ 4.0.1 2.13 false + 17 @@ -33,14 +34,6 @@ - - org.apache.maven.plugins - maven-compiler-plugin - 3.11.0 - - 17 - - org.apache.maven.plugins maven-shade-plugin diff --git a/spark-spanner-parent/pom.xml b/spark-spanner-parent/pom.xml index 21aa5460..fa91bf5a 100644 --- a/spark-spanner-parent/pom.xml +++ b/spark-spanner-parent/pom.xml @@ -74,6 +74,10 @@ + 1.8 + + 1.8 + 1.8 @@ -535,13 +539,14 @@ - org.apache.maven.plugins - maven-compiler-plugin - 3.11.0 - - 1.8 - 1.8 - + org.apache.maven.plugins + maven-compiler-plugin + 3.11.0 + + + ${toolchain.jdk.version} + + org.apache.maven.plugins From 28586ce395a2d85eb6b909c0e74bcf10b21206b7 Mon Sep 17 00:00:00 2001 From: stevelordbq Date: Tue, 21 Apr 2026 10:07:47 -0700 Subject: [PATCH 06/11] Spark 4.0 remove maven-jar-plugin version from 3.1 module --- spark-3.1-spanner-lib/pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/spark-3.1-spanner-lib/pom.xml b/spark-3.1-spanner-lib/pom.xml index bb936f88..09ddf085 100644 --- a/spark-3.1-spanner-lib/pom.xml +++ b/spark-3.1-spanner-lib/pom.xml @@ -57,7 +57,6 @@ org.apache.maven.plugins maven-jar-plugin - 3.3.0 **/org.apache.spark.sql.sources.DataSourceRegister From a489b452a631d0a48d3433def93237016b1c882c Mon Sep 17 00:00:00 2001 From: stevelordbq Date: Tue, 21 Apr 2026 10:44:07 -0700 Subject: [PATCH 07/11] Spark 4.0 remove extra maven-failsafe-plugin --- spark-spanner-lib-parent/pom.xml | 38 -------------------------------- 1 file changed, 38 deletions(-) diff --git a/spark-spanner-lib-parent/pom.xml b/spark-spanner-lib-parent/pom.xml index 652738ba..6be86bbd 100644 --- a/spark-spanner-lib-parent/pom.xml +++ b/spark-spanner-lib-parent/pom.xml @@ -32,42 +32,4 @@ provided - - - integration - - false - - - - - org.apache.maven.plugins - maven-failsafe-plugin - - ${argLine} - 7 - false - - **/*IntegrationTest.java - - - - - integration-test - - integration-test - - - - verify - - verify - - - - - - - - From 314d3c1e4b34b7da719f91781a6b665faabf9d35 Mon Sep 17 00:00:00 2001 From: stevelordbq Date: Tue, 21 Apr 2026 15:19:00 -0700 Subject: [PATCH 08/11] Spark 4.0 add 4.0 to job. Add java 17 toolchain --- .github/workflows/dev-ci.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dev-ci.yml b/.github/workflows/dev-ci.yml index d873fa10..117c39d8 100644 --- a/.github/workflows/dev-ci.yml +++ b/.github/workflows/dev-ci.yml @@ -11,11 +11,14 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 - - name: Set up JDK 8 + - name: Set up JDK 8 and 17 uses: actions/setup-java@v4 with: - java-version: '8' + java-version: | + 8 + 17 distribution: 'temurin' + check-latest: false - name: Cache Maven packages uses: actions/cache@v4 with: @@ -35,6 +38,9 @@ jobs: - name: Build for Spark 3.5 run: | mvn -B -e install -P3.5 -DskipTests + - name: Build for Spark 4.0 + run: | + mvn -B -e install -P4.0 -DskipTests - name: Get short commit hash id: vars run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT @@ -47,6 +53,12 @@ jobs: fi cp "$file" shaded-jars/ done + for file in spark-4.*/target/spark-4.*.jar; do + if [[ "$file" == *"lib"* || "$file" == *"original"* ]]; then + continue + fi + cp "$file" shaded-jars/ + done - name: Upload Shaded JARs uses: actions/upload-artifact@v4 with: From d8db79aaa633a9c9f425423807287ddc63621a14 Mon Sep 17 00:00:00 2001 From: stevelordbq Date: Wed, 22 Apr 2026 09:42:20 -0700 Subject: [PATCH 09/11] Spark 4.0 PR updates --- .../spanner/SpannerDataWriterTestBase.java | 3 - spark-4.0-spanner-lib/pom.xml | 97 +++++++++++-------- .../spanner/Spark40SpannerDataWriterTest.java | 8 +- 3 files changed, 61 insertions(+), 47 deletions(-) diff --git a/spark-3.1-spanner-lib/src/test/java/com/google/cloud/spark/spanner/SpannerDataWriterTestBase.java b/spark-3.1-spanner-lib/src/test/java/com/google/cloud/spark/spanner/SpannerDataWriterTestBase.java index cbef2a2c..8c7c51a5 100644 --- a/spark-3.1-spanner-lib/src/test/java/com/google/cloud/spark/spanner/SpannerDataWriterTestBase.java +++ b/spark-3.1-spanner-lib/src/test/java/com/google/cloud/spark/spanner/SpannerDataWriterTestBase.java @@ -459,9 +459,6 @@ public void testMutationTypeLowerCaseIsHonored() throws IOException { } protected InternalRow CreateInternalRow(long i) { - // ExpressionEncoder exprEncoder = (ExpressionEncoder) encoder; - // ExpressionEncoder.Serializer serializer = exprEncoder.createSerializer(); - // return serializer.apply(RowFactory.create(i, "row" + i)); return serializer.apply(RowFactory.create(i, "row" + i)); } } diff --git a/spark-4.0-spanner-lib/pom.xml b/spark-4.0-spanner-lib/pom.xml index e28d2a84..0c4dd968 100644 --- a/spark-4.0-spanner-lib/pom.xml +++ b/spark-4.0-spanner-lib/pom.xml @@ -17,6 +17,7 @@ 2.13 true 17 + 17 @@ -62,48 +63,60 @@ - - org.apache.maven.plugins - maven-failsafe-plugin - - 10 - false - - **/*IntegrationTest.java - - - --add-opens=java.base/java.lang=ALL-UNNAMED - --add-opens=java.base/java.lang.invoke=ALL-UNNAMED - --add-opens=java.base/java.lang.reflect=ALL-UNNAMED - --add-opens=java.base/java.io=ALL-UNNAMED - --add-opens=java.base/java.net=ALL-UNNAMED - --add-opens=java.base/java.nio=ALL-UNNAMED - --add-opens=java.base/java.util=ALL-UNNAMED - --add-opens=java.base/java.util.concurrent=ALL-UNNAMED - --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED - --add-opens=java.base/sun.nio.ch=ALL-UNNAMED - --add-opens=java.base/sun.nio.cs=ALL-UNNAMED - --add-opens=java.base/sun.security.action=ALL-UNNAMED - --add-opens=java.base/sun.util.calendar=ALL-UNNAMED - --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED - @{argLine} - - - - - integration-test - - integration-test - - - - verify - - verify - - - - + + + integration + + false + + + + + org.apache.maven.plugins + maven-failsafe-plugin + + 10 + false + + **/*IntegrationTest.java + + + --add-opens=java.base/java.lang=ALL-UNNAMED + --add-opens=java.base/java.lang.invoke=ALL-UNNAMED + --add-opens=java.base/java.lang.reflect=ALL-UNNAMED + --add-opens=java.base/java.io=ALL-UNNAMED + --add-opens=java.base/java.net=ALL-UNNAMED + --add-opens=java.base/java.nio=ALL-UNNAMED + --add-opens=java.base/java.util=ALL-UNNAMED + --add-opens=java.base/java.util.concurrent=ALL-UNNAMED + --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED + --add-opens=java.base/sun.nio.ch=ALL-UNNAMED + --add-opens=java.base/sun.nio.cs=ALL-UNNAMED + --add-opens=java.base/sun.security.action=ALL-UNNAMED + --add-opens=java.base/sun.util.calendar=ALL-UNNAMED + --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED + @{argLine} + + + + + integration-test + + integration-test + + + + verify + + verify + + + + + + + + diff --git a/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/Spark40SpannerDataWriterTest.java b/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/Spark40SpannerDataWriterTest.java index cdcc372e..8405a02a 100644 --- a/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/Spark40SpannerDataWriterTest.java +++ b/spark-4.0-spanner-lib/src/test/java/com/google/cloud/spark/spanner/Spark40SpannerDataWriterTest.java @@ -24,8 +24,12 @@ public class Spark40SpannerDataWriterTest extends SpannerDataWriterTestBase { + private scala.Function1 converter; + @Override - protected void localSetup() {} + protected void localSetup() { + converter = CatalystTypeConverters.createToCatalystConverter(schema); + } @Override protected Encoder getEncoder(StructType schema) { @@ -37,6 +41,6 @@ protected Encoder getEncoder(StructType schema) { protected InternalRow CreateInternalRow(long i) { Row row = RowFactory.create(i, "row" + i); - return (InternalRow) CatalystTypeConverters.createToCatalystConverter(schema).apply(row); + return (InternalRow) converter.apply(row); } } From e5af96456c07bec27173f8592883749333a901c0 Mon Sep 17 00:00:00 2001 From: stevelordbq Date: Wed, 22 Apr 2026 11:10:29 -0700 Subject: [PATCH 10/11] Spark 4.0 Single maven-toolchains-plugin driven by property. --- spark-3.1-spanner-lib/pom.xml | 11 ----------- spark-3.1-spanner/pom.xml | 11 ----------- spark-3.2-spanner-lib/pom.xml | 15 --------------- spark-3.2-spanner/pom.xml | 11 ----------- spark-3.3-spanner-lib/pom.xml | 15 --------------- spark-3.3-spanner/pom.xml | 11 ----------- spark-3.5-spanner-lib/pom.xml | 15 --------------- spark-3.5-spanner/pom.xml | 11 ----------- spark-4.0-spanner-lib/pom.xml | 18 +++--------------- spark-4.0-spanner/pom.xml | 11 ----------- spark-spanner-parent/pom.xml | 8 +++++++- 11 files changed, 10 insertions(+), 127 deletions(-) diff --git a/spark-3.1-spanner-lib/pom.xml b/spark-3.1-spanner-lib/pom.xml index 09ddf085..6be6eac8 100644 --- a/spark-3.1-spanner-lib/pom.xml +++ b/spark-3.1-spanner-lib/pom.xml @@ -70,17 +70,6 @@ - - org.apache.maven.plugins - maven-toolchains-plugin - - - - 8 - - - - diff --git a/spark-3.1-spanner/pom.xml b/spark-3.1-spanner/pom.xml index beb5c695..77378b57 100644 --- a/spark-3.1-spanner/pom.xml +++ b/spark-3.1-spanner/pom.xml @@ -43,17 +43,6 @@ org.apache.maven.plugins maven-shade-plugin - - org.apache.maven.plugins - maven-toolchains-plugin - - - - 8 - - - - diff --git a/spark-3.2-spanner-lib/pom.xml b/spark-3.2-spanner-lib/pom.xml index 901b84b1..3583b2de 100644 --- a/spark-3.2-spanner-lib/pom.xml +++ b/spark-3.2-spanner-lib/pom.xml @@ -47,19 +47,4 @@ test - - - - org.apache.maven.plugins - maven-toolchains-plugin - - - - 8 - - - - - - diff --git a/spark-3.2-spanner/pom.xml b/spark-3.2-spanner/pom.xml index f88393b3..7ee8b89e 100644 --- a/spark-3.2-spanner/pom.xml +++ b/spark-3.2-spanner/pom.xml @@ -36,17 +36,6 @@ org.apache.maven.plugins maven-shade-plugin - - org.apache.maven.plugins - maven-toolchains-plugin - - - - 8 - - - - diff --git a/spark-3.3-spanner-lib/pom.xml b/spark-3.3-spanner-lib/pom.xml index 9b9f4791..2153a512 100644 --- a/spark-3.3-spanner-lib/pom.xml +++ b/spark-3.3-spanner-lib/pom.xml @@ -47,19 +47,4 @@ ${project.version} - - - - org.apache.maven.plugins - maven-toolchains-plugin - - - - 8 - - - - - - diff --git a/spark-3.3-spanner/pom.xml b/spark-3.3-spanner/pom.xml index ccf29559..0c138099 100644 --- a/spark-3.3-spanner/pom.xml +++ b/spark-3.3-spanner/pom.xml @@ -36,17 +36,6 @@ org.apache.maven.plugins maven-shade-plugin - - org.apache.maven.plugins - maven-toolchains-plugin - - - - 8 - - - - diff --git a/spark-3.5-spanner-lib/pom.xml b/spark-3.5-spanner-lib/pom.xml index 3b0114c3..08e878e1 100644 --- a/spark-3.5-spanner-lib/pom.xml +++ b/spark-3.5-spanner-lib/pom.xml @@ -47,19 +47,4 @@ ${project.version} - - - - org.apache.maven.plugins - maven-toolchains-plugin - - - - 8 - - - - - - diff --git a/spark-3.5-spanner/pom.xml b/spark-3.5-spanner/pom.xml index f4b2d378..beb159cd 100644 --- a/spark-3.5-spanner/pom.xml +++ b/spark-3.5-spanner/pom.xml @@ -36,17 +36,6 @@ org.apache.maven.plugins maven-shade-plugin - - org.apache.maven.plugins - maven-toolchains-plugin - - - - 8 - - - - diff --git a/spark-4.0-spanner-lib/pom.xml b/spark-4.0-spanner-lib/pom.xml index 0c4dd968..479619e0 100644 --- a/spark-4.0-spanner-lib/pom.xml +++ b/spark-4.0-spanner-lib/pom.xml @@ -50,21 +50,6 @@ ${project.version} - - - - org.apache.maven.plugins - maven-toolchains-plugin - - - - 17 - - - - - - integration @@ -99,6 +84,9 @@ --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED @{argLine} + + ${toolchain.jdk.version} + diff --git a/spark-4.0-spanner/pom.xml b/spark-4.0-spanner/pom.xml index 78543488..63855dd8 100644 --- a/spark-4.0-spanner/pom.xml +++ b/spark-4.0-spanner/pom.xml @@ -38,17 +38,6 @@ org.apache.maven.plugins maven-shade-plugin - - org.apache.maven.plugins - maven-toolchains-plugin - - - - 17 - - - - diff --git a/spark-spanner-parent/pom.xml b/spark-spanner-parent/pom.xml index fa91bf5a..ef947076 100644 --- a/spark-spanner-parent/pom.xml +++ b/spark-spanner-parent/pom.xml @@ -530,7 +530,7 @@ - toolchain + select-jdk-toolchain @@ -563,6 +563,9 @@ but apparently the excludes filters above fail to exclude all integration tests (e.g., ReadIntegrationTestBase.java, SpannerInputPartitionReaderContextTest.java) --> + + ${toolchain.jdk.version} + @@ -766,6 +769,9 @@ **/*IntegrationTest.java + + ${toolchain.jdk.version} + From 379a725334aa5daa538212192f1bce34c1fffa73 Mon Sep 17 00:00:00 2001 From: stevelordbq Date: Wed, 22 Apr 2026 12:58:13 -0700 Subject: [PATCH 11/11] Spark 4.0 Single maven-toolchains-plugin driven by property. --- spark-4.0-spanner-lib/pom.xml | 2 +- spark-4.0-spanner/pom.xml | 2 +- spark-spanner-parent/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/spark-4.0-spanner-lib/pom.xml b/spark-4.0-spanner-lib/pom.xml index 479619e0..ff9383ff 100644 --- a/spark-4.0-spanner-lib/pom.xml +++ b/spark-4.0-spanner-lib/pom.xml @@ -16,7 +16,7 @@ 4.0.1 2.13 true - 17 + [17,18) 17 diff --git a/spark-4.0-spanner/pom.xml b/spark-4.0-spanner/pom.xml index 63855dd8..a87233b8 100644 --- a/spark-4.0-spanner/pom.xml +++ b/spark-4.0-spanner/pom.xml @@ -16,7 +16,7 @@ 4.0.1 2.13 false - 17 + [17,18) diff --git a/spark-spanner-parent/pom.xml b/spark-spanner-parent/pom.xml index ef947076..40e6fba4 100644 --- a/spark-spanner-parent/pom.xml +++ b/spark-spanner-parent/pom.xml @@ -74,7 +74,7 @@ - 1.8 + [1.8,9) 1.8 1.8