Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
540c3a4
update methods and testS
TimothyW553 Nov 12, 2025
80d4662
delete workspace
TimothyW553 Nov 12, 2025
7cf2c28
delete
TimothyW553 Nov 12, 2025
5fca199
change to catalogtable
TimothyW553 Nov 11, 2025
b0c0b6f
add tests
TimothyW553 Nov 12, 2025
fdec89e
fmt
TimothyW553 Nov 12, 2025
4807c4f
test fix
TimothyW553 Nov 13, 2025
8e992d9
storage props
TimothyW553 Nov 14, 2025
bbc37b5
merge props
TimothyW553 Nov 17, 2025
28e17b5
address comments
TimothyW553 Nov 17, 2025
afcdfb1
remove private method test
TimothyW553 Nov 18, 2025
e1aed3e
null check for storage properties
TimothyW553 Nov 18, 2025
d709460
Restore UC connection utils after rebase
TimothyW553 Dec 2, 2025
e7ef109
clean up...
TimothyW553 Dec 2, 2025
9835086
Restore UC connection info utils and add SparkUnityCatalogUtils suite
TimothyW553 Dec 2, 2025
d066f73
Update tests
TimothyW553 Dec 2, 2025
1f09ddc
spark
TimothyW553 Dec 2, 2025
c63731f
Clean up test utils and refactor
TimothyW553 Dec 2, 2025
fc13a3a
revert DeltaSQLConfV2.scala
TimothyW553 Dec 2, 2025
7ccf4aa
remove npe tests
TimothyW553 Dec 2, 2025
3be445e
remove more npe
TimothyW553 Dec 2, 2025
2ada816
Centralize utils
TimothyW553 Dec 4, 2025
3344671
fmt
TimothyW553 Dec 4, 2025
1028015
FMT
TimothyW553 Dec 4, 2025
5cddf21
Address comments - remove unecessary wrappers and add logging
TimothyW553 Dec 4, 2025
73c9ecd
Address comments - remove unecessary wrappers and add logging
TimothyW553 Dec 4, 2025
5cd8807
Add wrapper for map
TimothyW553 Dec 4, 2025
ab7f0fd
use SharedSparkSession
TimothyW553 Dec 4, 2025
37631c0
fix tests
TimothyW553 Dec 4, 2025
ca68661
fix tests
TimothyW553 Dec 4, 2025
e56a59f
annotate constant params
TimothyW553 Dec 5, 2025
5fc21cd
annotate constant params
TimothyW553 Dec 5, 2025
2db986b
Rename and follow naming conventions
TimothyW553 Dec 8, 2025
7e405bc
Rename and follow naming conventions
TimothyW553 Dec 8, 2025
28a6259
update context
TimothyW553 Dec 9, 2025
42b94dd
Merge branch 'master' into stack/ccv2-uc-utils
TimothyW553 Dec 9, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
/*
* Copyright (2025) The Delta Lake Project Authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.delta.kernel.spark.snapshot.unitycatalog;

import static java.util.Objects.requireNonNull;

/**
* Table information for Unity Catalog managed tables.
*
* <p>This POJO encapsulates all the information needed to interact with a Unity Catalog table
* without requiring Spark dependencies.
*/
public final class UCTableInfo {
private final String tableId;
private final String tablePath;
private final String ucUri;
private final String ucToken;

public UCTableInfo(String tableId, String tablePath, String ucUri, String ucToken) {
this.tableId = requireNonNull(tableId, "tableId is null");
this.tablePath = requireNonNull(tablePath, "tablePath is null");
this.ucUri = requireNonNull(ucUri, "ucUri is null");
this.ucToken = requireNonNull(ucToken, "ucToken is null");
}

public String getTableId() {
return tableId;
}

public String getTablePath() {
return tablePath;
}

public String getUcUri() {
return ucUri;
}

public String getUcToken() {
return ucToken;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
/*
* Copyright (2025) The Delta Lake Project Authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.delta.kernel.spark.snapshot.unitycatalog;

import static java.util.Objects.requireNonNull;

import io.delta.kernel.spark.utils.CatalogTableUtils;
import io.delta.storage.commit.uccommitcoordinator.UCCommitCoordinatorClient;
import java.util.Map;
import java.util.Optional;
import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.catalyst.catalog.CatalogTable;
import org.apache.spark.sql.delta.coordinatedcommits.UCCatalogConfig;
import org.apache.spark.sql.delta.coordinatedcommits.UCCommitCoordinatorBuilder$;

/**
* Utility class for extracting Unity Catalog table information from Spark catalog metadata.
*
* <p>This class isolates Spark dependencies, allowing {@link UCManagedSnapshotManager} to be
* created without Spark if table info is provided directly via {@link UCTableInfo}.
*/
public final class UCUtils {

// Utility class - no instances
private UCUtils() {}

/**
* Extracts Unity Catalog table information from Spark catalog table metadata.
*
* @param catalogTable Spark catalog table metadata
* @param spark SparkSession for resolving Unity Catalog configurations
* @return table info if table is UC-managed, empty otherwise
* @throws IllegalArgumentException if table is UC-managed but configuration is invalid
*/
public static Optional<UCTableInfo> extractTableInfo(
CatalogTable catalogTable, SparkSession spark) {
requireNonNull(catalogTable, "catalogTable is null");
requireNonNull(spark, "spark is null");

if (!CatalogTableUtils.isUnityCatalogManagedTable(catalogTable)) {
return Optional.empty();
}

String tableId = extractUCTableId(catalogTable);
String tablePath = extractTablePath(catalogTable);

// Get catalog name - require explicit catalog in identifier
scala.Option<String> catalogOption = catalogTable.identifier().catalog();
if (catalogOption.isEmpty()) {
throw new IllegalArgumentException(
"Unable to determine Unity Catalog for table "
+ catalogTable.identifier()
+ ": catalog name is missing. Use a fully-qualified table name with an explicit "
+ "catalog (e.g., catalog.schema.table).");
}
String catalogName = catalogOption.get();

// Get UC endpoint and token from Spark configs
scala.collection.immutable.Map<String, UCCatalogConfig> ucConfigs =
UCCommitCoordinatorBuilder$.MODULE$.getCatalogConfigMap(spark);

scala.Option<UCCatalogConfig> configOpt = ucConfigs.get(catalogName);

if (configOpt.isEmpty()) {
throw new IllegalArgumentException(
"Cannot create UC client for table "
+ catalogTable.identifier()
+ ": Unity Catalog configuration not found for catalog '"
+ catalogName
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

perhaps log the table identifier as well

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done, error message now includes the table identifier: "Cannot create UC client for table " + catalogTable.identifier() + ": ..."

+ "'.");
}

UCCatalogConfig config = configOpt.get();
String ucUri = config.uri();
String ucToken = config.token();

return Optional.of(new UCTableInfo(tableId, tablePath, ucUri, ucToken));
}

private static String extractUCTableId(CatalogTable catalogTable) {
Map<String, String> storageProperties =
scala.jdk.javaapi.CollectionConverters.asJava(catalogTable.storage().properties());

// TODO: UC constants should be consolidated in a shared location (future PR)
String ucTableId = storageProperties.get(UCCommitCoordinatorClient.UC_TABLE_ID_KEY);
if (ucTableId == null || ucTableId.isEmpty()) {
throw new IllegalArgumentException(
"Cannot extract ucTableId from table " + catalogTable.identifier());
}
return ucTableId;
}

private static String extractTablePath(CatalogTable catalogTable) {
if (catalogTable.location() == null) {
throw new IllegalArgumentException(
"Cannot extract table path: location is null for table " + catalogTable.identifier());
}
return catalogTable.location().toString();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
/*
* Copyright (2025) The Delta Lake Project Authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.delta.kernel.spark.snapshot.unitycatalog;

import static org.junit.jupiter.api.Assertions.assertEquals;

import org.junit.jupiter.api.Test;

/** Tests for {@link UCTableInfo}. */
class UCTableInfoTest {

@Test
void testConstructor_ValidInputs_StoresAllFields() {
// Use distinctive values that would fail if implementation had hardcoded defaults
String tableId = "uc_tbl_7f3a9b2c-e8d1-4f6a";
String tablePath = "abfss://[email protected]/delta/v2";
String ucUri = "https://uc-server.example.net/api/2.1/uc";
String ucToken = "dapi_Kx9mN$2pQr#7vWz";

UCTableInfo info = new UCTableInfo(tableId, tablePath, ucUri, ucToken);

assertEquals(tableId, info.getTableId(), "Table ID should be stored correctly");
assertEquals(tablePath, info.getTablePath(), "Table path should be stored correctly");
assertEquals(ucUri, info.getUcUri(), "UC URI should be stored correctly");
assertEquals(ucToken, info.getUcToken(), "UC token should be stored correctly");
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,15 @@
package io.delta.kernel.spark.utils;

import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;

import io.delta.storage.commit.uccommitcoordinator.UCCommitCoordinatorClient;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import org.apache.spark.sql.catalyst.catalog.CatalogTable;
import org.junit.jupiter.api.Test;
import scala.Option;

/** Tests for {@link CatalogTableUtils}. */
class CatalogTableUtilsTest {
Expand Down Expand Up @@ -97,22 +98,6 @@ void testIsUnityCatalogManaged_PreviewFlagMissingId_ReturnsFalse() {
"Preview flag without ID should not be considered Unity managed");
}

@Test
void testIsCatalogManaged_NullTable_ThrowsException() {
assertThrows(
NullPointerException.class,
() -> CatalogTableUtils.isCatalogManaged(null),
"Null table should throw NullPointerException");
}

@Test
void testIsUnityCatalogManaged_NullTable_ThrowsException() {
assertThrows(
NullPointerException.class,
() -> CatalogTableUtils.isUnityCatalogManagedTable(null),
"Null table should throw NullPointerException");
}

@Test
void testIsCatalogManaged_NullStorage_ReturnsFalse() {
CatalogTable table = catalogTableWithNullStorage(Collections.emptyMap());
Expand Down Expand Up @@ -151,15 +136,36 @@ void testIsUnityCatalogManaged_NullStorageProperties_ReturnsFalse() {

private static CatalogTable catalogTable(
Map<String, String> properties, Map<String, String> storageProperties) {
return CatalogTableTestUtils$.MODULE$.catalogTableWithProperties(properties, storageProperties);
return CatalogTableTestUtils$.MODULE$.createCatalogTable(
"tbl" /* tableName */,
Option.empty() /* catalogName */,
properties,
storageProperties,
Option.empty() /* locationUri */,
false /* nullStorage */,
false /* nullStorageProperties */);
}

private static CatalogTable catalogTableWithNullStorage(Map<String, String> properties) {
return CatalogTableTestUtils$.MODULE$.catalogTableWithNullStorage(properties);
return CatalogTableTestUtils$.MODULE$.createCatalogTable(
"tbl" /* tableName */,
Option.empty() /* catalogName */,
properties,
new HashMap<>() /* storageProperties */,
Option.empty() /* locationUri */,
true /* nullStorage */,
false /* nullStorageProperties */);
}

private static CatalogTable catalogTableWithNullStorageProperties(
Map<String, String> properties) {
return CatalogTableTestUtils$.MODULE$.catalogTableWithNullStorageProperties(properties);
return CatalogTableTestUtils$.MODULE$.createCatalogTable(
"tbl" /* tableName */,
Option.empty() /* catalogName */,
properties,
new HashMap<>() /* storageProperties */,
Option.empty() /* locationUri */,
false /* nullStorage */,
true /* nullStorageProperties */);
}
}
Loading
Loading