Skip to content

Commit 8e6d4b6

Browse files
murali-dbmurali-db
authored
[Kernel][#1] Implement table features and tests for iceberg writer compat v3 (delta-io#4733)
<!-- Thanks for sending a pull request! Here are some tips for you: 1. If this is your first time, please read our contributor guidelines: https://github.com/delta-io/delta/blob/master/CONTRIBUTING.md 2. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP] Your PR title ...'. 3. Be sure to keep the PR description updated to reflect all changes. 4. Please write your PR title to summarize what this PR proposes. 5. If possible, provide a concise example to reproduce the issue for a faster review. 6. If applicable, include the corresponding issue number in the PR title and link it in the body. --> #### Which Delta project/connector is this regarding? <!-- Please add the component selected below to the beginning of the pull request title For example: [Spark] Title of my pull request --> - [x] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [ ] Other (fill in here) ## Description <!-- - Describe what this PR changes. - Describe why we need the change. If this PR resolves an issue be sure to include "Resolves #XXX" to correctly link and close the issue upon merge. --> Add IcebergWriterCompatV3 table feature and unit tests. ## How was this patch tested? <!-- If tests were added, say they were added here. Please make sure to test the changes thoroughly including negative and positive cases if possible. If the changes were tested in any way other than unit tests, please clarify how you tested step by step (ideally copy and paste-able, so that other reviewers can test and check, and descendants can verify in the future). If the changes were not tested, please explain why. --> Unit tests. ## Does this PR introduce _any_ user-facing changes? <!-- If yes, please clarify the previous behavior and the change this PR proposes - provide the console output, description and/or an example to show the behavior difference if possible. If possible, please also clarify if this is a user-facing change compared to the released Delta Lake versions or within the unreleased branches such as master. If no, write 'No'. -->
1 parent ca288fd commit 8e6d4b6

File tree

3 files changed

+117
-1
lines changed

3 files changed

+117
-1
lines changed

‎kernel/kernel-api/src/main/java/io/delta/kernel/internal/TableConfig.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -285,6 +285,20 @@ public class TableConfig<T> {
285285
"needs to be a boolean.",
286286
true);
287287

288+
/**
289+
* Table property that enables modifying the table in accordance with the Delta-Iceberg Writer
290+
* Compatibility V3 ({@code icebergCompatWriterV3}) protocol. V2 is skipped to align with the
291+
* iceberg v3 spec.
292+
*/
293+
public static final TableConfig<Boolean> ICEBERG_WRITER_COMPAT_V3_ENABLED =
294+
new TableConfig<>(
295+
"delta.enableIcebergWriterCompatV3",
296+
"false",
297+
Boolean::valueOf,
298+
value -> true,
299+
"needs to be a boolean.",
300+
true);
301+
288302
public static class UniversalFormats {
289303

290304
/**
@@ -372,6 +386,7 @@ public static class UniversalFormats {
372386
addConfig(this, ICEBERG_COMPAT_V2_ENABLED);
373387
addConfig(this, ICEBERG_COMPAT_V3_ENABLED);
374388
addConfig(this, ICEBERG_WRITER_COMPAT_V1_ENABLED);
389+
addConfig(this, ICEBERG_WRITER_COMPAT_V3_ENABLED);
375390
addConfig(this, COLUMN_MAPPING_MAX_COLUMN_ID);
376391
addConfig(this, DATA_SKIPPING_NUM_INDEXED_COLS);
377392
addConfig(this, UNIVERSAL_FORMAT_ENABLED_FORMATS);

‎kernel/kernel-api/src/main/java/io/delta/kernel/internal/tablefeatures/TableFeatures.java‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -452,6 +452,24 @@ public boolean metadataRequiresFeatureToBeEnabled(Protocol protocol, Metadata me
452452
}
453453
}
454454

455+
public static final TableFeature ICEBERG_WRITER_COMPAT_V3 = new IcebergWriterCompatV3();
456+
457+
private static class IcebergWriterCompatV3 extends TableFeature.WriterFeature
458+
implements FeatureAutoEnabledByMetadata {
459+
IcebergWriterCompatV3() {
460+
super("icebergWriterCompatV3", /* minWriterVersion = */ 7);
461+
}
462+
463+
@Override
464+
public boolean metadataRequiresFeatureToBeEnabled(Protocol protocol, Metadata metadata) {
465+
return TableConfig.ICEBERG_WRITER_COMPAT_V3_ENABLED.fromMetadata(metadata);
466+
}
467+
468+
public @Override Set<TableFeature> requiredFeatures() {
469+
return Collections.singleton(ICEBERG_COMPAT_V3_W_FEATURE);
470+
}
471+
}
472+
455473
/////////////////////////////////////////////////////////////////////////////////
456474
/// END: Define the {@link TableFeature}s ///
457475
/////////////////////////////////////////////////////////////////////////////////
@@ -490,7 +508,8 @@ public boolean metadataRequiresFeatureToBeEnabled(Protocol protocol, Metadata me
490508
VARIANT_RW_FEATURE,
491509
VARIANT_RW_PREVIEW_FEATURE,
492510
VARIANT_SHREDDING_PREVIEW_RW_FEATURE,
493-
ICEBERG_WRITER_COMPAT_V1));
511+
ICEBERG_WRITER_COMPAT_V1,
512+
ICEBERG_WRITER_COMPAT_V3));
494513

495514
public static final Map<String, TableFeature> TABLE_FEATURE_MAP =
496515
Collections.unmodifiableMap(

‎kernel/kernel-api/src/test/scala/io/delta/kernel/internal/tablefeatures/TableFeaturesSuite.scala‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ class TableFeaturesSuite extends AnyFunSuite {
6464
"icebergCompatV3",
6565
"inCommitTimestamp",
6666
"icebergWriterCompatV1",
67+
"icebergWriterCompatV3",
6768
"clustering")
6869

6970
val legacyFeatures = Seq(
@@ -177,6 +178,14 @@ class TableFeaturesSuite extends AnyFunSuite {
177178
(
178179
"icebergWriterCompatV1",
179180
testMetadata(tblProps = Map("delta.enableIcebergWriterCompatV1" -> "false")),
181+
false),
182+
(
183+
"icebergWriterCompatV3",
184+
testMetadata(tblProps = Map("delta.enableIcebergWriterCompatV3" -> "true")),
185+
true),
186+
(
187+
"icebergWriterCompatV3",
188+
testMetadata(tblProps = Map("delta.enableIcebergWriterCompatV3" -> "false")),
180189
false)).foreach({ case (feature, metadata, expected) =>
181190
test(s"metadataRequiresFeatureToBeEnabled - $feature - $metadata") {
182191
val tableFeature = TableFeatures.getTableFeature(feature)
@@ -259,6 +268,7 @@ class TableFeaturesSuite extends AnyFunSuite {
259268
"typeWidening-preview",
260269
"typeWidening",
261270
"icebergWriterCompatV1",
271+
"icebergWriterCompatV3",
262272
"clustering")
263273

264274
assert(results.map(_.featureName()).toSet == expected.toSet)
@@ -594,6 +604,11 @@ class TableFeaturesSuite extends AnyFunSuite {
594604
new Protocol(3, 7, emptySet(), singleton("icebergWriterCompatV1")),
595605
testMetadata(tblProps = Map("delta.enableIcebergWriterCompatV1" -> "true")))
596606

607+
checkWriteSupported(
608+
"validateKernelCanWriteToTable: protocol 7 with icebergWriterCompatV3",
609+
new Protocol(3, 7, emptySet(), singleton("icebergWriterCompatV3")),
610+
testMetadata(tblProps = Map("delta.enableIcebergWriterCompatV3" -> "true")))
611+
597612
checkWriteSupported(
598613
"validateKernelCanWriteToTable: protocol 7 with clustering",
599614
new Protocol(3, 7, emptySet(), singleton("clustering")),
@@ -878,6 +893,73 @@ class TableFeaturesSuite extends AnyFunSuite {
878893
"icebergCompatV2",
879894
"icebergWriterCompatV1")),
880895
set("icebergCompatV2", "columnMapping", "icebergWriterCompatV1")),
896+
(
897+
testMetadata(tblProps = Map("delta.enableIcebergWriterCompatV3" -> "true")),
898+
new Protocol(1, 2),
899+
new Protocol(
900+
2,
901+
7,
902+
set(),
903+
set(
904+
"columnMapping",
905+
"appendOnly",
906+
"invariants",
907+
"icebergCompatV3",
908+
"icebergWriterCompatV3",
909+
"domainMetadata",
910+
"rowTracking")),
911+
set(
912+
"icebergCompatV3",
913+
"columnMapping",
914+
"icebergWriterCompatV3",
915+
"domainMetadata",
916+
"rowTracking")),
917+
(
918+
testMetadata(tblProps = Map(
919+
"delta.enableIcebergWriterCompatV3" -> "true",
920+
"delta.enableDeletionVectors" -> "true")),
921+
new Protocol(2, 5),
922+
new Protocol(
923+
3,
924+
7,
925+
set("columnMapping", "deletionVectors"),
926+
set(
927+
"columnMapping",
928+
"appendOnly",
929+
"deletionVectors",
930+
"invariants",
931+
"icebergCompatV3",
932+
"icebergWriterCompatV3",
933+
"checkConstraints",
934+
"generatedColumns",
935+
"changeDataFeed",
936+
"domainMetadata",
937+
"rowTracking")),
938+
set(
939+
"icebergCompatV3",
940+
"icebergWriterCompatV3",
941+
"deletionVectors",
942+
"domainMetadata",
943+
"rowTracking")),
944+
(
945+
testMetadata(tblProps = Map("delta.enableIcebergWriterCompatV3" -> "true")),
946+
new Protocol(1, 1), // Minimal starting protocol with no features
947+
new Protocol(
948+
2,
949+
7,
950+
set(),
951+
set(
952+
"columnMapping",
953+
"icebergCompatV3", // Added as dependency
954+
"icebergWriterCompatV3",
955+
"domainMetadata",
956+
"rowTracking")),
957+
set(
958+
"icebergCompatV3",
959+
"columnMapping",
960+
"icebergWriterCompatV3",
961+
"domainMetadata",
962+
"rowTracking")),
881963
(
882964
testMetadata(tblProps = Map(
883965
"delta.enableIcebergWriterCompatV1" -> "true",

0 commit comments

Comments
 (0)