Support nullable and non-nullable types with same name in the same template

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/typing/TypeRepository.java

@@ -23,8 +23,6 @@
 import com.apple.foundationdb.record.TupleFieldsProto;
 import com.google.common.base.Preconditions;
 import com.google.common.base.Verify;
-import com.google.common.collect.BiMap;
-import com.google.common.collect.HashBiMap;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
 import com.google.protobuf.DescriptorProtos;
@@ -435,13 +433,17 @@
     public static class Builder {
         private @Nonnull final FileDescriptorProto.Builder fileDescProtoBuilder;
         private @Nonnull final FileDescriptorSet.Builder fileDescSetBuilder;
-        private @Nonnull final BiMap<Type, String> typeToNameMap;
+        private @Nonnull final Map<Type, String> typeToNameMap;
+        private @Nonnull final Map<String, Type> nameToCanonicalTypeMap;
+        private @Nonnull final Set<String> typesWithBothNullabilityVariants;
 
         private Builder() {
             fileDescProtoBuilder = FileDescriptorProto.newBuilder();
             fileDescProtoBuilder.addAllDependency(DEPENDENCIES.stream().map(FileDescriptor::getFullName).collect(Collectors.toList()));
             fileDescSetBuilder = FileDescriptorSet.newBuilder();
-            typeToNameMap = HashBiMap.create();
+            typeToNameMap = new HashMap<>();
+            nameToCanonicalTypeMap = new HashMap<>();
+            typesWithBothNullabilityVariants = new HashSet<>();
         }
 
         @Nonnull
@@ -471,20 +473,47 @@
         @Nonnull
         public Builder addTypeIfNeeded(@Nonnull final Type type) {
             if (!typeToNameMap.containsKey(type)) {
+                // Check if we have a structurally identical type with different nullability already registered
+                Type canonicalType = findCanonicalTypeForStructure(type);
+                if (canonicalType != null) {
+                    // Use the same protobuf name as the canonical type
+                    String existingProtoName = typeToNameMap.get(canonicalType);
+                    if (existingProtoName != null) {
+                        typesWithBothNullabilityVariants.add(existingProtoName);
+                        typeToNameMap.put(type, existingProtoName);
+                        return this;
+                    }
+                }
+
+                // Standard case: define the protobuf type
                 type.defineProtoType(this);
             }
             return this;
         }
 
+        /**
+         * Finds a type that has the same structure as the given type but different nullability.
+         * Returns null if no such type exists.
+         */
+        private Type findCanonicalTypeForStructure(@Nonnull final Type type) {
+            for (Map.Entry<Type, String> entry : typeToNameMap.entrySet()) {
+                Type existingType = entry.getKey();
+                if (differsOnlyInNullability(type, existingType)) {
+                    return existingType;
+                }
+            }
+            return null;
+        }
+
         @Nonnull
         public Optional<String> getTypeName(@Nonnull final Type type) {
             return Optional.ofNullable(typeToNameMap.get(type));
         }
 
         @Nonnull
         public Optional<Type> getTypeByName(@Nonnull final String name) {
-            return Optional.ofNullable(typeToNameMap.inverse().get(name));
+            return Optional.ofNullable(nameToCanonicalTypeMap.get(name));
         }
 
         @Nonnull
         public Builder addMessageType(@Nonnull final DescriptorProtos.DescriptorProto descriptorProto) {
@@ -500,11 +529,55 @@
 
         @Nonnull
         public Builder registerTypeToTypeNameMapping(@Nonnull final Type type, @Nonnull final String protoTypeName) {
-            Verify.verify(!typeToNameMap.containsKey(type));
+            final String existingTypeName = typeToNameMap.get(type);
+            if (existingTypeName != null) {
+                // Type already registered, verify same protobuf name
+                Verify.verify(existingTypeName.equals(protoTypeName), "Type %s is already registered with name %s, cannot register with different name %s", type, existingTypeName, protoTypeName);
+                return this;
+            }
+
+            // Check if a type with same structure but different nullability is already registered
+            final Type existingTypeForName = nameToCanonicalTypeMap.get(protoTypeName);
+            if (existingTypeForName != null && differsOnlyInNullability(type, existingTypeForName)) {
+                // Allow both nullable and non-nullable variants to map to the same protobuf type
+                // Don't update nameToCanonicalTypeMap - keep the first registered type as canonical
+                typeToNameMap.put(type, protoTypeName);
+                return this;
+            }
+
+            // Standard case: new type with new name
             typeToNameMap.put(type, protoTypeName);
+            nameToCanonicalTypeMap.put(protoTypeName, type);
             return this;
         }
 
+        /**
+         * Checks if two types differ only in their nullability setting.
+         * This is used to allow both nullable and non-nullable variants of the same structural type
+         * to map to the same protobuf type name.
+         */
+        private boolean differsOnlyInNullability(@Nonnull final Type type1, @Nonnull final Type type2) {
+            if (type1.equals(type2)) {
+                return false; // Same type, doesn't differ
+            }
+
+            // Handle Type.Null specially - it can only be nullable, so it can't have a non-nullable variant
+            if (type1 instanceof Type.Null || type2 instanceof Type.Null) {
+                return false; // Type.Null can't have non-nullable variants
+            }
+
+            // Check if they have different nullability
+            if (type1.isNullable() == type2.isNullable()) {
+                return false; // Same nullability, so they differ in structure
+            }
+
+            // Create non-nullable versions to compare structure
+            final Type nonNullable1 = type1.isNullable() ? type1.notNullable() : type1;
+            final Type nonNullable2 = type2.isNullable() ? type2.notNullable() : type2;
+
+            return nonNullable1.equals(nonNullable2);
+        }
+
         @Nonnull
         public Builder addAllTypes(@Nonnull final Collection<Type> types) {
             types.forEach(this::addTypeIfNeeded);

fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java

@@ -366,6 +366,62 @@
         visitor.finishVisit(this);
     }
 
+    /**
+     * Manages nullable and non-nullable variants of a struct type with the same name.
+     * This allows a struct type to exist in both nullable (for table columns) and 
+     * non-nullable (for array elements) forms within the same schema template.
+     */
+    private static final class TypeVariants {
+        private DataType.Named nullableVariant;
+        private DataType.Named nonNullableVariant;
+
+        /**
+         * Adds or updates a type variant based on its nullability.
+         * 
+         * @param type The type to add/update
+         */
+        public void addVariant(@Nonnull DataType.Named type) {
+            // Cast to DataType to access isNullable() method
+            // All named types (StructType, EnumType, etc.) extend DataType and implement Named
+            DataType dataType = (DataType) type;
+            if (dataType.isNullable()) {
+                this.nullableVariant = type;
+            } else {
+                this.nonNullableVariant = type;
+            }
+        }
+
+        /**
+         * Gets the primary variant (nullable if available, otherwise non-nullable).
+         * This maintains backward compatibility by preferring nullable variants.
+         */
+        @Nonnull
+        public DataType.Named getPrimaryVariant() {
+            if (nullableVariant != null) {
+                return nullableVariant;
+            }
+            if (nonNullableVariant != null) {
+                return nonNullableVariant;
+            }
+            throw new RelationalException("No variants available", ErrorCode.INTERNAL_ERROR).toUncheckedWrappedException();
+        }
+
+        /**
+         * Gets all non-null variants.
+         */
+        @Nonnull
+        public Collection<DataType.Named> getAllVariants() {
+            final var variants = new ArrayList<DataType.Named>();
+            if (nullableVariant != null) {
+                variants.add(nullableVariant);
+            }
+            if (nonNullableVariant != null) {
+                variants.add(nonNullableVariant);
+            }
+            return variants;
+        }
+    }
+
     public static final class Builder {
         private static final String TABLE_ALREADY_EXISTS = "table '%s' already exists";
         private static final String TYPE_WITH_NAME_ALREADY_EXISTS = "type with name '%s' already exists";
@@ -384,7 +440,7 @@
         private final Map<String, RecordLayerTable> tables;
 
         @Nonnull
-        private final Map<String, DataType.Named> auxiliaryTypes; // for quick lookup
+        private final Map<String, TypeVariants> auxiliaryTypes; // for quick lookup
 
         @Nonnull
         private final Map<String, RecordLayerInvokedRoutine> invokedRoutines;
@@ -490,16 +546,27 @@
         /**
          * Adds an auxiliary type, an auxiliary type is a type that is merely created, so it can be referenced later on
          * in a table definition. Any {@link DataType.Named} data type can be added as an auxiliary type such as {@code enum}s
-         * and {@code struct}s.
+         * and {@code struct}s. For struct types, this method supports adding both nullable and non-nullable
+         * variants of the same named type.
          *
          * @param auxiliaryType The auxiliary {@link DataType} to add.
          * @return {@code this} {@link Builder}.
          */
         @Nonnull
         public Builder addAuxiliaryType(@Nonnull DataType.Named auxiliaryType) {
             Assert.thatUnchecked(!tables.containsKey(auxiliaryType.getName()), ErrorCode.INVALID_SCHEMA_TEMPLATE, TABLE_ALREADY_EXISTS, auxiliaryType.getName());
-            Assert.thatUnchecked(!auxiliaryTypes.containsKey(auxiliaryType.getName()), ErrorCode.INVALID_SCHEMA_TEMPLATE, TYPE_WITH_NAME_ALREADY_EXISTS, auxiliaryType.getName());
-            auxiliaryTypes.put(auxiliaryType.getName(), auxiliaryType);
+
+            // For struct types, allow both nullable and non-nullable variants
+            if (auxiliaryType instanceof DataType.StructType) {
+                TypeVariants variants = auxiliaryTypes.computeIfAbsent(auxiliaryType.getName(), k -> new TypeVariants());
+                variants.addVariant(auxiliaryType);
+            } else {
+                // For non-struct types (enums, etc.), maintain existing behavior
+                Assert.thatUnchecked(!auxiliaryTypes.containsKey(auxiliaryType.getName()), ErrorCode.INVALID_SCHEMA_TEMPLATE, TYPE_WITH_NAME_ALREADY_EXISTS, auxiliaryType.getName());
+                TypeVariants variants = new TypeVariants();
+                variants.addVariant(auxiliaryType);
+                auxiliaryTypes.put(auxiliaryType.getName(), variants);
+            }
             return this;
         }
 
@@ -543,7 +610,9 @@
             }
 
             if (auxiliaryTypes.containsKey(name)) {
-                return Optional.of((DataType) auxiliaryTypes.get(name));
+                // Return the primary variant (nullable if available, otherwise non-nullable)
+                // This maintains backward compatibility
+                return Optional.of((DataType) auxiliaryTypes.get(name).getPrimaryVariant());
             }
 
             return Optional.empty();
@@ -563,9 +632,14 @@
             }
 
             if (!needsResolution) {
-                for (final var auxiliaryType : auxiliaryTypes.values()) {
-                    if (!((DataType) auxiliaryType).isResolved()) {
-                        needsResolution = true;
+                for (final var typeVariants : auxiliaryTypes.values()) {
+                    for (final var variant : typeVariants.getAllVariants()) {
+                        if (!((DataType) variant).isResolved()) {
+                            needsResolution = true;
+                            break;
+                        }
+                    }
+                    if (needsResolution) {
                         break;
                     }
                 }
@@ -590,8 +664,11 @@
             for (final var table : tables.values()) {
                 mapBuilder.put(table.getName(), table.getDatatype());
             }
-            for (final var auxiliaryType : auxiliaryTypes.entrySet()) {
-                mapBuilder.put(auxiliaryType.getKey(), (DataType) auxiliaryType.getValue());
+            for (final var auxiliaryTypeEntry : auxiliaryTypes.entrySet()) {
+                // For each type name, add the primary variant to the map for backward compatibility
+                // This ensures existing type resolution logic works
+                TypeVariants typeVariants = auxiliaryTypeEntry.getValue();
+                mapBuilder.put(auxiliaryTypeEntry.getKey(), (DataType) typeVariants.getPrimaryVariant());
             }
             final var namedTypes = mapBuilder.build();
 
@@ -600,8 +677,12 @@
             for (final var table : tables.values()) {
                 depsBuilder.put(table.getDatatype(), getDependencies(table.getDatatype(), namedTypes));
             }
-            for (final var auxiliaryType : auxiliaryTypes.entrySet()) {
-                depsBuilder.put((DataType) auxiliaryType.getValue(), getDependencies((DataType) auxiliaryType.getValue(), namedTypes));
+            for (final var auxiliaryTypeEntry : auxiliaryTypes.entrySet()) {
+                TypeVariants typeVariants = auxiliaryTypeEntry.getValue();
+                // Add dependencies for all variants of this type
+                for (final var variant : typeVariants.getAllVariants()) {
+                    depsBuilder.put((DataType) variant, getDependencies((DataType) variant, namedTypes));
+                }
             }
             final var deps = depsBuilder.build();
 
@@ -640,14 +721,26 @@
             tables.clear();
             tables.putAll(resolvedTables.build());
 
-            final var resolvedAuxiliaryTypes = ImmutableMap.<String, DataType.Named>builder();
-            for (final var auxiliaryType : auxiliaryTypes.entrySet()) {
-                final var dataType = (DataType) auxiliaryType.getValue();
-                if (!dataType.isResolved()) {
-                    resolvedAuxiliaryTypes.put(auxiliaryType.getKey(), (DataType.Named) ((DataType) resolvedTypes.get(auxiliaryType.getKey())).withNullable(dataType.isNullable()));
-                } else {
-                    resolvedAuxiliaryTypes.put(auxiliaryType.getKey(), auxiliaryType.getValue());
+            final var resolvedAuxiliaryTypes = ImmutableMap.<String, TypeVariants>builder();
+            for (final var auxiliaryTypeEntry : auxiliaryTypes.entrySet()) {
+                final String typeName = auxiliaryTypeEntry.getKey();
+                final TypeVariants typeVariants = auxiliaryTypeEntry.getValue();
+
+                TypeVariants resolvedVariants = new TypeVariants();
+
+                // Resolve each variant
+                for (final var variant : typeVariants.getAllVariants()) {
+                    DataType.Named resolvedVariant;
+                    DataType variantAsDataType = (DataType) variant;
+                    if (!variantAsDataType.isResolved()) {
+                        resolvedVariant = (DataType.Named) ((DataType) resolvedTypes.get(typeName)).withNullable(variantAsDataType.isNullable());
+                    } else {
+                        resolvedVariant = variant;
+                    }
+                    resolvedVariants.addVariant(resolvedVariant);
                 }
+
+                resolvedAuxiliaryTypes.put(typeName, resolvedVariants);
             }
 
             auxiliaryTypes.clear();

fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java

@@ -1180,18 +1180,6 @@ void testNamingStructsSameType() throws Exception {
         }
     }
 
-    @Test
-    void testNamingStructsDifferentTypesThrows() throws Exception {
-        final String schemaTemplate = "CREATE TABLE T1(pk bigint, a bigint, b bigint, c bigint, PRIMARY KEY(pk))";
-        try (var ddl = Ddl.builder().database(URI.create("/TEST/QT")).relationalExtension(relationalExtension).schemaTemplate(schemaTemplate).build()) {
-            try (var statement = ddl.setSchemaAndGetConnection().createStatement()) {
-                statement.executeUpdate("insert into t1 values (42, 100, 500, 101)");
-                final var message = Assertions.assertThrows(SQLException.class, () -> statement.execute("select struct asd (a, 42, struct def (b, c), struct def(b, c, a)) as X from t1")).getMessage();
-                Assertions.assertTrue(message.contains("value already present: DEF")); // we could improve this error message.
-            }
-        }
-    }
-
     @Test
     void testNamingStructsSameTypeDifferentNestingLevels() throws Exception {
         final String schemaTemplate = "CREATE TABLE T1(pk bigint, a bigint, b bigint, c bigint, PRIMARY KEY(pk))";

yaml-tests/src/test/java/YamlIntegrationTests.java

@@ -226,6 +226,11 @@ void arrays(YamlTest.Runner runner) throws Exception {
         runner.runYamsql("arrays.yamsql");
     }
 
+    @TestTemplate
+    public void structTypeVariants(YamlTest.Runner runner) throws Exception {
+        runner.runYamsql("struct-type-variants.yamsql");
+    }
+
     @TestTemplate
     public void insertEnum(YamlTest.Runner runner) throws Exception {
         runner.runYamsql("insert-enum.yamsql");