Load CDK: Expanded namespace coverage

airbytehq · Jan 22, 2025 · b7cf463 · b7cf463
1 parent 60d2d64
commit b7cf463
Show file tree

Hide file tree

Showing 4 changed files with 51 additions and 18 deletions.
diff --git a/...ad/src/testFixtures/kotlin/io/airbyte/cdk/load/write/BasicFunctionalityIntegrationTest.kt b/...ad/src/testFixtures/kotlin/io/airbyte/cdk/load/write/BasicFunctionalityIntegrationTest.kt
@@ -449,9 +449,9 @@ abstract class BasicFunctionalityIntegrationTest(
     @Test
     open fun testNamespaces() {
         assumeTrue(verifyDataWriting)
-        fun makeStream(namespace: String) =
+        fun makeStream(namespace: String?, name: String = "test_stream") =
             DestinationStream(
-                DestinationStream.Descriptor(namespace, "test_stream"),
+                DestinationStream.Descriptor(namespace, name),
                 Append,
                 ObjectType(linkedMapOf("id" to intType)),
                 generationId = 0,
@@ -460,12 +460,14 @@ abstract class BasicFunctionalityIntegrationTest(
             )
         val stream1 = makeStream(randomizedNamespace + "_1")
         val stream2 = makeStream(randomizedNamespace + "_2")
+        val streamWithNullNamespace = makeStream(null, randomizedNamespace + "_stream")
         runSync(
             configContents,
             DestinationCatalog(
                 listOf(
                     stream1,
                     stream2,
+                    streamWithNullNamespace,
                 )
             ),
             listOf(
@@ -481,6 +483,12 @@ abstract class BasicFunctionalityIntegrationTest(
                     data = """{"id": 5678}""",
                     emittedAtMs = 1234,
                 ),
+                InputRecord(
+                    namespace = streamWithNullNamespace.descriptor.namespace,
+                    name = streamWithNullNamespace.descriptor.name,
+                    data = """{"id": 91011}""",
+                    emittedAtMs = 1234,
+                ),
             )
         )
         assertAll(
@@ -515,6 +523,22 @@ abstract class BasicFunctionalityIntegrationTest(
                     listOf(listOf("id")),
                     cursor = null
                 )
+            },
+            {
+                dumpAndDiffRecords(
+                    parsedConfig,
+                    listOf(
+                        OutputRecord(
+                            extractedAt = 1234,
+                            generationId = 0,
+                            data = mapOf("id" to 91011),
+                            airbyteMeta = OutputRecord.Meta(syncId = 42)
+                        )
+                    ),
+                    streamWithNullNamespace,
+                    listOf(listOf("id")),
+                    cursor = null
+                )
             }
         )
     }
@@ -525,9 +549,10 @@ abstract class BasicFunctionalityIntegrationTest(
         fun makeStream(
             name: String,
             schema: LinkedHashMap<String, FieldType> = linkedMapOf("id" to intType),
+            namespaceSuffix: String = "",
         ) =
             DestinationStream(
-                DestinationStream.Descriptor(randomizedNamespace, name),
+                DestinationStream.Descriptor(randomizedNamespace + namespaceSuffix, name),
                 Append,
                 ObjectType(schema),
                 generationId = 0,
@@ -569,6 +594,11 @@ abstract class BasicFunctionalityIntegrationTest(
                         "groups",
                         linkedMapOf("id" to intType, "authorization" to stringType)
                     ),
+                    makeStream(
+                        "streamWithSpecialCharactersInNamespace",
+                        namespaceSuffix = "_spøcial"
+                    ),
+                    makeStream("streamWithOperatorInNamespace", namespaceSuffix = "_operator-1"),
                 )
             )
         // For each stream, generate a record containing every field in the schema.

diff --git a/...ad-avro/src/testFixtures/kotlin/io/airbyte/cdk/load/data/avro/AvroExpectedRecordMapper.kt b/...ad-avro/src/testFixtures/kotlin/io/airbyte/cdk/load/data/avro/AvroExpectedRecordMapper.kt
@@ -34,7 +34,10 @@ object AvroExpectedRecordMapper : ExpectedRecordMapper {
     ): DestinationStream.Descriptor {
         // Map the special character but not the '+', because only the former is replaced in file
         // paths.
-        return descriptor.copy(name = descriptor.name.replace("é", "e"))
+        return descriptor.copy(
+            namespace = descriptor.namespace?.replace("ø", "o"),
+            name = descriptor.name.replace("é", "e")
+        )
     }
 
     private fun fieldNameMangler(value: AirbyteValue): AirbyteValue =

diff --git a/...ntegration/kotlin/io/airbyte/integrations/destination/s3_data_lake/S3DataLakeWriteTest.kt b/...ntegration/kotlin/io/airbyte/integrations/destination/s3_data_lake/S3DataLakeWriteTest.kt
@@ -112,6 +112,12 @@ class GlueWriteTest :
     override fun testFunkyCharacters() {
         super.testFunkyCharacters()
     }
+
+    @Test
+    @Disabled("https://github.com/airbytehq/airbyte-internal-issues/issues/11439")
+    override fun testNamespaces() {
+        super.testNamespaces()
+    }
 }
 
 class GlueAssumeRoleWriteTest :
@@ -126,12 +132,16 @@ class GlueAssumeRoleWriteTest :
         S3DataLakeTestUtil.getAWSSystemCredentialsAsMap()
     ) {
     @Test
-    @Disabled(
-        "https://github.com/airbytehq/airbyte-internal-issues/issues/11439"
-    )
+    @Disabled("https://github.com/airbytehq/airbyte-internal-issues/issues/11439")
     override fun testFunkyCharacters() {
         super.testFunkyCharacters()
     }
+
+    @Test
+    @Disabled("https://github.com/airbytehq/airbyte-internal-issues/issues/11439")
+    override fun testNamespaces() {
+        super.testNamespaces()
+    }
 }
 
 @Disabled(

diff --git a/...s3/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_v2/S3V2WriteTest.kt b/...s3/src/test-integration/kotlin/io/airbyte/integrations/destination/s3_v2/S3V2WriteTest.kt
@@ -177,17 +177,7 @@ class S3V2WriteTestAvroUncompressed :
         allTypesBehavior = StronglyTyped(integerCanBeLarge = false),
         nullEqualsUnset = true,
         nullUnknownTypes = true,
-    ) {
-    @Test
-    override fun testUnknownTypes() {
-        super.testUnknownTypes()
-    }
-
-    @Test
-    override fun testFunkyCharacters() {
-        super.testFunkyCharacters()
-    }
-}
+    )
 
 class S3V2WriteTestAvroBzip2 :
     S3V2WriteTest(