From 35816ec0bab5b720851455c511651a18dc711270 Mon Sep 17 00:00:00 2001 From: Marcia Schulman Date: Tue, 23 Apr 2024 13:51:07 -0400 Subject: [PATCH 01/11] use DefaultAzureCredential --- .../fn-receiver-debatcher/pom.xml | 19 ++++++++------- .../kotlin/gov/cdc/dex/hl7/FunctionConfig.kt | 23 +++++++++++-------- local_libs/lib-dex-commons/build.gradle.kts | 3 ++- .../gov/cdc/dex/azure/AzureBlobProxy.kt | 7 ++++-- .../cdc/dex/azure/DedicatedEventHubSender.kt | 6 +++-- .../kotlin/gov/cdc/dex/TestAzureBlobProxy.kt | 18 +++++++++++++++ .../kotlin/test/gov/cdc/dex/MetadataTest.kt | 6 ++--- 7 files changed, 55 insertions(+), 27 deletions(-) create mode 100644 local_libs/lib-dex-commons/src/test/kotlin/gov/cdc/dex/TestAzureBlobProxy.kt diff --git a/fns-hl7-pipeline/fn-receiver-debatcher/pom.xml b/fns-hl7-pipeline/fn-receiver-debatcher/pom.xml index 8188def12..09e737cb5 100644 --- a/fns-hl7-pipeline/fn-receiver-debatcher/pom.xml +++ b/fns-hl7-pipeline/fn-receiver-debatcher/pom.xml @@ -5,7 +5,7 @@ 4.0.0 gov.cdc.dataexchange receiver-debatcher - 0.0.43-SNAPSHOT + 0.0.47-SNAPSHOT jar DEX Fns-Pipeline :: receiver-debatcher @@ -25,7 +25,7 @@ ${project.build.directory}/azure-functions/${functionAppName} 2.10.1 1.2.7.1 - 0.0.46-SNAPSHOT + 0.0.47-SNAPSHOT 5.9.2 @@ -82,26 +82,25 @@ com.azure azure-messaging-eventhubs - - - com.google.code.gson - gson - ${gson.version} + com.azure + azure-identity - - com.azure azure-storage-blob + + com.google.code.gson + gson + ${gson.version} + org.junit.jupiter junit-jupiter ${junitjupiter.version} test - com.googlecode.json-simple json-simple diff --git a/fns-hl7-pipeline/fn-receiver-debatcher/src/main/kotlin/gov/cdc/dex/hl7/FunctionConfig.kt b/fns-hl7-pipeline/fn-receiver-debatcher/src/main/kotlin/gov/cdc/dex/hl7/FunctionConfig.kt index cbea085af..06b69a190 100644 --- a/fns-hl7-pipeline/fn-receiver-debatcher/src/main/kotlin/gov/cdc/dex/hl7/FunctionConfig.kt +++ b/fns-hl7-pipeline/fn-receiver-debatcher/src/main/kotlin/gov/cdc/dex/hl7/FunctionConfig.kt @@ -1,5 +1,7 @@ package gov.cdc.dex.hl7 +import com.azure.identity.DefaultAzureCredential +import com.azure.identity.DefaultAzureCredentialBuilder import gov.cdc.dex.azure.AzureBlobProxy import gov.cdc.dex.azure.DedicatedEventHubSender import org.slf4j.Logger @@ -9,7 +11,10 @@ class FunctionConfig { val azBlobProxy: AzureBlobProxy val evHubSenderOut: DedicatedEventHubSender val evHubSenderReports: DedicatedEventHubSender + private val tokenCredential : DefaultAzureCredential = DefaultAzureCredentialBuilder().build() private val logger : Logger = LoggerFactory.getLogger(this.javaClass.simpleName) + + val blobIngestContName: String = try { System.getenv("BlobIngestContainerName") } catch (e: NullPointerException) { @@ -31,21 +36,21 @@ class FunctionConfig { init { //Init Azure Storage connection - val ingestBlobConnStr = try { - System.getenv("BlobIngestConnectionString") + val ingestBlobUrl = try { + System.getenv("BlobIngestUrl") } catch (e: NullPointerException) { - logger.error("FATAL: Missing environment variable BlobIngestConnectionString") + logger.error("FATAL: Missing environment variable BlobIngestUrl") throw e } - azBlobProxy = AzureBlobProxy(ingestBlobConnStr, blobIngestContName) - val evHubConnStr = try { - System.getenv("EventHubConnectionString") + azBlobProxy = AzureBlobProxy(ingestBlobUrl, blobIngestContName, tokenCredential) + val evHubNamespace = try { + System.getenv("EventHubNamespaceUrl") } catch (e: NullPointerException) { - logger.error("FATAL: Missing environment variable EventHubConnectionString") + logger.error("FATAL: Missing environment variable EventHubNamespaceUrl") throw e } - evHubSenderOut = DedicatedEventHubSender(evHubConnStr, evHubSendName) - evHubSenderReports = DedicatedEventHubSender(evHubConnStr, evReportsHubName) + evHubSenderOut = DedicatedEventHubSender(evHubNamespace, evHubSendName, tokenCredential) + evHubSenderReports = DedicatedEventHubSender(evHubNamespace, evReportsHubName, tokenCredential) } } \ No newline at end of file diff --git a/local_libs/lib-dex-commons/build.gradle.kts b/local_libs/lib-dex-commons/build.gradle.kts index c75519571..844a98ecd 100644 --- a/local_libs/lib-dex-commons/build.gradle.kts +++ b/local_libs/lib-dex-commons/build.gradle.kts @@ -9,7 +9,7 @@ plugins { } group = "gov.cdc.dex" -version = "0.0.46-SNAPSHOT" +version = "0.0.47-SNAPSHOT" repositories { maven { @@ -27,6 +27,7 @@ dependencies { implementation("com.azure:azure-messaging-eventhubs") implementation("com.azure:azure-messaging-servicebus") implementation("com.azure:azure-storage-blob") + implementation("com.azure:azure-identity") implementation("redis.clients:jedis:5.1.0") implementation("com.azure:azure-cosmos:4.55.1") diff --git a/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/azure/AzureBlobProxy.kt b/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/azure/AzureBlobProxy.kt index 5874bd99f..d32bf02d9 100644 --- a/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/azure/AzureBlobProxy.kt +++ b/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/azure/AzureBlobProxy.kt @@ -1,15 +1,18 @@ package gov.cdc.dex.azure + +import com.azure.identity.DefaultAzureCredential import com.azure.storage.blob.BlobClient import com.azure.storage.blob.BlobContainerClient import com.azure.storage.blob.BlobServiceClientBuilder -class AzureBlobProxy(connectionStr: String, container: String) { +class AzureBlobProxy(blobStorageUrl: String, container: String, tokenCredential: DefaultAzureCredential) { private val blobContainerClient: BlobContainerClient init { blobContainerClient = BlobServiceClientBuilder() - .connectionString(connectionStr) + .endpoint(blobStorageUrl) + .credential(tokenCredential) .buildClient() .getBlobContainerClient(container) } diff --git a/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/azure/DedicatedEventHubSender.kt b/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/azure/DedicatedEventHubSender.kt index 434b993ec..67fba35d8 100644 --- a/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/azure/DedicatedEventHubSender.kt +++ b/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/azure/DedicatedEventHubSender.kt @@ -1,14 +1,16 @@ package gov.cdc.dex.azure import com.azure.core.amqp.exception.AmqpException +import com.azure.identity.DefaultAzureCredential import com.azure.messaging.eventhubs.EventData import com.azure.messaging.eventhubs.EventHubClientBuilder import com.azure.messaging.eventhubs.EventHubProducerClient -class DedicatedEventHubSender( evHubConnStr: String, evHubTopicName: String) { +class DedicatedEventHubSender( evHubNamespaceUrl: String, evHubTopicName: String, tokenCredential: DefaultAzureCredential) { private val producer: EventHubProducerClient = EventHubClientBuilder() - .connectionString(evHubConnStr, evHubTopicName) + // .connectionString(evHubConnStr, evHubTopicName) + .credential(evHubNamespaceUrl, evHubTopicName, tokenCredential) .buildProducerClient() fun disconnect() { diff --git a/local_libs/lib-dex-commons/src/test/kotlin/gov/cdc/dex/TestAzureBlobProxy.kt b/local_libs/lib-dex-commons/src/test/kotlin/gov/cdc/dex/TestAzureBlobProxy.kt new file mode 100644 index 000000000..00f803419 --- /dev/null +++ b/local_libs/lib-dex-commons/src/test/kotlin/gov/cdc/dex/TestAzureBlobProxy.kt @@ -0,0 +1,18 @@ +package gov.cdc.dex +import com.azure.identity.DefaultAzureCredential +import com.azure.identity.DefaultAzureCredentialBuilder +import gov.cdc.dex.azure.AzureBlobProxy +import org.junit.jupiter.api.Test + + +class TestAzureBlobProxy { + @Test + fun testBlobProxyCredential() { + val url = "https://ocioedemessagesadev.core.windows.net" + val container = "hl7ingress" + val credential = DefaultAzureCredentialBuilder().build() + + val blobProxy = AzureBlobProxy(url, container, credential) + } + +} \ No newline at end of file diff --git a/local_libs/lib-dex-commons/src/test/kotlin/test/gov/cdc/dex/MetadataTest.kt b/local_libs/lib-dex-commons/src/test/kotlin/test/gov/cdc/dex/MetadataTest.kt index 73e5d853b..f58bb41ed 100644 --- a/local_libs/lib-dex-commons/src/test/kotlin/test/gov/cdc/dex/MetadataTest.kt +++ b/local_libs/lib-dex-commons/src/test/kotlin/test/gov/cdc/dex/MetadataTest.kt @@ -55,11 +55,11 @@ class MetadataTest { // val provenance = DEXProvenance("senderID", "system", "filePath", "fileTS", 10, null) // val fileMD = DEXFileMetadata(reportingJurisdiction = "13", uploadID = "", provenance = provenance, dataStream = dataStream, tracing = tracing) - val routingInfo = RoutingMetadata("filePPath", "fileTS", 10, "orgA", "uploadID", "dsid", "dsroute", "trace", "span") + val routingInfo = RoutingMetadata("filePPath", "fileTS", "10", "orgA", "uploadID", "dsid", "dsroute", "trace", "span", "span") val stageMD = MockEventGridStageMetadata("stgName", "stgVersion", "sttus", listOf(), "ts") - val messageMD = MessageMetadata("msgUUID","SINGLE", 1, "hash", "MSH|") + val messageMD = MessageMetadata("msgUUID","SINGLE", 1, "hash") val summary = SummaryInfo("All good!") - val fullMD = DexHL7Metadata(messageMetadata = messageMD, routingInfo = routingInfo, stage = stageMD, summary = summary) + val fullMD = DexHL7Metadata(messageMetadata = messageMD, routingMetadata = routingInfo, stage = stageMD, summary = summary, content="") val json = gson.toJson(fullMD) println(json) From 3986c3bf749a463938df27c9deeb44ef991204d8 Mon Sep 17 00:00:00 2001 From: Marcia Schulman Date: Tue, 23 Apr 2024 16:54:44 -0400 Subject: [PATCH 02/11] add queue connection info --- .../fn-receiver-debatcher/pom.xml | 8 ++++---- .../main/kotlin/gov/cdc/dex/hl7/Function.kt | 2 +- .../kotlin/gov/cdc/dex/hl7/FunctionConfig.kt | 4 ++-- local_libs/lib-dex-commons/build.gradle.kts | 1 + .../cdc/dex/azure/DedicatedEventHubSender.kt | 5 ++--- .../kotlin/gov/cdc/dex/TestAzureBlobProxy.kt | 3 ++- .../gov/cdc/dex/TestDedicatedEHSender.kt | 19 +++++++++++++++++++ 7 files changed, 31 insertions(+), 11 deletions(-) create mode 100644 local_libs/lib-dex-commons/src/test/kotlin/gov/cdc/dex/TestDedicatedEHSender.kt diff --git a/fns-hl7-pipeline/fn-receiver-debatcher/pom.xml b/fns-hl7-pipeline/fn-receiver-debatcher/pom.xml index 09e737cb5..0c1bcd04d 100644 --- a/fns-hl7-pipeline/fn-receiver-debatcher/pom.xml +++ b/fns-hl7-pipeline/fn-receiver-debatcher/pom.xml @@ -128,10 +128,10 @@ ${functionRegion} ${subscriptionID} - - service_principal - azure-service-principal - + + + + linux diff --git a/fns-hl7-pipeline/fn-receiver-debatcher/src/main/kotlin/gov/cdc/dex/hl7/Function.kt b/fns-hl7-pipeline/fn-receiver-debatcher/src/main/kotlin/gov/cdc/dex/hl7/Function.kt index e30dc0ec6..abee7129f 100644 --- a/fns-hl7-pipeline/fn-receiver-debatcher/src/main/kotlin/gov/cdc/dex/hl7/Function.kt +++ b/fns-hl7-pipeline/fn-receiver-debatcher/src/main/kotlin/gov/cdc/dex/hl7/Function.kt @@ -46,7 +46,7 @@ class Function { @QueueTrigger( name = "message", queueName = "%queueName%", - connection = "BlobIngestConnectionString" + connection = "QueueConnection" ) message: String?, context: ExecutionContext ): DexHL7Metadata? { diff --git a/fns-hl7-pipeline/fn-receiver-debatcher/src/main/kotlin/gov/cdc/dex/hl7/FunctionConfig.kt b/fns-hl7-pipeline/fn-receiver-debatcher/src/main/kotlin/gov/cdc/dex/hl7/FunctionConfig.kt index 06b69a190..9fa2d3396 100644 --- a/fns-hl7-pipeline/fn-receiver-debatcher/src/main/kotlin/gov/cdc/dex/hl7/FunctionConfig.kt +++ b/fns-hl7-pipeline/fn-receiver-debatcher/src/main/kotlin/gov/cdc/dex/hl7/FunctionConfig.kt @@ -44,9 +44,9 @@ class FunctionConfig { } azBlobProxy = AzureBlobProxy(ingestBlobUrl, blobIngestContName, tokenCredential) val evHubNamespace = try { - System.getenv("EventHubNamespaceUrl") + System.getenv("EventHubNamespace") } catch (e: NullPointerException) { - logger.error("FATAL: Missing environment variable EventHubNamespaceUrl") + logger.error("FATAL: Missing environment variable EventHubNamespace") throw e } evHubSenderOut = DedicatedEventHubSender(evHubNamespace, evHubSendName, tokenCredential) diff --git a/local_libs/lib-dex-commons/build.gradle.kts b/local_libs/lib-dex-commons/build.gradle.kts index 844a98ecd..81bc62cc0 100644 --- a/local_libs/lib-dex-commons/build.gradle.kts +++ b/local_libs/lib-dex-commons/build.gradle.kts @@ -28,6 +28,7 @@ dependencies { implementation("com.azure:azure-messaging-servicebus") implementation("com.azure:azure-storage-blob") implementation("com.azure:azure-identity") + implementation("com.azure:azure-identity-broker") implementation("redis.clients:jedis:5.1.0") implementation("com.azure:azure-cosmos:4.55.1") diff --git a/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/azure/DedicatedEventHubSender.kt b/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/azure/DedicatedEventHubSender.kt index 67fba35d8..e5ed79e67 100644 --- a/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/azure/DedicatedEventHubSender.kt +++ b/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/azure/DedicatedEventHubSender.kt @@ -7,10 +7,9 @@ import com.azure.messaging.eventhubs.EventHubClientBuilder import com.azure.messaging.eventhubs.EventHubProducerClient -class DedicatedEventHubSender( evHubNamespaceUrl: String, evHubTopicName: String, tokenCredential: DefaultAzureCredential) { +class DedicatedEventHubSender( evHubNamespace: String, evHubTopicName: String, tokenCredential: DefaultAzureCredential) { private val producer: EventHubProducerClient = EventHubClientBuilder() - // .connectionString(evHubConnStr, evHubTopicName) - .credential(evHubNamespaceUrl, evHubTopicName, tokenCredential) + .credential(evHubNamespace, evHubTopicName, tokenCredential) .buildProducerClient() fun disconnect() { diff --git a/local_libs/lib-dex-commons/src/test/kotlin/gov/cdc/dex/TestAzureBlobProxy.kt b/local_libs/lib-dex-commons/src/test/kotlin/gov/cdc/dex/TestAzureBlobProxy.kt index 00f803419..e2ab6c6d6 100644 --- a/local_libs/lib-dex-commons/src/test/kotlin/gov/cdc/dex/TestAzureBlobProxy.kt +++ b/local_libs/lib-dex-commons/src/test/kotlin/gov/cdc/dex/TestAzureBlobProxy.kt @@ -1,5 +1,5 @@ package gov.cdc.dex -import com.azure.identity.DefaultAzureCredential + import com.azure.identity.DefaultAzureCredentialBuilder import gov.cdc.dex.azure.AzureBlobProxy import org.junit.jupiter.api.Test @@ -13,6 +13,7 @@ class TestAzureBlobProxy { val credential = DefaultAzureCredentialBuilder().build() val blobProxy = AzureBlobProxy(url, container, credential) + blobProxy.getBlobClient("unitTests/BatchedMessage.doNotDelete.txt") } } \ No newline at end of file diff --git a/local_libs/lib-dex-commons/src/test/kotlin/gov/cdc/dex/TestDedicatedEHSender.kt b/local_libs/lib-dex-commons/src/test/kotlin/gov/cdc/dex/TestDedicatedEHSender.kt new file mode 100644 index 000000000..7929c3b00 --- /dev/null +++ b/local_libs/lib-dex-commons/src/test/kotlin/gov/cdc/dex/TestDedicatedEHSender.kt @@ -0,0 +1,19 @@ +package gov.cdc.dex +import com.azure.identity.DefaultAzureCredentialBuilder +import gov.cdc.dex.azure.DedicatedEventHubSender +import org.junit.jupiter.api.Test + + +class TestDedicatedEHSender { + @Test + fun testDedicatedSenderCredential() { + val ehNamespace = "ocio-ede-dev-eventhub-namespace.servicebus.windows.net" + val ehHubName = "hl7-recdeb-reports" + val token = DefaultAzureCredentialBuilder().build() + + val sender = DedicatedEventHubSender(ehNamespace, ehHubName, token) + val ids = sender.getPartitionIds() + ids.forEach { println(it) } + sender.disconnect() + } +} \ No newline at end of file From 4361c45d320bbc46402a021d35844b591a17255f Mon Sep 17 00:00:00 2001 From: Marcia Schulman Date: Wed, 24 Apr 2024 15:57:30 -0400 Subject: [PATCH 03/11] update redactor, lib dex commons classes to use managed identity authentication --- fns-hl7-pipeline/fn-redactor/deployZip.sh | 2 +- fns-hl7-pipeline/fn-redactor/pom.xml | 12 ++++++++++-- .../src/main/kotlin/gov/cdc/dex/hl7/Function.kt | 2 +- .../kotlin/gov/cdc/dex/hl7/FunctionConfig.kt | 9 +++++++-- .../gov/cdc/dex/hl7/HealthCheckFunction.kt | 13 +++++++------ .../kotlin/gov/cdc/dex/azure/AzureBlobProxy.kt | 3 ++- .../cdc/dex/azure/DedicatedEventHubSender.kt | 3 ++- .../cdc/dex/azure/health/DependencyChecker.kt | 17 +++++++++++++++++ 8 files changed, 47 insertions(+), 14 deletions(-) diff --git a/fns-hl7-pipeline/fn-redactor/deployZip.sh b/fns-hl7-pipeline/fn-redactor/deployZip.sh index 9c2371564..422664bfd 100644 --- a/fns-hl7-pipeline/fn-redactor/deployZip.sh +++ b/fns-hl7-pipeline/fn-redactor/deployZip.sh @@ -21,6 +21,6 @@ cd ../../.. echo "Deploying Zip..." -az functionapp deployment source config-zip -g $hl7RG -n $function --src $base_name### Set FN_VERSION: +az functionapp deployment source config-zip -g $hl7RG -n $function --src $base_name fn_version=$(cat pom.xml |grep -oPm1 "(?<=)[^<]+") az functionapp config appsettings set --name $function --resource-group $hl7RG --settings FN_VERSION=$fn_version \ No newline at end of file diff --git a/fns-hl7-pipeline/fn-redactor/pom.xml b/fns-hl7-pipeline/fn-redactor/pom.xml index 95afeee09..b630541e0 100644 --- a/fns-hl7-pipeline/fn-redactor/pom.xml +++ b/fns-hl7-pipeline/fn-redactor/pom.xml @@ -5,7 +5,7 @@ gov.cdc.dataexchange redactor - 0.0.43-SNAPSHOT + 0.0.44-SNAPSHOT jar Azure Java Functions @@ -27,7 +27,7 @@ 2.0.5 1.2.9 1.9.0 - 0.0.45-SNAPSHOT + 0.0.47-SNAPSHOT @@ -41,6 +41,14 @@ + + com.azure + azure-identity + + + com.azure + azure-identity-broker + com.microsoft.azure.functions azure-functions-java-library diff --git a/fns-hl7-pipeline/fn-redactor/src/main/kotlin/gov/cdc/dex/hl7/Function.kt b/fns-hl7-pipeline/fn-redactor/src/main/kotlin/gov/cdc/dex/hl7/Function.kt index 5efffd60c..60bc3ced3 100644 --- a/fns-hl7-pipeline/fn-redactor/src/main/kotlin/gov/cdc/dex/hl7/Function.kt +++ b/fns-hl7-pipeline/fn-redactor/src/main/kotlin/gov/cdc/dex/hl7/Function.kt @@ -32,7 +32,7 @@ class Function { @EventHubTrigger( name = "msg", eventHubName = "%EventHubReceiveName%", - connection = "EventHubConnectionString", + connection = "EventHubConnection", consumerGroup = "%EventHubConsumerGroup%", ) message: List, diff --git a/fns-hl7-pipeline/fn-redactor/src/main/kotlin/gov/cdc/dex/hl7/FunctionConfig.kt b/fns-hl7-pipeline/fn-redactor/src/main/kotlin/gov/cdc/dex/hl7/FunctionConfig.kt index d58e7d920..a91aa0296 100644 --- a/fns-hl7-pipeline/fn-redactor/src/main/kotlin/gov/cdc/dex/hl7/FunctionConfig.kt +++ b/fns-hl7-pipeline/fn-redactor/src/main/kotlin/gov/cdc/dex/hl7/FunctionConfig.kt @@ -1,5 +1,6 @@ package gov.cdc.dex.hl7 +import com.azure.identity.DefaultAzureCredentialBuilder import gov.cdc.dex.azure.DedicatedEventHubSender import gov.cdc.dex.util.JsonHelper import gov.cdc.dex.util.ProfileConfiguration @@ -12,8 +13,12 @@ class FunctionConfig { init { //Init Event Hub connections - val evHubConnStr = System.getenv("EventHubConnectionString") - evHubSender = DedicatedEventHubSender(evHubConnStr, evHubSendName) + val evHubNamespace = System.getenv("EventHubConnection__fullyQualifiedNamespace") + val entraClientId = System.getenv("EventHubConnection__clientId") + val tokenCredential = DefaultAzureCredentialBuilder() + .managedIdentityClientId(entraClientId) + .build() + evHubSender = DedicatedEventHubSender(evHubNamespace, evHubSendName, tokenCredential) val profileConfigJson = FunctionConfig::class.java.getResource("/$PROFILE_CONFIG_FILE_PATH")?.readText() profileConfig = JsonHelper.gson.fromJson(profileConfigJson, ProfileConfiguration::class.java) diff --git a/fns-hl7-pipeline/fn-redactor/src/main/kotlin/gov/cdc/dex/hl7/HealthCheckFunction.kt b/fns-hl7-pipeline/fn-redactor/src/main/kotlin/gov/cdc/dex/hl7/HealthCheckFunction.kt index 54e26dd1d..0de37edfe 100644 --- a/fns-hl7-pipeline/fn-redactor/src/main/kotlin/gov/cdc/dex/hl7/HealthCheckFunction.kt +++ b/fns-hl7-pipeline/fn-redactor/src/main/kotlin/gov/cdc/dex/hl7/HealthCheckFunction.kt @@ -1,12 +1,9 @@ package gov.cdc.dex.hl7 +import com.azure.identity.DefaultAzureCredentialBuilder import com.microsoft.azure.functions.* import com.microsoft.azure.functions.annotation.* -import com.google.gson.JsonObject -import gov.cdc.dex.util.JsonHelper.toJsonElement import gov.cdc.dex.azure.health.* -import java.time.LocalDate - import java.util.* import kotlin.time.measureTime @@ -22,9 +19,13 @@ class HealthCheckFunction { ): HttpResponseMessage { val result = HealthCheckResult() val evHubSendName: String = System.getenv("EventHubSendName") - val evHubConnStr = System.getenv("EventHubConnectionString") + val evHubNamespace = System.getenv("EventHubConnection__fullyQualifiedNamespace") + val entraClientId = System.getenv("EventHubConnection__clientId") + val tokenCredential = DefaultAzureCredentialBuilder() + .managedIdentityClientId(entraClientId) + .build() val time = measureTime { - val ehHealthData = DependencyChecker().checkEventHub(evHubConnStr, evHubSendName) + val ehHealthData = DependencyChecker().checkEventHub(evHubNamespace, evHubSendName, tokenCredential) result.dependencyHealthChecks.add(ehHealthData) result.status = if (ehHealthData.status == "UP") "UP" else "DOWNGRADED" } diff --git a/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/azure/AzureBlobProxy.kt b/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/azure/AzureBlobProxy.kt index d32bf02d9..af55ba205 100644 --- a/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/azure/AzureBlobProxy.kt +++ b/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/azure/AzureBlobProxy.kt @@ -1,12 +1,13 @@ package gov.cdc.dex.azure +import com.azure.core.credential.TokenCredential import com.azure.identity.DefaultAzureCredential import com.azure.storage.blob.BlobClient import com.azure.storage.blob.BlobContainerClient import com.azure.storage.blob.BlobServiceClientBuilder -class AzureBlobProxy(blobStorageUrl: String, container: String, tokenCredential: DefaultAzureCredential) { +class AzureBlobProxy(blobStorageUrl: String, container: String, tokenCredential: TokenCredential) { private val blobContainerClient: BlobContainerClient init { diff --git a/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/azure/DedicatedEventHubSender.kt b/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/azure/DedicatedEventHubSender.kt index e5ed79e67..e98b3f1c3 100644 --- a/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/azure/DedicatedEventHubSender.kt +++ b/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/azure/DedicatedEventHubSender.kt @@ -1,13 +1,14 @@ package gov.cdc.dex.azure import com.azure.core.amqp.exception.AmqpException +import com.azure.core.credential.TokenCredential import com.azure.identity.DefaultAzureCredential import com.azure.messaging.eventhubs.EventData import com.azure.messaging.eventhubs.EventHubClientBuilder import com.azure.messaging.eventhubs.EventHubProducerClient -class DedicatedEventHubSender( evHubNamespace: String, evHubTopicName: String, tokenCredential: DefaultAzureCredential) { +class DedicatedEventHubSender( evHubNamespace: String, evHubTopicName: String, tokenCredential: TokenCredential) { private val producer: EventHubProducerClient = EventHubClientBuilder() .credential(evHubNamespace, evHubTopicName, tokenCredential) .buildProducerClient() diff --git a/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/azure/health/DependencyChecker.kt b/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/azure/health/DependencyChecker.kt index 0282f1a77..68a23b7ea 100644 --- a/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/azure/health/DependencyChecker.kt +++ b/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/azure/health/DependencyChecker.kt @@ -2,6 +2,7 @@ package gov.cdc.dex.azure.health import com.azure.core.amqp.AmqpRetryMode import com.azure.core.amqp.AmqpRetryOptions +import com.azure.core.credential.TokenCredential import com.azure.cosmos.CosmosClient import com.azure.cosmos.CosmosClientBuilder import com.azure.messaging.eventhubs.EventHubClientBuilder @@ -52,6 +53,22 @@ class DependencyChecker { } } + fun checkEventHub(eventHubNamespace: String, eventHubName: String, tokenCredential: TokenCredential) : DependencyHealthData { + val retryOptions = AmqpRetryOptions() + .setMaxRetries(1) + .setMode(AmqpRetryMode.FIXED) + .setTryTimeout(Duration.ofSeconds(10)) + + return checkDependency(AzureDependency.EVENT_HUB) { + val client: EventHubProducerClient = EventHubClientBuilder() + .credential(eventHubNamespace, eventHubName, tokenCredential) + .retryOptions(retryOptions) + .buildProducerClient() + client.partitionIds.count() + client.close() + } + } + fun checkServiceBus(connectionString: String, queueName: String) : DependencyHealthData { return checkDependency(AzureDependency.SERVICE_BUS) { val serviceBusClient: ServiceBusReceiverClient = ServiceBusClientBuilder() From 34f638667bbfb5bbcb3bc014d40ab008e85bfc7b Mon Sep 17 00:00:00 2001 From: Marcia Schulman Date: Wed, 24 Apr 2024 18:46:52 -0400 Subject: [PATCH 04/11] update receiver-debatcher to use managed identity with correct environment variables, update ServiceBusProxy --- .../fn-receiver-debatcher/pom.xml | 8 +++--- .../main/kotlin/gov/cdc/dex/hl7/Function.kt | 2 +- .../kotlin/gov/cdc/dex/hl7/FunctionConfig.kt | 26 ++++++++++++------- .../gov/cdc/dex/azure/ServiceBusProxy.kt | 5 ++-- 4 files changed, 25 insertions(+), 16 deletions(-) diff --git a/fns-hl7-pipeline/fn-receiver-debatcher/pom.xml b/fns-hl7-pipeline/fn-receiver-debatcher/pom.xml index 0c1bcd04d..09e737cb5 100644 --- a/fns-hl7-pipeline/fn-receiver-debatcher/pom.xml +++ b/fns-hl7-pipeline/fn-receiver-debatcher/pom.xml @@ -128,10 +128,10 @@ ${functionRegion} ${subscriptionID} - - - - + + service_principal + azure-service-principal + linux diff --git a/fns-hl7-pipeline/fn-receiver-debatcher/src/main/kotlin/gov/cdc/dex/hl7/Function.kt b/fns-hl7-pipeline/fn-receiver-debatcher/src/main/kotlin/gov/cdc/dex/hl7/Function.kt index abee7129f..70bd9b915 100644 --- a/fns-hl7-pipeline/fn-receiver-debatcher/src/main/kotlin/gov/cdc/dex/hl7/Function.kt +++ b/fns-hl7-pipeline/fn-receiver-debatcher/src/main/kotlin/gov/cdc/dex/hl7/Function.kt @@ -66,7 +66,7 @@ class Function { ) ) // create blob client - val blobName = event.eventData.url.substringAfter("/${fnConfig.blobIngestContName}/") + val blobName = event.eventData.url.substringAfter("/${fnConfig.blobIngestContainerName}/") logger.info("DEX::Reading blob: $blobName") val blobClient = fnConfig.azBlobProxy.getBlobClient(blobName) diff --git a/fns-hl7-pipeline/fn-receiver-debatcher/src/main/kotlin/gov/cdc/dex/hl7/FunctionConfig.kt b/fns-hl7-pipeline/fn-receiver-debatcher/src/main/kotlin/gov/cdc/dex/hl7/FunctionConfig.kt index 9fa2d3396..9faf77b06 100644 --- a/fns-hl7-pipeline/fn-receiver-debatcher/src/main/kotlin/gov/cdc/dex/hl7/FunctionConfig.kt +++ b/fns-hl7-pipeline/fn-receiver-debatcher/src/main/kotlin/gov/cdc/dex/hl7/FunctionConfig.kt @@ -11,11 +11,9 @@ class FunctionConfig { val azBlobProxy: AzureBlobProxy val evHubSenderOut: DedicatedEventHubSender val evHubSenderReports: DedicatedEventHubSender - private val tokenCredential : DefaultAzureCredential = DefaultAzureCredentialBuilder().build() - private val logger : Logger = LoggerFactory.getLogger(this.javaClass.simpleName) + private val logger: Logger = LoggerFactory.getLogger(this.javaClass.simpleName) - - val blobIngestContName: String = try { + val blobIngestContainerName : String = try { System.getenv("BlobIngestContainerName") } catch (e: NullPointerException) { logger.error("FATAL: Missing environment variable BlobIngestContainerName") @@ -23,26 +21,36 @@ class FunctionConfig { } val evHubSendName: String = try { System.getenv("EventHubSendName") - } catch(e: NullPointerException) { + } catch (e: NullPointerException) { logger.error("FATAL: Missing environment variable EventHubSendName") throw e } val evReportsHubName: String = try { System.getenv("EventReportsHubName") } catch (e: NullPointerException) { - logger.error ("FATAL: Missing environment variable EventReportsHubName") + logger.error("FATAL: Missing environment variable EventReportsHubName") throw e } - init { - //Init Azure Storage connection + val identityClientId : String = try { + System.getenv("QueueConnection__clientId") + } catch (e: NullPointerException) { + logger.error("FATAL: Missing environment variable QueueConnection__clientId") + throw e + } + // set up the application to use a Managed Identity + val tokenCredential = DefaultAzureCredentialBuilder() + .managedIdentityClientId(identityClientId) + .build() + + //Init Azure Storage connection val ingestBlobUrl = try { System.getenv("BlobIngestUrl") } catch (e: NullPointerException) { logger.error("FATAL: Missing environment variable BlobIngestUrl") throw e } - azBlobProxy = AzureBlobProxy(ingestBlobUrl, blobIngestContName, tokenCredential) + azBlobProxy = AzureBlobProxy(ingestBlobUrl, blobIngestContainerName, tokenCredential) val evHubNamespace = try { System.getenv("EventHubNamespace") } catch (e: NullPointerException) { diff --git a/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/azure/ServiceBusProxy.kt b/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/azure/ServiceBusProxy.kt index 35848f6ab..0d84f4398 100644 --- a/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/azure/ServiceBusProxy.kt +++ b/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/azure/ServiceBusProxy.kt @@ -1,11 +1,12 @@ package gov.cdc.dex.azure +import com.azure.core.credential.TokenCredential import com.azure.messaging.servicebus.ServiceBusClientBuilder import com.azure.messaging.servicebus.ServiceBusSenderClient -class ServiceBusProxy(val sbConnString: String, val sbQueue: String) { +class ServiceBusProxy(serviceBusNamespace: String, sbQueue: String, tokenCredential: TokenCredential) { private val serviceBusSender: ServiceBusSenderClient = ServiceBusClientBuilder() - .connectionString(sbConnString) + .credential(serviceBusNamespace, tokenCredential) .sender() .queueName(sbQueue) .buildClient() From 436811bb852c97a8d0009141953362803aaf7f7d Mon Sep 17 00:00:00 2001 From: Marcia Schulman Date: Thu, 25 Apr 2024 11:39:46 -0400 Subject: [PATCH 05/11] clean up imports and pom --- fns-hl7-pipeline/fn-receiver-debatcher/deployZip.sh | 2 +- fns-hl7-pipeline/fn-receiver-debatcher/pom.xml | 6 +++++- .../src/main/kotlin/gov/cdc/dex/hl7/FunctionConfig.kt | 1 - 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/fns-hl7-pipeline/fn-receiver-debatcher/deployZip.sh b/fns-hl7-pipeline/fn-receiver-debatcher/deployZip.sh index 62fd3e52a..bf8b920f3 100644 --- a/fns-hl7-pipeline/fn-receiver-debatcher/deployZip.sh +++ b/fns-hl7-pipeline/fn-receiver-debatcher/deployZip.sh @@ -22,6 +22,6 @@ cd ../../.. echo "Deploying Zip..." az functionapp deployment source config-zip -g $hl7RG -n $function --src $base_name -### Set FN_VERSION: + fn_version=$(cat pom.xml |grep -oPm1 "(?<=)[^<]+") az functionapp config appsettings set --name $function --resource-group $hl7RG --settings FN_VERSION=$fn_version \ No newline at end of file diff --git a/fns-hl7-pipeline/fn-receiver-debatcher/pom.xml b/fns-hl7-pipeline/fn-receiver-debatcher/pom.xml index 09e737cb5..a2be4689c 100644 --- a/fns-hl7-pipeline/fn-receiver-debatcher/pom.xml +++ b/fns-hl7-pipeline/fn-receiver-debatcher/pom.xml @@ -5,7 +5,7 @@ 4.0.0 gov.cdc.dataexchange receiver-debatcher - 0.0.47-SNAPSHOT + 0.0.44-SNAPSHOT jar DEX Fns-Pipeline :: receiver-debatcher @@ -143,6 +143,10 @@ FN_VERSION ${project.version} + + WEBSITE_RUN_FROM_PACKAGE + 1 + diff --git a/fns-hl7-pipeline/fn-receiver-debatcher/src/main/kotlin/gov/cdc/dex/hl7/FunctionConfig.kt b/fns-hl7-pipeline/fn-receiver-debatcher/src/main/kotlin/gov/cdc/dex/hl7/FunctionConfig.kt index 9faf77b06..1486b92cf 100644 --- a/fns-hl7-pipeline/fn-receiver-debatcher/src/main/kotlin/gov/cdc/dex/hl7/FunctionConfig.kt +++ b/fns-hl7-pipeline/fn-receiver-debatcher/src/main/kotlin/gov/cdc/dex/hl7/FunctionConfig.kt @@ -1,6 +1,5 @@ package gov.cdc.dex.hl7 -import com.azure.identity.DefaultAzureCredential import com.azure.identity.DefaultAzureCredentialBuilder import gov.cdc.dex.azure.AzureBlobProxy import gov.cdc.dex.azure.DedicatedEventHubSender From cc9edd7c823067dcbe3889403b3e0ed0e99914fa Mon Sep 17 00:00:00 2001 From: Marcia Schulman Date: Thu, 25 Apr 2024 11:49:49 -0400 Subject: [PATCH 06/11] update blob proxy unit test --- .../src/test/kotlin/gov/cdc/dex/TestAzureBlobProxy.kt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/local_libs/lib-dex-commons/src/test/kotlin/gov/cdc/dex/TestAzureBlobProxy.kt b/local_libs/lib-dex-commons/src/test/kotlin/gov/cdc/dex/TestAzureBlobProxy.kt index e2ab6c6d6..0f675a933 100644 --- a/local_libs/lib-dex-commons/src/test/kotlin/gov/cdc/dex/TestAzureBlobProxy.kt +++ b/local_libs/lib-dex-commons/src/test/kotlin/gov/cdc/dex/TestAzureBlobProxy.kt @@ -10,7 +10,10 @@ class TestAzureBlobProxy { fun testBlobProxyCredential() { val url = "https://ocioedemessagesadev.core.windows.net" val container = "hl7ingress" - val credential = DefaultAzureCredentialBuilder().build() + val clientId = System.getenv("BlobStorageClientId") + val credential = DefaultAzureCredentialBuilder() + .managedIdentityClientId(clientId) + .build() val blobProxy = AzureBlobProxy(url, container, credential) blobProxy.getBlobClient("unitTests/BatchedMessage.doNotDelete.txt") From 7bdba62b79806bc58f4c76caae21ce59566552e7 Mon Sep 17 00:00:00 2001 From: Marcia Schulman Date: Thu, 25 Apr 2024 11:53:44 -0400 Subject: [PATCH 07/11] update DedicatedEventHub sender unit test --- .../src/test/kotlin/gov/cdc/dex/TestDedicatedEHSender.kt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/local_libs/lib-dex-commons/src/test/kotlin/gov/cdc/dex/TestDedicatedEHSender.kt b/local_libs/lib-dex-commons/src/test/kotlin/gov/cdc/dex/TestDedicatedEHSender.kt index 7929c3b00..4322a05a8 100644 --- a/local_libs/lib-dex-commons/src/test/kotlin/gov/cdc/dex/TestDedicatedEHSender.kt +++ b/local_libs/lib-dex-commons/src/test/kotlin/gov/cdc/dex/TestDedicatedEHSender.kt @@ -9,7 +9,11 @@ class TestDedicatedEHSender { fun testDedicatedSenderCredential() { val ehNamespace = "ocio-ede-dev-eventhub-namespace.servicebus.windows.net" val ehHubName = "hl7-recdeb-reports" - val token = DefaultAzureCredentialBuilder().build() + val clientId = System.getenv("EventHubClientId") + + val token = DefaultAzureCredentialBuilder() + .managedIdentityClientId(clientId) + .build() val sender = DedicatedEventHubSender(ehNamespace, ehHubName, token) val ids = sender.getPartitionIds() From 20ce9facfece0305fd8cb1ec8ede41074ac105bb Mon Sep 17 00:00:00 2001 From: Marcia Schulman Date: Thu, 25 Apr 2024 13:34:48 -0400 Subject: [PATCH 08/11] update routingMetadata to match develop branch --- .../src/main/kotlin/gov/cdc/dex/metadata/RoutingMetadata.kt | 4 ++-- .../src/test/kotlin/test/gov/cdc/dex/MetadataTest.kt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/metadata/RoutingMetadata.kt b/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/metadata/RoutingMetadata.kt index ff25e3dda..b83143c0d 100644 --- a/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/metadata/RoutingMetadata.kt +++ b/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/metadata/RoutingMetadata.kt @@ -11,7 +11,7 @@ class RoutingMetadata ( @SerializedName("upload_id") val uploadId: String, @SerializedName("data_stream_id") val dataStreamId: String, @SerializedName("data_stream_route") val dataStreamRoute: String, - @SerializedName("trace_id") val traceId: String, - @SerializedName("span_id") var spanId: String, + @SerializedName("sender_id") val senderId: String, + @SerializedName("received_filename") val receivedFilename: String, @SerializedName("supporting_metadata") var supportingMetadata: Map? = null ) \ No newline at end of file diff --git a/local_libs/lib-dex-commons/src/test/kotlin/test/gov/cdc/dex/MetadataTest.kt b/local_libs/lib-dex-commons/src/test/kotlin/test/gov/cdc/dex/MetadataTest.kt index f58bb41ed..a55e8aeff 100644 --- a/local_libs/lib-dex-commons/src/test/kotlin/test/gov/cdc/dex/MetadataTest.kt +++ b/local_libs/lib-dex-commons/src/test/kotlin/test/gov/cdc/dex/MetadataTest.kt @@ -55,7 +55,7 @@ class MetadataTest { // val provenance = DEXProvenance("senderID", "system", "filePath", "fileTS", 10, null) // val fileMD = DEXFileMetadata(reportingJurisdiction = "13", uploadID = "", provenance = provenance, dataStream = dataStream, tracing = tracing) - val routingInfo = RoutingMetadata("filePPath", "fileTS", "10", "orgA", "uploadID", "dsid", "dsroute", "trace", "span", "span") + val routingInfo = RoutingMetadata("filePPath", "fileTS", "10", "orgA", "orgB","uploadID", "dsid", "dsroute", "sender1", "testfile.txt") val stageMD = MockEventGridStageMetadata("stgName", "stgVersion", "sttus", listOf(), "ts") val messageMD = MessageMetadata("msgUUID","SINGLE", 1, "hash") val summary = SummaryInfo("All good!") From 1caebace83665e2d4bdf5777393f32026dc050b7 Mon Sep 17 00:00:00 2001 From: Marcia Schulman Date: Thu, 25 Apr 2024 13:41:19 -0400 Subject: [PATCH 09/11] update routing metadata to match develop --- .../main/kotlin/gov/cdc/dex/hl7/Function.kt | 25 +++++++++---------- .../src/test/kotlin/DebatcherTest.kt | 12 ++++----- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/fns-hl7-pipeline/fn-receiver-debatcher/src/main/kotlin/gov/cdc/dex/hl7/Function.kt b/fns-hl7-pipeline/fn-receiver-debatcher/src/main/kotlin/gov/cdc/dex/hl7/Function.kt index 70bd9b915..26ff0c6ea 100644 --- a/fns-hl7-pipeline/fn-receiver-debatcher/src/main/kotlin/gov/cdc/dex/hl7/Function.kt +++ b/fns-hl7-pipeline/fn-receiver-debatcher/src/main/kotlin/gov/cdc/dex/hl7/Function.kt @@ -27,14 +27,13 @@ class Function { const val METADATA = "Metadata" val gson: Gson = GsonBuilder().serializeNulls().create() val knownMetadata: Set = setOf( - "data_stream_id", "meta_destination_id", - "data_stream_route", "meta_ext_event", + "data_stream_id", + "data_stream_route", "data_producer_id", - "jurisdiction", "reporting_jurisdiction", "meta_organization", - "sender_id", "user_id", "meta_username", + "jurisdiction", + "sender_id", "upload_id", "tus_tguid", - "trace_id", - "parent_span_id", "span_id" + "received_filename" ) val fnConfig = FunctionConfig() @@ -224,8 +223,8 @@ class Function { uploadId = newId, dataStreamId = currentMetadata.dataStreamId, dataStreamRoute = currentMetadata.dataStreamRoute, - traceId = currentMetadata.dataStreamId, - spanId = currentMetadata.spanId, + senderId = currentMetadata.senderId, + receivedFilename = currentMetadata.receivedFilename, supportingMetadata = currentMetadata.supportingMetadata ) } @@ -353,13 +352,13 @@ class Function { dataProducerId = metaDataMap["data_producer_id"] ?: "", jurisdiction = getValueOrDefaultString( metaDataMap, - listOf("jurisdiction", "reporting_jurisdiction", "meta_organization") + listOf("jurisdiction") ), uploadId = getValueOrDefaultString(metaDataMap, listOf("upload_id", "tus_tguid")), - dataStreamId = getValueOrDefaultString(metaDataMap, listOf("data_stream_id", "meta_destination_id")), - dataStreamRoute = getValueOrDefaultString(metaDataMap, listOf("data_stream_route", "meta_ext_event")), - traceId = getValueOrDefaultString(metaDataMap, listOf("trace_id")), - spanId = getValueOrDefaultString(metaDataMap, listOf("parent_span_id", "span_id")), + dataStreamId = getValueOrDefaultString(metaDataMap, listOf("data_stream_id")), + dataStreamRoute = getValueOrDefaultString(metaDataMap, listOf("data_stream_route")), + senderId = metaDataMap["sender_id"] ?: "", + receivedFilename = metaDataMap["received_filename"] ?: "", supportingMetadata = supportingMetadata ) } diff --git a/fns-hl7-pipeline/fn-receiver-debatcher/src/test/kotlin/DebatcherTest.kt b/fns-hl7-pipeline/fn-receiver-debatcher/src/test/kotlin/DebatcherTest.kt index 7b3273d79..2fbdfbda0 100644 --- a/fns-hl7-pipeline/fn-receiver-debatcher/src/test/kotlin/DebatcherTest.kt +++ b/fns-hl7-pipeline/fn-receiver-debatcher/src/test/kotlin/DebatcherTest.kt @@ -54,8 +54,8 @@ class DebatcherTest { try { // initialize event report metadata val eventMetadata = ReceiverEventMetadata(stage = - ReceiverEventStageMetadata(startProcessingTime = startTime, - eventTimestamp = Date().toIsoString())) + ReceiverEventStageMetadata(startProcessingTime = startTime, + eventTimestamp = Date().toIsoString())) val testFileIS = this::class.java.getResource(filePath).openStream() // Add routing data to Report object for this file val eventReport = ReceiverEventReport() @@ -190,8 +190,8 @@ class DebatcherTest { uploadId = uploadId, dataStreamId = currentMetadata.dataStreamId, dataStreamRoute = currentMetadata.dataStreamRoute, - traceId = currentMetadata.dataStreamId, - spanId = currentMetadata.spanId, + senderId = currentMetadata.senderId, + receivedFilename = currentMetadata.receivedFilename, supportingMetadata = currentMetadata.supportingMetadata ) } @@ -295,8 +295,8 @@ class DebatcherTest { uploadId = getValueOrDefaultString(metaDataMap, listOf("upload_id", "tus_tguid")), dataStreamId = getValueOrDefaultString(metaDataMap, listOf("data_stream_id", "meta_destination_id")), dataStreamRoute = getValueOrDefaultString(metaDataMap, listOf("data_stream_route", "meta_ext_event")), - traceId = getValueOrDefaultString(metaDataMap, listOf("trace_id")), - spanId = getValueOrDefaultString(metaDataMap, listOf("parent_span_id", "span_id")), + senderId = metaDataMap["sender_id"] ?: "", + receivedFilename = metaDataMap["received_filename"] ?: "", supportingMetadata = supportingMetadata ) } From 19bcb40c24d1488f4faf766add3e0562d6e638bb Mon Sep 17 00:00:00 2001 From: Marcia Schulman Date: Thu, 25 Apr 2024 13:46:23 -0400 Subject: [PATCH 10/11] optimize imports --- .../src/main/kotlin/gov/cdc/dex/azure/AzureBlobProxy.kt | 1 - .../src/main/kotlin/gov/cdc/dex/azure/DedicatedEventHubSender.kt | 1 - 2 files changed, 2 deletions(-) diff --git a/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/azure/AzureBlobProxy.kt b/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/azure/AzureBlobProxy.kt index af55ba205..6401cf451 100644 --- a/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/azure/AzureBlobProxy.kt +++ b/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/azure/AzureBlobProxy.kt @@ -2,7 +2,6 @@ package gov.cdc.dex.azure import com.azure.core.credential.TokenCredential -import com.azure.identity.DefaultAzureCredential import com.azure.storage.blob.BlobClient import com.azure.storage.blob.BlobContainerClient import com.azure.storage.blob.BlobServiceClientBuilder diff --git a/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/azure/DedicatedEventHubSender.kt b/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/azure/DedicatedEventHubSender.kt index e98b3f1c3..2b5690881 100644 --- a/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/azure/DedicatedEventHubSender.kt +++ b/local_libs/lib-dex-commons/src/main/kotlin/gov/cdc/dex/azure/DedicatedEventHubSender.kt @@ -2,7 +2,6 @@ package gov.cdc.dex.azure import com.azure.core.amqp.exception.AmqpException import com.azure.core.credential.TokenCredential -import com.azure.identity.DefaultAzureCredential import com.azure.messaging.eventhubs.EventData import com.azure.messaging.eventhubs.EventHubClientBuilder import com.azure.messaging.eventhubs.EventHubProducerClient From b7ca4f1f7173f5af1286118c72fcdf327c9d3625 Mon Sep 17 00:00:00 2001 From: Marcia Schulman Date: Fri, 26 Apr 2024 16:21:14 -0400 Subject: [PATCH 11/11] remove retry --- .../src/main/kotlin/gov/cdc/dex/hl7/Function.kt | 1 - .../src/main/kotlin/gov/cdc/dex/hl7/FunctionConfig.kt | 7 ++++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fns-hl7-pipeline/fn-redactor/src/main/kotlin/gov/cdc/dex/hl7/Function.kt b/fns-hl7-pipeline/fn-redactor/src/main/kotlin/gov/cdc/dex/hl7/Function.kt index 60bc3ced3..60947e78a 100644 --- a/fns-hl7-pipeline/fn-redactor/src/main/kotlin/gov/cdc/dex/hl7/Function.kt +++ b/fns-hl7-pipeline/fn-redactor/src/main/kotlin/gov/cdc/dex/hl7/Function.kt @@ -121,7 +121,6 @@ class Function { } catch (e: Exception) { logger.error("Unable to send to event hub ${fnConfig.evHubSendName}: ${e.message}") } - return inputEvent } diff --git a/fns-hl7-pipeline/fn-redactor/src/main/kotlin/gov/cdc/dex/hl7/FunctionConfig.kt b/fns-hl7-pipeline/fn-redactor/src/main/kotlin/gov/cdc/dex/hl7/FunctionConfig.kt index a91aa0296..735702778 100644 --- a/fns-hl7-pipeline/fn-redactor/src/main/kotlin/gov/cdc/dex/hl7/FunctionConfig.kt +++ b/fns-hl7-pipeline/fn-redactor/src/main/kotlin/gov/cdc/dex/hl7/FunctionConfig.kt @@ -12,9 +12,9 @@ class FunctionConfig { val evHubSendName: String = System.getenv("EventHubSendName") init { - //Init Event Hub connections - val evHubNamespace = System.getenv("EventHubConnection__fullyQualifiedNamespace") - val entraClientId = System.getenv("EventHubConnection__clientId") + //Init Event Hub connection + val entraClientId : String = System.getenv("EventHubConnection__clientId") + val evHubNamespace: String = System.getenv("EventHubConnection__fullyQualifiedNamespace") val tokenCredential = DefaultAzureCredentialBuilder() .managedIdentityClientId(entraClientId) .build() @@ -23,4 +23,5 @@ class FunctionConfig { val profileConfigJson = FunctionConfig::class.java.getResource("/$PROFILE_CONFIG_FILE_PATH")?.readText() profileConfig = JsonHelper.gson.fromJson(profileConfigJson, ProfileConfiguration::class.java) } + } \ No newline at end of file