From 22fa87d70f0e87f86147ec4e719f29e05bda2f23 Mon Sep 17 00:00:00 2001 From: Sanjay Vasandani Date: Thu, 7 Mar 2024 13:08:03 -0800 Subject: [PATCH] Wait for Kingdom restart in EmptyClusterPanelMatchCorrectnessTest. This fixes a race condition where the port forwarding may be pointing to the Kingdom pod used for resource setup. --- src/main/k8s/local/testing/BUILD.bazel | 17 +- src/main/k8s/panelmatch/local/BUILD.bazel | 1 - .../panelmatch/integration/k8s/BUILD.bazel | 1 + .../EmptyClusterPanelMatchCorrectnessTest.kt | 161 ++++++++++++------ 4 files changed, 123 insertions(+), 57 deletions(-) diff --git a/src/main/k8s/local/testing/BUILD.bazel b/src/main/k8s/local/testing/BUILD.bazel index 956281125dc..c26acb23d36 100644 --- a/src/main/k8s/local/testing/BUILD.bazel +++ b/src/main/k8s/local/testing/BUILD.bazel @@ -6,6 +6,7 @@ package( default_testonly = True, default_visibility = [ "//src/test/kotlin/org/wfanet/measurement/integration/k8s:__pkg__", + "//src/test/kotlin/org/wfanet/panelmatch/integration/k8s:__pkg__", ], ) @@ -75,7 +76,6 @@ kustomization_dir( kustomization_dir( name = "cmms", - testonly = True, srcs = [ ":duchies", ":edp_simulators", @@ -91,3 +91,18 @@ kustomization_dir( "//src/main/k8s/testing/secretfiles:kustomization", ], ) + +# Minimal CMMS for panel match. +kustomization_dir( + name = "cmms_for_panelmatch", + srcs = [ + "//src/main/k8s/local:kingdom", + "//src/main/k8s/local:panelmatch_emulators", + ], + generate_kustomization = True, + tags = ["manual"], + deps = [ + ":config_files", + "//src/main/k8s/testing/secretfiles:kustomization", + ], +) diff --git a/src/main/k8s/panelmatch/local/BUILD.bazel b/src/main/k8s/panelmatch/local/BUILD.bazel index 458bd27347e..44434a1c57e 100644 --- a/src/main/k8s/panelmatch/local/BUILD.bazel +++ b/src/main/k8s/panelmatch/local/BUILD.bazel @@ -50,7 +50,6 @@ kustomization_dir( name = "edp_daemon", srcs = [ ":forwarded_storage_edp_daemon", - "//src/main/k8s/local:kingdom", ], generate_kustomization = True, tags = ["manual"], diff --git a/src/test/kotlin/org/wfanet/panelmatch/integration/k8s/BUILD.bazel b/src/test/kotlin/org/wfanet/panelmatch/integration/k8s/BUILD.bazel index d3c37d2c7e1..13bddab9a2f 100644 --- a/src/test/kotlin/org/wfanet/panelmatch/integration/k8s/BUILD.bazel +++ b/src/test/kotlin/org/wfanet/panelmatch/integration/k8s/BUILD.bazel @@ -68,6 +68,7 @@ java_test( "//src/main/docker:push_all_local_images", "//src/main/docker/panel_exchange_client:push_all_images", "//src/main/k8s/local:kingdom_for_panelmatch_setup.tar", + "//src/main/k8s/local/testing:cmms_for_panelmatch.tar", "//src/main/k8s/panelmatch/local:edp_daemon.tar", "//src/main/k8s/panelmatch/local:mp_daemon.tar", ], diff --git a/src/test/kotlin/org/wfanet/panelmatch/integration/k8s/EmptyClusterPanelMatchCorrectnessTest.kt b/src/test/kotlin/org/wfanet/panelmatch/integration/k8s/EmptyClusterPanelMatchCorrectnessTest.kt index 062d770eacb..d6f553dc74a 100644 --- a/src/test/kotlin/org/wfanet/panelmatch/integration/k8s/EmptyClusterPanelMatchCorrectnessTest.kt +++ b/src/test/kotlin/org/wfanet/panelmatch/integration/k8s/EmptyClusterPanelMatchCorrectnessTest.kt @@ -158,28 +158,31 @@ class EmptyClusterPanelMatchCorrectnessTest : AbstractPanelMatchCorrectnessTest( // https://github.com/kubernetes/kubernetes/issues/66689. k8sClient.waitForServiceAccount("default", timeout = READY_TIMEOUT) - loadKingdomForPanelMatch() + loadKingdomForSetup() val dataProviderEntityContent = withContext(Dispatchers.IO) { createEntityContent("edp1") } val modelProviderEntityContent = withContext(Dispatchers.IO) { createEntityContent("mp1") } - return runResourceSetup(dataProviderEntityContent, modelProviderEntityContent) + val resourceSetupOutput = + runResourceSetup(dataProviderEntityContent, modelProviderEntityContent) + loadCmms(resourceSetupOutput.akidPrincipalMap) + + return resourceSetupOutput.entitiesData } + /** Runs resource setup, leaving port forwarding active for forwarded storage. */ private suspend fun runResourceSetup( dataProviderContent: EntityContent, modelProviderContent: EntityContent, - ): EntitiesData { + ): ResourceSetupOutput { val outputDir = withContext(Dispatchers.IO) { tempDir.newFolder("resource-setup") } val kingdomInternalPod = getPod(KINGDOM_INTERNAL_DEPLOYMENT_NAME) val mpPrivateStoragePod = getPod(MP_PRIVATE_STORAGE_DEPLOYMENT_NAME) val dpPrivateStoragePod = getPod(DP_PRIVATE_STORAGE_DEPLOYMENT_NAME) val sharedStoragePod = getPod(SHARED_STORAGE_DEPLOYMENT_NAME) - var entitiesData: EntitiesData - PortForwarder(kingdomInternalPod, SERVER_PORT).use { internalForward -> + return PortForwarder(kingdomInternalPod, SERVER_PORT).use { internalForward -> val internalAddress: InetSocketAddress = withContext(Dispatchers.IO) { internalForward.start() } - .also { portForwarders.add(internalForward) } val internalChannel = buildMutualTlsChannel(internalAddress.toTarget(), KINGDOM_SIGNING_CERTS).also { channels.add(it) @@ -193,14 +196,14 @@ class EmptyClusterPanelMatchCorrectnessTest : AbstractPanelMatchCorrectnessTest( outputDir, ) val panelMatchResourceKey = - withContext(Dispatchers.IO) { - panelMatchResourceSetup.process( + panelMatchResourceSetup + .process( dataProviderContent = dataProviderContent, modelProviderContent = modelProviderContent, exchangeDate = EXCHANGE_DATE.toProtoDate(), exchangeSchedule = SCHEDULE, ) - } + .also { internalChannel.shutdown() } dataProviderKey = panelMatchResourceKey.dataProviderKey modelProviderKey = panelMatchResourceKey.modelProviderKey @@ -287,6 +290,7 @@ class EmptyClusterPanelMatchCorrectnessTest : AbstractPanelMatchCorrectnessTest( } val akidPrincipalMap = outputDir.resolve(PanelMatchResourceSetup.AKID_PRINCIPAL_MAP_FILE) + loadDpDaemonForPanelMatch( k8sClient, panelMatchResourceKey.dataProviderKey, @@ -298,16 +302,17 @@ class EmptyClusterPanelMatchCorrectnessTest : AbstractPanelMatchCorrectnessTest( akidPrincipalMap, ) - entitiesData = + ResourceSetupOutput( EntitiesData( apiIdToExternalId(panelMatchResourceKey.dataProviderKey.dataProviderId), apiIdToExternalId(panelMatchResourceKey.modelProviderKey.modelProviderId), - ) + ), + akidPrincipalMap, + ) } - return entitiesData } - private suspend fun loadKingdomForPanelMatch() { + private suspend fun loadKingdomForSetup() { val appliedObjects: List = withContext(Dispatchers.IO) { val outputDir = tempDir.newFolder("kingdom-for-panelmatch-setup") @@ -327,9 +332,36 @@ class EmptyClusterPanelMatchCorrectnessTest : AbstractPanelMatchCorrectnessTest( kubectlApply(config, k8sClient) } - appliedObjects.filterIsInstance().forEach { - k8sClient.waitUntilDeploymentReady(checkNotNull(it.metadata?.name)) - } + waitUntilDeploymentsReady(appliedObjects) + } + + private suspend fun loadCmms(akidPrincipalMap: File) { + val appliedObjects: List = + withContext(Dispatchers.IO) { + val outputDir = tempDir.newFolder("cmms") + extractTar( + getRuntimePath(LOCAL_K8S_TESTING_PATH.resolve("cmms_for_panelmatch.tar")).toFile(), + outputDir, + ) + + val configFilesDir = outputDir.toPath().resolve(CONFIG_FILES_PATH).toFile() + logger.info("Copying $akidPrincipalMap to $CONFIG_FILES_PATH") + akidPrincipalMap.copyTo(configFilesDir.resolve(akidPrincipalMap.name)) + + val configFile: File = outputDir.resolve("config.yaml") + kustomize( + outputDir + .toPath() + .resolve(LOCAL_K8S_TESTING_PATH) + .resolve("cmms_for_panelmatch") + .toFile(), + configFile, + ) + + kubectlApply(configFile, k8sClient) + } + + waitUntilDeploymentsReady(appliedObjects) } private suspend fun loadDpDaemonForPanelMatch( @@ -337,28 +369,31 @@ class EmptyClusterPanelMatchCorrectnessTest : AbstractPanelMatchCorrectnessTest( dataProviderKey: DataProviderKey, akidPrincipalMap: File, ) { - withContext(Dispatchers.IO) { - val outputDir = tempDir.newFolder("edp_daemon") - extractTar( - getRuntimePath(LOCAL_K8S_PANELMATCH_PATH.resolve("edp_daemon.tar")).toFile(), - outputDir, - ) + val appliedObjects: List = + withContext(Dispatchers.IO) { + val outputDir = tempDir.newFolder("edp_daemon") + extractTar( + getRuntimePath(LOCAL_K8S_PANELMATCH_PATH.resolve("edp_daemon.tar")).toFile(), + outputDir, + ) - val configFilesDir = outputDir.toPath().resolve(CONFIG_FILES_PATH).toFile() - akidPrincipalMap.copyTo(configFilesDir.resolve(akidPrincipalMap.name)) - val configTemplate: File = outputDir.resolve("config.yaml") - kustomize( - outputDir.toPath().resolve(LOCAL_K8S_PANELMATCH_PATH).resolve("edp_daemon").toFile(), - configTemplate, - ) + val configFilesDir = outputDir.toPath().resolve(PANELMATCH_CONFIG_FILES_PATH).toFile() + akidPrincipalMap.copyTo(configFilesDir.resolve(akidPrincipalMap.name)) + val configTemplate: File = outputDir.resolve("config.yaml") + kustomize( + outputDir.toPath().resolve(LOCAL_K8S_PANELMATCH_PATH).resolve("edp_daemon").toFile(), + configTemplate, + ) - val configContent = - configTemplate - .readText(StandardCharsets.UTF_8) - .replace("{party_name}", dataProviderKey.dataProviderId) + val configContent = + configTemplate + .readText(StandardCharsets.UTF_8) + .replace("{party_name}", dataProviderKey.dataProviderId) - kubectlApply(configContent, k8sClient) - } + kubectlApply(configContent, k8sClient) + } + + waitUntilDeploymentsReady(appliedObjects) } private suspend fun loadMpDaemonForPanelMatch( @@ -366,28 +401,31 @@ class EmptyClusterPanelMatchCorrectnessTest : AbstractPanelMatchCorrectnessTest( modelProviderKey: ModelProviderKey, akidPrincipalMap: File, ) { - withContext(Dispatchers.IO) { - val outputDir = tempDir.newFolder("mp_daemon") - extractTar( - getRuntimePath(LOCAL_K8S_PANELMATCH_PATH.resolve("mp_daemon.tar")).toFile(), - outputDir, - ) + val appliedObjects: List = + withContext(Dispatchers.IO) { + val outputDir = tempDir.newFolder("mp_daemon") + extractTar( + getRuntimePath(LOCAL_K8S_PANELMATCH_PATH.resolve("mp_daemon.tar")).toFile(), + outputDir, + ) - val configFilesDir = outputDir.toPath().resolve(CONFIG_FILES_PATH).toFile() - akidPrincipalMap.copyTo(configFilesDir.resolve(akidPrincipalMap.name)) + val configFilesDir = outputDir.toPath().resolve(PANELMATCH_CONFIG_FILES_PATH).toFile() + akidPrincipalMap.copyTo(configFilesDir.resolve(akidPrincipalMap.name)) - val configTemplate: File = outputDir.resolve("config.yaml") - kustomize( - outputDir.toPath().resolve(LOCAL_K8S_PANELMATCH_PATH).resolve("mp_daemon").toFile(), - configTemplate, - ) - val configContent = - configTemplate - .readText(StandardCharsets.UTF_8) - .replace("{party_name}", modelProviderKey.modelProviderId) + val configTemplate: File = outputDir.resolve("config.yaml") + kustomize( + outputDir.toPath().resolve(LOCAL_K8S_PANELMATCH_PATH).resolve("mp_daemon").toFile(), + configTemplate, + ) + val configContent = + configTemplate + .readText(StandardCharsets.UTF_8) + .replace("{party_name}", modelProviderKey.modelProviderId) - kubectlApply(configContent, k8sClient) - } + kubectlApply(configContent, k8sClient) + } + + waitUntilDeploymentsReady(appliedObjects) } private suspend fun createTestHarness(entitiesData: EntitiesData): PanelMatchSimulator { @@ -436,6 +474,11 @@ class EmptyClusterPanelMatchCorrectnessTest : AbstractPanelMatchCorrectnessTest( portForwarder.stop() } } + + private data class ResourceSetupOutput( + val entitiesData: EntitiesData, + val akidPrincipalMap: File, + ) } companion object { @@ -449,8 +492,10 @@ class EmptyClusterPanelMatchCorrectnessTest : AbstractPanelMatchCorrectnessTest( var modelProviderKey: ModelProviderKey? = null private val LOCAL_K8S_PATH = Paths.get("src", "main", "k8s", "local") + private val LOCAL_K8S_TESTING_PATH = LOCAL_K8S_PATH.resolve("testing") + private val CONFIG_FILES_PATH = LOCAL_K8S_TESTING_PATH.resolve("config_files") private val LOCAL_K8S_PANELMATCH_PATH = Paths.get("src", "main", "k8s", "panelmatch", "local") - private val CONFIG_FILES_PATH = LOCAL_K8S_PANELMATCH_PATH.resolve("config_files") + private val PANELMATCH_CONFIG_FILES_PATH = LOCAL_K8S_PANELMATCH_PATH.resolve("config_files") private const val KINGDOM_INTERNAL_DEPLOYMENT_NAME = "gcp-kingdom-data-server-deployment" private const val KINGDOM_PUBLIC_DEPLOYMENT_NAME = "v2alpha-public-api-server-deployment" @@ -513,5 +558,11 @@ class EmptyClusterPanelMatchCorrectnessTest : AbstractPanelMatchCorrectnessTest( private fun extractTar(archive: File, outputDirectory: File) { Processes.runCommand("tar", "-xf", archive.toString(), "-C", outputDirectory.toString()) } + + private suspend fun waitUntilDeploymentsReady(appliedObjects: Iterable) { + appliedObjects.filterIsInstance().forEach { + k8sClient.waitUntilDeploymentReady(checkNotNull(it.metadata?.name)) + } + } } }