Push panel exchange container images as part of GitHub workflow. (#1079)

.github/workflows/create-cmm-images.yml → .github/workflows/push-images.yml

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-name: Build and Push CMM Images
+name: Build and push container images
 
 on:
   workflow_dispatch:
@@ -32,14 +32,20 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Export image tag
-        run: ./.github/workflows/export-image-tag.sh
-
-      - name: Output image tag
-        id: output-image-tag
-        run: echo "image-tag=$IMAGE_TAG" >> $GITHUB_OUTPUT
+      - id: get-image-tag
+        name: Get image tag
+        run: |
+          declare image_tag
+          if [[ "$GITHUB_REF_TYPE" == 'tag' ]]; then
+            image_tag="${GITHUB_REF_NAME#v}"
+          else
+            image_tag="$GITHUB_SHA"
+          fi
+          echo "image-tag=${image_tag}" >> "$GITHUB_OUTPUT"
 
       - name: Write ~/.bazelrc
+        env:
+          IMAGE_TAG: ${{ steps.get-image-tag.outputs.image-tag }}
         run: |
           cat << EOF > ~/.bazelrc
           build -c opt 
@@ -70,5 +76,13 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ github.token }}
 
+      - name: Build images
+        run: >
+          bazelisk build
+          //src/main/docker:push_all_gke_images 
+          //src/main/docker/panel_exchange_client:push_all_images
+
       - name: Push images
-        run: bazelisk run //src/main/docker:push_all_gke_images
+        run: |
+          bazelisk run //src/main/docker:push_all_gke_images 
+          bazelisk run //src/main/docker/panel_exchange_client:push_all_images

.github/workflows/update-cmms.yml

@@ -38,7 +38,7 @@ on:
 
 jobs:
   publish-images:
-    uses: ./.github/workflows/create-cmm-images.yml
+    uses: ./.github/workflows/push-images.yml
 
   terraform:
     uses: ./.github/workflows/terraform-cmms.yml

docs/panelmatch/deploying.md

@@ -7,14 +7,14 @@ discusses which parts can or should be customized in each deployment.
 
 Each Model Provider or Event Data Provider that wishes to use the reference
 implementation of the Panel Exchange Client should implement a binary that runs
-[ExchangeWorkflowDaemon](https://github.com/world-federation-of-advertisers/panel-exchange-client/blob/main/src/main/kotlin/org/wfanet/panelmatch/client/deploy/ExchangeWorkflowDaemon.kt).
+[ExchangeWorkflowDaemon](../../src/main/kotlin/org/wfanet/panelmatch/client/deploy/ExchangeWorkflowDaemon.kt).
 
 For some examples, see:
 
-*   [ExampleDaemon](https://github.com/world-federation-of-advertisers/panel-exchange-client/blob/main/src/main/kotlin/org/wfanet/panelmatch/client/deploy/example/ExampleDaemon.kt)
-*   [FilesystemExampleDaemonMain](https://github.com/world-federation-of-advertisers/panel-exchange-client/blob/main/src/main/kotlin/org/wfanet/panelmatch/client/deploy/example/filesystem/FilesystemExampleDaemonMain.kt)
-*   [GoogleCloudExampleDaemonMain](https://github.com/world-federation-of-advertisers/panel-exchange-client/blob/main/src/main/kotlin/org/wfanet/panelmatch/client/deploy/example/gcloud/GoogleCloudExampleDaemonMain.kt)
-*   [ExchangeWorkflowDaemonForTest](https://github.com/world-federation-of-advertisers/panel-exchange-client/blob/main/src/main/kotlin/org/wfanet/panelmatch/integration/ExchangeWorkflowDaemonForTest.kt)
+*   [ExampleDaemon](../..//src/main/kotlin/org/wfanet/panelmatch/client/deploy/example/ExampleDaemon.kt)
+*   [FilesystemExampleDaemonMain](../..//src/main/kotlin/org/wfanet/panelmatch/client/deploy/example/filesystem/FilesystemExampleDaemonMain.kt)
+*   [GoogleCloudExampleDaemonMain](../..//src/main/kotlin/org/wfanet/panelmatch/client/deploy/example/gcloud/GoogleCloudExampleDaemonMain.kt)
+*   [ExchangeWorkflowDaemonForTest](../..//src/main/kotlin/org/wfanet/panelmatch/integration/ExchangeWorkflowDaemonForTest.kt)
 
 The differences between these examples should illustrate the difference
 components that need to be customized.
@@ -72,7 +72,7 @@ necessary. If there is already a process to generate certificates outside the
 reference implementation, that can be used instead.
 
 In this situation, implement a custom
-[CertificateAuthority](https://github.com/world-federation-of-advertisers/panel-exchange-client/blob/main/src/main/kotlin/org/wfanet/panelmatch/common/certificates/CertificateAuthority.kt)
+[CertificateAuthority](../..//src/main/kotlin/org/wfanet/panelmatch/common/certificates/CertificateAuthority.kt)
 subclass that picks the appropriate pre-generated certificate.
 
 ### Storage

src/main/docker/BUILD.bazel

@@ -93,23 +93,27 @@ docker_push(
 docker_push(
     name = "push_all_gke_images",
     bundle = ":all_gke_images",
+    sequential = True,
     tags = ["manual"],
 )
 
 docker_push(
     name = "push_all_kingdom_gke_images",
     bundle = ":kingdom_gke_images",
+    sequential = True,
     tags = ["manual"],
 )
 
 docker_push(
     name = "push_all_duchy_gke_images",
     bundle = ":duchy_gke_images",
+    sequential = True,
     tags = ["manual"],
 )
 
 docker_push(
     name = "push_all_reporting_gke_images",
     bundle = ":reporting_gke_images",
+    sequential = True,
     tags = ["manual"],
 )

src/main/docker/panel_exchange_client/BUILD.bazel

@@ -1,9 +1,11 @@
 load(
     "@io_bazel_rules_docker//container:container.bzl",
+    "container_bundle",
     "container_push",
 )
+load("@io_bazel_rules_docker//contrib:push-all.bzl", "docker_push")
 load("//build:variables.bzl", "IMAGE_REPOSITORY_SETTINGS")
-load("//src/main/docker/panel_exchange_client:images.bzl", "ALL_GKE_IMAGES")
+load("//src/main/docker/panel_exchange_client:images.bzl", "AWS_IMAGES", "GCLOUD_IMAGES")
 
 package(
     default_testonly = True,  #TODO: delete when InMemoryKeyStore and FakeHybridCipher are not used.
@@ -12,6 +14,8 @@ package(
     ],
 )
 
+ALL_IMAGES = GCLOUD_IMAGES + AWS_IMAGES
+
 # These rules push Docker images to the container registry.
 # 'bazel build' builds the image, 'bazel run' will actually push it.
 [
@@ -23,5 +27,23 @@ package(
         repository = image_spec.repository,
         tags = ["manual"],
     )
-    for image_spec in ALL_GKE_IMAGES
+    for image_spec in ALL_IMAGES
 ]
+
+container_bundle(
+    name = "all_images",
+    images = {"{registry}/{name}:{tag}".format(
+        name = image_spec.repository,
+        registry = IMAGE_REPOSITORY_SETTINGS.container_registry,
+        tag = IMAGE_REPOSITORY_SETTINGS.image_tag,
+    ): image_spec.image for image_spec in ALL_IMAGES},
+    tags = ["manual"],
+    visibility = ["//visibility:private"],
+)
+
+docker_push(
+    name = "push_all_images",
+    bundle = ":all_images",
+    sequential = True,
+    tags = ["manual"],
+)

src/main/docker/panel_exchange_client/images.bzl

@@ -12,24 +12,27 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Container image specs."""
+"""Container image specs.
+
+TODO(@MarcoPremier): Merge this with images.bzl in cross-media-measurement repo
+"""
 
 load("//build:variables.bzl", "IMAGE_REPOSITORY_SETTINGS")
 
 _PREFIX = IMAGE_REPOSITORY_SETTINGS.repository_prefix
 
-# List of specs for all Docker containers to push to a container registry.
-# These are only used on GKE.
-# TODO(@MarcoPremier): Merge this with images.bzl in cross-media-measurement repo
-ALL_GKE_IMAGES = [
+GCLOUD_IMAGES = [
     struct(
         name = "google_cloud_example_daemon_image",
         image = "//src/main/kotlin/org/wfanet/panelmatch/client/deploy/example/gcloud:google_cloud_example_daemon_image",
-        repository = _PREFIX + "/example-panel-exchange-daemon",
+        repository = _PREFIX + "/panel-exchange/gcloud-example-daemon",
     ),
+]
+
+AWS_IMAGES = [
     struct(
         name = "aws_example_daemon_image",
         image = "//src/main/kotlin/org/wfanet/panelmatch/client/deploy/example/aws:aws_example_daemon_image",
-        repository = "example-panel-exchange-daemon",
+        repository = _PREFIX + "/panel-exchange/aws-example-daemon",
     ),
 ]

src/main/k8s/panelmatch/dev/BUILD.bazel

@@ -1,5 +1,9 @@
 load("//build:variables.bzl", "EXAMPLE_PANEL_EXCHANGE_CLIENT_DAEMON_CONFIG")
-load("//src/main/docker/panel_exchange_client:images.bzl", "ALL_GKE_IMAGES")
+load(
+    "//src/main/docker/panel_exchange_client:images.bzl",
+    "AWS_IMAGES",
+    "GCLOUD_IMAGES",
+)
 load("@wfa_rules_cue//cue:defs.bzl", "cue_export", "cue_library")
 
 cue_library(
@@ -80,7 +84,7 @@ cue_export(
 
 ALL_IMAGE_ARCHIVES = [
     image_spec.image + ".tar"
-    for image_spec in ALL_GKE_IMAGES
+    for image_spec in GCLOUD_IMAGES + AWS_IMAGES
 ]
 
 filegroup(

src/main/k8s/panelmatch/dev/README.md

@@ -1,13 +1,14 @@
 # `dev` Kubernetes Environment
 
 K8s manifest generation for the `dev` environment, which is the `halo-cmm-dev`
-Google Cloud project. The matching configuration for the Cross-Media Measurement System is in
+Google Cloud project. The matching configuration for the Cross-Media Measurement
+System is in
 https://github.com/world-federation-of-advertisers/cross-media-measurement/tree/main/src/main/k8s/dev.
 
 ## Push image
 
 ```shell
-bazel run -c opt //src/main/docker:push_google_cloud_example_daemon_image \
+bazel run -c opt //src/main/docker/panel_exchange_client:push_google_cloud_example_daemon_image \
   --define=container_registry=gcr.io --define=image_repo_prefix=halo-cmm-dev
 ```
 
@@ -16,7 +17,7 @@ bazel run -c opt //src/main/docker:push_google_cloud_example_daemon_image \
 Use [testing `secretfiles`](../testing/secretfiles).
 
 ```shell
-bazel run //src/main/k8s/testing/secretfiles:apply_kustomization
+bazel run //src/main/k8s/panelmatch/testing/secretfiles:apply_kustomization
 ```
 
 Use the generated K8s secret name when building the `cue_export` targets.

src/main/k8s/panelmatch/dev/example_daemon_aws.cue

@@ -100,7 +100,7 @@ deployments: {
 			// nodeSelector: "iam.gke.io/gke-metadata-server-enabled": "true"
 		}
 		_podSpec: _container: {
-			image:           _defaultAwsConfig.containerPrefix + "/example-panel-exchange-daemon"
+			image:           _defaultAwsConfig.containerPrefix + "panel-exchange/aws-example-daemon"
 			imagePullPolicy: "Always"
 			args:            _exchangeDaemonConfig.args + [
 						"--cert-collection-file=/var/run/secrets/files/trusted_certs.pem",

src/main/k8s/panelmatch/dev/example_daemon_gke.cue

@@ -109,7 +109,7 @@ deployments: {
 			nodeSelector: "iam.gke.io/gke-metadata-server-enabled": "true"
 		}
 		_podSpec: _container: {
-			image:           #ContainerRegistryPrefix + "/example-panel-exchange-daemon"
+			image:           #ContainerRegistryPrefix + "/panel-exchange/gcloud-example-daemon"
 			imagePullPolicy: "Always"
 			args:            _exchangeDaemonConfig.args + [
 						"--cert-collection-file=/var/run/secrets/files/trusted_certs.pem",

src/main/terraform/panel-match/main.tf

@@ -17,25 +17,25 @@ module "panel_exchange_client" {
 
   # EKS vars
   availability_zones_count = 2
-  project = "tftest"
-  vpc_cidr = "10.0.0.0/16"
-  subnet_cidr_bits = 8
+  project                  = "tftest"
+  vpc_cidr                 = "10.0.0.0/16"
+  subnet_cidr_bits         = 8
 
   # General MP vars
-  bucket_name = "tf-test-blob-storage"
+  bucket_name    = "tf-test-blob-storage"
   kms_alias_name = "my-key-alias"
-  ca_org_name = "WFA"
+  ca_org_name    = "WFA"
   ca_common_name = "WFA AWS MP CA"
-  ca_dns = "example.com"
+  ca_dns         = "example.com"
 
   # EKS Config vars
-  use_test_secrets = true
-  image_name = "push_aws_example_daemon_image"
-  build_target_name = "example_mp_daemon_aws"
-  manifest_name = "example_mp_daemon_aws.yaml"
-  repository_name = "example-panel-exchange-daemon"
-  path_to_secrets = "../k8s/testing/secretfiles"
-  path_to_cue = "../k8s/dev/example_mp_daemon_aws.cue"
+  use_test_secrets         = true
+  image_name               = "push_aws_example_daemon_image"
+  build_target_name        = "example_mp_daemon_aws"
+  manifest_name            = "example_mp_daemon_aws.yaml"
+  repository_name          = "panel-exchange/aws-example-daemon"
+  path_to_secrets          = "../k8s/testing/secretfiles"
+  path_to_cue              = "../k8s/dev/example_mp_daemon_aws.cue"
   k8s_account_service_name = "mp-workflow"
-  kingdom_endpoint = "public.kingdom.dev.halo-cmm.org:8443"
+  kingdom_endpoint         = "public.kingdom.dev.halo-cmm.org:8443"
 }