diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index b3e1670af33..e2f33675ec8 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -43,10 +43,6 @@ on: type: boolean description: 'Build amd64 `without_adx` and `without_netgo_without_adx` images, and arm64 images' required: false - private_build: - type: boolean - description: 'Build private images' - required: false jobs: # matrix_builder generates a matrix that includes the roles selected in the input @@ -86,7 +82,7 @@ jobs: docker-push: name: ${{ matrix.role }} images runs-on: ubuntu-latest - environment: Production Docker Registry + environment: Private Registry Builds needs: matrix_builder # setup jobs for each role @@ -110,33 +106,20 @@ jobs: - id: auth uses: google-github-actions/auth@v1 with: - credentials_json: ${{ secrets.GCR_SERVICE_KEY_SECRET }} + credentials_json: ${{ secrets.PRIVATE_REGISTRY_UPLOAD_SECRET }} - name: Set up Google Cloud SDK uses: google-github-actions/setup-gcloud@v1 - name: Authenticate Docker with gcloud run: | - if [[ "${{ github.event.inputs.private_build }}" == "true" ]]; then - gcloud auth configure-docker us-central1-docker.pkg.dev - else - gcloud auth configure-docker - fi - - - name: Set CONTAINER_REGISTRY - id: set-registry - run: | - if [[ "${{ github.event.inputs.private_build }}" == "true" ]]; then - echo "CONTAINER_REGISTRY=${{ vars.PRIVATE_REGISTRY }}" >> $GITHUB_ENV - else - echo "CONTAINER_REGISTRY=${{ vars.PUBLIC_REGISTRY }}" >> $GITHUB_ENV - fi + gcloud auth configure-docker us-central1-docker.pkg.dev - name: Build/Push ${{ matrix.role }} amd64 images with adx (default) env: IMAGE_TAG: ${{ inputs.docker_tag }} CADENCE_DEPLOY_KEY: ${{ secrets.CADENCE_DEPLOY_KEY }} run: | - make docker-build-${{ matrix.role }}-with-adx docker-push-${{ matrix.role }}-with-adx CONTAINER_REGISTRY=$CONTAINER_REGISTRY + make docker-build-${{ matrix.role }}-with-adx docker-push-${{ matrix.role }}-with-adx CONTAINER_REGISTRY=${{ vars.PRIVATE_REGISTRY }} - name: Build/Push ${{ matrix.role }} amd64 images without netgo and without adx, arm64 images if: ${{ inputs.include_alternative_builds }} @@ -146,7 +129,5 @@ jobs: run: | make docker-build-${{ matrix.role }}-without-adx docker-push-${{ matrix.role }}-without-adx \ docker-build-${{ matrix.role }}-without-netgo-without-adx docker-push-${{ matrix.role }}-without-netgo-without-adx \ - docker-cross-build-${{ matrix.role }}-arm docker-push-${{ matrix.role }}-arm CONTAINER_REGISTRY=$CONTAINER_REGISTRY - + docker-cross-build-${{ matrix.role }}-arm docker-push-${{ matrix.role }}-arm CONTAINER_REGISTRY=${{ vars.PRIVATE_REGISTRY }} - diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 3b5620f43c2..95454631127 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -1,59 +1,165 @@ name: CD on: -# Workflow dispatch for now, while we're testing environments - # push: - # tags: - # - '*' - # - "!daily-*" workflow_dispatch: inputs: tag: - description: 'Tag/commit' + description: 'Tag to build & push' required: true type: string - - env: GO_VERSION: "1.22" + PRIVATE_REGISTRY_HOST: us-central1-docker.pkg.dev jobs: - docker-push: - name: Push to container registry + # Build and Push to Private Registry + private-build: + name: Build & Push to Private Container Registry + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + role: [access, collection, consensus, execution, observer, verification] + environment: Private Registry Builds + steps: + - name: Setup Go + uses: actions/setup-go@v4 + with: + go-version: ${{ env.GO_VERSION }} + + - name: Checkout repo + uses: actions/checkout@v3 + + - id: auth + uses: google-github-actions/auth@v1 + with: + credentials_json: ${{ secrets.PRIVATE_REGISTRY_UPLOAD_SECRET }} + + - name: Authenticate Docker with gcloud + run: gcloud auth configure-docker ${{ env.PRIVATE_REGISTRY_HOST }} + + - name: Build & Push ${{ matrix.role }} + env: + IMAGE_TAG: ${{ inputs.tag }} + CONTAINER_REGISTRY: ${{ vars.PRIVATE_REGISTRY }} + run: | + make docker-build-${{ matrix.role }}-with-adx docker-push-${{ matrix.role }}-with-adx CONTAINER_REGISTRY=${CONTAINER_REGISTRY} + make docker-build-${{ matrix.role }}-without-adx docker-push-${{ matrix.role }}-without-adx CONTAINER_REGISTRY=${CONTAINER_REGISTRY} + make docker-build-${{ matrix.role }}-without-netgo-without-adx docker-push-${{ matrix.role }}-without-netgo-without-adx CONTAINER_REGISTRY=${CONTAINER_REGISTRY} + make docker-cross-build-${{ matrix.role }}-arm docker-push-${{ matrix.role }}-arm CONTAINER_REGISTRY=${CONTAINER_REGISTRY} + + # Individual Promotion Jobs with unique environments enables individual image promotion + promote-access: + name: Promote Access Image to Public Registry + runs-on: ubuntu-latest + needs: private-build + environment: Public Access Image Promotion + steps: + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Promote Access + uses: ./actions/promote-images + with: + gcp_credentials: ${{ secrets.PUBLIC_REGISTRY_PROMOTION_SECRET }} + private_registry: ${{ vars.PRIVATE_REGISTRY }} + private_registry_host: ${{ env.PRIVATE_REGISTRY_HOST }} + public_registry: ${{ vars.PUBLIC_REGISTRY }} + role: access + tags: "${{ inputs.tag }},${{ inputs.tag }}-without-adx,${{ inputs.tag }}-without-netgo-without-adx,${{ inputs.tag }}-arm" + + promote-collection: + name: Promote Collection Image to Public Registry runs-on: ubuntu-latest - environment: Production Docker Registry + needs: private-build + environment: Public Collection Image Promotion steps: - - name: Setup Go - uses: actions/setup-go@v4 - timeout-minutes: 10 # fail fast. sometimes this step takes an extremely long time - with: - go-version: ${{ env.GO_VERSION }} - - name: Checkout repo - uses: actions/checkout@v2 - with: - ref: ${{ inputs.tag }} - # Provide Google Service Account credentials to Github Action, allowing interaction with the Google Container Registry - # Logging in as github-actions@dl-flow.iam.gserviceaccount.com - - id: auth - uses: google-github-actions/auth@v1 - with: - credentials_json: ${{ secrets.GCR_SERVICE_KEY_SECRET }} - - name: Set up Google Cloud SDK - uses: google-github-actions/setup-gcloud@v1 - - name: Authenticate docker with gcloud - run: | - gcloud auth configure-docker - - name: Docker build - env: - CADENCE_DEPLOY_KEY: ${{ secrets.CADENCE_DEPLOY_KEY }} - run: | - make docker-build-flow-with-adx - make docker-build-flow-without-adx - make docker-build-flow-without-netgo-without-adx - make docker-cross-build-flow-arm - - name: Docker push - run: | - make docker-push-flow-with-adx - make docker-push-flow-without-adx - make docker-push-flow-without-netgo-without-adx - make docker-push-flow-arm + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Promote Collection + uses: ./actions/promote-images + with: + gcp_credentials: ${{ secrets.PUBLIC_REGISTRY_PROMOTION_SECRET }} + private_registry: ${{ vars.PRIVATE_REGISTRY }} + private_registry_host: ${{ env.PRIVATE_REGISTRY_HOST }} + public_registry: ${{ vars.PUBLIC_REGISTRY }} + role: collection + tags: "${{ inputs.tag }},${{ inputs.tag }}-without-adx,${{ inputs.tag }}-without-netgo-without-adx,${{ inputs.tag }}-arm" + + promote-consensus: + name: Promote Consensus Image to Public Registry + runs-on: ubuntu-latest + needs: private-build + environment: Public Consensus Image Promotion + steps: + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Promote Consensus + uses: ./actions/promote-images + with: + gcp_credentials: ${{ secrets.PUBLIC_REGISTRY_PROMOTION_SECRET }} + private_registry: ${{ vars.PRIVATE_REGISTRY }} + private_registry_host: ${{ env.PRIVATE_REGISTRY_HOST }} + public_registry: ${{ vars.PUBLIC_REGISTRY }} + role: consensus + tags: "${{ inputs.tag }},${{ inputs.tag }}-without-adx,${{ inputs.tag }}-without-netgo-without-adx,${{ inputs.tag }}-arm" + + promote-execution: + name: Promote Execution Image to Public Registry + runs-on: ubuntu-latest + needs: private-build + environment: Public Execution Image Promotion + steps: + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Promote Execution + uses: ./actions/promote-images + with: + gcp_credentials: ${{ secrets.PUBLIC_REGISTRY_PROMOTION_SECRET }} + private_registry: ${{ vars.PRIVATE_REGISTRY }} + private_registry_host: ${{ env.PRIVATE_REGISTRY_HOST }} + public_registry: ${{ vars.PUBLIC_REGISTRY }} + role: execution + tags: "${{ inputs.tag }},${{ inputs.tag }}-without-adx,${{ inputs.tag }}-without-netgo-without-adx,${{ inputs.tag }}-arm" + + promote-observer: + name: Promote Observer Image to Public Registry + runs-on: ubuntu-latest + needs: private-build + environment: Public Observer Image Promotion + steps: + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Promote Observer + uses: ./actions/promote-images + with: + gcp_credentials: ${{ secrets.PUBLIC_REGISTRY_PROMOTION_SECRET }} + private_registry: ${{ vars.PRIVATE_REGISTRY }} + private_registry_host: ${{ env.PRIVATE_REGISTRY_HOST }} + public_registry: ${{ vars.PUBLIC_REGISTRY }} + role: observer + tags: "${{ inputs.tag }},${{ inputs.tag }}-without-adx,${{ inputs.tag }}-without-netgo-without-adx,${{ inputs.tag }}-arm" + + promote-verification: + name: Promote Verification Image to Public Registry + runs-on: ubuntu-latest + needs: private-build + environment: Public Verification Image Promotion + steps: + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Promote Verification + uses: ./actions/promote-images + with: + gcp_credentials: ${{ secrets.PUBLIC_REGISTRY_PROMOTION_SECRET }} + private_registry: ${{ vars.PRIVATE_REGISTRY }} + private_registry_host: ${{ env.PRIVATE_REGISTRY_HOST }} + public_registry: ${{ vars.PUBLIC_REGISTRY }} + role: verification + tags: "${{ inputs.tag }},${{ inputs.tag }}-without-adx,${{ inputs.tag }}-without-netgo-without-adx,${{ inputs.tag }}-arm" + diff --git a/actions/promote-images/action.yml b/actions/promote-images/action.yml new file mode 100644 index 00000000000..298f7096588 --- /dev/null +++ b/actions/promote-images/action.yml @@ -0,0 +1,69 @@ +name: Promote Image to Public Registry +description: Pull image from private registry and push to public registry + +inputs: + gcp_credentials: + description: 'GCP Credentials JSON' + required: true + private_registry: + description: 'Private container registry URL' + required: true + private_registry_host: + description: 'Private Google Artifact Registry hostname' + required: true + public_registry: + description: 'Public container registry URL' + required: true + role: + description: 'Role to promote' + required: true + tags: + description: 'Comma-separated list of tags to use' + required: true + +runs: + using: "composite" + steps: + - name: Authenticate with Google Cloud + uses: google-github-actions/auth@v1 + with: + credentials_json: ${{ inputs.gcp_credentials }} + + - name: Set up Google Cloud SDK + uses: google-github-actions/setup-gcloud@v1 + + - name: Authenticate with Private Docker Registry + run: | + gcloud auth configure-docker ${{ inputs.private_registry_host }} + shell: bash + + - name: Pull and Tag Images + shell: bash + run: | + # Convert comma-separated tags input into an array + IFS=',' read -ra TAGS <<< "${{ inputs.tags }}" + + for TAG in "${TAGS[@]}"; do + IMAGE_PRIVATE="${{ inputs.private_registry }}/${{ inputs.role }}:${TAG}" + IMAGE_PUBLIC="${{ inputs.public_registry }}/${{ inputs.role }}:${TAG}" + echo "Processing ${IMAGE_PRIVATE} -> ${IMAGE_PUBLIC}" + docker pull "${IMAGE_PRIVATE}" + docker tag "${IMAGE_PRIVATE}" "${IMAGE_PUBLIC}" + done + + - name: Authenticate with Public Registry + run: | + gcloud auth configure-docker + shell: bash + + - name: Push Images to Public Registry + shell: bash + run: | + # Convert comma-separated tags input into an array + IFS=',' read -ra TAGS <<< "${{ inputs.tags }}" + for TAG in "${TAGS[@]}"; do + IMAGE_PUBLIC="${{ inputs.public_registry }}/${{ inputs.role }}:${TAG}" + echo "Pushing Image ${IMAGE_PUBLIC} to Public registry" + docker push "${IMAGE_PUBLIC}" + done +