Merge branch 'develop'

.github/workflows/azure_e2e_run_workflow.yml

@@ -0,0 +1,117 @@
+name: 'Azure e2e - Run Workflow'
+on:
+  schedule:
+    - cron: '0 16 * * *' # UTC 4pm, EST 11am, EDT 12pm
+  workflow_dispatch:
+
+env:
+  BROADBOT_TOKEN: '${{ secrets.BROADBOT_GITHUB_TOKEN }}' # github token for access to kick off a job in the private repo
+  RUN_NAME_SUFFIX: '${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt }}'
+
+jobs:
+
+  # This job provisions useful parameters for e2e tests
+  params-gen:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+    outputs:
+      project-name: ${{ steps.gen.outputs.project_name }}
+      bee-name: '${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt}}-dev'
+    steps:
+      - name: Generate a random billing project name
+        id: 'gen'
+        run: |
+          project_name=$(echo "tmp-billing-project-$(uuidgen)" | cut -c -30)
+          echo "project_name=${project_name}" >> $GITHUB_OUTPUT
+  
+  create-bee-workflow:
+    runs-on: ubuntu-latest
+    needs: [params-gen]
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+    steps:
+      - name: Dispatch to terra-github-workflows
+        uses: broadinstitute/workflow-dispatch@v3
+        with:
+          workflow: bee-create
+          repo: broadinstitute/terra-github-workflows
+          ref: refs/heads/main
+          token: ${{ env.BROADBOT_TOKEN }}
+          # NOTE: Opting to use "prod" instead of custom tag since I specifically want to test against the current prod state
+          # NOTE: For testing/development purposes I'm using dev
+          inputs: '{ "bee-name": "${{ needs.params-gen.outputs.bee-name }}", "version-template": "dev", "bee-template-name": "rawls-e2e-azure-tests"}'
+
+  create-and-attach-billing-project-to-landing-zone-workflow:
+    runs-on: ubuntu-latest
+    needs: [create-bee-workflow, params-gen]
+    steps:
+      - name: dispatch to terra-github-workflows
+        uses: broadinstitute/workflow-dispatch@v3
+        with:
+          workflow: attach-billing-project-to-landing-zone.yaml
+          repo: broadinstitute/terra-github-workflows
+          ref: refs/heads/main
+          token: ${{ env.BROADBOT_TOKEN }}
+          inputs: '{
+            "run-name": "attach-billing-project-to-landing-zone-${{ env.RUN_NAME_SUFFIX }}",
+            "bee-name": "${{ needs.params-gen.outputs.bee-name }}",
+            "billing-project": "${{ needs.params-gen.outputs.project-name }}",
+            "service-account": "[email protected]" }'
+
+  run-cromwell-az-e2e:
+    needs: [params-gen, create-and-attach-billing-project-to-landing-zone-workflow]
+    permissions:
+      contents: read
+      id-token: write
+    uses: "broadinstitute/dsp-reusable-workflows/.github/workflows/cromwell-az-e2e-test.yaml@main"
+    with:
+      bee-name: "${{ needs.params-gen.outputs.bee-name }}"
+      billing-project-name: "${{ needs.params-gen.outputs.project-name }}"
+
+  delete-billing-project-v2-from-bee-workflow:
+    continue-on-error: true
+    runs-on: ubuntu-latest
+    needs: [run-cromwell-az-e2e, create-and-attach-billing-project-to-landing-zone-workflow, params-gen]
+    if: always()
+    steps:
+      - name: dispatch to terra-github-workflows
+        uses: broadinstitute/workflow-dispatch@v3
+        with:
+          workflow: .github/workflows/delete-billing-project-v2-from-bee.yaml
+          repo: broadinstitute/terra-github-workflows
+          ref: refs/heads/main
+          token: ${{ env.BROADBOT_TOKEN }}
+          inputs: '{ 
+            "run-name": "delete-billing-project-v2-from-bee-${{ env.RUN_NAME_SUFFIX }}",
+            "bee-name": "${{ needs.params-gen.outputs.bee-name }}", 
+            "billing-project": "${{ needs.params-gen.outputs.project-name }}", 
+            "service-account": "[email protected]", 
+            "silent-on-failure": "false" }'
+
+  destroy-bee-workflow:
+    runs-on: ubuntu-latest
+    needs: [params-gen, create-bee-workflow, delete-billing-project-v2-from-bee-workflow]
+    if: always()
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+    steps:
+      - name: dispatch to terra-github-workflows
+        uses: broadinstitute/workflow-dispatch@v3
+        with:
+          workflow: bee-destroy.yaml
+          repo: broadinstitute/terra-github-workflows
+          ref: refs/heads/main
+          token: ${{ env.BROADBOT_TOKEN }}
+          inputs: '{ "bee-name": "${{ needs.params-gen.outputs.bee-name }}" }'
+          wait-for-completion: true
+
+  report-workflow:
+    uses: broadinstitute/sherlock/.github/workflows/client-report-workflow.yaml@main
+    with:
+      notify-slack-channels-upon-workflow-failure: "#cromwell_jenkins_ci_errors"
+    permissions:
+      id-token: write

.github/workflows/chart_update_on_merge.yml

@@ -60,7 +60,7 @@ jobs:
       run: |
         set -e
         cd cromwell
-        sbt -Dproject.isSnapshot=false -Dproject.isRelease=false dockerBuildAndPush
+        sbt -Dproject.isSnapshot=false dockerBuildAndPush
     - name: Deploy to dev and board release train (Cromwell)
       uses: broadinstitute/repository-dispatch@master
       with:

.github/workflows/consumer_contract_tests.yml

@@ -29,18 +29,25 @@ name: Consumer contract tests
 #     Consumer kicks off can-i-deploy on process to determine if changes can be promoted and used for deployment.
 #
 # NOTE: The publish-contracts workflow will use the latest commit of the branch that triggers this workflow to publish the unique consumer contract version to Pact Broker.
-
 on:
   pull_request:
+    branches:
+      - develop
     paths-ignore:
       - 'README.md'
   push:
+    branches:
+      - develop
     paths-ignore:
       - 'README.md'
   merge_group:
     branches:
       - develop
 
+env:
+  PUBLISH_CONTRACTS_RUN_NAME: 'publish-contracts-${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt }}'
+  CAN_I_DEPLOY_RUN_NAME: 'can-i-deploy-${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt }}'
+
 jobs:
   init-github-context:
     runs-on: ubuntu-latest
@@ -52,6 +59,23 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      # Construct a version string like `87-9c6c439`. Adapted from `chart_update_on_merge.yml`.
+      - name: Find Cromwell short SHA
+        run: |
+          set -e
+          echo "CROMWELL_SHORT_SHA=`git rev-parse --short $GITHUB_SHA`" >> $GITHUB_ENV
+      - name: Find Cromwell release number
+        run: |
+          set -e
+          previous_version=$(curl -X GET https://api.github.com/repos/broadinstitute/cromwell/releases/latest | jq .tag_name | xargs)
+          if ! [[ "${previous_version}" =~ ^[0-9][0-9]+$ ]]; then
+            exit 1
+          fi
+          echo "CROMWELL_NUMBER=$((previous_version + 1))" >> $GITHUB_ENV
+      - name: Save complete image ID
+        run: |
+          echo "CROMWELL_VERSION=`echo "$CROMWELL_NUMBER-$CROMWELL_SHORT_SHA"`" >> $GITHUB_ENV
+
       - name: Extract branch
         id: extract-branch
         run: |
@@ -70,9 +94,9 @@ jobs:
           fi
           echo "CURRENT_BRANCH=${GITHUB_REF/refs\/heads\//""}" >> $GITHUB_ENV
           echo "CURRENT_SHA=$GITHUB_SHA" >> $GITHUB_ENV
-          
+
           echo "repo-branch=${GITHUB_REF/refs\/heads\//""}" >> $GITHUB_OUTPUT
-          echo "repo-version=${GITHUB_SHA}" >> $GITHUB_OUTPUT
+          echo "repo-version=${CROMWELL_VERSION}" >> $GITHUB_OUTPUT
           echo "fork=${FORK}" >> $GITHUB_OUTPUT
 
       - name: Is PR triggered by forked repo?
@@ -88,11 +112,12 @@ jobs:
           echo "repo-version=${{ steps.extract-branch.outputs.repo-version }}"
           echo "fork=${{ steps.extract-branch.outputs.fork }}"
 
-  cromwell-consumer-contract-tests:
+  cromwell-contract-tests:
     runs-on: ubuntu-latest
     needs: [init-github-context]
     outputs:
-      pact-b64: ${{ steps.encode-pact.outputs.pact-b64 }}
+      pact-b64-drshub: ${{ steps.encode-pact.outputs.pact-b64-drshub }}
+      pact-b64-cbas: ${{ steps.encode-pact.outputs.pact-b64-cbas }}
 
     steps:
       - uses: actions/checkout@v3
@@ -108,36 +133,68 @@ jobs:
       - name: Output consumer contract as non-breaking base64 string
         id: encode-pact
         run: |
+          set -e
           cd pact4s
-          NON_BREAKING_B64=$(cat target/pacts/cromwell-consumer-drshub-provider.json | base64 -w 0)
-          echo "pact-b64=${NON_BREAKING_B64}" >> $GITHUB_OUTPUT
+          NON_BREAKING_B64_DRSHUB=$(cat target/pacts/cromwell-drshub.json | base64 -w 0)
+          NON_BREAKING_B64_CBAS=$(cat target/pacts/cromwell-cbas.json | base64 -w 0)
+          echo "pact-b64-drshub=${NON_BREAKING_B64_DRSHUB}" >> $GITHUB_OUTPUT
+          echo "pact-b64-cbas=${NON_BREAKING_B64_CBAS}" >> $GITHUB_OUTPUT
 
   # Prevent untrusted sources from using PRs to publish contracts
   # since access to secrets is not allowed.
   publish-contracts:
     runs-on: ubuntu-latest
     if: ${{ needs.init-github-context.outputs.fork == 'false' ||  needs.init-github-context.outputs.fork == ''}}
-    needs: [init-github-context, cromwell-consumer-contract-tests]
+    needs: [init-github-context, cromwell-contract-tests]
     steps:
-      - name: Dispatch to terra-github-workflows
-        uses: broadinstitute/workflow-dispatch@v3
+      - name: Dispatch drshub to terra-github-workflows
+        uses: broadinstitute/[email protected]
+        with:
+          run-name: "${{ env.PUBLISH_CONTRACTS_RUN_NAME }}"
+          workflow: .github/workflows/publish-contracts.yaml
+          repo: broadinstitute/terra-github-workflows
+          ref: refs/heads/main
+          token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} # github token for access to kick off a job in the private repo
+          inputs: '{
+            "run-name": "${{ env.PUBLISH_CONTRACTS_RUN_NAME }}",
+            "pact-b64": "${{ needs.cromwell-contract-tests.outputs.pact-b64-drshub }}",
+            "repo-owner": "${{ github.repository_owner }}",
+            "repo-name": "${{ github.event.repository.name }}",
+            "repo-branch": "${{ needs.init-github-context.outputs.repo-branch }}",
+            "release-tag": "${{ needs.init-github-context.outputs.repo-version }}"
+          }'
+      - name: Dispatch cbas to terra-github-workflows
+        uses: broadinstitute/[email protected]
         with:
+          run-name: "${{ env.PUBLISH_CONTRACTS_RUN_NAME }}"
           workflow: .github/workflows/publish-contracts.yaml
           repo: broadinstitute/terra-github-workflows
           ref: refs/heads/main
           token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} # github token for access to kick off a job in the private repo
-          inputs: '{ "pact-b64": "${{ needs.cromwell-consumer-contract-tests.outputs.pact-b64 }}", "repo-owner": "${{ github.repository_owner }}", "repo-name": "${{ github.event.repository.name }}", "repo-branch": "${{ needs.init-github-context.outputs.repo-branch }}" }'
+          inputs: '{
+            "run-name": "${{ env.PUBLISH_CONTRACTS_RUN_NAME }}",
+            "pact-b64": "${{ needs.cromwell-contract-tests.outputs.pact-b64-cbas }}",
+            "repo-owner": "${{ github.repository_owner }}",
+            "repo-name": "${{ github.event.repository.name }}",
+            "repo-branch": "${{ needs.init-github-context.outputs.repo-branch }}",
+            "release-tag": "${{ needs.init-github-context.outputs.repo-version }}"
+          }'
 
   can-i-deploy:
     runs-on: ubuntu-latest
     if: ${{ needs.init-github-context.outputs.fork == 'false' ||  needs.init-github-context.outputs.fork == ''}}
     needs: [ init-github-context, publish-contracts ]
     steps:
       - name: Dispatch to terra-github-workflows
-        uses: broadinstitute/workflow-dispatch@v3
+        uses: broadinstitute/workflow-dispatch@v4.0.0
         with:
+          run-name: "${{ env.CAN_I_DEPLOY_RUN_NAME }}"
           workflow: .github/workflows/can-i-deploy.yaml
           repo: broadinstitute/terra-github-workflows
           ref: refs/heads/main
           token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} # github token for access to kick off a job in the private repo
-          inputs: '{ "pacticipant": "cromwell-consumer", "version": "${{ needs.init-github-context.outputs.repo-version }}" }'
+          inputs: '{
+            "run-name": "${{ env.CAN_I_DEPLOY_RUN_NAME }}",
+            "pacticipant": "cromwell",
+            "version": "${{ needs.init-github-context.outputs.repo-version }}"
+          }'

.github/workflows/docker_build_test.yml

@@ -34,4 +34,10 @@ jobs:
         run: |
           set -e
           cd cromwell
-          sbt -Dproject.isSnapshot=false -Dproject.isRelease=false docker
+          sbt -Dproject.isSnapshot=false docker
+      # Rarely used but we really want it always working for emergencies
+      - name: Build Cromwell Debug Docker
+        run: |
+          set -e
+          cd cromwell
+          sbt -Dproject.isDebug=true docker

.github/workflows/integration_tests.yml

@@ -16,6 +16,12 @@ on:
 permissions:
   contents: read
 
+concurrency:
+  # Don't run this workflow concurrently on the same branch
+  group: ${{ github.workflow }}-${{ github.ref }}
+  # For PRs, don't wait for completion of existing runs, cancel them instead
+  cancel-in-progress: ${{ github.ref != 'develop' }}
+
 jobs:
   integration-tests:
     strategy:
@@ -32,6 +38,9 @@ jobs:
           - build_type: centaurPapiV2beta
             build_mysql: 5.7
             friendly_name: Centaur Papi V2 Beta with MySQL 5.7
+          - build_type: centaurPapiV2betaRestart
+            build_mysql: 5.7
+            friendly_name: Centaur Papi V2 Beta (restart)
           - build_type: dbms
             friendly_name: DBMS
           - build_type: centaurTes
@@ -101,6 +110,10 @@ jobs:
         set -e
         echo Running test.sh
         ./src/ci/bin/test.sh
+    - uses: codecov/codecov-action@v4
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        fail_ci_if_error: false # Tolerate missing codecov reports, since not all suites generate them.
     # always() is some github magic that forces the following step to run, even when the previous fails.
     # Without it, the if statement won't be evaluated on a test failure.
     - uses: ravsamhq/notify-slack-action@v2

.github/workflows/scalafmt-check.yml

@@ -0,0 +1,31 @@
+name: 'ScalaFmt Check'
+
+# This GitHub Action runs the ScalaFmt linting tool on the entire codebase.
+# It fails if any files are not formatted properly.
+# If it is triggered by someone commenting 'scalafmt' on a PR, it will first format, commit, and push formatted code
+# to the branch.
+
+run-name: ${{ format('ScalaFmt Check on {0}', github.ref_name) }}
+
+on:
+  workflow_dispatch:
+  push:
+
+permissions:
+  contents: read
+
+jobs:
+  run-scalafmt-check:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.target-branch }}
+      - uses: ./.github/set_up_cromwell_action
+        with:
+          cromwell_repo_token: ${{ secrets.BROADBOT_GITHUB_TOKEN }}
+      - name: Run ScalaFmt
+        run: |
+          sbt scalafmtCheckAll
+        working-directory: ${{ github.workspace }}