-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Upgrading the development process to reduse time taken for the docker…
… build and e2e run# This is a combination of 2 commits. test fix test fix test fix
- Loading branch information
1 parent
17d4891
commit cd32830
Showing
8 changed files
with
275 additions
and
250 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -116,31 +116,6 @@ jobs: | |
| echo LAST_COMMIT_GH=runner >> ./runner/.env | ||
| cat ./runner/.env | ||
| - name: Run Unit test before docker build | ||
| run: | | ||
| echo "-- Git submodule initialize into local --" | ||
| git submodule update --init | ||
| echo "-- Pulling git submodules into local --" | ||
| git pull --recurse-submodules | ||
| echo "-- Installing digital-form-builder-adapter locally --" | ||
| node update-package.js | ||
| yarn install | ||
| echo "-- Building digital-form-builder-adapter locally --" | ||
| yarn setup | ||
| echo "-- Building digital-form-builder locally --" | ||
| # shellcheck disable=SC2164 | ||
| cd digital-form-builder | ||
| yarn | ||
| echo "-- Building digital-form-builder model locally --" | ||
| yarn model build | ||
| echo "-- Building digital-form-builder queue-model locally --" | ||
| yarn queue-model build | ||
| echo "-- Building digital-form-builder-adapter model locally --" | ||
| cd .. | ||
| yarn model build | ||
| echo "-- Running unit tests --" | ||
| yarn runner test-cov | ||
| - name: Set up Docker Buildx property | ||
| run: echo "DOCKER_BUILDKIT=1" >> $GITHUB_ENV | ||
|
|
||
|
|
@@ -240,6 +215,11 @@ jobs: | |
| e2e-test: | ||
| needs: [ setup, docker-runner-build, docker-designer-build ] | ||
| runs-on: ubuntu-latest | ||
| if: github.ref != 'refs/heads/main' | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| containers: [ 0, 1, 2, 3, 4 ] | ||
| name: run e2e | ||
| outputs: | ||
| tag: ${{ steps.hashFile.outputs.tag }} | ||
|
|
@@ -298,28 +278,19 @@ jobs: | |
| - name: Run e2e tests | ||
| id: e2e | ||
| run: yarn e2e-test cypress run | ||
| run: yarn e2e-test cypress run --spec $(node e2e-test/cypress-parallel.js ${{ matrix.containers }} 5) --reporter junit --reporter-options "mochaFile=./cypress/screenshots/results/test-output-[hash].xml,toConsole=true,overwrite=false" | ||
| continue-on-error: true | ||
|
|
||
| - name: Create Folders | ||
| run: mkdir -p ./e2e-test/cypress/screenshots/logs | ||
|
|
||
| - name: Get Logs from localstack | ||
| run: docker logs localstack > ./e2e-test/cypress/screenshots/logs/localstack.log | ||
|
|
||
| - name: Get Logs from runner | ||
| run: docker logs runner > ./e2e-test/cypress/screenshots/logs/runner.log | ||
|
|
||
| - name: Get all the uploaded files in the bucket | ||
| run: docker exec localstack /bin/sh -c "awslocal s3api list-objects --bucket fsd-bucket" | ||
| run: mkdir -p ./e2e-test/cypress/screenshots/results | ||
|
|
||
| - name: Upload E2E Test Report Application | ||
| if: success() || failure() | ||
| uses: actions/[email protected] | ||
| with: | ||
| name: e2e-test-report-application | ||
| name: e2e-test-report-application-${{ matrix.containers }} | ||
| path: ./e2e-test/cypress/screenshots | ||
| retention-days: 5 | ||
| retention-days: 1 | ||
|
|
||
| - name: Check for errors & exit the job | ||
| run: | | ||
|
|
@@ -328,6 +299,55 @@ jobs: | |
| exit 1 | ||
| fi | ||
| aggregate-test-results: | ||
| name: Aggregate test results | ||
| needs: e2e-test # Make sure this job runs after 'e2e-test' job | ||
| if: github.ref != 'refs/heads/main' | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - name: Download All Reports | ||
| run: | | ||
| for i in {0..4}; do | ||
| gh run download --name "e2e-test-report-application-$i" --dir ./result || echo "Report $i not found" | ||
| done | ||
| env: | ||
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | ||
| - name: Install XML Tools | ||
| run: sudo apt-get update && sudo apt-get install -y libxml2-utils | ||
| - name: Generate Test Suite Report | ||
| run: | | ||
| echo "Aggregating test suite results..." | ||
| echo -e "\n+---------------------------------------------------------------------------------------------------------------------------" | ||
| printf "| %-60s | %-19s | %-19s |\n" "File Name" "Tests Count" "Status" | ||
| echo -e "----------------------------------------------------------------------------------------------------------------------------" | ||
| total_tests=0 | ||
| total_features_tests=0 | ||
| while read file; do | ||
| file_name=$(xmllint --xpath 'string(//testsuite/@file)' "./$file" -o 2>/dev/null) | ||
| tests_in_file=$(xmllint --xpath 'string(//*[local-name()='\''testsuite'\''][2]/@tests)' "./$file" -o 2>/dev/null) | ||
| failures=$(xmllint --xpath 'string(//testsuite[2]/@failures)' "./$file" -o 2>/dev/null) | ||
| status="Passed" | ||
| [ "$failures" -gt 0 ] && status="Failed" | ||
| tests_in_file=${tests_in_file:-0} | ||
| tests_in_file=$((tests_in_file + 0)) | ||
| total_tests=$((total_tests + tests_in_file)) | ||
| total_features_tests=$((total_features_tests + 1)) | ||
| printf "| %-60s | %-19s | %-19s |\n" "$file_name" "$tests_in_file" "$status" | ||
| done < <(find "result" -name '*.xml') | ||
| echo -e "----------------------------------------------------------------------------------------------------------------------------" | ||
| echo "Total Tests: $total_tests" | ||
| echo "Total Features: $total_features_tests" | ||
| echo -e "----------------------------------------------------------------------------------------------------------------------------" | ||
| dev_deploy: | ||
| needs: [ setup, docker-designer-build, docker-runner-build ] | ||
| if: ${{ always() && contains(fromJSON(needs.setup.outputs.jobs_to_run), 'dev') && (! contains(needs.*.result, 'failure') ) && (! contains(needs.*.result, 'cancelled') ) }} | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,94 @@ | ||
| name: Publish base image | ||
|
|
||
| on: | ||
| schedule: | ||
| - cron: '0 3 * * *' # Run daily at 3:00 AM UTC | ||
| workflow_dispatch: # Allow manual trigger | ||
|
|
||
| env: | ||
| DOCKER_REGISTRY: ghcr.io | ||
| IMAGE_NAME: "funding-service-design-form-runner-adapter-base" | ||
| IMAGE_REPO_PATH: "ghcr.io/${{github.repository_owner}}" | ||
|
|
||
| jobs: | ||
| form-runner-base-image: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Checkout repo | ||
| uses: actions/checkout@v4 | ||
|
|
||
| - name: "Set version in env" | ||
| id: set-version | ||
| run: | | ||
| source version | ||
| echo "VERSION=$VERSION" >> $GITHUB_ENV | ||
| - name: Docker metadata | ||
| id: metadata | ||
| uses: docker/metadata-action@v4 | ||
| with: | ||
| images: ${{env.IMAGE_REPO_PATH}}/${{env.IMAGE_NAME}} | ||
| tags: | | ||
| type=sha,format=long | ||
| type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }} | ||
| type=raw,value=${{env.VERSION}},enable=${{ github.ref == format('refs/heads/{0}', 'main') }} | ||
| type=raw,value=latest | ||
| type=ref,event=branch | ||
| - name: Log in to the Container registry | ||
| uses: docker/login-action@v1 | ||
| with: | ||
| registry: ${{ env.DOCKER_REGISTRY }} | ||
| username: ${{ github.actor }} | ||
| password: ${{ secrets.GITHUB_TOKEN }} | ||
|
|
||
| - uses: actions/checkout@v4 | ||
| - name: Use Node.js | ||
| uses: actions/setup-node@v3 | ||
| with: | ||
| node-version: "20.x" | ||
|
|
||
| - name: Get yarn cache directory path | ||
| id: yarn-cache-dir-path | ||
| run: echo "::set-output name=dir::$(yarn config get cacheFolder)" | ||
|
|
||
| - name: Get into the directory | ||
| id: change-dir | ||
| run: yarn config set enableImmutableInstalls false | ||
|
|
||
| - name: Create .env for runner workspace | ||
| run: | | ||
| touch ./runner/.env | ||
| echo LAST_TAG_GH=runner >> ./runner/.env | ||
| echo LAST_COMMIT_GH=runner >> ./runner/.env | ||
| cat ./runner/.env | ||
| - name: Set up Docker Buildx property | ||
| run: echo "DOCKER_BUILDKIT=1" >> $GITHUB_ENV | ||
|
|
||
| - name: Cache Docker layers | ||
| uses: actions/cache@v4 | ||
| with: | ||
| path: ~/.buildx/cache | ||
| key: ${{ runner.os }}-buildx-runner-${{ hashFiles('./runner/Dockerfile.base') }}-${{ hashFiles('./runner/package.json') }} | ||
| restore-keys: | | ||
| ${{ runner.os }}-buildx-runner-${{ hashFiles('./runner/Dockerfile.base') }}-${{ hashFiles('./runner/package.json') }} | ||
| - name: Set up Docker Buildx | ||
| uses: docker/setup-buildx-action@v3 | ||
| with: | ||
| driver: docker-container | ||
|
|
||
| - name: Build and push docker image | ||
| uses: docker/build-push-action@v4 | ||
| with: | ||
| context: . | ||
| tags: ${{ steps.metadata.outputs.tags}} | ||
| labels: ${{ steps.metadata.outputs.labels }} | ||
| push: true | ||
| file: ./runner/Dockerfile.base | ||
| build-args: | | ||
| LAST_TAG='${{env.VERSION}}' | ||
| LAST_COMMIT='${{ github.sha }}' | ||
| cache-from: type=gha,scope=buildx | ||
| cache-to: type=gha,mode=max,scope=buildx |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| const fs = require('fs'); | ||
| const path = require('path'); | ||
|
|
||
| const NODE_INDEX = Number(process.argv[2] || 1); | ||
| const NODE_TOTAL = Number(process.argv[3] || 1); | ||
|
|
||
| const TEST_FOLDER = './e2e-test/cypress/e2e'; | ||
|
|
||
| console.log(getSpecFiles().join(',')) | ||
|
|
||
| function getSpecFiles() { | ||
| const allSpecFiles = traverse(TEST_FOLDER); | ||
| const node_index = NODE_INDEX + 1; | ||
| return allSpecFiles.sort() | ||
| .filter((_, index) => (index % NODE_TOTAL) === (node_index - 1)) | ||
| .map(file => file.replace(/^e2e-test\//, '')); | ||
| } | ||
|
|
||
| function traverse(dir) { | ||
| let files = fs.readdirSync(dir); | ||
| files = files.map(file => { | ||
| const filePath = path.join(dir, file); | ||
| const stats = fs.statSync(filePath); | ||
| if (stats.isDirectory()) return traverse(filePath); | ||
| else if (stats.isFile()) return filePath; | ||
| }); | ||
|
|
||
| return files | ||
| .reduce((all, folderContents) => all.concat(folderContents), []); | ||
|
|
||
| } |
Oops, something went wrong.