diff --git a/.github/workflows/copilot_deploy.yml b/.github/workflows/copilot_deploy.yml index 85e79ea3..668c90ac 100644 --- a/.github/workflows/copilot_deploy.yml +++ b/.github/workflows/copilot_deploy.yml @@ -240,6 +240,11 @@ jobs: e2e-test: needs: [ setup, docker-runner-build, docker-designer-build ] runs-on: ubuntu-latest + if: github.ref != 'refs/heads/main' + strategy: + fail-fast: false + matrix: + containers: [ 0, 1, 2, 3, 4 ] name: run e2e outputs: tag: ${{ steps.hashFile.outputs.tag }} @@ -298,28 +303,19 @@ jobs: - name: Run e2e tests id: e2e - run: yarn e2e-test cypress run + run: yarn e2e-test cypress run --spec $(node e2e-test/cypress-parallel.js ${{ matrix.containers }} 5) --reporter junit --reporter-options "mochaFile=./cypress/screenshots/results/test-output-[hash].xml,toConsole=true,overwrite=false" continue-on-error: true - name: Create Folders - run: mkdir -p ./e2e-test/cypress/screenshots/logs - - - name: Get Logs from localstack - run: docker logs localstack > ./e2e-test/cypress/screenshots/logs/localstack.log - - - name: Get Logs from runner - run: docker logs runner > ./e2e-test/cypress/screenshots/logs/runner.log - - - name: Get all the uploaded files in the bucket - run: docker exec localstack /bin/sh -c "awslocal s3api list-objects --bucket fsd-bucket" + run: mkdir -p ./e2e-test/cypress/screenshots/results - name: Upload E2E Test Report Application if: success() || failure() uses: actions/upload-artifact@v4.4.3 with: - name: e2e-test-report-application + name: e2e-test-report-application-${{ matrix.containers }} path: ./e2e-test/cypress/screenshots - retention-days: 5 + retention-days: 1 - name: Check for errors & exit the job run: | @@ -328,6 +324,55 @@ jobs: exit 1 fi + aggregate-test-results: + name: Aggregate test results + needs: e2e-test # Make sure this job runs after 'e2e-test' job + if: github.ref != 'refs/heads/main' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Download All Reports + run: | + for i in {0..4}; do + gh run download --name "e2e-test-report-application-$i" --dir ./result || echo "Report $i not found" + done + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Install XML Tools + run: sudo apt-get update && sudo apt-get install -y libxml2-utils + - name: Generate Test Suite Report + run: | + echo "Aggregating test suite results..." + echo -e "\n+---------------------------------------------------------------------------------------------------------------------------" + printf "| %-60s | %-19s | %-19s |\n" "File Name" "Tests Count" "Status" + echo -e "----------------------------------------------------------------------------------------------------------------------------" + total_tests=0 + total_features_tests=0 + + while read file; do + + file_name=$(xmllint --xpath 'string(//testsuite/@file)' "./$file" -o 2>/dev/null) + tests_in_file=$(xmllint --xpath 'string(//*[local-name()='\''testsuite'\''][2]/@tests)' "./$file" -o 2>/dev/null) + failures=$(xmllint --xpath 'string(//testsuite[2]/@failures)' "./$file" -o 2>/dev/null) + + status="Passed" + [ "$failures" -gt 0 ] && status="Failed" + + tests_in_file=${tests_in_file:-0} + tests_in_file=$((tests_in_file + 0)) + + total_tests=$((total_tests + tests_in_file)) + total_features_tests=$((total_features_tests + 1)) + + printf "| %-60s | %-19s | %-19s |\n" "$file_name" "$tests_in_file" "$status" + + done < <(find "result" -name '*.xml') + + echo -e "----------------------------------------------------------------------------------------------------------------------------" + echo "Total Tests: $total_tests" + echo "Total Features: $total_features_tests" + echo -e "----------------------------------------------------------------------------------------------------------------------------" + dev_deploy: needs: [ setup, docker-designer-build, docker-runner-build ] if: ${{ always() && contains(fromJSON(needs.setup.outputs.jobs_to_run), 'dev') && (! contains(needs.*.result, 'failure') ) && (! contains(needs.*.result, 'cancelled') ) }} diff --git a/e2e-test/cypress-parallel.js b/e2e-test/cypress-parallel.js new file mode 100644 index 00000000..e6ec56fb --- /dev/null +++ b/e2e-test/cypress-parallel.js @@ -0,0 +1,31 @@ +const fs = require('fs'); +const path = require('path'); + +const NODE_INDEX = Number(process.argv[2] || 1); +const NODE_TOTAL = Number(process.argv[3] || 1); + +const TEST_FOLDER = './e2e-test/cypress/e2e'; + +console.log(getSpecFiles().join(',')) + +function getSpecFiles() { + const allSpecFiles = traverse(TEST_FOLDER); + const node_index = NODE_INDEX + 1; + return allSpecFiles.sort() + .filter((_, index) => (index % NODE_TOTAL) === (node_index - 1)) + .map(file => file.replace(/^e2e-test\//, '')); +} + +function traverse(dir) { + let files = fs.readdirSync(dir); + files = files.map(file => { + const filePath = path.join(dir, file); + const stats = fs.statSync(filePath); + if (stats.isDirectory()) return traverse(filePath); + else if (stats.isFile()) return filePath; + }); + + return files + .reduce((all, folderContents) => all.concat(folderContents), []); + +} diff --git a/e2e-test/cypress/e2e/runner/exit(not-in-use-in-adapter).feature b/e2e-test/cypress/e2e/runner/exit(not-in-use-in-adapter).feature deleted file mode 100644 index 67bab51e..00000000 --- a/e2e-test/cypress/e2e/runner/exit(not-in-use-in-adapter).feature +++ /dev/null @@ -1,38 +0,0 @@ -Feature: Exit - As a user, - I want to be able to exit the service, - so that I can save my progress and return at a later date. - - Background: - Given the form "exit-expiry" exists - - @wip - Scenario: Service can be exited with date displayed - When I navigate to the "exit-expiry" form - Then I see "Save and come back later" - When I choose "lisbon" - And I select the button "Save and come back later" - And I enter "test@test.com" for "Enter your email address" - And I select the button "Save and exit" - Then I see "9 July 2024" - And I see "test@test.com" - - @wip - Scenario: A user can start exiting, then go back to the form - When I navigate to the "exit-expiry" form - Then I see "Save and come back later" - When I choose "lisbon" - And I select the button "Save and come back later" - And I go back to application overview - Then I see "First page" - - @wip - Scenario: An initialised session can be exited - Given the session is initialised for the exit form - When I go to the initialised session URL - And I select the button "Save and come back later" - And I enter "test@test.com" for "Enter your email address" - And I select the button "Save and exit" - Then I see "Your application to exit test has been saved" - # TODO: Mock the API in the e2e process so we can check for correct data sent. - diff --git a/e2e-test/cypress/e2e/runner/imageQualityPlayback(not-in-use-in-adapter).feature b/e2e-test/cypress/e2e/runner/imageQualityPlayback(not-in-use-in-adapter).feature deleted file mode 100644 index 33282f04..00000000 --- a/e2e-test/cypress/e2e/runner/imageQualityPlayback(not-in-use-in-adapter).feature +++ /dev/null @@ -1,26 +0,0 @@ -Feature: Image quality playback page - - Background: - Given the form "image-quality-playback" exists - - @wip - Scenario Outline: Handling upload - Given I navigate to the "image-quality-playback" form - When I upload a file that "" - Then I see the heading "" - Examples: - | case | heading | - | fails-ocr | Check your image | - | passes | Summary | - - @wip - Scenario Outline: Navigating away from the playback page - Given I navigate to the "image-quality-playback" form - When I upload a file that "fails-ocr" - And I choose "