diff --git a/tests/end2end/runner.sh b/tests/end2end/runner.sh index d0d6aad..c256650 100755 --- a/tests/end2end/runner.sh +++ b/tests/end2end/runner.sh @@ -4,21 +4,46 @@ source ./utils.sh # Create an associated array mapping test name to exit code. declare -A TESTS +# Map testname to the number of times the test was run. +declare -A RUNS +TRIALS=3 + +# Each sub-directory in the current directory is a test. for test in ./*/ do + # Remove things like dir / from the test name testname="${test%*/}" testname="${testname#./}" yellow_echo "Running Test: ${testname}" - echo "============================================================" + echo "=================================================================" + # Move into the test dir pushd ${testname} - if [[ -f ./clean.sh ]]; then - ./clean.sh - else - ../clean.sh - fi - ./test.sh - TESTS[${testname}]=${?} + # Run the test up to ${TRIALS} times, stopping when it has a return of 0 + i=0 + # This is just a non-zero value, just to get us into the loop, it doesn't + # matter what the value is as it will be overwritten by the test return code. + return_code=-1 + while [[ "${i}" < "${TRIALS}" && ${return_code} != 0 ]]; do + # If there is a local clean script run that, otherwise run the global one. + # This is in the loop to ensure it is cleaned before each attempt. + if [[ -f ./clean.sh ]]; then + ./clean.sh + else + ../clean.sh + fi + if [[ "${i}" > 0 ]]; then + red_echo "${testname} failed, running trial $((i + 1))" + fi + ./test.sh + return_code="${?}" + i=$((i + 1)) + done + # Save the tests return value + TESTS[${testname}]="${return_code}" + # Save the number of times a test had to be run. + RUNS[${testname}]="${i}" + # Cleanup after the test, again, look for a local clean and use if found. if [[ -f ./clean.sh ]]; then ./clean.sh else @@ -32,7 +57,13 @@ echo "Test Summary:" for test in "${!TESTS[@]}" do if [[ "${TESTS[$test]}" == 0 ]]; then - green_echo "${test}" + green_echo "${test}" "n" + # Check if we had to re-run tests. + if [[ "${RUNS[$test]}" != 1 ]]; then + yellow_echo " (Had to run test '${test}' ${RUNS[$test]} times to pass)." + else + echo + fi else red_echo "${test}" fi diff --git a/tests/end2end/utils.sh b/tests/end2end/utils.sh index fb2f5fa..0c5c0e5 100644 --- a/tests/end2end/utils.sh +++ b/tests/end2end/utils.sh @@ -9,27 +9,37 @@ NORMAL="\033[0m" function color_echo { local text="${1}" local color="${2}" - echo -e "${color}${text}${NORMAL}" + local newline="${3}" + # If we provide a 3rd argument, don't include the newline, this lets us + # do two colors on one line easily. + if [[ -z "${newline}" ]]; then + echo -e "${color}${text}${NORMAL}" + else + echo -en "${color}${text}${NORMAL}" + fi } function green_echo { local text="${1}" - color_echo "${text}" "${GREEN}" + local newline="${2}" + color_echo "${text}" "${GREEN}" "${newline}" } function red_echo { local text="${1}" - color_echo "${text}" "${RED}" + local newline="${2}" + color_echo "${text}" "${RED}" "${newline}" } function yellow_echo { local text="${1}" - color_echo "${text}" "${YELLOW}" + local newline="${2}" + color_echo "${text}" "${YELLOW}" "${newline}" } function make_repo { echo "Making Git Repo." - git init + git init 2> /dev/null git branch -m main # Set the git user/email for the generated test repo git config --local user.email "git-theta-tester@example.com"