Add retries to the end-2-end tests. (#253)

This PR adds the ability to do a limited number of retries (3) for
end2end tests. Often the windows tests will fail during CI (an error
about the event loop being closed) crashing the whole end2end ci
pipeline.

As these are flaky, not broken, re-running the failed tests often fixes
the problem. But this is annoying to have to get and email and click
re-run each time it happens. This new version will rerun failed tests
automatically, logging when a test that passed needed multiple tires.
Hopefully this makes the windows issue less annoying going forward.

Author: blester125

tests/end2end/runner.sh

@@ -4,21 +4,46 @@ source ./utils.sh
 
 # Create an associated array mapping test name to exit code.
 declare -A TESTS
+# Map testname to the number of times the test was run.
+declare -A RUNS
 
+TRIALS=3
+
+# Each sub-directory in the current directory is a test.
 for test in ./*/
 do
+    # Remove things like dir / from the test name
     testname="${test%*/}"
     testname="${testname#./}"
     yellow_echo "Running Test: ${testname}"
-    echo "============================================================"
+    echo "================================================================="
+    # Move into the test dir
     pushd ${testname}
-    if [[ -f ./clean.sh ]]; then
-        ./clean.sh
-    else
-        ../clean.sh
-    fi
-    ./test.sh
-    TESTS[${testname}]=${?}
+    # Run the test up to ${TRIALS} times, stopping when it has a return of 0
+    i=0
+    # This is just a non-zero value, just to get us into the loop, it doesn't
+    # matter what the value is as it will be overwritten by the test return code.
+    return_code=-1
+    while [[ "${i}" < "${TRIALS}" && ${return_code} != 0 ]]; do
+        # If there is a local clean script run that, otherwise run the global one.
+        # This is in the loop to ensure it is cleaned before each attempt.
+        if [[ -f ./clean.sh ]]; then
+            ./clean.sh
+        else
+            ../clean.sh
+        fi
+        if [[ "${i}" > 0 ]]; then
+            red_echo "${testname} failed, running trial $((i + 1))"
+        fi
+        ./test.sh
+        return_code="${?}"
+        i=$((i + 1))
+    done
+    # Save the tests return value
+    TESTS[${testname}]="${return_code}"
+    # Save the number of times a test had to be run.
+    RUNS[${testname}]="${i}"
+    # Cleanup after the test, again, look for a local clean and use if found.
     if [[ -f ./clean.sh ]]; then
         ./clean.sh
     else
@@ -32,7 +57,13 @@ echo "Test Summary:"
 for test in "${!TESTS[@]}"
 do
     if [[ "${TESTS[$test]}" == 0 ]]; then
-        green_echo "${test}"
+        green_echo "${test}" "n"
+        # Check if we had to re-run tests.
+        if [[ "${RUNS[$test]}" != 1 ]]; then
+            yellow_echo " (Had to run test '${test}' ${RUNS[$test]} times to pass)."
+        else
+            echo
+        fi
     else
         red_echo "${test}"
     fi

tests/end2end/utils.sh

@@ -9,27 +9,37 @@ NORMAL="\033[0m"
 function color_echo {
     local text="${1}"
     local color="${2}"
-    echo -e "${color}${text}${NORMAL}"
+    local newline="${3}"
+    # If we provide a 3rd argument, don't include the newline, this lets us
+    # do two colors on one line easily.
+    if [[ -z "${newline}" ]]; then
+      echo -e "${color}${text}${NORMAL}"
+    else
+      echo -en "${color}${text}${NORMAL}"
+    fi
 }
 
 function green_echo {
     local text="${1}"
-    color_echo "${text}" "${GREEN}"
+    local newline="${2}"
+    color_echo "${text}" "${GREEN}" "${newline}"
 }
 
 function red_echo {
     local text="${1}"
-    color_echo "${text}" "${RED}"
+    local newline="${2}"
+    color_echo "${text}" "${RED}" "${newline}"
 }
 
 function yellow_echo {
     local text="${1}"
-    color_echo "${text}" "${YELLOW}"
+    local newline="${2}"
+    color_echo "${text}" "${YELLOW}" "${newline}"
 }
 
 function make_repo {
     echo "Making Git Repo."
-    git init
+    git init 2> /dev/null
     git branch -m main
     # Set the git user/email for the generated test repo
     git config --local user.email "[email protected]"