Add retries to the end-2-end tests.

tests/end2end/runner.sh

@@ -4,21 +4,46 @@ source ./utils.sh
 
 # Create an associated array mapping test name to exit code.
 declare -A TESTS
+# Map testname to the number of times the test was run.
+declare -A RUNS
 
+TRIALS=3
+
+# Each sub-directory in the current directory is a test.
 for test in ./*/
 do
+    # Remove things like dir / from the test name
     testname="${test%*/}"
     testname="${testname#./}"
     yellow_echo "Running Test: ${testname}"
-    echo "============================================================"
+    echo "================================================================="
+    # Move into the test dir
     pushd ${testname}
-    if [[ -f ./clean.sh ]]; then
-        ./clean.sh
-    else
-        ../clean.sh
-    fi
-    ./test.sh
-    TESTS[${testname}]=${?}
+    # Run the test up to ${TRIALS} times, stopping when it has a return of 0
+    i=0
+    # This is just a non-zero value, just to get us into the loop, it doesn't
+    # matter what the value is as it will be overwritten by the test return code.
+    return_code=-1
+    while [[ "${i}" < "${TRIALS}" && ${return_code} != 0 ]]; do
+        # If there is a local clean script run that, otherwise run the global one.
+        # This is in the loop to ensure it is cleaned before each attempt.
+        if [[ -f ./clean.sh ]]; then
+            ./clean.sh
+        else
+            ../clean.sh
+        fi
+        if [[ "${i}" > 0 ]]; then
+            red_echo "${testname} failed, running trial $((i + 1))"
+        fi
+        ./test.sh
+        return_code="${?}"
+        i=$((i + 1))
+    done
+    # Save the tests return value
+    TESTS[${testname}]="${return_code}"
+    # Save the number of times a test had to be run.
+    RUNS[${testname}]="${i}"
+    # Cleanup after the test, again, look for a local clean and use if found.
     if [[ -f ./clean.sh ]]; then
         ./clean.sh
     else
@@ -32,7 +57,13 @@ echo "Test Summary:"
 for test in "${!TESTS[@]}"
 do
     if [[ "${TESTS[$test]}" == 0 ]]; then
-        green_echo "${test}"
+        green_echo "${test}" "n"
+        # Check if we had to re-run tests.
+        if [[ "${RUNS[$test]}" != 1 ]]; then
+            yellow_echo " (Had to run test '${test}' ${RUNS[$test]} times to pass)."
+        else
+            echo
+        fi
     else
         red_echo "${test}"
     fi

tests/end2end/utils.sh

@@ -9,27 +9,37 @@ NORMAL="\033[0m"
 function color_echo {
     local text="${1}"
     local color="${2}"
-    echo -e "${color}${text}${NORMAL}"
+    local newline="${3}"
+    # If we provide a 3rd argument, don't include the newline, this lets us
+    # do two colors on one line easily.
+    if [[ -z "${newline}" ]]; then
+      echo -e "${color}${text}${NORMAL}"
+    else
+      echo -en "${color}${text}${NORMAL}"
+    fi
 }
 
 function green_echo {
     local text="${1}"
-    color_echo "${text}" "${GREEN}"
+    local newline="${2}"
+    color_echo "${text}" "${GREEN}" "${newline}"
 }
 
 function red_echo {
     local text="${1}"
-    color_echo "${text}" "${RED}"
+    local newline="${2}"
+    color_echo "${text}" "${RED}" "${newline}"
 }
 
 function yellow_echo {
     local text="${1}"
-    color_echo "${text}" "${YELLOW}"
+    local newline="${2}"
+    color_echo "${text}" "${YELLOW}" "${newline}"
 }
 
 function make_repo {
     echo "Making Git Repo."
-    git init
+    git init 2> /dev/null
     git branch -m main
     # Set the git user/email for the generated test repo
     git config --local user.email "[email protected]"