retry aemm installs (#237)

Author: bwagner5

test/e2e/cordon-only-test

@@ -57,7 +57,7 @@ aemm_helm_args=(
     aemm_helm_args+=("${common_helm_args[@]}")
 
 set -x
-helm "${aemm_helm_args[@]}"
+retry 5 helm "${aemm_helm_args[@]}"
 set +x
 
 emtp_helm_args=(

test/e2e/imds-v2-test

@@ -80,7 +80,7 @@ aemm_helm_args=(
     aemm_helm_args+=("${common_helm_args[@]}")
 
 set -x
-helm "${aemm_helm_args[@]}"
+retry 5 helm "${aemm_helm_args[@]}"
 set +x
 
 TAINT_CHECK_CYCLES=15

test/e2e/maintenance-event-cancellation-test

@@ -84,7 +84,7 @@ aemm_helm_args=(
     aemm_helm_args+=("${common_helm_args[@]}")
 
 set -x
-helm "${aemm_helm_args[@]}"
+retry 5 helm "${aemm_helm_args[@]}"
 set +x
 
 TAINT_CHECK_CYCLES=15

test/e2e/maintenance-event-dry-run-test

@@ -84,7 +84,7 @@ aemm_helm_args=(
     aemm_helm_args+=("${common_helm_args[@]}")
 
 set -x
-helm "${aemm_helm_args[@]}"
+retry 5 helm "${aemm_helm_args[@]}"
 set +x
 
 logs=0

test/e2e/maintenance-event-reboot-test

@@ -78,7 +78,7 @@ aemm_helm_args=(
     aemm_helm_args+=("${common_helm_args[@]}")
 
 set -x
-helm "${aemm_helm_args[@]}"
+retry 5 helm "${aemm_helm_args[@]}"
 set +x
 
 TAINT_CHECK_CYCLES=15

test/e2e/maintenance-event-test

@@ -86,7 +86,7 @@ aemm_helm_args=(
     aemm_helm_args+=("${common_helm_args[@]}")
 
 set -x
-helm "${aemm_helm_args[@]}"
+retry 5 helm "${aemm_helm_args[@]}"
 set +x
 
 TAINT_CHECK_CYCLES=15

test/e2e/prometheus-metrics-test

@@ -20,6 +20,10 @@ common_helm_args=()
 [[ "${TEST_WINDOWS-}" == "true" ]] && common_helm_args+=(--set targetNodeOs="windows")
 [[ -n "${NTH_WORKER_LABEL-}" ]] && common_helm_args+=(--set nodeSelector."$NTH_WORKER_LABEL")
 
+helm repo add stable https://kubernetes-charts.storage.googleapis.com
+helm repo update
+retry 5 helm install prometheus-operator stable/prometheus-operator --set prometheusOperator.admissionWebhooks.enabled="false"
+
 anth_helm_args=(
   upgrade
   --install
@@ -35,6 +39,7 @@ anth_helm_args=(
   --set enableSpotInterruptionDraining="true"
   --set taintNode="true"
   --set enablePrometheusServer="true"
+  --set podMonitor.create="true"
   --set tolerations=""
 )
 [[ -n "${NODE_TERMINATION_HANDLER_DOCKER_PULL_POLICY-}" ]] &&
@@ -80,7 +85,7 @@ aemm_helm_args=(
     aemm_helm_args+=("${common_helm_args[@]}")
 
 set -x
-helm "${aemm_helm_args[@]}"
+retry 5 helm "${aemm_helm_args[@]}"
 set +x
 
 TAINT_CHECK_CYCLES=15

test/e2e/spot-interruption-dry-run-test

@@ -82,7 +82,7 @@ aemm_helm_args=(
     aemm_helm_args+=("${common_helm_args[@]}")
 
 set -x
-helm "${aemm_helm_args[@]}"
+retry 5 helm "${aemm_helm_args[@]}"
 set +x
 
 logs=0

test/e2e/spot-interruption-test

@@ -86,7 +86,7 @@ aemm_helm_args=(
     aemm_helm_args+=("${common_helm_args[@]}")
 
 set -x
-helm "${aemm_helm_args[@]}"
+retry 5 helm "${aemm_helm_args[@]}"
 set +x
 
 TAINT_CHECK_CYCLES=15

test/e2e/webhook-http-proxy-test

@@ -93,7 +93,7 @@ aemm_helm_args=(
     aemm_helm_args+=("${common_helm_args[@]}")
 
 set -x
-helm "${aemm_helm_args[@]}"
+retry 5 helm "${aemm_helm_args[@]}"
 set +x
 
 emtp_helm_args=(

test/e2e/webhook-secret-test

@@ -42,7 +42,7 @@ aemm_helm_args=(
     aemm_helm_args+=("${common_helm_args[@]}")
 
 set -x
-helm "${aemm_helm_args[@]}"
+retry 5 helm "${aemm_helm_args[@]}"
 set +x
 
 emtp_helm_args=(

test/e2e/webhook-test

@@ -83,7 +83,7 @@ aemm_helm_args=(
     aemm_helm_args+=("${common_helm_args[@]}")
 
 set -x
-helm "${aemm_helm_args[@]}"
+retry 5 helm "${aemm_helm_args[@]}"
 set +x
 
 TAINT_CHECK_CYCLES=15

test/eks-cluster-test/run-test

@@ -62,6 +62,31 @@ function exit_and_fail {
     exit 1
 }
 
+function retry {
+  local retries=$1
+  shift
+
+  local count=0
+  set +e
+  trap "set -e" RETURN
+  until "$@"; do
+    exit=$?
+    set -e
+    wait=$((2 ** $count))
+    count=$(($count + 1))
+    if [ $count -lt $retries ]; then
+      echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
+      sleep $wait
+    else
+      echo "Retry $count/$retries exited $exit, no more retries left."
+      return $exit
+    fi
+    set +e
+  done
+  return 0
+}
+export -f retry
+
 function clean_up {
     if [[ "$PRESERVE" == false ]]; then
       echo "====================================================================================================="

test/k8s-local-cluster-test/provision-cluster

@@ -95,21 +95,21 @@ if [[ OVERRIDE_PATH -eq 1 ]]; then
 else
   if [ ! -x "$TMP_DIR/kubectl" ]; then
       echoerr "🥑 Downloading the \"kubectl\" binary"
-      curl -Lo $TMP_DIR/kubectl "https://storage.googleapis.com/kubernetes-release/release/$KUBECTL_VERSION/bin/$PLATFORM/amd64/kubectl"
+      retry 5 curl -Lo $TMP_DIR/kubectl "https://storage.googleapis.com/kubernetes-release/release/$KUBECTL_VERSION/bin/$PLATFORM/amd64/kubectl"
       chmod +x $TMP_DIR/kubectl
       echoerr "👍 Downloaded the \"kubectl\" binary"
   fi
 
   if [ ! -x "$TMP_DIR/kind" ]; then
       echoerr "🥑 Downloading the \"kind\" binary"
-      curl -Lo $TMP_DIR/kind https://github.com/kubernetes-sigs/kind/releases/download/v$KIND_VERSION/kind-$PLATFORM-amd64
+      retry 5 curl -Lo $TMP_DIR/kind https://github.com/kubernetes-sigs/kind/releases/download/v$KIND_VERSION/kind-$PLATFORM-amd64
       chmod +x $TMP_DIR/kind
       echoerr "👍 Downloaded the \"kind\" binary"
   fi
 
   if [ ! -x "$TMP_DIR/helm" ]; then
       echoerr "🥑 Downloading the \"helm\" binary"
-      curl -L https://get.helm.sh/helm-v$HELM_VERSION-$PLATFORM-amd64.tar.gz | tar zxf - -C $TMP_DIR
+      retry 5 curl -L https://get.helm.sh/helm-v$HELM_VERSION-$PLATFORM-amd64.tar.gz | tar zxf - -C $TMP_DIR
       mv $TMP_DIR/$PLATFORM-amd64/helm $TMP_DIR/.
       chmod +x $TMP_DIR/helm
       echoerr "👍 Downloaded the \"helm\" binary"

test/k8s-local-cluster-test/run-test

@@ -28,6 +28,31 @@ function relpath() {
   perl -e 'use File::Spec; print File::Spec->abs2rel(@ARGV) . "\n"' $1 $2
 }
 
+function retry {
+  local retries=$1
+  shift
+
+  local count=0
+  set +e
+  trap "set -e" RETURN
+  until "$@"; do
+    exit=$?
+    set -e
+    wait=$((2 ** $count))
+    count=$(($count + 1))
+    if [ $count -lt $retries ]; then
+      echo "Retry $count/$retries exited $exit, retrying in $wait seconds..."
+      sleep $wait
+    else
+      echo "Retry $count/$retries exited $exit, no more retries left."
+      return $exit
+    fi
+    set +e
+  done
+  return 0
+}
+export -f retry
+
 function clean_up {
     if [[ "$PRESERVE" == false ]]; then
         $SCRIPTPATH/../k8s-local-cluster-test/delete-cluster $DELETE_CLUSTER_ARGS || :