chore(ci): update for e2e on new GKE and ephemeral AKS cluster (#2242)

* Delete tekton pipelines

* Better k8s cleanups

Update gke.sh

* Unify `helm upgrade -i` multiline

* Helm chart version *

* Delete unused file

* Update values for spot AKS cluster

* Update values for GKE cluster

* Use separate trap

Revert "Use separate trap"

This reverts commit 69e93ce4bd501b8e4854faee6dc1033a1999ecce.

Reapply "Use separate trap"

This reverts commit 6cb3fb86e3fbc52fb5c19aed8425a8b55e9bc48a.

Reapply trap

* Tekton pipelines install with kubectl

* Better detection of ephemeral cluster

* Longer wait time for backstage up for ephemeral cluster

* Note the chart version for releases

* Remove cluster url GKE - already in openshift/release

* Service token and better exporting

* GKE Cluster url

* AKS use IP instead of name

* Install tekton pipelines earlier

* Tekton pipelines wait for endpoints instead

* Conditional AKS cleanup

* Fix auth keys

* Tolerations for AKS

* Old chart with new probes

Author: zdrapela

.ibm/pipelines/cluster/aks/deployment.sh

@@ -1,10 +1,10 @@
 #!/bin/bash
 
 initiate_aks_deployment() {
+  install_tekton_pipelines
   add_helm_repos
   delete_namespace "${NAME_SPACE_RBAC_K8S}"
   configure_namespace "${NAME_SPACE_K8S}"
-  install_tekton_pipelines
   uninstall_helmchart "${NAME_SPACE_K8S}" "${RELEASE_NAME}"
   cd "${DIR}" || exit
   local rhdh_base_url="https://${K8S_CLUSTER_ROUTER_BASE}"
@@ -13,18 +13,19 @@ initiate_aks_deployment() {
   mkdir -p "${ARTIFACT_DIR}/${NAME_SPACE_K8S}"
   cp -a "/tmp/${HELM_CHART_K8S_MERGED_VALUE_FILE_NAME}" "${ARTIFACT_DIR}/${NAME_SPACE_K8S}/" # Save the final value-file into the artifacts directory.
   echo "Deploying image from repository: ${QUAY_REPO}, TAG_NAME: ${TAG_NAME}, in NAME_SPACE: ${NAME_SPACE_K8S}"
-  helm upgrade -i "${RELEASE_NAME}" -n "${NAME_SPACE_K8S}" "${HELM_REPO_NAME}/${HELM_IMAGE_NAME}" --version "${CHART_VERSION}" \
+  helm upgrade -i "${RELEASE_NAME}" -n "${NAME_SPACE_K8S}" \
+    "${HELM_REPO_NAME}/${HELM_IMAGE_NAME}" --version "${CHART_VERSION}" \
     -f "/tmp/${HELM_CHART_K8S_MERGED_VALUE_FILE_NAME}" \
     --set global.host="${K8S_CLUSTER_ROUTER_BASE}" \
     --set upstream.backstage.image.repository="${QUAY_REPO}" \
     --set upstream.backstage.image.tag="${TAG_NAME}"
 }
 
 initiate_rbac_aks_deployment() {
+  install_tekton_pipelines
   add_helm_repos
   delete_namespace "${NAME_SPACE_K8S}"
   configure_namespace "${NAME_SPACE_RBAC_K8S}"
-  install_tekton_pipelines
   uninstall_helmchart "${NAME_SPACE_RBAC_K8S}" "${RELEASE_NAME_RBAC}"
   cd "${DIR}" || exit
   local rbac_rhdh_base_url="https://${K8S_CLUSTER_ROUTER_BASE}"
@@ -33,7 +34,8 @@ initiate_rbac_aks_deployment() {
   mkdir -p "${ARTIFACT_DIR}/${NAME_SPACE_RBAC_K8S}"
   cp -a "/tmp/${HELM_CHART_RBAC_K8S_MERGED_VALUE_FILE_NAME}" "${ARTIFACT_DIR}/${NAME_SPACE_RBAC_K8S}/" # Save the final value-file into the artifacts directory.
   echo "Deploying image from repository: ${QUAY_REPO}, TAG_NAME: ${TAG_NAME}, in NAME_SPACE: ${NAME_SPACE_RBAC_K8S}"
-  helm upgrade -i "${RELEASE_NAME_RBAC}" -n "${NAME_SPACE_RBAC_K8S}" "${HELM_REPO_NAME}/${HELM_IMAGE_NAME}" --version "${CHART_VERSION}" \
+  helm upgrade -i "${RELEASE_NAME_RBAC}" -n "${NAME_SPACE_RBAC_K8S}" \
+    "${HELM_REPO_NAME}/${HELM_IMAGE_NAME}" --version "${CHART_VERSION}" \
     -f "/tmp/${HELM_CHART_RBAC_K8S_MERGED_VALUE_FILE_NAME}" \
     --set global.host="${K8S_CLUSTER_ROUTER_BASE}" \
     --set upstream.backstage.image.repository="${QUAY_REPO}" \

.ibm/pipelines/cluster/gke/deployment.sh

@@ -2,10 +2,10 @@
 
 initiate_gke_deployment() {
   gcloud_ssl_cert_create $GKE_CERT_NAME $GKE_INSTANCE_DOMAIN_NAME $GOOGLE_CLOUD_PROJECT
+  install_tekton_pipelines
   add_helm_repos
   delete_namespace "${NAME_SPACE_RBAC_K8S}"
   configure_namespace "${NAME_SPACE_K8S}"
-  install_tekton_pipelines
   uninstall_helmchart "${NAME_SPACE_K8S}" "${RELEASE_NAME}"
   cd "${DIR}" || exit
   local rhdh_base_url="https://${K8S_CLUSTER_ROUTER_BASE}"
@@ -15,7 +15,8 @@ initiate_gke_deployment() {
   mkdir -p "${ARTIFACT_DIR}/${NAME_SPACE_K8S}"
   cp -a "/tmp/${HELM_CHART_K8S_MERGED_VALUE_FILE_NAME}" "${ARTIFACT_DIR}/${NAME_SPACE_K8S}/" # Save the final value-file into the artifacts directory.
   echo "Deploying image from repository: ${QUAY_REPO}, TAG_NAME: ${TAG_NAME}, in NAME_SPACE: ${NAME_SPACE_K8S}"
-  helm upgrade -i "${RELEASE_NAME}" -n "${NAME_SPACE_K8S}" "${HELM_REPO_NAME}/${HELM_IMAGE_NAME}" --version "${CHART_VERSION}" \
+  helm upgrade -i "${RELEASE_NAME}" -n "${NAME_SPACE_K8S}" \
+    "${HELM_REPO_NAME}/${HELM_IMAGE_NAME}" --version "${CHART_VERSION}" \
     -f "/tmp/${HELM_CHART_K8S_MERGED_VALUE_FILE_NAME}" \
     --set global.host="${K8S_CLUSTER_ROUTER_BASE}" \
     --set upstream.backstage.image.repository="${QUAY_REPO}" \
@@ -25,10 +26,10 @@ initiate_gke_deployment() {
 
 initiate_rbac_gke_deployment() {
   gcloud_ssl_cert_create $GKE_CERT_NAME $GKE_INSTANCE_DOMAIN_NAME $GOOGLE_CLOUD_PROJECT
+  install_tekton_pipelines
   add_helm_repos
   delete_namespace "${NAME_SPACE_K8S}"
   configure_namespace "${NAME_SPACE_RBAC_K8S}"
-  install_tekton_pipelines
   uninstall_helmchart "${NAME_SPACE_RBAC_K8S}" "${RELEASE_NAME_RBAC}"
   cd "${DIR}" || exit
   local rbac_rhdh_base_url="https://${K8S_CLUSTER_ROUTER_BASE}"
@@ -37,7 +38,8 @@ initiate_rbac_gke_deployment() {
   mkdir -p "${ARTIFACT_DIR}/${NAME_SPACE_RBAC_K8S}"
   cp -a "/tmp/${HELM_CHART_RBAC_K8S_MERGED_VALUE_FILE_NAME}" "${ARTIFACT_DIR}/${NAME_SPACE_RBAC_K8S}/" # Save the final value-file into the artifacts directory.
   echo "Deploying image from repository: ${QUAY_REPO}, TAG_NAME: ${TAG_NAME}, in NAME_SPACE: ${NAME_SPACE_RBAC_K8S}"
-  helm upgrade -i "${RELEASE_NAME_RBAC}" -n "${NAME_SPACE_RBAC_K8S}" "${HELM_REPO_NAME}/${HELM_IMAGE_NAME}" --version "${CHART_VERSION}" \
+  helm upgrade -i "${RELEASE_NAME_RBAC}" -n "${NAME_SPACE_RBAC_K8S}" \
+    "${HELM_REPO_NAME}/${HELM_IMAGE_NAME}" --version "${CHART_VERSION}" \
     -f "/tmp/${HELM_CHART_RBAC_K8S_MERGED_VALUE_FILE_NAME}" \
     --set global.host="${K8S_CLUSTER_ROUTER_BASE}" \
     --set upstream.backstage.image.repository="${QUAY_REPO}" \

.ibm/pipelines/env_variables.sh

@@ -24,7 +24,7 @@ NAME_SPACE_RUNTIME="${NAME_SPACE_RUNTIME:-showcase-runtime}"
 NAME_SPACE_POSTGRES_DB="${NAME_SPACE_POSTGRES_DB:-postgress-external-db}"
 NAME_SPACE_RDS="showcase-rds-nightly"
 OPERATOR_MANAGER='rhdh-operator'
-CHART_VERSION="2.15.2"
+CHART_VERSION="2.15.2" # Fixed version should be used for release branches.
 GITHUB_APP_APP_ID=$(cat /tmp/secrets/GITHUB_APP_3_APP_ID)
 GITHUB_APP_CLIENT_ID=$(cat /tmp/secrets/GITHUB_APP_3_CLIENT_ID)
 GITHUB_APP_PRIVATE_KEY=$(cat /tmp/secrets/GITHUB_APP_3_PRIVATE_KEY)
@@ -93,7 +93,6 @@ ARM_CLIENT_ID=$(cat /tmp/secrets/ARM_CLIENT_ID)
 ARM_CLIENT_SECRET=$(cat /tmp/secrets/ARM_CLIENT_SECRET)
 AKS_NIGHTLY_CLUSTER_NAME=$(cat /tmp/secrets/AKS_NIGHTLY_CLUSTER_NAME)
 AKS_NIGHTLY_CLUSTER_RESOURCEGROUP=$(cat /tmp/secrets/AKS_NIGHTLY_CLUSTER_RESOURCEGROUP)
-AKS_INSTANCE_DOMAIN_NAME=$(cat /tmp/secrets/AKS_INSTANCE_DOMAIN_NAME)
 
 GKE_CLUSTER_NAME=$(cat /tmp/secrets/GKE_CLUSTER_NAME)
 GKE_CLUSTER_REGION=$(cat /tmp/secrets/GKE_CLUSTER_REGION)

.ibm/pipelines/jobs/aks.sh

@@ -4,14 +4,15 @@ handle_aks() {
   echo "Starting AKS deployment"
   for file in ${DIR}/cluster/aks/*.sh; do source $file; done
 
-  export K8S_CLUSTER_ROUTER_BASE=$AKS_INSTANCE_DOMAIN_NAME
+  export K8S_CLUSTER_ROUTER_BASE=$(kubectl get svc nginx --namespace app-routing-system -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
   export NAME_SPACE_K8S="showcase-k8s-ci-nightly"
   export NAME_SPACE_RBAC_K8S="showcase-rbac-k8s-ci-nightly"
 
   url="https://${K8S_CLUSTER_ROUTER_BASE}"
 
-  if kubectl auth whoami > /dev/null 2>&1; then
+  if oc whoami --show-server > /dev/null 2>&1; then
     echo "Using an ephemeral AKS cluster."
+    export EPHEMERAL="true"
   else
     echo "Falling back to a long-running AKS cluster."
     export K8S_CLUSTER_TOKEN=$(cat /tmp/secrets/AKS_CLUSTER_TOKEN)
@@ -30,10 +31,16 @@ handle_aks() {
   fi
 
   initiate_aks_deployment
-  check_and_test "${RELEASE_NAME}" "${NAME_SPACE_K8S}" "${url}"
+  check_and_test "${RELEASE_NAME}" "${NAME_SPACE_K8S}" "${url}" 50 30
   delete_namespace "${NAME_SPACE_K8S}"
   initiate_rbac_aks_deployment
   local rbac_rhdh_base_url="https://${K8S_CLUSTER_ROUTER_BASE}"
   check_and_test "${RELEASE_NAME_RBAC}" "${NAME_SPACE_RBAC_K8S}" "${rbac_rhdh_base_url}"
   delete_namespace "${NAME_SPACE_RBAC_K8S}"
 }
+
+cleanup_aks() {
+  if [ "$EPHEMERAL" != "true" ]; then
+    az_aks_stop "${AKS_NIGHTLY_CLUSTER_NAME}" "${AKS_NIGHTLY_CLUSTER_RESOURCEGROUP}"
+  fi
+}

.ibm/pipelines/jobs/gke.sh

@@ -4,30 +4,51 @@ handle_gke() {
   echo "Starting GKE deployment"
   for file in ${DIR}/cluster/gke/*.sh; do source $file; done
 
-  export K8S_CLUSTER_TOKEN=$(cat /tmp/secrets/GKE_CLUSTER_TOKEN)
-  export K8S_CLUSTER_TOKEN_ENCODED=$(printf "%s" $K8S_CLUSTER_TOKEN | base64 | tr -d '\n')
-  export K8S_SERVICE_ACCOUNT_TOKEN=$K8S_CLUSTER_TOKEN_ENCODED
-  export OCM_CLUSTER_TOKEN=$K8S_CLUSTER_TOKEN_ENCODED
-
-  export K8S_CLUSTER_ROUTER_BASE=$GKE_INSTANCE_DOMAIN_NAME
-  export NAME_SPACE_K8S="showcase-k8s-ci-nightly"
-  export NAME_SPACE_RBAC_K8S="showcase-rbac-k8s-ci-nightly"
+  K8S_CLUSTER_ROUTER_BASE=$GKE_INSTANCE_DOMAIN_NAME
+  NAME_SPACE_K8S="showcase-k8s-ci-nightly"
+  NAME_SPACE_RBAC_K8S="showcase-rbac-k8s-ci-nightly"
+  export K8S_CLUSTER_ROUTER_BASE NAME_SPACE_K8S NAME_SPACE_RBAC_K8S
 
   url="https://${K8S_CLUSTER_ROUTER_BASE}"
 
   gcloud_auth "${GKE_SERVICE_ACCOUNT_NAME}" "/tmp/secrets/GKE_SERVICE_ACCOUNT_KEY"
   gcloud_gke_get_credentials "${GKE_CLUSTER_NAME}" "${GKE_CLUSTER_REGION}" "${GOOGLE_CLOUD_PROJECT}"
 
-  export K8S_CLUSTER_URL=$(oc whoami --show-server)
-  export K8S_CLUSTER_API_SERVER_URL=$(printf "%s" "$K8S_CLUSTER_URL" | base64 | tr -d '\n')
-  export OCM_CLUSTER_URL=$(printf "%s" "$K8S_CLUSTER_URL" | base64 | tr -d '\n')
+  K8S_CLUSTER_URL=$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')
+  K8S_CLUSTER_API_SERVER_URL=$(printf "%s" "$K8S_CLUSTER_URL" | base64 | tr -d '\n')
+  OCM_CLUSTER_URL=$(printf "%s" "$K8S_CLUSTER_URL" | base64 | tr -d '\n')
+  export K8S_CLUSTER_URL K8S_CLUSTER_API_SERVER_URL OCM_CLUSTER_URL
+
+# Create a service account and assign token
+  SA_NAME="tester-sa-2"
+  SA_NAMESPACE="default"
+  SA_BINDING_NAME="${SA_NAME}-binding"
+  if ! kubectl get serviceaccount ${SA_NAME} -n ${SA_NAMESPACE} &> /dev/null; then
+    echo "Creating service account ${SA_NAME}..."
+    kubectl create serviceaccount ${SA_NAME} -n ${SA_NAMESPACE}
+    echo "Creating cluster role binding..."
+    kubectl create clusterrolebinding ${SA_BINDING_NAME} \
+        --clusterrole=cluster-admin \
+        --serviceaccount=${SA_NAMESPACE}:${SA_NAME}
+    echo "Service account and binding created successfully"
+  else
+    echo "Service account ${SA_NAME} already exists in namespace ${SA_NAMESPACE}"
+  fi
+  K8S_CLUSTER_TOKEN=$(kubectl create token tester-sa-2 -n default)  
+  K8S_CLUSTER_TOKEN_ENCODED=$(printf "%s" $K8S_CLUSTER_TOKEN | base64 | tr -d '\n')
+  K8S_SERVICE_ACCOUNT_TOKEN=$K8S_CLUSTER_TOKEN_ENCODED
+  OCM_CLUSTER_TOKEN=$K8S_CLUSTER_TOKEN_ENCODED
+  export K8S_CLUSTER_TOKEN K8S_CLUSTER_TOKEN_ENCODED K8S_SERVICE_ACCOUNT_TOKEN OCM_CLUSTER_TOKEN
 
   initiate_gke_deployment
-  check_and_test "${RELEASE_NAME}" "${NAME_SPACE_K8S}" "${url}"
+  check_and_test "${RELEASE_NAME}" "${NAME_SPACE_K8S}" "${url}" 50 30
   delete_namespace "${NAME_SPACE_K8S}"
   initiate_rbac_gke_deployment
   local rbac_rhdh_base_url="https://${K8S_CLUSTER_ROUTER_BASE}"
   check_and_test "${RELEASE_NAME_RBAC}" "${NAME_SPACE_RBAC_K8S}" "${rbac_rhdh_base_url}"
   delete_namespace "${NAME_SPACE_RBAC_K8S}"
+}
 
+cleanup_gke() {
+  delete_tekton_pipelines
 }

.ibm/pipelines/jobs/periodic.sh

@@ -45,17 +45,21 @@ run_runtime_config_change_tests() {
   local runtime_url="https://${RELEASE_NAME}-backstage-${NAME_SPACE_RUNTIME}.${K8S_CLUSTER_ROUTER_BASE}"
 
   apply_yaml_files "${DIR}" "${NAME_SPACE_RUNTIME}" "${runtime_url}"
-  helm upgrade -i "${RELEASE_NAME}" -n "${NAME_SPACE_RUNTIME}" "${HELM_REPO_NAME}/${HELM_IMAGE_NAME}" --version "${CHART_VERSION}" -f "${DIR}/value_files/${HELM_CHART_VALUE_FILE_NAME}" --set global.clusterRouterBase="${K8S_CLUSTER_ROUTER_BASE}" --set upstream.backstage.image.repository="${QUAY_REPO}" --set upstream.backstage.image.tag="${TAG_NAME}"
+  helm upgrade -i "${RELEASE_NAME}" -n "${NAME_SPACE_RUNTIME}" \
+    "${HELM_REPO_NAME}/${HELM_IMAGE_NAME}" --version "${CHART_VERSION}" \
+    -f "${DIR}/value_files/${HELM_CHART_VALUE_FILE_NAME}" \
+    --set global.clusterRouterBase="${K8S_CLUSTER_ROUTER_BASE}" \
+    --set upstream.backstage.image.repository="${QUAY_REPO}" \
+    --set upstream.backstage.image.tag="${TAG_NAME}" \
   check_and_test "${RELEASE_NAME}" "${NAME_SPACE_RUNTIME}" "${runtime_url}"
 }
 
 add_sanity_plugins_check() {
-  helm upgrade -i "${RELEASE_NAME}" \
-     -n "${NAME_SPACE}" "${HELM_REPO_NAME}/${HELM_IMAGE_NAME}" \
-     --version "${CHART_VERSION}" \
-     -f "${DIR}/value_files/${HELM_CHART_VALUE_FILE_NAME}" \
-     -f "${DIR}/value_files/sanity-check-plugins.yaml" \
-     --set global.clusterRouterBase="${K8S_CLUSTER_ROUTER_BASE}" \
-     --set upstream.backstage.image.repository="${QUAY_REPO}" \
-     --set upstream.backstage.image.tag="${TAG_NAME}"
+  helm upgrade -i "${RELEASE_NAME}" -n "${NAME_SPACE}" \
+    "${HELM_REPO_NAME}/${HELM_IMAGE_NAME}" --version "${CHART_VERSION}" \
+    -f "${DIR}/value_files/${HELM_CHART_VALUE_FILE_NAME}" \
+    -f "${DIR}/value_files/sanity-check-plugins.yaml" \
+    --set global.clusterRouterBase="${K8S_CLUSTER_ROUTER_BASE}" \
+    --set upstream.backstage.image.repository="${QUAY_REPO}" \
+    --set upstream.backstage.image.tag="${TAG_NAME}"
 }

.ibm/pipelines/kubernetes-tests.sh

@@ -12,9 +12,17 @@ OVERALL_RESULT=0
 # shellcheck disable=SC2317
 cleanup() {
   echo "Cleaning up before exiting"
-  if [[ "$JOB_NAME" == *aks* && "${OPENSHIFT_CI}" == "true" ]]; then
-    # If the job is for Azure Kubernetes Service (AKS), stop the AKS cluster.
-    az_aks_stop "${AKS_NIGHTLY_CLUSTER_NAME}" "${AKS_NIGHTLY_CLUSTER_RESOURCEGROUP}"
+  if [[ "${OPENSHIFT_CI}" == "true" ]]; then
+    case "$JOB_NAME" in
+      *aks*)
+        echo "Calling handle_aks"
+        cleanup_aks
+        ;;
+      *gke*)
+        echo "Calling cleanup_gke"
+        cleanup_gke
+        ;;
+    esac
   fi
   rm -rf ~/tmpbin
 }