cloud-e2e.yml

name: Cloud E2E

on:
  workflow_dispatch:
    inputs:
      TEST_TYPE:
        description: "The specify version of GO (e.g. openapi-test|adminapi-test|engine-test)"
        type: string
        required: false
        default: 'openapi-test'
      TEST_ENGINES:
        description: "The specify version of GO (e.g. mysql)"
        type: string
        required: false
        default: 'mysql'
      CLOUD_ENV_NAME:
        description: "The cloud env name of test (e.g. dev) "
        type: string
        required: false
        default: 'dev'
      CLOUD_BRANCH:
        description: "The cloud branch name (e.g. main) "
        type: string
        required: false
        default: 'main'
      GO_VERSION:
        description: "The specify version of GO (e.g. 1.23)"
        type: string
        required: false
        default: '1.23'
      K3S_VERSION:
        description: 'k3s cluster version (e.g. 1.30)'
        type: string
        required: false
        default: '1.30'
      APECD_REF:
        description: "The branch name of apecloud-cd"
        type: string
        required: false
        default: 'main'
      CURRENT_VERSION:
        description: "The current release version (e.g. v0.30) "
        type: string
        required: false
        default: ''
  workflow_call:
    inputs:
      TEST_TYPE:
        description: "The specify version of GO (e.g. openapi-test|adminapi-test|engine-test)"
        type: string
        required: false
        default: 'openapi-test'
      TEST_ENGINES:
        description: "The specify version of GO (e.g. mysql)"
        type: string
        required: false
        default: 'mysql'
      CLOUD_ENV_NAME:
        description: "The cloud env name of test (e.g. dev) "
        type: string
        required: false
        default: 'dev'
      CLOUD_BRANCH:
        description: "The cloud branch name (e.g. main) "
        type: string
        required: false
        default: 'main'
      GO_VERSION:
        description: "The specify version of GO (e.g. 1.23)"
        type: string
        required: false
        default: '1.23'
      K3S_VERSION:
        description: 'k3s cluster version (e.g. 1.30)'
        type: string
        required: false
        default: '1.30'
      APECD_REF:
        description: "The branch name of apecloud-cd"
        type: string
        required: false
        default: 'main'
      CURRENT_VERSION:
        description: "The current release version (e.g. v0.30) "
        type: string
        required: false
        default: ''

run-name: E2E Test ${{ inputs.TEST_TYPE }} ${{ inputs.TEST_ENGINES }} on Env:${{ inputs.CLOUD_ENV_NAME }} Ref:${{ inputs.CLOUD_BRANCH }}

env:
  ACK_KUBECONFIG_DEV: ${{ secrets.ACK_KUBECONFIG_DEV }}
  ACK_KUBECONFIG_DEMO: ${{ secrets.ACK_KUBECONFIG_DEMO }}
  IDC_KUBECONFIG_1: ${{ secrets.IDC_KUBECONFIG_1 }}
  IDC_KUBECONFIG_2: ${{ secrets.IDC_KUBECONFIG_2 }}
  IDC_KUBECONFIG_4: ${{ secrets.IDC_KUBECONFIG_4 }}
  ACK_KUBECONFIG_PROD: ${{ secrets.ACK_KUBECONFIG_PROD }}
  ACK_KUBECONFIG_INTL_PROD: ${{ secrets.ACK_KUBECONFIG_INTL_PROD }}
  VKE_KUBECONFIG_TEST: ${{ secrets.VKE_KUBECONFIG_TEST }}
  GITHUB_USER: ${{ secrets.PERSONAL_ACCESS_USER }}
  GH_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
  K3D_NAME: kbcloud
  CLOUD_LICENSE: ${{ secrets.CLOUD_LICENSE }}
  DOCKER_REGISTRY_URL: docker.io
  DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }}
  DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }}

jobs:
  get-test-type:
    runs-on: ubuntu-latest
    outputs:
      test-type: ${{ steps.get_test_type.outputs.test-type }}
      cloud-branch: ${{ steps.get_test_type.outputs.cloud-branch }}
      cloud-env-name: ${{ steps.get_test_type.outputs.cloud-env-name }}
      test-engines: ${{ steps.get_test_type.outputs.test-engines }}
      test-e2e: ${{ steps.get_test_type.outputs.test-e2e }}
      test-installer: ${{ steps.get_test_type.outputs.test-installer }}
    steps:
      - name: Get test type
        id: get_test_type
        run: |
          TEST_TYPES="${{ inputs.TEST_TYPE }}"
          test_type=""
          for test_type_tmp in $(echo "${TEST_TYPES}" | sed 's/|/ /g' ); do
              if [[ -z "${test_type}" ]]; then
                  test_type="{\"test-type\":\"${test_type_tmp}\"}"
              else
                  test_type="${test_type},{\"test-type\":\"${test_type_tmp}\"}"
              fi
          done
          echo "${test_type}"
          echo "test-type={\"include\":[${test_type}]}" >> $GITHUB_OUTPUT
          
          CLOUD_BRANCH="${{ inputs.CLOUD_BRANCH }}"
          if [[ -z "$CLOUD_BRANCH" ]]; then
              CLOUD_BRANCH="main"
          fi
          echo cloud-branch="$CLOUD_BRANCH" >> $GITHUB_OUTPUT
          
          CLOUD_ENV_NAME="${{ inputs.CLOUD_ENV_NAME }}"
          if [[ -z "$CLOUD_ENV_NAME" ]]; then
              CLOUD_ENV_NAME="dev"
          fi
          echo cloud-env-name="$CLOUD_ENV_NAME" >> $GITHUB_OUTPUT
          
          TEST_ENGINES="${{ inputs.TEST_ENGINES }}"
          if [[ -z "$TEST_ENGINES" ]]; then
              TEST_ENGINES="mysql"
          fi
          echo test-engines="$TEST_ENGINES" >> $GITHUB_OUTPUT
          
          # check test installer
          TEST_INSTALLER="false"
          TEST_E2E="false"
          if [[ "${CLOUD_BRANCH}" == "v"*"."*"."* ]]; then
              TEST_INSTALLER="true"
          else
              TEST_E2E="true"
          fi
          echo test-installer="$TEST_INSTALLER" >> $GITHUB_OUTPUT
          
          # check test e2e on dev
          CURRENT_VERSION="${{ inputs.CURRENT_VERSION }}"
          if [[ -n "${CURRENT_VERSION}" && "${CURRENT_VERSION}" != "v"* ]]; then
              CURRENT_VERSION="v${CURRENT_VERSION}"
          fi
          
          if [[ -n "${CURRENT_VERSION}" && "${CLOUD_BRANCH}" == "${CURRENT_VERSION}."* ]]; then
              TEST_E2E="true"
          fi
          echo test-e2e="$TEST_E2E" >> $GITHUB_OUTPUT

  e2e-test:
    needs: [ get-test-type ]
    if: ${{ needs.get-test-type.outputs.test-e2e == 'true' }}
    name: ${{ matrix.test-type }}-${{ needs.get-test-type.outputs.cloud-env-name }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.get-test-type.outputs.test-type) }}
    outputs:
      openapi-test-result: ${{ steps.get_test_result.outputs.openapi-test-result }}
      adminapi-test-result: ${{ steps.get_test_result.outputs.adminapi-test-result }}
      engine-test-result: ${{ steps.get_test_result.outputs.engine-test-result }}
      test-result: ${{ steps.get_test_result.outputs.test-result }}
    runs-on: ubuntu-latest
    steps:
      - name: get go version
        run: |
          GO_VERSION="${{ inputs.GO_VERSION }}"
          if [[ -z "$GO_VERSION" ]]; then
              GO_VERSION="1.23"
          fi
          echo GO_VERSION="$GO_VERSION" >> $GITHUB_ENV

      - name: Checkout apecloud Code
        uses: actions/checkout@v4
        with:
          repository: apecloud/apecloud
          path: ./
          token: ${{ env.GH_TOKEN }}
          ref: ${{ env.CLOUD_BRANCH }}

      - name: checkout apecloud-cd code
        uses: actions/checkout@v4
        with:
          repository: apecloud/apecloud-cd
          path: ./apecloud-cd
          ref: ${{ inputs.APECD_REF }}

      - name: Setup Go specify version
        uses: actions/setup-go@v3
        with:
          go-version: "${{ env.GO_VERSION }}"

      - name: git config
        run: |
          git config --global url."https://${{ env.GITHUB_USER }}:${{ env.GH_TOKEN }}@github.com".insteadof "https://github.com"

      - name: Configure ACK Context ${{ inputs.CLOUD_ENV_NAME }}
        id: cloud_env
        run: |
          mkdir -p $HOME/.kube
          touch $HOME/.kube/config
          CLOUD_ENV_NAME="${{ inputs.CLOUD_ENV_NAME }}"
          if [[ -z "$CLOUD_ENV_NAME" ]]; then
              CLOUD_ENV_NAME="dev"
          fi
          echo CLOUD_ENV_NAME="$CLOUD_ENV_NAME" >> $GITHUB_ENV
          
          E2E_ENV_VARS=""
          case "$CLOUD_ENV_NAME" in
              dev)
                  echo '${{ env.ACK_KUBECONFIG_DEV }}' > $HOME/.kube/config
                  E2E_ENV_VARS="${{ vars.E2E_ENV_VARS_DEV }}"
              ;;
              demo)
                  echo '${{ env.ACK_KUBECONFIG_DEMO }}' > $HOME/.kube/config
                  E2E_ENV_VARS="${{ vars.E2E_ENV_VARS_DEMO }}"
              ;;
              idc1)
                  echo '${{ env.IDC_KUBECONFIG_1 }}' > $HOME/.kube/config
                  E2E_ENV_VARS="${{ vars.E2E_ENV_VARS_IDC1 }}"
              ;;
              idc2)
                  echo '${{ env.IDC_KUBECONFIG_2 }}' > $HOME/.kube/config
                  E2E_ENV_VARS="${{ vars.E2E_ENV_VARS_IDC2 }}"
              ;;
              idc4)
                  echo '${{ env.IDC_KUBECONFIG_4 }}' > $HOME/.kube/config
                  E2E_ENV_VARS="${{ vars.E2E_ENV_VARS_IDC4 }}"
              ;;
              prod)
                  echo '${{ env.ACK_KUBECONFIG_PROD }}' > $HOME/.kube/config
                  E2E_ENV_VARS="${{ vars.E2E_ENV_VARS_PROD }}"
              ;;
              intl_prod)
                  echo '${{ env.ACK_KUBECONFIG_INTL_PROD }}' > $HOME/.kube/config
                  E2E_ENV_VARS="${{ vars.E2E_ENV_VARS_INTL_PROD }}"
              ;;
              vke_test)
                  echo '${{ env.VKE_KUBECONFIG_TEST }}' > $HOME/.kube/config
                  E2E_ENV_VARS="${{ vars.E2E_ENV_VARS_VKE_TEST }}"
              ;;
          esac
          echo E2E_ENV_VARS="$E2E_ENV_VARS" >> $GITHUB_ENV

      - name: install ginkgo
        run: |
          cd e2e
          make module
          make install-ginkgo

      - name: ${{ matrix.test-type }}
        id: get_test_result
        run: |
          cd e2e
          file_log="test_result.log"
          touch ${file_log} 
          
          for env_vars in $(echo "${E2E_ENV_VARS}" | sed 's/|/ /g'); do
              eval_cmd="export $env_vars"
              echo "$eval_cmd"
              eval "$eval_cmd"
          done
          
          TEST_ENGINES="${{ needs.get-test-type.outputs.test-engines }}"
          echo "export KB_CLOUD_TEST_ENGINES=${TEST_ENGINES}"
          export KB_CLOUD_TEST_ENGINES=${TEST_ENGINES}
          
          TEST_TYPE="${{ matrix.test-type }}"
          if [[ -z "$TEST_TYPE" ]]; then
              TEST_TYPE="openapi-test"
          fi
          make ${TEST_TYPE} | tee -a ${file_log}
          
          test_ret="$( grep "Test Suite Failed" ${file_log} || true )"
          test_result="[PASSED]"
          if [[ -n "$test_ret" ]]; then
              test_result="[FAILED]"
          fi
          
          case "$TEST_TYPE" in
              openapi-test)
                  echo openapi-test-result="${test_result}" >> $GITHUB_OUTPUT
              ;;
              adminapi-test)
                  echo adminapi-test-result="${test_result}" >> $GITHUB_OUTPUT
              ;;
              engine-test)
                  echo engine-test-result="${test_result}" >> $GITHUB_OUTPUT
              ;;
              *)
                  echo test-result="${test_result}" >> $GITHUB_OUTPUT
              ;;
          esac
          
          if [[ -n "$test_ret" ]]; then
              exit 1
          fi

  installer-test:
    needs: [ get-test-type ]
    if: ${{ needs.get-test-type.outputs.test-installer == 'true' }}
    runs-on: ubuntu-latest
    outputs:
      installer-result: ${{ steps.install_kb_cloud.outputs.installer-result }}
      bootstrapper-result: ${{ steps.deploy_kb_cloud_env.outputs.bootstrapper-result }}
    steps:
      - name: Checkout apecloud-cd Code
        uses: actions/checkout@v4
        with:
          repository: apecloud/apecloud-cd
          path: apecloud-cd
          ref: ${{ inputs.APECD_REF }}

      - name: Setup kubectl
        uses: azure/setup-kubectl@v3
        with:
          version: "v1.30.4"

      - name: Install Helm
        uses: azure/setup-helm@v3
        with:
          version: "v3.16.3"

      - name: Setup Go
        uses: actions/setup-go@v3
        with:
          go-version: "${{ env.GO_VERSION }}"

      - name: setup k3d k3s
        uses: apecloud-inc/setup-k3d-k3s@v2
        with:
          k3d-name: "${{ env.K3D_NAME }}"
          version: "v${{ inputs.K3S_VERSION }}"
          github-token: ${{ env.GH_TOKEN }}
          k3d-args: -p 443:443@loadbalancer -p 80:80@loadbalancer --agents 2

      - name: update k3d coredns cm
        run: |
          for i in {1..3}; do
              bash ${{ github.workspace }}/apecloud-cd/.github/utils/utils.sh --type 37
              sleep 1
          done

      - name: install cert-manager
        run: |
          helm repo add jetstack https://charts.jetstack.io --force-update
          helm install \
              cert-manager jetstack/cert-manager \
              --namespace cert-manager \
              --create-namespace \
              --version v1.16.0 \
              --set crds.enabled=true

      - name: install kb cloud
        run: |
          helm repo add kb-chart https://apecloud.github.io/helm-charts
          
          helm install installer kb-chart/kb-cloud-installer \
              --namespace kb-cloud \
              --create-namespace \
              --version ${{ needs.get-test-type.outputs.cloud-branch }} \
              --set image.registry="docker.io" \
              --set version=${{ needs.get-test-type.outputs.cloud-branch }} \
              --set kubernetesProvider=k3d \
              --set fqdn.domain=mytest.kubeblocks.com \
              --set tls.enabled=true

      - if: ${{ env.DOCKER_REGISTRY_USER != '' && env.DOCKER_REGISTRY_PASSWORD != '' }}
        name: Login to Docker Hub
        uses: docker/login-action@v2
        with:
          registry: ${{ env.DOCKER_REGISTRY_URL }}
          username: ${{ env.DOCKER_REGISTRY_USER }}
          password: ${{ env.DOCKER_REGISTRY_PASSWORD }}

      - name: check kb-cloud-installer pod Completed
        id: install_kb_cloud
        run: |
          sleep 10
          check_status=0
          for i in {1..200}; do
              if kubectl get pods -n kb-cloud | grep "kb-cloud-installer" | grep Completed ; then
                  echo "KB Cloud is ready"
                  check_status=1
                  break
              fi
              echo "Waiting for KB Cloud to be ready..."
              kubectl get pods -n kb-cloud
              echo ""
              sleep 5
          done
          
          installer_result="[PASSED]"
          if [[ $check_status -eq 0 ]]; then
              echo "KB Cloud is not ready"
              installer_result="[FAILED]"
              echo installer-result="${installer_result}" >> $GITHUB_OUTPUT
          
              installer_pods=$(kubectl get pods -n kb-cloud | grep "kb-cloud-installer" )
              echo "installer pod: $installer_pods"
              installer_pod_names="$(echo "${installer_pods}" | awk '{print $1}')"
              for installer_pod_name in $(echo "${installer_pod_names}"); do
                  echo "==================== pod ${installer_pod_name} logs ===================="
                  kubectl logs -n kb-cloud ${installer_pod_name}
              done 
          
              exit 1
          else
              echo installer-result="${installer_result}" >> $GITHUB_OUTPUT
          fi

      - name: deploy KB Cloud Env
        run: |
          echo "get k8s config"
          k8s_cluster_name=$(kubectl config get-clusters | sed '1d')
          kubectl config set-cluster ${k8s_cluster_name} --insecure-skip-tls-verify=true
          k8s_kubeconfig_yaml=$(kubectl config view --minify -o yaml --raw)
          kubeconfig_file_name="k8s_kubeconfig_file.yaml"
          touch ${kubeconfig_file_name}
          echo "${k8s_kubeconfig_yaml}" > ${kubeconfig_file_name}
          yq e -i '.clusters[0].cluster.server = "https://kubernetes.default:443"' ${kubeconfig_file_name}
          k8s_kubeconfig=$(cat ${kubeconfig_file_name})
          echo "------------------------------------------------------------"
          echo "${k8s_kubeconfig}"
          echo "------------------------------------------------------------"
          K8S_KUBECONFIG="$(echo "${k8s_kubeconfig}" | base64)"
          
          CLOUD_LICENSE="${{ env.CLOUD_LICENSE }}"
          echo "deploy KB Cloud Env"
          
          helm install bootstrapper kb-chart/kb-cloud-bootstrapper \
            --namespace kb-system \
            --create-namespace \
            --version ${{ needs.get-test-type.outputs.cloud-branch }} \
            --set image.registry="docker.io" \
            --set license="${CLOUD_LICENSE}" \
            --set kubeconfig="${K8S_KUBECONFIG}"

      - name: check kb-cloud-bootstrapper pod Completed
        id: deploy_kb_cloud_env
        run: |
          sleep 10
          check_status=0
          for i in {1..200}; do
              if kubectl get pods -n kb-system | grep "kb-cloud-bootstrapper" | grep Completed ; then
                  echo "KB Cloud Env is ready"
                  check_status=1
                  break
              fi
              echo "Waiting for KB Cloud Env to be ready..."
              kubectl get pods -n kb-system
              echo ""
              sleep 5
          done
          
          bootstrapper_result="[PASSED]"
          if [[ $check_status -eq 0 ]]; then
              echo "KB Cloud Env is not ready"
              bootstrapper_result="[FAILED]"
              echo bootstrapper-result="${bootstrapper_result}" >> $GITHUB_OUTPUT
          
              bootstrapper_pods=$(kubectl get pods -n kb-system | grep "kb-cloud-bootstrapper" )
              echo "bootstrapper pod: $bootstrapper_pods"
              bootstrapper_pod_names="$(echo "${bootstrapper_pods}" | awk '{print $1}')"
              for bootstrapper_pod_name in $(echo "${bootstrapper_pod_names}"); do
                  echo "==================== pod ${bootstrapper_pod_name} logs ===================="
                  kubectl logs -n kb-system ${bootstrapper_pod_name}
              done 
          
              exit 1
          else
              echo bootstrapper-result="${bootstrapper_result}" >> $GITHUB_OUTPUT
          fi

  upgrade-test:
    needs: [ get-test-type ]
    if: ${{ needs.get-test-type.outputs.test-installer == 'true' }}
    runs-on: ubuntu-latest
    outputs:
      installer-result: ${{ steps.install_kb_cloud.outputs.installer-result }}
      upgrade-result: ${{ steps.upgrade_kb_cloud.outputs.upgrade-result }}
      cloud-pre-version: ${{ steps.cloud_pre_version.outputs.cloud-pre-version }}
    steps:
      - name: Checkout apecloud-cd Code
        uses: actions/checkout@v4
        with:
          repository: apecloud/apecloud-cd
          path: apecloud-cd
          ref: ${{ inputs.APECD_REF }}

      - name: Setup kubectl
        uses: azure/setup-kubectl@v3
        with:
          version: "v1.30.4"

      - name: Install Helm
        uses: azure/setup-helm@v3
        with:
          version: "v3.16.3"

      - name: Setup Go
        uses: actions/setup-go@v3
        with:
          go-version: "${{ env.GO_VERSION }}"

      - name: setup k3d k3s
        uses: apecloud-inc/setup-k3d-k3s@v2
        with:
          k3d-name: "${{ env.K3D_NAME }}"
          version: "v${{ inputs.K3S_VERSION }}"
          github-token: ${{ env.GH_TOKEN }}
          k3d-args: -p 443:443@loadbalancer -p 80:80@loadbalancer --agents 2

      - name: update k3d coredns cm
        run: |
          for i in {1..3}; do
              bash ${{ github.workspace }}/apecloud-cd/.github/utils/utils.sh --type 37
              sleep 1
          done

      - name: install cert-manager
        run: |
          helm repo add jetstack https://charts.jetstack.io --force-update
          helm install \
              cert-manager jetstack/cert-manager \
              --namespace cert-manager \
              --create-namespace \
              --version v1.16.0 \
              --set crds.enabled=true

      - name: get cloud pre version
        id: cloud_pre_version
        run: |
          UPGRADE_VERSION="${{ needs.get-test-type.outputs.cloud-branch }}"
          if [[ "$UPGRADE_VERSION" != "v"* ]]; then
              UPGRADE_VERSION="v${UPGRADE_VERSION}"
          fi
          CLOUD_PRE_VERSION=$(bash ${{ github.workspace }}/apecloud-cd/.github/utils/utils.sh --type 39 \
              --github-repo "apecloud/apecloud" \
              --github-token "${{ env.GH_TOKEN }}" \
              --version "${UPGRADE_VERSION}")
          echo "CLOUD_PRE_VERSION:$CLOUD_PRE_VERSION"
          echo "cloud-pre-version=${CLOUD_PRE_VERSION}" >> $GITHUB_OUTPUT

      - name: install kb cloud with ${{ steps.cloud_pre_version.outputs.cloud-pre-version }}
        run: |
          helm repo add kb-chart https://apecloud.github.io/helm-charts
          CLOUD_PRE_VERSION="${{ steps.cloud_pre_version.outputs.cloud-pre-version }}"
          HELM_CHART_PKG_NAME="kb-cloud-installer-${CLOUD_PRE_VERSION/v/}.tgz"
          helm_pull_cmd="helm pull kb-chart/kb-cloud-installer --version ${CLOUD_PRE_VERSION}"
          for i in {1..200}; do
              eval "$helm_pull_cmd" || true
              # check helm chart package
              echo "helm chart package checking..."
              if [[ -f "${HELM_CHART_PKG_NAME}" ]]; then
                  echo "found helm chart package ${HELM_CHART_PKG_NAME}"
                  break
              fi
              sleep 1
              helm repo update kb-chart
          done
          
          helm install installer kb-chart/kb-cloud-installer \
              --namespace kb-cloud \
              --create-namespace \
              --version "${CLOUD_PRE_VERSION}" \
              --set "version=${CLOUD_PRE_VERSION}" \
              --set image.registry="docker.io" \
              --set kubernetesProvider=k3d \
              --set fqdn.domain=mytest.kubeblocks.com \
              --set tls.enabled=true

      - name: check kb-cloud-installer pod Completed
        id: install_kb_cloud
        run: |
          sleep 10
          check_status=0
          for i in {1..200}; do
              if kubectl get pods -n kb-cloud | grep "kb-cloud-installer" | grep Completed ; then
                  echo "KB Cloud is ready"
                  check_status=1
                  break
              fi
              echo "Waiting for KB Cloud to be ready..."
              kubectl get pods -n kb-cloud
              echo ""
              sleep 5
          done
          
          installer_result="[PASSED]"
          if [[ $check_status -eq 0 ]]; then
              echo "KB Cloud is not ready"
              installer_result="[FAILED]"
              echo installer-result="${installer_result}" >> $GITHUB_OUTPUT
          
              installer_pods=$(kubectl get pods -n kb-cloud | grep "kb-cloud-installer" )
              echo "installer pod: $installer_pods"
              installer_pod_names="$(echo "${installer_pods}" | awk '{print $1}')"
              for installer_pod_name in $(echo "${installer_pod_names}"); do
                  echo "==================== pod ${installer_pod_name} logs ===================="
                  kubectl logs -n kb-cloud ${installer_pod_name}
              done 
          
              exit 1
          else
              echo installer-result="${installer_result}" >> $GITHUB_OUTPUT
          fi

      - name: upgrade kb cloud to ${{ needs.get-test-type.outputs.cloud-branch }}
        run: |
          helm uninstall installer --namespace kb-cloud
          sleep 2
          
          CLOUD_VERSION="${{ needs.get-test-type.outputs.cloud-branch }}"
          helm install installer-upgrade kb-chart/kb-cloud-installer \
              --namespace kb-cloud \
              --create-namespace \
              --version ${CLOUD_VERSION} \
              --set version=${CLOUD_VERSION} \
              --set image.registry="docker.io" \
              --set kubernetesProvider=k3d \
              --set fqdn.domain=mytest.kubeblocks.com \
              --set tls.enabled=true \
              --set action=upgrade

      - name: check kb-cloud-installer upgrade pod Completed
        id: upgrade_kb_cloud
        run: |
          sleep 10
          check_status=0
          for i in {1..200}; do
              if kubectl get pods -n kb-cloud | grep "kb-cloud-installer" | grep Completed ; then
                  echo "KB Cloud is ready"
                  check_status=1
                  break
              fi
              echo "Waiting for KB Cloud to be ready..."
              kubectl get pods -n kb-cloud
              echo ""
              sleep 5
          done
          
          upgrade_result="[PASSED]"
          if [[ $check_status -eq 0 ]]; then
              echo "KB Cloud is not ready"
              upgrade_result="[FAILED]"
              echo upgrade-result="${upgrade_result}" >> $GITHUB_OUTPUT
          
              installer_pods=$(kubectl get pods -n kb-cloud | grep "kb-cloud-installer" )
              echo "installer pod: $installer_pods"
              installer_pod_names="$(echo "${installer_pods}" | awk '{print $1}')"
              for installer_pod_name in $(echo "${installer_pod_names}"); do
                  echo "==================== pod ${installer_pod_name} logs ===================="
                  kubectl logs -n kb-cloud ${installer_pod_name}
              done 
          
              exit 1
          else
              echo upgrade-result="${upgrade_result}" >> $GITHUB_OUTPUT
          fi

  enable-runner:
    needs: [ get-test-type ]
    if: ${{ needs.get-test-type.outputs.test-installer == 'true' }}
    uses: ./.github/workflows/enable-self-runner.yml
    with:
      CLOUD_PROVIDER: "eks"
      GITHUB_REPO: "${{ github.repository }}"
      RUNNER_NAMESPACE: "default"
      RUNNER_LABEL: "eks-k3d-runner"
    secrets: inherit

  openapi-test-k3s:
    if: ${{ needs.get-test-type.outputs.test-installer == 'true' && contains(inputs.TEST_TYPE, 'openapi-test') }}
    needs: [ get-test-type, enable-runner ]
    uses: ./.github/workflows/cloud-e2e-k3s-test.yml
    with:
      TEST_TYPE: "openapi-test"
      TEST_ENGINES: "${{ needs.get-test-type.outputs.test-engines }}"
      CLOUD_BRANCH: "${{ needs.get-test-type.outputs.cloud-branch }}"
      GO_VERSION: "${{ inputs.GO_VERSION }}"
      K3S_VERSION: "${{ inputs.K3S_VERSION }}"
      APECD_REF: ${{ inputs.APECD_REF }}
    secrets: inherit

  adminapi-test-k3s:
    if: ${{ needs.get-test-type.outputs.test-installer == 'true' && contains(inputs.TEST_TYPE, 'adminapi-test') }}
    needs: [ get-test-type, enable-runner ]
    uses: ./.github/workflows/cloud-e2e-k3s-test.yml
    with:
      TEST_TYPE: "adminapi-test"
      TEST_ENGINES: "${{ needs.get-test-type.outputs.test-engines }}"
      CLOUD_BRANCH: "${{ needs.get-test-type.outputs.cloud-branch }}"
      GO_VERSION: "${{ inputs.GO_VERSION }}"
      K3S_VERSION: "${{ inputs.K3S_VERSION }}"
      APECD_REF: ${{ inputs.APECD_REF }}
    secrets: inherit

  engine-test-k3s:
    if: ${{ needs.get-test-type.outputs.test-installer == 'true' && contains(inputs.TEST_TYPE, 'engine-test') }}
    needs: [ get-test-type, enable-runner ]
    uses: ./.github/workflows/cloud-e2e-k3s-test.yml
    with:
      TEST_TYPE: "engine-test"
      TEST_ENGINES: "${{ needs.get-test-type.outputs.test-engines }}"
      CLOUD_BRANCH: "${{ needs.get-test-type.outputs.cloud-branch }}"
      GO_VERSION: "${{ inputs.GO_VERSION }}"
      K3S_VERSION: "${{ inputs.K3S_VERSION }}"
      APECD_REF: ${{ inputs.APECD_REF }}
    secrets: inherit

  send-message:
    needs: [ get-test-type, e2e-test, openapi-test-k3s, adminapi-test-k3s, engine-test-k3s, installer-test, upgrade-test ]
    runs-on: ubuntu-latest
    if: ${{ always() }}
    steps:
      - name: Checkout apecloud-cd Code
        uses: actions/checkout@v4
        with:
          repository: apecloud/apecloud-cd
          ref: ${{ inputs.APECD_REF }}

      - name: send test result message
        run: |
          CLOUD_BRANCH="${{ needs.get-test-type.outputs.cloud-branch }}"
          CLOUD_ENV_NAME="${{ needs.get-test-type.outputs.cloud-env-name }}"
          TEST_INSTALLER="${{ needs.get-test-type.outputs.test-installer }}"
          TEST_ENGINES="${{ needs.get-test-type.outputs.test-engines }}"
          
          TEST_RESULT=""
          TEST_TYPES="${{ inputs.TEST_TYPE }}"
          for test_type_tmp in $(echo "${TEST_TYPES}" | sed 's/|/ /g' ); do
              case "$test_type_tmp" in
                  openapi-test)
                      TEST_RESULT="${TEST_RESULT}##${test_type_tmp}-${CLOUD_ENV_NAME}|${{ needs.e2e-test.outputs.openapi-test-result }}"
                  ;;
                  adminapi-test)
                      TEST_RESULT="${TEST_RESULT}##${test_type_tmp}-${CLOUD_ENV_NAME}|${{ needs.e2e-test.outputs.adminapi-test-result }}"
                  ;;
                  engine-test)
                      TEST_RESULT="${TEST_RESULT}##${test_type_tmp}-${CLOUD_ENV_NAME}|${{ needs.e2e-test.outputs.engine-test-result }}"
                  ;;
                  *)
                      TEST_RESULT="${TEST_RESULT}##${test_type_tmp}-${CLOUD_ENV_NAME}|${{ needs.e2e-test.outputs.test-result }}"
                  ;;
              esac
          done
          
          echo "TEST_RESULT:${TEST_RESULT}"
          TEST_RESULT=$( bash .github/utils/utils.sh --type 12 \
              --github-repo "${{ github.repository }}" \
              --github-token "${{ env.GH_TOKEN }}" \
              --test-result "${TEST_RESULT}" \
              --run-id "$GITHUB_RUN_ID" )
          echo "TEST_RESULT:${TEST_RESULT}"
          
          TEST_RESULT_INSTALLER=""
          TEST_RESULT_K3S=""
          if [[ "${TEST_INSTALLER}" == "true" ]]; then
              if [[ "$TEST_TYPES" == *"openapi-test"* ]]; then
                  TEST_RESULT_K3S="${TEST_RESULT_K3S}##openapi-test-k3s|${{ needs.openapi-test-k3s.outputs.test-result }}"
              fi
          
              if [[ "$TEST_TYPES" == *"adminapi-test"* ]]; then
              TEST_RESULT_K3S="${TEST_RESULT_K3S}##adminapi-test-k3s|${{ needs.adminapi-test-k3s.outputs.test-result }}"
              fi
          
              if [[ "$TEST_TYPES" == *"engine-test"* ]]; then
              TEST_RESULT_K3S="${TEST_RESULT_K3S}##engine-test-k3s|${{ needs.engine-test-k3s.outputs.test-result }}"
              fi
          
              echo "TEST_RESULT_K3S:${TEST_RESULT_K3S}"
              TEST_RESULT_K3S=$( bash .github/utils/utils.sh --type 12 \
                  --github-repo "${{ github.repository }}" \
                  --github-token "${{ env.GH_TOKEN }}" \
                  --test-result "${TEST_RESULT_K3S}" \
                  --run-id "$GITHUB_RUN_ID" )
          
              echo "get installer test result"
              INSTALLER_RESULT="${{ needs.installer-test.outputs.installer-result }}"
              if [[ -z "${INSTALLER_RESULT}" ]]; then
                  INSTALLER_RESULT="[FAILED]"
              fi
              BOOTSTRAPPER_RESULT="${{ needs.installer-test.outputs.bootstrapper-result }}"
              if [[ -z "${BOOTSTRAPPER_RESULT}" ]]; then
                  BOOTSTRAPPER_RESULT="[FAILED]"
              fi
              
              echo "get upgrade test result"
              CLOUD_PRE_VERSION="${{ needs.upgrade-test.outputs.cloud-pre-version }}"
              INSTALLER_PRE_RESULT="${{ needs.upgrade-test.outputs.installer-result }}"
              if [[ -z "${INSTALLER_PRE_RESULT}" ]]; then
                  INSTALLER_PRE_RESULT="[FAILED]"
              fi
              UPGRADE_RESULT="${{ needs.upgrade-test.outputs.upgrade-result }}"
              if [[ -z "${UPGRADE_RESULT}" ]]; then
                  UPGRADE_RESULT="[FAILED]"
              fi
              
              TEST_RESULT_INSTALLER="installer-test|installer-${CLOUD_BRANCH}|${INSTALLER_RESULT}"
              TEST_RESULT_INSTALLER="${TEST_RESULT_INSTALLER}##installer-test|bootstrapper-${CLOUD_BRANCH}|${BOOTSTRAPPER_RESULT}"
          
              TEST_RESULT_INSTALLER="${TEST_RESULT_INSTALLER}##upgrade-test|installer-${CLOUD_PRE_VERSION}|${INSTALLER_PRE_RESULT}"
              TEST_RESULT_INSTALLER="${TEST_RESULT_INSTALLER}##upgrade-test|upgrade-${CLOUD_BRANCH}|${UPGRADE_RESULT}"
              echo "TEST_RESULT_INSTALLER:${TEST_RESULT_INSTALLER}" 
          
              TEST_RESULT_INSTALLER=$( bash .github/utils/utils.sh --type 38 \
                  --github-repo "${{ github.repository }}" \
                  --github-token "${{ env.GH_TOKEN }}" \
                  --test-result "${TEST_RESULT_INSTALLER}" \
                  --run-id "$GITHUB_RUN_ID" )
              echo "TEST_RESULT_INSTALLER:$TEST_RESULT_INSTALLER"
          fi
          
          if [[ -n "${TEST_RESULT_K3S}" ]]; then
              TEST_RESULT="${TEST_RESULT_K3S}##${TEST_RESULT}"
              echo "TEST_RESULT:${TEST_RESULT}"
          fi
          
          if [[ -n "${TEST_RESULT_INSTALLER}" ]]; then
              TEST_RESULT="${TEST_RESULT_INSTALLER}##${TEST_RESULT}"
              echo "TEST_RESULT:${TEST_RESULT}"
          fi
          
          date_ret=$(date +%Y-%m-%d-%T)
          TEST_TITLE="[${CLOUD_BRANCH}] Cloud E2E Test ${TEST_ENGINES} [${date_ret}]"
          if [[ "${TEST_INSTALLER}" == "true" ]]; then
              TEST_TITLE="[${CLOUD_BRANCH}] Cloud E2E Test ${TEST_ENGINES} & Cloud Installer [${date_ret}]"
          fi
          
          python3 .github/utils/send_mesage.py \
              --url "${{ vars.CLOUD_E2E_BOT_WEBHOOK }}" \
              --title "$TEST_TITLE" \
              --result "$TEST_RESULT"

  disable-runner:
    if: ${{ always() && needs.get-test-type.outputs.test-installer == 'true' }}
    needs: [ get-test-type, openapi-test-k3s, adminapi-test-k3s, engine-test-k3s ]
    uses: ./.github/workflows/enable-self-runner.yml
    with:
      CLOUD_PROVIDER: "eks"
      GITHUB_REPO: "${{ github.repository }}"
      RUNNER_NAMESPACE: "default"
      RUNNER_LABEL: "eks-k3d-runner"
      ENABLE: "disable"
    secrets: inherit