diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index c4986646..fee5028a 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -20,14 +20,14 @@ jobs: install_only: true - name: Set Up Clusters run: | - hack/local-dev-setup.sh + hack/e2e-test/build-clusters.sh - name: Install Helm uses: azure/setup-helm@v3 with: version: v3.10.1 - name: Init kurator cluster run: | - hack/e2e-test/install.sh + hack/e2e-test/install-kurator.sh - name: fleet-clusters e2e test run: | hack/e2e-test/run-e2e.sh diff --git a/e2e/attachedcluster_test.go b/e2e/attachedcluster_test.go index 4b278848..35b51ab1 100644 --- a/e2e/attachedcluster_test.go +++ b/e2e/attachedcluster_test.go @@ -30,74 +30,49 @@ import ( var _ = ginkgo.Describe("[AttachedClusters] AttachedClusters testing", func() { var ( - namespace string - fleetname string - memberClusterName1 string - memberClusterName2 string - kubeconfig1Path string - kubeconfig2Path string - secret1 *corev1.Secret - secret2 *corev1.Secret - attachedcluster1 *clusterv1a1.AttachedCluster - attachedcluster2 *clusterv1a1.AttachedCluster + namespace string + fleetname string + memberClusterName string + kubeconfigPath string + secret *corev1.Secret + attachedcluster *clusterv1a1.AttachedCluster ) ginkgo.BeforeEach(func() { namespace = "default" fleetname = "e2etest" - memberClusterName1 = "kurator-member1" - memberClusterName2 = "kurator-member2" - kubeconfig1Path = "/root/.kube/kurator-member1.config" - kubeconfig2Path = "/root/.kube/kurator-member2.config" + memberClusterName = "kurator-member" + kubeconfigPath = "/root/.kube/kurator-member.config" - // build two secrets - kubeconfig1, readfileErr1 := os.ReadFile(kubeconfig1Path) - gomega.Expect(readfileErr1).ShouldNot(gomega.HaveOccurred()) - data1 := make(map[string][]byte) - data1[memberClusterName1] = kubeconfig1 - secret1 = resources.NewSecret(namespace, memberClusterName1, data1) - - kubeconfig2, readfileErr2 := os.ReadFile(kubeconfig2Path) - gomega.Expect(readfileErr2).ShouldNot(gomega.HaveOccurred()) - data2 := make(map[string][]byte) - data2[memberClusterName2] = kubeconfig2 - secret2 = resources.NewSecret(namespace, memberClusterName2, data2) + // build secrets use member cluster kubeconfig + kubeconfig, readfileErr := os.ReadFile(kubeconfigPath) + gomega.Expect(readfileErr).ShouldNot(gomega.HaveOccurred()) + data := make(map[string][]byte) + data[memberClusterName] = kubeconfig + secret = resources.NewSecret(namespace, memberClusterName, data) // build two attachedclusters - secretKeyRef1 := clusterv1a1.SecretKeyRef{ - Name: memberClusterName1, - Key: memberClusterName1, - } - secretKeyRef2 := clusterv1a1.SecretKeyRef{ - Name: memberClusterName2, - Key: memberClusterName2, + secretKeyRef := clusterv1a1.SecretKeyRef{ + Name: memberClusterName, + Key: memberClusterName, } - attachedcluster1 = resources.NewAttachedCluster(namespace, memberClusterName1, secretKeyRef1) - attachedcluster2 = resources.NewAttachedCluster(namespace, memberClusterName2, secretKeyRef2) + attachedcluster = resources.NewAttachedCluster(namespace, memberClusterName, secretKeyRef) }) ginkgo.It("Create Fleet", func() { // step 1.create secrets - secretCreateErr1 := resources.CreateSecret(kubeClient, secret1) - secretCreateErr2 := resources.CreateSecret(kubeClient, secret2) - gomega.Expect(secretCreateErr1).ShouldNot(gomega.HaveOccurred()) - gomega.Expect(secretCreateErr2).ShouldNot(gomega.HaveOccurred()) + secretCreateErr := resources.CreateSecret(kubeClient, secret) + gomega.Expect(secretCreateErr).ShouldNot(gomega.HaveOccurred()) // step 2.create attachedclusters - attachedCreateErr1 := resources.CreateAttachedCluster(kuratorClient, attachedcluster1) - attachedCreateErr2 := resources.CreateAttachedCluster(kuratorClient, attachedcluster2) - gomega.Expect(attachedCreateErr1).ShouldNot(gomega.HaveOccurred()) - gomega.Expect(attachedCreateErr2).ShouldNot(gomega.HaveOccurred()) + attachedCreateErr := resources.CreateAttachedCluster(kuratorClient, attachedcluster) + gomega.Expect(attachedCreateErr).ShouldNot(gomega.HaveOccurred()) time.Sleep(5 * time.Second) // step 3.create fleet clusters := []*corev1.ObjectReference{ { - Name: memberClusterName1, - Kind: "AttachedCluster", - }, - { - Name: memberClusterName2, + Name: memberClusterName, Kind: "AttachedCluster", }, } diff --git a/e2e/suite.go b/e2e/suite.go new file mode 100644 index 00000000..0a0fd843 --- /dev/null +++ b/e2e/suite.go @@ -0,0 +1,17 @@ +/* +Copyright Kurator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e diff --git a/hack/e2e-test/build-clusters.sh b/hack/e2e-test/build-clusters.sh new file mode 100755 index 00000000..cbea3a39 --- /dev/null +++ b/hack/e2e-test/build-clusters.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash + +# shellcheck disable=SC2086,SC1090,SC2206,SC1091 +set -o errexit +set -o nounset +set -o pipefail + +# This script starts a local karmada control plane based on current codebase and with a certain number of clusters joined. +# Parameters: [HOST_IPADDRESS](optional) if you want to export clusters' API server port to specific IP address +# This script depends on utils in: ${REPO_ROOT}/hack/util.sh +# 1. used by developer to setup develop environment quickly. +# 2. used by e2e testing to setup test environment automatically. +ROOT_DIR=$(git rev-parse --show-toplevel)/hack +KIND_CONFIGS_ROOT=${ROOT_DIR}/kind-configs +source "${ROOT_DIR}"/util.sh + +KIND_VERSION=${KIND_VERSION:-"kindest/node:v1.27.3"} + +# variable define +KUBECONFIG_PATH=${KUBECONFIG_PATH:-"${HOME}/.kube"} +MAIN_KUBECONFIG=${MAIN_KUBECONFIG:-"${KUBECONFIG_PATH}/kurator-host.config"} +HOST_CLUSTER_NAME=${HOST_CLUSTER_NAME:-"kurator-host"} +MEMBER_CLUSTER_KUBECONFIG=${MEMBER_CLUSTER_KUBECONFIG:-"${KUBECONFIG_PATH}/kurator-member.config"} +MEMBER_CLUSTER_NAME=${MEMBER_CLUSTER_NAME:-"kurator-member"} +ENABLE_KIND_WITH_WORKER=${ENABLE_KIND_WITH_WORKER:-"false"} + +#prepare for kind cluster config +TEMP_PATH=$(mktemp -d) +echo -e "Preparing kind config in path: ${TEMP_PATH}" +#When the Enable worker option is turned on, select to copy the configuration that contains the worker. +if [ ${ENABLE_KIND_WITH_WORKER} = "true" ]; then + cp -rf ${ROOT_DIR}/kind-configs-with-worker/*.yaml "${TEMP_PATH}"/ +else + cp -rf "${KIND_CONFIGS_ROOT}"/*.yaml "${TEMP_PATH}"/ +fi + +util::create_cluster "${HOST_CLUSTER_NAME}" "${MAIN_KUBECONFIG}" "${KIND_VERSION}" "${TEMP_PATH}" "${TEMP_PATH}"/host.yaml +util::create_cluster "${MEMBER_CLUSTER_NAME}" "${MEMBER_CLUSTER_KUBECONFIG}" "${KIND_VERSION}" "${TEMP_PATH}" "${TEMP_PATH}"/member1.yaml + +util::check_clusters_ready "${MAIN_KUBECONFIG}" "${HOST_CLUSTER_NAME}" +sleep 5s +util::check_clusters_ready "${MEMBER_CLUSTER_KUBECONFIG}" "${MEMBER_CLUSTER_NAME}" +sleep 10s + +# connecting networks between primary, remote clusters +echo "connect primary <-> remote" +util::connect_kind_clusters "${HOST_CLUSTER_NAME}" "${MAIN_KUBECONFIG}" "${MEMBER_CLUSTER_NAME}" "${MEMBER_CLUSTER_KUBECONFIG}" 1 + +echo "cluster networks connected" + +echo "install metallb in host cluster" +util::install_metallb ${MAIN_KUBECONFIG} ${HOST_CLUSTER_NAME} "ipv4" "255" + + +echo "starting install metallb in member clusters" +MEMBER_CLUSTERS=(${MEMBER_CLUSTER_NAME}) +MEMBER_KUBECONFIGS=(${MEMBER_CLUSTER_KUBECONFIG}) +MEMBER_IPSPACES=("254" "253") +echo "install metallb in ${MEMBER_CLUSTERS}" +util::install_metallb ${MEMBER_KUBECONFIGS} ${MEMBER_CLUSTERS} "ipv4" ${MEMBER_IPSPACES} + +function print_success() { + echo "Local clusters is running." + echo -e "\nTo start using your host cluster, run:" + echo -e " export KUBECONFIG=${MAIN_KUBECONFIG}" + echo -e "\nTo manage your remote clusters, run:" + echo -e " export KUBECONFIG=${MEMBER_CLUSTER_KUBECONFIG}" +} + +print_success \ No newline at end of file diff --git a/hack/e2e-test/install.sh b/hack/e2e-test/install-kurator.sh similarity index 93% rename from hack/e2e-test/install.sh rename to hack/e2e-test/install-kurator.sh index aa7aa3da..a82d67bb 100755 --- a/hack/e2e-test/install.sh +++ b/hack/e2e-test/install-kurator.sh @@ -9,8 +9,8 @@ set -o pipefail KUBECONFIG_PATH=${KUBECONFIG_PATH:-"${HOME}/.kube"} MAIN_KUBECONFIG=${MAIN_KUBECONFIG:-"${KUBECONFIG_PATH}/kurator-host.config"} export KUBECONFIG=${MAIN_KUBECONFIG} -COMMIT_ID=$(git rev-parse --short HEAD) -VERSION=${COMMIT_ID} +COMMIT_ID=$(git rev-parse --short HEAD) +VERSION=$(echo "$COMMIT_ID" | grep -o '^[0-9]') sleep 5s diff --git a/hack/local-dev-setup.sh b/hack/local-dev-setup.sh index a9fc34d3..bee5432c 100755 --- a/hack/local-dev-setup.sh +++ b/hack/local-dev-setup.sh @@ -14,7 +14,7 @@ ROOT_DIR=$(git rev-parse --show-toplevel)/hack KIND_CONFIGS_ROOT=${ROOT_DIR}/kind-configs source "${ROOT_DIR}"/util.sh -KIND_VERSION=${KIND_VERSION:-"kindest/node:v1.27.3"} +KIND_VERSION=${KIND_VERSION:-"kindest/node:v1.25.3"} # variable define KUBECONFIG_PATH=${KUBECONFIG_PATH:-"${HOME}/.kube"}