placement_test.go

package e2e

import (
	"context"
	"fmt"

	"github.com/onsi/ginkgo/v2"
	"github.com/onsi/gomega"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	utilerrors "k8s.io/apimachinery/pkg/util/errors"
	"k8s.io/apimachinery/pkg/util/rand"

	clusterapiv1 "open-cluster-management.io/api/cluster/v1"
	clusterapiv1beta1 "open-cluster-management.io/api/cluster/v1beta1"
	clusterapiv1beta2 "open-cluster-management.io/api/cluster/v1beta2"

	"open-cluster-management.io/ocm/test/integration/util"
)

const (
	e2eTestLabel             = "created-by"
	e2eTestLabelValue        = "placement-e2e-test"
	maxNumOfClusterDecisions = 100
)

// Test cases with lable "sanity-check" could be ran as sanity check on an existing environment with
// placement controller installed and well configured . Resource leftovers should be cleaned up on
// the hub cluster.
var _ = ginkgo.Describe("Placement", ginkgo.Label("placement", "sanity-check"), func() {
	var namespace string
	var placementName string
	var clusterSet1Name string
	var clusterSetGlobal string
	var suffix string
	var err error

	ginkgo.BeforeEach(func() {
		suffix = rand.String(5)
		namespace = fmt.Sprintf("ns-%s", suffix)
		placementName = fmt.Sprintf("placement-%s", suffix)
		clusterSet1Name = fmt.Sprintf("clusterset-%s", suffix)
		clusterSetGlobal = "global"

		// create testing namespace
		ns := &corev1.Namespace{
			ObjectMeta: metav1.ObjectMeta{
				Name: namespace,
				Labels: map[string]string{
					e2eTestLabel: e2eTestLabelValue,
				},
			},
		}
		_, err := hub.KubeClient.CoreV1().Namespaces().Create(context.Background(), ns, metav1.CreateOptions{})
		gomega.Expect(err).ToNot(gomega.HaveOccurred())
	})

	ginkgo.AfterEach(func() {
		var errs []error
		ginkgo.By("Delete managedclustersets")
		err := hub.ClusterClient.ClusterV1beta2().ManagedClusterSets().DeleteCollection(context.Background(), metav1.DeleteOptions{}, metav1.ListOptions{
			LabelSelector: e2eTestLabel + "=" + e2eTestLabelValue,
		})
		if err != nil {
			errs = append(errs, err)
		}

		ginkgo.By("Delete managedclusters")
		err = hub.ClusterClient.ClusterV1().ManagedClusters().DeleteCollection(context.Background(), metav1.DeleteOptions{}, metav1.ListOptions{
			LabelSelector: e2eTestLabel + "=" + e2eTestLabelValue,
		})
		if err != nil {
			errs = append(errs, err)
		}

		ginkgo.By("Delete namespace")
		err = hub.KubeClient.CoreV1().Namespaces().Delete(context.Background(), namespace, metav1.DeleteOptions{})
		if err != nil {
			errs = append(errs, err)
		}
		gomega.Expect(utilerrors.NewAggregate(errs)).ToNot(gomega.HaveOccurred())

	})

	assertPlacementDecisionCreated := func(placement *clusterapiv1beta1.Placement) {
		ginkgo.By("Check if placementdecision is created")
		gomega.Eventually(func() bool {
			pdl, err := hub.ClusterClient.ClusterV1beta1().PlacementDecisions(namespace).List(context.Background(), metav1.ListOptions{
				LabelSelector: clusterapiv1beta1.PlacementLabel + "=" + placement.Name,
			})
			if err != nil {
				return false
			}
			if len(pdl.Items) == 0 {
				return false
			}
			for _, pd := range pdl.Items {
				objectMeta := pd.ObjectMeta
				if controlled := metav1.IsControlledBy(&objectMeta, placement); !controlled {
					return false
				}
			}
			return true
		}).Should(gomega.BeTrue())
	}

	assertNumberOfDecisions := func(placementName string, desiredNOD int) {
		ginkgo.By("Check the number of decisions in placementdecisions")
		desiredNOPD := desiredNOD/maxNumOfClusterDecisions + 1
		gomega.Eventually(func() bool {
			pdl, err := hub.ClusterClient.ClusterV1beta1().PlacementDecisions(namespace).List(context.Background(), metav1.ListOptions{
				LabelSelector: clusterapiv1beta1.PlacementLabel + "=" + placementName,
			})
			if err != nil {
				return false
			}
			if len(pdl.Items) != desiredNOPD {
				return false
			}
			actualNOD := 0
			for _, pd := range pdl.Items {
				actualNOD += len(pd.Status.Decisions)
			}
			return actualNOD == desiredNOD
		}).Should(gomega.BeTrue())
	}

	assertPlacementStatus := func(placementName string, numOfSelectedClusters int, satisfied bool) {
		ginkgo.By("Check the status of placement")
		gomega.Eventually(func() bool {
			placement, err := hub.ClusterClient.ClusterV1beta1().Placements(namespace).Get(context.Background(), placementName, metav1.GetOptions{})
			if err != nil {
				return false
			}
			status := metav1.ConditionFalse
			if satisfied {
				status = metav1.ConditionTrue
			}
			if !util.HasCondition(
				placement.Status.Conditions,
				clusterapiv1beta1.PlacementConditionSatisfied,
				"",
				status,
			) {
				return false
			}

			if !util.HasCondition(
				placement.Status.Conditions,
				clusterapiv1beta1.PlacementConditionMisconfigured,
				"Succeedconfigured",
				metav1.ConditionFalse,
			) {
				return false
			}

			return placement.Status.NumberOfSelectedClusters == int32(numOfSelectedClusters)
		}).Should(gomega.BeTrue())
	}

	assertCreatingClusterSet := func(clusterSetName string, matchLabel map[string]string) {
		ginkgo.By("Create clusterset")
		clusterset := &clusterapiv1beta2.ManagedClusterSet{
			ObjectMeta: metav1.ObjectMeta{
				Name: clusterSetName,
				Labels: map[string]string{
					e2eTestLabel: e2eTestLabelValue,
				},
			},
		}
		if matchLabel != nil {
			clusterset.Spec.ClusterSelector = clusterapiv1beta2.ManagedClusterSelector{
				SelectorType: clusterapiv1beta2.LabelSelector,
				LabelSelector: &metav1.LabelSelector{
					MatchLabels: matchLabel,
				},
			}
		}
		_, err = hub.ClusterClient.ClusterV1beta2().ManagedClusterSets().Create(context.Background(), clusterset, metav1.CreateOptions{})
		gomega.Expect(err).ToNot(gomega.HaveOccurred())
	}

	assertCreatingClusterSetBinding := func(clusterSetName string) {
		ginkgo.By("Create clustersetbinding")
		csb := &clusterapiv1beta2.ManagedClusterSetBinding{
			ObjectMeta: metav1.ObjectMeta{
				Namespace: namespace,
				Name:      clusterSetName,
				Labels: map[string]string{
					e2eTestLabel: e2eTestLabelValue,
				},
			},
			Spec: clusterapiv1beta2.ManagedClusterSetBindingSpec{
				ClusterSet: clusterSetName,
			},
		}
		_, err = hub.ClusterClient.ClusterV1beta2().ManagedClusterSetBindings(namespace).Create(context.Background(), csb, metav1.CreateOptions{})
		gomega.Expect(err).ToNot(gomega.HaveOccurred())
	}

	assertBindingClusterSet := func(clusterSetName string, matchLabel map[string]string) {
		ginkgo.By("Create clusterset/clustersetbinding")
		assertCreatingClusterSet(clusterSetName, matchLabel)
		assertCreatingClusterSetBinding(clusterSetName)
	}

	assertCreatingClusters := func(clusterSetName string, num int) {
		ginkgo.By(fmt.Sprintf("Create %d clusters", num))
		for i := 0; i < num; i++ {
			labels := map[string]string{
				e2eTestLabel: e2eTestLabelValue,
			}
			if len(clusterSetName) > 0 {
				labels[clusterapiv1beta2.ClusterSetLabel] = clusterSetName
			}
			cluster := &clusterapiv1.ManagedCluster{
				ObjectMeta: metav1.ObjectMeta{
					GenerateName: "cluster-",
					Labels:       labels,
				},
			}
			_, err = hub.ClusterClient.ClusterV1().ManagedClusters().Create(context.Background(), cluster, metav1.CreateOptions{})
			gomega.Expect(err).ToNot(gomega.HaveOccurred())
		}
	}

	assertCreatingPlacement := func(name string, noc *int32, nod int) {
		ginkgo.By("Create placement")
		placement := &clusterapiv1beta1.Placement{
			ObjectMeta: metav1.ObjectMeta{
				Namespace: namespace,
				Name:      name,
				Labels: map[string]string{
					e2eTestLabel: e2eTestLabelValue,
				},
			},
			Spec: clusterapiv1beta1.PlacementSpec{
				NumberOfClusters: noc,
				Tolerations: []clusterapiv1beta1.Toleration{
					{
						Key: clusterapiv1.ManagedClusterTaintUnreachable,
					},
				},
			},
		}

		placement, err = hub.ClusterClient.ClusterV1beta1().Placements(namespace).Create(context.Background(), placement, metav1.CreateOptions{})
		gomega.Expect(err).ToNot(gomega.HaveOccurred())

		assertPlacementDecisionCreated(placement)
		assertNumberOfDecisions(placementName, nod)
		if noc != nil {
			assertPlacementStatus(placementName, nod, nod == int(*noc))
		}
	}

	ginkgo.It("Should schedule successfully", func() {
		assertBindingClusterSet(clusterSet1Name, nil)
		assertCreatingClusters(clusterSet1Name, 5)
		assertCreatingPlacement(placementName, noc(10), 5)

		ginkgo.By("Reduce NOC of the placement")
		gomega.Eventually(func() error {
			placement, err := hub.ClusterClient.ClusterV1beta1().Placements(namespace).Get(context.Background(), placementName, metav1.GetOptions{})
			if err != nil {
				return err
			}
			noc := int32(6)
			placement.Spec.NumberOfClusters = &noc
			_, err = hub.ClusterClient.ClusterV1beta1().Placements(namespace).Update(context.Background(), placement, metav1.UpdateOptions{})
			return err
		}).ShouldNot(gomega.HaveOccurred())

		assertNumberOfDecisions(placementName, 5)
		assertPlacementStatus(placementName, 5, false)

		// create global clusterset if necessary
		assertCreatingClusterSetBinding(clusterSetGlobal)

		// create 2 more clusters belong to global clusterset
		assertCreatingClusters("", 2)
		assertNumberOfDecisions(placementName, 6)
		assertPlacementStatus(placementName, 6, true)

		ginkgo.By("Delete placement")
		err = hub.ClusterClient.ClusterV1beta1().Placements(namespace).Delete(context.TODO(), placementName, metav1.DeleteOptions{})
		gomega.Expect(err).ToNot(gomega.HaveOccurred())

		ginkgo.By("Check if placementdecisions are deleted as well")
		gomega.Eventually(func() bool {
			placementDecisions, err := hub.ClusterClient.ClusterV1beta1().PlacementDecisions(namespace).List(context.TODO(), metav1.ListOptions{
				LabelSelector: fmt.Sprintf("%s=%s", clusterapiv1beta1.PlacementLabel, placementName),
			})
			if err != nil {
				return false
			}

			return len(placementDecisions.Items) == 0
		}).Should(gomega.BeTrue())
	})

	ginkgo.It("Should delete placementdecision successfully", func() {
		assertBindingClusterSet(clusterSet1Name, nil)
		assertCreatingClusters(clusterSet1Name, 1)
		assertCreatingPlacement(placementName, nil, 1)

		ginkgo.By("Add cluster predicate")
		gomega.Eventually(func() error {
			placement, err := hub.ClusterClient.ClusterV1beta1().Placements(namespace).Get(context.Background(), placementName, metav1.GetOptions{})
			if err != nil {
				return err
			}
			placement.Spec.Predicates = []clusterapiv1beta1.ClusterPredicate{
				{
					RequiredClusterSelector: clusterapiv1beta1.ClusterSelector{
						LabelSelector: metav1.LabelSelector{
							MatchLabels: map[string]string{
								"a": "b",
							},
						},
					},
				},
			}
			_, err = hub.ClusterClient.ClusterV1beta1().Placements(namespace).Update(context.Background(), placement, metav1.UpdateOptions{})
			return err
		}).ShouldNot(gomega.HaveOccurred())

		ginkgo.By("Create empty placement decision")
		assertNumberOfDecisions(placementName, 0)
		assertPlacementStatus(placementName, 0, false)

		ginkgo.By("Delete placement")
		err = hub.ClusterClient.ClusterV1beta1().Placements(namespace).Delete(context.TODO(), placementName, metav1.DeleteOptions{})
		gomega.Expect(err).ToNot(gomega.HaveOccurred())
	})

	//update this case when version upgrade
	ginkgo.It("Should support v1beta1 placement", func() {
		ginkgo.By("Create v1beta1 placement with v1beta1 client")
		placement := &clusterapiv1beta1.Placement{
			ObjectMeta: metav1.ObjectMeta{
				Namespace: namespace,
				Name:      "test",
			},
			Spec: clusterapiv1beta1.PlacementSpec{
				Tolerations: []clusterapiv1beta1.Toleration{
					{
						Key: clusterapiv1.ManagedClusterTaintUnreachable,
					},
				},
			},
		}

		_, err = hub.ClusterClient.ClusterV1beta1().Placements(namespace).Create(context.Background(), placement, metav1.CreateOptions{})
		gomega.Expect(err).ToNot(gomega.HaveOccurred())
	})
})

func noc(n int) *int32 {
	noc := int32(n)
	return &noc
}