debug e2e

Signed-off-by: root <[email protected]>

Author: ldpliu

agent/pkg/spec/syncers/clusterlabel_syncer.go

@@ -20,6 +20,8 @@ import (
 	"github.com/stolostron/multicluster-global-hub/pkg/utils"
 )
 
+var log = logger.DefaultZapLogger()
+
 const (
 	// periodicApplyInterval = 5 * time.Second
 	hohFieldManager = "mgh-agent"
@@ -51,6 +53,7 @@ func (syncer *managedClusterLabelsBundleSyncer) Sync(ctx context.Context, payloa
 	if err := json.Unmarshal(payload, bundle); err != nil {
 		return err
 	}
+	syncer.log.Debugf("start sync bundle: %v", bundle)
 	syncer.setLatestBundle(bundle) // uses latestBundle
 	syncer.handleBundle()
 
@@ -69,6 +72,8 @@ func (syncer *managedClusterLabelsBundleSyncer) handleBundle() {
 	defer syncer.latestBundleLock.Unlock()
 
 	for _, managedClusterLabelsSpec := range syncer.latestBundle.Objects {
+		log.Debugf("managedClusterLabelsSpec:%v", *managedClusterLabelsSpec)
+
 		lastProcessedTimestampPtr := syncer.getManagedClusterLastProcessedTimestamp(managedClusterLabelsSpec.ClusterName)
 		if managedClusterLabelsSpec.UpdateTimestamp.After(*lastProcessedTimestampPtr) { // handle (success) once
 			syncer.bundleProcessingWaitingGroup.Add(1)
@@ -115,11 +120,13 @@ func (s *managedClusterLabelsBundleSyncer) updateManagedClusterAsync(
 		for key, value := range labelsSpec.Labels {
 			managedCluster.Labels[key] = value
 		}
+		log.Debugf("managedCluster.Labels: %v", managedCluster.Labels)
 
 		// delete labels by key
 		for _, labelKey := range labelsSpec.DeletedLabelKeys {
 			delete(managedCluster.Labels, labelKey)
 		}
+		log.Debugf("managedCluster.Labels: %v", managedCluster.Labels)
 
 		if err := s.updateManagedFieldEntry(managedCluster, labelsSpec); err != nil {
 			s.log.Error(err, "failed to update managed cluster", "name", labelsSpec.ClusterName)

agent/pkg/status/generic/generic_handler.go

@@ -6,8 +6,11 @@ import (
 
 	"github.com/stolostron/multicluster-global-hub/agent/pkg/status/interfaces"
 	genericpayload "github.com/stolostron/multicluster-global-hub/pkg/bundle/generic"
+	"github.com/stolostron/multicluster-global-hub/pkg/logger"
 )
 
+var log = logger.ZapLogger("generic-handler")
+
 type genericHandler struct {
 	eventData *genericpayload.GenericObjectBundle
 	// isSpec is to let the handler only update the event when spec is changed.
@@ -32,23 +35,28 @@ func NewGenericHandler(eventData *genericpayload.GenericObjectBundle, opts ...Ha
 }
 
 func (h *genericHandler) Get() interface{} {
+	log.Debugf("get obj:%v", h.eventData)
 	return h.eventData
 }
 
 func (h *genericHandler) Update(obj client.Object) bool {
+	log.Debugf("update obj:%v", obj)
+
 	if h.shouldUpdate != nil {
 		if updated := h.shouldUpdate(obj); !updated {
+			log.Debugf("h.shouldUpdate false")
 			return false
 		}
 	}
-
 	index := getObjectIndexByUID(obj.GetUID(), (*h.eventData))
 	if index == -1 { // object not found, need to add it to the bundle
 		(*h.eventData) = append((*h.eventData), obj)
 		return true
 	}
 
 	old := (*h.eventData)[index]
+	log.Debugf("obj: %v", old)
+
 	if h.isSpec && old.GetGeneration() == obj.GetGeneration() {
 		return false
 	}
@@ -121,6 +129,8 @@ func WithSpec(onlySpec bool) HandlerOption {
 }
 
 func WithShouldUpdate(shouldUpdate func(client.Object) bool) HandlerOption {
+	log.Debugf("g.shouldUpdate:%v, shouldUpdate:%v", shouldUpdate, shouldUpdate)
+
 	return func(g *genericHandler) {
 		g.shouldUpdate = shouldUpdate
 	}

manager/pkg/spec/syncers/managedcluster_labels_syncer.go

@@ -105,12 +105,14 @@ func getUpdatedManagedClusterLabelsBundles(timestamp *time.Time,
 func getManagedClusterLabelBundleByRows(db *gorm.DB, rows *sql.Rows) (
 	map[string]*spec.ManagedClusterLabelsSpecBundle, error,
 ) {
+	log.Debugf("getManagedClusterLabelBundleByRows")
 	leafHubToLabelsSpecBundleMap := make(map[string]*spec.ManagedClusterLabelsSpecBundle)
 	for rows.Next() {
 		managedClusterLabel := models.ManagedClusterLabel{}
 		if err := db.ScanRows(rows, &managedClusterLabel); err != nil {
 			return nil, fmt.Errorf("error reading managed cluster label from table - %w", err)
 		}
+		log.Debugf("managedClusterLabel: %v", managedClusterLabel)
 
 		// create ManagedClusterLabelsSpecBundle if not mapped for leafHub
 		managedClusterLabelsSpecBundle, found := leafHubToLabelsSpecBundleMap[managedClusterLabel.LeafHubName]
@@ -127,12 +129,14 @@ func getManagedClusterLabelBundleByRows(db *gorm.DB, rows *sql.Rows) (
 		if err != nil {
 			return nil, fmt.Errorf("error to unmarshal labels - %w", err)
 		}
+		log.Debugf("labels: %v", labels)
 
 		deletedKeys := []string{}
 		err = json.Unmarshal(managedClusterLabel.DeletedLabelKeys, &deletedKeys)
 		if err != nil {
 			return nil, fmt.Errorf("error to unmarshal deletedKeys - %w", err)
 		}
+		log.Debugf("deletedKeys: %v", deletedKeys)
 
 		managedClusterLabelsSpecBundle.Objects = append(managedClusterLabelsSpecBundle.Objects,
 			&spec.ManagedClusterLabelsSpec{

manager/pkg/spec/syncers/managedcluster_labels_watcher.go

@@ -18,6 +18,8 @@ import (
 	"github.com/stolostron/multicluster-global-hub/pkg/logger"
 )
 
+var log = logger.ZapLogger("labels-watcher")
+
 const (
 	labelsTableName             = "managed_clusters_labels"
 	clusterTableName            = "managed_clusters"
@@ -66,7 +68,7 @@ func (watcher *managedClusterLabelsStatusWatcher) updateDeletedLabelsPeriodicall
 			return
 
 		case <-labelsTrimmerTicker.C:
-
+			log.Debugf("labelsTrimmerTicker")
 			_, cancelFunc := context.WithTimeout(ctx, watcher.intervalPolicy.GetMaxInterval())
 
 			// update the deleted label keys to label table by the managed cluster table
@@ -106,10 +108,12 @@ func (watcher *managedClusterLabelsStatusWatcher) updateDeletedLabelsPeriodicall
 
 func (watcher *managedClusterLabelsStatusWatcher) updateDeletedLabelsByManagedCluster() bool {
 	leafHubToLabelsSpecBundleMap, err := getLabelBundleWithDeletedKey()
+	log.Debugf("leafHubToLabelsSpecBundleMap: %v, err: %v", leafHubToLabelsSpecBundleMap, err)
 	if err != nil {
 		watcher.log.Error(err, "trimming cycle skipped")
 		return false
 	}
+	log.Debugf("updateDeletedLabelsByManagedCluster")
 	conn := database.GetConn()
 	err = database.Lock(conn)
 	if err != nil {
@@ -120,9 +124,12 @@ func (watcher *managedClusterLabelsStatusWatcher) updateDeletedLabelsByManagedCl
 
 	result := true
 	// iterate over entries
+	log.Debugf("updateDeletedLabelsByManagedCluster")
+
 	for _, managedClusterLabelsSpecBundle := range leafHubToLabelsSpecBundleMap {
 		// fetch actual labels status reflected in status DB
 		for _, managedClusterLabelsSpec := range managedClusterLabelsSpecBundle.Objects {
+			log.Debugf("managedClusterLabelsSpec: %v", managedClusterLabelsSpec)
 			labelsStatus, err := getLabelsFromManagedCluster(managedClusterLabelsSpecBundle.LeafHubName,
 				managedClusterLabelsSpec.ClusterName)
 			if err != nil {
@@ -153,13 +160,16 @@ func (watcher *managedClusterLabelsStatusWatcher) updateDeletedLabelsByManagedCl
 				result = false
 				continue
 			}
+			log.Debugf("update label table by managed cluster table successfully", "leafHub",
+				managedClusterLabelsSpecBundle.LeafHubName,
+				"managedCluster", managedClusterLabelsSpec.ClusterName, "version", managedClusterLabelsSpec.Version)
 
 			watcher.log.Debugw("update label table by managed cluster table successfully", "leafHub",
 				managedClusterLabelsSpecBundle.LeafHubName,
 				"managedCluster", managedClusterLabelsSpec.ClusterName, "version", managedClusterLabelsSpec.Version)
 		}
 	}
-
+	log.Debugf("result: %v", result)
 	return result
 }
 

test/e2e/cluster_test.go

@@ -14,6 +14,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/klog"
 	addonapiv1alpha1 "open-cluster-management.io/api/addon/v1alpha1"
 	clusterv1 "open-cluster-management.io/api/cluster/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -87,7 +88,7 @@ var _ = Describe("Managed Clusters", Label("e2e-test-cluster"), Ordered, func()
 					return fmt.Errorf("want messsage %s, got %s", eventMessage, clusterEvent.Message)
 				}
 				return nil
-			}, 3*time.Minute, 1*time.Second).ShouldNot(HaveOccurred())
+			}, 3*time.Minute, 10*time.Second).ShouldNot(HaveOccurred())
 
 			By("Delete the cluster event from the leafhub")
 			Expect(hubClient.Delete(ctx, clusterEvent)).To(Succeed())
@@ -273,8 +274,10 @@ func getManagedClusterByName(client *http.Client, managedClusterName string) (
 	if err != nil {
 		return nil, err
 	}
-
+	klog.Infof("len(managedClusterList):%v \n", len(managedClusterList.Items))
 	for _, managedCluster := range managedClusterList.Items {
+		klog.Infof("managedcluster: %v, label: %v \n", managedCluster.Name, managedCluster.Labels)
+		klog.Infof("managedClusterName: %v \n", managedClusterName)
 		if managedCluster.Name == managedClusterName {
 			return &managedCluster, nil
 		}
@@ -292,6 +295,7 @@ func assertAddLabel(cluster clusterv1.ManagedCluster, labelKey, labelVal string)
 		},
 	}
 	By("Check the label is added")
+	count := 1
 	Eventually(func() error {
 		err := updateClusterLabelByAPI(httpClient, patches, GetClusterID(cluster))
 		if err != nil {
@@ -306,8 +310,14 @@ func assertAddLabel(cluster clusterv1.ManagedCluster, labelKey, labelVal string)
 				return nil
 			}
 		}
+		count++
+		if count > 150 {
+			time.Sleep(200 * time.Minute)
+		}
+		klog.Infof("count: %v\n", count)
+		klog.Infof("failed to add label [%s: %s] to cluster %s \n", labelKey, labelVal, cluster.Name)
 		return fmt.Errorf("failed to add label [%s: %s] to cluster %s", labelKey, labelVal, cluster.Name)
-	}, 3*time.Minute, 1*time.Second).ShouldNot(HaveOccurred())
+	}, 3*time.Minute, 10*time.Second).ShouldNot(HaveOccurred())
 }
 
 func assertRemoveLabel(cluster clusterv1.ManagedCluster, labelKey, labelVal string) {
@@ -334,7 +344,7 @@ func assertRemoveLabel(cluster clusterv1.ManagedCluster, labelKey, labelVal stri
 			}
 		}
 		return nil
-	}, 3*time.Minute, 1*time.Second).ShouldNot(HaveOccurred())
+	}, 3*time.Minute, 10*time.Second).ShouldNot(HaveOccurred())
 }
 
 func updateClusterLabelByAPI(client *http.Client, patches []patch, managedClusterID string) error {