diff --git a/controllers/component_manager.go b/controllers/component_manager.go index 026b2990b..2e6e341b5 100644 --- a/controllers/component_manager.go +++ b/controllers/component_manager.go @@ -49,13 +49,13 @@ func NewComponentManager( return allComponents } - d := components.NewDiscovery(cfgen, ytsaurus) m := components.NewMaster(cfgen, ytsaurus) var hps []components.Component for _, hpSpec := range ytsaurus.GetResource().Spec.HTTPProxies { hps = append(hps, components.NewHTTPProxy(cfgen, ytsaurus, m, hpSpec)) } yc := components.NewYtsaurusClient(cfgen, ytsaurus, hps[0], getAllComponents) + d := components.NewDiscovery(cfgen, ytsaurus, yc) var dnds []components.Component if len(resource.Spec.DataNodes) > 0 { @@ -120,7 +120,7 @@ func NewComponentManager( } if resource.Spec.ControllerAgents != nil { - ca := components.NewControllerAgent(cfgen, ytsaurus, m) + ca := components.NewControllerAgent(cfgen, ytsaurus, m, yc) allComponents = append(allComponents, ca) } @@ -147,7 +147,7 @@ func NewComponentManager( } if resource.Spec.MasterCaches != nil { - mc := components.NewMasterCache(cfgen, ytsaurus) + mc := components.NewMasterCache(cfgen, ytsaurus, yc) allComponents = append(allComponents, mc) } diff --git a/pkg/components/controller_agent.go b/pkg/components/controller_agent.go index 47003e516..b691868a1 100644 --- a/pkg/components/controller_agent.go +++ b/pkg/components/controller_agent.go @@ -2,7 +2,10 @@ package components import ( "context" + "fmt" + "go.ytsaurus.tech/yt/go/ypath" + "go.ytsaurus.tech/yt/go/yt" corev1 "k8s.io/api/core/v1" ytv1 "github.com/ytsaurus/ytsaurus-k8s-operator/api/v1" @@ -14,11 +17,12 @@ import ( type ControllerAgent struct { localServerComponent - cfgen *ytconfig.Generator - master Component + cfgen *ytconfig.Generator + master Component + ytsaurusClient internalYtsaurusClient } -func NewControllerAgent(cfgen *ytconfig.Generator, ytsaurus *apiproxy.Ytsaurus, master Component) *ControllerAgent { +func NewControllerAgent(cfgen *ytconfig.Generator, ytsaurus *apiproxy.Ytsaurus, master Component, yc internalYtsaurusClient) *ControllerAgent { l := cfgen.GetComponentLabeller(consts.ControllerAgentType, "") resource := ytsaurus.GetResource() @@ -41,6 +45,7 @@ func NewControllerAgent(cfgen *ytconfig.Generator, ytsaurus *apiproxy.Ytsaurus, localServerComponent: newLocalServerComponent(l, ytsaurus, srv), cfgen: cfgen, master: master, + ytsaurusClient: yc, } } @@ -56,8 +61,23 @@ func (ca *ControllerAgent) doSync(ctx context.Context, dry bool) (ComponentStatu } if ca.ytsaurus.GetClusterState() == ytv1.ClusterStateUpdating { - if status, err := handleUpdatingClusterState(ctx, ca.ytsaurus, ca, &ca.localComponent, ca.server, dry); status != nil { - return *status, err + if IsUpdatingComponent(ca.ytsaurus, ca) { + switch getComponentUpdateStrategy(ca.ytsaurus, consts.ControllerAgentType, ca.GetShortName()) { + case ytv1.ComponentUpdateModeTypeOnDelete: + if status, err := handleOnDeleteUpdatingClusterState(ctx, ca.ytsaurus, ca, &ca.localComponent, ca.server, dry); status != nil { + return *status, err + } + default: + if status, err := handleBulkUpdatingClusterState(ctx, ca.ytsaurus, ca, &ca.localComponent, ca.server, dry); status != nil { + return *status, err + } + } + + if ca.ytsaurus.GetUpdateState() != ytv1.UpdateStateWaitingForPodsCreation { + return ComponentStatusReady(), err + } + } else { + return ComponentStatusReadyAfter("Not updating component"), nil } } @@ -91,3 +111,59 @@ func (ca *ControllerAgent) Sync(ctx context.Context) error { _, err := ca.doSync(ctx, false) return err } + +type ControllerAgentsWithMaintenance struct { + Address string `yson:",value"` + Maintenance bool `yson:"maintenance,attr"` + Alerts []string `yson:"alerts,attr"` +} + +func (ca *ControllerAgent) UpdatePreCheck(ctx context.Context) ComponentStatus { + if ca.ytsaurusClient == nil { + return ComponentStatusBlocked("YtsaurusClient component is not available") + } + + ytClient := ca.ytsaurusClient.GetYtClient() + if ytClient == nil { + return ComponentStatusBlocked("YT client is not available") + } + + // Check that the number of instances in YT matches the expected instanceCount + expectedCount := int(ca.ytsaurus.GetResource().Spec.ControllerAgents.InstanceCount) + if err := IsInstanceCountEqualYTSpec(ctx, ytClient, consts.ControllerAgentType, expectedCount); err != nil { + return ComponentStatusBlocked(err.Error()) + } + + controllerAgentsWithMaintenance := make([]ControllerAgentsWithMaintenance, 0) + cypressPath := consts.ComponentCypressPath(consts.ControllerAgentType) + + err := ca.ytsaurusClient.GetYtClient().ListNode(ctx, ypath.Path(cypressPath), &controllerAgentsWithMaintenance, &yt.ListNodeOptions{ + Attributes: []string{"maintenance", "alerts"}}) + if err != nil { + return ComponentStatusBlocked(err.Error()) + } + + for _, controllerAgent := range controllerAgentsWithMaintenance { + var connected bool + err = ca.ytsaurusClient.GetYtClient().GetNode( + ctx, + ypath.Path(fmt.Sprintf("%v/%v/orchid/controller_agent/service/connected", cypressPath, controllerAgent.Address)), + &connected, + nil) + if err != nil { + return ComponentStatusBlocked(err.Error()) + } + + if !connected { + msg := fmt.Sprintf("Controller agent is not connected: %v", controllerAgent.Address) + return ComponentStatusBlocked(msg) + } + + if controllerAgent.Maintenance { + msg := fmt.Sprintf("There is a controller agent in maintenance: %v", controllerAgent.Address) + return ComponentStatusBlocked(msg) + } + } + + return ComponentStatusReady() +} diff --git a/pkg/components/discovery.go b/pkg/components/discovery.go index b447883d1..4d3371803 100644 --- a/pkg/components/discovery.go +++ b/pkg/components/discovery.go @@ -2,6 +2,7 @@ package components import ( "context" + "fmt" corev1 "k8s.io/api/core/v1" @@ -10,14 +11,16 @@ import ( "github.com/ytsaurus/ytsaurus-k8s-operator/pkg/consts" "github.com/ytsaurus/ytsaurus-k8s-operator/pkg/resources" "github.com/ytsaurus/ytsaurus-k8s-operator/pkg/ytconfig" + "go.ytsaurus.tech/yt/go/ypath" ) type Discovery struct { localServerComponent - cfgen *ytconfig.Generator + cfgen *ytconfig.Generator + ytsaurusClient internalYtsaurusClient } -func NewDiscovery(cfgen *ytconfig.Generator, ytsaurus *apiproxy.Ytsaurus) *Discovery { +func NewDiscovery(cfgen *ytconfig.Generator, ytsaurus *apiproxy.Ytsaurus, yc internalYtsaurusClient) *Discovery { l := cfgen.GetComponentLabeller(consts.DiscoveryType, "") resource := ytsaurus.GetResource() @@ -41,6 +44,7 @@ func NewDiscovery(cfgen *ytconfig.Generator, ytsaurus *apiproxy.Ytsaurus) *Disco return &Discovery{ localServerComponent: newLocalServerComponent(l, ytsaurus, srv), cfgen: cfgen, + ytsaurusClient: yc, } } @@ -56,8 +60,23 @@ func (d *Discovery) doSync(ctx context.Context, dry bool) (ComponentStatus, erro } if d.ytsaurus.GetClusterState() == ytv1.ClusterStateUpdating { - if status, err := handleUpdatingClusterState(ctx, d.ytsaurus, d, &d.localComponent, d.server, dry); status != nil { - return *status, err + if IsUpdatingComponent(d.ytsaurus, d) { + switch getComponentUpdateStrategy(d.ytsaurus, consts.DiscoveryType, d.GetShortName()) { + case ytv1.ComponentUpdateModeTypeOnDelete: + if status, err := handleOnDeleteUpdatingClusterState(ctx, d.ytsaurus, d, &d.localComponent, d.server, dry); status != nil { + return *status, err + } + default: + if status, err := handleBulkUpdatingClusterState(ctx, d.ytsaurus, d, &d.localComponent, d.server, dry); status != nil { + return *status, err + } + } + + if d.ytsaurus.GetUpdateState() != ytv1.UpdateStateWaitingForPodsCreation { + return ComponentStatusReady(), err + } + } else { + return ComponentStatusReadyAfter("Not updating component"), nil } } @@ -83,3 +102,39 @@ func (d *Discovery) Sync(ctx context.Context) error { _, err := d.doSync(ctx, false) return err } + +func (d *Discovery) UpdatePreCheck(ctx context.Context) ComponentStatus { + if d.ytsaurusClient == nil { + return ComponentStatusBlocked("YtsaurusClient component is not available") + } + + ytClient := d.ytsaurusClient.GetYtClient() + if ytClient == nil { + return ComponentStatusBlocked("YT client is not available") + } + + // Check that the number of instances in YT matches the expected instanceCount + expectedCount := int(d.ytsaurus.GetResource().Spec.Discovery.InstanceCount) + if err := IsInstanceCountEqualYTSpec(ctx, ytClient, consts.DiscoveryType, expectedCount); err != nil { + return ComponentStatusBlocked(err.Error()) + } + + var discoveryServers []string + cypressPath := consts.ComponentCypressPath(consts.DiscoveryType) + + err := d.ytsaurusClient.GetYtClient().ListNode(ctx, ypath.Path(cypressPath), &discoveryServers, nil) + if err != nil { + return ComponentStatusBlocked(err.Error()) + } + + // Check that all discovery_servers are alive + for _, server := range discoveryServers { + orchidPath := ypath.Path(fmt.Sprintf("%s/%s/orchid", cypressPath, server)) + var orchidData map[string]interface{} + if err := ytClient.GetNode(ctx, orchidPath, &orchidData, nil); err != nil { + return ComponentStatusBlocked(fmt.Sprintf("Failed to get discovery orchid data for %s: %v", server, err)) + } + } + + return ComponentStatusReady() +} diff --git a/pkg/components/master_caches.go b/pkg/components/master_caches.go index 9361bac00..5f5b33d40 100644 --- a/pkg/components/master_caches.go +++ b/pkg/components/master_caches.go @@ -14,10 +14,11 @@ import ( type MasterCache struct { localServerComponent - cfgen *ytconfig.Generator + cfgen *ytconfig.Generator + ytsaurusClient internalYtsaurusClient } -func NewMasterCache(cfgen *ytconfig.Generator, ytsaurus *apiproxy.Ytsaurus) *MasterCache { +func NewMasterCache(cfgen *ytconfig.Generator, ytsaurus *apiproxy.Ytsaurus, yc internalYtsaurusClient) *MasterCache { l := cfgen.GetComponentLabeller(consts.MasterCacheType, "") resource := ytsaurus.GetResource() @@ -39,6 +40,7 @@ func NewMasterCache(cfgen *ytconfig.Generator, ytsaurus *apiproxy.Ytsaurus) *Mas return &MasterCache{ localServerComponent: newLocalServerComponent(l, ytsaurus, srv), cfgen: cfgen, + ytsaurusClient: yc, } } @@ -54,8 +56,23 @@ func (mc *MasterCache) doSync(ctx context.Context, dry bool) (ComponentStatus, e } if mc.ytsaurus.GetClusterState() == ytv1.ClusterStateUpdating { - if status, err := handleUpdatingClusterState(ctx, mc.ytsaurus, mc, &mc.localComponent, mc.server, dry); status != nil { - return *status, err + if IsUpdatingComponent(mc.ytsaurus, mc) { + switch getComponentUpdateStrategy(mc.ytsaurus, consts.MasterCacheType, mc.GetShortName()) { + case ytv1.ComponentUpdateModeTypeOnDelete: + if status, err := handleOnDeleteUpdatingClusterState(ctx, mc.ytsaurus, mc, &mc.localComponent, mc.server, dry); status != nil { + return *status, err + } + default: + if status, err := handleBulkUpdatingClusterState(ctx, mc.ytsaurus, mc, &mc.localComponent, mc.server, dry); status != nil { + return *status, err + } + } + + if mc.ytsaurus.GetUpdateState() != ytv1.UpdateStateWaitingForPodsCreation { + return ComponentStatusReady(), err + } + } else { + return ComponentStatusReadyAfter("Not updating component"), nil } } @@ -99,3 +116,22 @@ func (mc *MasterCache) getHostAddressLabel() string { } return defaultHostAddressLabel } + +func (mc *MasterCache) UpdatePreCheck(ctx context.Context) ComponentStatus { + if mc.ytsaurusClient == nil { + return ComponentStatusBlocked("YtsaurusClient component is not available") + } + + ytClient := mc.ytsaurusClient.GetYtClient() + if ytClient == nil { + return ComponentStatusBlocked("YT client is not available") + } + + // Check that the number of instances in YT matches the expected instanceCount + expectedCount := int(mc.ytsaurus.GetResource().Spec.MasterCaches.InstanceCount) + if err := IsInstanceCountEqualYTSpec(ctx, ytClient, consts.MasterCacheType, expectedCount); err != nil { + return ComponentStatusBlocked(err.Error()) + } + + return ComponentStatusReady() +} diff --git a/test/e2e/helpers_test.go b/test/e2e/helpers_test.go index 9d50439d3..b985f89db 100644 --- a/test/e2e/helpers_test.go +++ b/test/e2e/helpers_test.go @@ -170,6 +170,16 @@ func getChangedPods(before, after map[string]corev1.Pod) changedObjects { return ret } +func getExpectedUpdatedPods(podsBeforeUpdate map[string]corev1.Pod, stsName string) []string { + expectedUpdated := make([]string, 0) + for name := range podsBeforeUpdate { + if strings.HasPrefix(name, stsName+"-") { + expectedUpdated = append(expectedUpdated, name) + } + } + return expectedUpdated +} + // updateSpecToTriggerAllComponentUpdate is a helper // that introduce spec change which should trigger change in all component static configs // and thus trigger all components update. diff --git a/test/e2e/ytsaurus_controller_test.go b/test/e2e/ytsaurus_controller_test.go index c3840ce72..c9906b779 100644 --- a/test/e2e/ytsaurus_controller_test.go +++ b/test/e2e/ytsaurus_controller_test.go @@ -1895,11 +1895,20 @@ exec "$@"` }) // integration Context("update plan strategy testing", Label("update", "plan", "strategy"), func() { + var podsBeforeUpdate map[string]corev1.Pod + + BeforeEach(func() { + By("Adding base components") + ytBuilder.WithBaseComponents() + }) + + JustBeforeEach(func(ctx context.Context) { + By("Getting pods before actions") + podsBeforeUpdate = getComponentPods(ctx, namespace) + }) DescribeTableSubtree("bulk strategy", Label("bulk"), func(componentType consts.ComponentType, stsName string) { BeforeEach(func() { - ytBuilder.WithBaseComponents() - switch componentType { case consts.QueryTrackerType: ytBuilder.WithQueryTracker() @@ -1912,7 +1921,21 @@ exec "$@"` case consts.MasterType: ytsaurus.Spec.CoreImage = testutil.YtsaurusImagePrevious ytsaurus.Spec.PrimaryMasters.InstanceCount = 3 + case consts.MasterCacheType: + ytsaurus.Spec.MasterCaches = &ytv1.MasterCachesSpec{ + InstanceSpec: ytv1.InstanceSpec{ + Image: ptr.To(testutil.YtsaurusImagePrevious), + InstanceCount: 3, + }, + } + case consts.DiscoveryType: + ytsaurus.Spec.Discovery.Image = ptr.To(testutil.YtsaurusImagePrevious) + ytsaurus.Spec.Discovery.InstanceCount = 3 + case consts.ControllerAgentType: + ytsaurus.Spec.ControllerAgents.Image = ptr.To(testutil.YtsaurusImagePrevious) + ytsaurus.Spec.ControllerAgents.InstanceCount = 3 } + ytsaurus.Spec.UpdatePlan = []ytv1.ComponentUpdateSelector{ { Component: ytv1.Component{Type: componentType}, @@ -1922,47 +1945,49 @@ exec "$@"` }, } }) + It("Should update "+stsName+" in bulkUpdate mode and have Running state", func(ctx context.Context) { By("Trigger " + stsName + " update") - switch componentType { - case consts.QueryTrackerType: - ytsaurus.Spec.QueryTrackers.Image = ptr.To(testutil.QueryTrackerImageCurrent) - case consts.MasterType: - ytsaurus.Spec.CoreImage = testutil.YtsaurusImageCurrent - } - + updateSpecToTriggerAllComponentUpdate(ytsaurus) UpdateObject(ctx, ytsaurus) EventuallyYtsaurus(ctx, ytsaurus, reactionTimeout).Should(HaveObservedGeneration()) + Expect(ytsaurus).Should(HaveClusterUpdatingComponents(componentType)) + + By("Verify bulk update condition is set") + bulkUpdateCondition := fmt.Sprintf("%s%s", componentType, consts.ConditionBulkUpdateModeStarted) + EventuallyYtsaurus(ctx, ytsaurus, reactionTimeout).Should(WithTransform( + func(yts *ytv1.Ytsaurus) []metav1.Condition { + return yts.Status.UpdateStatus.Conditions + }, + ContainElement(SatisfyAll( + HaveField("Type", bulkUpdateCondition), + HaveField("Status", metav1.ConditionTrue), + )), + )) + By("Waiting cluster update completes") EventuallyYtsaurus(ctx, ytsaurus, upgradeTimeout).Should(HaveClusterStateRunning()) podsAfter := getComponentPods(ctx, namespace) + changedPods := getChangedPods(podsBeforeUpdate, podsAfter) + expectedUpdated := getExpectedUpdatedPods(podsBeforeUpdate, stsName) - for name, pod := range podsAfter { - if strings.HasPrefix(name, stsName+"-") { - switch componentType { - case consts.QueryTrackerType: - Expect(pod.Spec.Containers[0].Image).To(Equal(*ytsaurus.Spec.QueryTrackers.Image)) - case consts.MasterType: - Expect(pod.Spec.Containers[0].Image).To(Equal(ytsaurus.Spec.CoreImage)) - } - } - } - - checkClusterHealth(ctx, ytClient) - checkChunkLocations(ytClient) + Expect(changedPods.Created).To(BeEmpty(), "created") + Expect(changedPods.Deleted).To(BeEmpty(), "deleted") + Expect(changedPods.Updated).To(ConsistOf(expectedUpdated), "updated") }) }, - Entry("update query tracker", Label("qt"), consts.QueryTrackerType, consts.GetStatefulSetPrefix(consts.QueryTrackerType)), - Entry("update master", Label("ms"), consts.MasterType, consts.GetStatefulSetPrefix(consts.MasterType)), + Entry("update query tracker", Label(consts.GetStatefulSetPrefix(consts.QueryTrackerType)), consts.QueryTrackerType, consts.GetStatefulSetPrefix(consts.QueryTrackerType)), + Entry("update master", Label(consts.GetStatefulSetPrefix(consts.MasterType)), consts.MasterType, consts.GetStatefulSetPrefix(consts.MasterType)), + Entry("update controller agent", Label(consts.GetStatefulSetPrefix(consts.ControllerAgentType)), consts.ControllerAgentType, consts.GetStatefulSetPrefix(consts.ControllerAgentType)), + Entry("update discovery", Label(consts.GetStatefulSetPrefix(consts.DiscoveryType)), consts.DiscoveryType, consts.GetStatefulSetPrefix(consts.DiscoveryType)), + Entry("update master cache", Label(consts.GetStatefulSetPrefix(consts.MasterCacheType)), consts.MasterCacheType, consts.GetStatefulSetPrefix(consts.MasterCacheType)), ) - DescribeTableSubtree("on-delete strategy", Label("ondelete"), func(componentType consts.ComponentType, stsName string) { BeforeEach(func() { - ytBuilder.WithBaseComponents() switch componentType { case consts.SchedulerType: ytsaurus.Spec.Schedulers = &ytv1.SchedulersSpec{ @@ -1986,16 +2011,8 @@ exec "$@"` } }) It("should update "+stsName+" with OnDelete strategy and have cluster Running state", func(ctx context.Context) { - podsBefore := getComponentPods(ctx, namespace) - By("Trigger " + stsName + " update") - switch componentType { - case consts.SchedulerType: - ytsaurus.Spec.Schedulers.Image = ptr.To(testutil.YtsaurusImageCurrent) - case consts.MasterType: - ytsaurus.Spec.CoreImage = testutil.YtsaurusImageCurrent - } - + updateSpecToTriggerAllComponentUpdate(ytsaurus) UpdateObject(ctx, ytsaurus) EventuallyYtsaurus(ctx, ytsaurus, reactionTimeout).Should(HaveObservedGeneration()) @@ -2005,31 +2022,34 @@ exec "$@"` ) By("Verify StatefulSet has OnDelete update strategy") - stsObject := appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: stsName}} - EventuallyObject(ctx, &stsObject, reactionTimeout).Should(HaveField("Spec.UpdateStrategy.Type", appsv1.OnDeleteStatefulSetStrategyType)) - - By("Verify pods are NOT updated after 10 sec") - Consistently(func() bool { - podsNow := getComponentPods(ctx, namespace) - for name, pod := range podsNow { - if strings.HasPrefix(name, stsName+"-") { - switch componentType { - case consts.SchedulerType: - if pod.Spec.Containers[0].Image == *ytsaurus.Spec.Schedulers.Image { - return false // Pod was updated, which shouldn't happen - } - case consts.MasterType: - if pod.Spec.Containers[0].Image == ytsaurus.Spec.CoreImage { - return false // Pod was updated, which shouldn't happen - } - } - } - } - return true // All pods still on old image - }, 10*time.Second).Should(BeTrue()) + sts := appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: stsName}} + EventuallyObject(ctx, &sts, reactionTimeout).Should(HaveField("Spec.UpdateStrategy.Type", appsv1.OnDeleteStatefulSetStrategyType)) + + // By("Verify StatefulSet has a new revision but pods stay on the old one") + // EventuallyObject(ctx, &sts, reactionTimeout).Should(WithTransform( + // func(current *appsv1.StatefulSet) bool { + // return current.Status.CurrentRevision != "" && + // current.Status.UpdateRevision != "" && + // current.Status.UpdateRevision != current.Status.CurrentRevision + // }, + // BeTrue(), + // )) + + // oldRev := sts.Status.CurrentRevision + // Consistently(func() bool { + // podsNow := getComponentPods(ctx, namespace) + // for name, pod := range podsNow { + // if strings.HasPrefix(name, stsName+"-") { + // if pod.Labels[appsv1.StatefulSetRevisionLabel] != oldRev { + // return false + // } + // } + // } + // return true + // }, 10*time.Second).Should(BeTrue()) By("Manually delete component pods") - for name := range podsBefore { + for name := range podsBeforeUpdate { if strings.HasPrefix(name, stsName+"-") { var pod corev1.Pod err := k8sClient.Get(ctx, types.NamespacedName{ @@ -2046,21 +2066,16 @@ exec "$@"` EventuallyYtsaurus(ctx, ytsaurus, upgradeTimeout).Should(HaveClusterStateRunning()) podsAfter := getComponentPods(ctx, namespace) + changedPods := getChangedPods(podsBeforeUpdate, podsAfter) + expectedUpdated := getExpectedUpdatedPods(podsBeforeUpdate, stsName) - for name, pod := range podsAfter { - if strings.HasPrefix(name, stsName+"-") { - switch componentType { - case consts.SchedulerType: - Expect(pod.Spec.Containers[0].Image).To(Equal(*ytsaurus.Spec.Schedulers.Image)) - case consts.MasterType: - Expect(pod.Spec.Containers[0].Image).To(Equal(ytsaurus.Spec.CoreImage)) - } - } - } + Expect(changedPods.Created).To(BeEmpty(), "created") + Expect(changedPods.Deleted).To(BeEmpty(), "deleted") + Expect(changedPods.Updated).To(ConsistOf(expectedUpdated), "updated") }) }, - Entry("update scheduler", Label("sch"), consts.SchedulerType, consts.GetStatefulSetPrefix(consts.SchedulerType)), - Entry("update master", Label("ms"), consts.MasterType, consts.GetStatefulSetPrefix(consts.MasterType)), + Entry("update scheduler", Label(consts.GetStatefulSetPrefix(consts.SchedulerType)), consts.SchedulerType, consts.GetStatefulSetPrefix(consts.SchedulerType)), + Entry("update master", Label(consts.GetStatefulSetPrefix(consts.MasterType)), consts.MasterType, consts.GetStatefulSetPrefix(consts.MasterType)), ) }) })