diff --git a/api/v1alpha1/databaseclusterbackup_types.go b/api/v1alpha1/databaseclusterbackup_types.go index 9c6f1d1e..873ab651 100644 --- a/api/v1alpha1/databaseclusterbackup_types.go +++ b/api/v1alpha1/databaseclusterbackup_types.go @@ -32,9 +32,10 @@ const ( // BackupState is used to represent the backup's state. type BackupState string +// Known Backup states. const ( - BackupNew BackupState = "" //nolint:revive - BackupStarting BackupState = "Starting" //nolint:revive + BackupNew BackupState = "" + BackupStarting BackupState = "Starting" BackupRunning BackupState = "Running" BackupFailed BackupState = "Failed" BackupSucceeded BackupState = "Succeeded" diff --git a/api/v1alpha1/databaseclusterrestore_types.go b/api/v1alpha1/databaseclusterrestore_types.go index 96f49c35..7afbab19 100644 --- a/api/v1alpha1/databaseclusterrestore_types.go +++ b/api/v1alpha1/databaseclusterrestore_types.go @@ -29,9 +29,10 @@ import ( // RestoreState represents state of restoration. type RestoreState string +// Known Restore states. const ( - RestoreNew RestoreState = "" //nolint:revive - RestoreStarting RestoreState = "Starting" //nolint:revive + RestoreNew RestoreState = "" + RestoreStarting RestoreState = "Starting" RestoreRunning RestoreState = "Restoring" RestoreFailed RestoreState = "Failed" RestoreSucceeded RestoreState = "Succeeded" diff --git a/cmd/main.go b/cmd/main.go index dd88e5f7..c51cd892 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -27,6 +27,9 @@ import ( "strings" vmv1beta1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1" + pgv2 "github.com/percona/percona-postgresql-operator/pkg/apis/pgv2.percona.com/v2" + psmdbv1 "github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1" + pxcv1 "github.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -98,6 +101,10 @@ func init() { utilruntime.Must(everestv1alpha1.AddToScheme(scheme)) utilruntime.Must(vmv1beta1.AddToScheme(scheme)) + + utilruntime.Must(pgv2.SchemeBuilder.AddToScheme(scheme)) + utilruntime.Must(psmdbv1.SchemeBuilder.AddToScheme(scheme)) + utilruntime.Must(pxcv1.SchemeBuilder.AddToScheme(scheme)) // +kubebuilder:scaffold:scheme } @@ -225,36 +232,42 @@ func main() { os.Exit(1) } } - podRef := corev1.ObjectReference{Name: cfg.PodName, Namespace: cfg.SystemNamespace} // Initialise the controllers. - if err = (&controllers.DatabaseClusterReconciler{ + clusterReconciler := &controllers.DatabaseClusterReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { + Cache: mgr.GetCache(), + } + if err := clusterReconciler.SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "DatabaseCluster") os.Exit(1) } - if err = (&controllers.DatabaseEngineReconciler{ + restoreReconciler := &controllers.DatabaseClusterRestoreReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr, podRef, dbNamespaces); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "DatabaseEngine") - os.Exit(1) + Cache: mgr.GetCache(), } - if err = (&controllers.DatabaseClusterRestoreReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { + if err := restoreReconciler.SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "DatabaseClusterRestore") os.Exit(1) } - if err = (&controllers.DatabaseClusterBackupReconciler{ + backupReconciler := &controllers.DatabaseClusterBackupReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { + Cache: mgr.GetCache(), + } + if err := backupReconciler.SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "DatabaseClusterBackup") os.Exit(1) } + if err = (&controllers.DatabaseEngineReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Controllers: []controllers.DatabaseController{clusterReconciler, restoreReconciler, backupReconciler}, + }).SetupWithManager(mgr, dbNamespaces); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "DatabaseEngine") + os.Exit(1) + } if err = (&controllers.BackupStorageReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), diff --git a/internal/controller/backupstorage_controller.go b/internal/controller/backupstorage_controller.go index 960b884e..70a60a69 100644 --- a/internal/controller/backupstorage_controller.go +++ b/internal/controller/backupstorage_controller.go @@ -98,6 +98,7 @@ func (r *BackupStorageReconciler) Reconcile(ctx context.Context, req ctrl.Reques // SetupWithManager sets up the controller with the Manager. func (r *BackupStorageReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). + Named("BackupStorage"). For(&everestv1alpha1.BackupStorage{}). Owns(&corev1.Secret{}). Watches( diff --git a/internal/controller/controller_watcher_registry.go b/internal/controller/controller_watcher_registry.go new file mode 100644 index 00000000..606f855b --- /dev/null +++ b/internal/controller/controller_watcher_registry.go @@ -0,0 +1,59 @@ +// everest-operator +// Copyright (C) 2022 Percona LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package controllers contains a set of controllers for everest +package controllers + +import ( + "sync" + + "github.com/go-logr/logr" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +// controllerWatcherRegistry is a wrapper arond controller.Controller that provides a way to +// store and keep track of the sources that have been added to the controller. +type controllerWatcherRegistry struct { + controller.Controller + store sync.Map + log logr.Logger +} + +func newControllerWatcherRegistry(log logr.Logger, c controller.Controller) *controllerWatcherRegistry { + return &controllerWatcherRegistry{ + Controller: c, + log: log, + } +} + +// addWatchers adds the provided sources to the controller's watch, and stores the name of the sources in a map to avoid adding them again. +func (c *controllerWatcherRegistry) addWatchers( + name string, + sources ...source.Source, +) error { + _, ok := c.store.Load(name) + if ok { + return nil // watcher group already exists with this name, so skip. + } + for _, src := range sources { + if err := c.Controller.Watch(src); err != nil { + return err + } + } + c.log.Info("Added watchers", "name", name) + c.store.Store(name, struct{}{}) + return nil +} diff --git a/internal/controller/databasecluster_controller.go b/internal/controller/databasecluster_controller.go index 9cbc0628..6da5a082 100644 --- a/internal/controller/databasecluster_controller.go +++ b/internal/controller/databasecluster_controller.go @@ -33,19 +33,19 @@ import ( corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" everestv1alpha1 "github.com/percona/everest-operator/api/v1alpha1" "github.com/percona/everest-operator/internal/controller/common" @@ -83,7 +83,10 @@ var everestFinalizers = []string{ // DatabaseClusterReconciler reconciles a DatabaseCluster object. type DatabaseClusterReconciler struct { client.Client + Cache cache.Cache Scheme *runtime.Scheme + + controller *controllerWatcherRegistry } // dbProvider provides an abstraction for managing the reconciliation @@ -418,85 +421,28 @@ func (r *DatabaseClusterReconciler) reconcileLabels( return nil } -func (r *DatabaseClusterReconciler) addPXCKnownTypes(scheme *runtime.Scheme) error { - pxcSchemeGroupVersion := schema.GroupVersion{Group: common.PXCAPIGroup, Version: "v1"} - scheme.AddKnownTypes(pxcSchemeGroupVersion, - &pxcv1.PerconaXtraDBCluster{}, &pxcv1.PerconaXtraDBClusterList{}, - &pxcv1.PerconaXtraDBClusterRestore{}, &pxcv1.PerconaXtraDBClusterRestoreList{}) - - metav1.AddToGroupVersion(scheme, pxcSchemeGroupVersion) - return nil -} - -func (r *DatabaseClusterReconciler) addPSMDBKnownTypes(scheme *runtime.Scheme) error { - psmdbSchemeGroupVersion := schema.GroupVersion{Group: common.PSMDBAPIGroup, Version: "v1"} - scheme.AddKnownTypes(psmdbSchemeGroupVersion, - &psmdbv1.PerconaServerMongoDB{}, &psmdbv1.PerconaServerMongoDBList{}) - - metav1.AddToGroupVersion(scheme, psmdbSchemeGroupVersion) - return nil -} - -func (r *DatabaseClusterReconciler) addPGKnownTypes(scheme *runtime.Scheme) error { - pgSchemeGroupVersion := schema.GroupVersion{Group: common.PGAPIGroup, Version: "v2"} - scheme.AddKnownTypes(pgSchemeGroupVersion, - &pgv2.PerconaPGCluster{}, &pgv2.PerconaPGClusterList{}) - - metav1.AddToGroupVersion(scheme, pgSchemeGroupVersion) - return nil -} - -func (r *DatabaseClusterReconciler) addPSMDBToScheme(scheme *runtime.Scheme) error { - builder := runtime.NewSchemeBuilder(r.addPSMDBKnownTypes) - return builder.AddToScheme(scheme) -} - -func (r *DatabaseClusterReconciler) addPXCToScheme(scheme *runtime.Scheme) error { - builder := runtime.NewSchemeBuilder(r.addPXCKnownTypes) - return builder.AddToScheme(scheme) -} - -func (r *DatabaseClusterReconciler) addPGToScheme(scheme *runtime.Scheme) error { - builder := runtime.NewSchemeBuilder(r.addPGKnownTypes) - return builder.AddToScheme(scheme) -} - // SetupWithManager sets up the controller with the Manager. func (r *DatabaseClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { if err := r.initIndexers(context.Background(), mgr); err != nil { return err } - unstructuredResource := &unstructured.Unstructured{} - unstructuredResource.SetGroupVersionKind(schema.GroupVersionKind{ - Group: "apiextensions.k8s.io", - Kind: "CustomResourceDefinition", - Version: "v1", - }) - controller := ctrl.NewControllerManagedBy(mgr). + ctrlBuilder := ctrl.NewControllerManagedBy(mgr). + Named("DatabaseCluster"). For(&everestv1alpha1.DatabaseCluster{}) - err := r.Get(context.Background(), types.NamespacedName{Name: pxcCRDName}, unstructuredResource) - if err == nil { - if err := r.addPXCToScheme(r.Scheme); err == nil { - controller.Owns(&pxcv1.PerconaXtraDBCluster{}) - } - } - err = r.Get(context.Background(), types.NamespacedName{Name: psmdbCRDName}, unstructuredResource) - if err == nil { - if err := r.addPSMDBToScheme(r.Scheme); err == nil { - controller.Owns(&psmdbv1.PerconaServerMongoDB{}) - } - } - err = r.Get(context.Background(), types.NamespacedName{Name: pgCRDName}, unstructuredResource) - if err == nil { - if err := r.addPGToScheme(r.Scheme); err == nil { - controller.Owns(&pgv2.PerconaPGCluster{}) - } - } - r.initWatchers(controller) - controller.WithEventFilter(common.DefaultNamespaceFilter) - return controller.Complete(r) + r.initWatchers(ctrlBuilder) + ctrlBuilder.WithEventFilter(common.DefaultNamespaceFilter) + + // Normally we would call `Complete()`, however, with `Build()`, we get a handle to the underlying controller, + // so that we can dynamically add watchers from the DatabaseEngine reconciler. + ctrl, err := ctrlBuilder.Build(r) + if err != nil { + return err + } + log := mgr.GetLogger().WithName("DynamicWatcher").WithValues("controller", "DatabaseCluster") + r.controller = newControllerWatcherRegistry(log, ctrl) + return nil } func (r *DatabaseClusterReconciler) initIndexers(ctx context.Context, mgr ctrl.Manager) error { @@ -769,53 +715,6 @@ func (r *DatabaseClusterReconciler) initWatchers(controller *builder.Builder) { }), builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), ) - - if r.Scheme.Recognizes(common.PXCRGVK) { - controller.Watches( - // need to watch pxc-restore to be sure the db is reconciled on every pxc-restore status update. - // watching the dbr is not enough since the operator merges the statuses but we need to pause the db exactly when - // the pxc-restore got to the pxcv1.RestoreStopCluster status - &pxcv1.PerconaXtraDBClusterRestore{}, - handler.EnqueueRequestsFromMapFunc(func(_ context.Context, obj client.Object) []reconcile.Request { - pxcRestore, ok := obj.(*pxcv1.PerconaXtraDBClusterRestore) - if !ok { - return []reconcile.Request{} - } - return []reconcile.Request{ - { - NamespacedName: types.NamespacedName{ - // db name to reconcile is the same as the pxc cluster name - Name: pxcRestore.Spec.PXCCluster, - Namespace: obj.GetNamespace(), - }, - }, - } - }), - builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), - ) - } - - if r.Scheme.Recognizes(common.PSMDBGVK) { - controller.Watches( - &psmdbv1.PerconaServerMongoDB{}, - &handler.EnqueueRequestForObject{}, - builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), - ) - } - if r.Scheme.Recognizes(common.PGGVK) { - controller.Watches( - &pgv2.PerconaPGCluster{}, - &handler.EnqueueRequestForObject{}, - builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), - ) - } - if r.Scheme.Recognizes(common.PXCGVK) { - controller.Watches( - &pxcv1.PerconaXtraDBCluster{}, - &handler.EnqueueRequestForObject{}, - builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), - ) - } } func (r *DatabaseClusterReconciler) databaseClustersInObjectNamespace(ctx context.Context, obj client.Object) []reconcile.Request { @@ -953,3 +852,74 @@ func (r *DatabaseClusterReconciler) ensureFinalizers( } return nil } + +// ReconcileWatchers reconciles the watchers for the DatabaseCluster controller. +func (r *DatabaseClusterReconciler) ReconcileWatchers(ctx context.Context) error { + dbEngines := &everestv1alpha1.DatabaseEngineList{} + if err := r.List(ctx, dbEngines); err != nil { + return err + } + + log := log.FromContext(ctx) + addWatcher := func(dbEngineType everestv1alpha1.EngineType, obj client.Object) error { + sources := []source.Source{ + source.TypedKind(r.Cache, obj, &handler.EnqueueRequestForObject{}), + } + + // special case for PXC - we need to watch pxc-restore to be sure the db is reconciled on every pxc-restore status update. + // watching the dbr is not enough since the operator merges the statuses but we need to pause the db exactly when + // the pxc-restore got to the pxcv1.RestoreStopCluster status + if dbEngineType == everestv1alpha1.DatabaseEnginePXC { + sources = append(sources, newPXCRestoreWatchSource(r.Cache)) + } + + if err := r.controller.addWatchers(string(dbEngineType), sources...); err != nil { + return err + } + return nil + } + + for _, dbEngine := range dbEngines.Items { + if dbEngine.Status.State != everestv1alpha1.DBEngineStateInstalled { + continue + } + switch t := dbEngine.Spec.Type; t { + case everestv1alpha1.DatabaseEnginePXC: + if err := addWatcher(t, &pxcv1.PerconaXtraDBCluster{}); err != nil { + return err + } + case everestv1alpha1.DatabaseEnginePostgresql: + if err := addWatcher(t, &pgv2.PerconaPGCluster{}); err != nil { + return err + } + case everestv1alpha1.DatabaseEnginePSMDB: + if err := addWatcher(t, &psmdbv1.PerconaServerMongoDB{}); err != nil { + return err + } + default: + log.Info("Unknown database engine type", "type", dbEngine.Spec.Type) + continue + } + } + return nil +} + +func newPXCRestoreWatchSource(cache cache.Cache) source.Source { //nolint:ireturn + return source.TypedKind[client.Object](cache, &pxcv1.PerconaXtraDBClusterRestore{}, + handler.EnqueueRequestsFromMapFunc(func(_ context.Context, obj client.Object) []reconcile.Request { + pxcRestore, ok := obj.(*pxcv1.PerconaXtraDBClusterRestore) + if !ok { + return []reconcile.Request{} + } + return []reconcile.Request{ + { + NamespacedName: types.NamespacedName{ + Name: pxcRestore.Spec.PXCCluster, + Namespace: obj.GetNamespace(), + }, + }, + } + }), + predicate.ResourceVersionChangedPredicate{}, + ) +} diff --git a/internal/controller/databaseclusterbackup_controller.go b/internal/controller/databaseclusterbackup_controller.go index 30d0519e..a53733a4 100644 --- a/internal/controller/databaseclusterbackup_controller.go +++ b/internal/controller/databaseclusterbackup_controller.go @@ -36,18 +36,18 @@ import ( corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/workqueue" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" everestv1alpha1 "github.com/percona/everest-operator/api/v1alpha1" "github.com/percona/everest-operator/internal/controller/common" @@ -85,6 +85,9 @@ var ErrBackupStorageUndefined = errors.New("backup storage is not defined in the type DatabaseClusterBackupReconciler struct { client.Client Scheme *runtime.Scheme + Cache cache.Cache + + controller *controllerWatcherRegistry } //+kubebuilder:rbac:groups=everest.percona.com,resources=databaseclusterbackups,verbs=get;list;watch;create;update;patch;delete @@ -218,13 +221,6 @@ func (r *DatabaseClusterBackupReconciler) reconcileMeta( // SetupWithManager sets up the controller with the Manager. func (r *DatabaseClusterBackupReconciler) SetupWithManager(mgr ctrl.Manager) error { - unstructuredResource := &unstructured.Unstructured{} - unstructuredResource.SetGroupVersionKind(schema.GroupVersionKind{ - Group: "apiextensions.k8s.io", - Kind: "CustomResourceDefinition", - Version: "v1", - }) - // Index the dbClusterName field in DatabaseClusterBackup. err := mgr.GetFieldIndexer().IndexField( context.Background(), &everestv1alpha1.DatabaseClusterBackup{}, dbClusterBackupDBClusterNameField, @@ -242,46 +238,24 @@ func (r *DatabaseClusterBackupReconciler) SetupWithManager(mgr ctrl.Manager) err return err } - controller := ctrl.NewControllerManagedBy(mgr). + ctrlBuilder := ctrl.NewControllerManagedBy(mgr). + Named("DatabaseClusterBackup"). For(&everestv1alpha1.DatabaseClusterBackup{}) - controller.Watches( + ctrlBuilder.Watches( &corev1.Namespace{}, common.EnqueueObjectsInNamespace(r.Client, &everestv1alpha1.DatabaseClusterBackupList{}), ) - ctx := context.Background() - - err = r.Get(ctx, types.NamespacedName{Name: pxcBackupCRDName}, unstructuredResource) - if err == nil { - if err := r.addPXCToScheme(r.Scheme); err == nil { - controller.Watches( - &pxcv1.PerconaXtraDBClusterBackup{}, - r.watchHandler(r.tryCreatePXC), - ) - } - } + ctrlBuilder.WithEventFilter(common.DefaultNamespaceFilter) - err = r.Get(ctx, types.NamespacedName{Name: psmdbBackupCRDName}, unstructuredResource) - if err == nil { - if err := r.addPSMDBToScheme(r.Scheme); err == nil { - controller.Watches( - &psmdbv1.PerconaServerMongoDBBackup{}, - r.watchHandler(r.tryCreatePSMDB), - ) - } - } - - err = r.Get(ctx, types.NamespacedName{Name: pgBackupCRDName}, unstructuredResource) - if err == nil { - if err := r.addPGToScheme(r.Scheme); err == nil { - controller.Watches( - &pgv2.PerconaPGBackup{}, - r.watchHandler(r.tryCreatePG), - ) - } + // Normally we would call `Complete()`, however, with `Build()`, we get a handle to the underlying controller, + // so that we can dynamically add watchers from the DatabaseEngine reconciler. + ctrl, err := ctrlBuilder.Build(r) + if err != nil { + return err } - - controller.WithEventFilter(common.DefaultNamespaceFilter) - return controller.Complete(r) + log := mgr.GetLogger().WithName("DynamicWatcher").WithValues("controller", "DatabaseClusterBackup") + r.controller = newControllerWatcherRegistry(log, ctrl) + return nil } func (r *DatabaseClusterBackupReconciler) watchHandler(creationFunc func(ctx context.Context, obj client.Object) error) handler.Funcs { @@ -533,48 +507,6 @@ func (r *DatabaseClusterBackupReconciler) tryCreatePSMDB(ctx context.Context, ob return r.Create(ctx, backup) } -func (r *DatabaseClusterBackupReconciler) addPXCToScheme(scheme *runtime.Scheme) error { - builder := runtime.NewSchemeBuilder(r.addPXCKnownTypes) - return builder.AddToScheme(scheme) -} - -func (r *DatabaseClusterBackupReconciler) addPXCKnownTypes(scheme *runtime.Scheme) error { - pxcSchemeGroupVersion := schema.GroupVersion{Group: "pxc.percona.com", Version: "v1"} - scheme.AddKnownTypes(pxcSchemeGroupVersion, - &pxcv1.PerconaXtraDBClusterBackup{}, &pxcv1.PerconaXtraDBClusterBackupList{}) - - metav1.AddToGroupVersion(scheme, pxcSchemeGroupVersion) - return nil -} - -func (r *DatabaseClusterBackupReconciler) addPSMDBToScheme(scheme *runtime.Scheme) error { - builder := runtime.NewSchemeBuilder(r.addPSMDBKnownTypes) - return builder.AddToScheme(scheme) -} - -func (r *DatabaseClusterBackupReconciler) addPSMDBKnownTypes(scheme *runtime.Scheme) error { - psmdbSchemeGroupVersion := schema.GroupVersion{Group: "psmdb.percona.com", Version: "v1"} - scheme.AddKnownTypes(psmdbSchemeGroupVersion, - &psmdbv1.PerconaServerMongoDBBackup{}, &psmdbv1.PerconaServerMongoDBBackupList{}) - - metav1.AddToGroupVersion(scheme, psmdbSchemeGroupVersion) - return nil -} - -func (r *DatabaseClusterBackupReconciler) addPGToScheme(scheme *runtime.Scheme) error { - builder := runtime.NewSchemeBuilder(r.addPGKnownTypes) - return builder.AddToScheme(scheme) -} - -func (r *DatabaseClusterBackupReconciler) addPGKnownTypes(scheme *runtime.Scheme) error { - pgSchemeGroupVersion := schema.GroupVersion{Group: "pgv2.percona.com", Version: "v2"} - scheme.AddKnownTypes(pgSchemeGroupVersion, - &pgv2.PerconaPGBackup{}, &pgv2.PerconaPGBackupList{}) - - metav1.AddToGroupVersion(scheme, pgSchemeGroupVersion) - return nil -} - // Reconcile PXC. // Returns: (requeue(bool), error). func (r *DatabaseClusterBackupReconciler) reconcilePXC( @@ -946,3 +878,43 @@ func (r *DatabaseClusterBackupReconciler) handleStorageProtectionFinalizer( } return nil } + +// ReconcileWatchers reconciles the watchers for the DatabaseClusterBackup controller. +func (r *DatabaseClusterBackupReconciler) ReconcileWatchers(ctx context.Context) error { + dbEngines := &everestv1alpha1.DatabaseEngineList{} + if err := r.List(ctx, dbEngines); err != nil { + return err + } + + log := log.FromContext(ctx) + addWatcher := func(dbEngineType everestv1alpha1.EngineType, obj client.Object, f func(context.Context, client.Object) error) error { + if err := r.controller.addWatchers(string(dbEngineType), source.Kind(r.Cache, obj, r.watchHandler(f))); err != nil { + return err + } + return nil + } + + for _, dbEngine := range dbEngines.Items { + if dbEngine.Status.State != everestv1alpha1.DBEngineStateInstalled { + continue + } + switch t := dbEngine.Spec.Type; t { + case everestv1alpha1.DatabaseEnginePXC: + if err := addWatcher(t, &pxcv1.PerconaXtraDBClusterBackup{}, r.tryCreatePXC); err != nil { + return err + } + case everestv1alpha1.DatabaseEnginePostgresql: + if err := addWatcher(t, &pgv2.PerconaPGBackup{}, r.tryCreatePG); err != nil { + return err + } + case everestv1alpha1.DatabaseEnginePSMDB: + if err := addWatcher(t, &psmdbv1.PerconaServerMongoDBBackup{}, r.tryCreatePSMDB); err != nil { + return err + } + default: + log.Info("Unknown database engine type", "type", dbEngine.Spec.Type) + continue + } + } + return nil +} diff --git a/internal/controller/databaseclusterrestore_controller.go b/internal/controller/databaseclusterrestore_controller.go index a4690f65..5423075c 100644 --- a/internal/controller/databaseclusterrestore_controller.go +++ b/internal/controller/databaseclusterrestore_controller.go @@ -29,15 +29,16 @@ import ( pxcv1 "github.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" everestv1alpha1 "github.com/percona/everest-operator/api/v1alpha1" "github.com/percona/everest-operator/internal/controller/common" @@ -66,6 +67,9 @@ var ( type DatabaseClusterRestoreReconciler struct { client.Client Scheme *runtime.Scheme + Cache cache.Cache + + controller *controllerWatcherRegistry } //+kubebuilder:rbac:groups=everest.percona.com,resources=databaseclusterrestores,verbs=get;list;watch;create;update;patch;delete @@ -509,81 +513,16 @@ func (r *DatabaseClusterRestoreReconciler) restorePG(ctx context.Context, restor return r.Status().Update(ctx, restore) } -func (r *DatabaseClusterRestoreReconciler) addPXCKnownTypes(scheme *runtime.Scheme) error { - pxcSchemeGroupVersion := schema.GroupVersion{Group: "pxc.percona.com", Version: "v1"} - scheme.AddKnownTypes(pxcSchemeGroupVersion, - &pxcv1.PerconaXtraDBClusterRestore{}, &pxcv1.PerconaXtraDBClusterRestoreList{}) - - metav1.AddToGroupVersion(scheme, pxcSchemeGroupVersion) - return nil -} - -func (r *DatabaseClusterRestoreReconciler) addPSMDBKnownTypes(scheme *runtime.Scheme) error { - psmdbSchemeGroupVersion := schema.GroupVersion{Group: "psmdb.percona.com", Version: "v1"} - scheme.AddKnownTypes(psmdbSchemeGroupVersion, - &psmdbv1.PerconaServerMongoDBRestore{}, &psmdbv1.PerconaServerMongoDBRestoreList{}) - - metav1.AddToGroupVersion(scheme, psmdbSchemeGroupVersion) - return nil -} - -func (r *DatabaseClusterRestoreReconciler) addPGKnownTypes(scheme *runtime.Scheme) error { - pgSchemeGroupVersion := schema.GroupVersion{Group: "pgv2.percona.com", Version: "v2"} - scheme.AddKnownTypes(pgSchemeGroupVersion, - &pgv2.PerconaPGRestore{}, &pgv2.PerconaPGRestoreList{}) - - metav1.AddToGroupVersion(scheme, pgSchemeGroupVersion) - return nil -} - -func (r *DatabaseClusterRestoreReconciler) addPXCToScheme(scheme *runtime.Scheme) error { - builder := runtime.NewSchemeBuilder(r.addPXCKnownTypes) - return builder.AddToScheme(scheme) -} - -func (r *DatabaseClusterRestoreReconciler) addPSMDBToScheme(scheme *runtime.Scheme) error { - builder := runtime.NewSchemeBuilder(r.addPSMDBKnownTypes) - return builder.AddToScheme(scheme) -} - -func (r *DatabaseClusterRestoreReconciler) addPGToScheme(scheme *runtime.Scheme) error { - builder := runtime.NewSchemeBuilder(r.addPGKnownTypes) - return builder.AddToScheme(scheme) -} - // SetupWithManager sets up the controller with the Manager. func (r *DatabaseClusterRestoreReconciler) SetupWithManager(mgr ctrl.Manager) error { - unstructuredResource := &unstructured.Unstructured{} - unstructuredResource.SetGroupVersionKind(schema.GroupVersionKind{ - Group: "apiextensions.k8s.io", - Kind: "CustomResourceDefinition", - Version: "v1", - }) - controller := ctrl.NewControllerManagedBy(mgr). + ctrlBuilder := ctrl.NewControllerManagedBy(mgr). + Named("DatabaseClusterRestore"). For(&everestv1alpha1.DatabaseClusterRestore{}). Watches( &corev1.Namespace{}, common.EnqueueObjectsInNamespace(r.Client, &everestv1alpha1.DatabaseClusterRestoreList{}), ) - err := r.Get(context.Background(), types.NamespacedName{Name: pxcRestoreCRDName}, unstructuredResource) - if err == nil { - if err := r.addPXCToScheme(r.Scheme); err == nil { - controller.Owns(&pxcv1.PerconaXtraDBClusterRestore{}) - } - } - err = r.Get(context.Background(), types.NamespacedName{Name: psmdbRestoreCRDName}, unstructuredResource) - if err == nil { - if err := r.addPSMDBToScheme(r.Scheme); err == nil { - controller.Owns(&psmdbv1.PerconaServerMongoDBRestore{}) - } - } - err = r.Get(context.Background(), types.NamespacedName{Name: pgRestoreCRDName}, unstructuredResource) - if err == nil { - if err := r.addPGToScheme(r.Scheme); err == nil { - controller.Owns(&pgv2.PerconaPGRestore{}) - } - } - err = mgr.GetFieldIndexer().IndexField( + if err := mgr.GetFieldIndexer().IndexField( context.Background(), &everestv1alpha1.DatabaseClusterRestore{}, dbClusterRestoreDBClusterNameField, @@ -591,13 +530,20 @@ func (r *DatabaseClusterRestoreReconciler) SetupWithManager(mgr ctrl.Manager) er res := rawObj.(*everestv1alpha1.DatabaseClusterRestore) //nolint:forcetypeassert return []string{res.Spec.DBClusterName} }, - ) - if err != nil { + ); err != nil { return err } + ctrlBuilder.WithEventFilter(common.DefaultNamespaceFilter) - controller.WithEventFilter(common.DefaultNamespaceFilter) - return controller.Complete(r) + // Normally we would call `Complete()`, however, with `Build()`, we get a handle to the underlying controller, + // so that we can dynamically add watchers from the DatabaseEngine reconciler. + ctrl, err := ctrlBuilder.Build(r) + if err != nil { + return err + } + log := mgr.GetLogger().WithName("DynamicWatcher").WithValues("controller", "DatabaseClusterRestore") + r.controller = newControllerWatcherRegistry(log, ctrl) + return nil } func parsePrefixFromDestination(url string) string { @@ -712,3 +658,44 @@ func validatePitrRestoreSpec(dataSource everestv1alpha1.DataSource) error { } return nil } + +// ReconcileWatchers reconciles the watchers for the DatabaseClusterRestore controller. +func (r *DatabaseClusterRestoreReconciler) ReconcileWatchers(ctx context.Context) error { + dbEngines := &everestv1alpha1.DatabaseEngineList{} + if err := r.List(ctx, dbEngines); err != nil { + return err + } + + log := log.FromContext(ctx) + addWatcher := func(dbEngineType everestv1alpha1.EngineType, obj client.Object) error { + src := source.TypedKind(r.Cache, obj, handler.EnqueueRequestForOwner(r.Scheme, r.RESTMapper(), &everestv1alpha1.DatabaseCluster{})) + if err := r.controller.addWatchers(string(dbEngineType), src); err != nil { + return err + } + return nil + } + + for _, dbEngine := range dbEngines.Items { + if dbEngine.Status.State != everestv1alpha1.DBEngineStateInstalled { + continue + } + switch t := dbEngine.Spec.Type; t { + case everestv1alpha1.DatabaseEnginePXC: + if err := addWatcher(t, &pxcv1.PerconaXtraDBClusterRestore{}); err != nil { + return err + } + case everestv1alpha1.DatabaseEnginePostgresql: + if err := addWatcher(t, &pgv2.PerconaPGRestore{}); err != nil { + return err + } + case everestv1alpha1.DatabaseEnginePSMDB: + if err := addWatcher(t, &psmdbv1.PerconaServerMongoDBRestore{}); err != nil { + return err + } + default: + log.Info("Unknown database engine type", "type", dbEngine.Spec.Type) + continue + } + } + return nil +} diff --git a/internal/controller/databaseengine_controller.go b/internal/controller/databaseengine_controller.go index 9d135e7a..ff1a8b83 100644 --- a/internal/controller/databaseengine_controller.go +++ b/internal/controller/databaseengine_controller.go @@ -25,9 +25,6 @@ import ( "time" opfwv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1" - pgv2 "github.com/percona/percona-postgresql-operator/pkg/apis/pgv2.percona.com/v2" - psmdbv1 "github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1" - pxcv1 "github.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1" "golang.org/x/mod/semver" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -65,18 +62,19 @@ var operatorEngine = map[string]everestv1alpha1.EngineType{ common.PGDeploymentName: everestv1alpha1.DatabaseEnginePostgresql, } -var operatorEngineTypeToCRDGroup = map[everestv1alpha1.EngineType]string{ - everestv1alpha1.DatabaseEnginePXC: pxcv1.SchemeGroupVersion.Group, - everestv1alpha1.DatabaseEnginePSMDB: psmdbv1.SchemeGroupVersion.Group, - everestv1alpha1.DatabaseEnginePostgresql: pgv2.GroupVersion.Group, -} - // DatabaseEngineReconciler reconciles a DatabaseEngine object. type DatabaseEngineReconciler struct { client.Client Scheme *runtime.Scheme versionService *version.Service - podSelf corev1.ObjectReference // reference to self pod. + + Controllers []DatabaseController +} + +// DatabaseController provides an abstraction for the DatabaseEngine controller +// to orchestrate operations across various database controllers (like DBCluster, DBBackup, etc.). +type DatabaseController interface { + ReconcileWatchers(ctx context.Context) error } //+kubebuilder:rbac:groups=everest.percona.com,resources=databaseengines,verbs=get;list;watch;create;update;patch;delete @@ -189,10 +187,8 @@ func (r *DatabaseEngineReconciler) Reconcile(ctx context.Context, req ctrl.Reque return ctrl.Result{}, err } - if requeue, err := r.restartIfNeeded(ctx); err != nil { - return ctrl.Result{}, err - } else if requeue { - return ctrl.Result{Requeue: true}, nil + if err := r.reconcileWatchers(ctx); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to reconcile watchers: %w", err) } return ctrl.Result{ @@ -200,42 +196,13 @@ func (r *DatabaseEngineReconciler) Reconcile(ctx context.Context, req ctrl.Reque }, nil } -// restartIfNeeded checks if the operator pod needs to be restarted. -// It does so by checking if there are any running DBEngines whose CRDs are not registered with the operator. -// Returns: [requeue(bool), error]. -func (r *DatabaseEngineReconciler) restartIfNeeded(ctx context.Context) (bool, error) { - if r.podSelf.Name == "" || r.podSelf.Namespace == "" { - return false, nil - } - dbEngines := &everestv1alpha1.DatabaseEngineList{} - if err := r.List(ctx, dbEngines); err != nil { - return false, fmt.Errorf("failed to list DatabaseEngines: %w", err) - } - for _, dbEngine := range dbEngines.Items { - // Wait until all DB engines are either installed/not installed. - // This way we can avoid redundant restarts. - if dbEngine.Status.State == "" || - dbEngine.Status.State == everestv1alpha1.DBEngineStateInstalling || - dbEngine.Status.State == everestv1alpha1.DBEngineStateUpgrading { - return true, nil - } - if dbEngine.Status.State == everestv1alpha1.DBEngineStateNotInstalled { - continue - } - group, found := operatorEngineTypeToCRDGroup[dbEngine.Spec.Type] - if !found { - return false, fmt.Errorf("unknown engine type '%s'", dbEngine.Spec.Type) - } - // Ideally we would also like to check if all registered controllers are also watching the CRs, - // but since that's tricky to accomplish, we will only check if the CRDs are registered to the scheme, - // since we typically perform that step along with configuring the watches. - if !r.Scheme.IsGroupRegistered(group) { - return false, r.Delete(ctx, &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{Name: r.podSelf.Name, Namespace: r.podSelf.Namespace}, - }) +func (r *DatabaseEngineReconciler) reconcileWatchers(ctx context.Context) error { + for _, c := range r.Controllers { + if err := c.ReconcileWatchers(ctx); err != nil { + return err } } - return false, nil + return nil } func (r *DatabaseEngineReconciler) reconcileOperatorUpgradeStatus( @@ -500,13 +467,13 @@ func (r *DatabaseEngineReconciler) ensureDBEnginesInNamespaces(ctx context.Conte } // SetupWithManager sets up the controller with the Manager. -func (r *DatabaseEngineReconciler) SetupWithManager(mgr ctrl.Manager, selfPodRef corev1.ObjectReference, namespaces []string) error { - r.podSelf = selfPodRef +func (r *DatabaseEngineReconciler) SetupWithManager(mgr ctrl.Manager, namespaces []string) error { if _, err := r.ensureDBEnginesInNamespaces(context.Background(), namespaces); err != nil { return err } r.versionService = version.NewVersionService() c := ctrl.NewControllerManagedBy(mgr). + Named("DatabaseEngine"). For(&everestv1alpha1.DatabaseEngine{}). Watches(&appsv1.Deployment{}, &handler.EnqueueRequestForObject{}). Watches( diff --git a/internal/controller/monitoringconfig_controller.go b/internal/controller/monitoringconfig_controller.go index 8f906487..78dc3657 100644 --- a/internal/controller/monitoringconfig_controller.go +++ b/internal/controller/monitoringconfig_controller.go @@ -255,6 +255,7 @@ func (r *MonitoringConfigReconciler) genVMAgentSpec(ctx context.Context, monitor func (r *MonitoringConfigReconciler) SetupWithManager(mgr ctrl.Manager, monitoringNamespace string) error { r.monitoringNamespace = monitoringNamespace return ctrl.NewControllerManagedBy(mgr). + Named("MonitoringConfig"). For(&everestv1alpha1.MonitoringConfig{}). Watches(&vmv1beta1.VMAgent{}, r.enqueueMonitoringConfigs()). Watches(&corev1.Namespace{},