diff --git a/README.md b/README.md index d32fd2a..6da15b0 100644 --- a/README.md +++ b/README.md @@ -145,9 +145,11 @@ For each CareInstruction, a dedicated Shoot controller is dynamically created an - Extracts cluster connection details (API server URL, CA certificate) - Creates or updates corresponding Secret resources with OIDC configuration - Generates Greenhouse Cluster resources with appropriate labels -- Optionally configures OIDC authentication on Shoot clusters for Greenhouse access. Also see respective [Greenhouse docs](https://cloudoperators.github.io/greenhouse/docs/user-guides/cluster/oidc_connectivity/) and [Gardener docs](https://gardener.cloud/docs/guides/administer-shoots/oidc-login/#configure-the-shoot-cluster) +- Optionally configures OIDC authentication on Shoot clusters for Greenhouse access. Also see respective [Greenhouse docs](https://cloudoperators.github.io/greenhouse/docs/user-guides/cluster/oidc-login/) and [Gardener docs](https://gardener.cloud/docs/guides/administer-shoots/oidc-login/#configure-the-shoot-cluster) - Optionally configures RBAC on the Shoot cluster for Greenhouse access +> **Auth ConfigMap labeling & watch**: When `authenticationConfigMapName` is set, the shoot controller labels the referenced Greenhouse ConfigMap with `shoot-grafter.cloudoperators.dev/auth-configmap: "true"` and `shoot-grafter.cloudoperators.dev/careinstruction: ` on first interaction (creation or update). The controller also watches these labeled ConfigMaps on the Greenhouse cluster; any change to the auth ConfigMap automatically re-enqueues all matching Shoots so the Garden-side OIDC configuration stays in sync without waiting for the next Shoot event. + ## Custom Resource: CareInstruction A `CareInstruction` defines the configuration for onboarding Shoots from a specific Garden cluster. @@ -208,7 +210,7 @@ spec: | `shootSelector.expression` | string | No | CEL expression for filtering shoots by status or other fields (max 1024 chars). The shoot object is available as `object` | | `propagateLabels` | []string | No | List of label keys to copy from Shoot to Greenhouse Cluster | | `additionalLabels` | map[string]string | No | Additional labels to add to all created Greenhouse Clusters | -| `authenticationConfigMapName` | string | No | Name of ConfigMap in Greenhouse cluster containing AuthenticationConfiguration [(config.yaml with apiserver.config.k8s.io/v1beta1 content)](https://gardener.cloud/docs/guides/administer-shoots/oidc-login/#configure-the-shoot-cluster)| +| `authenticationConfigMapName` | string | No | Name of ConfigMap in Greenhouse cluster containing AuthenticationConfiguration [(config.yaml with apiserver.config.k8s.io/v1beta1 content)](https://gardener.cloud/docs/guides/administer-shoots/oidc-login/#configure-the-shoot-cluster). The ConfigMap is labeled by the shoot controller on first interaction and watched for changes to trigger automatic re-reconciliation of Shoots. | | `enableRBAC` | bool | No | When false, skips automatic RBAC setup on Shoot clusters (default: true‚) | *Note: Either `gardenClusterName` or `gardenClusterKubeConfigSecretName` must be provided (priority: kubeconfig secret > cluster name) diff --git a/api/v1alpha1/careinstruction_types.go b/api/v1alpha1/careinstruction_types.go index 4d4eae4..8033882 100644 --- a/api/v1alpha1/careinstruction_types.go +++ b/api/v1alpha1/careinstruction_types.go @@ -36,7 +36,7 @@ const ( CareInstructionLabel = "shoot-grafter.cloudoperators.dev/careinstruction" // AuthConfigMapLabel is the label used to identify AuthenticationConfiguration ConfigMaps - AuthConfigMapLabel = "shoot-grafter.cloudoperators/authconfigmap" + AuthConfigMapLabel = "shoot-grafter.cloudoperators.dev/auth-configmap" // ShootStatusOnboarded indicates the shoot has been onboarded as a Greenhouse Cluster. ShootStatusOnboarded = "Onboarded" diff --git a/controller/careinstruction/careinstruction_controller.go b/controller/careinstruction/careinstruction_controller.go index e569605..e7e8dad 100644 --- a/controller/careinstruction/careinstruction_controller.go +++ b/controller/careinstruction/careinstruction_controller.go @@ -288,9 +288,11 @@ func (r *CareInstructionReconciler) reconcileManager(ctx context.Context, careIn } // Register the ShootController with the garden manager - // Note: EventRecorder is obtained from the Greenhouse manager to emit events on the Greenhouse cluster + // Note: EventRecorder is obtained from the Greenhouse manager to emit events on the Greenhouse cluster. + // GreenhouseMgr is passed so the ShootController can watch Greenhouse cluster resources (e.g. auth CMs). sc := &shoot.ShootController{ GreenhouseClient: r.Client, + GreenhouseMgr: r.Manager, GardenClient: gardenClient, Logger: r.WithValues("careInstruction", careInstruction.Name), Name: shoot.GenerateName(careInstruction.Name), diff --git a/controller/shoot/auth.go b/controller/shoot/auth.go index 891d703..ad3c832 100644 --- a/controller/shoot/auth.go +++ b/controller/shoot/auth.go @@ -34,14 +34,38 @@ func (r *ShootController) configureOIDCAuthentication(ctx context.Context, shoot r.CareInstruction.Spec.AuthenticationConfigMapName, err) } - // Add the auth ConfigMap label if it doesn't exist + // Label the ConfigMap so the watch predicate can identify it and associate it with this CareInstruction. + // Take a snapshot before any mutations so the patch only touches metadata.labels. + base := greenhouseAuthConfigMap.DeepCopy() if greenhouseAuthConfigMap.Labels == nil { greenhouseAuthConfigMap.Labels = make(map[string]string) } - if _, hasLabel := greenhouseAuthConfigMap.Labels[v1alpha1.AuthConfigMapLabel]; !hasLabel { - greenhouseAuthConfigMap.Labels[v1alpha1.AuthConfigMapLabel] = "true" - if err := r.GreenhouseClient.Update(ctx, &greenhouseAuthConfigMap); err != nil { - r.Info("failed to add auth ConfigMap label", "configMap", greenhouseAuthConfigMap.Name, "error", err) + labelsNeedUpdate := false + + // If the CM is already owned by a different CareInstruction, skip relabelling to + // avoid breaking its watch predicate. OIDC merging still proceeds normally. + existingOwner, hasCILabel := greenhouseAuthConfigMap.Labels[v1alpha1.CareInstructionLabel] + if hasCILabel && existingOwner != r.CareInstruction.Name { + r.Info("auth ConfigMap is already owned by another CareInstruction; skipping relabel to avoid breaking its watch", + "configMap", greenhouseAuthConfigMap.Name, + "existingOwner", existingOwner, + "thisCareInstruction", r.CareInstruction.Name) + } else { + if _, hasAuthLabel := greenhouseAuthConfigMap.Labels[v1alpha1.AuthConfigMapLabel]; !hasAuthLabel { + greenhouseAuthConfigMap.Labels[v1alpha1.AuthConfigMapLabel] = "true" + labelsNeedUpdate = true + } + if !hasCILabel { + greenhouseAuthConfigMap.Labels[v1alpha1.CareInstructionLabel] = r.CareInstruction.Name + labelsNeedUpdate = true + } + } + + if labelsNeedUpdate { + // Use a merge patch so only metadata.labels is sent to the API server. + // This avoids overwriting concurrent changes to config.yaml or other fields. + if err := r.GreenhouseClient.Patch(ctx, &greenhouseAuthConfigMap, client.MergeFrom(base)); err != nil { + r.Info("failed to patch labels on auth ConfigMap", "configMap", greenhouseAuthConfigMap.Name, "error", err) // Don't fail the reconciliation for this, just log it } } diff --git a/controller/shoot/shoot_controller.go b/controller/shoot/shoot_controller.go index bd071a4..f74865e 100644 --- a/controller/shoot/shoot_controller.go +++ b/controller/shoot/shoot_controller.go @@ -26,7 +26,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" ) const ( @@ -36,6 +39,7 @@ const ( type ShootController struct { GreenhouseClient client.Client + GreenhouseMgr ctrl.Manager // GreenhouseMgr provides the Greenhouse cluster cache for watches GardenClient client.Client logr.Logger Name string @@ -75,11 +79,60 @@ func (r *ShootController) SetupWithManager(mgr ctrl.Manager) error { r.Error(nil, "EventRecorder is not set for ShootController", "name", r.Name) } - // Setup the shoot controller with the manager - return ctrl.NewControllerManagedBy(mgr). + b := ctrl.NewControllerManagedBy(mgr). Named(r.Name). - For(&gardenerv1beta1.Shoot{}, builder.WithPredicates(predicates...)). - Complete(r) + For(&gardenerv1beta1.Shoot{}, builder.WithPredicates(predicates...)) + + // Watch the auth ConfigMap on the Greenhouse cluster; re-enqueue all Shoots when it changes + // so the Garden-side OIDC config stays in sync. + if r.CareInstruction.Spec.AuthenticationConfigMapName != "" && r.GreenhouseMgr != nil { + authCMPredicate := predicate.TypedFuncs[*corev1.ConfigMap]{ + CreateFunc: func(_ event.TypedCreateEvent[*corev1.ConfigMap]) bool { return false }, + UpdateFunc: func(e event.TypedUpdateEvent[*corev1.ConfigMap]) bool { + oldCM, newCM := e.ObjectOld, e.ObjectNew + if newCM.GetName() != r.CareInstruction.Spec.AuthenticationConfigMapName || + newCM.GetNamespace() != r.CareInstruction.GetNamespace() { + return false + } + if newCM.GetLabels()[v1alpha1.AuthConfigMapLabel] != "true" || + newCM.GetLabels()[v1alpha1.CareInstructionLabel] != r.CareInstruction.Name { + return false + } + return !maps.Equal(oldCM.Data, newCM.Data) + }, + DeleteFunc: func(_ event.TypedDeleteEvent[*corev1.ConfigMap]) bool { return false }, + } + b = b.WatchesRawSource(source.Kind( + r.GreenhouseMgr.GetCache(), + &corev1.ConfigMap{}, + handler.TypedEnqueueRequestsFromMapFunc(r.enqueueShootsForAuthConfigMap), + authCMPredicate, + )) + } + + // Setup the shoot controller with the manager + return b.Complete(r) +} + +func (r *ShootController) enqueueShootsForAuthConfigMap(ctx context.Context, _ *corev1.ConfigMap) []reconcile.Request { + shoots, err := r.CareInstruction.ListShoots(ctx, r.GardenClient) + if err != nil { + r.Info("failed to list shoots for auth ConfigMap change", "error", err) + return nil + } + requests := make([]reconcile.Request, 0, len(shoots.Items)) + for _, shoot := range shoots.Items { + if !r.matchesCEL(&shoot) { + continue + } + requests = append(requests, reconcile.Request{ + NamespacedName: client.ObjectKey{ + Namespace: shoot.Namespace, + Name: shoot.Name, + }, + }) + } + return requests } func (r *ShootController) newCELPredicate() predicate.Predicate { diff --git a/controller/shoot/shoot_controller_test.go b/controller/shoot/shoot_controller_test.go index 9a8749c..cf56f38 100644 --- a/controller/shoot/shoot_controller_test.go +++ b/controller/shoot/shoot_controller_test.go @@ -6,6 +6,7 @@ package shoot_test import ( "context" "encoding/base64" + "time" "shoot-grafter/api/v1alpha1" "shoot-grafter/controller/shoot" @@ -30,9 +31,10 @@ import ( ) var ( - careInstruction *v1alpha1.CareInstruction - mgrCtx context.Context - mgrCancel context.CancelFunc + careInstruction *v1alpha1.CareInstruction + mgrCtx context.Context + mgrCancel context.CancelFunc + greenhouseMgrCancel context.CancelFunc ) var _ = Describe("Shoot Controller", func() { JustBeforeEach(func() { @@ -73,10 +75,12 @@ var _ = Describe("Shoot Controller", func() { }) Expect(err).NotTo(HaveOccurred(), "there must be no error creating the greenhouse manager") - // Create ShootController with EventRecorder from Greenhouse manager - Expect(err).NotTo(HaveOccurred(), "there must be no error creating the manager") + // Create ShootController with EventRecorder and GreenhouseMgr from the Greenhouse manager. + // GreenhouseMgr is required so the ShootController can watch Greenhouse auth CMs and + // re-enqueue Shoots when they change. Expect((&shoot.ShootController{ GreenhouseClient: test.K8sClient, + GreenhouseMgr: greenhouseMgr, // Provide Greenhouse manager for cross-cluster CM watch GardenClient: test.GardenK8sClient, Logger: ctrl.Log.WithName("controllers").WithName("ShootController"), Name: "ShootController", @@ -88,7 +92,22 @@ var _ = Describe("Shoot Controller", func() { Expect(careInstructionWebhook.SetupWebhookWithManager(mgr)).To(Succeed(), "there must be no error setting up the webhook with the manager") mgrCtx, mgrCancel = context.WithCancel(test.Ctx) - // start the manager + + // start the Greenhouse manager so its cache is populated for the auth CM watch + ghCtx, ghCancel := context.WithCancel(test.Ctx) + greenhouseMgrCancel = ghCancel + go func() { + defer GinkgoRecover() + Expect(greenhouseMgr.Start(ghCtx)).To(Succeed(), "there must be no error starting the greenhouse manager") + }() + + // Wait for the Greenhouse cache to sync before proceeding so the auth CM watch is established. + syncCtx, syncCancel := context.WithTimeout(ghCtx, 30*time.Second) + defer syncCancel() + Expect(greenhouseMgr.GetCache().WaitForCacheSync(syncCtx)).To(BeTrue(), + "the greenhouse manager cache must be synced before running watch-dependent assertions") + + // start the garden manager go func() { defer GinkgoRecover() Expect(mgr.Start(mgrCtx)).To(Succeed(), "there must be no error starting the manager") @@ -191,8 +210,12 @@ var _ = Describe("Shoot Controller", func() { return len(events.Items) == 0 }).Should(BeTrue(), "should eventually not find Event resources") - // stop the manager + // stop both managers mgrCancel() + if greenhouseMgrCancel != nil { + greenhouseMgrCancel() + greenhouseMgrCancel = nil + } }) @@ -1734,7 +1757,7 @@ jwt: } Expect(test.GardenK8sClient.Create(test.Ctx, cm)).To(Succeed(), "should create CA ConfigMap") - // Eventually verify the label was added by the controller + // Eventually verify both labels were added by the controller Eventually(func(g Gomega) bool { var updatedConfigMap corev1.ConfigMap err := test.K8sClient.Get(test.Ctx, client.ObjectKey{ @@ -1744,10 +1767,173 @@ jwt: if err != nil { return false } - // Verify the label was added - g.Expect(updatedConfigMap.Labels).To(HaveKeyWithValue(v1alpha1.AuthConfigMapLabel, "true")) + // Verify AuthConfigMapLabel was added to enable the watch predicate + g.Expect(updatedConfigMap.Labels).To(HaveKeyWithValue(v1alpha1.AuthConfigMapLabel, "true"), + "controller should add auth-configmap label so the watch predicate can match") + // Verify CareInstructionLabel was added to associate the CM with its owning CareInstruction + g.Expect(updatedConfigMap.Labels).To(HaveKeyWithValue(v1alpha1.CareInstructionLabel, careInstruction.Name), + "controller should add careinstruction label to identify the owning CareInstruction") return true - }).Should(BeTrue(), "controller should add auth ConfigMap label when not initially present") + }).Should(BeTrue(), "controller should add both auth and careinstruction labels when not initially present") + }) + }) + + When("testing OIDC configuration watch triggering reconciliation", func() { + var greenhouseAuthConfigMap *corev1.ConfigMap + + BeforeEach(func() { + careInstruction = &v1alpha1.CareInstruction{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-careinstruction-watch", + Namespace: "default", + }, + Spec: v1alpha1.CareInstructionSpec{ + ShootSelector: &v1alpha1.ShootSelector{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "test": "watch", + }, + }, + }, + AuthenticationConfigMapName: "greenhouse-auth-config-watch", + }, + } + + // Create the Greenhouse AuthenticationConfiguration ConfigMap with initial OIDC config. + // The watch predicate requires both AuthConfigMapLabel and CareInstructionLabel. + // The controller will add CareInstructionLabel on first reconciliation; we pre-set + // AuthConfigMapLabel here so the CM is already identifiable as an auth CM. + greenhouseAuthConfigMap = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "greenhouse-auth-config-watch", + Namespace: "default", + Labels: map[string]string{ + v1alpha1.AuthConfigMapLabel: "true", + }, + }, + Data: map[string]string{ + "config.yaml": `apiVersion: apiserver.config.k8s.io/v1beta1 +kind: AuthenticationConfiguration +jwt: +- issuer: + url: https://greenhouse-watch.test.example.com + audiences: + - audience-v1 + claimMappings: + username: + claim: sub + prefix: 'greenhouse-v1:' +`, + }, + } + Expect(test.K8sClient.Create(test.Ctx, greenhouseAuthConfigMap)).To(Succeed(), "should create Greenhouse auth ConfigMap") + }) + + It("should re-reconcile shoots when Greenhouse auth CM is updated", func() { + // Create a shoot that matches the selector + shoot := &gardenerv1beta1.Shoot{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-shoot-watch", + Namespace: "default", + Labels: map[string]string{ + "test": "watch", + }, + }, + } + Expect(test.GardenK8sClient.Create(test.Ctx, shoot)).To(Succeed(), "should create Shoot resource") + + shoot.Status = gardenerv1beta1.ShootStatus{ + AdvertisedAddresses: []gardenerv1beta1.ShootAdvertisedAddress{ + { + Name: "external", + URL: "https://api-server.test-shoot-watch.example.com", + }, + }, + } + Expect(test.GardenK8sClient.Status().Update(test.Ctx, shoot)).To(Succeed(), "should update Shoot status") + + // Create CA ConfigMap + caCM := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-shoot-watch.ca-cluster", + Namespace: "default", + }, + Data: map[string]string{"ca.crt": "test-ca-data"}, + } + Expect(test.GardenK8sClient.Create(test.Ctx, caCM)).To(Succeed(), "should create CA ConfigMap") + + gardenAuthCMName := "test-careinstruction-watch-greenhouse-auth" + + // Step 1: wait for the initial Garden auth CM to be created with the v1 audience + Eventually(func(g Gomega) { + authCM := &corev1.ConfigMap{} + g.Expect(test.GardenK8sClient.Get(test.Ctx, client.ObjectKey{ + Name: gardenAuthCMName, + Namespace: "default", + }, authCM)).To(Succeed(), "should find Garden auth ConfigMap") + + var authConfig apiserverv1beta1.AuthenticationConfiguration + g.Expect(yaml.Unmarshal([]byte(authCM.Data["config.yaml"]), &authConfig)).To(Succeed()) + g.Expect(authConfig.JWT).To(HaveLen(1)) + g.Expect(authConfig.JWT[0].Issuer.URL).To(Equal("https://greenhouse-watch.test.example.com")) + g.Expect(authConfig.JWT[0].Issuer.Audiences).To(ConsistOf("audience-v1")) + }).Should(Succeed(), "initial Garden auth CM should contain v1 audience") + + // Step 2: wait for the controller to label the Greenhouse auth CM. + // This adds CareInstructionLabel, enabling the watch predicate to match future updates. + Eventually(func(g Gomega) { + var ghCM corev1.ConfigMap + g.Expect(test.K8sClient.Get(test.Ctx, client.ObjectKey{ + Name: "greenhouse-auth-config-watch", + Namespace: "default", + }, &ghCM)).To(Succeed()) + g.Expect(ghCM.Labels).To(HaveKeyWithValue(v1alpha1.CareInstructionLabel, careInstruction.Name), + "controller should have added CareInstructionLabel to Greenhouse auth CM") + }).Should(Succeed(), "Greenhouse auth CM should be labeled with CareInstructionLabel before proceeding") + + // Step 3: update the Greenhouse auth CM — same issuer URL, but new audience and prefix. + // Keeping the same URL ensures mergeAuthenticationConfigurations replaces the existing + // entry in-place rather than appending a second issuer. The watch (source.Kind on the + // Greenhouse cache) fires because the CM has both required labels and its content changed. + Expect(test.K8sClient.Get(test.Ctx, client.ObjectKey{ + Name: "greenhouse-auth-config-watch", + Namespace: "default", + }, greenhouseAuthConfigMap)).To(Succeed(), "should fetch latest Greenhouse auth CM") + greenhouseAuthConfigMap.Data["config.yaml"] = `apiVersion: apiserver.config.k8s.io/v1beta1 +kind: AuthenticationConfiguration +jwt: +- issuer: + url: https://greenhouse-watch.test.example.com + audiences: + - audience-v2 + claimMappings: + username: + claim: sub + prefix: 'greenhouse-v2:' +` + Expect(test.K8sClient.Update(test.Ctx, greenhouseAuthConfigMap)).To(Succeed(), + "should update Greenhouse auth CM to trigger watch-based reconciliation") + + // Step 4: the watch fires -> shoots are re-enqueued -> controller reconciles -> + // Garden auth CM is updated in-place to reflect the new audience (v2). + // The issuer count stays at 1 because the URL matches and the entry is replaced. + Eventually(func(g Gomega) { + authCM := &corev1.ConfigMap{} + g.Expect(test.GardenK8sClient.Get(test.Ctx, client.ObjectKey{ + Name: gardenAuthCMName, + Namespace: "default", + }, authCM)).To(Succeed()) + + var authConfig apiserverv1beta1.AuthenticationConfiguration + g.Expect(yaml.Unmarshal([]byte(authCM.Data["config.yaml"]), &authConfig)).To(Succeed()) + g.Expect(authConfig.JWT).To(HaveLen(1), "should still have exactly one issuer (replaced in-place)") + g.Expect(authConfig.JWT[0].Issuer.URL).To(Equal("https://greenhouse-watch.test.example.com")) + // audience-v2 proves the Garden CM was re-reconciled from the updated Greenhouse CM + g.Expect(authConfig.JWT[0].Issuer.Audiences).To(ConsistOf("audience-v2"), + "Garden auth CM should have updated audience after Greenhouse CM change triggers watch") + g.Expect(authConfig.JWT[0].ClaimMappings.Username.Prefix).NotTo(BeNil()) + g.Expect(*authConfig.JWT[0].ClaimMappings.Username.Prefix).To(Equal("greenhouse-v2:")) + }).Should(Succeed(), "Garden auth CM should be re-reconciled to v2 config after Greenhouse CM change triggers watch") }) })