Merge pull request #15 from weaveworks/trigger-on-provisioned

Trigger on provisioned

Author: bigkevmcd

Dockerfile

@@ -1,5 +1,5 @@
 # Build the manager binary
-FROM golang:1.17 as builder
+FROM golang:1.19 as builder
 
 ARG GITHUB_BUILD_USERNAME
 ARG GITHUB_BUILD_TOKEN

Makefile

@@ -40,7 +40,7 @@ IMAGE_TAG := $(shell tools/image-tag)
 IMG ?= $(IMAGE_TAG_BASE):$(IMAGE_TAG)
 CRD_OPTIONS ?= "crd"
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
-ENVTEST_K8S_VERSION = 1.21
+ENVTEST_K8S_VERSION = 1.25
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))

api/v1alpha1/clusterbootstrapconfig_types.go

@@ -45,7 +45,7 @@ type JobTemplate struct {
 	// guarantees (e.g. finalizers) will be honored. If this field is unset,
 	// the Job won't be automatically deleted. If this field is set to zero,
 	// the Job becomes eligible to be deleted immediately after it finishes.
-	// +optional
+	//+optional
 	TTLSecondsAfterFinished *int32 `json:"ttlSecondsAfterFinished,omitempty"`
 
 	// A batch/v1 Job is created with the Spec as the PodSpec.
@@ -57,6 +57,18 @@ type ClusterBootstrapConfigSpec struct {
 	ClusterSelector metav1.LabelSelector `json:"clusterSelector"`
 	Template        JobTemplate          `json:"jobTemplate"`
 
+	// Trigger the bootstrapping when the linked cluster has a True
+	// "ClusterProvisioned" condition.
+	//
+	// A new job will not be triggered when the cluster is finally "Ready"
+	// because it will already have the annotation that indicates the cluster
+	// has been bootstrapped.
+	//
+	// Defaults to false.
+	//+kubebuilder:default:false
+	//+optional
+	RequireClusterProvisioned bool `json:"requireClusterProvisioned"`
+
 	// Wait for the remote cluster to be "ready" before creating the jobs.
 	// Defaults to false.
 	//+kubebuilder:default:false

api/v1alpha1/groupversion_info.go

@@ -15,8 +15,8 @@ limitations under the License.
 */
 
 // Package v1alpha1 contains API Schema definitions for the capi v1alpha1 API group
-//+kubebuilder:object:generate=true
-//+groupName=capi.weave.works
+// +kubebuilder:object:generate=true
+// +groupName=capi.weave.works
 package v1alpha1
 
 import (

config/crd/bases/capi.weave.works_clusterbootstrapconfigs.yaml

@@ -7196,6 +7196,13 @@ spec:
                 - generateName
                 - spec
                 type: object
+              requireClusterProvisioned:
+                description: "Trigger the bootstrapping when the linked cluster has
+                  a True \"ClusterProvisioned\" condition. \n A new job will not be
+                  triggered when the cluster is finally \"Ready\" because it will
+                  already have the annotation that indicates the cluster has been
+                  bootstrapped. \n Defaults to false."
+                type: boolean
               requireClusterReady:
                 description: Wait for the remote cluster to be "ready" before creating
                   the jobs. Defaults to false.

config/default/manager_auth_proxy_patch.yaml

@@ -25,3 +25,8 @@ spec:
         - "--health-probe-bind-address=:8081"
         - "--metrics-bind-address=127.0.0.1:8080"
         - "--leader-elect"
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL

config/rbac/role.yaml

@@ -61,3 +61,12 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - gitops.weave.works
+  resources:
+  - gitopsclusters
+  verbs:
+  - get
+  - list
+  - patch
+  - watch

controllers/clusterbootstrapconfig_controller.go

@@ -21,6 +21,7 @@ import (
 	"encoding/json"
 	"fmt"
 
+	"github.com/fluxcd/pkg/runtime/conditions"
 	gitopsv1alpha1 "github.com/weaveworks/cluster-controller/api/v1alpha1"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -62,6 +63,7 @@ func NewClusterBootstrapConfigReconciler(c client.Client, s *runtime.Scheme) *Cl
 //+kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters,verbs=get;list;watch;update;patch
 //+kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch
+// +kubebuilder:rbac:groups="gitops.weave.works",resources=gitopsclusters,verbs=get;watch;list;patch
 
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
 // move the current state of the cluster closer to the desired state.
@@ -76,15 +78,15 @@ func (r *ClusterBootstrapConfigReconciler) Reconcile(ctx context.Context, req ct
 	}
 	logger.Info("cluster bootstrap config loaded", "name", clusterBootstrapConfig.ObjectMeta.Name)
 
-	clusters, err := r.getClustersBySelector(ctx, req.Namespace, clusterBootstrapConfig.Spec.ClusterSelector)
+	clusters, err := r.getClustersBySelector(ctx, req.Namespace, clusterBootstrapConfig.Spec)
 	if err != nil {
 		return ctrl.Result{}, fmt.Errorf("failed to getClustersBySelector for bootstrap config %s: %w", req, err)
 	}
 	logger.Info("identified clusters for reconciliation", "clusterCount", len(clusters))
 
-	for _, c := range clusters {
+	for _, cluster := range clusters {
 		if clusterBootstrapConfig.Spec.RequireClusterReady {
-			clusterName := types.NamespacedName{Name: c.GetName(), Namespace: c.GetNamespace()}
+			clusterName := types.NamespacedName{Name: cluster.GetName(), Namespace: cluster.GetNamespace()}
 			clusterClient, err := r.clientForCluster(ctx, clusterName)
 			if err != nil {
 				if apierrors.IsNotFound(err) {
@@ -105,7 +107,7 @@ func (r *ClusterBootstrapConfigReconciler) Reconcile(ctx context.Context, req ct
 				return ctrl.Result{RequeueAfter: clusterBootstrapConfig.ClusterReadinessRequeue()}, nil
 			}
 		}
-		if err := bootstrapClusterWithConfig(ctx, logger, r.Client, c, &clusterBootstrapConfig); err != nil {
+		if err := bootstrapClusterWithConfig(ctx, logger, r.Client, cluster, &clusterBootstrapConfig); err != nil {
 			return ctrl.Result{}, fmt.Errorf("failed to bootstrap cluster config: %w", err)
 		}
 
@@ -119,8 +121,8 @@ func (r *ClusterBootstrapConfigReconciler) Reconcile(ctx context.Context, req ct
 		if err != nil {
 			return ctrl.Result{}, fmt.Errorf("failed to create a patch to update the cluster annotations: %w", err)
 		}
-		if err := r.Client.Patch(ctx, c, client.RawPatch(types.MergePatchType, mergePatch)); err != nil {
-			return ctrl.Result{}, fmt.Errorf("failed to annotate cluster %s/%s as bootstrapped: %w", c.ObjectMeta.Name, c.ObjectMeta.Namespace, err)
+		if err := r.Client.Patch(ctx, cluster, client.RawPatch(types.MergePatchType, mergePatch)); err != nil {
+			return ctrl.Result{}, fmt.Errorf("failed to annotate cluster %s/%s as bootstrapped: %w", cluster.ObjectMeta.Name, cluster.ObjectMeta.Namespace, err)
 		}
 	}
 	return ctrl.Result{}, nil
@@ -137,9 +139,9 @@ func (r *ClusterBootstrapConfigReconciler) SetupWithManager(mgr ctrl.Manager) er
 		Complete(r)
 }
 
-func (r *ClusterBootstrapConfigReconciler) getClustersBySelector(ctx context.Context, ns string, ls metav1.LabelSelector) ([]*gitopsv1alpha1.GitopsCluster, error) {
+func (r *ClusterBootstrapConfigReconciler) getClustersBySelector(ctx context.Context, ns string, spec capiv1alpha1.ClusterBootstrapConfigSpec) ([]*gitopsv1alpha1.GitopsCluster, error) {
 	logger := ctrl.LoggerFrom(ctx)
-	selector, err := metav1.LabelSelectorAsSelector(&ls)
+	selector, err := metav1.LabelSelectorAsSelector(&spec.ClusterSelector)
 	if err != nil {
 		return nil, fmt.Errorf("unable to convert selector: %w", err)
 	}
@@ -156,23 +158,24 @@ func (r *ClusterBootstrapConfigReconciler) getClustersBySelector(ctx context.Con
 	logger.Info("identified clusters with selector", "selector", selector, "count", len(clusterList.Items))
 	clusters := []*gitopsv1alpha1.GitopsCluster{}
 	for i := range clusterList.Items {
-		c := &clusterList.Items[i]
+		cluster := &clusterList.Items[i]
 
-		clusterFound := false
-		for _, condition := range c.Status.Conditions {
-			if condition.Type == "Ready" && condition.Status == metav1.ConditionTrue {
-				clusterFound = true
-			}
-		}
-		if !clusterFound {
-			logger.Info("cluster discarded - not provisioned", "phase", c.Status)
+		if !conditions.IsReady(cluster) && !spec.RequireClusterProvisioned {
+			logger.Info("cluster discarded - not ready", "phase", cluster.Status)
 			continue
 		}
-		if metav1.HasAnnotation(c.ObjectMeta, capiv1alpha1.BootstrappedAnnotation) {
+		if spec.RequireClusterProvisioned {
+			if !isProvisioned(cluster) {
+				logger.Info("waiting for cluster to be provisioned", "cluster", cluster.Name)
+				continue
+			}
+		}
+
+		if metav1.HasAnnotation(cluster.ObjectMeta, capiv1alpha1.BootstrappedAnnotation) {
 			continue
 		}
-		if c.DeletionTimestamp.IsZero() {
-			clusters = append(clusters, c)
+		if cluster.DeletionTimestamp.IsZero() {
+			clusters = append(clusters, cluster)
 		}
 	}
 	return clusters, nil
@@ -270,3 +273,7 @@ func kubeConfigBytesToClient(b []byte) (client.Client, error) {
 	}
 	return client, nil
 }
+
+func isProvisioned(from conditions.Getter) bool {
+	return conditions.IsTrue(from, gitopsv1alpha1.ClusterProvisionedCondition)
+}

controllers/clusterbootstrapconfig_controller_test.go

@@ -138,6 +138,82 @@ func TestReconcile_when_cluster_ready(t *testing.T) {
 	}
 }
 
+func TestReconcile_when_cluster_provisioned(t *testing.T) {
+	bc := makeTestClusterBootstrapConfig(func(c *capiv1alpha1.ClusterBootstrapConfig) {
+		c.Spec.RequireClusterProvisioned = true
+	})
+	cl := makeTestCluster(func(c *gitopsv1alpha1.GitopsCluster) {
+		c.ObjectMeta.Labels = bc.Spec.ClusterSelector.MatchLabels
+		c.Status.Conditions = append(c.Status.Conditions, makeNotReadyCondition(), makeClusterProvisionedCondition())
+	})
+	secret := makeTestSecret(types.NamespacedName{
+		Name:      cl.GetName() + "-kubeconfig",
+		Namespace: cl.GetNamespace(),
+	}, map[string][]byte{"value": []byte("testing")})
+	// This cheats by using the local client as the remote client to simplify
+	// getting the value from the remote client.
+	reconciler := makeTestReconciler(t, bc, cl, secret)
+	reconciler.configParser = func(b []byte) (client.Client, error) {
+		return reconciler.Client, nil
+	}
+
+	result, err := reconciler.Reconcile(context.TODO(), ctrl.Request{NamespacedName: types.NamespacedName{
+		Name:      bc.GetName(),
+		Namespace: bc.GetNamespace(),
+	}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !result.IsZero() {
+		t.Fatalf("want empty result, got %v", result)
+	}
+	var jobs batchv1.JobList
+	if err := reconciler.List(context.TODO(), &jobs, client.InNamespace(testNamespace)); err != nil {
+		t.Fatal(err)
+	}
+	if l := len(jobs.Items); l != 1 {
+		t.Fatalf("found %d jobs, want %d", l, 1)
+	}
+}
+
+func TestReconcile_when_cluster_not_provisioned(t *testing.T) {
+	bc := makeTestClusterBootstrapConfig(func(c *capiv1alpha1.ClusterBootstrapConfig) {
+		c.Spec.RequireClusterProvisioned = true
+	})
+	cl := makeTestCluster(func(c *gitopsv1alpha1.GitopsCluster) {
+		c.ObjectMeta.Labels = bc.Spec.ClusterSelector.MatchLabels
+		c.Status.Conditions = append(c.Status.Conditions, makeNotReadyCondition())
+	})
+	secret := makeTestSecret(types.NamespacedName{
+		Name:      cl.GetName() + "-kubeconfig",
+		Namespace: cl.GetNamespace(),
+	}, map[string][]byte{"value": []byte("testing")})
+	// This cheats by using the local client as the remote client to simplify
+	// getting the value from the remote client.
+	reconciler := makeTestReconciler(t, bc, cl, secret)
+	reconciler.configParser = func(b []byte) (client.Client, error) {
+		return reconciler.Client, nil
+	}
+
+	result, err := reconciler.Reconcile(context.TODO(), ctrl.Request{NamespacedName: types.NamespacedName{
+		Name:      bc.GetName(),
+		Namespace: bc.GetNamespace(),
+	}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !result.IsZero() {
+		t.Fatalf("want empty result, got %v", result)
+	}
+	var jobs batchv1.JobList
+	if err := reconciler.List(context.TODO(), &jobs, client.InNamespace(testNamespace)); err != nil {
+		t.Fatal(err)
+	}
+	if l := len(jobs.Items); l != 0 {
+		t.Fatalf("found %d jobs, want %d", l, 1)
+	}
+}
+
 func TestReconcile_when_cluster_no_matching_labels(t *testing.T) {
 	bc := makeTestClusterBootstrapConfig(func(c *capiv1alpha1.ClusterBootstrapConfig) {
 		c.Spec.RequireClusterReady = true
@@ -320,6 +396,20 @@ func makeReadyCondition() metav1.Condition {
 	}
 }
 
+func makeClusterProvisionedCondition() metav1.Condition {
+	return metav1.Condition{
+		Type:   gitopsv1alpha1.ClusterProvisionedCondition,
+		Status: metav1.ConditionTrue,
+	}
+}
+
+func makeNotReadyCondition() metav1.Condition {
+	return metav1.Condition{
+		Type:   "Ready",
+		Status: metav1.ConditionFalse,
+	}
+}
+
 func makeTestReconciler(t *testing.T, objs ...runtime.Object) *ClusterBootstrapConfigReconciler {
 	s, tc := makeTestClientAndScheme(t, objs...)
 	return NewClusterBootstrapConfigReconciler(tc, s)

go.mod

@@ -1,16 +1,16 @@
 module github.com/weaveworks/cluster-bootstrap-controller
 
-go 1.17
+go 1.19
 
 require (
+	github.com/fluxcd/pkg/runtime v0.13.2
 	github.com/go-logr/logr v1.2.2
 	github.com/google/go-cmp v0.5.7
-	github.com/weaveworks/cluster-controller v0.0.0-20220412121721-313761dc9997
+	github.com/weaveworks/cluster-controller v1.3.1
 	k8s.io/api v0.23.4
 	k8s.io/apimachinery v0.23.4
 	k8s.io/client-go v0.23.4
 	k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9
-	sigs.k8s.io/cluster-api v1.1.3
 	sigs.k8s.io/controller-runtime v0.11.1
 	sigs.k8s.io/yaml v1.3.0
 )
@@ -24,7 +24,6 @@ require (
 	github.com/Azure/go-autorest/logger v0.2.1 // indirect
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/blang/semver v3.5.1+incompatible // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
@@ -43,6 +42,7 @@ require (
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/onsi/gomega v1.18.1 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/prometheus/client_golang v1.12.1 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect