Watch gitopscluster CRs (#11)

* Watch GitOps clusters
* Configure git for private modules
* Configure docker for private modules
* Add environment variables to docker build step
* Update readme

Co-authored-by: Kevin McDermott <[email protected]>
Co-authored-by: Simon Howe <[email protected]>

Author: 3 people

.github/workflows/build.yaml

@@ -3,7 +3,10 @@ name: "Build and push docker container"
 on:
   pull_request:
   workflow_dispatch:
-    
+
+env:
+  GOPRIVATE: github.com/weaveworks/cluster-controller
+
 jobs:
   cluster-bootstrap-controller:
     runs-on: ubuntu-latest
@@ -17,7 +20,15 @@ jobs:
       with:
         fetch-depth: 0
         ref: ${{ github.event.pull_request.head.sha }}
+    - name: Configure git for private modules
+      env:
+        GITHUB_BUILD_USERNAME: ${{ secrets.BUILD_BOT_USER }}
+        GITHUB_BUILD_TOKEN: ${{ secrets.BUILD_BOT_PERSONAL_ACCESS_TOKEN }}
+      run: git config --global url."https://${GITHUB_BUILD_USERNAME}:${GITHUB_BUILD_TOKEN}@github.com".insteadOf "https://github.com"
     - name: Build docker image
+      env:
+        GITHUB_BUILD_USERNAME: ${{ secrets.BUILD_BOT_USER }}
+        GITHUB_BUILD_TOKEN: ${{ secrets.BUILD_BOT_PERSONAL_ACCESS_TOKEN }}
       run: |
         make docker-build
     - name: Login to Docker Hub

Dockerfile

@@ -1,6 +1,10 @@
 # Build the manager binary
 FROM golang:1.17 as builder
 
+ARG GITHUB_BUILD_USERNAME
+ARG GITHUB_BUILD_TOKEN
+RUN git config --global url."https://${GITHUB_BUILD_USERNAME}:${GITHUB_BUILD_TOKEN}@github.com".insteadOf "https://github.com"
+
 WORKDIR /workspace
 # Copy the Go Modules manifests
 COPY go.mod go.mod

Makefile

@@ -100,7 +100,11 @@ run: manifests generate fmt vet ## Run a controller from your host.
 	go run ./main.go
 
 docker-build: test ## Build docker image with the manager.
-	docker build -t ${IMG} .
+	docker build \
+	--build-arg=GITHUB_BUILD_TOKEN=$(GITHUB_BUILD_TOKEN) \
+	--build-arg=GITHUB_BUILD_USERNAME=$(GITHUB_BUILD_USERNAME) \
+	-t ${IMG} . \
+	
 
 docker-push: ## Push docker image with the manager.
 	docker push ${IMG}

README.md

@@ -1,10 +1,10 @@
 # cluster-bootstrap-controller
 
-This is a controller that tracks [CAPI](https://github.com/kubernetes-sigs/cluster-api) [Cluster](https://cluster-api.sigs.k8s.io/developer/architecture/controllers/cluster.html) objects.
+This is a controller that tracks [GitopsCluster] objects.
 
 It provides a CR for a `ClusterBootstrapConfig` which provides a [Job](https://kubernetes.io/docs/concepts/workloads/controllers/job/) template.
 
-When a CAPI Cluster is "provisioned" a Job is created from the template, the template can access multiple fields.
+When a GitopsCluster is "Ready" a Job is created from the template, the template can access multiple fields.
 
 ```yaml
 apiVersion: capi.weave.works/v1alpha1
@@ -34,7 +34,7 @@ spec:
           secretName: '{{ .ObjectMeta.Name }}-kubeconfig'
 ```
 
-This is using Go [templating](https://pkg.go.dev/text/template) and the `Cluster` object is provided as the context, this means that expressions like `{{ .ObjectMeta.Name }}` will get the _name_ of the Cluster that has transitioned to "provisioned".
+This is using Go [templating](https://pkg.go.dev/text/template) and the `GitopsCluster` object is provided as the context, this means that expressions like `{{ .ObjectMeta.Name }}` will get the _name_ of the GitopsCluster that has transitioned to "Ready".
 
 ## Annotations
 
@@ -55,8 +55,6 @@ e.g.
 
 ## Installation
 
-You will need to have CAPI installed first, see the [CAPI Quick Start](https://cluster-api.sigs.k8s.io/user/quick-start.html).
-
 Release files are available https://github.com/weaveworks/cluster-bootstrap-controller/releases
 
 You can install these e.g.

controllers/bootstrap.go

@@ -4,10 +4,10 @@ import (
 	"context"
 	"fmt"
 
+	gitopsv1alpha1 "github.com/weaveworks/cluster-controller/api/v1alpha1"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
@@ -16,7 +16,7 @@ import (
 )
 
 // bootstrapCluster applies the jobs from a ClusterBootstrapConfig to a cluster.
-func bootstrapClusterWithConfig(ctx context.Context, logger logr.Logger, c client.Client, cl *clusterv1.Cluster, bc *capiv1alpha1.ClusterBootstrapConfig) error {
+func bootstrapClusterWithConfig(ctx context.Context, logger logr.Logger, c client.Client, cl *gitopsv1alpha1.GitopsCluster, bc *capiv1alpha1.ClusterBootstrapConfig) error {
 	job, err := renderTemplates(cl, jobFromTemplate(cl, bc.Spec.Template))
 	if err != nil {
 		return fmt.Errorf("failed to render job from template: %w", err)
@@ -31,7 +31,7 @@ func bootstrapClusterWithConfig(ctx context.Context, logger logr.Logger, c clien
 	return nil
 }
 
-func jobFromTemplate(cl *clusterv1.Cluster, jt capiv1alpha1.JobTemplate) *batchv1.Job {
+func jobFromTemplate(cl *gitopsv1alpha1.GitopsCluster, jt capiv1alpha1.JobTemplate) *batchv1.Job {
 	return &batchv1.Job{
 		ObjectMeta: metav1.ObjectMeta{
 			GenerateName: jt.GenerateName,

controllers/bootstrap_test.go

@@ -7,13 +7,13 @@ import (
 	"github.com/go-logr/logr"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
+	gitopsv1alpha1 "github.com/weaveworks/cluster-controller/api/v1alpha1"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	ptrutils "k8s.io/utils/pointer"
-	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 
@@ -76,8 +76,8 @@ func Test_bootstrapClusterWithConfig_sets_owner(t *testing.T) {
 
 	want := []metav1.OwnerReference{
 		{
-			APIVersion: "cluster.x-k8s.io/v1beta1",
-			Kind:       "Cluster",
+			APIVersion: "gitops.weave.works/v1alpha1",
+			Kind:       "GitopsCluster",
 			Name:       testClusterName,
 		},
 	}
@@ -90,7 +90,7 @@ func Test_bootstrapClusterWithConfig_fail_to_create_job(t *testing.T) {
 	// This is a hacky test for making Create fail because of an unregistered
 	// type.
 	s := runtime.NewScheme()
-	test.AssertNoError(t, clusterv1.AddToScheme(s))
+	test.AssertNoError(t, gitopsv1alpha1.AddToScheme(s))
 	tc := fake.NewClientBuilder().WithScheme(s).Build()
 	bc := makeTestClusterBootstrapConfig()
 	cl := makeTestCluster()
@@ -120,13 +120,13 @@ func makeTestPodSpecWithVolumes(volumes ...corev1.Volume) corev1.PodSpec {
 	}
 }
 
-func makeTestCluster(opts ...func(*clusterv1.Cluster)) *clusterv1.Cluster {
-	c := &clusterv1.Cluster{
+func makeTestCluster(opts ...func(*gitopsv1alpha1.GitopsCluster)) *gitopsv1alpha1.GitopsCluster {
+	c := &gitopsv1alpha1.GitopsCluster{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      testClusterName,
 			Namespace: testNamespace,
 		},
-		Spec: clusterv1.ClusterSpec{},
+		Spec: gitopsv1alpha1.GitopsClusterSpec{},
 	}
 	for _, o := range opts {
 		o(c)
@@ -188,8 +188,8 @@ func makeTestClientAndScheme(t *testing.T, objs ...runtime.Object) (*runtime.Sch
 	s := runtime.NewScheme()
 	test.AssertNoError(t, clientgoscheme.AddToScheme(s))
 	test.AssertNoError(t, capiv1alpha1.AddToScheme(s))
-	test.AssertNoError(t, clusterv1.AddToScheme(s))
 	test.AssertNoError(t, batchv1.AddToScheme(s))
+	test.AssertNoError(t, gitopsv1alpha1.AddToScheme(s))
 	return s, fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(objs...).Build()
 }
 

controllers/clusterbootstrapconfig_controller.go

@@ -21,14 +21,14 @@ import (
 	"encoding/json"
 	"fmt"
 
+	gitopsv1alpha1 "github.com/weaveworks/cluster-controller/api/v1alpha1"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/clientcmd"
-	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
@@ -131,13 +131,13 @@ func (r *ClusterBootstrapConfigReconciler) SetupWithManager(mgr ctrl.Manager) er
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&capiv1alpha1.ClusterBootstrapConfig{}).
 		Watches(
-			&source.Kind{Type: &clusterv1.Cluster{}},
+			&source.Kind{Type: &gitopsv1alpha1.GitopsCluster{}},
 			handler.EnqueueRequestsFromMapFunc(r.clusterToClusterBootstrapConfig),
 		).
 		Complete(r)
 }
 
-func (r *ClusterBootstrapConfigReconciler) getClustersBySelector(ctx context.Context, ns string, ls metav1.LabelSelector) ([]*clusterv1.Cluster, error) {
+func (r *ClusterBootstrapConfigReconciler) getClustersBySelector(ctx context.Context, ns string, ls metav1.LabelSelector) ([]*gitopsv1alpha1.GitopsCluster, error) {
 	logger := ctrl.LoggerFrom(ctx)
 	selector, err := metav1.LabelSelectorAsSelector(&ls)
 	if err != nil {
@@ -148,17 +148,24 @@ func (r *ClusterBootstrapConfigReconciler) getClustersBySelector(ctx context.Con
 		logger.Info("empty ClusterBootstrapConfig selector: no clusters are selected")
 		return nil, nil
 	}
-	clusterList := &clusterv1.ClusterList{}
+	clusterList := &gitopsv1alpha1.GitopsClusterList{}
 	if err := r.Client.List(ctx, clusterList, client.InNamespace(ns), client.MatchingLabelsSelector{Selector: selector}); err != nil {
 		return nil, fmt.Errorf("failed to list clusters: %w", err)
 	}
 
 	logger.Info("identified clusters with selector", "selector", selector, "count", len(clusterList.Items))
-	clusters := []*clusterv1.Cluster{}
+	clusters := []*gitopsv1alpha1.GitopsCluster{}
 	for i := range clusterList.Items {
 		c := &clusterList.Items[i]
-		if clusterv1.ClusterPhase(c.Status.Phase) != clusterv1.ClusterPhaseProvisioned {
-			logger.Info("cluster discarded - not provisioned", "phase", c.Status.Phase)
+
+		clusterFound := false
+		for _, condition := range c.Status.Conditions {
+			if condition.Type == "Ready" && condition.Status == metav1.ConditionTrue {
+				clusterFound = true
+			}
+		}
+		if !clusterFound {
+			logger.Info("cluster discarded - not provisioned", "phase", c.Status)
 			continue
 		}
 		if metav1.HasAnnotation(c.ObjectMeta, capiv1alpha1.BootstrappedAnnotation) {
@@ -175,7 +182,7 @@ func (r *ClusterBootstrapConfigReconciler) getClustersBySelector(ctx context.Con
 // ClusterBootstrapConfig.
 func (r *ClusterBootstrapConfigReconciler) clusterToClusterBootstrapConfig(o client.Object) []ctrl.Request {
 	result := []ctrl.Request{}
-	cluster, ok := o.(*clusterv1.Cluster)
+	cluster, ok := o.(*gitopsv1alpha1.GitopsCluster)
 	if !ok {
 		panic(fmt.Sprintf("Expected a Cluster but got a %T", o))
 	}

controllers/clusterbootstrapconfig_controller_test.go

@@ -15,12 +15,12 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/clientcmd"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
-	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	capiv1alpha1 "github.com/weaveworks/cluster-bootstrap-controller/api/v1alpha1"
 	"github.com/weaveworks/cluster-bootstrap-controller/test"
+	gitopsv1alpha1 "github.com/weaveworks/cluster-controller/api/v1alpha1"
 )
 
 const testWaitDuration = time.Second * 55
@@ -35,9 +35,9 @@ func TestReconcile_when_cluster_not_ready(t *testing.T) {
 		"node-role.kubernetes.io/control-plane": "",
 	}, corev1.NodeCondition{Type: "Ready", Status: "False", LastHeartbeatTime: metav1.Now(), LastTransitionTime: metav1.Now(), Reason: "KubeletReady", Message: "kubelet is posting ready status"})
 
-	cl := makeTestCluster(func(c *clusterv1.Cluster) {
+	cl := makeTestCluster(func(c *gitopsv1alpha1.GitopsCluster) {
 		c.ObjectMeta.Labels = bc.Spec.ClusterSelector.MatchLabels
-		c.Status.Phase = string(clusterv1.ClusterPhaseProvisioned)
+		c.Status.Conditions = append(c.Status.Conditions, makeReadyCondition())
 	})
 	secret := makeTestSecret(types.NamespacedName{
 		Name:      cl.GetName() + "-kubeconfig",
@@ -69,9 +69,9 @@ func TestReconcile_when_cluster_secret_not_available(t *testing.T) {
 	bc := makeTestClusterBootstrapConfig(func(c *capiv1alpha1.ClusterBootstrapConfig) {
 		c.Spec.RequireClusterReady = true
 	})
-	cl := makeTestCluster(func(c *clusterv1.Cluster) {
+	cl := makeTestCluster(func(c *gitopsv1alpha1.GitopsCluster) {
 		c.ObjectMeta.Labels = bc.Spec.ClusterSelector.MatchLabels
-		c.Status.Phase = string(clusterv1.ClusterPhaseProvisioned)
+		c.Status.Conditions = append(c.Status.Conditions, makeReadyCondition())
 	})
 	reconciler := makeTestReconciler(t, bc, cl)
 
@@ -104,9 +104,9 @@ func TestReconcile_when_cluster_ready(t *testing.T) {
 	}, corev1.NodeCondition{
 		Type: "Ready", Status: "True", LastHeartbeatTime: metav1.Now(), LastTransitionTime: metav1.Now(), Reason: "KubeletReady", Message: "kubelet is posting ready status"})
 
-	cl := makeTestCluster(func(c *clusterv1.Cluster) {
+	cl := makeTestCluster(func(c *gitopsv1alpha1.GitopsCluster) {
 		c.ObjectMeta.Labels = bc.Spec.ClusterSelector.MatchLabels
-		c.Status.Phase = string(clusterv1.ClusterPhaseProvisioned)
+		c.Status.Conditions = append(c.Status.Conditions, makeReadyCondition())
 	})
 	secret := makeTestSecret(types.NamespacedName{
 		Name:      cl.GetName() + "-kubeconfig",
@@ -142,11 +142,11 @@ func TestReconcile_when_cluster_no_matching_labels(t *testing.T) {
 	bc := makeTestClusterBootstrapConfig(func(c *capiv1alpha1.ClusterBootstrapConfig) {
 		c.Spec.RequireClusterReady = true
 	})
-	cl := makeTestCluster(func(c *clusterv1.Cluster) {
+	cl := makeTestCluster(func(c *gitopsv1alpha1.GitopsCluster) {
 		c.ObjectMeta.Labels = map[string]string{
 			"will-not-match": "",
 		}
-		c.Status.Phase = string(clusterv1.ClusterPhaseProvisioned)
+		c.Status.Conditions = append(c.Status.Conditions, makeReadyCondition())
 	})
 	// This cheats by using the local client as the remote client to simplify
 	// getting the value from the remote client.
@@ -178,11 +178,11 @@ func TestReconcile_when_empty_label_selector(t *testing.T) {
 		}
 
 	})
-	cl := makeTestCluster(func(c *clusterv1.Cluster) {
+	cl := makeTestCluster(func(c *gitopsv1alpha1.GitopsCluster) {
 		c.ObjectMeta.Labels = map[string]string{
 			"will-not-match": "",
 		}
-		c.Status.Phase = string(clusterv1.ClusterPhaseProvisioned)
+		c.Status.Conditions = append(c.Status.Conditions, makeReadyCondition())
 	})
 	// This cheats by using the local client as the remote client to simplify
 	// getting the value from the remote client.
@@ -214,9 +214,9 @@ func TestReconcile_when_cluster_ready_and_old_label(t *testing.T) {
 		LastTransitionTime: metav1.Now(), Reason: "KubeletReady",
 		Message: "kubelet is posting ready status"})
 
-	cl := makeTestCluster(func(c *clusterv1.Cluster) {
+	cl := makeTestCluster(func(c *gitopsv1alpha1.GitopsCluster) {
 		c.ObjectMeta.Labels = bc.Spec.ClusterSelector.MatchLabels
-		c.Status.Phase = string(clusterv1.ClusterPhaseProvisioned)
+		c.Status.Conditions = append(c.Status.Conditions, makeReadyCondition())
 	})
 	secret := makeTestSecret(types.NamespacedName{
 		Name:      cl.GetName() + "-kubeconfig",
@@ -313,6 +313,13 @@ func Test_kubeConfigBytesToClient_with_invalidkubeconfig(t *testing.T) {
 	}
 }
 
+func makeReadyCondition() metav1.Condition {
+	return metav1.Condition{
+		Type:   "Ready",
+		Status: metav1.ConditionTrue,
+	}
+}
+
 func makeTestReconciler(t *testing.T, objs ...runtime.Object) *ClusterBootstrapConfigReconciler {
 	s, tc := makeTestClientAndScheme(t, objs...)
 	return NewClusterBootstrapConfigReconciler(tc, s)

controllers/templating.go

@@ -5,8 +5,8 @@ import (
 	"fmt"
 	"text/template"
 
+	gitopsv1alpha1 "github.com/weaveworks/cluster-controller/api/v1alpha1"
 	batchv1 "k8s.io/api/batch/v1"
-	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	"sigs.k8s.io/yaml"
 )
 
@@ -16,14 +16,14 @@ func lookup(m map[string]string) func(s string) string {
 	}
 }
 
-func makeFuncMap(cl *clusterv1.Cluster) template.FuncMap {
+func makeFuncMap(cl *gitopsv1alpha1.GitopsCluster) template.FuncMap {
 	return template.FuncMap{
 		"annotation": lookup(cl.ObjectMeta.GetAnnotations()),
 		"label":      lookup(cl.ObjectMeta.GetLabels()),
 	}
 }
 
-func renderTemplates(cl *clusterv1.Cluster, j *batchv1.Job) (*batchv1.Job, error) {
+func renderTemplates(cl *gitopsv1alpha1.GitopsCluster, j *batchv1.Job) (*batchv1.Job, error) {
 	raw, err := yaml.Marshal(j)
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse job as YAML: %w", err)