Skip to content

Commit d36def5

Browse files
bigkevmcdbigkevmcd
authored
Merge pull request #16 from weaveworks/multi-bootstrap-annotation
Allow multiple bootstraps with different configs.
2 parents 5fee3fb + f442912 commit d36def5

File tree

3 files changed

+175
-8
lines changed

3 files changed

+175
-8
lines changed

‎api/v1alpha1/clusterbootstrapconfig_types.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,10 @@ import (
2525

2626
const defaultWaitDuration = time.Second * 60
2727

28-
const BootstrappedAnnotation = "capi.weave.works/bootstrapped"
28+
const (
29+
BootstrappedAnnotation = "capi.weave.works/bootstrapped"
30+
BootstrapConfigsAnnotation = "capi.weave.works/bootstrap-configs"
31+
)
2932

3033
// JobTemplate describes a job to create
3134
type JobTemplate struct {

‎controllers/clusterbootstrapconfig_controller.go‎

Lines changed: 27 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"context"
2121
"encoding/json"
2222
"fmt"
23+
"strings"
2324

2425
"github.com/fluxcd/pkg/runtime/conditions"
2526
gitopsv1alpha1 "github.com/weaveworks/cluster-controller/api/v1alpha1"
@@ -29,6 +30,7 @@ import (
2930
"k8s.io/apimachinery/pkg/labels"
3031
"k8s.io/apimachinery/pkg/runtime"
3132
"k8s.io/apimachinery/pkg/types"
33+
"k8s.io/apimachinery/pkg/util/sets"
3234
"k8s.io/client-go/tools/clientcmd"
3335
ctrl "sigs.k8s.io/controller-runtime"
3436
"sigs.k8s.io/controller-runtime/pkg/client"
@@ -78,7 +80,7 @@ func (r *ClusterBootstrapConfigReconciler) Reconcile(ctx context.Context, req ct
7880
}
7981
logger.Info("cluster bootstrap config loaded", "name", clusterBootstrapConfig.ObjectMeta.Name)
8082

81-
clusters, err := r.getClustersBySelector(ctx, req.Namespace, clusterBootstrapConfig.Spec)
83+
clusters, err := r.getClustersBySelector(ctx, req.Namespace, clusterBootstrapConfig)
8284
if err != nil {
8385
return ctrl.Result{}, fmt.Errorf("failed to getClustersBySelector for bootstrap config %s: %w", req, err)
8486
}
@@ -114,7 +116,8 @@ func (r *ClusterBootstrapConfigReconciler) Reconcile(ctx context.Context, req ct
114116
mergePatch, err := json.Marshal(map[string]interface{}{
115117
"metadata": map[string]interface{}{
116118
"annotations": map[string]interface{}{
117-
capiv1alpha1.BootstrappedAnnotation: "yes",
119+
capiv1alpha1.BootstrappedAnnotation: "yes",
120+
capiv1alpha1.BootstrapConfigsAnnotation: appendClusterConfigToBootstrappedList(clusterBootstrapConfig, cluster),
118121
},
119122
},
120123
})
@@ -128,6 +131,14 @@ func (r *ClusterBootstrapConfigReconciler) Reconcile(ctx context.Context, req ct
128131
return ctrl.Result{}, nil
129132
}
130133

134+
func appendClusterConfigToBootstrappedList(config capiv1alpha1.ClusterBootstrapConfig, cluster *gitopsv1alpha1.GitopsCluster) string {
135+
current := cluster.GetAnnotations()[capiv1alpha1.BootstrapConfigsAnnotation]
136+
set := sets.NewString(strings.Split(current, ",")...)
137+
id := fmt.Sprintf("%s/%s", config.GetNamespace(), config.GetName())
138+
set.Insert(id)
139+
return strings.Join(set.List(), ",")
140+
}
141+
131142
// SetupWithManager sets up the controller with the Manager.
132143
func (r *ClusterBootstrapConfigReconciler) SetupWithManager(mgr ctrl.Manager) error {
133144
return ctrl.NewControllerManagedBy(mgr).
@@ -139,9 +150,9 @@ func (r *ClusterBootstrapConfigReconciler) SetupWithManager(mgr ctrl.Manager) er
139150
Complete(r)
140151
}
141152

142-
func (r *ClusterBootstrapConfigReconciler) getClustersBySelector(ctx context.Context, ns string, spec capiv1alpha1.ClusterBootstrapConfigSpec) ([]*gitopsv1alpha1.GitopsCluster, error) {
153+
func (r *ClusterBootstrapConfigReconciler) getClustersBySelector(ctx context.Context, ns string, config capiv1alpha1.ClusterBootstrapConfig) ([]*gitopsv1alpha1.GitopsCluster, error) {
143154
logger := ctrl.LoggerFrom(ctx)
144-
selector, err := metav1.LabelSelectorAsSelector(&spec.ClusterSelector)
155+
selector, err := metav1.LabelSelectorAsSelector(&config.Spec.ClusterSelector)
145156
if err != nil {
146157
return nil, fmt.Errorf("unable to convert selector: %w", err)
147158
}
@@ -160,19 +171,21 @@ func (r *ClusterBootstrapConfigReconciler) getClustersBySelector(ctx context.Con
160171
for i := range clusterList.Items {
161172
cluster := &clusterList.Items[i]
162173

163-
if !conditions.IsReady(cluster) && !spec.RequireClusterProvisioned {
174+
if !conditions.IsReady(cluster) && !config.Spec.RequireClusterProvisioned {
164175
logger.Info("cluster discarded - not ready", "phase", cluster.Status)
165176
continue
166177
}
167-
if spec.RequireClusterProvisioned {
178+
if config.Spec.RequireClusterProvisioned {
168179
if !isProvisioned(cluster) {
169180
logger.Info("waiting for cluster to be provisioned", "cluster", cluster.Name)
170181
continue
171182
}
172183
}
173184

174185
if metav1.HasAnnotation(cluster.ObjectMeta, capiv1alpha1.BootstrappedAnnotation) {
175-
continue
186+
if alreadyBootstrappedWithConfig(cluster, config) {
187+
continue
188+
}
176189
}
177190
if cluster.DeletionTimestamp.IsZero() {
178191
clusters = append(clusters, cluster)
@@ -181,6 +194,13 @@ func (r *ClusterBootstrapConfigReconciler) getClustersBySelector(ctx context.Con
181194
return clusters, nil
182195
}
183196

197+
func alreadyBootstrappedWithConfig(cluster *gitopsv1alpha1.GitopsCluster, config capiv1alpha1.ClusterBootstrapConfig) bool {
198+
current := cluster.GetAnnotations()[capiv1alpha1.BootstrapConfigsAnnotation]
199+
set := sets.NewString(strings.Split(current, ",")...)
200+
id := fmt.Sprintf("%s/%s", config.GetNamespace(), config.GetName())
201+
return set.Has(id)
202+
}
203+
184204
// clusterToClusterBootstrapConfig is mapper function that maps clusters to
185205
// ClusterBootstrapConfig.
186206
func (r *ClusterBootstrapConfigReconciler) clusterToClusterBootstrapConfig(o client.Object) []ctrl.Request {

‎controllers/clusterbootstrapconfig_controller_test.go‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,100 @@ func TestReconcile_when_cluster_ready(t *testing.T) {
138138
}
139139
}
140140

141+
func TestReconcile_when_cluster_ready_bootstrapped_with_same_config(t *testing.T) {
142+
bc := makeTestClusterBootstrapConfig(func(c *capiv1alpha1.ClusterBootstrapConfig) {
143+
c.Spec.RequireClusterReady = true
144+
})
145+
readyNode := makeNode(map[string]string{
146+
"node-role.kubernetes.io/control-plane": "",
147+
}, corev1.NodeCondition{
148+
Type: "Ready", Status: "True", LastHeartbeatTime: metav1.Now(), LastTransitionTime: metav1.Now(), Reason: "KubeletReady", Message: "kubelet is posting ready status"})
149+
150+
cl := makeTestCluster(func(c *gitopsv1alpha1.GitopsCluster) {
151+
c.ObjectMeta.Labels = bc.Spec.ClusterSelector.MatchLabels
152+
c.Status.Conditions = append(c.Status.Conditions, makeReadyCondition())
153+
c.ObjectMeta.Annotations = map[string]string{
154+
capiv1alpha1.BootstrappedAnnotation: "true",
155+
capiv1alpha1.BootstrapConfigsAnnotation: fmt.Sprintf("%s/%s", bc.Namespace, bc.Name),
156+
}
157+
})
158+
secret := makeTestSecret(types.NamespacedName{
159+
Name: cl.GetName() + "-kubeconfig",
160+
Namespace: cl.GetNamespace(),
161+
}, map[string][]byte{"value": []byte("testing")})
162+
// This cheats by using the local client as the remote client to simplify
163+
// getting the value from the remote client.
164+
reconciler := makeTestReconciler(t, bc, cl, secret, readyNode)
165+
reconciler.configParser = func(b []byte) (client.Client, error) {
166+
return reconciler.Client, nil
167+
}
168+
169+
result, err := reconciler.Reconcile(context.TODO(), ctrl.Request{NamespacedName: types.NamespacedName{
170+
Name: bc.GetName(),
171+
Namespace: bc.GetNamespace(),
172+
}})
173+
if err != nil {
174+
t.Fatal(err)
175+
}
176+
if !result.IsZero() {
177+
t.Fatalf("want empty result, got %v", result)
178+
}
179+
var jobs batchv1.JobList
180+
if err := reconciler.List(context.TODO(), &jobs, client.InNamespace(testNamespace)); err != nil {
181+
t.Fatal(err)
182+
}
183+
if l := len(jobs.Items); l != 0 {
184+
t.Fatalf("found %d jobs, want %d", l, 0)
185+
}
186+
}
187+
188+
func TestReconcile_when_cluster_ready_bootstrapped_with_different_config(t *testing.T) {
189+
bc := makeTestClusterBootstrapConfig(func(c *capiv1alpha1.ClusterBootstrapConfig) {
190+
c.Spec.RequireClusterReady = true
191+
})
192+
readyNode := makeNode(map[string]string{
193+
"node-role.kubernetes.io/control-plane": "",
194+
}, corev1.NodeCondition{
195+
Type: "Ready", Status: "True", LastHeartbeatTime: metav1.Now(), LastTransitionTime: metav1.Now(), Reason: "KubeletReady", Message: "kubelet is posting ready status"})
196+
197+
cl := makeTestCluster(func(c *gitopsv1alpha1.GitopsCluster) {
198+
c.ObjectMeta.Labels = bc.Spec.ClusterSelector.MatchLabels
199+
c.ObjectMeta.Annotations = map[string]string{
200+
capiv1alpha1.BootstrappedAnnotation: "true",
201+
capiv1alpha1.BootstrapConfigsAnnotation: "unknown/unknown",
202+
}
203+
c.Status.Conditions = append(c.Status.Conditions, makeReadyCondition())
204+
})
205+
secret := makeTestSecret(types.NamespacedName{
206+
Name: cl.GetName() + "-kubeconfig",
207+
Namespace: cl.GetNamespace(),
208+
}, map[string][]byte{"value": []byte("testing")})
209+
// This cheats by using the local client as the remote client to simplify
210+
// getting the value from the remote client.
211+
reconciler := makeTestReconciler(t, bc, cl, secret, readyNode)
212+
reconciler.configParser = func(b []byte) (client.Client, error) {
213+
return reconciler.Client, nil
214+
}
215+
216+
result, err := reconciler.Reconcile(context.TODO(), ctrl.Request{NamespacedName: types.NamespacedName{
217+
Name: bc.GetName(),
218+
Namespace: bc.GetNamespace(),
219+
}})
220+
if err != nil {
221+
t.Fatal(err)
222+
}
223+
if !result.IsZero() {
224+
t.Fatalf("want empty result, got %v", result)
225+
}
226+
var jobs batchv1.JobList
227+
if err := reconciler.List(context.TODO(), &jobs, client.InNamespace(testNamespace)); err != nil {
228+
t.Fatal(err)
229+
}
230+
if l := len(jobs.Items); l != 1 {
231+
t.Fatalf("found %d jobs, want %d", l, 1)
232+
}
233+
}
234+
141235
func TestReconcile_when_cluster_provisioned(t *testing.T) {
142236
bc := makeTestClusterBootstrapConfig(func(c *capiv1alpha1.ClusterBootstrapConfig) {
143237
c.Spec.RequireClusterProvisioned = true
@@ -244,6 +338,56 @@ func TestReconcile_when_cluster_no_matching_labels(t *testing.T) {
244338
assertNoJobsCreated(t, reconciler.Client)
245339
}
246340

341+
func TestReconcile_when_cluster_ready_bootstrapped_with_multiple_config(t *testing.T) {
342+
// Multiple configs can bootstrap the same cluster
343+
// If the reconciled cluster is in that list (anywhere) then we don't create
344+
// jobs.
345+
bc := makeTestClusterBootstrapConfig(func(c *capiv1alpha1.ClusterBootstrapConfig) {
346+
c.Spec.RequireClusterReady = true
347+
})
348+
readyNode := makeNode(map[string]string{
349+
"node-role.kubernetes.io/control-plane": "",
350+
}, corev1.NodeCondition{
351+
Type: "Ready", Status: "True", LastHeartbeatTime: metav1.Now(), LastTransitionTime: metav1.Now(), Reason: "KubeletReady", Message: "kubelet is posting ready status"})
352+
353+
cl := makeTestCluster(func(c *gitopsv1alpha1.GitopsCluster) {
354+
c.ObjectMeta.Labels = bc.Spec.ClusterSelector.MatchLabels
355+
c.ObjectMeta.Annotations = map[string]string{
356+
capiv1alpha1.BootstrappedAnnotation: "true",
357+
capiv1alpha1.BootstrapConfigsAnnotation: fmt.Sprintf("%s,%s/%s", "unknown/unknown", bc.GetNamespace(), bc.GetName()),
358+
}
359+
c.Status.Conditions = append(c.Status.Conditions, makeReadyCondition())
360+
})
361+
secret := makeTestSecret(types.NamespacedName{
362+
Name: cl.GetName() + "-kubeconfig",
363+
Namespace: cl.GetNamespace(),
364+
}, map[string][]byte{"value": []byte("testing")})
365+
// This cheats by using the local client as the remote client to simplify
366+
// getting the value from the remote client.
367+
reconciler := makeTestReconciler(t, bc, cl, secret, readyNode)
368+
reconciler.configParser = func(b []byte) (client.Client, error) {
369+
return reconciler.Client, nil
370+
}
371+
372+
result, err := reconciler.Reconcile(context.TODO(), ctrl.Request{NamespacedName: types.NamespacedName{
373+
Name: bc.GetName(),
374+
Namespace: bc.GetNamespace(),
375+
}})
376+
if err != nil {
377+
t.Fatal(err)
378+
}
379+
if !result.IsZero() {
380+
t.Fatalf("want empty result, got %v", result)
381+
}
382+
var jobs batchv1.JobList
383+
if err := reconciler.List(context.TODO(), &jobs, client.InNamespace(testNamespace)); err != nil {
384+
t.Fatal(err)
385+
}
386+
if l := len(jobs.Items); l != 0 {
387+
t.Fatalf("found %d jobs, want %d", l, 0)
388+
}
389+
}
390+
247391
func TestReconcile_when_empty_label_selector(t *testing.T) {
248392
// When the label selector is empty, we don't want any jobs created, rather
249393
// than a job for all clusters.

0 commit comments

Comments
 (0)