diff --git a/api/v1alpha2/zz_generated.conversion.go b/api/v1alpha2/zz_generated.conversion.go index 12cb30171423..dcb31e9bd5b8 100644 --- a/api/v1alpha2/zz_generated.conversion.go +++ b/api/v1alpha2/zz_generated.conversion.go @@ -439,6 +439,7 @@ func autoConvert_v1alpha2_ClusterSpec_To_v1alpha3_ClusterSpec(in *ClusterSpec, o func autoConvert_v1alpha3_ClusterSpec_To_v1alpha2_ClusterSpec(in *v1alpha3.ClusterSpec, out *ClusterSpec, s conversion.Scope) error { // WARNING: in.Paused requires manual conversion: does not exist in peer-type + // WARNING: in.PostApplyAddons requires manual conversion: does not exist in peer-type out.ClusterNetwork = (*ClusterNetwork)(unsafe.Pointer(in.ClusterNetwork)) // WARNING: in.ControlPlaneEndpoint requires manual conversion: does not exist in peer-type // WARNING: in.ControlPlaneRef requires manual conversion: does not exist in peer-type diff --git a/api/v1alpha3/cluster_types.go b/api/v1alpha3/cluster_types.go index 34c5d8312213..94bb8b0a6e5b 100644 --- a/api/v1alpha3/cluster_types.go +++ b/api/v1alpha3/cluster_types.go @@ -18,7 +18,6 @@ package v1alpha3 import ( "fmt" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" capierrors "sigs.k8s.io/cluster-api/errors" @@ -26,6 +25,9 @@ import ( const ( ClusterFinalizer = "cluster.cluster.x-k8s.io" + + // PostApplyAnnotationPrefix is the prefix of the annotations that is used to determine if an addon is applied or not. + PostApplyAnnotationPrefix = "cluster.x-k8s.io/postapply-secret" ) // ANCHOR: ClusterSpec @@ -36,6 +38,9 @@ type ClusterSpec struct { // +optional Paused bool `json:"paused,omitempty"` + // PostApplyAddons is a list of Secrets in YAML format to be applied to remote clusters. + PostApplyAddons []PostApplyAddon `json:"postApplyAddons,omitempty"` + // Cluster network configuration. // +optional ClusterNetwork *ClusterNetwork `json:"clusterNetwork,omitempty"` @@ -57,6 +62,18 @@ type ClusterSpec struct { // ANCHOR_END: ClusterSpec +// ANCHOR: PostApplyAddon + +// PostApplyAddon specifies the addon's Secret parameters. +type PostApplyAddon struct { + // Name is the name of the secret. + Name string `json:"name,omitempty"` + // Namespace is the namespace of the secret. + Namespace string `json:"namespace,omitempty"` +} + +// ANCHOR_END: PostApplyAddon + // ANCHOR: ClusterNetwork // ClusterNetwork specifies the different networking diff --git a/api/v1alpha3/zz_generated.deepcopy.go b/api/v1alpha3/zz_generated.deepcopy.go index ec11404f2a95..4f7d2d628f1b 100644 --- a/api/v1alpha3/zz_generated.deepcopy.go +++ b/api/v1alpha3/zz_generated.deepcopy.go @@ -165,6 +165,11 @@ func (in *ClusterNetwork) DeepCopy() *ClusterNetwork { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { *out = *in + if in.PostApplyAddons != nil { + in, out := &in.PostApplyAddons, &out.PostApplyAddons + *out = make([]PostApplyAddon, len(*in)) + copy(*out, *in) + } if in.ClusterNetwork != nil { in, out := &in.ClusterNetwork, &out.ClusterNetwork *out = new(ClusterNetwork) @@ -1002,6 +1007,21 @@ func (in *ObjectMeta) DeepCopy() *ObjectMeta { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PostApplyAddon) DeepCopyInto(out *PostApplyAddon) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PostApplyAddon. +func (in *PostApplyAddon) DeepCopy() *PostApplyAddon { + if in == nil { + return nil + } + out := new(PostApplyAddon) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *UnhealthyCondition) DeepCopyInto(out *UnhealthyCondition) { *out = *in diff --git a/config/crd/bases/cluster.x-k8s.io_clusters.yaml b/config/crd/bases/cluster.x-k8s.io_clusters.yaml index 021aea5a69d5..24962cc36f64 100644 --- a/config/crd/bases/cluster.x-k8s.io_clusters.yaml +++ b/config/crd/bases/cluster.x-k8s.io_clusters.yaml @@ -185,6 +185,22 @@ spec: spec: description: ClusterSpec defines the desired state of Cluster properties: + postApplyAddons: + description: PostApplyAddons is the list of addons to be applied after cluster is created + items: + description: PostApplyAddon represents a secret for post apply. + properties: + name: + description: 'Name of the PostApply addon secret' + type: string + namespace: + description: 'Namespace of the PostApply addon secret' + type: string + required: + - name + - namespace + type: object + type: array clusterNetwork: description: Cluster network configuration. properties: diff --git a/controllers/cluster_controller.go b/controllers/cluster_controller.go index 4b98b77f989c..d18cec3a465d 100644 --- a/controllers/cluster_controller.go +++ b/controllers/cluster_controller.go @@ -153,6 +153,7 @@ func (r *ClusterReconciler) reconcile(ctx context.Context, cluster *clusterv1.Cl r.reconcileControlPlane(ctx, cluster), r.reconcileKubeconfig(ctx, cluster), r.reconcileControlPlaneInitialized(ctx, cluster), + r.reconcilePostApply(ctx, cluster), } // Parse the errors, making sure we record if there is a RequeueAfterError. diff --git a/controllers/cluster_controller_postapply.go b/controllers/cluster_controller_postapply.go new file mode 100644 index 000000000000..83b6638ed603 --- /dev/null +++ b/controllers/cluster_controller_postapply.go @@ -0,0 +1,199 @@ +package controllers + +import ( + "bufio" + "bytes" + "context" + "fmt" + "github.com/pkg/errors" + "io" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + utilyaml "k8s.io/apimachinery/pkg/util/yaml" + clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" + "sigs.k8s.io/cluster-api/controllers/remote" + "sigs.k8s.io/cluster-api/util/secret" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" +) + +var ( + errUninitializedControlPlane = errors.New("control plane is uninitialized") +) + +func (r *ClusterReconciler) reconcilePostApply(ctx context.Context, cluster *clusterv1.Cluster) error { + logger := r.Log.WithValues("cluster", cluster.Name, "namespace", cluster.Namespace) + + if !cluster.Status.ControlPlaneInitialized { + return errUninitializedControlPlane + } + + c, err := remote.NewClusterClient(ctx, r.Client, cluster, r.scheme) + // Failed to get remote cluster client: Kubeconfig secret may be missing for the cluster. + if err != nil { + return err + } + + addons := cluster.Spec.PostApplyAddons + for _, addon := range addons { + annotations := cluster.GetAnnotations() + PostApplyAnnotation := clusterv1.PostApplyAnnotationPrefix + "-" + addon.Namespace + "-" + addon.Name + // Do not apply if the current addon has already been applied. + if _, exists := annotations[PostApplyAnnotation]; exists { + continue + } + logger.Info("Trying to post-apply ", "secret", addon.Name) + + typedName := types.NamespacedName{Name: addon.Name, Namespace: addon.Namespace} + + addonSecret, err := secret.GetAnySecretFromNamespacedName(ctx, r.Client, typedName) + if err != nil { + logger.Error(err, "Failed to fetch PostApply addon secret", "Secret", addon.Namespace+"/"+addon.Name) + continue + } + + data, ok := addonSecret.Data[secret.PostApplyDataKey] + if !ok { + err = errors.New(fmt.Sprintf("Missing key %q in secret %q", secret.PostApplyDataKey, addon.Name)) + logger.Error(err, "Failed to retrieve PostApply data") + continue + } + err = ApplyYAMLWithNamespace(ctx, c, data, "") + if err != nil { + logger.Error(err, "Failed applying PostApply secret to cluster") + continue + } + + logger.Info("Successfully applied post-apply addon", "Secret", addon.Namespace+"/"+addon.Name) + + cluster.GetAnnotations() + + if annotations == nil { + annotations = make(map[string]string) + } + annotations[PostApplyAnnotation] = time.Now().Format(time.RFC3339) + } + return nil +} + +// ApplyYAMLWithNamespace applies the provided YAML as unstructured data with the given client. +// The data may be a single YAML document or multidoc YAML. This function is idempotent. +// When a non-empty namespace is provided then all objects are assigned the namespace prior to being created. +func ApplyYAMLWithNamespace(ctx context.Context, c client.Client, data []byte, namespace string) error { + return ForEachObjectInYAML(ctx, c, data, namespace, func(ctx context.Context, c client.Client, obj *unstructured.Unstructured) error { + // Create the object on the API server. + if err := c.Create(ctx, obj); err != nil { + // The create call is idempotent, so if the object already exists + // then do not consider it to be an error. + if !apierrors.IsAlreadyExists(err) { + return errors.Wrapf( + err, + "failed to create object %s %s/%s", + obj.GroupVersionKind(), + obj.GetNamespace(), + obj.GetName()) + } + } + return nil + }) +} + +// ForEachObjectInYAMLActionFunc is a function that is executed against each +// object found in a YAML document. +// When a non-empty namespace is provided then the object is assigned the +// namespace prior to any other actions being performed with or to the object. +type ForEachObjectInYAMLActionFunc func(context.Context, client.Client, *unstructured.Unstructured) error + +// ForEachObjectInYAML excutes actionFn for each object in the provided YAML. +// If an error is returned then no further objects are processed. +// The data may be a single YAML document or multidoc YAML. +// When a non-empty namespace is provided then all objects are assigned the +// the namespace prior to any other actions being performed with or to the +// object. +func ForEachObjectInYAML( + ctx context.Context, + c client.Client, + data []byte, + namespace string, + actionFn ForEachObjectInYAMLActionFunc) error { + + chanObj, chanErr := DecodeYAML(data) + for { + select { + case obj := <-chanObj: + if obj == nil { + return nil + } + if namespace != "" { + obj.SetNamespace(namespace) + } + if err := actionFn(ctx, c, obj); err != nil { + return err + } + case err := <-chanErr: + if err == nil { + return nil + } + return errors.Wrap(err, "received error while decoding yaml to delete from server") + } + } +} + +// DecodeYAML unmarshals a YAML document or multidoc YAML as unstructured +// objects, placing each decoded object into a channel. +func DecodeYAML(data []byte) (<-chan *unstructured.Unstructured, <-chan error) { + + var ( + chanErr = make(chan error) + chanObj = make(chan *unstructured.Unstructured) + multidocReader = utilyaml.NewYAMLReader(bufio.NewReader(bytes.NewReader(data))) + ) + + go func() { + defer close(chanErr) + defer close(chanObj) + + // Iterate over the data until Read returns io.EOF. Every successful + // read returns a complete YAML document. + for { + buf, err := multidocReader.Read() + if err != nil { + if err == io.EOF { + return + } + chanErr <- errors.Wrap(err, "failed to read yaml data") + return + } + + // Do not use this YAML doc if it is unkind. + var typeMeta runtime.TypeMeta + if err := yaml.Unmarshal(buf, &typeMeta); err != nil { + continue + } + if typeMeta.Kind == "" { + continue + } + + // Define the unstructured object into which the YAML document will be + // unmarshaled. + obj := &unstructured.Unstructured{ + Object: map[string]interface{}{}, + } + + // Unmarshal the YAML document into the unstructured object. + if err := yaml.Unmarshal(buf, &obj.Object); err != nil { + chanErr <- errors.Wrap(err, "failed to unmarshal yaml data") + return + } + + // Place the unstructured object into the channel. + chanObj <- obj + } + }() + + return chanObj, chanErr +} diff --git a/util/secret/consts.go b/util/secret/consts.go index cf3463083b9d..4ade81945b7d 100644 --- a/util/secret/consts.go +++ b/util/secret/consts.go @@ -46,4 +46,7 @@ const ( // APIServerEtcdClient is the secret name of user-supplied secret containing the apiserver-etcd-client key/cert APIServerEtcdClient Purpose = "apiserver-etcd-client" + + // PostApplyDataKey is the key used to store a PostApply addon in the secret's data field. + PostApplyDataKey = "addon.yaml" ) diff --git a/util/secret/secret.go b/util/secret/secret.go index abbee1fb7856..b699348619b8 100644 --- a/util/secret/secret.go +++ b/util/secret/secret.go @@ -53,3 +53,19 @@ func GetFromNamespacedName(ctx context.Context, c client.Client, clusterName typ func Name(cluster string, suffix Purpose) string { return fmt.Sprintf("%s-%s", cluster, suffix) } + +// GetAnySecretFromNamespacedName retrieves any Secret from the given +// secret name and namespace. +func GetAnySecretFromNamespacedName(ctx context.Context, c client.Client, secretName types.NamespacedName) (*corev1.Secret, error) { + secret := &corev1.Secret{} + secretKey := client.ObjectKey{ + Namespace: secretName.Namespace, + Name: secretName.Name, + } + + if err := c.Get(ctx, secretKey, secret); err != nil { + return nil, err + } + + return secret, nil +} \ No newline at end of file