MAJOR: crd: add job for custom resource definition handling

CRDs are not properly handled with external tools, Helm and similar
options cannot handle upgrading of custom resource definitions

Author: oktalz

.gitignore

@@ -4,3 +4,4 @@ kubernetes-ingress
 dist/
 .code-generator/
 bin/golangci-lint
+.local/*

crs/definition/backend.yaml renamed to crs/definition/backends.core.haproxy.org.yaml

@@ -0,0 +1,37 @@
+package definition
+
+import _ "embed"
+
+//go:embed defaults.core.haproxy.org.yaml
+var DefaultsV1alpha2 []byte
+
+//go:embed globals.core.haproxy.org.yaml
+var GlobalsV1alpha2 []byte
+
+//go:embed backends.core.haproxy.org.yaml
+var BackendsV1alpha2 []byte
+
+//go:embed upgrade/defaults.core.haproxy.org.yaml
+var DefaultsV1alpha1V1alpha2 []byte
+
+//go:embed upgrade/globals.core.haproxy.org.yaml
+var GlobalsV1alpha1V1alpha2 []byte
+
+//go:embed upgrade/backends.core.haproxy.org.yaml
+var BackendsV1alpha1V1alpha2 []byte
+
+func GetCRDs() map[string][]byte {
+	return map[string][]byte{
+		"defaults.core.haproxy.org": DefaultsV1alpha2,
+		"globals.core.haproxy.org":  GlobalsV1alpha2,
+		"backends.core.haproxy.org": BackendsV1alpha2,
+	}
+}
+
+func GetCRDsUpgrade() map[string][]byte {
+	return map[string][]byte{
+		"defaults.core.haproxy.org": DefaultsV1alpha1V1alpha2,
+		"globals.core.haproxy.org":  GlobalsV1alpha1V1alpha2,
+		"backends.core.haproxy.org": BackendsV1alpha1V1alpha2,
+	}
+}

crs/definition/defaults.yaml renamed to crs/definition/defaults.core.haproxy.org.yaml

@@ -0,0 +1,16 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: haproxy-ingress-crd # each deploymnent should have a unique name, in example we always recreate the custer
+  namespace: haproxy-controller
+spec:
+  template:
+    spec:
+      serviceAccountName: haproxy-kubernetes-ingress-crd
+      containers:
+      - name: haproxy-ingress-crd
+        image: haproxytech/kubernetes-ingress:latest
+        imagePullPolicy: Never
+        command: ["./haproxy-ingress-controller","--job-check-crd"]
+      restartPolicy: Never
+  backoffLimit: 0

crs/definition/embed.go

@@ -0,0 +1,31 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: haproxy-kubernetes-ingress-crd
+  namespace: haproxy-controller
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: haproxy-kubernetes-ingress-crd
+rules:
+  - apiGroups:
+      - "apiextensions.k8s.io"
+    resources:
+      - customresourcedefinitions
+    verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: haproxy-kubernetes-ingress-crd
+  namespace: haproxy-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: haproxy-kubernetes-ingress-crd
+subjects:
+  - kind: ServiceAccount
+    name: haproxy-kubernetes-ingress-crd
+    namespace: haproxy-controller

crs/definition/global.yaml renamed to crs/definition/globals.core.haproxy.org.yaml

@@ -12,7 +12,6 @@ if [ -n "${CI_ENV}" ]; then
   echo "building image for ingress controller"
   if [ -n "${GITLAB_CI}" ]; then
     echo "haproxytech/kubernetes-ingress image already available from previous stage"
-    #docker build --build-arg TARGETPLATFORM="linux/amd64" -t haproxytech/kubernetes-ingress -f build/Dockerfile .
   else
     docker build --build-arg TARGETPLATFORM="linux/amd64" -t haproxytech/kubernetes-ingress -f build/Dockerfile .
   fi
@@ -46,11 +45,14 @@ fi
 echo "loading image http-echo in kind"
 kind load docker-image haproxytech/http-echo:latest  --name=$clustername
 
+printf %80s |tr " " "="; echo ""
+echo "Create HAProxy namespace ..."
+kubectl apply -f $DIR/config/0.namespace.yaml
 printf %80s |tr " " "="; echo ""
 echo "Install custom resource definitions ..."
-kubectl apply -f $DIR/../../crs/definition/backend.yaml
-kubectl apply -f $DIR/../../crs/definition/defaults.yaml
-kubectl apply -f $DIR/../../crs/definition/global.yaml
+kubectl apply -f $DIR/../../deploy/tests/config/crd/rbac.yaml
+kubectl apply -f $DIR/../../deploy/tests/config/crd/job-crd.yaml
+kubectl wait --for=condition=complete --timeout=60s job/haproxy-ingress-crd -n haproxy-controller
 
 if [ "$EXPERIMENTAL_GWAPI" = "1" ]; then
   printf %80s |tr " " "="; echo ""
@@ -80,7 +82,6 @@ fi
 
 printf %80s |tr " " "="; echo ""
 echo "deploying Ingress Controller ..."
-kubectl apply -f $DIR/config/0.namespace.yaml
 kubectl apply -f $DIR/config/1.rbac.yaml
 if [ "$EXPERIMENTAL_GWAPI" = "1" ]; then
   printf %80s |tr " " "="; echo ""

crs/definition/upgrade/backend.yaml renamed to crs/definition/upgrade/backends.core.haproxy.org.yaml

@@ -21,6 +21,7 @@ require (
 	k8s.io/client-go v0.28.1
 	sigs.k8s.io/controller-runtime v0.16.2
 	sigs.k8s.io/gateway-api v0.5.0
+	sigs.k8s.io/yaml v1.4.0
 )
 
 require (
@@ -87,6 +88,5 @@ require (
 	k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
 	k8s.io/utils v0.0.0-20230505201702-9f6742963106 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect
-	sigs.k8s.io/yaml v1.3.0 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 )

crs/definition/upgrade/defaults.yaml renamed to crs/definition/upgrade/defaults.core.haproxy.org.yaml

@@ -489,7 +489,7 @@ sigs.k8s.io/gateway-api v0.5.0 h1:ze+k9fJqvmL8s1t3e4q1ST8RnN+f09dEv+gfacahlAE=
 sigs.k8s.io/gateway-api v0.5.0/go.mod h1:x0AP6gugkFV8fC/oTlnOMU0pnmuzIR8LfIPRVUjxSqA=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
-sigs.k8s.io/structured-merge-diff/v4 v4.3.0 h1:UZbZAZfX0wV2zr7YZorDz6GXROfDFj6LvqCRm4VUVKk=
-sigs.k8s.io/structured-merge-diff/v4 v4.3.0/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=
-sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
-sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
+sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4=
+sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=
+sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
+sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=

crs/definition/upgrade/global.yaml renamed to crs/definition/upgrade/globals.core.haproxy.org.yaml

@@ -34,6 +34,7 @@ import (
 
 	"github.com/haproxytech/kubernetes-ingress/pkg/annotations"
 	"github.com/haproxytech/kubernetes-ingress/pkg/controller"
+	"github.com/haproxytech/kubernetes-ingress/pkg/job"
 	"github.com/haproxytech/kubernetes-ingress/pkg/k8s"
 	"github.com/haproxytech/kubernetes-ingress/pkg/store"
 	"github.com/haproxytech/kubernetes-ingress/pkg/utils"
@@ -70,6 +71,22 @@ func main() {
 		parser.WriteHelp(os.Stdout)
 		return
 	}
+	if osArgs.JobCheckCRD {
+		logger.Print(IngressControllerInfo)
+		logger.Print(job.IngressControllerCRDUpdater)
+		logger.Infof("HAProxy Ingress Controller CRD Updater %s %s%s", GitTag, GitCommit, GitDirty)
+		logger.Infof("Build from: %s", GitRepo)
+		logger.Infof("Build date: %s\n", GitCommitDate)
+
+		err := job.CRDRefresh(logger, osArgs)
+		if err != nil {
+			logger.Error(err)
+			os.Exit(1) //nolint:gocritic
+		}
+		// exit, this is just a job
+		os.Exit(0)
+	}
+
 	logger.ShowFilename(false)
 	exit := logInfo(logger, osArgs)
 	if exit {

deploy/tests/config/crd/job-crd.yaml

@@ -0,0 +1,103 @@
+// Copyright 2023 HAProxy Technologies LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package job
+
+import (
+	"context"
+
+	"github.com/haproxytech/kubernetes-ingress/crs/definition"
+	"github.com/haproxytech/kubernetes-ingress/pkg/k8s"
+	"github.com/haproxytech/kubernetes-ingress/pkg/utils"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	apiError "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/yaml"
+)
+
+func CRDRefresh(log utils.Logger, osArgs utils.OSArgs) error {
+	log.Info("checking CRDS")
+	config, err := k8s.GetRestConfig(osArgs)
+	if err != nil {
+		return err
+	}
+
+	// Create a new clientset for the apiextensions API group
+	clientset := apiextensionsclientset.NewForConfigOrDie(config)
+
+	// Check if the CRD exists
+	crds := definition.GetCRDs()
+	crdsUpgrade := definition.GetCRDsUpgrade()
+	for crdName, crdDef := range crds {
+		// CustomResourceDefinition object
+		var crd apiextensionsv1.CustomResourceDefinition
+		err = yaml.Unmarshal(crdDef, &crd)
+		if err != nil {
+			return err
+		}
+		log.Info("")
+		log.Infof("checking CRD %s", crdName)
+
+		existingVersion, err := clientset.ApiextensionsV1().CustomResourceDefinitions().Get(context.Background(), crdName, metav1.GetOptions{})
+		if err != nil {
+			if !apiError.IsNotFound(err) {
+				return err
+			}
+			log.Infof("CRD %s does not exist", crdName)
+			// Create the CRD
+			_, err = clientset.ApiextensionsV1().CustomResourceDefinitions().Create(context.Background(), &crd, metav1.CreateOptions{})
+			if err != nil {
+				return err
+			}
+			log.Infof("CRD %s created", crdName)
+			continue
+		}
+		log.Infof("CRD %s exists", crdName)
+		versions := existingVersion.Spec.Versions
+		if len(versions) == 2 {
+			log.Infof("CRD %s exists as v1alpha1 and v1alpha2, nothing to do", crdName)
+			continue
+		}
+		// check if we have alpha 2 or we need to upgrade for alpha2
+		crd.ObjectMeta.ResourceVersion = existingVersion.ObjectMeta.ResourceVersion
+		if versions[0].Name == "v1alpha2" {
+			log.Infof("CRD %s exists as v1alpha2, nothing to do", crdName)
+			continue
+		}
+		err = yaml.Unmarshal(crdsUpgrade[crdName], &crd)
+		if err != nil {
+			return err
+		}
+		// Upgrade the CRDl
+		_, err = clientset.ApiextensionsV1().CustomResourceDefinitions().Update(context.Background(), &crd, metav1.UpdateOptions{})
+		if err != nil {
+			return err
+		}
+	}
+
+	log.Info("")
+	log.Info("CRD update done")
+	return nil
+}
+
+// IngressControllerCRDUpdater console pretty print
+const IngressControllerCRDUpdater = `
+  ____ ____  ____    _   _           _       _
+ / ___|  _ \|  _ \  | | | |_ __   __| | __ _| |_ ___ _ __
+| |   | |_) | | | | | | | | '_ \ / _` + "`" + ` |/ _` + "`" + ` | __/ _ \ '__|
+| |___|  _ <| |_| | | |_| | |_) | (_| | (_| | ||  __/ |
+ \____|_| \_\____/   \___/| .__/ \__,_|\__,_|\__\___|_|
+                          |_|
+
+`

deploy/tests/config/crd/rbac.yaml

@@ -0,0 +1,106 @@
+// Copyright 2019 HAProxy Technologies LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package k8s
+
+import (
+	"strings"
+	"time"
+
+	"k8s.io/client-go/tools/cache"
+
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	// apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	apiextensionsinformers "k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions"
+)
+
+func (k k8s) runCRDefinitionsInformer(eventChan chan string, stop chan struct{}) { //nolint:ireturn
+	// Create a new informer factory with the clientset.
+
+	factory := apiextensionsinformers.NewSharedInformerFactoryWithOptions(k.apiExtensionsClient, k.cacheResyncPeriod)
+	informer := factory.Apiextensions().V1().CustomResourceDefinitions().Informer()
+	_, err := informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
+		AddFunc: func(obj interface{}) {
+			crd := obj.(*apiextensionsv1.CustomResourceDefinition)
+			if crd.Spec.Group != "core.haproxy.org" {
+				return
+			}
+			if !(crd.Spec.Names.Kind == "Global" || crd.Spec.Names.Kind == "Defaults" || crd.Spec.Names.Kind == "Backend") {
+				return
+			}
+			for _, version := range crd.Spec.Versions {
+				if version.Name == "v1alpha2" {
+					time.Sleep(time.Second * 5) // a little delay is needed to let CRD API be created
+					eventChan <- crd.Spec.Names.Kind
+					return
+				}
+			}
+		},
+	})
+
+	go informer.Run(stop)
+
+	if !cache.WaitForCacheSync(stop, informer.HasSynced) {
+		logger.Error("Caches are not populated due to an underlying error, cannot monitor CRS creation")
+	}
+
+	logger.Error(err)
+}
+
+func (k k8s) RunCRSCreationMonitoring(eventChan chan SyncDataEvent, stop chan struct{}) {
+	count := 0
+	for key := range k.crs {
+		if strings.Contains(key, "core.haproxy.org/v1alpha2") {
+			count++
+		}
+	}
+	if count > 2 {
+		// all crds are already in list
+		return
+	}
+
+	eventCRS := make(chan string)
+	k.runCRDefinitionsInformer(eventCRS, stop)
+	go func(chan string) {
+		for {
+			select {
+			case crdName := <-eventCRS:
+				if _, ok := k.crs["core.haproxy.org/v1alpha2 - "+crdName]; ok {
+					// we have already created watchers for this CRD
+					continue
+				}
+				informersSyncedEvent := &[]cache.InformerSynced{}
+				for _, namespace := range k.whiteListedNS {
+					crs := map[string]CR{}
+					switch crdName {
+					case "Backend":
+						crs[crdName] = NewBackendCR()
+					case "Defaults":
+						crs[crdName] = NewDefaultsCR()
+					case "Global":
+						crs[crdName] = NewGlobalCR()
+					}
+					logger.Info("Custom resource definition created, adding CR watcher for " + crs[crdName].GetKind())
+					k.runCRInformers(eventChan, stop, namespace, informersSyncedEvent, crs)
+				}
+
+				if !cache.WaitForCacheSync(stop, *informersSyncedEvent...) {
+					logger.Error("Caches are not populated due to an underlying error, cannot monitor new CRDs")
+				}
+			case <-stop:
+				return
+			}
+		}
+	}(eventCRS)
+}