MEDIUM: delete certificates through runtime

hdurand0710 · oktalz · commit 350c5074a188 · 2025-02-19T10:58:57.000Z
Creation of certificates through runtime is already done. Adding now the deletion though runtime
diff --git a/crs/api/ingress/v1/backend.go b/crs/api/ingress/v1/backend.go
@@ -22,7 +22,7 @@ import (
 
 // +genclient
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
-// +kubebuilder:metadata:annotations="haproxy.org/client-native=v5.1.15-0.20241106115115-75c12953fe1f"
+// +kubebuilder:metadata:annotations="haproxy.org/client-native=v5.1.16-0.20250120132445-428abe2a833d"
 
 // Backend is a specification for a Backend resource
 type Backend struct {
diff --git a/crs/api/ingress/v1/defaults.go b/crs/api/ingress/v1/defaults.go
@@ -22,7 +22,7 @@ import (
 
 // +genclient
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
-// +kubebuilder:metadata:annotations="haproxy.org/client-native=v5.1.15-0.20241106115115-75c12953fe1f"
+// +kubebuilder:metadata:annotations="haproxy.org/client-native=v5.1.16-0.20250120132445-428abe2a833d"
 
 // Defaults is a specification for a Defaults resource
 type Defaults struct {
diff --git a/crs/api/ingress/v1/global.go b/crs/api/ingress/v1/global.go
@@ -22,7 +22,7 @@ import (
 
 // +genclient
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
-// +kubebuilder:metadata:annotations="haproxy.org/client-native=v5.1.15-0.20241106115115-75c12953fe1f"
+// +kubebuilder:metadata:annotations="haproxy.org/client-native=v5.1.16-0.20250120132445-428abe2a833d"
 // +kubebuilder:validation:XValidation:rule="!has(self.spec.config.default_path)", message="spec.config.default_path is set by ingress controller internally"
 // +kubebuilder:validation:XValidation:rule="!has(self.spec.config.master__dash__worker)", message="spec.config.master-worker is set by ingress controller internally"
 // +kubebuilder:validation:XValidation:rule="!has(self.spec.config.pidfile)", message="spec.config.pidfile is set by ingress controller internally"
diff --git a/crs/api/ingress/v1/tcp.go b/crs/api/ingress/v1/tcp.go
@@ -23,7 +23,7 @@ import (
 
 // +genclient
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
-// +kubebuilder:metadata:annotations="haproxy.org/client-native=v5.1.15-0.20241106115115-75c12953fe1f"
+// +kubebuilder:metadata:annotations="haproxy.org/client-native=v5.1.16-0.20250120132445-428abe2a833d"
 
 // TCP is a specification for a TCP resource
 type TCP struct {
diff --git a/crs/definition/ingress.v1.haproxy.org_backends.yaml b/crs/definition/ingress.v1.haproxy.org_backends.yaml
@@ -4,7 +4,7 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.17.0
-    haproxy.org/client-native: v5.1.15-0.20241106115115-75c12953fe1f
+    haproxy.org/client-native: v5.1.16-0.20250120132445-428abe2a833d
   name: backends.ingress.v1.haproxy.org
 spec:
   group: ingress.v1.haproxy.org
diff --git a/crs/definition/ingress.v1.haproxy.org_defaults.yaml b/crs/definition/ingress.v1.haproxy.org_defaults.yaml
@@ -4,7 +4,7 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.17.0
-    haproxy.org/client-native: v5.1.15-0.20241106115115-75c12953fe1f
+    haproxy.org/client-native: v5.1.16-0.20250120132445-428abe2a833d
   name: defaults.ingress.v1.haproxy.org
 spec:
   group: ingress.v1.haproxy.org
diff --git a/crs/definition/ingress.v1.haproxy.org_globals.yaml b/crs/definition/ingress.v1.haproxy.org_globals.yaml
@@ -4,7 +4,7 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.17.0
-    haproxy.org/client-native: v5.1.15-0.20241106115115-75c12953fe1f
+    haproxy.org/client-native: v5.1.16-0.20250120132445-428abe2a833d
   name: globals.ingress.v1.haproxy.org
 spec:
   group: ingress.v1.haproxy.org
diff --git a/crs/definition/ingress.v1.haproxy.org_tcps.yaml b/crs/definition/ingress.v1.haproxy.org_tcps.yaml
@@ -4,7 +4,7 @@ kind: CustomResourceDefinition
 metadata:
   annotations:
     controller-gen.kubebuilder.io/version: v0.17.0
-    haproxy.org/client-native: v5.1.15-0.20241106115115-75c12953fe1f
+    haproxy.org/client-native: v5.1.16-0.20250120132445-428abe2a833d
   name: tcps.ingress.v1.haproxy.org
 spec:
   group: ingress.v1.haproxy.org
diff --git a/go.mod b/go.mod
@@ -10,7 +10,7 @@ require (
 	github.com/google/go-cmp v0.6.0
 	github.com/google/renameio v1.0.1
 	github.com/haproxytech/client-native/v3 v3.1.2-0.20230607075433-231591da68ed
-	github.com/haproxytech/client-native/v5 v5.1.15-0.20241106115115-75c12953fe1f
+	github.com/haproxytech/client-native/v5 v5.1.16-0.20250120132445-428abe2a833d
 	github.com/jessevdk/go-flags v1.4.0
 	github.com/pires/go-proxyproto v0.8.0
 	github.com/prometheus/client_golang v1.20.5
@@ -78,6 +78,7 @@ require (
 	golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 // indirect
 	golang.org/x/net v0.34.0 // indirect
 	golang.org/x/oauth2 v0.25.0 // indirect
+	golang.org/x/sync v0.10.0 // indirect
 	golang.org/x/sys v0.29.0 // indirect
 	golang.org/x/term v0.28.0 // indirect
 	golang.org/x/text v0.21.0 // indirect
diff --git a/go.sum b/go.sum
@@ -72,8 +72,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/haproxytech/client-native/v3 v3.1.2-0.20230607075433-231591da68ed h1:qjKxpoe4wYQmpGrrooqau88Fgusy9VQfulpXghBDqx8=
 github.com/haproxytech/client-native/v3 v3.1.2-0.20230607075433-231591da68ed/go.mod h1:xRVluo27FAjK4ag33+jUfpnNf9olXQlTRHUl3VMvq98=
-github.com/haproxytech/client-native/v5 v5.1.15-0.20241106115115-75c12953fe1f h1:v0eQBUs3g70gPHWpP4xBb7wP1qLSs3OCFdDTP/vGbww=
-github.com/haproxytech/client-native/v5 v5.1.15-0.20241106115115-75c12953fe1f/go.mod h1:dhwpFkOsWxJRtwNs+LTJtsUTf9pX3uWHKKip9loIK3c=
+github.com/haproxytech/client-native/v5 v5.1.16-0.20250120132445-428abe2a833d h1:CSIkA9flJLOYabKfDR7DUJ8qL+SkWh1rqXXjBqctWn4=
+github.com/haproxytech/client-native/v5 v5.1.16-0.20250120132445-428abe2a833d/go.mod h1:/ms7QkqUYwCBm31zYl/qT/0r/TBDRt60pusERVZ2j0Q=
 github.com/haproxytech/go-logger v1.1.0 h1:HgGtYaI1ApkvbQdsm7f9AzQQoxTB7w37criTflh7IQE=
 github.com/haproxytech/go-logger v1.1.0/go.mod h1:OekUd8HCb7ubxMplzHUPBTHNxZmddOWfOjWclZsqIeM=
 github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA=
@@ -178,6 +178,8 @@ golang.org/x/oauth2 v0.25.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbht
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
+golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
diff --git a/pkg/handler/refresh.go b/pkg/handler/refresh.go
@@ -32,7 +32,7 @@ func (handler Refresh) Update(k store.K8s, h haproxy.HAProxy, a annotations.Anno
 	}
 	// Certs
 	if cleanCrts {
-		h.RefreshCerts()
+		h.RefreshCerts(h.HAProxyClient)
 	}
 	// Rules
 	h.RefreshRules(h.HAProxyClient)
diff --git a/pkg/haproxy/api/api.go b/pkg/haproxy/api/api.go
@@ -102,11 +102,17 @@ type HAProxyClient interface { //nolint:interfacebloat
 	UserListDeleteAll() error
 	UserListExistsByGroup(group string) (bool, error)
 	UserListCreateByGroup(group string, userPasswordMap map[string][]byte) error
+	Cert
+}
+
+type Cert interface {
 	CertEntryCreate(filename string) error
 	CertEntrySet(filename string, payload []byte) error
 	CertEntryCommit(filename string) error
 	CertEntryAbort(filename string) error
 	CrtListEntryAdd(crtList string, entry runtime.CrtListEntry) error
+	CrtListEntryDelete(crtList, filename string, linenumber *int64) error
+	CertEntryDelete(filename string) error
 }
 
 type clientNative struct {
diff --git a/pkg/haproxy/api/runtime.go b/pkg/haproxy/api/runtime.go
@@ -273,3 +273,19 @@ func (c *clientNative) CrtListEntryAdd(crtList string, entry runtime.CrtListEntr
 	}
 	return runtime.AddCrtListEntry(crtList, entry)
 }
+
+func (c *clientNative) CrtListEntryDelete(crtList, filename string, linenumber *int64) error {
+	runtime, err := c.nativeAPI.Runtime()
+	if err != nil {
+		return err
+	}
+	return runtime.DeleteCrtListEntry(crtList, filename, nil)
+}
+
+func (c *clientNative) CertEntryDelete(filename string) error {
+	runtime, err := c.nativeAPI.Runtime()
+	if err != nil {
+		return err
+	}
+	return runtime.DeleteCertEntry(filename)
+}
diff --git a/pkg/haproxy/certs/main.go b/pkg/haproxy/certs/main.go
@@ -35,7 +35,7 @@ type Certificates interface {
 	// Updated returns true if there is any updadted/created certificate
 	CertsUpdated() bool
 	// Refresh removes unused certs from HAProxyCertDir
-	RefreshCerts()
+	RefreshCerts(api api.HAProxyClient)
 	// Clean cleans certificates state
 	CleanCerts()
 	SetAPI(api api.HAProxyClient)
@@ -215,6 +215,28 @@ func (c *certs) updateRuntime(filename string, payload []byte) (bool, error) {
 	return updated, nil
 }
 
+func (c *certs) deleteRuntime(crtList, filename string) error {
+	// Only 1 transaction in parallel is possible for now in haproxy
+	// Keep this mutex for now to ensure that we perform 1 transaction at a time
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	var err error
+	certFile := path.Join(crtList, filename)
+	err = c.client.CrtListEntryDelete(crtList, certFile, nil)
+	if err != nil {
+		return err
+	}
+	utils.GetLogger().Debugf("del ssl crt-list` ok [%s %s]", crtList, certFile)
+
+	err = c.client.CertEntryDelete(certFile)
+	if err != nil {
+		return err
+	}
+	utils.GetLogger().Debugf("del ssl cert` ok [%s]", certFile)
+	return nil
+}
+
 func (c *certs) CleanCerts() {
 	for i := range c.frontend {
 		c.frontend[i].inUse = false
@@ -243,11 +265,12 @@ func (c *certs) FrontCertsInUse() bool {
 	return false
 }
 
-func (c *certs) RefreshCerts() {
-	refreshCerts(c.frontend, env.FrontendDir)
-	refreshCerts(c.backend, env.BackendDir)
-	refreshCerts(c.ca, env.CaDir)
-	refreshCerts(c.TCPCR, env.TCPCRDir)
+func (c *certs) RefreshCerts(api api.HAProxyClient) {
+	c.SetAPI(api)
+	c.refreshCerts(c.frontend, env.FrontendDir)
+	c.refreshCerts(c.backend, env.BackendDir)
+	c.refreshCerts(c.ca, env.CaDir)
+	c.refreshCerts(c.TCPCR, env.TCPCRDir)
 }
 
 func (c *certs) CertsUpdated() (reload bool) {
@@ -262,7 +285,7 @@ func (c *certs) CertsUpdated() (reload bool) {
 	return reload
 }
 
-func refreshCerts(certs map[string]*cert, certDir string) {
+func (c *certs) refreshCerts(certs map[string]*cert, certDir string) {
 	files, err := os.ReadDir(certDir)
 	if err != nil {
 		logger.Error(err)
@@ -277,11 +300,16 @@ func refreshCerts(certs map[string]*cert, certDir string) {
 		certName := strings.Split(filename, ".pem")[0]
 		crt, crtOk := certs[certName]
 		if !crtOk || !crt.inUse {
+			err := c.deleteRuntime(certDir, filename)
+			if err != nil {
+				instance.Reload("Runtime delete of cert file '%s' failed : %s", filename, err.Error())
+			} else {
+				utils.GetLogger().Debugf("Runtime delete of cert ok [%s]", filename)
+			}
 			fs.AddDelayedFunc(filename, func() {
 				logger.Error(os.Remove(path.Join(certDir, filename)))
 			})
 			delete(certs, certName)
-			instance.Reload("secret %s removed", certName)
 		}
 	}
 }
