Merge pull request #4 from davidepasquero/codex/forzare-eliminazione-controller-in-terminating

Fix stuck pods cleanup

Author: davidepasquero

cmd/controller/run.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"log"
 	"net/http"
+	"time"
 
 	"github.com/rancher/wrangler/pkg/leader"
 	"github.com/rancher/wrangler/pkg/signals"
@@ -16,6 +17,7 @@ import (
 	"github.com/harvester/vm-dhcp-controller/pkg/config"
 	"github.com/harvester/vm-dhcp-controller/pkg/controller"
 	"github.com/harvester/vm-dhcp-controller/pkg/server"
+	"github.com/harvester/vm-dhcp-controller/pkg/util"
 )
 
 var (
@@ -46,6 +48,8 @@ func run(options *config.ControllerOptions) error {
 		logrus.Fatalf("Error building controllers: %s", err.Error())
 	}
 
+	go util.CleanupTerminatingPods(ctx, client, options.AgentNamespace, "controller", time.Minute)
+
 	callback := func(ctx context.Context) {
 		if err := management.Register(ctx, cfg, controller.RegisterFuncList); err != nil {
 			panic(err)

cmd/webhook/run.go

@@ -2,11 +2,14 @@ package main
 
 import (
 	"context"
+	"time"
 
+	"github.com/harvester/vm-dhcp-controller/pkg/util"
 	"github.com/harvester/webhook/pkg/config"
 	"github.com/harvester/webhook/pkg/server"
 	"github.com/rancher/wrangler/pkg/start"
 	"github.com/sirupsen/logrus"
+	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 
 	ctlcore "github.com/harvester/vm-dhcp-controller/pkg/generated/controllers/core"
@@ -70,6 +73,13 @@ func run(ctx context.Context, cfg *rest.Config, options *config.Options) error {
 		return err
 	}
 
+	client, err := kubernetes.NewForConfig(cfg)
+	if err != nil {
+		return err
+	}
+
+	go util.CleanupTerminatingPods(ctx, client, options.Namespace, "webhook", time.Minute)
+
 	webhookServer := server.NewWebhookServer(ctx, cfg, name, options)
 
 	if err := webhookServer.RegisterValidators(

pkg/util/pod.go

@@ -0,0 +1,54 @@
+package util
+
+import (
+	"context"
+	"time"
+
+	"github.com/sirupsen/logrus"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/client-go/kubernetes"
+)
+
+// CleanupTerminatingPods forcibly deletes pods with the specified component label
+// that have been stuck in the Terminating state for longer than the given
+// threshold.
+func CleanupTerminatingPods(ctx context.Context, client kubernetes.Interface, namespace, component string, threshold time.Duration) {
+	if client == nil || namespace == "" || component == "" {
+		return
+	}
+
+	selector := labels.Set{"app.kubernetes.io/component": component}.AsSelector().String()
+	ticker := time.NewTicker(threshold / 2)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			pods, err := client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: selector})
+			if err != nil {
+				logrus.Errorf("(CleanupTerminatingPods) list pods error: %v", err)
+				continue
+			}
+
+			for i := range pods.Items {
+				pod := &pods.Items[i]
+				if pod.DeletionTimestamp == nil {
+					continue
+				}
+				if time.Since(pod.DeletionTimestamp.Time) < threshold {
+					continue
+				}
+
+				logrus.Infof("(CleanupTerminatingPods) force deleting stuck pod %s/%s", pod.Namespace, pod.Name)
+				grace := int64(0)
+				if err := client.CoreV1().Pods(pod.Namespace).Delete(ctx, pod.Name, metav1.DeleteOptions{GracePeriodSeconds: &grace}); err != nil && !apierrors.IsNotFound(err) {
+					logrus.Errorf("(CleanupTerminatingPods) delete pod %s/%s error: %v", pod.Namespace, pod.Name, err)
+				}
+			}
+		}
+	}
+}