Feat/cordon, drain

kubernetes/controller/project-controller/templates/crd.yaml

@@ -189,6 +189,75 @@ spec:
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
+metadata:
+  name: nodemaintenances.project.aipub.ten1010.io
+spec:
+  scope: Cluster
+  names:
+    plural: nodemaintenances
+    singular: nodemaintenance
+    kind: NodeMaintenance
+    shortNames:
+      - nm
+  group: project.aipub.ten1010.io
+  versions:
+    - name: v1alpha1
+      served: true
+      storage: true
+      subresources:
+        status: { }
+      schema:
+        openAPIV3Schema:
+          type: object
+          required:
+            - metadata
+            - spec
+          properties:
+            metadata:
+              type: object
+            spec:
+              type: object
+              properties:
+                targetNodes:
+                  type: array
+                  nullable: false
+                  items:
+                    type: string
+                action:
+                  type: object
+                  properties:
+                    type:
+                      type: string
+                    ignoreDaemonSets:
+                      type: boolean
+                      default: true
+                    force:
+                      type: boolean
+                      default: false
+            status:
+              type: object
+              properties:
+                allEffectedNodes:
+                  type: array
+                  default: []
+                  items:
+                    type: string
+                action:
+                  type: object
+                  properties:
+                    type:
+                      type: string
+                    ignoreDaemonSets:
+                      type: boolean
+                      default: true
+                    force:
+                      type: boolean
+                      default: false
+                status:
+                  type: string
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
 metadata:
   name: aipubusers.project.aipub.ten1010.io
 spec:

kubernetes/examples/node-maintain.yaml

@@ -0,0 +1,11 @@
+apiVersion: project.aipub.ten1010.io/v1alpha1
+kind: NodeMaintenance
+metadata:
+  name: node1-maintenance
+spec:
+  targetNodes:
+    - minikube-m02
+  action:
+    type: cordon
+    ignoreDaemonSets: true
+    force: false

kubernetes/examples/pod1.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nginx-pod
+spec:
+  containers:
+    - name: my-nginx
+      image: nginx

kubernetes/examples/pod2.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nginx-pod2
+spec:
+  containers:
+    - name: my-nginx
+      image: nginx

kubernetes/examples/pod3.yaml

@@ -0,0 +1,17 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: nginx-pod-daemonset
+  namespace: project-controller
+spec:
+  selector:
+    matchLabels:
+      app: nginx-pod-daemonset
+  template:
+    metadata:
+      labels:
+        app: nginx-pod-daemonset
+    spec:
+      containers:
+        - name: my-nginx
+          image: nginx

src/main/java/io/ten1010/aipub/projectcontroller/configuration/ControllerConfiguration.java

@@ -8,10 +8,7 @@
 import io.kubernetes.client.informer.SharedInformerFactory;
 import io.ten1010.aipub.projectcontroller.controller.cluster.NamespaceControllerFactory;
 import io.ten1010.aipub.projectcontroller.controller.cluster.NodeControllerFactory;
-import io.ten1010.aipub.projectcontroller.controller.cr.AipubUserControllerFactory;
-import io.ten1010.aipub.projectcontroller.controller.cr.ImageHubControllerFactory;
-import io.ten1010.aipub.projectcontroller.controller.cr.NodeGroupControllerFactory;
-import io.ten1010.aipub.projectcontroller.controller.cr.ProjectControllerFactory;
+import io.ten1010.aipub.projectcontroller.controller.cr.*;
 import io.ten1010.aipub.projectcontroller.controller.namespaced.ImagePullSecretReconcilerFactory;
 import io.ten1010.aipub.projectcontroller.controller.namespaced.ResourceQuotaControllerFactory;
 import io.ten1010.aipub.projectcontroller.controller.rbac.aipub.AipubUserClusterRoleBindingControllerFactory;
@@ -39,7 +36,6 @@ public class ControllerConfiguration {
     @Bean
     public ControllerManager controllerManager(
             SharedInformerFactory sharedInformerFactory, List<Controller> controllers, List<WorkloadControllerFactory<?>> workloadControllerFactories) {
-        System.out.println(controllers);
         ControllerManagerBuilder builder = ControllerBuilder.controllerManagerBuilder(sharedInformerFactory);
         controllers.forEach(builder::addController);
         workloadControllerFactories.forEach(f -> builder.addController(f.createController()));
@@ -59,6 +55,14 @@ public Controller projectController(SharedInformerFactory sharedInformerFactory,
                 .createController();
     }
 
+    @Bean
+    public Controller nodeMaintenanceController(SharedInformerFactory sharedInformerFactory,
+                                        K8sApiProvider k8sApiProvider,
+                                        ReconciliationService reconciliationService) {
+        return new NodeMaintenanceControllerFactory(sharedInformerFactory, k8sApiProvider, reconciliationService)
+                .createController();
+    }
+
     @Bean
     public Controller aipubUserController(SharedInformerFactory sharedInformerFactory,
                                           K8sApiProvider k8sApiProvider,

src/main/java/io/ten1010/aipub/projectcontroller/controller/cr/NodeMaintenanceControllerFactory.java

@@ -0,0 +1,48 @@
+package io.ten1010.aipub.projectcontroller.controller.cr;
+
+import io.kubernetes.client.extended.controller.Controller;
+import io.kubernetes.client.extended.controller.ControllerWatch;
+import io.kubernetes.client.extended.controller.builder.ControllerBuilder;
+import io.kubernetes.client.extended.controller.reconciler.Request;
+import io.kubernetes.client.extended.workqueue.WorkQueue;
+import io.kubernetes.client.informer.SharedInformerFactory;
+import io.ten1010.aipub.projectcontroller.controller.ControllerFactory;
+import io.ten1010.aipub.projectcontroller.controller.watch.DefaultControllerWatch;
+import io.ten1010.aipub.projectcontroller.controller.watch.OnUpdateFilterFactory;
+import io.ten1010.aipub.projectcontroller.domain.k8s.K8sApiProvider;
+import io.ten1010.aipub.projectcontroller.domain.k8s.ReconciliationService;
+import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1alpha1NodeMaintenance;
+
+public class NodeMaintenanceControllerFactory implements ControllerFactory {
+
+    private final SharedInformerFactory sharedInformerFactory;
+    private final K8sApiProvider k8sApiProvider;
+    private final OnUpdateFilterFactory onUpdateFilterFactory;
+
+    public NodeMaintenanceControllerFactory(
+            SharedInformerFactory sharedInformerFactory,
+            K8sApiProvider k8sApiProvider,
+            ReconciliationService reconciliationService) {
+        this.sharedInformerFactory = sharedInformerFactory;
+        this.k8sApiProvider = k8sApiProvider;
+        this.onUpdateFilterFactory = new OnUpdateFilterFactory();
+    }
+
+    @Override
+    public Controller createController() {
+        return ControllerBuilder.defaultBuilder(this.sharedInformerFactory)
+                .withName("node-maintenance-controller")
+                .withWorkerCount(1)
+                .withReadyFunc(this.sharedInformerFactory.getExistingSharedIndexInformer(V1alpha1NodeMaintenance.class)::hasSynced)
+                .watch(this::createNodeMaintenanceWatch)
+                .withReconciler(new NodeMaintenanceReconciler(this.sharedInformerFactory, this.k8sApiProvider))
+                .build();
+    }
+
+    private ControllerWatch<V1alpha1NodeMaintenance> createNodeMaintenanceWatch(WorkQueue<Request> workQueue) {
+        DefaultControllerWatch<V1alpha1NodeMaintenance> watch = new DefaultControllerWatch<>(workQueue, V1alpha1NodeMaintenance.class);
+        watch.setOnUpdateFilter(this.onUpdateFilterFactory.nodeMaintenanceSpecFieldFilter());
+        return watch;
+    }
+
+}

src/main/java/io/ten1010/aipub/projectcontroller/controller/cr/NodeMaintenanceReconciler.java

@@ -0,0 +1,180 @@
+package io.ten1010.aipub.projectcontroller.controller.cr;
+
+import io.kubernetes.client.extended.controller.reconciler.Request;
+import io.kubernetes.client.extended.controller.reconciler.Result;
+import io.kubernetes.client.informer.SharedInformerFactory;
+import io.kubernetes.client.informer.cache.Indexer;
+import io.kubernetes.client.openapi.ApiException;
+import io.kubernetes.client.openapi.ApiResponse;
+import io.kubernetes.client.openapi.apis.CoreV1Api;
+import io.kubernetes.client.openapi.models.*;
+import io.ten1010.aipub.projectcontroller.controller.AbstractReconciler;
+import io.ten1010.aipub.projectcontroller.controller.RequestHelper;
+import io.ten1010.aipub.projectcontroller.domain.k8s.K8sApiProvider;
+import io.ten1010.aipub.projectcontroller.domain.k8s.K8sObjectTypeConstants;
+import io.ten1010.aipub.projectcontroller.domain.k8s.KeyResolver;
+import io.ten1010.aipub.projectcontroller.domain.k8s.ProjectApiConstants;
+import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1alpha1NodeMaintenance;
+import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1alpha1NodeMaintenanceAction;
+import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1alpha1NodeMaintenanceStatus;
+import io.ten1010.aipub.projectcontroller.domain.k8s.util.K8sObjectUtils;
+import io.ten1010.aipub.projectcontroller.domain.k8s.util.StatusPatchHelper;
+import lombok.extern.slf4j.Slf4j;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+
+@Slf4j
+public class NodeMaintenanceReconciler extends AbstractReconciler {
+
+    private final KeyResolver keyResolver;
+    private final Indexer<V1alpha1NodeMaintenance> projectIndexer;
+    private final CoreV1Api coreV1Api;
+    private final StatusPatchHelper<V1alpha1NodeMaintenance> statusPatchHelper;
+    private final String namespace = "project-controller";
+
+    public NodeMaintenanceReconciler(
+            SharedInformerFactory sharedInformerFactory,
+            K8sApiProvider k8sApiProvider) {
+        this.keyResolver = new KeyResolver();
+        this.projectIndexer = sharedInformerFactory
+                .getExistingSharedIndexInformer(V1alpha1NodeMaintenance.class)
+                .getIndexer();
+        this.coreV1Api = new CoreV1Api(k8sApiProvider.getApiClient());
+        this.statusPatchHelper = new StatusPatchHelper<>(
+                k8sApiProvider.getApiClient(),
+                K8sObjectTypeConstants.NODE_MAINTENANCE_V1ALPHA1,
+                ProjectApiConstants.NODE_MAINTENANCE_RESOURCE_PLURAL);
+    }
+
+    @Override
+    protected Result reconcileInternal(Request request) throws ApiException {
+        String projectKey = new RequestHelper(this.keyResolver).resolveKey(request);
+        Optional<V1alpha1NodeMaintenance> nodeMaintenanceOpt = Optional.ofNullable(this.projectIndexer.getByKey(projectKey));
+        if (nodeMaintenanceOpt.isEmpty()) {
+            return new Result(false);
+        }
+
+        var nodeMaintenance = nodeMaintenanceOpt.get();
+        var spec = Objects.requireNonNull(nodeMaintenance.getSpec());
+        var status = nodeMaintenance.getStatus();
+        var action = Objects.requireNonNull(spec.getAction());
+        var actionType = Objects.requireNonNull(action.getType());
+        String nodeMaintenanceName = K8sObjectUtils.getName(nodeMaintenance);
+        List<String> targetNodeNames = Objects.requireNonNull(spec.getTargetNodes());
+
+        if (status != null && (
+                targetNodeNames.equals(status.getAllEffectedNodes()) && action.equals(status.getAction())
+        )) {
+            return new Result(false);
+        }
+
+        List<String> effectedNodes = new ArrayList<>();
+        for (String targetNodeName : targetNodeNames) {
+            var targetNodeRequest = coreV1Api.readNode(targetNodeName);
+            var targetNodeResponse = targetNodeRequest.executeWithHttpInfo();
+            if (targetNodeResponse.getStatusCode() != 200) {
+                return new Result(false);
+            }
+            V1Node targetNode = targetNodeResponse.getData();
+            Objects.requireNonNull(targetNode.getSpec());
+
+            //
+            switch (actionType) {
+                case "cordon":
+                    //
+                    executeSchedulable(targetNode, true, nodeMaintenanceName);
+                    effectedNodes.add(targetNodeName);
+                    break;
+                case "uncordon":
+                    //
+                    executeSchedulable(targetNode, false, nodeMaintenanceName);
+                    effectedNodes.add(targetNodeName);
+                    break;
+                case "drain":
+                    //
+                    if (targetNode.getSpec().getUnschedulable() == null || Boolean.FALSE.equals(targetNode.getSpec().getUnschedulable())) {
+                        log.error("Node {} isn't cordon state", nodeMaintenanceName);
+                        return new Result(false);
+                    }
+
+                    var podRequest = coreV1Api.listNamespacedPod(namespace);
+                    var podResponse = podRequest.executeWithHttpInfo();
+                    if (podResponse.getStatusCode() != 200) {
+                        return new Result(false);
+                    }
+                    executePodDelete(podResponse, targetNodeName, action, nodeMaintenanceName);
+                    effectedNodes.add(targetNodeName);
+                    break;
+                default:
+            }
+        }
+
+        //
+        V1alpha1NodeMaintenanceStatus edited = new V1alpha1NodeMaintenanceStatus();
+        edited.setAllEffectedNodes(targetNodeNames);
+        edited.setAction(action);
+        edited.setStatus("COMPLETED");
+        nodeMaintenance.setStatus(edited);
+
+        this.updateNodeMaintenanceStatus(nodeMaintenance);
+
+        return new Result(false);
+    }
+
+    private void executePodDelete(ApiResponse<V1PodList> podResponse, String targetNodeName, V1alpha1NodeMaintenanceAction action, String maintenanceName) throws ApiException {
+        boolean isIgnoreDaemonSets = Boolean.TRUE.equals(action.getIgnoreDaemonSets());
+        boolean isForce = Boolean.TRUE.equals(action.getForce());
+        for (V1Pod _item : podResponse.getData().getItems()) {
+            V1PodSpec _spec = Objects.requireNonNull(_item.getSpec());
+            if (targetNodeName.equals(_spec.getNodeName())) {
+                V1ObjectMeta _metadata = Objects.requireNonNull(_item.getMetadata());
+                String podName = _metadata.getName();
+                String podNamespace = _metadata.getNamespace();
+                int gracePeriodSeconds = isForce ? 0 : Objects.requireNonNull(_spec.getTerminationGracePeriodSeconds()).intValue();
+
+                if (isIgnoreDaemonSets) {
+                    var ownerReferences = Objects.requireNonNull(_metadata.getOwnerReferences());
+                    boolean isDaemonset = false;
+                    for (V1OwnerReference ownerReference : ownerReferences) {
+                        if (ownerReference.getKind().equalsIgnoreCase("DaemonSet")) {
+                            isDaemonset = true;
+                            break;
+                        }
+                    }
+                    if (!isDaemonset) {
+                        log.info("drain daemonSet ignore : pod name - {} / namespace - {}", podName, podNamespace);
+                        coreV1Api.deleteNamespacedPod(podName, podNamespace)
+                                .gracePeriodSeconds(gracePeriodSeconds)
+                                .execute();
+                    }
+                } else {
+                    log.info("drain all pods : pod name - {} / namespace - {}", podName, podNamespace);
+                    coreV1Api.deleteNamespacedPod(podName, podNamespace)
+                            .gracePeriodSeconds(gracePeriodSeconds)
+                            .execute();
+                }
+            }
+        }
+    }
+
+    private void executeSchedulable(V1Node targetNode, boolean isUnschedulable, String maintenanceName) throws ApiException {
+        String nodeName = targetNode.getMetadata().getName();
+        targetNode.getSpec().setUnschedulable(isUnschedulable);
+
+        if (isUnschedulable) {
+            log.info("cordon : node name - {} / yaml - {}", nodeName, maintenanceName);
+        } else {
+            log.info("uncordon : node name - {} / yaml - {}", nodeName, maintenanceName);
+        }
+        coreV1Api.replaceNode(nodeName, targetNode).execute();
+    }
+
+    private void updateNodeMaintenanceStatus(V1alpha1NodeMaintenance nodeMaintenance) throws ApiException {
+        Objects.requireNonNull(nodeMaintenance.getStatus());
+        this.statusPatchHelper.patchStatus(null, K8sObjectUtils.getName(nodeMaintenance), nodeMaintenance.getStatus());
+    }
+
+}

src/main/java/io/ten1010/aipub/projectcontroller/controller/watch/OnUpdateFilterFactory.java

@@ -27,6 +27,10 @@ public BiPredicate<V1alpha1Project, V1alpha1Project> projectSpecFieldFilter() {
         return (oldObj, newObj) -> !Objects.equals(oldObj.getSpec(), newObj.getSpec());
     }
 
+    public BiPredicate<V1alpha1NodeMaintenance, V1alpha1NodeMaintenance> nodeMaintenanceSpecFieldFilter() {
+        return (oldObj, newObj) -> !Objects.equals(oldObj.getSpec(), newObj.getSpec());
+    }
+
     public BiPredicate<V1alpha1Project, V1alpha1Project> projectSpecQuotaFieldFilter() {
         return (oldObj, newObj) -> !ProjectUtils.getSpecQuota(oldObj).equals(ProjectUtils.getSpecQuota(newObj));
     }