Feat/cordon, drain

kubernetes/controller/project-controller/kustomization.yaml

@@ -28,5 +28,6 @@ resources:
   - templates/deployment.yaml
   - templates/service.yaml
   - templates/mutating-webhook-configuration.yaml
+  - templates/validating-webhook-configuration.yaml
 patchesStrategicMerge:
   - patches.yaml

kubernetes/controller/project-controller/patches.yaml

@@ -33,3 +33,12 @@ webhooks:
     clientConfig:
       caBundle: ### To be patched ###
 ---
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: project-controller.project.aipub.ten1010.io
+webhooks:
+  - name: nodemaintenances.project-controller.project.aipub.ten1010.io
+    clientConfig:
+      caBundle: ### To be patched ###
+---

kubernetes/controller/project-controller/templates/crd.yaml

@@ -189,6 +189,75 @@ spec:
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
+metadata:
+  name: nodemaintenances.project.aipub.ten1010.io
+spec:
+  scope: Cluster
+  names:
+    plural: nodemaintenances
+    singular: nodemaintenance
+    kind: NodeMaintenance
+    shortNames:
+      - nm
+  group: project.aipub.ten1010.io
+  versions:
+    - name: v1alpha1
+      served: true
+      storage: true
+      subresources:
+        status: { }
+      schema:
+        openAPIV3Schema:
+          type: object
+          required:
+            - metadata
+            - spec
+          properties:
+            metadata:
+              type: object
+            spec:
+              type: object
+              properties:
+                targetNodes:
+                  type: array
+                  nullable: false
+                  items:
+                    type: string
+                actions:
+                  type: array
+                  default: []
+                  items:
+                    type: object
+                    properties:
+                      type:
+                        type: string
+                      ignoreDaemonSets:
+                        type: boolean
+                        default: true
+                      force:
+                        type: boolean
+                        default: false
+            status:
+              type: object
+              properties:
+                allEffectedNodes:
+                  type: array
+                  default: []
+                  items:
+                    type: string
+                actions:
+                  type: array
+                  default: []
+                  items:
+                    type: object
+                    properties:
+                      type:
+                        type: string
+                      status:
+                        type: string
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
 metadata:
   name: aipubusers.project.aipub.ten1010.io
 spec:

kubernetes/controller/project-controller/templates/validating-webhook-configuration.yaml

@@ -0,0 +1,27 @@
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: project-controller.project.aipub.ten1010.io
+webhooks:
+  - name: nodemaintenances.project-controller.project.aipub.ten1010.io
+    admissionReviewVersions: [ "v1" ]
+    clientConfig:
+      caBundle: <CA_BUNDLE>
+      service:
+        namespace: project-controller
+        name: project-controller
+        path: /api/v1/admissionreviews
+        port: 8080
+    failurePolicy: Fail
+    matchPolicy: Equivalent
+    namespaceSelector: { }
+    objectSelector: { }
+    rules:
+      - apiGroups: [ "project.aipub.ten1010.io" ]
+        apiVersions: [ "v1alpha1" ]
+        operations: [ "CREATE", "UPDATE" ]
+        resources: [ "nodemaintenances" ]
+        scope: "*"
+    sideEffects: None
+    timeoutSeconds: 10
+---

kubernetes/examples/node-maintain.yaml

@@ -0,0 +1,13 @@
+apiVersion: project.aipub.ten1010.io/v1alpha1
+kind: NodeMaintenance
+metadata:
+  name: node1-maintenance
+spec:
+  targetNodes:
+    - minikube-m02
+    - minikube-m03
+  actions:
+    - type: uncordon
+    - type: drain
+      ignoreDaemonSets: false
+      force: true

kubernetes/examples/pod1-rs.yaml

@@ -0,0 +1,20 @@
+apiVersion: apps/v1
+kind: ReplicaSet
+metadata:
+  name: nginx-pod-rs3
+  labels:
+    app: nginx
+    tier: fe
+spec:
+  replicas: 4
+  selector:
+      matchLabels:
+        tier: fe
+  template:
+    metadata:
+      labels:
+        tier: fe
+    spec:
+      containers:
+        - name: my-nginx
+          image: nginx

kubernetes/examples/pod1.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nginx-pod
+spec:
+  containers:
+    - name: my-nginx
+      image: nginx

kubernetes/examples/pod2.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nginx-pod2
+spec:
+  containers:
+    - name: my-nginx
+      image: nginx

kubernetes/examples/pod3.yaml

@@ -0,0 +1,17 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: nginx-pod-daemonset
+  namespace: project-controller
+spec:
+  selector:
+    matchLabels:
+      app: nginx-pod-daemonset
+  template:
+    metadata:
+      labels:
+        app: nginx-pod-daemonset
+    spec:
+      containers:
+        - name: my-nginx
+          image: nginx

src/main/java/io/ten1010/aipub/projectcontroller/configuration/ControllerConfiguration.java

@@ -8,10 +8,7 @@
 import io.kubernetes.client.informer.SharedInformerFactory;
 import io.ten1010.aipub.projectcontroller.controller.cluster.NamespaceControllerFactory;
 import io.ten1010.aipub.projectcontroller.controller.cluster.NodeControllerFactory;
-import io.ten1010.aipub.projectcontroller.controller.cr.AipubUserControllerFactory;
-import io.ten1010.aipub.projectcontroller.controller.cr.ImageHubControllerFactory;
-import io.ten1010.aipub.projectcontroller.controller.cr.NodeGroupControllerFactory;
-import io.ten1010.aipub.projectcontroller.controller.cr.ProjectControllerFactory;
+import io.ten1010.aipub.projectcontroller.controller.cr.*;
 import io.ten1010.aipub.projectcontroller.controller.namespaced.ImagePullSecretReconcilerFactory;
 import io.ten1010.aipub.projectcontroller.controller.namespaced.ResourceQuotaControllerFactory;
 import io.ten1010.aipub.projectcontroller.controller.rbac.aipub.AipubUserClusterRoleBindingControllerFactory;
@@ -39,7 +36,6 @@ public class ControllerConfiguration {
     @Bean
     public ControllerManager controllerManager(
             SharedInformerFactory sharedInformerFactory, List<Controller> controllers, List<WorkloadControllerFactory<?>> workloadControllerFactories) {
-        System.out.println(controllers);
         ControllerManagerBuilder builder = ControllerBuilder.controllerManagerBuilder(sharedInformerFactory);
         controllers.forEach(builder::addController);
         workloadControllerFactories.forEach(f -> builder.addController(f.createController()));
@@ -59,6 +55,14 @@ public Controller projectController(SharedInformerFactory sharedInformerFactory,
                 .createController();
     }
 
+    @Bean
+    public Controller nodeMaintenanceController(SharedInformerFactory sharedInformerFactory,
+                                        K8sApiProvider k8sApiProvider,
+                                        ReconciliationService reconciliationService) {
+        return new NodeMaintenanceControllerFactory(sharedInformerFactory, k8sApiProvider, reconciliationService)
+                .createController();
+    }
+
     @Bean
     public Controller aipubUserController(SharedInformerFactory sharedInformerFactory,
                                           K8sApiProvider k8sApiProvider,

src/main/java/io/ten1010/aipub/projectcontroller/configuration/MutatingConfiguration.java

@@ -63,4 +63,9 @@ public ImageReviewReviewHandler imageReviewReviewHandler(
         return new ImageReviewReviewHandler(repositoryService, artifactService, sharedInformerFactory);
     }
 
+    @Bean
+    public NodeMaintenanceReviewHandler nodeMaintenanceReviewHandler(AipubProperties aipubProperties, SubjectResolver subjectResolver, SharedInformerFactory sharedInformerFactory) {
+        return new NodeMaintenanceReviewHandler(aipubProperties, subjectResolver, sharedInformerFactory);
+    }
+
 }

src/main/java/io/ten1010/aipub/projectcontroller/controller/BoundObjectResolver.java

@@ -3,6 +3,8 @@
 import io.kubernetes.client.informer.SharedInformerFactory;
 import io.kubernetes.client.informer.cache.Indexer;
 import io.kubernetes.client.openapi.models.V1Node;
+import io.kubernetes.client.openapi.models.V1OwnerReference;
+import io.kubernetes.client.openapi.models.V1Pod;
 import io.ten1010.aipub.projectcontroller.domain.k8s.KeyResolver;
 import io.ten1010.aipub.projectcontroller.domain.k8s.dto.*;
 import io.ten1010.aipub.projectcontroller.domain.k8s.util.*;
@@ -26,6 +28,8 @@ private static <T> List<T> getIntersection(List<T> list1, List<T> list2) {
     private final Indexer<V1alpha1ImageHub> imageHubIndexer;
     private final Indexer<V1Node> nodeIndexer;
     private final Indexer<V1alpha1ResourceSet> resourceSetIndexer;
+    private final Indexer<V1alpha1NodeMaintenance> nodeMaintenanceIndexer;
+    private final Indexer<V1Pod> podIndexer;
 
     public BoundObjectResolver(SharedInformerFactory sharedInformerFactory) {
         this.keyResolver = new KeyResolver();
@@ -47,6 +51,12 @@ public BoundObjectResolver(SharedInformerFactory sharedInformerFactory) {
         this.resourceSetIndexer = sharedInformerFactory
                 .getExistingSharedIndexInformer(V1alpha1ResourceSet.class)
                 .getIndexer();
+        this.nodeMaintenanceIndexer = sharedInformerFactory
+                .getExistingSharedIndexInformer(V1alpha1NodeMaintenance.class)
+                .getIndexer();
+        this.podIndexer = sharedInformerFactory
+                .getExistingSharedIndexInformer(V1Pod.class)
+                .getIndexer();
     }
 
     public List<V1alpha1AipubUser> getAllBoundAipubUsers(V1alpha1Project project) {
@@ -133,6 +143,28 @@ public List<V1alpha1NodeGroup> getAllBoundNodeGroups(V1Node node) {
         return K8sObjectUtils.distinctByKey(this.keyResolver, allBoundNodeGroups);
     }
 
+    public List<V1alpha1NodeMaintenance> getAllBoundNodeMaintenances(V1Node node) {
+        List<V1alpha1NodeMaintenance> allBoundNodeGroups = this.nodeMaintenanceIndexer.byIndex(
+                IndexerConstants.NODE_NAME_TO_NODE_MAINTENANCE_INDEXER_NAME,
+                K8sObjectUtils.getName(node));
+        return K8sObjectUtils.distinctByKey(this.keyResolver, allBoundNodeGroups);
+    }
+
+    public List<V1Node> getAllBoundNodeByNodeMaintenances(V1alpha1NodeMaintenance node) {
+        List<V1Node> allBoundNodeGroups = new ArrayList<>();
+        for (String targetNode : node.getSpec().getTargetNodes()) {
+            Optional<V1Node> opt = Optional.ofNullable(this.nodeIndexer.getByKey(targetNode));
+            if (opt.isPresent()) {
+                allBoundNodeGroups.add(opt.get());
+            }
+        }
+        return K8sObjectUtils.distinctByKey(this.keyResolver, allBoundNodeGroups);
+    }
+
+    public List<V1Pod> getAllBoundPods(String nodeName) {
+        return podIndexer.byIndex(IndexerConstants.NODE_NAME_TO_POD_INDEXER_NAME, nodeName);
+    }
+
     public List<V1alpha1ResourceSet> getAllBoundResourceSets(V1alpha1Project project) {
         List<V1Node> allBoundNodes = new ArrayList<>(getDirectlyBoundNodes(project));
         List<V1alpha1ResourceSet> allBoundResourceSets = new ArrayList<>();

src/main/java/io/ten1010/aipub/projectcontroller/controller/cluster/NodeControllerFactory.java

@@ -14,6 +14,7 @@
 import io.ten1010.aipub.projectcontroller.domain.k8s.K8sApiProvider;
 import io.ten1010.aipub.projectcontroller.domain.k8s.ReconciliationService;
 import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1alpha1NodeGroup;
+import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1alpha1NodeMaintenance;
 import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1alpha1Project;
 
 public class NodeControllerFactory implements ControllerFactory {
@@ -43,9 +44,11 @@ public Controller createController() {
                 .withReadyFunc(this.sharedInformerFactory.getExistingSharedIndexInformer(V1Node.class)::hasSynced)
                 .withReadyFunc(this.sharedInformerFactory.getExistingSharedIndexInformer(V1alpha1Project.class)::hasSynced)
                 .withReadyFunc(this.sharedInformerFactory.getExistingSharedIndexInformer(V1alpha1NodeGroup.class)::hasSynced)
+                .withReadyFunc(this.sharedInformerFactory.getExistingSharedIndexInformer(V1alpha1NodeMaintenance.class)::hasSynced)
                 .watch(this::createNodeWatch)
                 .watch(this::createProjectWatch)
                 .watch(this::createNodeGroupWatch)
+                .watch(this::updateNodeMaintenanceWatch)
                 .withReconciler(new NodeReconciler(this.sharedInformerFactory, this.k8sApiProvider, this.reconciliationService))
                 .build();
     }
@@ -70,4 +73,11 @@ private ControllerWatch<V1alpha1NodeGroup> createNodeGroupWatch(WorkQueue<Reques
         return watch;
     }
 
+    private ControllerWatch<V1alpha1NodeMaintenance> updateNodeMaintenanceWatch(WorkQueue<Request> workQueue) {
+        DefaultControllerWatch<V1alpha1NodeMaintenance> watch = new DefaultControllerWatch<>(workQueue, V1alpha1NodeMaintenance.class);
+        watch.setOnUpdateFilter(this.onUpdateFilterFactory.nodeMaintenanceCreateFilter());
+        watch.setRequestBuilder(this.requestBuilderFactory.nodeMaintenanceToBoundNodes());
+        return watch;
+    }
+
 }

src/main/java/io/ten1010/aipub/projectcontroller/controller/cluster/NodeReconciler.java

@@ -14,9 +14,9 @@
 import io.ten1010.aipub.projectcontroller.controller.RequestHelper;
 import io.ten1010.aipub.projectcontroller.domain.k8s.K8sApiProvider;
 import io.ten1010.aipub.projectcontroller.domain.k8s.KeyResolver;
+import io.ten1010.aipub.projectcontroller.domain.k8s.NodeMaintenanceConstants;
 import io.ten1010.aipub.projectcontroller.domain.k8s.ReconciliationService;
-import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1alpha1NodeGroup;
-import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1alpha1Project;
+import io.ten1010.aipub.projectcontroller.domain.k8s.dto.*;
 import io.ten1010.aipub.projectcontroller.domain.k8s.util.K8sObjectUtils;
 import io.ten1010.aipub.projectcontroller.domain.k8s.util.NodeUtils;
 
@@ -61,6 +61,11 @@ protected Result reconcileInternal(Request request) throws ApiException {
         Map<String, String> reconciledAnnotations = this.reconciliationService.reconcileNodeAnnotations(node, boundProjects, boundNodeGroups);
         List<V1Taint> reconciledTaints = this.reconciliationService.reconcileTaints(node);
 
+        List<V1alpha1NodeMaintenance> allBoundNodeMaintenances = this.boundObjectResolver.getAllBoundNodeMaintenances(node);
+        if (!allBoundNodeMaintenances.isEmpty()) {
+            return executeSchedulable(node, allBoundNodeMaintenances);
+        }
+
         return reconcileExistingNode(nodeOpt.get(), reconciledLabels, reconciledAnnotations, reconciledTaints);
     }
 
@@ -87,6 +92,29 @@ private Result reconcileExistingNode(
         return new Result(false);
     }
 
+    private Result executeSchedulable(V1Node targetNode, List<V1alpha1NodeMaintenance> nodeMaintenances) throws ApiException {
+        String nodeName = K8sObjectUtils.getName(targetNode);
+        for (V1alpha1NodeMaintenance nodeMaintenance : nodeMaintenances) {
+            var progressList = nodeMaintenance.getStatus().getActions().stream()
+                    .filter(x -> !x.getType().equals(NodeMaintenanceConstants.NN_DRAIN)
+                            && x.getStatus().equals(NodeMaintenanceConstants.NN_PROGRESS))
+                    .toList();
+            if (progressList.isEmpty()) {
+                break;
+            }
+
+            for (V1alpha1NodeMaintenanceAction action : nodeMaintenance.getSpec().getActions()) {
+                if (action.getType().equals(NodeMaintenanceConstants.NN_CORDON)) {
+                    targetNode.getSpec().setUnschedulable(true);
+                } else if (action.getType().equals(NodeMaintenanceConstants.NN_UNCORDON)) {
+                    targetNode.getSpec().setUnschedulable(false);
+                }
+            }
+        }
+        updateNode(nodeName, targetNode);
+        return new Result(false);
+    }
+
     private void updateNode(String objName, V1Node node) throws ApiException {
         this.coreV1Api
                 .replaceNode(objName, node)