diff --git a/kubernetes/controller/project-controller/kustomization.yaml b/kubernetes/controller/project-controller/kustomization.yaml index 4b03f523..b800ed1d 100644 --- a/kubernetes/controller/project-controller/kustomization.yaml +++ b/kubernetes/controller/project-controller/kustomization.yaml @@ -28,5 +28,6 @@ resources: - templates/deployment.yaml - templates/service.yaml - templates/mutating-webhook-configuration.yaml + - templates/validating-webhook-configuration.yaml patchesStrategicMerge: - patches.yaml diff --git a/kubernetes/controller/project-controller/patches.yaml b/kubernetes/controller/project-controller/patches.yaml index 078e1546..8246e6d6 100644 --- a/kubernetes/controller/project-controller/patches.yaml +++ b/kubernetes/controller/project-controller/patches.yaml @@ -33,3 +33,12 @@ webhooks: clientConfig: caBundle: ### To be patched ### --- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: project-controller.project.aipub.ten1010.io +webhooks: + - name: nodemaintenances.project-controller.project.aipub.ten1010.io + clientConfig: + caBundle: ### To be patched ### +--- diff --git a/kubernetes/controller/project-controller/templates/crd.yaml b/kubernetes/controller/project-controller/templates/crd.yaml index a5234a08..dffdfcbf 100644 --- a/kubernetes/controller/project-controller/templates/crd.yaml +++ b/kubernetes/controller/project-controller/templates/crd.yaml @@ -189,6 +189,75 @@ spec: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + name: nodemaintenances.project.aipub.ten1010.io +spec: + scope: Cluster + names: + plural: nodemaintenances + singular: nodemaintenance + kind: NodeMaintenance + shortNames: + - nm + group: project.aipub.ten1010.io + versions: + - name: v1alpha1 + served: true + storage: true + subresources: + status: { } + schema: + openAPIV3Schema: + type: object + required: + - metadata + - spec + properties: + metadata: + type: object + spec: + type: object + properties: + targetNodes: + type: array + nullable: false + items: + type: string + actions: + type: array + default: [] + items: + type: object + properties: + type: + type: string + ignoreDaemonSets: + type: boolean + default: true + force: + type: boolean + default: false + status: + type: object + properties: + allEffectedNodes: + type: array + default: [] + items: + type: string + actions: + type: array + default: [] + items: + type: object + properties: + type: + type: string + status: + type: string +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: name: aipubusers.project.aipub.ten1010.io spec: diff --git a/kubernetes/controller/project-controller/templates/validating-webhook-configuration.yaml b/kubernetes/controller/project-controller/templates/validating-webhook-configuration.yaml new file mode 100644 index 00000000..fa0488a0 --- /dev/null +++ b/kubernetes/controller/project-controller/templates/validating-webhook-configuration.yaml @@ -0,0 +1,27 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: project-controller.project.aipub.ten1010.io +webhooks: + - name: nodemaintenances.project-controller.project.aipub.ten1010.io + admissionReviewVersions: [ "v1" ] + clientConfig: + caBundle: + service: + namespace: project-controller + name: project-controller + path: /api/v1/admissionreviews + port: 8080 + failurePolicy: Fail + matchPolicy: Equivalent + namespaceSelector: { } + objectSelector: { } + rules: + - apiGroups: [ "project.aipub.ten1010.io" ] + apiVersions: [ "v1alpha1" ] + operations: [ "CREATE", "UPDATE" ] + resources: [ "nodemaintenances" ] + scope: "*" + sideEffects: None + timeoutSeconds: 10 +--- diff --git a/kubernetes/examples/node-maintain.yaml b/kubernetes/examples/node-maintain.yaml new file mode 100644 index 00000000..9cccf808 --- /dev/null +++ b/kubernetes/examples/node-maintain.yaml @@ -0,0 +1,13 @@ +apiVersion: project.aipub.ten1010.io/v1alpha1 +kind: NodeMaintenance +metadata: + name: node1-maintenance +spec: + targetNodes: + - minikube-m02 + - minikube-m03 + actions: + - type: uncordon + - type: drain + ignoreDaemonSets: false + force: true diff --git a/kubernetes/examples/pod1-rs.yaml b/kubernetes/examples/pod1-rs.yaml new file mode 100644 index 00000000..d84e27e6 --- /dev/null +++ b/kubernetes/examples/pod1-rs.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: ReplicaSet +metadata: + name: nginx-pod-rs3 + labels: + app: nginx + tier: fe +spec: + replicas: 4 + selector: + matchLabels: + tier: fe + template: + metadata: + labels: + tier: fe + spec: + containers: + - name: my-nginx + image: nginx diff --git a/kubernetes/examples/pod1.yaml b/kubernetes/examples/pod1.yaml new file mode 100644 index 00000000..217f6977 --- /dev/null +++ b/kubernetes/examples/pod1.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Pod +metadata: + name: nginx-pod +spec: + containers: + - name: my-nginx + image: nginx diff --git a/kubernetes/examples/pod2.yaml b/kubernetes/examples/pod2.yaml new file mode 100644 index 00000000..17c9bad4 --- /dev/null +++ b/kubernetes/examples/pod2.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Pod +metadata: + name: nginx-pod2 +spec: + containers: + - name: my-nginx + image: nginx diff --git a/kubernetes/examples/pod3.yaml b/kubernetes/examples/pod3.yaml new file mode 100644 index 00000000..700e91c0 --- /dev/null +++ b/kubernetes/examples/pod3.yaml @@ -0,0 +1,17 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: nginx-pod-daemonset + namespace: project-controller +spec: + selector: + matchLabels: + app: nginx-pod-daemonset + template: + metadata: + labels: + app: nginx-pod-daemonset + spec: + containers: + - name: my-nginx + image: nginx diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/configuration/ControllerConfiguration.java b/src/main/java/io/ten1010/aipub/projectcontroller/configuration/ControllerConfiguration.java index 143fb290..6bccb32b 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/configuration/ControllerConfiguration.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/configuration/ControllerConfiguration.java @@ -8,10 +8,7 @@ import io.kubernetes.client.informer.SharedInformerFactory; import io.ten1010.aipub.projectcontroller.controller.cluster.NamespaceControllerFactory; import io.ten1010.aipub.projectcontroller.controller.cluster.NodeControllerFactory; -import io.ten1010.aipub.projectcontroller.controller.cr.AipubUserControllerFactory; -import io.ten1010.aipub.projectcontroller.controller.cr.ImageHubControllerFactory; -import io.ten1010.aipub.projectcontroller.controller.cr.NodeGroupControllerFactory; -import io.ten1010.aipub.projectcontroller.controller.cr.ProjectControllerFactory; +import io.ten1010.aipub.projectcontroller.controller.cr.*; import io.ten1010.aipub.projectcontroller.controller.namespaced.ImagePullSecretReconcilerFactory; import io.ten1010.aipub.projectcontroller.controller.namespaced.ResourceQuotaControllerFactory; import io.ten1010.aipub.projectcontroller.controller.rbac.aipub.AipubUserClusterRoleBindingControllerFactory; @@ -39,7 +36,6 @@ public class ControllerConfiguration { @Bean public ControllerManager controllerManager( SharedInformerFactory sharedInformerFactory, List controllers, List> workloadControllerFactories) { - System.out.println(controllers); ControllerManagerBuilder builder = ControllerBuilder.controllerManagerBuilder(sharedInformerFactory); controllers.forEach(builder::addController); workloadControllerFactories.forEach(f -> builder.addController(f.createController())); @@ -59,6 +55,14 @@ public Controller projectController(SharedInformerFactory sharedInformerFactory, .createController(); } + @Bean + public Controller nodeMaintenanceController(SharedInformerFactory sharedInformerFactory, + K8sApiProvider k8sApiProvider, + ReconciliationService reconciliationService) { + return new NodeMaintenanceControllerFactory(sharedInformerFactory, k8sApiProvider, reconciliationService) + .createController(); + } + @Bean public Controller aipubUserController(SharedInformerFactory sharedInformerFactory, K8sApiProvider k8sApiProvider, diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/configuration/MutatingConfiguration.java b/src/main/java/io/ten1010/aipub/projectcontroller/configuration/MutatingConfiguration.java index 2daa33f9..a8892058 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/configuration/MutatingConfiguration.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/configuration/MutatingConfiguration.java @@ -63,4 +63,9 @@ public ImageReviewReviewHandler imageReviewReviewHandler( return new ImageReviewReviewHandler(repositoryService, artifactService, sharedInformerFactory); } + @Bean + public NodeMaintenanceReviewHandler nodeMaintenanceReviewHandler(AipubProperties aipubProperties, SubjectResolver subjectResolver, SharedInformerFactory sharedInformerFactory) { + return new NodeMaintenanceReviewHandler(aipubProperties, subjectResolver, sharedInformerFactory); + } + } diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/controller/BoundObjectResolver.java b/src/main/java/io/ten1010/aipub/projectcontroller/controller/BoundObjectResolver.java index 7791b6a7..eacfca3e 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/controller/BoundObjectResolver.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/controller/BoundObjectResolver.java @@ -3,6 +3,8 @@ import io.kubernetes.client.informer.SharedInformerFactory; import io.kubernetes.client.informer.cache.Indexer; import io.kubernetes.client.openapi.models.V1Node; +import io.kubernetes.client.openapi.models.V1OwnerReference; +import io.kubernetes.client.openapi.models.V1Pod; import io.ten1010.aipub.projectcontroller.domain.k8s.KeyResolver; import io.ten1010.aipub.projectcontroller.domain.k8s.dto.*; import io.ten1010.aipub.projectcontroller.domain.k8s.util.*; @@ -26,6 +28,8 @@ private static List getIntersection(List list1, List list2) { private final Indexer imageHubIndexer; private final Indexer nodeIndexer; private final Indexer resourceSetIndexer; + private final Indexer nodeMaintenanceIndexer; + private final Indexer podIndexer; public BoundObjectResolver(SharedInformerFactory sharedInformerFactory) { this.keyResolver = new KeyResolver(); @@ -47,6 +51,12 @@ public BoundObjectResolver(SharedInformerFactory sharedInformerFactory) { this.resourceSetIndexer = sharedInformerFactory .getExistingSharedIndexInformer(V1alpha1ResourceSet.class) .getIndexer(); + this.nodeMaintenanceIndexer = sharedInformerFactory + .getExistingSharedIndexInformer(V1alpha1NodeMaintenance.class) + .getIndexer(); + this.podIndexer = sharedInformerFactory + .getExistingSharedIndexInformer(V1Pod.class) + .getIndexer(); } public List getAllBoundAipubUsers(V1alpha1Project project) { @@ -133,6 +143,28 @@ public List getAllBoundNodeGroups(V1Node node) { return K8sObjectUtils.distinctByKey(this.keyResolver, allBoundNodeGroups); } + public List getAllBoundNodeMaintenances(V1Node node) { + List allBoundNodeGroups = this.nodeMaintenanceIndexer.byIndex( + IndexerConstants.NODE_NAME_TO_NODE_MAINTENANCE_INDEXER_NAME, + K8sObjectUtils.getName(node)); + return K8sObjectUtils.distinctByKey(this.keyResolver, allBoundNodeGroups); + } + + public List getAllBoundNodeByNodeMaintenances(V1alpha1NodeMaintenance node) { + List allBoundNodeGroups = new ArrayList<>(); + for (String targetNode : node.getSpec().getTargetNodes()) { + Optional opt = Optional.ofNullable(this.nodeIndexer.getByKey(targetNode)); + if (opt.isPresent()) { + allBoundNodeGroups.add(opt.get()); + } + } + return K8sObjectUtils.distinctByKey(this.keyResolver, allBoundNodeGroups); + } + + public List getAllBoundPods(String nodeName) { + return podIndexer.byIndex(IndexerConstants.NODE_NAME_TO_POD_INDEXER_NAME, nodeName); + } + public List getAllBoundResourceSets(V1alpha1Project project) { List allBoundNodes = new ArrayList<>(getDirectlyBoundNodes(project)); List allBoundResourceSets = new ArrayList<>(); diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/controller/cluster/NodeControllerFactory.java b/src/main/java/io/ten1010/aipub/projectcontroller/controller/cluster/NodeControllerFactory.java index 935fe209..b121bf46 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/controller/cluster/NodeControllerFactory.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/controller/cluster/NodeControllerFactory.java @@ -14,6 +14,7 @@ import io.ten1010.aipub.projectcontroller.domain.k8s.K8sApiProvider; import io.ten1010.aipub.projectcontroller.domain.k8s.ReconciliationService; import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1alpha1NodeGroup; +import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1alpha1NodeMaintenance; import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1alpha1Project; public class NodeControllerFactory implements ControllerFactory { @@ -43,9 +44,11 @@ public Controller createController() { .withReadyFunc(this.sharedInformerFactory.getExistingSharedIndexInformer(V1Node.class)::hasSynced) .withReadyFunc(this.sharedInformerFactory.getExistingSharedIndexInformer(V1alpha1Project.class)::hasSynced) .withReadyFunc(this.sharedInformerFactory.getExistingSharedIndexInformer(V1alpha1NodeGroup.class)::hasSynced) + .withReadyFunc(this.sharedInformerFactory.getExistingSharedIndexInformer(V1alpha1NodeMaintenance.class)::hasSynced) .watch(this::createNodeWatch) .watch(this::createProjectWatch) .watch(this::createNodeGroupWatch) + .watch(this::updateNodeMaintenanceWatch) .withReconciler(new NodeReconciler(this.sharedInformerFactory, this.k8sApiProvider, this.reconciliationService)) .build(); } @@ -70,4 +73,11 @@ private ControllerWatch createNodeGroupWatch(WorkQueue updateNodeMaintenanceWatch(WorkQueue workQueue) { + DefaultControllerWatch watch = new DefaultControllerWatch<>(workQueue, V1alpha1NodeMaintenance.class); + watch.setOnUpdateFilter(this.onUpdateFilterFactory.nodeMaintenanceCreateFilter()); + watch.setRequestBuilder(this.requestBuilderFactory.nodeMaintenanceToBoundNodes()); + return watch; + } + } diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/controller/cluster/NodeReconciler.java b/src/main/java/io/ten1010/aipub/projectcontroller/controller/cluster/NodeReconciler.java index 1c9dad27..e5bf27e4 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/controller/cluster/NodeReconciler.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/controller/cluster/NodeReconciler.java @@ -14,9 +14,9 @@ import io.ten1010.aipub.projectcontroller.controller.RequestHelper; import io.ten1010.aipub.projectcontroller.domain.k8s.K8sApiProvider; import io.ten1010.aipub.projectcontroller.domain.k8s.KeyResolver; +import io.ten1010.aipub.projectcontroller.domain.k8s.NodeMaintenanceConstants; import io.ten1010.aipub.projectcontroller.domain.k8s.ReconciliationService; -import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1alpha1NodeGroup; -import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1alpha1Project; +import io.ten1010.aipub.projectcontroller.domain.k8s.dto.*; import io.ten1010.aipub.projectcontroller.domain.k8s.util.K8sObjectUtils; import io.ten1010.aipub.projectcontroller.domain.k8s.util.NodeUtils; @@ -61,6 +61,11 @@ protected Result reconcileInternal(Request request) throws ApiException { Map reconciledAnnotations = this.reconciliationService.reconcileNodeAnnotations(node, boundProjects, boundNodeGroups); List reconciledTaints = this.reconciliationService.reconcileTaints(node); + List allBoundNodeMaintenances = this.boundObjectResolver.getAllBoundNodeMaintenances(node); + if (!allBoundNodeMaintenances.isEmpty()) { + return executeSchedulable(node, allBoundNodeMaintenances); + } + return reconcileExistingNode(nodeOpt.get(), reconciledLabels, reconciledAnnotations, reconciledTaints); } @@ -87,6 +92,29 @@ private Result reconcileExistingNode( return new Result(false); } + private Result executeSchedulable(V1Node targetNode, List nodeMaintenances) throws ApiException { + String nodeName = K8sObjectUtils.getName(targetNode); + for (V1alpha1NodeMaintenance nodeMaintenance : nodeMaintenances) { + var progressList = nodeMaintenance.getStatus().getActions().stream() + .filter(x -> !x.getType().equals(NodeMaintenanceConstants.NN_DRAIN) + && x.getStatus().equals(NodeMaintenanceConstants.NN_PROGRESS)) + .toList(); + if (progressList.isEmpty()) { + break; + } + + for (V1alpha1NodeMaintenanceAction action : nodeMaintenance.getSpec().getActions()) { + if (action.getType().equals(NodeMaintenanceConstants.NN_CORDON)) { + targetNode.getSpec().setUnschedulable(true); + } else if (action.getType().equals(NodeMaintenanceConstants.NN_UNCORDON)) { + targetNode.getSpec().setUnschedulable(false); + } + } + } + updateNode(nodeName, targetNode); + return new Result(false); + } + private void updateNode(String objName, V1Node node) throws ApiException { this.coreV1Api .replaceNode(objName, node) diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/controller/cr/NodeMaintenanceControllerFactory.java b/src/main/java/io/ten1010/aipub/projectcontroller/controller/cr/NodeMaintenanceControllerFactory.java new file mode 100644 index 00000000..1d89c59e --- /dev/null +++ b/src/main/java/io/ten1010/aipub/projectcontroller/controller/cr/NodeMaintenanceControllerFactory.java @@ -0,0 +1,66 @@ +package io.ten1010.aipub.projectcontroller.controller.cr; + +import io.kubernetes.client.extended.controller.Controller; +import io.kubernetes.client.extended.controller.ControllerWatch; +import io.kubernetes.client.extended.controller.builder.ControllerBuilder; +import io.kubernetes.client.extended.controller.reconciler.Request; +import io.kubernetes.client.extended.workqueue.WorkQueue; +import io.kubernetes.client.informer.SharedInformerFactory; +import io.kubernetes.client.openapi.models.V1Node; +import io.kubernetes.client.openapi.models.V1Pod; +import io.ten1010.aipub.projectcontroller.controller.ControllerFactory; +import io.ten1010.aipub.projectcontroller.controller.watch.DefaultControllerWatch; +import io.ten1010.aipub.projectcontroller.controller.watch.OnUpdateFilterFactory; +import io.ten1010.aipub.projectcontroller.domain.k8s.K8sApiProvider; +import io.ten1010.aipub.projectcontroller.domain.k8s.ReconciliationService; +import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1alpha1NodeMaintenance; + +public class NodeMaintenanceControllerFactory implements ControllerFactory { + + private final SharedInformerFactory sharedInformerFactory; + private final K8sApiProvider k8sApiProvider; + private final OnUpdateFilterFactory onUpdateFilterFactory; + + public NodeMaintenanceControllerFactory( + SharedInformerFactory sharedInformerFactory, + K8sApiProvider k8sApiProvider, + ReconciliationService reconciliationService) { + this.sharedInformerFactory = sharedInformerFactory; + this.k8sApiProvider = k8sApiProvider; + this.onUpdateFilterFactory = new OnUpdateFilterFactory(); + } + + @Override + public Controller createController() { + return ControllerBuilder.defaultBuilder(this.sharedInformerFactory) + .withName("node-maintenance-controller") + .withWorkerCount(1) + .withReadyFunc(this.sharedInformerFactory.getExistingSharedIndexInformer(V1alpha1NodeMaintenance.class)::hasSynced) + .withReadyFunc(this.sharedInformerFactory.getExistingSharedIndexInformer(V1Node.class)::hasSynced) + .withReadyFunc(this.sharedInformerFactory.getExistingSharedIndexInformer(V1Pod.class)::hasSynced) + .watch(this::createNodeMaintenanceWatch) + .watch(this::updateNodeStatus) + .watch(this::updatePodStatus) + .withReconciler(new NodeMaintenanceReconciler(this.sharedInformerFactory, this.k8sApiProvider)) + .build(); + } + + private ControllerWatch updateNodeStatus(WorkQueue workQueue) { + DefaultControllerWatch watch = new DefaultControllerWatch<>(workQueue, V1Node.class); + watch.setOnUpdateFilter(this.onUpdateFilterFactory.nodeFilter()); + return watch; + } + + private ControllerWatch updatePodStatus(WorkQueue workQueue) { + DefaultControllerWatch watch = new DefaultControllerWatch<>(workQueue, V1Pod.class); + watch.setOnUpdateFilter(this.onUpdateFilterFactory.podNodeNameFieldFilter()); + return watch; + } + + private ControllerWatch createNodeMaintenanceWatch(WorkQueue workQueue) { + DefaultControllerWatch watch = new DefaultControllerWatch<>(workQueue, V1alpha1NodeMaintenance.class); + watch.setOnUpdateFilter(this.onUpdateFilterFactory.nodeMaintenanceCreateFilter()); + return watch; + } + +} diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/controller/cr/NodeMaintenanceReconciler.java b/src/main/java/io/ten1010/aipub/projectcontroller/controller/cr/NodeMaintenanceReconciler.java new file mode 100644 index 00000000..70adf37f --- /dev/null +++ b/src/main/java/io/ten1010/aipub/projectcontroller/controller/cr/NodeMaintenanceReconciler.java @@ -0,0 +1,159 @@ +package io.ten1010.aipub.projectcontroller.controller.cr; + +import io.kubernetes.client.extended.controller.reconciler.Request; +import io.kubernetes.client.extended.controller.reconciler.Result; +import io.kubernetes.client.informer.SharedInformerFactory; +import io.kubernetes.client.informer.cache.Indexer; +import io.kubernetes.client.openapi.ApiException; +import io.kubernetes.client.openapi.models.V1Node; +import io.kubernetes.client.openapi.models.V1Pod; +import io.ten1010.aipub.projectcontroller.controller.AbstractReconciler; +import io.ten1010.aipub.projectcontroller.controller.BoundObjectResolver; +import io.ten1010.aipub.projectcontroller.controller.RequestHelper; +import io.ten1010.aipub.projectcontroller.domain.k8s.*; +import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1alpha1NodeMaintenance; +import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1alpha1NodeMaintenanceAction; +import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1alpha1NodeMaintenanceStatus; +import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1alpha1NodeMaintenanceStatusAction; +import io.ten1010.aipub.projectcontroller.domain.k8s.util.K8sObjectUtils; +import io.ten1010.aipub.projectcontroller.domain.k8s.util.StatusPatchHelper; +import lombok.extern.slf4j.Slf4j; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +@Slf4j +public class NodeMaintenanceReconciler extends AbstractReconciler { + + private final KeyResolver keyResolver; + private final Indexer projectIndexer; + private final BoundObjectResolver boundObjectResolver; + private final StatusPatchHelper statusPatchHelper; + + public NodeMaintenanceReconciler( + SharedInformerFactory sharedInformerFactory, + K8sApiProvider k8sApiProvider) { + this.keyResolver = new KeyResolver(); + this.projectIndexer = sharedInformerFactory + .getExistingSharedIndexInformer(V1alpha1NodeMaintenance.class) + .getIndexer(); + this.boundObjectResolver = new BoundObjectResolver(sharedInformerFactory); + this.statusPatchHelper = new StatusPatchHelper<>( + k8sApiProvider.getApiClient(), + K8sObjectTypeConstants.NODE_MAINTENANCE_V1ALPHA1, + ProjectApiConstants.NODE_MAINTENANCE_RESOURCE_PLURAL); + } + + @Override + protected Result reconcileInternal(Request request) throws ApiException { + String projectKey = new RequestHelper(this.keyResolver).resolveKey(request); + Optional nodeMaintenanceOpt = Optional.ofNullable(this.projectIndexer.getByKey(projectKey)); + if (nodeMaintenanceOpt.isEmpty()) { + return new Result(false); + } + + var nodeMaintenance = nodeMaintenanceOpt.get(); + var spec = Objects.requireNonNull(nodeMaintenance.getSpec()); + var status = nodeMaintenance.getStatus(); + List targetNodeNames = Objects.requireNonNull(spec.getTargetNodes()); + + if (status != null && targetNodeNames.equals(status.getAllEffectedNodes())) { + List specAction = spec.getActions().stream().map(V1alpha1NodeMaintenanceAction::getType).toList(); + List statusAction = new ArrayList<>(); + List statusNotCompleted = new ArrayList<>(); + for (V1alpha1NodeMaintenanceStatusAction action : status.getActions()) { + statusAction.add(action.getType()); + if (action.getStatus() == null || !action.getStatus().equals(NodeMaintenanceConstants.NN_COMPLETED)) { + statusNotCompleted.add("NOT COMPLETED"); + } + } + if (specAction.equals(statusAction) && statusNotCompleted.isEmpty()) { + return new Result(false); + } + } + + V1alpha1NodeMaintenanceStatus edited = new V1alpha1NodeMaintenanceStatus(); + edited.setAllEffectedNodes(targetNodeNames); + List actionList = new ArrayList<>(); + for (V1alpha1NodeMaintenanceAction action : nodeMaintenance.getSpec().getActions()) { + V1alpha1NodeMaintenanceStatusAction _action = new V1alpha1NodeMaintenanceStatusAction(); + _action.setType(action.getType()); + _action.setStatus(NodeMaintenanceConstants.NN_PROGRESS); + actionList.add(_action); + } + edited.setActions(actionList); + nodeMaintenance.setStatus(edited); + this.updateNodeMaintenanceStatus(nodeMaintenance); + + if (nodeMaintenance.getStatus() != null) { + List nodeList = this.boundObjectResolver.getAllBoundNodeByNodeMaintenances(nodeMaintenance); + for (V1alpha1NodeMaintenanceStatusAction action : nodeMaintenance.getStatus().getActions()) { + if (action.getType().equals(NodeMaintenanceConstants.NN_CORDON) && action.getStatus().equals(NodeMaintenanceConstants.NN_PROGRESS)) { + for (V1Node _node : nodeList) { + if (_node.getSpec().getUnschedulable() != null && _node.getSpec().getUnschedulable()) { + action.setStatus(NodeMaintenanceConstants.NN_COMPLETED); + } + } + } + if (action.getType().equals(NodeMaintenanceConstants.NN_UNCORDON) && action.getStatus().equals(NodeMaintenanceConstants.NN_PROGRESS)) { + for (V1Node _node : nodeList) { + if (_node.getSpec().getUnschedulable() == null || !_node.getSpec().getUnschedulable()) { + action.setStatus(NodeMaintenanceConstants.NN_COMPLETED); + } + } + } + if (action.getType().equals(NodeMaintenanceConstants.NN_DRAIN) && action.getStatus().equals(NodeMaintenanceConstants.NN_PROGRESS)) { + for (V1Node _node : nodeList) { + boolean isDeleted = isDrainedTargetNode(_node); + if (isDeleted) { + action.setStatus(NodeMaintenanceConstants.NN_COMPLETED); + } + } + } + } + this.updateNodeMaintenanceStatus(nodeMaintenance); + } + + return new Result(true, Duration.ofSeconds(1)); + } + + private boolean isDrainedTargetNode(V1Node node) { + List nodeMaintenanceList = this.boundObjectResolver.getAllBoundNodeMaintenances(node); + List pods = this.boundObjectResolver.getAllBoundPods(node.getMetadata().getName()); + if (nodeMaintenanceList.isEmpty()) { + return false; + } + int resultCnt = 0; + for (V1alpha1NodeMaintenance allBoundNodeGroup : nodeMaintenanceList) { + if (allBoundNodeGroup.getSpec().getTargetNodes().contains(node.getMetadata().getName())) { + var actions = allBoundNodeGroup.getSpec().getActions(); + for (V1Pod pod : pods) { + var ownerReferences = Objects.requireNonNull(pod.getMetadata().getOwnerReferences()); + boolean isDaemonSet = ownerReferences.stream() + .anyMatch(x -> x.getKind().equalsIgnoreCase(NodeMaintenanceConstants.NN_DAEMONSET)); + for (V1alpha1NodeMaintenanceAction action : actions) { + if (action.getType().equals(NodeMaintenanceConstants.NN_DRAIN)) { + if (isDaemonSet) { + if (action.getIgnoreDaemonSets()) { + resultCnt++; + } + } else { + resultCnt++; + } + } + } + } + } + } + return resultCnt == 0 ? true : false; + } + + private void updateNodeMaintenanceStatus(V1alpha1NodeMaintenance nodeMaintenance) throws ApiException { + Objects.requireNonNull(nodeMaintenance.getStatus()); + this.statusPatchHelper.patchStatus(null, K8sObjectUtils.getName(nodeMaintenance), nodeMaintenance.getStatus()); + } + +} diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/controller/watch/OnUpdateFilterFactory.java b/src/main/java/io/ten1010/aipub/projectcontroller/controller/watch/OnUpdateFilterFactory.java index 20abab3d..2a30b1a6 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/controller/watch/OnUpdateFilterFactory.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/controller/watch/OnUpdateFilterFactory.java @@ -5,6 +5,8 @@ import io.ten1010.aipub.projectcontroller.domain.k8s.dto.*; import io.ten1010.aipub.projectcontroller.domain.k8s.util.*; +import java.util.ArrayList; +import java.util.List; import java.util.Objects; import java.util.Set; import java.util.function.BiPredicate; @@ -27,6 +29,21 @@ public BiPredicate projectSpecFieldFilter() { return (oldObj, newObj) -> !Objects.equals(oldObj.getSpec(), newObj.getSpec()); } + public BiPredicate nodeMaintenanceCreateFilter() { + return (oldObj, newObj) -> { + if (newObj.getStatus() != null) { + List actionTypes = newObj.getSpec().getActions().stream().map(V1alpha1NodeMaintenanceAction::getType).toList(); + List statusTypes = newObj.getStatus().getActions().stream().map(V1alpha1NodeMaintenanceStatusAction::getType).toList(); + + return !Objects.equals(oldObj.getSpec(), newObj.getSpec()) + || !newObj.getSpec().getTargetNodes().equals(newObj.getStatus().getAllEffectedNodes()) + || !actionTypes.equals(statusTypes); + } else { + return oldObj.getSpec() == null && newObj.getSpec() != null; + } + }; + } + public BiPredicate projectSpecQuotaFieldFilter() { return (oldObj, newObj) -> !ProjectUtils.getSpecQuota(oldObj).equals(ProjectUtils.getSpecQuota(newObj)); } diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/controller/watch/RequestBuilderFactory.java b/src/main/java/io/ten1010/aipub/projectcontroller/controller/watch/RequestBuilderFactory.java index 5f67ffb0..bc93304b 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/controller/watch/RequestBuilderFactory.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/controller/watch/RequestBuilderFactory.java @@ -11,6 +11,7 @@ import io.ten1010.aipub.projectcontroller.domain.k8s.util.K8sObjectUtils; import io.ten1010.aipub.projectcontroller.informer.IndexerConstants; +import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.function.Function; @@ -174,6 +175,27 @@ public Function> nodeGroupToBoundNodes() { .toList(); } + public Function> nodeMaintenanceToBoundNodes() { + return nodeMaintenance -> this.boundObjectResolver.getAllBoundNodeByNodeMaintenances(nodeMaintenance).stream() + .map(K8sObjectUtils::getName) + .map(Request::new) + .toList(); + } + + public Function> podListByNodeMaintenance() { + Indexer podIndexer = this.sharedInformerFactory.getExistingSharedIndexInformer(V1Pod.class).getIndexer(); + return nodeMaintenance -> { + List requestList = new ArrayList<>(); + for (String targetNode : nodeMaintenance.getSpec().getTargetNodes()) { + List pods = podIndexer.byIndex(IndexerConstants.NODE_NAME_TO_POD_INDEXER_NAME, targetNode); + for (V1Pod pod : pods) { + requestList.add(new Request(K8sObjectUtils.getNamespace(pod), K8sObjectUtils.getName(pod))); + } + } + return requestList; + }; + } + public Function> nodeGroupToNamespacedObjects(Class objectClass) { Function> projectToNamespacedObjects = projectToNamespacedObjects(objectClass); return nodeGroup -> { diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/controller/workload/PodControllerFactory.java b/src/main/java/io/ten1010/aipub/projectcontroller/controller/workload/PodControllerFactory.java index 888894c0..1d0f0d92 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/controller/workload/PodControllerFactory.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/controller/workload/PodControllerFactory.java @@ -14,6 +14,7 @@ import io.ten1010.aipub.projectcontroller.controller.watch.RequestBuilderFactory; import io.ten1010.aipub.projectcontroller.domain.k8s.K8sApiProvider; import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1alpha1NodeGroup; +import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1alpha1NodeMaintenance; import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1alpha1Project; public class PodControllerFactory implements ControllerFactory { @@ -44,11 +45,13 @@ public Controller createController() { .withReadyFunc(this.sharedInformerFactory.getExistingSharedIndexInformer(V1alpha1Project.class)::hasSynced) .withReadyFunc(this.sharedInformerFactory.getExistingSharedIndexInformer(V1alpha1NodeGroup.class)::hasSynced) .withReadyFunc(this.sharedInformerFactory.getExistingSharedIndexInformer(V1Node.class)::hasSynced) + .withReadyFunc(this.sharedInformerFactory.getExistingSharedIndexInformer(V1alpha1NodeMaintenance.class)::hasSynced) .watch(this::createPodWatch) .watch(this::createProjectWatch) .watch(this::createNodeGroupWatch) .watch(this::createNodeWatch) .watch(this::createBoundPodNodeWatch) + .watch(this::updateNodeMaintenanceWatch) .withReconciler(new PodReconciler( this.sharedInformerFactory, this.k8sApiProvider, @@ -90,4 +93,11 @@ private ControllerWatch createBoundPodNodeWatch(WorkQueue workQ return watch; } + private ControllerWatch updateNodeMaintenanceWatch(WorkQueue workQueue) { + DefaultControllerWatch watch = new DefaultControllerWatch<>(workQueue, V1alpha1NodeMaintenance.class); + watch.setOnUpdateFilter(this.onUpdateFilterFactory.nodeMaintenanceCreateFilter()); + watch.setRequestBuilder(this.requestBuilderFactory.podListByNodeMaintenance()); + return watch; + } + } diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/controller/workload/PodReconciler.java b/src/main/java/io/ten1010/aipub/projectcontroller/controller/workload/PodReconciler.java index e22b2fe0..477986ca 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/controller/workload/PodReconciler.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/controller/workload/PodReconciler.java @@ -7,18 +7,24 @@ import io.kubernetes.client.openapi.ApiException; import io.kubernetes.client.openapi.apis.CoreV1Api; import io.kubernetes.client.openapi.models.V1Node; +import io.kubernetes.client.openapi.models.V1OwnerReference; import io.kubernetes.client.openapi.models.V1Pod; import io.ten1010.aipub.projectcontroller.controller.AbstractReconciler; +import io.ten1010.aipub.projectcontroller.controller.BoundObjectResolver; import io.ten1010.aipub.projectcontroller.controller.RequestHelper; import io.ten1010.aipub.projectcontroller.domain.k8s.K8sApiProvider; import io.ten1010.aipub.projectcontroller.domain.k8s.KeyResolver; import io.ten1010.aipub.projectcontroller.domain.k8s.NamespaceNameResolver; +import io.ten1010.aipub.projectcontroller.domain.k8s.NodeMaintenanceConstants; +import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1alpha1NodeMaintenance; +import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1alpha1NodeMaintenanceAction; import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1alpha1Project; import io.ten1010.aipub.projectcontroller.domain.k8s.util.K8sObjectUtils; import io.ten1010.aipub.projectcontroller.domain.k8s.util.NodeUtils; import io.ten1010.aipub.projectcontroller.domain.k8s.util.WorkloadUtils; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -32,6 +38,7 @@ public class PodReconciler extends AbstractReconciler { private final Indexer projectIndexer; private final CoreV1Api coreV1Api; private final PodNodesResolver podNodesResolver; + private final BoundObjectResolver boundObjectResolver; public PodReconciler( SharedInformerFactory sharedInformerFactory, @@ -50,6 +57,7 @@ public PodReconciler( .getIndexer(); this.coreV1Api = new CoreV1Api(k8sApiProvider.getApiClient()); this.podNodesResolver = podNodesResolver; + this.boundObjectResolver = new BoundObjectResolver(sharedInformerFactory); } @Override @@ -75,12 +83,54 @@ protected Result reconcileInternal(Request request) throws ApiException { return processCaseThatProjectManagedNode(node, pod); } + List targetNodes = this.boundObjectResolver.getAllBoundNodeMaintenances(node); + if (!targetNodes.isEmpty()) { + return nodeMaintenancePodDrain(targetNodes, node, pod); + } + return processCaseThatNotProjectManagedNode(pod); } - private void deletePod(V1Pod pod) throws ApiException { + private Result nodeMaintenancePodDrain(List targetNodes, V1Node node, V1Pod pod) throws ApiException { + for (V1alpha1NodeMaintenance nodeMaintenance : targetNodes) { + if (nodeMaintenance.getSpec().getTargetNodes().contains(node.getMetadata().getName())) { + var statusActions = nodeMaintenance.getStatus().getActions().stream() + .filter(a -> a.getType().equals(NodeMaintenanceConstants.NN_DRAIN) + && a.getStatus().equals(NodeMaintenanceConstants.NN_PROGRESS)) + .toList(); + if (statusActions.isEmpty()) { + return new Result(false); + } + + var ownerReferences = Objects.requireNonNull(pod.getMetadata().getOwnerReferences()); + boolean isDaemonSet = ownerReferences.stream() + .anyMatch(x -> x.getKind().equalsIgnoreCase(NodeMaintenanceConstants.NN_DAEMONSET)); + + var specActions = nodeMaintenance.getSpec().getActions().stream() + .filter(a -> a.getType().equals(NodeMaintenanceConstants.NN_DRAIN)).toList(); + for (V1alpha1NodeMaintenanceAction action : specActions) { + if (isDaemonSet) { + if (action.getIgnoreDaemonSets()) { + deletePod(pod, action.getForce()); + } + } else { + deletePod(pod, action.getForce()); + } + } + } + } + return new Result(false); + } + + private void deletePod(V1Pod pod, boolean isForce) throws ApiException { if (K8sObjectUtils.getDeletionTimestamp(pod).isEmpty()) { + int terminationGrace = pod.getSpec().getTerminationGracePeriodSeconds().intValue(); + if (isForce) { + terminationGrace = 0; + } + this.coreV1Api.deleteNamespacedPod(K8sObjectUtils.getName(pod), K8sObjectUtils.getNamespace(pod)) + .gracePeriodSeconds(terminationGrace) .execute(); } } @@ -94,7 +144,7 @@ private Result processCaseThatProjectManagedNode(V1Node node, V1Pod pod) throws try { allowedProjectNodeObjects = this.podNodesResolver.getNodes(pod); } catch (UnsupportedControllerException e) { - deletePod(pod); + deletePod(pod, false); return new Result(false); } @@ -102,7 +152,7 @@ private Result processCaseThatProjectManagedNode(V1Node node, V1Pod pod) throws .map(K8sObjectUtils::getName) .collect(Collectors.toSet()); if (!allowedProjectNodes.contains(K8sObjectUtils.getName(node))) { - deletePod(pod); + deletePod(pod, false); return new Result(false); } return new Result(false); @@ -114,7 +164,7 @@ private Result processCaseThatNotProjectManagedNode(V1Pod pod) throws ApiExcepti V1alpha1Project project = this.projectIndexer.getByKey(projKey); if (project != null) { - deletePod(pod); + deletePod(pod, false); } return new Result(false); } diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/K8sObjectTypeConstants.java b/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/K8sObjectTypeConstants.java index faddb2e7..4746ab3c 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/K8sObjectTypeConstants.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/K8sObjectTypeConstants.java @@ -17,6 +17,12 @@ public final class K8sObjectTypeConstants { ProjectApiConstants.VERSION, ProjectApiConstants.NODE_GROUP_RESOURCE_KIND), V1alpha1NodeGroup.class); + public static final K8sObjectType NODE_MAINTENANCE_V1ALPHA1 = new K8sObjectType<>( + new K8sObjectTypeKey( + ProjectApiConstants.PROJECT_GROUP, + ProjectApiConstants.VERSION, + ProjectApiConstants.NODE_MAINTENANCE_RESOURCE_KIND), + V1alpha1NodeMaintenance.class); public static final K8sObjectType AIPUB_USER_V1ALPHA1 = new K8sObjectType( new K8sObjectTypeKey( ProjectApiConstants.PROJECT_GROUP, diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/NodeMaintenanceConstants.java b/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/NodeMaintenanceConstants.java new file mode 100644 index 00000000..b07d4c97 --- /dev/null +++ b/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/NodeMaintenanceConstants.java @@ -0,0 +1,15 @@ +package io.ten1010.aipub.projectcontroller.domain.k8s; + +public final class NodeMaintenanceConstants { + + public static final String NN_DRAIN = "drain"; + public static final String NN_CORDON = "cordon"; + public static final String NN_UNCORDON = "uncordon"; + public static final String NN_PROGRESS = "PROGRESS"; + public static final String NN_COMPLETED = "COMPLETED"; + public static final String NN_DAEMONSET = "DaemonSet"; + + private NodeMaintenanceConstants() { + } + +} diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/ProjectApiConstants.java b/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/ProjectApiConstants.java index 380d4a19..ae35cb04 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/ProjectApiConstants.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/ProjectApiConstants.java @@ -16,6 +16,9 @@ public final class ProjectApiConstants { public static final String NODE_GROUP_RESOURCE_KIND = "NodeGroup"; public static final String NODE_GROUP_RESOURCE_PLURAL = "nodegroups"; + public static final String NODE_MAINTENANCE_RESOURCE_KIND = "NodeMaintenance"; + public static final String NODE_MAINTENANCE_RESOURCE_PLURAL = "nodemaintenances"; + public static final String AIPUB_USER_RESOURCE_KIND = "AipubUser"; public static final String AIPUB_USER_RESOURCE_PLURAL = "aipubusers"; diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/dto/V1alpha1NodeMaintenance.java b/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/dto/V1alpha1NodeMaintenance.java new file mode 100644 index 00000000..fcce5a62 --- /dev/null +++ b/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/dto/V1alpha1NodeMaintenance.java @@ -0,0 +1,22 @@ +package io.ten1010.aipub.projectcontroller.domain.k8s.dto; + +import io.kubernetes.client.common.KubernetesObject; +import io.kubernetes.client.openapi.models.V1ObjectMeta; +import lombok.Data; +import org.jspecify.annotations.Nullable; + +@Data +public class V1alpha1NodeMaintenance implements KubernetesObject { + + @Nullable + private String apiVersion; + @Nullable + private String kind; + @Nullable + private V1ObjectMeta metadata; + @Nullable + private V1alpha1NodeMaintenanceSpec spec; + @Nullable + private V1alpha1NodeMaintenanceStatus status; + +} diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/dto/V1alpha1NodeMaintenanceAction.java b/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/dto/V1alpha1NodeMaintenanceAction.java new file mode 100644 index 00000000..097d35f0 --- /dev/null +++ b/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/dto/V1alpha1NodeMaintenanceAction.java @@ -0,0 +1,18 @@ +package io.ten1010.aipub.projectcontroller.domain.k8s.dto; + +import lombok.Data; +import org.jspecify.annotations.Nullable; + +@Data +public class V1alpha1NodeMaintenanceAction { + + @Nullable + private String type; + + @Nullable + private Boolean ignoreDaemonSets; + + @Nullable + private Boolean force; + +} diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/dto/V1alpha1NodeMaintenanceList.java b/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/dto/V1alpha1NodeMaintenanceList.java new file mode 100644 index 00000000..62464244 --- /dev/null +++ b/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/dto/V1alpha1NodeMaintenanceList.java @@ -0,0 +1,22 @@ +package io.ten1010.aipub.projectcontroller.domain.k8s.dto; + +import io.kubernetes.client.common.KubernetesListObject; +import io.kubernetes.client.openapi.models.V1ListMeta; +import lombok.Data; +import org.jspecify.annotations.Nullable; + +import java.util.List; + +@Data +public class V1alpha1NodeMaintenanceList implements KubernetesListObject { + + @Nullable + private String apiVersion; + @Nullable + private String kind; + @Nullable + private V1ListMeta metadata; + @Nullable + private List items; + +} diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/dto/V1alpha1NodeMaintenanceSpec.java b/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/dto/V1alpha1NodeMaintenanceSpec.java new file mode 100644 index 00000000..498b3490 --- /dev/null +++ b/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/dto/V1alpha1NodeMaintenanceSpec.java @@ -0,0 +1,16 @@ +package io.ten1010.aipub.projectcontroller.domain.k8s.dto; + +import lombok.Data; +import org.jspecify.annotations.Nullable; + +import java.util.List; + +@Data +public class V1alpha1NodeMaintenanceSpec { + + @Nullable + private List targetNodes; + @Nullable + private List actions; + +} diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/dto/V1alpha1NodeMaintenanceStatus.java b/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/dto/V1alpha1NodeMaintenanceStatus.java new file mode 100644 index 00000000..f70d74a7 --- /dev/null +++ b/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/dto/V1alpha1NodeMaintenanceStatus.java @@ -0,0 +1,17 @@ +package io.ten1010.aipub.projectcontroller.domain.k8s.dto; + +import lombok.Data; +import org.jspecify.annotations.Nullable; + +import java.util.List; + +@Data +public class V1alpha1NodeMaintenanceStatus { + + @Nullable + private List allEffectedNodes; + + @Nullable + private List actions; + +} diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/dto/V1alpha1NodeMaintenanceStatusAction.java b/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/dto/V1alpha1NodeMaintenanceStatusAction.java new file mode 100644 index 00000000..4ec8d703 --- /dev/null +++ b/src/main/java/io/ten1010/aipub/projectcontroller/domain/k8s/dto/V1alpha1NodeMaintenanceStatusAction.java @@ -0,0 +1,15 @@ +package io.ten1010.aipub.projectcontroller.domain.k8s.dto; + +import lombok.Data; +import org.jspecify.annotations.Nullable; + +@Data +public class V1alpha1NodeMaintenanceStatusAction { + + @Nullable + private String type; + + @Nullable + private String status; + +} diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/informer/IndexerConstants.java b/src/main/java/io/ten1010/aipub/projectcontroller/informer/IndexerConstants.java index 4d46aae7..ed1e81fc 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/informer/IndexerConstants.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/informer/IndexerConstants.java @@ -13,6 +13,8 @@ public final class IndexerConstants { public static final String ENABLE_NODE_SELECTOR_TO_NODE_GROUPS_INDEXER_NAME = "ENABLE_NODE_SELECTOR_TO_NODE_GROUPS"; public static final String NODE_NAME_TO_NODE_GROUPS_INDEXER_NAME = "NODE_NAME_TO_NODE_GROUPS"; + public static final String NODE_NAME_TO_NODE_MAINTENANCE_INDEXER_NAME = "NODE_NAME_TO_NODE_MAINTENANCE"; + public static final String NODE_NAME_TO_RESOURCE_SETS_INDEXER_NAME = "NODE_NAME_TO_RESOURCE_SETS_INDEXER_NAME"; public static final String NODE_NAME_TO_NODE_RESOURCE_STATUSES_INDEXER_NAME = "NODE_NAME_TO_NODE_RESOURCE_STATUSES_INDEXER_NAME"; diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/informer/SharedInformerFactoryProvider.java b/src/main/java/io/ten1010/aipub/projectcontroller/informer/SharedInformerFactoryProvider.java index db91ea84..3ac05788 100644 --- a/src/main/java/io/ten1010/aipub/projectcontroller/informer/SharedInformerFactoryProvider.java +++ b/src/main/java/io/ten1010/aipub/projectcontroller/informer/SharedInformerFactoryProvider.java @@ -4,11 +4,13 @@ import io.kubernetes.client.informer.SharedInformerFactory; import io.kubernetes.client.openapi.ApiClient; import io.kubernetes.client.openapi.apis.CoreV1Api; +import io.kubernetes.client.openapi.apis.CustomObjectsApi; import io.kubernetes.client.openapi.apis.RbacAuthorizationV1Api; import io.kubernetes.client.openapi.models.*; import io.kubernetes.client.util.CallGeneratorParams; import io.ten1010.aipub.projectcontroller.domain.k8s.K8sApiProvider; import io.ten1010.aipub.projectcontroller.domain.k8s.KeyResolver; +import io.ten1010.aipub.projectcontroller.domain.k8s.ProjectApiConstants; import io.ten1010.aipub.projectcontroller.domain.k8s.dto.*; import io.ten1010.aipub.projectcontroller.domain.k8s.util.*; @@ -32,6 +34,7 @@ public SharedInformerFactoryProvider(K8sApiProvider k8sApiProvider, List informer = informerFactory.sharedIndexInformerFor( + (CallGeneratorParams params) -> new CustomObjectsApi(apiClient) + .listClusterCustomObject(ProjectApiConstants.PROJECT_GROUP, ProjectApiConstants.VERSION, ProjectApiConstants.NODE_MAINTENANCE_RESOURCE_PLURAL) + .resourceVersion(params.resourceVersion) + .watch(params.watch) + .timeoutSeconds(params.timeoutSeconds) + .buildCall(null), + V1alpha1NodeMaintenance.class, + V1alpha1NodeMaintenanceList.class); + informer.addIndexers(Map.of( + IndexerConstants.NODE_NAME_TO_NODE_MAINTENANCE_INDEXER_NAME, + x -> x.getSpec().getTargetNodes()) + ); + } + private void registerAipubUserInformer(SharedInformerFactory informerFactory) { informerFactory.sharedIndexInformerFor( this.k8sApiProvider.getAipubUserApi(), diff --git a/src/main/java/io/ten1010/aipub/projectcontroller/mutating/service/NodeMaintenanceReviewHandler.java b/src/main/java/io/ten1010/aipub/projectcontroller/mutating/service/NodeMaintenanceReviewHandler.java new file mode 100644 index 00000000..b5980b35 --- /dev/null +++ b/src/main/java/io/ten1010/aipub/projectcontroller/mutating/service/NodeMaintenanceReviewHandler.java @@ -0,0 +1,53 @@ +package io.ten1010.aipub.projectcontroller.mutating.service; + +import io.kubernetes.client.informer.SharedInformerFactory; +import io.kubernetes.client.informer.cache.Indexer; +import io.ten1010.aipub.projectcontroller.configuration.AipubProperties; +import io.ten1010.aipub.projectcontroller.domain.k8s.K8sObjectTypeConstants; +import io.ten1010.aipub.projectcontroller.domain.k8s.KeyResolver; +import io.ten1010.aipub.projectcontroller.domain.k8s.SubjectResolver; +import io.ten1010.aipub.projectcontroller.domain.k8s.dto.V1alpha1NodeMaintenance; +import io.ten1010.aipub.projectcontroller.mutating.V1AdmissionReviewUtils; +import io.ten1010.aipub.projectcontroller.mutating.dto.V1AdmissionReview; +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.HttpStatus; + +import java.util.Objects; + +@Slf4j +public class NodeMaintenanceReviewHandler extends AbstractReviewHandler { + + private final AipubProperties aipubProperties; + private final SubjectResolver subjectResolver; + private final KeyResolver keyResolver; + private final Indexer nodeMaintenanceIndexer; + + public NodeMaintenanceReviewHandler(AipubProperties aipubProperties, SubjectResolver subjectResolver, SharedInformerFactory sharedInformerFactory) { + super(K8sObjectTypeConstants.NODE_MAINTENANCE_V1ALPHA1); + this.aipubProperties = aipubProperties; + this.keyResolver = new KeyResolver(); + this.subjectResolver = subjectResolver; + this.nodeMaintenanceIndexer = sharedInformerFactory + .getExistingSharedIndexInformer(V1alpha1NodeMaintenance.class) + .getIndexer(); + } + + @Override + public void handle(V1AdmissionReview review) { + Objects.requireNonNull(review.getRequest()); + Objects.requireNonNull(review.getRequest().getUserInfo()); + + V1alpha1NodeMaintenance nodeMaintenance = getRequestObject(review); + var spec = Objects.requireNonNull(nodeMaintenance.getSpec()); + var actions = Objects.requireNonNull(spec.getActions()); + long uncordonCount = actions.stream().filter(x -> x.getType().equals("uncordon")).count(); + long drainCount = actions.stream().filter(x -> x.getType().equals("drain")).count(); + + if (uncordonCount > 0 && drainCount > 0) { + V1AdmissionReviewUtils.reject(review, HttpStatus.FORBIDDEN.value(), "Forbidden"); + return; + } + V1AdmissionReviewUtils.allow(review); + } + +}