From 24ed436f9fd1fe877a0387f15c493aef958ec641 Mon Sep 17 00:00:00 2001
From: jonathannewman <jonathan.newman@puppetlabs.com>
Date: Tue, 3 May 2022 09:32:40 -0700
Subject: [PATCH] (JENKINS-68371) improve asynchrony of
 StandardPlannedNodeBuilder

When the NodeProvisioner is building agents in the provision step: https://github.com/jenkinsci/kubernetes-plugin/blob/307d9791dcf7dfc3bbbcbdf1a7eab44ed752a4c8/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesCloud.java#L536 it currently loops through the number of
nodes to provision. This is implemented in the StandardPlannedNodeBuilder,
and is done as a blocking operation, while using a Future interface
to satisfy the consumers.  In our testing, it was seen that this
operation could take upwards of 100 seconds when under load, causing
provisioning to be effectively stopped for periods of time.

This change introduces a configurable thread pool that will cause
the Agent creation step to occur in a separate thread.  This, in
our testing resolves the serial bottleneck issue and allows provisioning
to continue while the blocking operations occur separately.
---
 .../StandardPlannedNodeBuilder.java           | 33 ++++++++++++-------
 1 file changed, 21 insertions(+), 12 deletions(-)

diff --git a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/StandardPlannedNodeBuilder.java b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/StandardPlannedNodeBuilder.java
index 3c41e20e27..591a703ca3 100644
--- a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/StandardPlannedNodeBuilder.java
+++ b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/StandardPlannedNodeBuilder.java
@@ -1,35 +1,44 @@
 package org.csanchez.jenkins.plugins.kubernetes;
 
 import hudson.Util;
+import hudson.util.NamingThreadFactory;
 import hudson.model.Descriptor;
 import hudson.slaves.NodeProvisioner;
-
 import java.io.IOException;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.logging.Logger;
 
 /**
  * The default {@link PlannedNodeBuilder} implementation, in case there is other registered.
  */
 public class StandardPlannedNodeBuilder extends PlannedNodeBuilder {
+    private static final Logger LOGGER = Logger.getLogger(StandardPlannedNodeBuilder.class.getName());
+    private static final int THREAD_POOL_SIZE = Integer.parseInt(System.getProperty("org.csanchez.jenkins.plugins.kubernetes.plannedNodeBuilderThreadPoolSize", "100"));
+    private static final ExecutorService EXECUTOR_SERVICE = Executors.newFixedThreadPool(THREAD_POOL_SIZE, new NamingThreadFactory(Executors.defaultThreadFactory(), "StandardPlannedNodeBuilderAgent"));
+
     @Override
     public NodeProvisioner.PlannedNode build() {
+        LOGGER.finer("Start build");
+        long start = System.currentTimeMillis();
         KubernetesCloud cloud = getCloud();
         PodTemplate t = getTemplate();
-        CompletableFuture f;
-        String displayName;
-        try {
+        Future f = EXECUTOR_SERVICE.submit(() -> {
+            long insideThreadStart = System.currentTimeMillis();
+            LOGGER.fine("Creating agent");
             KubernetesSlave agent = KubernetesSlave
                     .builder()
                     .podTemplate(cloud.getUnwrappedTemplate(t))
                     .cloud(cloud)
                     .build();
-            displayName = agent.getDisplayName();
-            f = CompletableFuture.completedFuture(agent);
-        } catch (IOException | Descriptor.FormException e) {
-            displayName = null;
-            f = new CompletableFuture();
-            f.completeExceptionally(e);
-        }
-        return new NodeProvisioner.PlannedNode(Util.fixNull(displayName), f, getNumExecutors());
+            LOGGER.fine("Created agent in " + (System.currentTimeMillis() - insideThreadStart) + " milliseconds");
+            return agent;
+        });
+        LOGGER.finer("Created future after " + (System.currentTimeMillis() - start) + " milliseconds");
+        NodeProvisioner.PlannedNode result = new NodeProvisioner.PlannedNode(Util.fixNull("Kubernetes Agent"), f, getNumExecutors());
+        LOGGER.finer("Exiting build after " + (System.currentTimeMillis() - start) + " milliseconds");
+        return result;
     }
 }