diff --git a/plugin/src/main/java/jenkins/plugins/openstack/compute/JCloudsCloud.java b/plugin/src/main/java/jenkins/plugins/openstack/compute/JCloudsCloud.java index bd2ec320..1eee29fb 100644 --- a/plugin/src/main/java/jenkins/plugins/openstack/compute/JCloudsCloud.java +++ b/plugin/src/main/java/jenkins/plugins/openstack/compute/JCloudsCloud.java @@ -254,14 +254,23 @@ private void injectReferenceIntoTemplates() { return queue; // more slaves then declared - no need to query openstack } - - final List runningNodes = getOpenstack().getRunningNodes(); - - int serverCount = runningNodes.size(); - if (serverCount >= globalMax) { - return queue; // more servers than needed - no need to proceed any further - } - + // Check the number of current servers + int serverCount = 0; + List runningNodes = new ArrayList(); + try { + // get the running nodes + runningNodes = getOpenstack().getRunningNodes(); + + serverCount = runningNodes.size(); + if (serverCount >= globalMax) { + return queue; // more servers than needed - no need to proceed any further + } + } catch (JCloudsCloud.LoginFailure ex) { + LOGGER.log(Level.WARNING, "Login failure: " + ex.getMessage()); + return queue; + } + + int globalCapacity = globalMax - Math.max(nodeCount, serverCount); assert globalCapacity > 0; @@ -299,8 +308,19 @@ public Collection provision(Label label, int excess final JCloudsSlaveTemplate template = templateProvider.poll(); if (template == null) { - LOGGER.info("Instance cap exceeded for cloud " + name + " while provisioning for label " + label); - break; + // two cases + // cloud authentication issue or Instance cap exceeded + try { + // try to authenticate + getOpenstack(); + // if okay, then the problem is related to the instance cap + LOGGER.info("Instance cap exceeded for cloud " + name + " while provisioning for label " + label); + break; + } catch (JCloudsCloud.LoginFailure ex) { + // no need to log here because it has already been logged in the getAvailableTemplates method. + break; + } + } LOGGER.fine("Provisioning slave for " + label + " from template " + template.getName()); @@ -313,9 +333,13 @@ public Collection provision(Label label, int excess excessWorkload -= numExecutors; } + + return plannedNodeList; } + + private static final class NodeCallable implements Callable { private final JCloudsCloud cloud; private final JCloudsSlaveTemplate template; @@ -614,7 +638,8 @@ public ProvisioningFailedException(String msg) { } } - /*package*/ static final class LoginFailure extends RuntimeException { + /*package*/ + public static final class LoginFailure extends RuntimeException { private static final long serialVersionUID = 4085466675398031930L; @@ -626,7 +651,7 @@ private LoginFailure(String name, AuthenticationException ex) { super("Failure to authenticate for cloud " + name + ": " + ex.toString()); } - private LoginFailure(String msg) { + public LoginFailure(String msg) { super(msg); } } diff --git a/plugin/src/test/java/jenkins/plugins/openstack/PluginTestRule.java b/plugin/src/test/java/jenkins/plugins/openstack/PluginTestRule.java index 1416d049..23f9b324 100644 --- a/plugin/src/test/java/jenkins/plugins/openstack/PluginTestRule.java +++ b/plugin/src/test/java/jenkins/plugins/openstack/PluginTestRule.java @@ -310,6 +310,13 @@ public JCloudsCloud dummyCloud(SlaveOptions opts, JCloudsSlaveTemplate... templa return cloud; } + public JCloudsCloud unavailableDummyCloud(SlaveOptions opts, JCloudsSlaveTemplate... templates) { + JCloudsCloud cloud = new MockJCloudsCloud(opts,false,templates); + + jenkins.clouds.add(cloud); + return cloud; + } + public JCloudsCloud configureSlaveLaunchingWithFloatingIP(String labels) { return configureSlaveLaunchingWithFloatingIP(dummyCloud(dummySlaveTemplate(labels))); } @@ -642,9 +649,18 @@ public MockJCloudsCloud(SlaveOptions opts, JCloudsSlaveTemplate... templates) { super("openstack", "endPointUrl", false,"zone", opts, Arrays.asList(templates), "credentialsId"); } + public MockJCloudsCloud(SlaveOptions opts,boolean available, JCloudsSlaveTemplate... templates) { + super("openstack", "endPointUrl", false,"zone", opts, Arrays.asList(templates), null); + } + @Override public @Nonnull Openstack getOpenstack() { - return os; + if ( getCredentialsId() != null){ + return os ; + } else { + throw new LoginFailure("Login failure"); + } + } @Override diff --git a/plugin/src/test/java/jenkins/plugins/openstack/compute/ProvisioningTest.java b/plugin/src/test/java/jenkins/plugins/openstack/compute/ProvisioningTest.java index aba5ac2b..027ff888 100644 --- a/plugin/src/test/java/jenkins/plugins/openstack/compute/ProvisioningTest.java +++ b/plugin/src/test/java/jenkins/plugins/openstack/compute/ProvisioningTest.java @@ -32,6 +32,7 @@ import java.io.IOException; import java.net.URL; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -439,6 +440,99 @@ public void failIfNoAccessIpFound() { } } + @Test + public void provisionWhenSharedLabel() throws Exception { + + // Test 1 - premiere assertion + /* ○ Test1: Everything fine + § Two clouds configured and up + § Two jobs - excessWorkload = 2 + § Should work fine + */ + + SlaveOptions init = j.defaultSlaveOptions(); + JCloudsSlaveTemplate template1 = j.dummySlaveTemplate(init.getBuilder().instanceCap(1).build(), "generic"); + JCloudsSlaveTemplate template2 = j.dummySlaveTemplate(init.getBuilder().instanceCap(1).build(), "generic"); + JCloudsCloud cloud = j.dummyCloud(init.getBuilder().instanceCap(2).build(), template1); + JCloudsCloud cloud2 = j.dummyCloud(init.getBuilder().instanceCap(2).build(), template2); + + Label generic = Label.get("generic"); + + // Simulate the provisioning process used in NodeProvisioner (https://github.com/jenkinsci/jenkins/blob/master/core/src/main/java/hudson/slaves/NodeProvisioner.java#L628) + List clouds = new ArrayList(); + clouds.add(cloud); clouds.add(cloud2); + int jobsCount = 2; + // Until there are no more jobs to build + while(jobsCount>0){ + // try provisioning from the clouds + for (JCloudsCloud c : clouds){ + if (c.canProvision(generic)){ + // update the number of remaining jobs to build + Collection plannedNodeList = c.provision(generic,jobsCount); + jobsCount -= plannedNodeList.size(); + } + } + } + assertEquals(0,jobsCount); + + // Test 2 - second assertion + /* ○ Test2: First Cloud down + § First Cloud down - simulate outage by providing invalid credentials + § Two jobs + § Should work fine - It should build both jobs using the remaining cloud + */ + clouds.clear(); + cloud = cloud2 = null; + JCloudsCloud cloud3 = j.unavailableDummyCloud(init.getBuilder().instanceCap(1).build(), template1); + JCloudsCloud cloud4 = j.dummyCloud(init.getBuilder().instanceCap(2).build(), template2); + + + + // // Simulate the provisioning process used in NodeProvisioner (https://github.com/jenkinsci/jenkins/blob/master/core/src/main/java/hudson/slaves/NodeProvisioner.java#L628) + + clouds.add(cloud3); clouds.add(cloud4); + int jobsCount2 = 2; + // // Until there are no more jobs to build + while(jobsCount2>0){ + // try provisioning from the clouds + for (JCloudsCloud c : clouds){ + if (c.canProvision(generic)){ + // update the number of remaining jobs to build + jobsCount2 -= c.provision(generic,jobsCount2).size(); + } + } + } + + assertEquals(0,jobsCount2); + // Test 3 - third assertion + /* ○ Test 3: second Cloud down + § First Cloud down - simulate outage by providing invalid credentials + § Two jobs + § Should work fine - It should build both jobs using the remaining cloud + */ + clouds.clear(); + cloud3 = cloud4 = null; + JCloudsCloud cloud5 = j.dummyCloud(init.getBuilder().instanceCap(2).build(), template1); + JCloudsCloud cloud6 = j.unavailableDummyCloud(init.getBuilder().instanceCap(1).build(), template2); + + // // Simulate the provisioning process used in NodeProvisioner (https://github.com/jenkinsci/jenkins/blob/master/core/src/main/java/hudson/slaves/NodeProvisioner.java#L628) + clouds.add(cloud5); clouds.add(cloud6); + int jobsCount3 = 2; + // // Until there are no more jobs to build + while(jobsCount3>0){ + // try provisioning from the clouds + for (JCloudsCloud c : clouds){ + if (c.canProvision(generic)){ + // update the number of remaining jobs to build + jobsCount3 -= c.provision(generic,jobsCount3).size(); + } + } + } + + assertEquals(0,jobsCount3); + } + + private void verifyPreferredAddressUsed(String expectedAddress, Collection addresses) throws Exception { CloudStatistics cs = CloudStatistics.get(); assertThat(cs.getActivities(), Matchers.iterableWithSize(0));