diff --git a/pom.xml b/pom.xml index b6d60b65..7221587d 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ org.jenkins-ci.plugins plugin - 3.12 + 3.13 io.jenkins.plugins @@ -19,7 +19,7 @@ 2.0.3 2.121 8 - 2.28-rc341.90cc5dc659de + 2.28-rc343.e9b9e0610374 true @@ -69,6 +69,11 @@ aws-java-sdk 1.11.329 + + org.jenkins-ci.plugins + apache-httpcomponents-client-4-api + 4.5.5-2.2-rc32.4a9f3bcc3908 + + 2.8-rc353.ae434696120f test @@ -159,17 +164,6 @@ 1.7 test - - com.github.rholder - guava-retrying - 2.0.0 - - - com.google.guava - guava - - - diff --git a/src/main/java/io/jenkins/plugins/artifact_manager_jclouds/JCloudsArtifactManager.java b/src/main/java/io/jenkins/plugins/artifact_manager_jclouds/JCloudsArtifactManager.java index a6664754..f2a0ada6 100644 --- a/src/main/java/io/jenkins/plugins/artifact_manager_jclouds/JCloudsArtifactManager.java +++ b/src/main/java/io/jenkins/plugins/artifact_manager_jclouds/JCloudsArtifactManager.java @@ -24,13 +24,24 @@ package io.jenkins.plugins.artifact_manager_jclouds; -import com.github.rholder.retry.Attempt; -import com.github.rholder.retry.AttemptTimeLimiters; +import hudson.AbortException; +import hudson.EnvVars; +import hudson.FilePath; +import hudson.Launcher; +import hudson.Util; +import hudson.model.BuildListener; +import hudson.model.Run; +import hudson.model.TaskListener; +import hudson.remoting.VirtualChannel; +import hudson.slaves.WorkspaceList; +import hudson.util.DirScanner; +import hudson.util.io.ArchiverFactory; +import io.jenkins.plugins.artifact_manager_jclouds.BlobStoreProvider.HttpMethod; +import io.jenkins.plugins.httpclient.RobustHTTPClient; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.net.HttpURLConnection; import java.net.URL; import java.nio.file.Files; import java.nio.file.InvalidPathException; @@ -39,10 +50,12 @@ import java.util.HashMap; import java.util.Iterator; import java.util.Map; -import java.util.concurrent.ExecutionException; import java.util.logging.Level; import java.util.logging.Logger; - +import jenkins.MasterToSlaveFileCallable; +import jenkins.model.ArtifactManager; +import jenkins.util.VirtualFile; +import org.apache.http.client.methods.HttpGet; import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.BlobStoreContext; import org.jclouds.blobstore.domain.Blob; @@ -50,36 +63,6 @@ import org.jclouds.blobstore.options.CopyOptions; import org.jclouds.blobstore.options.ListContainerOptions; import org.jenkinsci.plugins.workflow.flow.StashManager; - -import com.github.rholder.retry.RetryException; -import com.github.rholder.retry.RetryListener; -import com.github.rholder.retry.RetryerBuilder; -import com.github.rholder.retry.StopStrategies; -import com.github.rholder.retry.WaitStrategies; -import com.google.common.util.concurrent.UncheckedTimeoutException; -import hudson.AbortException; -import hudson.EnvVars; -import hudson.FilePath; -import hudson.Launcher; -import hudson.Util; -import hudson.model.BuildListener; -import hudson.model.Computer; -import hudson.model.Run; -import hudson.model.TaskListener; -import hudson.remoting.VirtualChannel; -import hudson.slaves.WorkspaceList; -import hudson.util.DirScanner; -import hudson.util.io.ArchiverFactory; -import io.jenkins.plugins.artifact_manager_jclouds.BlobStoreProvider.HttpMethod; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; -import jenkins.MasterToSlaveFileCallable; -import jenkins.model.ArtifactManager; -import jenkins.util.JenkinsJVM; -import jenkins.util.VirtualFile; -import org.apache.commons.io.IOUtils; import org.kohsuke.accmod.Restricted; import org.kohsuke.accmod.restrictions.NoExternalUse; @@ -91,6 +74,8 @@ public final class JCloudsArtifactManager extends ArtifactManager implements Sta private static final Logger LOGGER = Logger.getLogger(JCloudsArtifactManager.class.getName()); + static RobustHTTPClient client = new RobustHTTPClient(); + private final BlobStoreProvider provider; private transient String key; // e.g. myorg/myrepo/master/123 @@ -148,11 +133,8 @@ private static class UploadToBlobStorage extends MasterToSlaveFileCallable private final Map artifactUrls; // e.g. "target/x.war", "http://..." private final TaskListener listener; - // Bind when constructed on the master side; on the agent side, deserialize those values. - private final int stopAfterAttemptNumber = UPLOAD_STOP_AFTER_ATTEMPT_NUMBER; - private final long waitMultiplier = UPLOAD_WAIT_MULTIPLIER; - private final long waitMaximum = UPLOAD_WAIT_MAXIMUM; - private final long timeout = UPLOAD_TIMEOUT; + // Bind when constructed on the master side; on the agent side, deserialize the same configuration. + private final RobustHTTPClient client = JCloudsArtifactManager.client; UploadToBlobStorage(Map artifactUrls, TaskListener listener) { this.artifactUrls = artifactUrls; @@ -162,9 +144,7 @@ private static class UploadToBlobStorage extends MasterToSlaveFileCallable @Override public Void invoke(File f, VirtualChannel channel) throws IOException, InterruptedException { for (Map.Entry entry : artifactUrls.entrySet()) { - Path local = f.toPath().resolve(entry.getKey()); - URL url = entry.getValue(); - uploadFile(local, url, listener, stopAfterAttemptNumber, waitMultiplier, waitMaximum, timeout); + client.uploadFile(new File(f, entry.getKey()), entry.getValue(), listener); } return null; } @@ -229,10 +209,7 @@ private static final class Stash extends MasterToSlaveFileCallable { private final boolean useDefaultExcludes; private final String tempDir; private final TaskListener listener; - private final int stopAfterAttemptNumber = UPLOAD_STOP_AFTER_ATTEMPT_NUMBER; - private final long waitMultiplier = UPLOAD_WAIT_MULTIPLIER; - private final long waitMaximum = UPLOAD_WAIT_MAXIMUM; - private final long timeout = UPLOAD_TIMEOUT; + private final RobustHTTPClient client = JCloudsArtifactManager.client; Stash(URL url, String includes, String excludes, boolean useDefaultExcludes, String tempDir, TaskListener listener) throws IOException { this.url = url; @@ -245,7 +222,7 @@ private static final class Stash extends MasterToSlaveFileCallable { @Override public Integer invoke(File f, VirtualChannel channel) throws IOException, InterruptedException { - // TODO JCLOUDS-769 streaming upload is not currently straightforward, so using a temp file pending rewrite to use multipart uploads + // TODO use streaming upload rather than a temp file; is it necessary to set the content length in advance? // (we prefer not to upload individual files for stashes, so as to preserve symlinks & file permissions, as StashManager’s default does) Path tempDirP = Paths.get(tempDir); Files.createDirectories(tempDirP); @@ -258,7 +235,7 @@ public Integer invoke(File f, VirtualChannel channel) throws IOException, Interr throw new IOException(e); } if (count > 0) { - uploadFile(tmp, url, listener, stopAfterAttemptNumber, waitMultiplier, waitMaximum, timeout); + client.uploadFile(tmp.toFile(), url, listener); } return count; } finally { @@ -279,24 +256,29 @@ public void unstash(String name, FilePath workspace, Launcher launcher, EnvVars String.format("No such saved stash ‘%s’ found at %s/%s", name, provider.getContainer(), blobPath)); } URL url = provider.toExternalURL(blob, HttpMethod.GET); - workspace.act(new Unstash(url)); + workspace.act(new Unstash(url, listener)); listener.getLogger().printf("Unstashed file(s) from %s%n", provider.toURI(provider.getContainer(), blobPath)); } private static final class Unstash extends MasterToSlaveFileCallable { private static final long serialVersionUID = 1L; private final URL url; + private final TaskListener listener; + private final RobustHTTPClient client = JCloudsArtifactManager.client; - Unstash(URL url) throws IOException { + Unstash(URL url, TaskListener listener) throws IOException { this.url = url; + this.listener = listener; } @Override public Void invoke(File f, VirtualChannel channel) throws IOException, InterruptedException { - try (InputStream is = url.openStream()) { - new FilePath(f).untarFrom(is, FilePath.TarCompression.GZIP); - // Note that this API currently offers no count of files in the tarball we could report. - } + client.connect("download", "download " + RobustHTTPClient.sanitize(url) + " into " + f, c -> c.execute(new HttpGet(url.toString())), response -> { + try (InputStream is = response.getEntity().getContent()) { + new FilePath(f).untarFrom(is, FilePath.TarCompression.GZIP); + // Note that this API currently offers no count of files in the tarball we could report. + } + }, listener); return null; } } @@ -359,91 +341,4 @@ private BlobStoreContext getContext() throws IOException { return provider.getContext(); } - private static final class HTTPAbortException extends AbortException { - final int code; - HTTPAbortException(int code, String message) { - super(message); - this.code = code; - } - } - - /** - * Number of upload attempts of nonfatal errors before giving up. - */ - static int UPLOAD_STOP_AFTER_ATTEMPT_NUMBER = Integer.getInteger(JCloudsArtifactManager.class.getName() + ".UPLOAD_STOP_AFTER_ATTEMPT_NUMBER", 10); - /** - * Initial number of milliseconds between first and second upload attempts. - * Subsequent ones increase exponentially. - * Note that this is not a randomized exponential backoff; - * and the base of the exponent is hard-coded to 2. - */ - static long UPLOAD_WAIT_MULTIPLIER = Long.getLong(JCloudsArtifactManager.class.getName() + ".UPLOAD_WAIT_MULTIPLIER", 100); - /** - * Maximum number of seconds between upload attempts. - */ - static long UPLOAD_WAIT_MAXIMUM = Long.getLong(JCloudsArtifactManager.class.getName() + ".UPLOAD_WAIT_MAXIMUM", 300); - /** - * Number of seconds to permit a single upload attempt to take. - */ - static long UPLOAD_TIMEOUT = Long.getLong(JCloudsArtifactManager.class.getName() + ".UPLOAD_TIMEOUT", /* 15m */15 * 60); - - private static final ExecutorService executors = JenkinsJVM.isJenkinsJVM() ? Computer.threadPoolForRemoting : Executors.newCachedThreadPool(); - - /** - * Upload a file to a URL - */ - @SuppressWarnings("Convert2Lambda") // bogus use of generics (type variable should have been on class); cannot be made into a lambda - private static void uploadFile(Path f, URL url, final TaskListener listener, int stopAfterAttemptNumber, long waitMultiplier, long waitMaximum, long timeout) throws IOException, InterruptedException { - String urlSafe = url.toString().replaceFirst("[?].+$", "?…"); - try { - AtomicReference lastError = new AtomicReference<>(); - RetryerBuilder.newBuilder(). - retryIfException(x -> x instanceof IOException && (!(x instanceof HTTPAbortException) || ((HTTPAbortException) x).code >= 500) || x instanceof UncheckedTimeoutException). - withRetryListener(new RetryListener() { - @Override - public void onRetry(Attempt attempt) { - if (attempt.hasException()) { - lastError.set(attempt.getExceptionCause()); - } - } - }). - withStopStrategy(StopStrategies.stopAfterAttempt(stopAfterAttemptNumber)). - withWaitStrategy(WaitStrategies.exponentialWait(waitMultiplier, waitMaximum, TimeUnit.SECONDS)). - withAttemptTimeLimiter(AttemptTimeLimiters.fixedTimeLimit(timeout, TimeUnit.SECONDS, executors)). - build().call(() -> { - Throwable t = lastError.get(); - if (t != null) { - listener.getLogger().println("Retrying upload after: " + (t instanceof AbortException ? t.getMessage() : t.toString())); - } - HttpURLConnection connection = (HttpURLConnection) url.openConnection(); - connection.setDoOutput(true); - connection.setRequestMethod("PUT"); - connection.setFixedLengthStreamingMode(Files.size(f)); // prevent loading file in memory - try (OutputStream out = connection.getOutputStream()) { - Files.copy(f, out); - } - int responseCode = connection.getResponseCode(); - if (responseCode < 200 || responseCode >= 300) { - String diag; - try (InputStream err = connection.getErrorStream()) { - diag = err != null ? IOUtils.toString(err, connection.getContentEncoding()) : null; - } - throw new HTTPAbortException(responseCode, String.format("Failed to upload %s to %s, response: %d %s, body: %s", f.toAbsolutePath(), urlSafe, responseCode, connection.getResponseMessage(), diag)); - } - return null; - }); - } catch (ExecutionException | RetryException x) { // *sigh*, checked exceptions - Throwable x2 = x.getCause(); - if (x2 instanceof IOException) { - throw (IOException) x2; - } else if (x2 instanceof RuntimeException) { - throw (RuntimeException) x2; - } else if (x2 instanceof InterruptedException) { - throw (InterruptedException) x2; - } else { // Error? - throw new RuntimeException(x); - } - } - } - } diff --git a/src/main/java/io/jenkins/plugins/artifact_manager_jclouds/JCloudsVirtualFile.java b/src/main/java/io/jenkins/plugins/artifact_manager_jclouds/JCloudsVirtualFile.java index 1abf9e22..4aec4edd 100644 --- a/src/main/java/io/jenkins/plugins/artifact_manager_jclouds/JCloudsVirtualFile.java +++ b/src/main/java/io/jenkins/plugins/artifact_manager_jclouds/JCloudsVirtualFile.java @@ -46,7 +46,6 @@ import java.util.Spliterators; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.stream.Stream; import java.util.stream.StreamSupport; import jenkins.util.VirtualFile; import org.jclouds.blobstore.BlobStore; @@ -182,15 +181,15 @@ public boolean exists() throws IOException { /** * List all the blobs under this one * - * @return a stream of blobs StorageMetadata + * @return some blobs + * @throws RuntimeException either now or when the stream is processed; wrap in {@link IOException} if desired */ - private Stream listStorageMetadata(boolean recursive) throws IOException { + private Iterator listStorageMetadata(boolean recursive) throws IOException { ListContainerOptions options = prefix(key + "/"); if (recursive) { options.recursive(); } - PageSetIterable it = new PageSetIterable(getContext().getBlobStore(), getContainer(), options); - return StreamSupport.stream(Spliterators.spliteratorUnknownSize(it, Spliterator.ORDERED), false); + return new PageSetIterable(getContext().getBlobStore(), getContainer(), options); } @Override @@ -207,9 +206,14 @@ public VirtualFile[] list() throws IOException { map(simple -> new JCloudsVirtualFile(provider, container, keyS + simple)). // direct children toArray(VirtualFile[]::new); } - VirtualFile[] list = listStorageMetadata(false) + VirtualFile[] list; + try { + list = StreamSupport.stream(Spliterators.spliteratorUnknownSize(listStorageMetadata(false), Spliterator.ORDERED), false) .map(meta -> new JCloudsVirtualFile(provider, getContainer(), meta.getName().replaceFirst("/$", ""))) .toArray(VirtualFile[]::new); + } catch (RuntimeException x) { + throw new IOException(x); + } LOGGER.log(Level.FINEST, "Listing files from {0} {1}: {2}", new String[] { getContainer(), getKey(), Arrays.toString(list) }); return list; @@ -279,6 +283,9 @@ static class PageSetIterable implements Iterator { private PageSet set; private Iterator iterator; + /** + * @throws RuntimeException either now or when iterating; wrap in {@link IOException} if desired + */ PageSetIterable(@NonNull BlobStore blobStore, @NonNull String container, @NonNull ListContainerOptions options) { this.blobStore = blobStore; @@ -363,13 +370,19 @@ public V run(Callable callable) throws IOException { Deque stack = cacheFrames(); Map saved = new HashMap<>(); int prefixLength = key.length() + /* / */1; - listStorageMetadata(true).forEach(sm -> { - Long length = sm.getSize(); - if (length != null) { - Date lastModified = sm.getLastModified(); - saved.put(sm.getName().substring(prefixLength), new CachedMetadata(length, lastModified != null ? lastModified.getTime() : 0)); + try { + Iterator it = listStorageMetadata(true); + while (it.hasNext()) { + StorageMetadata sm = it.next(); + Long length = sm.getSize(); + if (length != null) { + Date lastModified = sm.getLastModified(); + saved.put(sm.getName().substring(prefixLength), new CachedMetadata(length, lastModified != null ? lastModified.getTime() : 0)); + } } - }); + } catch (RuntimeException x) { + throw new IOException(x); + } stack.push(new CacheFrame(key + "/", saved)); try { LOGGER.log(Level.FINE, "using cache {0} / {1}: {2} file entries", new Object[] {container, key, saved.size()}); diff --git a/src/test/java/io/jenkins/plugins/artifact_manager_jclouds/MockApiMetadata.java b/src/test/java/io/jenkins/plugins/artifact_manager_jclouds/MockApiMetadata.java index 7df59333..66c3b8e3 100644 --- a/src/test/java/io/jenkins/plugins/artifact_manager_jclouds/MockApiMetadata.java +++ b/src/test/java/io/jenkins/plugins/artifact_manager_jclouds/MockApiMetadata.java @@ -42,6 +42,7 @@ import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.logging.Level; import java.util.logging.Logger; @@ -148,6 +149,23 @@ protected void configure() { } + @FunctionalInterface + interface GetBlobKeysInsideContainerHandler { + Iterable run() throws IOException; + } + + private static final Map getBlobKeysInsideContainerHandlers = new ConcurrentHashMap<>(); + + static void handleGetBlobKeysInsideContainer(String container, GetBlobKeysInsideContainerHandler handler) { + getBlobKeysInsideContainerHandlers.put(container, handler); + } + + private static final Map removeBlobHandlers = new ConcurrentHashMap<>(); + + static void handleRemoveBlob(String container, String key, Runnable handler) { + removeBlobHandlers.put(container + '/' + key, handler); + } + /** Like {@link TransientStorageStrategy}. */ public static final class MockStrategy implements LocalStorageStrategy { @@ -205,6 +223,10 @@ public boolean blobExists(String container, String key) { @Override public Iterable getBlobKeysInsideContainer(String container) throws IOException { + GetBlobKeysInsideContainerHandler handler = getBlobKeysInsideContainerHandlers.remove(container); + if (handler != null) { + return handler.run(); + } return blobsByContainer.get(container).keySet(); } @@ -231,6 +253,11 @@ public String putBlob(String containerName, Blob blob) throws IOException { @Override public void removeBlob(String container, String key) { + Runnable handler = removeBlobHandlers.remove(container + '/' + key); + if (handler != null) { + handler.run(); + return; + } blobsByContainer.get(container).remove(key); } diff --git a/src/test/java/io/jenkins/plugins/artifact_manager_jclouds/NetworkTest.java b/src/test/java/io/jenkins/plugins/artifact_manager_jclouds/NetworkTest.java index fc195901..30e87f38 100644 --- a/src/test/java/io/jenkins/plugins/artifact_manager_jclouds/NetworkTest.java +++ b/src/test/java/io/jenkins/plugins/artifact_manager_jclouds/NetworkTest.java @@ -23,23 +23,36 @@ */ package io.jenkins.plugins.artifact_manager_jclouds; +import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException; +import com.google.common.base.Throwables; import hudson.model.Result; +import hudson.model.Run; +import hudson.tasks.LogRotator; +import io.jenkins.plugins.httpclient.RobustHTTPClient; +import java.util.Objects; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.stream.Collectors; import jenkins.model.ArtifactManagerConfiguration; import org.apache.http.ConnectionClosedException; import org.apache.http.HttpVersion; import org.apache.http.entity.StringEntity; import org.apache.http.message.BasicStatusLine; +import static org.hamcrest.Matchers.*; +import org.jclouds.blobstore.ContainerNotFoundException; import org.jenkinsci.plugins.workflow.cps.CpsFlowDefinition; import org.jenkinsci.plugins.workflow.job.WorkflowJob; import org.jenkinsci.plugins.workflow.job.WorkflowRun; import org.jenkinsci.plugins.workflow.steps.TimeoutStepExecution; -import org.junit.ClassRule; -import org.junit.Test; +import static org.junit.Assert.*; import org.junit.Before; +import org.junit.ClassRule; import org.junit.Rule; +import org.junit.Test; import org.jvnet.hudson.test.BuildWatcher; import org.jvnet.hudson.test.Issue; import org.jvnet.hudson.test.JenkinsRule; +import org.jvnet.hudson.test.LoggerRule; /** * Explores responses to edge cases such as server errors and hangs. @@ -51,7 +64,7 @@ * For example, some S3 mock frameworks completely ignore authentication. * Conversely, some failure modes we want to test may not be supported by an S3 mock framework. *
  • S3 mock frameworks are not written to expect the jclouds abstraction layer, and vice-versa. - * The jclouds {@code AWSS3ProviderMetadata} cannot accept a custom {@link AmazonS3} which {@code S3MockRule} would supply; + * The jclouds {@code AWSS3ProviderMetadata} cannot accept a custom {@code AmazonS3} which {@code S3MockRule} would supply; * it reimplements the S3 REST API from scratch, not using the AWS SDK. * It would be necessary to run Dockerized mocks with local HTTP ports. * @@ -65,6 +78,9 @@ public class NetworkTest { @Rule public JenkinsRule r = new JenkinsRule(); + @Rule + public LoggerRule loggerRule = new LoggerRule(); + @Before public void configureManager() throws Exception { MockBlobStore mockBlobStore = new MockBlobStore(); @@ -72,10 +88,14 @@ public void configureManager() throws Exception { ArtifactManagerConfiguration.get().getArtifactManagerFactories().add(new JCloudsArtifactManagerFactory(mockBlobStore)); } + @Before + public void createAgent() throws Exception { + r.createSlave("remote", null, null); + } + @Test - public void unrecoverableExceptionArchiving() throws Exception { + public void unrecoverableErrorArchiving() throws Exception { WorkflowJob p = r.createProject(WorkflowJob.class, "p"); - r.createSlave("remote", null, null); failIn(BlobStoreProvider.HttpMethod.PUT, "p/1/artifacts/f", 403, 0); p.setDefinition(new CpsFlowDefinition("node('remote') {writeFile file: 'f', text: '.'; archiveArtifacts 'f'}", true)); WorkflowRun b = r.assertBuildStatus(Result.FAILURE, p.scheduleBuild2(0)); @@ -86,9 +106,8 @@ public void unrecoverableExceptionArchiving() throws Exception { } @Test - public void recoverableExceptionArchiving() throws Exception { + public void recoverableErrorArchiving() throws Exception { WorkflowJob p = r.createProject(WorkflowJob.class, "p"); - r.createSlave("remote", null, null); failIn(BlobStoreProvider.HttpMethod.PUT, "p/1/artifacts/f", 500, 0); p.setDefinition(new CpsFlowDefinition("node('remote') {writeFile file: 'f', text: '.'; archiveArtifacts 'f'}", true)); WorkflowRun b = r.buildAndAssertSuccess(p); @@ -100,7 +119,6 @@ public void recoverableExceptionArchiving() throws Exception { @Test public void networkExceptionArchiving() throws Exception { WorkflowJob p = r.createProject(WorkflowJob.class, "p"); - r.createSlave("remote", null, null); failIn(BlobStoreProvider.HttpMethod.PUT, "p/1/artifacts/f", 0, 0); p.setDefinition(new CpsFlowDefinition("node('remote') {writeFile file: 'f', text: '.'; archiveArtifacts 'f'}", true)); WorkflowRun b = r.buildAndAssertSuccess(p); @@ -110,13 +128,12 @@ public void networkExceptionArchiving() throws Exception { } @Test - public void repeatedRecoverableExceptionArchiving() throws Exception { + public void repeatedRecoverableErrorArchiving() throws Exception { WorkflowJob p = r.createProject(WorkflowJob.class, "p"); - r.createSlave("remote", null, null); - int origStopAfterAttemptNumber = JCloudsArtifactManager.UPLOAD_STOP_AFTER_ATTEMPT_NUMBER; - JCloudsArtifactManager.UPLOAD_STOP_AFTER_ATTEMPT_NUMBER = 3; + JCloudsArtifactManager.client = new RobustHTTPClient(); + JCloudsArtifactManager.client.setStopAfterAttemptNumber(3); try { - failIn(BlobStoreProvider.HttpMethod.PUT, "p/1/artifacts/f", 500, JCloudsArtifactManager.UPLOAD_STOP_AFTER_ATTEMPT_NUMBER); + failIn(BlobStoreProvider.HttpMethod.PUT, "p/1/artifacts/f", 500, 3); p.setDefinition(new CpsFlowDefinition("node('remote') {writeFile file: 'f', text: '.'; archiveArtifacts 'f'}", true)); WorkflowRun b = r.assertBuildStatus(Result.FAILURE, p.scheduleBuild2(0)); r.assertLogContains("ERROR: Failed to upload", b); @@ -124,16 +141,15 @@ public void repeatedRecoverableExceptionArchiving() throws Exception { r.assertLogContains("Retrying upload", b); r.assertLogNotContains("\tat hudson.tasks.ArtifactArchiver.perform", b); } finally { - JCloudsArtifactManager.UPLOAD_STOP_AFTER_ATTEMPT_NUMBER = origStopAfterAttemptNumber; + JCloudsArtifactManager.client = new RobustHTTPClient(); } } @Test public void hangArchiving() throws Exception { WorkflowJob p = r.createProject(WorkflowJob.class, "p"); - r.createSlave("remote", null, null); - long origTimeout = JCloudsArtifactManager.UPLOAD_TIMEOUT; - JCloudsArtifactManager.UPLOAD_TIMEOUT = 5; + JCloudsArtifactManager.client = new RobustHTTPClient(); + JCloudsArtifactManager.client.setTimeout(5); try { hangIn(BlobStoreProvider.HttpMethod.PUT, "p/1/artifacts/f"); p.setDefinition(new CpsFlowDefinition("node('remote') {writeFile file: 'f', text: '.'; archiveArtifacts 'f'}", true)); @@ -147,14 +163,13 @@ public void hangArchiving() throws Exception { r.assertLogContains("Retrying upload", b); r.assertLogNotContains("\tat hudson.tasks.ArtifactArchiver.perform", b); } finally { - JCloudsArtifactManager.UPLOAD_TIMEOUT = origTimeout; + JCloudsArtifactManager.client = new RobustHTTPClient(); } } @Test public void interruptedArchiving() throws Exception { WorkflowJob p = r.createProject(WorkflowJob.class, "p"); - r.createSlave("remote", null, null); hangIn(BlobStoreProvider.HttpMethod.PUT, "p/1/artifacts/f"); p.setDefinition(new CpsFlowDefinition("node('remote') {writeFile file: 'f', text: '.'; archiveArtifacts 'f'}", true)); WorkflowRun b = p.scheduleBuild2(0).waitForStart(); @@ -169,6 +184,188 @@ public void interruptedArchiving() throws Exception { r.assertLogContains(new TimeoutStepExecution.ExceededTimeout().getShortDescription(), r.assertBuildStatus(Result.ABORTED, p.scheduleBuild2(0))); } + @Test + public void unrecoverableErrorUnstashing() throws Exception { + WorkflowJob p = r.createProject(WorkflowJob.class, "p"); + failIn(BlobStoreProvider.HttpMethod.GET, "p/1/stashes/f.tgz", 403, 0); + p.setDefinition(new CpsFlowDefinition("node('remote') {writeFile file: 'f', text: '.'; stash 'f'; unstash 'f'}", true)); + WorkflowRun b = r.assertBuildStatus(Result.FAILURE, p.scheduleBuild2(0)); + r.assertLogContains("ERROR: Failed to download", b); + r.assertLogContains("/container/p/1/stashes/f.tgz?…", b); + r.assertLogContains("response: 403 simulated 403 failure, body: Detailed explanation of 403.", b); + r.assertLogNotContains("Retrying download", b); + r.assertLogNotContains("\tat org.jenkinsci.plugins.workflow.flow.StashManager.unstash", b); + } + + @Test + public void recoverableErrorUnstashing() throws Exception { + WorkflowJob p = r.createProject(WorkflowJob.class, "p"); + failIn(BlobStoreProvider.HttpMethod.GET, "p/1/stashes/f.tgz", 500, 0); + p.setDefinition(new CpsFlowDefinition("node('remote') {writeFile file: 'f', text: '.'; stash 'f'; unstash 'f'}", true)); + WorkflowRun b = r.buildAndAssertSuccess(p); + r.assertLogContains("/container/p/1/stashes/f.tgz?…", b); + r.assertLogContains("response: 500 simulated 500 failure, body: Detailed explanation of 500.", b); + r.assertLogContains("Retrying download", b); + r.assertLogNotContains("\tat org.jenkinsci.plugins.workflow.flow.StashManager.unstash", b); + } + + @Test + public void networkExceptionUnstashing() throws Exception { + WorkflowJob p = r.createProject(WorkflowJob.class, "p"); + // failIn does not work: URL connection gets a 200 status despite a ConnectionClosedException being thrown; a new connection is made. + MockBlobStore.speciallyHandle(BlobStoreProvider.HttpMethod.GET, "p/1/stashes/f.tgz", (request, response, context) -> {}); + p.setDefinition(new CpsFlowDefinition("node('remote') {writeFile file: 'f', text: '.'; stash 'f'; unstash 'f'}", true)); + WorkflowRun b = r.buildAndAssertSuccess(p); + r.assertLogContains("Retrying download", b); + // Currently catches an error from FilePath.untarFrom: java.io.IOException: Failed to extract input stream + r.assertLogNotContains("\tat org.jenkinsci.plugins.workflow.flow.StashManager.unstash", b); + } + + @Test + public void repeatedRecoverableErrorUnstashing() throws Exception { + WorkflowJob p = r.createProject(WorkflowJob.class, "p"); + JCloudsArtifactManager.client = new RobustHTTPClient(); + JCloudsArtifactManager.client.setStopAfterAttemptNumber(3); + try { + failIn(BlobStoreProvider.HttpMethod.GET, "p/1/stashes/f.tgz", 500, 3); + p.setDefinition(new CpsFlowDefinition("node('remote') {writeFile file: 'f', text: '.'; stash 'f'; unstash 'f'}", true)); + WorkflowRun b = r.assertBuildStatus(Result.FAILURE, p.scheduleBuild2(0)); + r.assertLogContains("ERROR: Failed to download", b); + r.assertLogContains("/container/p/1/stashes/f.tgz?…", b); + r.assertLogContains("response: 500 simulated 500 failure, body: Detailed explanation of 500.", b); + r.assertLogContains("Retrying download", b); + r.assertLogNotContains("\tat org.jenkinsci.plugins.workflow.flow.StashManager.unstash", b); + } finally { + JCloudsArtifactManager.client = new RobustHTTPClient(); + } + } + + @Test + public void hangUnstashing() throws Exception { + WorkflowJob p = r.createProject(WorkflowJob.class, "p"); + JCloudsArtifactManager.client = new RobustHTTPClient(); + JCloudsArtifactManager.client.setTimeout(5); + try { + hangIn(BlobStoreProvider.HttpMethod.GET, "p/1/stashes/f.tgz"); + p.setDefinition(new CpsFlowDefinition("node('remote') {writeFile file: 'f', text: '.'; stash 'f'; unstash 'f'}", true)); + WorkflowRun b = r.buildAndAssertSuccess(p); + r.assertLogContains("Retrying download", b); + r.assertLogNotContains("\tat org.jenkinsci.plugins.workflow.flow.StashManager.unstash", b); + hangIn(BlobStoreProvider.HttpMethod.GET, "p/2/stashes/f.tgz"); + p.setDefinition(new CpsFlowDefinition("node('master') {writeFile file: 'f', text: '.'; stash 'f'; unstash 'f'}", true)); + b = r.buildAndAssertSuccess(p); + r.assertLogContains("Retrying download", b); + r.assertLogNotContains("\tat org.jenkinsci.plugins.workflow.flow.StashManager.unstash", b); + } finally { + JCloudsArtifactManager.client = new RobustHTTPClient(); + } + } + + @Test + public void interruptedUnstashing() throws Exception { + WorkflowJob p = r.createProject(WorkflowJob.class, "p"); + hangIn(BlobStoreProvider.HttpMethod.GET, "p/1/stashes/f.tgz"); + p.setDefinition(new CpsFlowDefinition("node('remote') {writeFile file: 'f', text: '.'; stash 'f'; unstash 'f'}", true)); + WorkflowRun b = p.scheduleBuild2(0).waitForStart(); + r.waitForMessage("[Pipeline] unstash", b); + Thread.sleep(2000); + b.getExecutor().interrupt(); + r.assertBuildStatus(Result.ABORTED, r.waitForCompletion(b)); + hangIn(BlobStoreProvider.HttpMethod.GET, "p/2/stashes/f.tgz"); + p.setDefinition(new CpsFlowDefinition("node('master') {writeFile file: 'f', text: '.'; stash 'f'; timeout(time: 3, unit: 'SECONDS') {unstash 'f'}}", true)); + r.assertLogContains(new TimeoutStepExecution.ExceededTimeout().getShortDescription(), r.assertBuildStatus(Result.ABORTED, p.scheduleBuild2(0))); + } + + @Test + public void recoverableErrorUnarchiving() throws Exception { + WorkflowJob p = r.createProject(WorkflowJob.class, "p"); + failIn(BlobStoreProvider.HttpMethod.GET, "p/1/artifacts/f", 500, 0); + p.setDefinition(new CpsFlowDefinition("node('remote') {writeFile file: 'f', text: '.'; archiveArtifacts 'f'; unarchive mapping: ['f': 'f']}", true)); + WorkflowRun b = r.buildAndAssertSuccess(p); + r.assertLogContains("/container/p/1/artifacts/f?…", b); + r.assertLogContains("response: 500 simulated 500 failure, body: Detailed explanation of 500.", b); + r.assertLogContains("Retrying download", b); + r.assertLogNotContains("\tat org.jenkinsci.plugins.workflow.steps.ArtifactUnarchiverStepExecution.run", b); + } + + // TBD if jclouds, or its S3 provider, is capable of differentiating recoverable from nonrecoverable errors. The error simulated here: + // org.jclouds.blobstore.ContainerNotFoundException: nonexistent.s3.amazonaws.com not found: The specified bucket does not exist + // at org.jclouds.s3.handlers.ParseS3ErrorFromXmlContent.refineException(ParseS3ErrorFromXmlContent.java:81) + // at org.jclouds.aws.handlers.ParseAWSErrorFromXmlContent.handleError(ParseAWSErrorFromXmlContent.java:89) + // at … + // Disconnecting network in the middle of some operations produces a bunch of warnings from BackoffLimitedRetryHandler.ifReplayableBackoffAndReturnTrue + // followed by a org.jclouds.http.HttpResponseException: Network is unreachable (connect failed) connecting to GET … + // Also not testing hangs here since org.jclouds.Constants.PROPERTY_SO_TIMEOUT/PROPERTY_CONNECTION_TIMEOUT probably handle this. + @Test + public void errorListing() throws Exception { + WorkflowJob p = r.createProject(WorkflowJob.class, "p"); + MockApiMetadata.handleGetBlobKeysInsideContainer("container", () -> {throw new ContainerNotFoundException("container", "sorry");}); + p.setDefinition(new CpsFlowDefinition("node('remote') {writeFile file: 'f', text: '.'; archiveArtifacts 'f'; unarchive mapping: ['f': 'f']}", true)); + WorkflowRun b = r.assertBuildStatus(Result.FAILURE, p.scheduleBuild2(0)); + r.assertLogContains(ContainerNotFoundException.class.getName(), b); + // Currently prints a stack trace, OK. + } + + // Interrupts during a network operation seem to have no effect; when retrying during network disconnection, + // BackoffLimitedRetryHandler.imposeBackoffExponentialDelay throws InterruptedException wrapped in RuntimeException. + @Test + public void interruptedListing() throws Exception { + WorkflowJob p = r.createProject(WorkflowJob.class, "p"); + MockApiMetadata.handleGetBlobKeysInsideContainer("container", () -> { + try { + Thread.sleep(Long.MAX_VALUE); + return null; // to satisfy compiler + } catch (InterruptedException x) { + throw new RuntimeException(x); + } + }); + p.setDefinition(new CpsFlowDefinition("node('remote') {writeFile file: 'f', text: '.'; archiveArtifacts 'f'; timeout(time: 3, unit: 'SECONDS') {unarchive mapping: ['f': 'f']}}", true)); + r.assertLogContains(new TimeoutStepExecution.ExceededTimeout().getShortDescription(), r.assertBuildStatus(Result.ABORTED, p.scheduleBuild2(0))); + } + + @Test + public void errorCleaning() throws Exception { + loggerRule.record(WorkflowRun.class, Level.WARNING).capture(10); + WorkflowJob p = r.createProject(WorkflowJob.class, "p"); + p.setDefinition(new CpsFlowDefinition("node('remote') {writeFile file: 'f', text: '.'; archiveArtifacts 'f'; stash 'stuff'}", true)); + MockApiMetadata.handleRemoveBlob("container", "p/1/stashes/stuff.tgz", () -> {throw new ContainerNotFoundException("container", "sorry about your stashes");}); + r.buildAndAssertSuccess(p); + p.setBuildDiscarder(new LogRotator(-1, -1, -1, 0)); + MockApiMetadata.handleRemoveBlob("container", "p/1/artifacts/f", () -> {throw new ContainerNotFoundException("container", "sorry about your artifacts");}); + r.buildAndAssertSuccess(p); + assertThat(loggerRule.getRecords().stream().map(LogRecord::getThrown).filter(Objects::nonNull).map(Throwables::getRootCause).map(Throwable::getMessage).collect(Collectors.toSet()), + containsInAnyOrder("container not found: sorry about your stashes", "container not found: sorry about your artifacts")); + } + + // Interrupts probably never delivered during HTTP requests (maybe depends on servlet container?). + // Hangs would be handled by jclouds code. + @Test + public void errorBrowsing() throws Exception { + WorkflowJob p = r.createProject(WorkflowJob.class, "p"); + p.setDefinition(new CpsFlowDefinition("node('remote') {writeFile file: 'f', text: '.'; archiveArtifacts 'f'}", true)); + WorkflowRun b = r.buildAndAssertSuccess(p); + MockApiMetadata.handleGetBlobKeysInsideContainer("container", () -> {throw new ContainerNotFoundException("container", "sorry");}); + JenkinsRule.WebClient wc = r.createWebClient(); + { + System.err.println("build root"); + loggerRule.record(Run.class, Level.WARNING).capture(10); + wc.getPage(b); + assertThat(loggerRule.getRecords().stream().map(LogRecord::getThrown).filter(Objects::nonNull).map(Throwables::getRootCause).map(Throwable::getMessage).collect(Collectors.toSet()), + containsInAnyOrder("container not found: sorry")); + } + { + System.err.println("artifact root"); + MockApiMetadata.handleGetBlobKeysInsideContainer("container", () -> {throw new ContainerNotFoundException("container", "sorry");}); + try { + wc.getPage(b, "artifact/"); + fail("Currently DirectoryBrowserSupport throws up storage exceptions."); + } catch (FailingHttpStatusCodeException x) { + assertEquals(500, x.getStatusCode()); + assertThat(x.getResponse().getContentAsString(), containsString("container not found: sorry")); + } + } + } + private static void failIn(BlobStoreProvider.HttpMethod method, String key, int code, int repeats) { MockBlobStore.speciallyHandle(method, key, (request, response, context) -> { if (repeats > 0) { diff --git a/src/test/java/io/jenkins/plugins/artifact_manager_s3/JCloudsArtifactManagerTest.java b/src/test/java/io/jenkins/plugins/artifact_manager_s3/JCloudsArtifactManagerTest.java index 22329d20..abeb044b 100644 --- a/src/test/java/io/jenkins/plugins/artifact_manager_s3/JCloudsArtifactManagerTest.java +++ b/src/test/java/io/jenkins/plugins/artifact_manager_s3/JCloudsArtifactManagerTest.java @@ -238,7 +238,7 @@ public void serializationProblem() throws Exception { S3BlobStore.BREAK_CREDS = true; try { WorkflowRun b = j.buildAndAssertSuccess(p); - j.assertLogContains("caught org.jclouds.aws.AWSResponseException", b); + j.assertLogContains("caught java.io.IOException: org.jclouds.aws.AWSResponseException", b); j.assertLogNotContains("java.io.NotSerializableException", b); } finally { S3BlobStore.BREAK_CREDS = false;