diff --git a/pom.xml b/pom.xml index ab5ad132..6102f7c7 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ org.jenkins-ci.plugins plugin - 3.20 + 3.24 io.jenkins.plugins @@ -16,10 +16,9 @@ 1.2 -SNAPSHOT - 2.1.1 + 2.1.1 2.121.3 8 - 2.29 4.0.0-rc2883.3399537b3d44 2.20 true @@ -44,6 +43,11 @@ + + io.jenkins.plugins + artifact-manager-jclouds + 1.0-SNAPSHOT + io.jenkins.plugins aws-global-configuration @@ -59,11 +63,6 @@ aws-credentials 1.23 - - org.jenkins-ci.plugins - apache-httpcomponents-client-4-api - 4.5.5-3.0 - org.mockito mockito-core @@ -84,12 +83,7 @@ org.jenkins-ci.plugins.workflow workflow-api - ${workflow-api-plugin.version} - - - org.jenkins-ci.plugins.workflow - workflow-api - ${workflow-api-plugin.version} + 2.29 tests test @@ -141,12 +135,6 @@ 2.9 test - - org.kohsuke.metainf-services - metainf-services - 1.7 - test - io.findify s3mock_2.12 @@ -161,6 +149,10 @@ com.amazonaws aws-java-sdk-s3 + + com.google.guava + guava + @@ -232,13 +224,13 @@ jenkins-core ${jenkins-core.version} - + com.google.guava guava - + com.google.inject.extensions guice-assistedinject 4.0 @@ -290,11 +282,13 @@ org.jenkins-ci.tools maven-hpi-plugin - + com.google.common. - diff --git a/src/main/java/io/jenkins/plugins/artifact_manager_jclouds/BlobStoreProvider.java b/src/main/java/io/jenkins/plugins/artifact_manager_jclouds/BlobStoreProvider.java deleted file mode 100644 index ba154d56..00000000 --- a/src/main/java/io/jenkins/plugins/artifact_manager_jclouds/BlobStoreProvider.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * The MIT License - * - * Copyright 2018 CloudBees, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -package io.jenkins.plugins.artifact_manager_jclouds; - -import java.io.IOException; -import java.io.Serializable; -import java.net.URI; -import java.net.URL; - -import org.jclouds.blobstore.BlobStoreContext; -import org.jclouds.blobstore.domain.Blob; -import org.kohsuke.accmod.Restricted; -import org.kohsuke.accmod.restrictions.Beta; - -import edu.umd.cs.findbugs.annotations.NonNull; -import hudson.ExtensionPoint; -import hudson.model.AbstractDescribableImpl; - -/** - * Provider for jclouds-based blob stores usable for artifact storage. - * An instance will be copied into a build record together with any fields it defines. - */ -@Restricted(Beta.class) -public abstract class BlobStoreProvider extends AbstractDescribableImpl implements ExtensionPoint, Serializable { - - private static final long serialVersionUID = -861350249543443493L; - - public enum HttpMethod { - GET, PUT; - } - - /** A constant for the blob path prefix to use. */ - @NonNull - public abstract String getPrefix(); - - /** A constant for the blob container name to use. */ - @NonNull - public abstract String getContainer(); - - /** A constant to define whether we should delete artifacts or leave them to be managed on the blob service side. */ - public abstract boolean isDeleteArtifacts(); - - /** A constant to define whether we should delete stashes or leave them to be managed on the blob service side. */ - public abstract boolean isDeleteStashes(); - - /** Creates the jclouds handle for working with blob. */ - @NonNull - public abstract BlobStoreContext getContext() throws IOException; - - /** - * Get a provider-specific URI. - * - * @param container - * container where this exists. - * @param key - * fully qualified name relative to the container. - * @return the URI - */ - @NonNull - public abstract URI toURI(@NonNull String container, @NonNull String key); - - /** - * Generate a URL valid for downloading OR uploading the blob for a limited period of time - * - * @param blob - * blob to generate the URL for - * @param httpMethod - * HTTP method to create a URL for (downloads or uploads) - * @return the URL - * @throws IOException - */ - @NonNull - public abstract URL toExternalURL(@NonNull Blob blob, @NonNull HttpMethod httpMethod) throws IOException; - - @Override - public BlobStoreProviderDescriptor getDescriptor() { - return (BlobStoreProviderDescriptor) super.getDescriptor(); - } - -} diff --git a/src/main/java/io/jenkins/plugins/artifact_manager_jclouds/BlobStoreProviderDescriptor.java b/src/main/java/io/jenkins/plugins/artifact_manager_jclouds/BlobStoreProviderDescriptor.java deleted file mode 100644 index 96fbb017..00000000 --- a/src/main/java/io/jenkins/plugins/artifact_manager_jclouds/BlobStoreProviderDescriptor.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * The MIT License - * - * Copyright 2018 CloudBees, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -package io.jenkins.plugins.artifact_manager_jclouds; - -import hudson.model.Descriptor; -import org.kohsuke.accmod.Restricted; -import org.kohsuke.accmod.restrictions.Beta; - -/** - * Descriptor type for {@link BlobStoreProvider}. - */ -@Restricted(Beta.class) -public abstract class BlobStoreProviderDescriptor extends Descriptor {} diff --git a/src/main/java/io/jenkins/plugins/artifact_manager_jclouds/JCloudsArtifactManager.java b/src/main/java/io/jenkins/plugins/artifact_manager_jclouds/JCloudsArtifactManager.java deleted file mode 100644 index e06da2a7..00000000 --- a/src/main/java/io/jenkins/plugins/artifact_manager_jclouds/JCloudsArtifactManager.java +++ /dev/null @@ -1,326 +0,0 @@ -/* - * The MIT License - * - * Copyright 2018 CloudBees, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -package io.jenkins.plugins.artifact_manager_jclouds; - -import hudson.AbortException; -import hudson.EnvVars; -import hudson.FilePath; -import hudson.Launcher; -import hudson.Util; -import hudson.model.BuildListener; -import hudson.model.Run; -import hudson.model.TaskListener; -import hudson.remoting.VirtualChannel; -import hudson.slaves.WorkspaceList; -import hudson.util.DirScanner; -import hudson.util.io.ArchiverFactory; -import edu.umd.cs.findbugs.annotations.NonNull; -import io.jenkins.plugins.artifact_manager_jclouds.BlobStoreProvider.HttpMethod; -import io.jenkins.plugins.httpclient.RobustHTTPClient; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.URI; -import java.net.URL; -import java.nio.file.Files; -import java.nio.file.InvalidPathException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.HashMap; -import java.util.Map; -import java.util.logging.Level; -import java.util.logging.Logger; -import jenkins.MasterToSlaveFileCallable; -import jenkins.model.ArtifactManager; -import jenkins.util.VirtualFile; -import org.apache.http.client.methods.HttpGet; -import org.jclouds.blobstore.BlobStore; -import org.jclouds.blobstore.BlobStoreContext; -import org.jclouds.blobstore.BlobStores; -import org.jclouds.blobstore.domain.Blob; -import org.jclouds.blobstore.domain.StorageMetadata; -import org.jclouds.blobstore.options.CopyOptions; -import org.jclouds.blobstore.options.ListContainerOptions; -import org.jenkinsci.plugins.workflow.flow.StashManager; -import org.kohsuke.accmod.Restricted; -import org.kohsuke.accmod.restrictions.NoExternalUse; - -/** - * Jenkins artifact/stash implementation using any blob store supported by Apache jclouds. - * To offer a new backend, implement {@link BlobStoreProvider}. - */ -@Restricted(NoExternalUse.class) -public final class JCloudsArtifactManager extends ArtifactManager implements StashManager.StashAwareArtifactManager { - - private static final Logger LOGGER = Logger.getLogger(JCloudsArtifactManager.class.getName()); - - static RobustHTTPClient client = new RobustHTTPClient(); - - private final BlobStoreProvider provider; - - private transient String key; // e.g. myorg/myrepo/master/123 - - JCloudsArtifactManager(@NonNull Run build, BlobStoreProvider provider) { - this.provider = provider; - onLoad(build); - } - - private Object readResolve() { - if (provider == null) { - throw new IllegalStateException("Missing provider field"); - } - return this; - } - - @Override - public void onLoad(@NonNull Run build) { - this.key = String.format("%s/%s", build.getParent().getFullName(), build.getNumber()); - } - - private String getBlobPath(String path) { - return getBlobPath(key, path); - } - - private String getBlobPath(String key, String path) { - return String.format("%s%s/%s", provider.getPrefix(), key, path); - } - - /* - * This could be called multiple times - */ - @Override - public void archive(FilePath workspace, Launcher launcher, BuildListener listener, Map artifacts) - throws IOException, InterruptedException { - LOGGER.log(Level.FINE, "Archiving from {0}: {1}", new Object[] { workspace, artifacts }); - Map artifactUrls = new HashMap<>(); - BlobStore blobStore = getContext().getBlobStore(); - - // Map artifacts to urls for upload - for (Map.Entry entry : artifacts.entrySet()) { - String path = "artifacts/" + entry.getKey(); - String blobPath = getBlobPath(path); - Blob blob = blobStore.blobBuilder(blobPath).build(); - blob.getMetadata().setContainer(provider.getContainer()); - artifactUrls.put(entry.getValue(), provider.toExternalURL(blob, HttpMethod.PUT)); - } - - workspace.act(new UploadToBlobStorage(artifactUrls, listener)); - listener.getLogger().printf("Uploaded %s artifact(s) to %s%n", artifactUrls.size(), provider.toURI(provider.getContainer(), getBlobPath("artifacts/"))); - } - - private static class UploadToBlobStorage extends MasterToSlaveFileCallable { - private static final long serialVersionUID = 1L; - - private final Map artifactUrls; // e.g. "target/x.war", "http://..." - private final TaskListener listener; - // Bind when constructed on the master side; on the agent side, deserialize the same configuration. - private final RobustHTTPClient client = JCloudsArtifactManager.client; - - UploadToBlobStorage(Map artifactUrls, TaskListener listener) { - this.artifactUrls = artifactUrls; - this.listener = listener; - } - - @Override - public Void invoke(File f, VirtualChannel channel) throws IOException, InterruptedException { - for (Map.Entry entry : artifactUrls.entrySet()) { - client.uploadFile(new File(f, entry.getKey()), entry.getValue(), listener); - } - return null; - } - } - - @Override - public boolean delete() throws IOException, InterruptedException { - String blobPath = getBlobPath(""); - if (!provider.isDeleteArtifacts()) { - LOGGER.log(Level.FINE, "Ignoring blob deletion: {0}", blobPath); - return false; - } - return JCloudsVirtualFile.delete(provider, getContext().getBlobStore(), blobPath); - } - - @Override - public VirtualFile root() { - return new JCloudsVirtualFile(provider, provider.getContainer(), getBlobPath("artifacts")); - } - - @Override - public void stash(String name, FilePath workspace, Launcher launcher, EnvVars env, TaskListener listener, String includes, String excludes, boolean useDefaultExcludes, boolean allowEmpty) throws IOException, InterruptedException { - BlobStore blobStore = getContext().getBlobStore(); - - // Map stash to url for upload - String path = getBlobPath("stashes/" + name + ".tgz"); - Blob blob = blobStore.blobBuilder(path).build(); - blob.getMetadata().setContainer(provider.getContainer()); - URL url = provider.toExternalURL(blob, HttpMethod.PUT); - workspace.act(new Stash(url, provider.toURI(provider.getContainer(), path), includes, excludes, useDefaultExcludes, allowEmpty, WorkspaceList.tempDir(workspace).getRemote(), listener)); - } - - private static final class Stash extends MasterToSlaveFileCallable { - private static final long serialVersionUID = 1L; - private final URL url; - private final URI uri; - private final String includes, excludes; - private final boolean useDefaultExcludes; - private final boolean allowEmpty; - private final String tempDir; - private final TaskListener listener; - private final RobustHTTPClient client = JCloudsArtifactManager.client; - - Stash(URL url, URI uri, String includes, String excludes, boolean useDefaultExcludes, boolean allowEmpty, String tempDir, TaskListener listener) throws IOException { - /** Actual destination as a presigned URL. */ - this.url = url; - /** Logical location for display purposes only. */ - this.uri = uri; - this.includes = includes; - this.excludes = excludes; - this.useDefaultExcludes = useDefaultExcludes; - this.allowEmpty = allowEmpty; - this.tempDir = tempDir; - this.listener = listener; - } - - @Override - public Void invoke(File f, VirtualChannel channel) throws IOException, InterruptedException { - // TODO use streaming upload rather than a temp file; is it necessary to set the content length in advance? - // (we prefer not to upload individual files for stashes, so as to preserve symlinks & file permissions, as StashManager’s default does) - Path tempDirP = Paths.get(tempDir); - Files.createDirectories(tempDirP); - Path tmp = Files.createTempFile(tempDirP, "stash", ".tgz"); - try { - int count; - try (OutputStream os = Files.newOutputStream(tmp)) { - count = new FilePath(f).archive(ArchiverFactory.TARGZ, os, new DirScanner.Glob(Util.fixEmpty(includes) == null ? "**" : includes, excludes, useDefaultExcludes)); - } catch (InvalidPathException e) { - throw new IOException(e); - } - if (count == 0 && !allowEmpty) { - throw new AbortException("No files included in stash"); - } - client.uploadFile(tmp.toFile(), url, listener); - listener.getLogger().printf("Stashed %d file(s) to %s%n", count, uri); - return null; - } finally { - Files.delete(tmp); - } - } - } - - @Override - public void unstash(String name, FilePath workspace, Launcher launcher, EnvVars env, TaskListener listener) throws IOException, InterruptedException { - BlobStore blobStore = getContext().getBlobStore(); - - // Map stash to url for download - String blobPath = getBlobPath("stashes/" + name + ".tgz"); - Blob blob = blobStore.getBlob(provider.getContainer(), blobPath); - if (blob == null) { - throw new AbortException( - String.format("No such saved stash ‘%s’ found at %s/%s", name, provider.getContainer(), blobPath)); - } - URL url = provider.toExternalURL(blob, HttpMethod.GET); - workspace.act(new Unstash(url, listener)); - listener.getLogger().printf("Unstashed file(s) from %s%n", provider.toURI(provider.getContainer(), blobPath)); - } - - private static final class Unstash extends MasterToSlaveFileCallable { - private static final long serialVersionUID = 1L; - private final URL url; - private final TaskListener listener; - private final RobustHTTPClient client = JCloudsArtifactManager.client; - - Unstash(URL url, TaskListener listener) throws IOException { - this.url = url; - this.listener = listener; - } - - @Override - public Void invoke(File f, VirtualChannel channel) throws IOException, InterruptedException { - client.connect("download", "download " + RobustHTTPClient.sanitize(url) + " into " + f, c -> c.execute(new HttpGet(url.toString())), response -> { - try (InputStream is = response.getEntity().getContent()) { - new FilePath(f).untarFrom(is, FilePath.TarCompression.GZIP); - // Note that this API currently offers no count of files in the tarball we could report. - } - }, listener); - return null; - } - } - - @Override - public void clearAllStashes(TaskListener listener) throws IOException, InterruptedException { - String stashPrefix = getBlobPath("stashes/"); - - if (!provider.isDeleteStashes()) { - LOGGER.log(Level.FINE, "Ignoring stash deletion: {0}", stashPrefix); - return; - } - - BlobStore blobStore = getContext().getBlobStore(); - int count = 0; - try { - for (StorageMetadata sm : BlobStores.listAll(blobStore, provider.getContainer(), ListContainerOptions.Builder.prefix(stashPrefix).recursive())) { - String path = sm.getName(); - assert path.startsWith(stashPrefix); - LOGGER.fine("deleting " + path); - blobStore.removeBlob(provider.getContainer(), path); - count++; - } - } catch (RuntimeException x) { - throw new IOException(x); - } - listener.getLogger().printf("Deleted %d stash(es) from %s%n", count, provider.toURI(provider.getContainer(), stashPrefix)); - } - - @Override - public void copyAllArtifactsAndStashes(Run to, TaskListener listener) throws IOException, InterruptedException { - ArtifactManager am = to.pickArtifactManager(); - if (!(am instanceof JCloudsArtifactManager)) { - throw new AbortException("Cannot copy artifacts and stashes to " + to + " using " + am.getClass().getName()); - } - JCloudsArtifactManager dest = (JCloudsArtifactManager) am; - String allPrefix = getBlobPath(""); - BlobStore blobStore = getContext().getBlobStore(); - int count = 0; - try { - for (StorageMetadata sm : BlobStores.listAll(blobStore, provider.getContainer(), ListContainerOptions.Builder.prefix(allPrefix).recursive())) { - String path = sm.getName(); - assert path.startsWith(allPrefix); - String destPath = getBlobPath(dest.key, path.substring(allPrefix.length())); - LOGGER.fine("copying " + path + " to " + destPath); - blobStore.copyBlob(provider.getContainer(), path, provider.getContainer(), destPath, CopyOptions.NONE); - count++; - } - } catch (RuntimeException x) { - throw new IOException(x); - } - listener.getLogger().printf("Copied %d artifact(s)/stash(es) from %s to %s%n", count, provider.toURI(provider.getContainer(), allPrefix), provider.toURI(provider.getContainer(), dest.getBlobPath(""))); - } - - private BlobStoreContext getContext() throws IOException { - return provider.getContext(); - } - -} diff --git a/src/main/java/io/jenkins/plugins/artifact_manager_jclouds/JCloudsArtifactManagerFactory.java b/src/main/java/io/jenkins/plugins/artifact_manager_jclouds/JCloudsArtifactManagerFactory.java deleted file mode 100644 index 7dfed389..00000000 --- a/src/main/java/io/jenkins/plugins/artifact_manager_jclouds/JCloudsArtifactManagerFactory.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * The MIT License - * - * Copyright 2018 CloudBees, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -package io.jenkins.plugins.artifact_manager_jclouds; - -import hudson.Extension; -import hudson.model.Run; -import jenkins.model.ArtifactManager; -import jenkins.model.ArtifactManagerFactory; -import jenkins.model.ArtifactManagerFactoryDescriptor; -import org.jenkinsci.Symbol; -import org.kohsuke.accmod.Restricted; -import org.kohsuke.accmod.restrictions.NoExternalUse; -import org.kohsuke.stapler.DataBoundConstructor; - -/** - * Factory for {@link ArtifactManager} - */ -@Restricted(NoExternalUse.class) -public class JCloudsArtifactManagerFactory extends ArtifactManagerFactory { - - private final BlobStoreProvider provider; - - @DataBoundConstructor - public JCloudsArtifactManagerFactory(BlobStoreProvider provider) { - if (provider == null) { - throw new IllegalArgumentException(); - } - this.provider = provider; - } - - private Object readResolve() { - if (provider == null) { - throw new IllegalStateException("Missing provider field"); - } - return this; - } - - public BlobStoreProvider getProvider() { - return provider; - } - - @Override - public ArtifactManager managerFor(Run build) { - return new JCloudsArtifactManager(build, provider); - } - - @Symbol("jclouds") - @Extension - public static final class DescriptorImpl extends ArtifactManagerFactoryDescriptor { - - @Override - public String getDisplayName() { - return "Cloud Artifact Storage"; - } - - } - -} diff --git a/src/main/java/io/jenkins/plugins/artifact_manager_jclouds/JCloudsVirtualFile.java b/src/main/java/io/jenkins/plugins/artifact_manager_jclouds/JCloudsVirtualFile.java deleted file mode 100644 index 00dce6e2..00000000 --- a/src/main/java/io/jenkins/plugins/artifact_manager_jclouds/JCloudsVirtualFile.java +++ /dev/null @@ -1,378 +0,0 @@ -/* - * The MIT License - * - * Copyright 2018 CloudBees, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -package io.jenkins.plugins.artifact_manager_jclouds; - -import static org.jclouds.blobstore.options.ListContainerOptions.Builder.*; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.net.URL; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.Deque; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.StreamSupport; - -import org.jclouds.blobstore.BlobStore; -import org.jclouds.blobstore.BlobStoreContext; -import org.jclouds.blobstore.BlobStores; -import org.jclouds.blobstore.domain.Blob; -import org.jclouds.blobstore.domain.MutableBlobMetadata; -import org.jclouds.blobstore.domain.StorageMetadata; -import org.jclouds.blobstore.options.ListContainerOptions; -import org.jclouds.rest.AuthorizationException; -import org.kohsuke.accmod.Restricted; -import org.kohsuke.accmod.restrictions.NoExternalUse; - -import edu.umd.cs.findbugs.annotations.CheckForNull; -import edu.umd.cs.findbugs.annotations.NonNull; -import hudson.AbortException; -import hudson.remoting.Callable; -import io.jenkins.plugins.artifact_manager_jclouds.BlobStoreProvider.HttpMethod; -import jenkins.util.VirtualFile; - -/** - * JClouds BlobStore Guide - */ -@Restricted(NoExternalUse.class) -public class JCloudsVirtualFile extends VirtualFile { - - private static final long serialVersionUID = -5126878907895121335L; - - private static final Logger LOGGER = Logger.getLogger(JCloudsVirtualFile.class.getName()); - - @NonNull - private BlobStoreProvider provider; - @NonNull - private final String container; - @NonNull - private final String key; - @CheckForNull - private transient Blob blob; - @CheckForNull - private transient BlobStoreContext context; - - public JCloudsVirtualFile(@NonNull BlobStoreProvider provider, @NonNull String container, @NonNull String key) { - this.provider = provider; - this.container = container; - this.key = key; - assert !key.isEmpty(); - assert !key.startsWith("/"); - assert !key.endsWith("/"); - } - - private JCloudsVirtualFile(@NonNull JCloudsVirtualFile related, @NonNull String key) { - this(related.provider, related.container, key); - context = related.context; - } - - /** - * Build jclouds blob context that is the base for all operations - */ - @Restricted(NoExternalUse.class) // testing only - BlobStoreContext getContext() throws IOException { - if (context == null) { - context = provider.getContext(); - } - return context; - } - - private String getContainer() { - return container; - } - - /** - * Returns the full name, directories included - */ - private String getKey() { - return key; - } - - /** - * Returns the base name - */ - @Override - public String getName() { - return key.replaceFirst(".+/", ""); - } - - private Blob getBlob() throws IOException { - if (blob == null) { - LOGGER.log(Level.FINE, "checking for existence of blob {0} / {1}", new Object[] {container, key}); - blob = getContext().getBlobStore().getBlob(getContainer(), getKey()); - if (blob == null) { - blob = getContext().getBlobStore().blobBuilder(getKey()).build(); - blob.getMetadata().setContainer(getContainer()); - } - } - return blob; - } - - @Override - public URI toURI() { - return provider.toURI(container, key); - } - - @Override - public URL toExternalURL() throws IOException { - return provider.toExternalURL(getBlob(), HttpMethod.GET); - } - - @Override - public VirtualFile getParent() { - // undefined to go outside …/artifacts - return new JCloudsVirtualFile(this, key.replaceFirst("/[^/]+$", "")); - } - - @Override - public boolean isDirectory() throws IOException { - String keyS = key + "/"; - CacheFrame frame = findCacheFrame(keyS); - if (frame != null) { - LOGGER.log(Level.FINER, "cache hit on directory status of {0} / {1}", new Object[] {container, key}); - String relSlash = keyS.substring(frame.root.length()); // "" or "sub/dir/" - return frame.children.keySet().stream().anyMatch(f -> f.startsWith(relSlash)); - } - LOGGER.log(Level.FINE, "checking directory status {0} / {1}", new Object[] {container, key}); - return !getContext().getBlobStore().list(getContainer(), prefix(key + "/")).isEmpty(); - } - - @Override - public boolean isFile() throws IOException { - CacheFrame frame = findCacheFrame(key); - if (frame != null) { - String rel = key.substring(frame.root.length()); - CachedMetadata metadata = frame.children.get(rel); - LOGGER.log(Level.FINER, "cache hit on file status of {0} / {1}", new Object[] {container, key}); - return metadata != null; - } - LOGGER.log(Level.FINE, "checking file status {0} / {1}", new Object[] {container, key}); - return getBlob().getMetadata().getSize() != null; - } - - @Override - public boolean exists() throws IOException { - return isDirectory() || isFile(); - } - - /** - * List all the blobs under this one - * - * @return some blobs - * @throws RuntimeException either now or when the stream is processed; wrap in {@link IOException} if desired - */ - private Iterable listStorageMetadata(boolean recursive) throws IOException { - ListContainerOptions options = prefix(key + "/"); - if (recursive) { - options.recursive(); - } - return BlobStores.listAll(getContext().getBlobStore(), getContainer(), options); - } - - @Override - public VirtualFile[] list() throws IOException { - String keyS = key + "/"; - CacheFrame frame = findCacheFrame(keyS); - if (frame != null) { - LOGGER.log(Level.FINER, "cache hit on listing of {0} / {1}", new Object[] {container, key}); - String relSlash = keyS.substring(frame.root.length()); // "" or "sub/dir/" - return frame.children.keySet().stream(). // filenames relative to frame root - filter(f -> f.startsWith(relSlash)). // those inside this dir - map(f -> f.substring(relSlash.length()).replaceFirst("/.+", "")). // just the file simple name, or direct subdir name - distinct(). // ignore duplicates if have multiple files under one direct subdir - map(simple -> new JCloudsVirtualFile(this, keyS + simple)). // direct children - toArray(VirtualFile[]::new); - } - VirtualFile[] list; - try { - list = StreamSupport.stream(listStorageMetadata(false).spliterator(), false) - .map(meta -> new JCloudsVirtualFile(this, meta.getName().replaceFirst("/$", ""))) - .toArray(VirtualFile[]::new); - } catch (RuntimeException x) { - throw new IOException(x); - } - LOGGER.log(Level.FINEST, "Listing files from {0} {1}: {2}", - new String[] { getContainer(), getKey(), Arrays.toString(list) }); - return list; - } - - @Override - public VirtualFile child(String name) { - return new JCloudsVirtualFile(this, key + "/" + name); - } - - @Override - public long length() throws IOException { - CacheFrame frame = findCacheFrame(key); - if (frame != null) { - String rel = key.substring(frame.root.length()); - CachedMetadata metadata = frame.children.get(rel); - LOGGER.log(Level.FINER, "cache hit on length of {0} / {1}", new Object[] {container, key}); - return metadata != null ? metadata.length : 0; - } - LOGGER.log(Level.FINE, "checking length {0} / {1}", new Object[] {container, key}); - MutableBlobMetadata metadata = getBlob().getMetadata(); - Long size = metadata == null ? Long.valueOf(0) : metadata.getSize(); - return size == null ? 0 : size; - } - - @Override - public long lastModified() throws IOException { - CacheFrame frame = findCacheFrame(key); - if (frame != null) { - String rel = key.substring(frame.root.length()); - CachedMetadata metadata = frame.children.get(rel); - LOGGER.log(Level.FINER, "cache hit on lastModified of {0} / {1}", new Object[] {container, key}); - return metadata != null ? metadata.lastModified : 0; - } - LOGGER.log(Level.FINE, "checking modification time {0} / {1}", new Object[] {container, key}); - MutableBlobMetadata metadata = getBlob().getMetadata(); - return metadata == null || metadata.getLastModified() == null ? 0 : metadata.getLastModified().getTime(); - } - - @Override - public boolean canRead() throws IOException { - return true; - } - - @Override - public InputStream open() throws IOException { - LOGGER.log(Level.FINE, "reading {0} / {1}", new Object[] {container, key}); - if (isDirectory()) { - // That is what java.io.FileInputStream.open throws - throw new FileNotFoundException(String.format("%s/%s (Is a directory)", getContainer(), getKey())); - } - if (!isFile()) { - throw new FileNotFoundException( - String.format("%s/%s (No such file or directory)", getContainer(), getKey())); - } - return getBlob().getPayload().openStream(); - } - - /** - * Cache of metadata collected during {@link #run}. - * Keys are {@link #container}. - * Values are a stack of cache frames, one per nested {@link #run} call. - */ - private static final ThreadLocal>> cache = ThreadLocal.withInitial(HashMap::new); - - private static final class CacheFrame { - /** {@link #key} of the root virtual file plus a trailing {@code /} */ - final String root; - /** - * Information about all known (recursive) child files (not directories). - * Keys are {@code /}-separated relative paths. - * If the root itself happened to be a file, that information is not cached. - */ - final Map children; - CacheFrame(String root, Map children) { - this.root = root; - this.children = children; - } - } - - /** - * Record that a given file exists. - */ - private static final class CachedMetadata { - final long length, lastModified; - CachedMetadata(long length, long lastModified) { - this.length = length; - this.lastModified = lastModified; - } - } - - @Override - public V run(Callable callable) throws IOException { - LOGGER.log(Level.FINE, "enter cache {0} / {1}", new Object[] {container, key}); - Deque stack = cacheFrames(); - Map saved = new HashMap<>(); - int prefixLength = key.length() + /* / */1; - try { - for (StorageMetadata sm : listStorageMetadata(true)) { - Long length = sm.getSize(); - if (length != null) { - Date lastModified = sm.getLastModified(); - saved.put(sm.getName().substring(prefixLength), new CachedMetadata(length, lastModified != null ? lastModified.getTime() : 0)); - } - } - } catch (AuthorizationException e) { - String cause = e.getCause() != null ? e.getCause().getMessage() : ""; - throw new AbortException(String.format("Authorization failed: %s %s", e.getMessage(), cause)); - } catch (RuntimeException x) { - throw new IOException(x); - } - stack.push(new CacheFrame(key + "/", saved)); - try { - LOGGER.log(Level.FINE, "using cache {0} / {1}: {2} file entries", new Object[] {container, key, saved.size()}); - return callable.call(); - } finally { - LOGGER.log(Level.FINE, "exit cache {0} / {1}", new Object[] {container, key}); - stack.pop(); - } - } - - private Deque cacheFrames() { - return cache.get().computeIfAbsent(container, c -> new ArrayDeque<>()); - } - - /** Finds a cache frame whose {@link CacheFrame#root} is a prefix of the given {@link #key} or {@code /}-appended variant. */ - private @CheckForNull CacheFrame findCacheFrame(String key) { - return cacheFrames().stream().filter(frame -> key.startsWith(frame.root)).findFirst().orElse(null); - } - - /** - * Delete all blobs starting with a given prefix. - */ - public static boolean delete(BlobStoreProvider provider, BlobStore blobStore, String prefix) throws IOException, InterruptedException { - try { - List paths = new ArrayList<>(); - for (StorageMetadata sm : BlobStores.listAll(blobStore, provider.getContainer(), ListContainerOptions.Builder.prefix(prefix).recursive())) { - String path = sm.getName(); - assert path.startsWith(prefix); - paths.add(path); - } - if (paths.isEmpty()) { - LOGGER.log(Level.FINE, "nothing to delete under {0}", prefix); - return false; - } else { - LOGGER.log(Level.FINE, "deleting {0} blobs under {1}", new Object[] {paths.size(), prefix}); - blobStore.removeBlobs(provider.getContainer(), paths); - return true; - } - } catch (RuntimeException x) { - throw new IOException(x); - } - } - -} diff --git a/src/main/java/io/jenkins/plugins/artifact_manager_jclouds/s3/S3BlobStoreConfig.java b/src/main/java/io/jenkins/plugins/artifact_manager_jclouds/s3/S3BlobStoreConfig.java index 2a3e4fd5..62958db0 100644 --- a/src/main/java/io/jenkins/plugins/artifact_manager_jclouds/s3/S3BlobStoreConfig.java +++ b/src/main/java/io/jenkins/plugins/artifact_manager_jclouds/s3/S3BlobStoreConfig.java @@ -29,7 +29,6 @@ import javax.annotation.Nonnull; -import com.amazonaws.services.s3.model.AmazonS3Exception; import org.apache.commons.lang.StringUtils; import org.kohsuke.stapler.DataBoundSetter; import org.kohsuke.stapler.QueryParameter; @@ -47,10 +46,11 @@ import hudson.Util; import hudson.model.Failure; import hudson.util.FormValidation; -import io.jenkins.plugins.artifact_manager_jclouds.JCloudsVirtualFile; import io.jenkins.plugins.aws.global_configuration.AbstractAwsGlobalConfiguration; import io.jenkins.plugins.aws.global_configuration.CredentialsAwsGlobalConfiguration; import jenkins.model.Jenkins; +import org.jclouds.blobstore.BlobStores; +import org.jclouds.blobstore.options.ListContainerOptions; import org.jenkinsci.Symbol; /** @@ -243,8 +243,7 @@ public FormValidation doValidateS3BucketConfig(@QueryParameter String container, FormValidation ret = FormValidation.ok("success"); try { S3BlobStore provider = new S3BlobStoreTester(container, prefix); - JCloudsVirtualFile jc = new JCloudsVirtualFile(provider, container, prefix); - jc.list(); + BlobStores.listAll(provider.getContext().getBlobStore(), getContainer(), ListContainerOptions.Builder.prefix(prefix + "/")); } catch (Throwable t){ String msg = processExceptionMessage(t); ret = FormValidation.error(StringUtils.abbreviate(msg, 200)); diff --git a/src/main/resources/io/jenkins/plugins/artifact_manager_jclouds/JCloudsArtifactManagerFactory/config.jelly b/src/main/resources/io/jenkins/plugins/artifact_manager_jclouds/JCloudsArtifactManagerFactory/config.jelly deleted file mode 100644 index 0d6f1ea3..00000000 --- a/src/main/resources/io/jenkins/plugins/artifact_manager_jclouds/JCloudsArtifactManagerFactory/config.jelly +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - - diff --git a/src/test/java/io/jenkins/plugins/artifact_manager_jclouds/MockApiMetadata.java b/src/test/java/io/jenkins/plugins/artifact_manager_jclouds/MockApiMetadata.java deleted file mode 100644 index 08f2b1bc..00000000 --- a/src/test/java/io/jenkins/plugins/artifact_manager_jclouds/MockApiMetadata.java +++ /dev/null @@ -1,257 +0,0 @@ -/* - * The MIT License - * - * Copyright 2018 CloudBees, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -package io.jenkins.plugins.artifact_manager_jclouds; - -import com.google.inject.AbstractModule; -import java.io.IOException; -import java.net.URI; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.logging.Logger; -import org.apache.commons.io.IOUtils; -import org.jclouds.apis.ApiMetadata; -import org.jclouds.apis.internal.BaseApiMetadata; -import org.jclouds.blobstore.BlobRequestSigner; -import org.jclouds.blobstore.BlobStore; -import org.jclouds.blobstore.BlobStoreContext; -import org.jclouds.blobstore.LocalBlobRequestSigner; -import org.jclouds.blobstore.LocalStorageStrategy; -import org.jclouds.blobstore.TransientApiMetadata; -import org.jclouds.blobstore.TransientStorageStrategy; -import org.jclouds.blobstore.attr.ConsistencyModel; -import org.jclouds.blobstore.config.BlobStoreObjectModule; -import org.jclouds.blobstore.config.LocalBlobStore; -import org.jclouds.blobstore.config.TransientBlobStoreContextModule; -import org.jclouds.blobstore.domain.Blob; -import org.jclouds.blobstore.domain.BlobAccess; -import org.jclouds.blobstore.domain.ContainerAccess; -import org.jclouds.blobstore.domain.StorageMetadata; -import org.jclouds.blobstore.options.CreateContainerOptions; -import org.jclouds.blobstore.options.ListContainerOptions; -import org.jclouds.domain.Location; -import org.jclouds.io.Payloads; -import org.kohsuke.MetaInfServices; - -/** - * A mock provider allowing control of all operations. - * Otherwise akin to a simplified version of {@link TransientApiMetadata}. - * Whereas the stock {@code transient} provider would merely implement the full SPI, - * we also want to allow particular metadata operations to fail or block at test-specified times. - */ -@MetaInfServices(ApiMetadata.class) -public final class MockApiMetadata extends BaseApiMetadata { - - private static final Logger LOGGER = Logger.getLogger(MockApiMetadata.class.getName()); - - public MockApiMetadata() { - this(new Builder()); - } - - private MockApiMetadata(Builder builder) { - super(builder); - } - - @Override - public Builder toBuilder() { - return new Builder().fromApiMetadata(this); - } - - private static final class Builder extends BaseApiMetadata.Builder { - - Builder() { - id("mock"); - name("mock"); - identityName("mock"); - documentation(URI.create("about:nothing")); - defaultIdentity("nobody"); - defaultCredential("anon"); - defaultEndpoint("http://nowhere.net/"); - view(BlobStoreContext.class); - defaultModule(MockModule.class); - } - - @Override - protected Builder self() { - return this; - } - - @Override - public ApiMetadata build() { - return new MockApiMetadata(this); - } - - } - - /** Like {@link TransientBlobStoreContextModule}. */ - public static final class MockModule extends AbstractModule { - - @Override - protected void configure() { - install(new BlobStoreObjectModule()); - bind(BlobStore.class).to(LocalBlobStore.class); - bind(ConsistencyModel.class).toInstance(ConsistencyModel.STRICT); - bind(LocalStorageStrategy.class).to(MockStrategy.class); - bind(BlobRequestSigner.class).to(LocalBlobRequestSigner.class); - } - - } - - @FunctionalInterface - interface GetBlobKeysInsideContainerHandler { - Iterable run() throws IOException; - } - - private static final Map getBlobKeysInsideContainerHandlers = new ConcurrentHashMap<>(); - - static void handleGetBlobKeysInsideContainer(String container, GetBlobKeysInsideContainerHandler handler) { - getBlobKeysInsideContainerHandlers.put(container, handler); - } - - private static final Map removeBlobHandlers = new ConcurrentHashMap<>(); - - static void handleRemoveBlob(String container, String key, Runnable handler) { - removeBlobHandlers.put(container + '/' + key, handler); - } - - /** Like {@link TransientStorageStrategy}. */ - public static final class MockStrategy implements LocalStorageStrategy { - - private final Map> blobsByContainer = new HashMap<>(); - - @Override - public boolean containerExists(String container) { - return blobsByContainer.containsKey(container); - } - - @Override - public Collection getAllContainerNames() { - return blobsByContainer.keySet(); - } - - @Override - public boolean createContainerInLocation(String container, Location location, CreateContainerOptions options) { - return blobsByContainer.putIfAbsent(container, new HashMap<>()) == null; - } - - @Override - public ContainerAccess getContainerAccess(String container) { - throw new UnsupportedOperationException(); // TODO - } - - @Override - public void setContainerAccess(String container, ContainerAccess access) { - throw new UnsupportedOperationException(); // TODO - } - - @Override - public void deleteContainer(String container) { - blobsByContainer.remove(container); - } - - @Override - public void clearContainer(String container) { - blobsByContainer.get(container).clear(); - } - - @Override - public void clearContainer(String container, ListContainerOptions options) { - throw new UnsupportedOperationException(); // TODO - } - - @Override - public StorageMetadata getContainerMetadata(String container) { - throw new UnsupportedOperationException(); // TODO - } - - @Override - public boolean blobExists(String container, String key) { - return blobsByContainer.get(container).containsKey(key); - } - - @Override - public Iterable getBlobKeysInsideContainer(String container) throws IOException { - GetBlobKeysInsideContainerHandler handler = getBlobKeysInsideContainerHandlers.remove(container); - if (handler != null) { - return handler.run(); - } - return blobsByContainer.get(container).keySet(); - } - - @Override - public Blob getBlob(String containerName, String blobName) { - Blob blob = blobsByContainer.get(containerName).get(blobName); - assert containerName.equals(blob.getMetadata().getContainer()) : blob; - return blob; - } - - @Override - public String putBlob(String containerName, Blob blob) throws IOException { - { - // When called from LocalBlobStore.copyBlob, there is no container, and it uses an InputStreamPayload which cannot be reused. - // TransientStorageStrategy has an elaborate createUpdatedCopyOfBlobInContainer here, but these two fixups seem to suffice. - blob.getMetadata().setContainer(containerName); - byte[] data = IOUtils.toByteArray(blob.getPayload().openStream()); - blob.getMetadata().setSize((long) data.length); - blob.setPayload(Payloads.newByteArrayPayload(data)); - } - blobsByContainer.get(containerName).put(blob.getMetadata().getName(), blob); - return null; - } - - @Override - public void removeBlob(String container, String key) { - Runnable handler = removeBlobHandlers.remove(container + '/' + key); - if (handler != null) { - handler.run(); - return; - } - blobsByContainer.get(container).remove(key); - } - - @Override - public BlobAccess getBlobAccess(String container, String key) { - return BlobAccess.PRIVATE; - } - - @Override - public void setBlobAccess(String container, String key, BlobAccess access) { - // ignore - } - - @Override - public Location getLocation(String containerName) { - return null; - } - - @Override - public String getSeparator() { - return "/"; - } - - } - -} diff --git a/src/test/java/io/jenkins/plugins/artifact_manager_jclouds/MockApiMetadataTest.java b/src/test/java/io/jenkins/plugins/artifact_manager_jclouds/MockApiMetadataTest.java deleted file mode 100644 index 945c67f4..00000000 --- a/src/test/java/io/jenkins/plugins/artifact_manager_jclouds/MockApiMetadataTest.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * The MIT License - * - * Copyright 2018 CloudBees, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -package io.jenkins.plugins.artifact_manager_jclouds; - -import java.util.stream.Collectors; -import org.jclouds.ContextBuilder; -import org.jclouds.blobstore.BlobStore; -import org.jclouds.blobstore.BlobStoreContext; -import org.jclouds.blobstore.domain.Blob; -import org.jclouds.blobstore.domain.StorageMetadata; -import static org.junit.Assert.*; -import org.junit.Test; - -public class MockApiMetadataTest { - - @Test - public void smokes() throws Exception { - BlobStoreContext bsc = ContextBuilder.newBuilder("mock").buildView(BlobStoreContext.class); - BlobStore bs = bsc.getBlobStore(); - bs.createContainerInLocation(null, "container"); - Blob blob = bs.blobBuilder("file.txt").payload("content").build(); - bs.putBlob("container", blob); - assertEquals("file.txt", bs.list("container").stream().map(StorageMetadata::getName).collect(Collectors.joining(":"))); - } - -} diff --git a/src/test/java/io/jenkins/plugins/artifact_manager_jclouds/MockBlobStore.java b/src/test/java/io/jenkins/plugins/artifact_manager_jclouds/MockBlobStore.java deleted file mode 100644 index 6a67f9bc..00000000 --- a/src/test/java/io/jenkins/plugins/artifact_manager_jclouds/MockBlobStore.java +++ /dev/null @@ -1,169 +0,0 @@ -/* - * The MIT License - * - * Copyright 2018 CloudBees, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -package io.jenkins.plugins.artifact_manager_jclouds; - -import java.io.IOException; -import java.net.URI; -import java.net.URL; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import org.apache.commons.io.IOUtils; -import org.apache.http.ConnectionClosedException; -import org.apache.http.HttpEntity; -import org.apache.http.HttpEntityEnclosingRequest; -import org.apache.http.HttpRequest; -import org.apache.http.HttpResponse; -import org.apache.http.entity.ByteArrayEntity; -import org.apache.http.impl.bootstrap.HttpServer; -import org.apache.http.impl.bootstrap.ServerBootstrap; -import org.apache.http.protocol.HttpContext; -import org.apache.http.protocol.HttpRequestHandler; -import org.jclouds.ContextBuilder; -import org.jclouds.blobstore.BlobStore; -import org.jclouds.blobstore.BlobStoreContext; -import org.jclouds.blobstore.domain.Blob; -import org.jclouds.blobstore.domain.StorageMetadata; - -/** - * A mock storage provider which keeps all blobs in memory. - * Presigned “external” URLs are supported. - * Allows tests to inject failures such as HTTP errors or hangs. - */ -public final class MockBlobStore extends BlobStoreProvider { - - private static final Logger LOGGER = Logger.getLogger(MockBlobStore.class.getName()); - - private transient BlobStoreContext context; - private transient URL baseURL; - - @Override - public String getPrefix() { - return ""; - } - - @Override - public String getContainer() { - return "container"; - } - - private static final Map specialHandlers = new ConcurrentHashMap<>(); - - /** - * Requests that the next HTTP access to a particular presigned URL should behave specially. - * @param method upload or download - * @param key the blob’s {@link StorageMetadata#getName} - * @param handler what to do instead - */ - static void speciallyHandle(HttpMethod method, String key, HttpRequestHandler handler) { - specialHandlers.put(method + ":" + key, handler); - } - - @Override - public synchronized BlobStoreContext getContext() throws IOException { - if (context == null) { - context = ContextBuilder.newBuilder("mock").buildView(BlobStoreContext.class); - HttpServer server = ServerBootstrap.bootstrap(). - registerHandler("*", (HttpRequest request, HttpResponse response, HttpContext _context) -> { - String method = request.getRequestLine().getMethod(); - Matcher m = Pattern.compile("/([^/]+)/(.+)[?]method=" + method).matcher(request.getRequestLine().getUri()); - if (!m.matches()) { - throw new IllegalStateException(); - } - String container = m.group(1); - String key = m.group(2); - HttpRequestHandler specialHandler = specialHandlers.remove(method + ":" + key); - if (specialHandler != null) { - specialHandler.handle(request, response, _context); - return; - } - BlobStore blobStore = context.getBlobStore(); - switch (method) { - case "GET": { - Blob blob = blobStore.getBlob(container, key); - if (blob == null) { - response.setStatusCode(404); - return; - } - byte[] data = IOUtils.toByteArray(blob.getPayload().openStream()); - response.setStatusCode(200); - response.setEntity(new ByteArrayEntity(data)); - LOGGER.log(Level.INFO, "Serving {0} bytes from {1}:{2}", new Object[] {data.length, container, key}); - return; - } case "PUT": { - HttpEntity entity = ((HttpEntityEnclosingRequest) request).getEntity(); - byte[] data = IOUtils.toByteArray(entity.getContent()); - Blob blob = blobStore.blobBuilder(key).payload(data).build(); - if (!blobStore.containerExists(container)) { - blobStore.createContainerInLocation(null, container); - } - blobStore.putBlob(container, blob); - response.setStatusCode(204); - LOGGER.log(Level.INFO, "Uploaded {0} bytes to {1}:{2}", new Object[] {data.length, container, key}); - return; - } default: { - throw new IllegalStateException(); - } - } - }). - setExceptionLogger(x -> { - if (x instanceof ConnectionClosedException) { - LOGGER.info(x.toString()); - } else { - LOGGER.log(Level.INFO, "error thrown in HTTP service", x); - } - }). - create(); - server.start(); - baseURL = new URL("http://" + server.getInetAddress().getHostName() + ":" + server.getLocalPort() + "/"); - LOGGER.log(Level.INFO, "Mock server running at {0}", baseURL); - } - return context; - } - - @Override - public URI toURI(String container, String key) { - return URI.create("mock://" + container + "/" + key); - } - - @Override - public URL toExternalURL(Blob blob, HttpMethod httpMethod) throws IOException { - return new URL(baseURL, blob.getMetadata().getContainer() + "/" + blob.getMetadata().getName() + "?method=" + httpMethod); - } - - @Override - public boolean isDeleteArtifacts() { - return true; - } - - @Override - public boolean isDeleteStashes() { - return true; - } - -} diff --git a/src/test/java/io/jenkins/plugins/artifact_manager_jclouds/MockBlobStoreTest.java b/src/test/java/io/jenkins/plugins/artifact_manager_jclouds/MockBlobStoreTest.java deleted file mode 100644 index 29de0fbe..00000000 --- a/src/test/java/io/jenkins/plugins/artifact_manager_jclouds/MockBlobStoreTest.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * The MIT License - * - * Copyright 2018 CloudBees, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -package io.jenkins.plugins.artifact_manager_jclouds; - -import org.jenkinsci.plugins.workflow.ArtifactManagerTest; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.jvnet.hudson.test.BuildWatcher; -import org.jvnet.hudson.test.JenkinsRule; - -public class MockBlobStoreTest { - - @ClassRule - public static BuildWatcher buildWatcher = new BuildWatcher(); - - @Rule - public JenkinsRule j = new JenkinsRule(); - - @Test - public void smokes() throws Exception { - ArtifactManagerTest.artifactArchiveAndDelete(j, new JCloudsArtifactManagerFactory(new MockBlobStore()), false, null); - ArtifactManagerTest.artifactStashAndDelete(j, new JCloudsArtifactManagerFactory(new MockBlobStore()), false, null); - } - -} diff --git a/src/test/java/io/jenkins/plugins/artifact_manager_jclouds/NetworkTest.java b/src/test/java/io/jenkins/plugins/artifact_manager_jclouds/NetworkTest.java deleted file mode 100644 index ca90f7da..00000000 --- a/src/test/java/io/jenkins/plugins/artifact_manager_jclouds/NetworkTest.java +++ /dev/null @@ -1,403 +0,0 @@ -/* - * The MIT License - * - * Copyright 2018 CloudBees, Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -package io.jenkins.plugins.artifact_manager_jclouds; - -import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException; -import com.google.common.base.Throwables; -import hudson.model.Result; -import hudson.model.Run; -import hudson.tasks.LogRotator; -import io.jenkins.plugins.httpclient.RobustHTTPClient; -import java.util.Objects; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.LogRecord; -import java.util.stream.Collectors; -import jenkins.model.ArtifactManagerConfiguration; -import org.apache.http.ConnectionClosedException; -import org.apache.http.HttpVersion; -import org.apache.http.entity.StringEntity; -import org.apache.http.message.BasicStatusLine; -import static org.hamcrest.Matchers.*; -import org.jclouds.blobstore.ContainerNotFoundException; -import org.jenkinsci.plugins.workflow.cps.CpsFlowDefinition; -import org.jenkinsci.plugins.workflow.job.WorkflowJob; -import org.jenkinsci.plugins.workflow.job.WorkflowRun; -import org.jenkinsci.plugins.workflow.steps.TimeoutStepExecution; -import static org.junit.Assert.*; -import org.junit.Before; -import org.junit.ClassRule; -import org.junit.Rule; -import org.junit.Test; -import org.jvnet.hudson.test.BuildWatcher; -import org.jvnet.hudson.test.Issue; -import org.jvnet.hudson.test.JenkinsRule; -import org.jvnet.hudson.test.LoggerRule; - -/** - * Explores responses to edge cases such as server errors and hangs. - * We do mocking at the jclouds level, rather than using (say) S3Mock, because: - *
    - *
  • We are interested here in the behavior of the generic Jenkins integration code in this package, - * as well as its dependencies in Jenkins core and Pipeline, not that of a particular blob store. - *
  • S3-specific failure modes are of interest but are generally hard to simulate using those mock frameworks anyway. - * For example, some S3 mock frameworks completely ignore authentication. - * Conversely, some failure modes we want to test may not be supported by an S3 mock framework. - *
  • S3 mock frameworks are not written to expect the jclouds abstraction layer, and vice-versa. - * The jclouds {@code AWSS3ProviderMetadata} cannot accept a custom {@code AmazonS3} which {@code S3MockRule} would supply; - * it reimplements the S3 REST API from scratch, not using the AWS SDK. - * It would be necessary to run Dockerized mocks with local HTTP ports. - *
- */ -@Issue("JENKINS-50597") -public class NetworkTest { - - @ClassRule - public static BuildWatcher buildWatcher = new BuildWatcher(); - - @Rule - public JenkinsRule r = new JenkinsRule(); - - @Rule - public LoggerRule loggerRule = new LoggerRule(); - - @Before - public void configureManager() throws Exception { - MockBlobStore mockBlobStore = new MockBlobStore(); - mockBlobStore.getContext().getBlobStore().createContainerInLocation(null, mockBlobStore.getContainer()); - ArtifactManagerConfiguration.get().getArtifactManagerFactories().add(new JCloudsArtifactManagerFactory(mockBlobStore)); - } - - @Before - public void createAgent() throws Exception { - r.createSlave("remote", null, null); - } - - @Test - public void unrecoverableErrorArchiving() throws Exception { - WorkflowJob p = r.createProject(WorkflowJob.class, "p"); - failIn(BlobStoreProvider.HttpMethod.PUT, "p/1/artifacts/f", 403, 0); - p.setDefinition(new CpsFlowDefinition("node('remote') {writeFile file: 'f', text: '.'; archiveArtifacts 'f'}", true)); - WorkflowRun b = r.assertBuildStatus(Result.FAILURE, p.scheduleBuild2(0)); - r.assertLogContains("ERROR: Failed to upload", b); - r.assertLogContains("/container/p/1/artifacts/f?…, response: 403 simulated 403 failure, body: Detailed explanation of 403.", b); - r.assertLogNotContains("Retrying upload", b); - r.assertLogNotContains("\tat hudson.tasks.ArtifactArchiver.perform", b); - } - - @Test - public void recoverableErrorArchiving() throws Exception { - WorkflowJob p = r.createProject(WorkflowJob.class, "p"); - failIn(BlobStoreProvider.HttpMethod.PUT, "p/1/artifacts/f", 500, 0); - p.setDefinition(new CpsFlowDefinition("node('remote') {writeFile file: 'f', text: '.'; archiveArtifacts 'f'}", true)); - WorkflowRun b = r.buildAndAssertSuccess(p); - r.assertLogContains("/container/p/1/artifacts/f?…, response: 500 simulated 500 failure, body: Detailed explanation of 500.", b); - r.assertLogContains("Retrying upload", b); - r.assertLogNotContains("\tat hudson.tasks.ArtifactArchiver.perform", b); - } - - @Test - public void networkExceptionArchiving() throws Exception { - WorkflowJob p = r.createProject(WorkflowJob.class, "p"); - failIn(BlobStoreProvider.HttpMethod.PUT, "p/1/artifacts/f", 0, 0); - p.setDefinition(new CpsFlowDefinition("node('remote') {writeFile file: 'f', text: '.'; archiveArtifacts 'f'}", true)); - WorkflowRun b = r.buildAndAssertSuccess(p); - // currently prints a ‘java.net.SocketException: Connection reset’ but not sure if we really care - r.assertLogContains("Retrying upload", b); - r.assertLogNotContains("\tat hudson.tasks.ArtifactArchiver.perform", b); - } - - @Test - public void repeatedRecoverableErrorArchiving() throws Exception { - WorkflowJob p = r.createProject(WorkflowJob.class, "p"); - JCloudsArtifactManager.client = new RobustHTTPClient(); - JCloudsArtifactManager.client.setStopAfterAttemptNumber(3); - try { - failIn(BlobStoreProvider.HttpMethod.PUT, "p/1/artifacts/f", 500, 3); - p.setDefinition(new CpsFlowDefinition("node('remote') {writeFile file: 'f', text: '.'; archiveArtifacts 'f'}", true)); - WorkflowRun b = r.assertBuildStatus(Result.FAILURE, p.scheduleBuild2(0)); - r.assertLogContains("ERROR: Failed to upload", b); - r.assertLogContains("/container/p/1/artifacts/f?…, response: 500 simulated 500 failure, body: Detailed explanation of 500.", b); - r.assertLogContains("Retrying upload", b); - r.assertLogNotContains("\tat hudson.tasks.ArtifactArchiver.perform", b); - } finally { - JCloudsArtifactManager.client = new RobustHTTPClient(); - } - } - - @Test - public void hangArchiving() throws Exception { - WorkflowJob p = r.createProject(WorkflowJob.class, "p"); - JCloudsArtifactManager.client = new RobustHTTPClient(); - JCloudsArtifactManager.client.setTimeout(5, TimeUnit.SECONDS); - try { - hangIn(BlobStoreProvider.HttpMethod.PUT, "p/1/artifacts/f"); - p.setDefinition(new CpsFlowDefinition("node('remote') {writeFile file: 'f', text: '.'; archiveArtifacts 'f'}", true)); - WorkflowRun b = r.buildAndAssertSuccess(p); - r.assertLogContains("Retrying upload", b); - r.assertLogNotContains("\tat hudson.tasks.ArtifactArchiver.perform", b); - // Also from master: - hangIn(BlobStoreProvider.HttpMethod.PUT, "p/2/artifacts/f"); - p.setDefinition(new CpsFlowDefinition("node('master') {writeFile file: 'f', text: '.'; archiveArtifacts 'f'}", true)); - b = r.buildAndAssertSuccess(p); - r.assertLogContains("Retrying upload", b); - r.assertLogNotContains("\tat hudson.tasks.ArtifactArchiver.perform", b); - } finally { - JCloudsArtifactManager.client = new RobustHTTPClient(); - } - } - - @Test - public void interruptedArchiving() throws Exception { - WorkflowJob p = r.createProject(WorkflowJob.class, "p"); - hangIn(BlobStoreProvider.HttpMethod.PUT, "p/1/artifacts/f"); - p.setDefinition(new CpsFlowDefinition("node('remote') {writeFile file: 'f', text: '.'; archiveArtifacts 'f'}", true)); - WorkflowRun b = p.scheduleBuild2(0).waitForStart(); - r.waitForMessage("Archiving artifacts", b); - Thread.sleep(2000); // wait for hangIn to sleep; OK if occasionally it has not gotten there yet, we still expect the same result - b.getExecutor().interrupt(); - r.assertBuildStatus(Result.ABORTED, r.waitForCompletion(b)); - // Currently prints a stack trace of java.lang.InterruptedException; good enough. - // Check the same from a timeout within the build, rather than a user abort, and also from master just for fun: - hangIn(BlobStoreProvider.HttpMethod.PUT, "p/2/artifacts/f"); - p.setDefinition(new CpsFlowDefinition("node('master') {writeFile file: 'f', text: '.'; timeout(time: 3, unit: 'SECONDS') {archiveArtifacts 'f'}}", true)); - r.assertLogContains(new TimeoutStepExecution.ExceededTimeout().getShortDescription(), r.assertBuildStatus(Result.ABORTED, p.scheduleBuild2(0))); - } - - @Test - public void unrecoverableErrorUnstashing() throws Exception { - WorkflowJob p = r.createProject(WorkflowJob.class, "p"); - failIn(BlobStoreProvider.HttpMethod.GET, "p/1/stashes/f.tgz", 403, 0); - p.setDefinition(new CpsFlowDefinition("node('remote') {writeFile file: 'f', text: '.'; stash 'f'; unstash 'f'}", true)); - WorkflowRun b = r.assertBuildStatus(Result.FAILURE, p.scheduleBuild2(0)); - r.assertLogContains("ERROR: Failed to download", b); - r.assertLogContains("/container/p/1/stashes/f.tgz?…", b); - r.assertLogContains("response: 403 simulated 403 failure, body: Detailed explanation of 403.", b); - r.assertLogNotContains("Retrying download", b); - r.assertLogNotContains("\tat org.jenkinsci.plugins.workflow.flow.StashManager.unstash", b); - } - - @Test - public void recoverableErrorUnstashing() throws Exception { - WorkflowJob p = r.createProject(WorkflowJob.class, "p"); - failIn(BlobStoreProvider.HttpMethod.GET, "p/1/stashes/f.tgz", 500, 0); - p.setDefinition(new CpsFlowDefinition("node('remote') {writeFile file: 'f', text: '.'; stash 'f'; unstash 'f'}", true)); - WorkflowRun b = r.buildAndAssertSuccess(p); - r.assertLogContains("/container/p/1/stashes/f.tgz?…", b); - r.assertLogContains("response: 500 simulated 500 failure, body: Detailed explanation of 500.", b); - r.assertLogContains("Retrying download", b); - r.assertLogNotContains("\tat org.jenkinsci.plugins.workflow.flow.StashManager.unstash", b); - } - - @Test - public void networkExceptionUnstashing() throws Exception { - WorkflowJob p = r.createProject(WorkflowJob.class, "p"); - // failIn does not work: URL connection gets a 200 status despite a ConnectionClosedException being thrown; a new connection is made. - MockBlobStore.speciallyHandle(BlobStoreProvider.HttpMethod.GET, "p/1/stashes/f.tgz", (request, response, context) -> {}); - p.setDefinition(new CpsFlowDefinition("node('remote') {writeFile file: 'f', text: '.'; stash 'f'; unstash 'f'}", true)); - WorkflowRun b = r.buildAndAssertSuccess(p); - r.assertLogContains("Retrying download", b); - // Currently catches an error from FilePath.untarFrom: java.io.IOException: Failed to extract input stream - r.assertLogNotContains("\tat org.jenkinsci.plugins.workflow.flow.StashManager.unstash", b); - } - - @Test - public void repeatedRecoverableErrorUnstashing() throws Exception { - WorkflowJob p = r.createProject(WorkflowJob.class, "p"); - JCloudsArtifactManager.client = new RobustHTTPClient(); - JCloudsArtifactManager.client.setStopAfterAttemptNumber(3); - try { - failIn(BlobStoreProvider.HttpMethod.GET, "p/1/stashes/f.tgz", 500, 3); - p.setDefinition(new CpsFlowDefinition("node('remote') {writeFile file: 'f', text: '.'; stash 'f'; unstash 'f'}", true)); - WorkflowRun b = r.assertBuildStatus(Result.FAILURE, p.scheduleBuild2(0)); - r.assertLogContains("ERROR: Failed to download", b); - r.assertLogContains("/container/p/1/stashes/f.tgz?…", b); - r.assertLogContains("response: 500 simulated 500 failure, body: Detailed explanation of 500.", b); - r.assertLogContains("Retrying download", b); - r.assertLogNotContains("\tat org.jenkinsci.plugins.workflow.flow.StashManager.unstash", b); - } finally { - JCloudsArtifactManager.client = new RobustHTTPClient(); - } - } - - @Test - public void hangUnstashing() throws Exception { - WorkflowJob p = r.createProject(WorkflowJob.class, "p"); - JCloudsArtifactManager.client = new RobustHTTPClient(); - JCloudsArtifactManager.client.setTimeout(5, TimeUnit.SECONDS); - try { - hangIn(BlobStoreProvider.HttpMethod.GET, "p/1/stashes/f.tgz"); - p.setDefinition(new CpsFlowDefinition("node('remote') {writeFile file: 'f', text: '.'; stash 'f'; unstash 'f'}", true)); - WorkflowRun b = r.buildAndAssertSuccess(p); - r.assertLogContains("Retrying download", b); - r.assertLogNotContains("\tat org.jenkinsci.plugins.workflow.flow.StashManager.unstash", b); - hangIn(BlobStoreProvider.HttpMethod.GET, "p/2/stashes/f.tgz"); - p.setDefinition(new CpsFlowDefinition("node('master') {writeFile file: 'f', text: '.'; stash 'f'; unstash 'f'}", true)); - b = r.buildAndAssertSuccess(p); - r.assertLogContains("Retrying download", b); - r.assertLogNotContains("\tat org.jenkinsci.plugins.workflow.flow.StashManager.unstash", b); - } finally { - JCloudsArtifactManager.client = new RobustHTTPClient(); - } - } - - @Test - public void interruptedUnstashing() throws Exception { - WorkflowJob p = r.createProject(WorkflowJob.class, "p"); - hangIn(BlobStoreProvider.HttpMethod.GET, "p/1/stashes/f.tgz"); - p.setDefinition(new CpsFlowDefinition("node('remote') {writeFile file: 'f', text: '.'; stash 'f'; unstash 'f'}", true)); - WorkflowRun b = p.scheduleBuild2(0).waitForStart(); - r.waitForMessage("[Pipeline] unstash", b); - Thread.sleep(2000); - b.getExecutor().interrupt(); - r.assertBuildStatus(Result.ABORTED, r.waitForCompletion(b)); - hangIn(BlobStoreProvider.HttpMethod.GET, "p/2/stashes/f.tgz"); - p.setDefinition(new CpsFlowDefinition("node('master') {writeFile file: 'f', text: '.'; stash 'f'; timeout(time: 3, unit: 'SECONDS') {unstash 'f'}}", true)); - r.assertLogContains(new TimeoutStepExecution.ExceededTimeout().getShortDescription(), r.assertBuildStatus(Result.ABORTED, p.scheduleBuild2(0))); - } - - @Test - public void recoverableErrorUnarchiving() throws Exception { - WorkflowJob p = r.createProject(WorkflowJob.class, "p"); - failIn(BlobStoreProvider.HttpMethod.GET, "p/1/artifacts/f", 500, 0); - p.setDefinition(new CpsFlowDefinition("node('remote') {writeFile file: 'f', text: '.'; archiveArtifacts 'f'; unarchive mapping: ['f': 'f']}", true)); - WorkflowRun b = r.buildAndAssertSuccess(p); - r.assertLogContains("/container/p/1/artifacts/f?…", b); - r.assertLogContains("response: 500 simulated 500 failure, body: Detailed explanation of 500.", b); - r.assertLogContains("Retrying download", b); - r.assertLogNotContains("\tat org.jenkinsci.plugins.workflow.steps.ArtifactUnarchiverStepExecution.run", b); - } - - // TBD if jclouds, or its S3 provider, is capable of differentiating recoverable from nonrecoverable errors. The error simulated here: - // org.jclouds.blobstore.ContainerNotFoundException: nonexistent.s3.amazonaws.com not found: The specified bucket does not exist - // at org.jclouds.s3.handlers.ParseS3ErrorFromXmlContent.refineException(ParseS3ErrorFromXmlContent.java:81) - // at org.jclouds.aws.handlers.ParseAWSErrorFromXmlContent.handleError(ParseAWSErrorFromXmlContent.java:89) - // at … - // Disconnecting network in the middle of some operations produces a bunch of warnings from BackoffLimitedRetryHandler.ifReplayableBackoffAndReturnTrue - // followed by a org.jclouds.http.HttpResponseException: Network is unreachable (connect failed) connecting to GET … - // Also not testing hangs here since org.jclouds.Constants.PROPERTY_SO_TIMEOUT/PROPERTY_CONNECTION_TIMEOUT probably handle this. - @Test - public void errorListing() throws Exception { - WorkflowJob p = r.createProject(WorkflowJob.class, "p"); - MockApiMetadata.handleGetBlobKeysInsideContainer("container", () -> {throw new ContainerNotFoundException("container", "sorry");}); - p.setDefinition(new CpsFlowDefinition("node('remote') {writeFile file: 'f', text: '.'; archiveArtifacts 'f'; unarchive mapping: ['f': 'f']}", true)); - WorkflowRun b = r.assertBuildStatus(Result.FAILURE, p.scheduleBuild2(0)); - r.assertLogContains(ContainerNotFoundException.class.getName(), b); - // Currently prints a stack trace, OK. - } - - // Interrupts during a network operation seem to have no effect; when retrying during network disconnection, - // BackoffLimitedRetryHandler.imposeBackoffExponentialDelay throws InterruptedException wrapped in RuntimeException. - @Test - public void interruptedListing() throws Exception { - WorkflowJob p = r.createProject(WorkflowJob.class, "p"); - MockApiMetadata.handleGetBlobKeysInsideContainer("container", () -> { - try { - Thread.sleep(Long.MAX_VALUE); - return null; // to satisfy compiler - } catch (InterruptedException x) { - throw new RuntimeException(x); - } - }); - p.setDefinition(new CpsFlowDefinition("node('remote') {writeFile file: 'f', text: '.'; archiveArtifacts 'f'; timeout(time: 3, unit: 'SECONDS') {unarchive mapping: ['f': 'f']}}", true)); - r.assertLogContains(new TimeoutStepExecution.ExceededTimeout().getShortDescription(), r.assertBuildStatus(Result.ABORTED, p.scheduleBuild2(0))); - } - - @Test - public void errorCleaningArtifacts() throws Exception { - loggerRule.record(WorkflowRun.class, Level.WARNING).capture(10); - WorkflowJob p = r.createProject(WorkflowJob.class, "p"); - p.setDefinition(new CpsFlowDefinition("node('remote') {writeFile file: 'f', text: '.'; archiveArtifacts 'f'}", true)); - r.buildAndAssertSuccess(p); - p.setBuildDiscarder(new LogRotator(-1, -1, -1, 0)); - MockApiMetadata.handleRemoveBlob("container", "p/1/artifacts/f", () -> {throw new ContainerNotFoundException("container", "sorry about your artifacts");}); - r.buildAndAssertSuccess(p); - assertThat(loggerRule.getRecords().stream().map(LogRecord::getThrown).filter(Objects::nonNull).map(Throwables::getRootCause).map(Throwable::getMessage).collect(Collectors.toSet()), - containsInAnyOrder("container not found: sorry about your artifacts")); - } - - @Test - public void errorCleaningStashes() throws Exception { - loggerRule.record(WorkflowRun.class, Level.WARNING).capture(10); - WorkflowJob p = r.createProject(WorkflowJob.class, "p"); - p.setDefinition(new CpsFlowDefinition("node('remote') {writeFile file: 'f', text: '.'; stash 'stuff'}", true)); - MockApiMetadata.handleRemoveBlob("container", "p/1/stashes/stuff.tgz", () -> {throw new ContainerNotFoundException("container", "sorry about your stashes");}); - r.buildAndAssertSuccess(p); - assertThat(loggerRule.getRecords().stream().map(LogRecord::getThrown).filter(Objects::nonNull).map(Throwables::getRootCause).map(Throwable::getMessage).collect(Collectors.toSet()), - containsInAnyOrder("container not found: sorry about your stashes")); - } - - // Interrupts probably never delivered during HTTP requests (maybe depends on servlet container?). - // Hangs would be handled by jclouds code. - @Test - public void errorBrowsing() throws Exception { - WorkflowJob p = r.createProject(WorkflowJob.class, "p"); - p.setDefinition(new CpsFlowDefinition("node('remote') {writeFile file: 'f', text: '.'; archiveArtifacts 'f'}", true)); - WorkflowRun b = r.buildAndAssertSuccess(p); - MockApiMetadata.handleGetBlobKeysInsideContainer("container", () -> {throw new ContainerNotFoundException("container", "sorry");}); - JenkinsRule.WebClient wc = r.createWebClient(); - { - System.err.println("build root"); - loggerRule.record(Run.class, Level.WARNING).capture(10); - wc.getPage(b); - assertThat(loggerRule.getRecords().stream().map(LogRecord::getThrown).filter(Objects::nonNull).map(Throwables::getRootCause).map(Throwable::getMessage).collect(Collectors.toSet()), - containsInAnyOrder("container not found: sorry")); - } - { - System.err.println("artifact root"); - MockApiMetadata.handleGetBlobKeysInsideContainer("container", () -> {throw new ContainerNotFoundException("container", "sorry");}); - try { - wc.getPage(b, "artifact/"); - fail("Currently DirectoryBrowserSupport throws up storage exceptions."); - } catch (FailingHttpStatusCodeException x) { - assertEquals(500, x.getStatusCode()); - assertThat(x.getResponse().getContentAsString(), containsString("container not found: sorry")); - } - } - } - - private static void failIn(BlobStoreProvider.HttpMethod method, String key, int code, int repeats) { - MockBlobStore.speciallyHandle(method, key, (request, response, context) -> { - if (repeats > 0) { - failIn(method, key, code, repeats - 1); - } - if (code == 0) { - throw new ConnectionClosedException("Refusing to even send a status code for " + key); - } - response.setStatusLine(new BasicStatusLine(HttpVersion.HTTP_1_0, code, "simulated " + code + " failure")); - response.setEntity(new StringEntity("Detailed explanation of " + code + ".")); - }); - } - - private static void hangIn(BlobStoreProvider.HttpMethod method, String key) { - MockBlobStore.speciallyHandle(method, key, (request, response, context) -> { - try { - Thread.sleep(Long.MAX_VALUE); - } catch (InterruptedException x) { - assert false : x; // on the server side, should not happen - } - }); - } - -}