diff --git a/Jenkinsfile b/Jenkinsfile index c31b5960..a229fa51 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1 +1 @@ -buildPlugin(jenkinsVersions: [null, '2.89']) +buildPlugin() diff --git a/pom.xml b/pom.xml index cfe4138d..8ccee401 100644 --- a/pom.xml +++ b/pom.xml @@ -64,10 +64,11 @@ 2.28 -SNAPSHOT - 2.60.3 + 2.121 8 false 2.14 + true @@ -140,5 +141,29 @@ structs 1.14 + + org.jenkins-ci.test + docker-fixtures + 1.8 + test + + + org.jenkins-ci.plugins + ssh-slaves + 1.26 + test + + + org.jenkins-ci.plugins + jdk-tool + 1.0 + test + + + org.jenkins-ci.plugins + apache-httpcomponents-client-4-api + 4.5.5-2.1 + test + diff --git a/src/main/java/org/jenkinsci/plugins/workflow/flow/StashManager.java b/src/main/java/org/jenkinsci/plugins/workflow/flow/StashManager.java index b8a8ef7b..54fe445d 100644 --- a/src/main/java/org/jenkinsci/plugins/workflow/flow/StashManager.java +++ b/src/main/java/org/jenkinsci/plugins/workflow/flow/StashManager.java @@ -26,12 +26,16 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import hudson.AbortException; +import hudson.EnvVars; import hudson.Extension; import hudson.ExtensionList; import hudson.ExtensionPoint; import hudson.FilePath; +import hudson.Launcher; import hudson.Launcher.LocalLauncher; import hudson.Util; +import hudson.model.Computer; +import hudson.model.Node; import hudson.model.Run; import hudson.model.TaskListener; import hudson.org.apache.tools.tar.TarInputStream; @@ -57,6 +61,7 @@ import org.apache.commons.io.IOUtils; import org.apache.tools.tar.TarEntry; import org.kohsuke.accmod.Restricted; +import org.kohsuke.accmod.restrictions.Beta; import org.kohsuke.accmod.restrictions.DoNotUse; import org.kohsuke.accmod.restrictions.NoExternalUse; @@ -84,20 +89,35 @@ public static void stash(@Nonnull Run build, @Nonnull String name, @Nonnull stash(build, name, workspace, listener, includes, excludes, useDefaultExcludes, false); } + @Deprecated + public static void stash(@Nonnull Run build, @Nonnull String name, @Nonnull FilePath workspace, @Nonnull TaskListener listener, + @CheckForNull String includes, @CheckForNull String excludes, boolean useDefaultExcludes, boolean allowEmpty) throws IOException, InterruptedException { + stash(build, name, workspace, launcherFor(workspace, listener), envFor(build, workspace, listener), listener, includes, excludes, useDefaultExcludes, allowEmpty); + } + /** * Saves a stash of some files from a build. * @param build a build to use as storage * @param name a simple name to assign to the stash (must follow {@link Jenkins#checkGoodName} constraints) * @param workspace a directory to use as a base + * @param launcher a way to launch processes, if required + * @param env environment to use when launching processes, if required + * @param listener a way to report progress or problems * @param includes a set of Ant-style file includes, separated by commas; null/blank is allowed as a synonym for {@code **} (i.e., everything) * @param excludes an optional set of Ant-style file excludes * @param useDefaultExcludes whether to use Ant default excludes * @param allowEmpty whether to allow an empty stash + * @see StashAwareArtifactManager#stash */ @SuppressFBWarnings(value="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE", justification="fine if mkdirs returns false") - public static void stash(@Nonnull Run build, @Nonnull String name, @Nonnull FilePath workspace, @Nonnull TaskListener listener, + public static void stash(@Nonnull Run build, @Nonnull String name, @Nonnull FilePath workspace, @Nonnull Launcher launcher, @Nonnull EnvVars env, @Nonnull TaskListener listener, @CheckForNull String includes, @CheckForNull String excludes, boolean useDefaultExcludes, boolean allowEmpty) throws IOException, InterruptedException { Jenkins.checkGoodName(name); + StashAwareArtifactManager saam = stashAwareArtifactManager(build); + if (saam != null) { + saam.stash(name, workspace, launcher, env, listener, includes, excludes, useDefaultExcludes, allowEmpty); + return; + } File storage = storage(build, name); storage.getParentFile().mkdirs(); if (storage.isFile()) { @@ -115,49 +135,88 @@ public static void stash(@Nonnull Run build, @Nonnull String name, @Nonnull } } + @Deprecated + public static void unstash(@Nonnull Run build, @Nonnull String name, @Nonnull FilePath workspace, @Nonnull TaskListener listener) throws IOException, InterruptedException { + unstash(build, name, workspace, launcherFor(workspace, listener), envFor(build, workspace, listener), listener); + } + /** * Restores a stash of some files from a build. * @param build a build used as storage * @param name a name passed previously to {@link #stash} * @param workspace a directory to copy into + * @param launcher a way to launch processes, if required + * @param env environment to use when launching processes, if required + * @param listener a way to report progress or problems + * @throws AbortException in case there is no such saved stash + * @see StashAwareArtifactManager#unstash */ - public static void unstash(@Nonnull Run build, @Nonnull String name, @Nonnull FilePath workspace, @Nonnull TaskListener listener) throws IOException, InterruptedException { + public static void unstash(@Nonnull Run build, @Nonnull String name, @Nonnull FilePath workspace, @Nonnull Launcher launcher, @Nonnull EnvVars env, @Nonnull TaskListener listener) throws IOException, InterruptedException { Jenkins.checkGoodName(name); + StashAwareArtifactManager saam = stashAwareArtifactManager(build); + if (saam != null) { + saam.unstash(name, workspace, launcher, env, listener); + return; + } File storage = storage(build, name); if (!storage.isFile()) { throw new AbortException("No such saved stash ‘" + name + "’"); } new FilePath(storage).untar(workspace, FilePath.TarCompression.GZIP); - // currently nothing to print; listener is a placeholder + } + + @Deprecated + public static void clearAll(@Nonnull Run build) throws IOException { + try { + clearAll(build, TaskListener.NULL); + } catch (InterruptedException x) { + throw new IOException(x); + } } /** * Delete any and all stashes in a build. * @param build a build possibly passed to {@link #stash} in the past + * @param listener a way to report progress or problems + * @see StashAwareArtifactManager#clearAllStashes */ - public static void clearAll(@Nonnull Run build) throws IOException { + public static void clearAll(@Nonnull Run build, @Nonnull TaskListener listener) throws IOException, InterruptedException { + StashAwareArtifactManager saam = stashAwareArtifactManager(build); + if (saam != null) { + saam.clearAllStashes(listener); + return; + } Util.deleteRecursive(storage(build)); } + @Deprecated + public static void maybeClearAll(@Nonnull Run build) throws IOException { + try { + maybeClearAll(build, TaskListener.NULL); + } catch (InterruptedException x) { + throw new IOException(x); + } + } + /** * Delete any and all stashes in a build unless told otherwise. * {@link StashBehavior#shouldClearAll} may cancel this. * @param build a build possibly passed to {@link #stash} in the past + * @see #clearAll(Run, TaskListener) */ - public static void maybeClearAll(@Nonnull Run build) throws IOException { + public static void maybeClearAll(@Nonnull Run build, @Nonnull TaskListener listener) throws IOException, InterruptedException { for (StashBehavior behavior : ExtensionList.lookup(StashBehavior.class)) { if (!behavior.shouldClearAll(build)) { return; } } - clearAll(build); + clearAll(build, listener); } /** - * Copy any stashes from one build to another. - * @param from a build possibly passed to {@link #stash} in the past - * @param to a new build + * @deprecated without replacement; only used from {@link CopyStashesAndArtifacts} anyway */ + @Deprecated public static void copyAll(@Nonnull Run from, @Nonnull Run to) throws IOException { File fromStorage = storage(from); if (!fromStorage.isDirectory()) { @@ -166,7 +225,7 @@ public static void copyAll(@Nonnull Run from, @Nonnull Run to) throws FileUtils.copyDirectory(fromStorage, storage(to)); } - @Restricted(DoNotUse.class) // currently just for tests + @Restricted(DoNotUse.class) // just for tests, and incompatible with StashAwareArtifactManager @SuppressFBWarnings(value="DM_DEFAULT_ENCODING", justification="test code") public static Map> stashesOf(@Nonnull Run build) throws IOException { Map> result = new TreeMap>(); @@ -196,11 +255,12 @@ public static Map> stashesOf(@Nonnull Run build) return result; } - private static @Nonnull File storage(@Nonnull Run build) { + private static @Nonnull File storage(@Nonnull Run build) throws IOException { + assert stashAwareArtifactManager(build) == null; return new File(build.getRootDir(), "stashes"); } - private static @Nonnull File storage(@Nonnull Run build, @Nonnull String name) { + private static @Nonnull File storage(@Nonnull Run build, @Nonnull String name) throws IOException { File dir = storage(build); File f = new File(dir, name + SUFFIX); if (!f.getParentFile().equals(dir)) { @@ -229,16 +289,91 @@ public boolean shouldClearAll(@Nonnull Run build) { } + /** + * Mixin interface for an {@link ArtifactManager} which supports specialized stash behavior as well. + * + *

When implementing off-Jenkins artifact storage, you should NOT extend this directly but instead use the + * {@code JCloudsArtifactManager} in the plugin currently named {@code artifact-manager-s3}. + * + * This is dangerous to directly extend if using remote storage unless you write a very robust handling of network failures including at least a base timeout and retries. + * The {@code JCloudsArtifactManager} implementation supports extensibility to various cloud providers and custom stores via the {@code BlobStoreProvider} ExtensionPoint. + * It handles all aspects of making cloud artifact storage work smoothly in Jenkins + * including the {@link VirtualFile} implementation, robust network error handling, overall configuration UI, and more. + * Implement this interface directly at your own risk. + * @see JEP-202 + */ + @Restricted(Beta.class) + public interface StashAwareArtifactManager /* extends ArtifactManager */ { + + /** @see StashManager#stash(Run, String, FilePath, Launcher, EnvVars, TaskListener, String, String, boolean, boolean) */ + void stash(@Nonnull String name, @Nonnull FilePath workspace, @Nonnull Launcher launcher, @Nonnull EnvVars env, @Nonnull TaskListener listener, @CheckForNull String includes, @CheckForNull String excludes, boolean useDefaultExcludes, boolean allowEmpty) throws IOException, InterruptedException; + + /** @see StashManager#unstash(Run, String, FilePath, Launcher, EnvVars, TaskListener) */ + void unstash(@Nonnull String name, @Nonnull FilePath workspace, @Nonnull Launcher launcher, @Nonnull EnvVars env, @Nonnull TaskListener listener) throws IOException, InterruptedException; + + /** @see StashManager#clearAll(Run, TaskListener) */ + void clearAllStashes(@Nonnull TaskListener listener) throws IOException, InterruptedException; + + /** + * Copy all stashes and artifacts from one build to another. + * The {@link ArtifactManager} configuration will be as of the origin build. + * If the implementation cannot handle {@code to} for whatever reason, it may throw {@link AbortException}. + * @see CopyStashesAndArtifacts + */ + void copyAllArtifactsAndStashes(@Nonnull Run to, @Nonnull TaskListener listener) throws IOException, InterruptedException; + + } + + private static @CheckForNull StashAwareArtifactManager stashAwareArtifactManager(@Nonnull Run build) throws IOException { + ArtifactManager am = build.pickArtifactManager(); + return am instanceof StashAwareArtifactManager ? (StashAwareArtifactManager) am : null; + } + + @Deprecated + private static @Nonnull Launcher launcherFor(@Nonnull FilePath workspace, @Nonnull TaskListener listener) { + Computer c = workspace.toComputer(); + if (c != null) { + Node n = c.getNode(); + if (n != null) { + return n.createLauncher(listener); + } else { + listener.error(c.getDisplayName() + " seems to be offline"); + return new LocalLauncher(listener); + } + } else { + listener.error(workspace + " seems to be offline"); + return new LocalLauncher(listener); + } + } + + @Deprecated + private static @Nonnull EnvVars envFor(@Nonnull Run build, @Nonnull FilePath workspace, @Nonnull TaskListener listener) throws IOException, InterruptedException { + Computer c = workspace.toComputer(); + if (c != null) { + EnvVars e = c.getEnvironment(); + e.putAll(c.buildEnvironment(listener)); + e.putAll(build.getEnvironment(listener)); + return e; + } else { + listener.error(workspace + " seems to be offline"); + return new EnvVars(); + } + } + @Restricted(NoExternalUse.class) @Extension public static class CopyStashesAndArtifacts extends FlowCopier.ByRun { @Override public void copy(Run original, Run copy, TaskListener listener) throws IOException, InterruptedException { - // TODO ArtifactManager should define an optimized operation to copy from another, or VirtualFile should define copyRecursive + StashAwareArtifactManager saam = stashAwareArtifactManager(original); + if (saam != null) { + saam.copyAllArtifactsAndStashes(copy, listener); + return; + } VirtualFile srcroot = original.getArtifactManager().root(); FilePath dstDir = createTmpDir(); try { Map files = new HashMap<>(); - for (String path : srcroot.list("**/*")) { + for (String path : srcroot.list("**/*", null, false)) { files.put(path, path); InputStream in = srcroot.child(path).open(); try { diff --git a/src/test/java/org/jenkinsci/plugins/workflow/ArtifactManagerTest.java b/src/test/java/org/jenkinsci/plugins/workflow/ArtifactManagerTest.java new file mode 100644 index 00000000..883a75d8 --- /dev/null +++ b/src/test/java/org/jenkinsci/plugins/workflow/ArtifactManagerTest.java @@ -0,0 +1,438 @@ +/* + * The MIT License + * + * Copyright 2018 CloudBees, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package org.jenkinsci.plugins.workflow; + +import hudson.AbortException; +import hudson.EnvVars; +import hudson.ExtensionList; +import hudson.FilePath; +import hudson.Functions; +import hudson.Launcher; +import hudson.Util; +import hudson.model.FreeStyleBuild; +import hudson.model.FreeStyleProject; +import hudson.model.TaskListener; +import hudson.plugins.sshslaves.SSHLauncher; +import hudson.remoting.Callable; +import hudson.slaves.DumbSlave; +import hudson.tasks.ArtifactArchiver; +import hudson.util.StreamTaskListener; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.util.Arrays; +import java.util.Collections; +import java.util.logging.Level; +import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; +import jenkins.model.ArtifactManager; +import jenkins.model.ArtifactManagerConfiguration; +import jenkins.model.ArtifactManagerFactory; +import jenkins.model.Jenkins; +import jenkins.model.StandardArtifactManager; +import jenkins.security.MasterToSlaveCallable; +import jenkins.util.VirtualFile; +import org.apache.commons.io.IOUtils; +import static org.hamcrest.Matchers.*; +import org.jenkinsci.plugins.workflow.flow.StashManager; +import org.jenkinsci.test.acceptance.docker.Docker; +import org.jenkinsci.test.acceptance.docker.DockerImage; +import org.jenkinsci.test.acceptance.docker.fixtures.JavaContainer; +import static org.junit.Assert.*; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.jvnet.hudson.test.JenkinsRule; +import org.jvnet.hudson.test.LoggerRule; + +/** + * {@link #artifactArchiveAndDelete} and variants allow an implementation of {@link ArtifactManager} plus {@link VirtualFile} to be run through a standard gantlet of tests. + */ +public class ArtifactManagerTest { + + @Rule public JenkinsRule r = new JenkinsRule(); + @Rule public LoggerRule logging = new LoggerRule(); + + private static DockerImage image; + + @BeforeClass public static void doPrepareImage() throws Exception { + image = prepareImage(); + } + + /** + * Sets up a Docker image, if Docker support is available in this environment. + * Used by {@link #artifactArchiveAndDelete} etc. + */ + public static @CheckForNull DockerImage prepareImage() throws Exception { + Docker docker = new Docker(); + if (docker.isAvailable()) { + return docker.build(JavaContainer.class); + } else { + System.err.println("No Docker support; falling back to running tests against an agent in a process on the same machine."); + return null; + } + } + + /** + * Creates an agent, in a Docker container when possible, calls {@link #setUpWorkspace}, then runs some tests. + */ + private static void wrapInContainer(@Nonnull JenkinsRule r, @CheckForNull ArtifactManagerFactory factory, + boolean weirdCharacters, TestFunction f) throws Exception { + if (factory != null) { + ArtifactManagerConfiguration.get().getArtifactManagerFactories().add(factory); + } + JavaContainer runningContainer = null; + try { + DumbSlave agent; + if (image != null) { + runningContainer = image.start(JavaContainer.class).start(); + agent = new DumbSlave("test-agent", "/home/test/slave", new SSHLauncher(runningContainer.ipBound(22), runningContainer.port(22), "test", "test", "", "")); + Jenkins.get().addNode(agent); + r.waitOnline(agent); + } else { + agent = r.createOnlineSlave(); + } + FreeStyleProject p = r.createFreeStyleProject(); + p.setAssignedNode(agent); + FilePath ws = agent.getWorkspaceFor(p); + setUpWorkspace(ws, weirdCharacters); + ArtifactArchiver aa = new ArtifactArchiver("**"); + aa.setDefaultExcludes(false); + p.getPublishersList().add(aa); + FreeStyleBuild b = r.buildAndAssertSuccess(p); + f.apply(agent, p, b, ws); + } finally { + if (runningContainer != null) { + runningContainer.close(); + } + } + } + + /** + * Test artifact archiving with a manager that does not honor deletion requests. + * @param weirdCharacters as in {@link #artifactArchiveAndDelete} + * @param image as in {@link #artifactArchiveAndDelete} + */ + public static void artifactArchive(@Nonnull JenkinsRule r, @CheckForNull ArtifactManagerFactory factory, boolean weirdCharacters, @CheckForNull DockerImage image) throws Exception { + wrapInContainer(r, factory, weirdCharacters, (agent, p, b, ws) -> { + VirtualFile root = b.getArtifactManager().root(); + new Verify(agent, root, weirdCharacters).run(); + // should not delete + assertFalse(b.getArtifactManager().delete()); + assertTrue(b.getArtifactManager().root().child("file").isFile()); + }); + } + + /** + * Test artifact archiving in a plain manager. + * @param weirdCharacters check behavior of files with Unicode and various unusual characters in the name + * @param image use {@link #prepareImage} in a {@link BeforeClass} block + */ + public static void artifactArchiveAndDelete(@Nonnull JenkinsRule r, @CheckForNull ArtifactManagerFactory factory, boolean weirdCharacters, @CheckForNull DockerImage image) throws Exception { + wrapInContainer(r, factory, weirdCharacters, (agent, p, b, ws) -> { + VirtualFile root = b.getArtifactManager().root(); + new Verify(agent, root, weirdCharacters).run(); + // Also check deletion: + assertTrue(b.getArtifactManager().delete()); + assertFalse(b.getArtifactManager().root().child("file").isFile()); + assertFalse(b.getArtifactManager().delete()); + }); + } + + /** + * Test stashing and unstashing with a {@link StashManager.StashAwareArtifactManager} that does not honor deletion requests. + * @param weirdCharacters as in {@link #artifactArchiveAndDelete} + * @param image as in {@link #artifactArchiveAndDelete} + */ + public static void artifactStash(@Nonnull JenkinsRule r, @CheckForNull ArtifactManagerFactory factory, boolean weirdCharacters, @CheckForNull DockerImage image) throws Exception { + wrapInContainer(r, factory, weirdCharacters, + new StashFunction(r, weirdCharacters, (p, b, ws, launcher, env, listener) -> { + // should not have deleted + StashManager.unstash(b, "stuff", ws, launcher, env, listener); + assertTrue(ws.child("file").exists()); + })); + } + + /** + * Test stashing and unstashing with a {@link StashManager.StashAwareArtifactManager} with standard behavior. + * @param weirdCharacters as in {@link #artifactArchiveAndDelete} + * @param image as in {@link #artifactArchiveAndDelete} + */ + public static void artifactStashAndDelete(@Nonnull JenkinsRule r, @CheckForNull ArtifactManagerFactory factory, boolean weirdCharacters, @CheckForNull DockerImage image) throws Exception { + wrapInContainer(r, factory, weirdCharacters, + new StashFunction(r, weirdCharacters, (p, b, ws, launcher, env, listener) -> { + try { + StashManager.unstash(b, "stuff", ws, launcher, env, listener); + fail("should not have succeeded in unstashing"); + } catch (AbortException x) { + System.err.println("caught as expected: " + x); + } + assertFalse(ws.child("file").exists()); + })); + } + + /** + * Creates a variety of files in a directory structure designed to exercise interesting aspects of {@link VirtualFile}. + */ + private static void setUpWorkspace(FilePath workspace, boolean weirdCharacters) throws Exception { + workspace.child("file").write("content", null); + workspace.child("some/deeply/nested/dir/subfile").write("content", null); + workspace.child(".git/config").write("whatever", null); + workspace.child("otherdir/somefile~").write("whatever", null); + if (weirdCharacters) { + assertEquals("UTF-8 vs. UTF-8", workspace.getChannel().call(new FindEncoding())); + workspace.child("otherdir/xxx#?:$&'\"<>čॐ").write("whatever", null); + } + // best to avoid scalability tests (large number of files, single large file) here—too fragile + // also avoiding tests of file mode and symlinks: will not work on Windows, and may or may not work in various providers + } + private static class FindEncoding extends MasterToSlaveCallable { + @Override public String call() throws Exception { + return System.getProperty("file.encoding") + " vs. " + System.getProperty("sun.jnu.encoding"); + } + } + + /** + * Block to run overall tests. + * @see #wrapInContainer + */ + @FunctionalInterface + private interface TestFunction { + void apply(DumbSlave agent, FreeStyleProject p, FreeStyleBuild b, FilePath ws) throws Exception; + } + + /** + * Block to run stash-specific tests. + * @see StashFunction + */ + @FunctionalInterface + private interface TestStashFunction { + void apply(FreeStyleProject p, FreeStyleBuild b, FilePath ws, Launcher launcher, EnvVars env, + TaskListener listener) throws Exception; + } + + /** + * Verifies behaviors of stash and unstash operations. + */ + private static class StashFunction implements TestFunction { + private final JenkinsRule r; + private final boolean weirdCharacters; + private final TestStashFunction f; + + StashFunction(@Nonnull JenkinsRule r, boolean weirdCharacters, TestStashFunction f) { + this.r = r; + this.weirdCharacters = weirdCharacters; + this.f = f; + } + + @Override + public void apply(DumbSlave agent, FreeStyleProject p, FreeStyleBuild b, FilePath ws) throws Exception { + TaskListener listener = StreamTaskListener.fromStderr(); + Launcher launcher = agent.createLauncher(listener); + EnvVars env = agent.toComputer().getEnvironment(); + env.putAll(agent.toComputer().buildEnvironment(listener)); + // Make sure we can stash and then unstash within a build: + StashManager.stash(b, "stuff", ws, launcher, env, listener, "file", null, false, false); + ws.child("file").delete(); + StashManager.unstash(b, "stuff", ws, launcher, env, listener); + assertEquals("content", ws.child("file").readToString()); + ws.child("file").delete(); + // Copy stashes and artifacts from one build to a second one: + p.getPublishersList().clear(); + FreeStyleBuild b2 = r.buildAndAssertSuccess(p); + ExtensionList.lookupSingleton(StashManager.CopyStashesAndArtifacts.class).copy(b, b2, listener); + // Verify the copied stashes: + StashManager.unstash(b2, "stuff", ws, launcher, env, listener); + assertEquals("content", ws.child("file").readToString()); + // And the copied artifacts: + VirtualFile root = b2.getArtifactManager().root(); + new Verify(agent, root, weirdCharacters).run(); + // Also delete the original: + StashManager.clearAll(b, listener); + // Stashes should have been deleted, but not artifacts: + assertTrue(b.getArtifactManager().root().child("file").isFile()); + ws.deleteContents(); + assertFalse(ws.child("file").exists()); + f.apply(p, b, ws, launcher, env, listener); + } + } + + /** + * Runs an assortment of verifications via {@link #test} on a remote directory. + */ + private static class Verify { + + private final DumbSlave agent; + private final VirtualFile root; + private final boolean weirdCharacters; + + Verify(DumbSlave agent, VirtualFile root, boolean weirdCharacters) { + this.agent = agent; + this.root = root; + this.weirdCharacters = weirdCharacters; + } + + /** + * Perform verification. + * When {@link VirtualFile#run} is overridden, uses {@link VerifyBatch} also. + */ + void run() throws Exception { + test(); + if (Util.isOverridden(VirtualFile.class, root.getClass(), "run", Callable.class)) { + for (VirtualFile f : Arrays.asList(root, root.child("some"), root.child("file"), root.child("does-not-exist"))) { + System.err.println("testing batch operations starting from " + f); + f.run(new VerifyBatch(this)); + } + } + } + + /** + * Performs verifications against a possibly cached set of metadata. + */ + private static class VerifyBatch extends MasterToSlaveCallable { + private final Verify verification; + VerifyBatch(Verify verification) { + this.verification = verification; + } + @Override public Void call() throws IOException { + try { + verification.test(); + } catch (RuntimeException | IOException x) { + throw x; + } catch (Exception x) { + throw new IOException(x); + } + return null; + } + } + + /** + * Verifies miscellaneous aspects of files in {@link root}. + * Checks that files are in the expected places, directories can be listed, etc. + * @see #setUpWorkspace + */ + private void test() throws Exception { + assertThat("root name is unspecified generally", root.getName(), not(endsWith("/"))); + VirtualFile file = root.child("file"); + assertEquals("file", file.getName()); + assertFile(file, "content"); + assertEquals(root, file.getParent()); + VirtualFile some = root.child("some"); + assertEquals("some", some.getName()); + assertDir(some); + assertEquals(root, some.getParent()); + assertThat(root.list(), arrayContainingInAnyOrder(file, some, root.child("otherdir"), root.child(".git"))); + assertThat(root.list("file", null, false), containsInAnyOrder("file")); + VirtualFile subfile = root.child("some/deeply/nested/dir/subfile"); + assertEquals("subfile", subfile.getName()); + assertFile(subfile, "content"); + VirtualFile someDeeplyNestedDir = some.child("deeply/nested/dir"); + assertEquals("dir", someDeeplyNestedDir.getName()); + assertDir(someDeeplyNestedDir); + assertEquals(some, someDeeplyNestedDir.getParent().getParent().getParent()); + assertEquals(Collections.singletonList(subfile), Arrays.asList(someDeeplyNestedDir.list())); + assertThat(someDeeplyNestedDir.list("subfile", null, false), containsInAnyOrder("subfile")); + assertThat(root.list("**/*file", null, false), containsInAnyOrder("file", "some/deeply/nested/dir/subfile")); + assertThat(some.list("**/*file", null, false), containsInAnyOrder("deeply/nested/dir/subfile")); + assertThat(root.list("**", "**/xxx*", true), containsInAnyOrder("file", "some/deeply/nested/dir/subfile")); + if (weirdCharacters) { + assertFile(root.child("otherdir/xxx#?:$&'\"<>čॐ"), "whatever"); + } + assertNonexistent(root.child("does-not-exist")); + assertNonexistent(root.child("some/deeply/nested/dir/does-not-exist")); + } + + /** + * Checks that a given file exists, as a plain file, with the specified contents. + * Checks both {@link VirtualFile#open} and, if implemented, {@link VirtualFile#toExternalURL}. + */ + private void assertFile(VirtualFile f, String contents) throws Exception { + System.err.println("Asserting file: " + f); + assertTrue("Not a file: " + f, f.isFile()); + assertFalse("Unexpected directory: " + f, f.isDirectory()); + assertTrue("Does not exist: " + f, f.exists()); + assertEquals(contents.length(), f.length()); + assertThat(f.lastModified(), not(is(0))); + try (InputStream is = f.open()) { + assertEquals(contents, IOUtils.toString(is)); + } + URL url = f.toExternalURL(); + if (url != null) { + System.err.println("opening " + url); + assertEquals(contents, agent.getChannel().call(new RemoteOpenURL(url))); + } + } + + private static final class RemoteOpenURL extends MasterToSlaveCallable { + private final URL u; + RemoteOpenURL(URL u) { + this.u = u; + } + @Override public String call() throws IOException { + return IOUtils.toString(u); + } + } + + } + + /** + * Checks that a given path exists as a directory. + */ + private static void assertDir(VirtualFile f) throws IOException { + System.err.println("Asserting dir: " + f); + assertFalse("Unexpected file: " + f, f.isFile()); + assertTrue("Not a directory: " + f, f.isDirectory()); + assertTrue("Does not exist: " + f, f.exists()); + // length & lastModified may or may not be defined + } + + /** + * Checks that a given path does not exist as either a file or a directory. + */ + private static void assertNonexistent(VirtualFile f) throws IOException { + System.err.println("Asserting nonexistent: " + f); + assertFalse("Unexpected file: " + f, f.isFile()); + assertFalse("Unexpected dir: " + f, f.isDirectory()); + assertFalse("Unexpectedly exists: " + f, f.exists()); + try { + assertEquals(0, f.length()); + } catch (IOException x) { + // also OK + } + try { + assertEquals(0, f.lastModified()); + } catch (IOException x) { + // also OK + } + } + + /** Run the standard one, as a control. */ + @Test public void standard() throws Exception { + logging.record(StandardArtifactManager.class, Level.FINE); + // Who knows about weird characters on NTFS; also case-sensitivity could confuse things + artifactArchiveAndDelete(r, null, !Functions.isWindows(), image); + } + +} diff --git a/src/test/java/org/jenkinsci/plugins/workflow/DirectArtifactManagerFactory.java b/src/test/java/org/jenkinsci/plugins/workflow/DirectArtifactManagerFactory.java new file mode 100644 index 00000000..d8273c3d --- /dev/null +++ b/src/test/java/org/jenkinsci/plugins/workflow/DirectArtifactManagerFactory.java @@ -0,0 +1,257 @@ +/* + * The MIT License + * + * Copyright 2018 CloudBees, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package org.jenkinsci.plugins.workflow; + +import hudson.Extension; +import hudson.FilePath; +import hudson.Launcher; +import hudson.Util; +import hudson.model.BuildListener; +import hudson.model.Run; +import hudson.remoting.Callable; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URL; +import java.net.URLConnection; +import java.net.URLDecoder; +import java.net.URLEncoder; +import java.net.URLStreamHandler; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collection; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Level; +import java.util.logging.Logger; +import jenkins.model.ArtifactManager; +import jenkins.model.ArtifactManagerFactory; +import jenkins.model.ArtifactManagerFactoryDescriptor; +import jenkins.model.Jenkins; +import jenkins.util.VirtualFile; +import org.apache.commons.io.IOUtils; +import org.apache.http.ConnectionClosedException; +import org.apache.http.HttpRequest; +import org.apache.http.HttpResponse; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.bootstrap.HttpServer; +import org.apache.http.impl.bootstrap.ServerBootstrap; +import org.apache.http.protocol.HttpContext; +import org.jvnet.hudson.test.JenkinsRule; + +/** + * A mock artifact manager which allows tests to exercise direct download of artifacts via HTTP URLs. + * Whereas {@link ArtifactManagerTest} allows you to test an implementation, this allows you to test a caller. + * Use {@link #whileBlockingOpen} to exercise the behavior. + * @see JENKINS-49635 + */ +public final class DirectArtifactManagerFactory extends ArtifactManagerFactory { + + private static final Logger LOGGER = Logger.getLogger(DirectArtifactManagerFactory.class.getName()); + private static final AtomicInteger blockOpen = new AtomicInteger(); + + private final transient URL baseURL; + + public DirectArtifactManagerFactory() throws Exception { + HttpServer server = ServerBootstrap.bootstrap(). + registerHandler("*", (HttpRequest request, HttpResponse response, HttpContext _context) -> { + String method = request.getRequestLine().getMethod(); + String contents = URLDecoder.decode(request.getRequestLine().getUri().substring(1), "UTF-8"); + switch (method) { + case "GET": { + response.setStatusCode(200); + response.setEntity(new StringEntity(contents)); + LOGGER.log(Level.INFO, "Serving ‘{0}’", contents); + return; + } + default: { + throw new IllegalStateException(); + } + } + }). + setExceptionLogger(x -> { + if (x instanceof ConnectionClosedException) { + LOGGER.info(x.toString()); + } else { + LOGGER.log(Level.INFO, "error thrown in HTTP service", x); + } + }). + create(); + server.start(); + baseURL = new URL("http://" + server.getInetAddress().getHostName() + ":" + server.getLocalPort() + "/"); + LOGGER.log(Level.INFO, "Mock server running at {0}", baseURL); + + } + + @Override public ArtifactManager managerFor(Run build) { + return new DirectArtifactManager(build, baseURL); + } + + /** + * Within this dynamic scope (not sensitive to a thread), prevent {@link VirtualFile#open} from being called. + * {@link VirtualFile#toExternalURL} may be called, but the URL may not be opened inside this JVM + * (so you must send it for example to {@link JenkinsRule#createOnlineSlave()}). + */ + public static T whileBlockingOpen(java.util.concurrent.Callable block) throws Exception { + blockOpen.incrementAndGet(); + try { + return block.call(); + } finally { + blockOpen.decrementAndGet(); + } + } + + @Extension public static final class DescriptorImpl extends ArtifactManagerFactoryDescriptor {} + + private static final class DirectArtifactManager extends ArtifactManager { + + private transient File dir; + private transient final URL baseURL; + + DirectArtifactManager(Run build, URL baseURL) { + this.baseURL = baseURL; + onLoad(build); + } + + @Override public void archive(FilePath workspace, Launcher launcher, BuildListener listener, Map artifacts) throws IOException, InterruptedException { + workspace.copyRecursiveTo(new FilePath.ExplicitlySpecifiedDirScanner(artifacts), new FilePath(dir), "copying"); + } + + @Override public VirtualFile root() { + return new NoOpenVF(VirtualFile.forFile(dir), baseURL); + } + + @Override public void onLoad(Run build) { + dir = new File(Jenkins.get().getRootDir(), Util.getDigestOf(build.getExternalizableId())); + } + + @Override public boolean delete() throws IOException, InterruptedException { + if (!dir.exists()) { + return false; + } + Util.deleteRecursive(dir); + return true; + } + + } + + private static final class NoOpenVF extends VirtualFile { + + private final VirtualFile delegate; + private final URL baseURL; + + NoOpenVF(VirtualFile delegate, URL baseURL) { + this.delegate = delegate; + this.baseURL = baseURL; + } + + @Override public InputStream open() throws IOException { + if (blockOpen.get() > 0) { + throw new IllegalStateException("should not be called; use toExternalURL instead"); + } else { + return delegate.open(); + } + } + + @Override public URL toExternalURL() throws IOException { + if (blockOpen.get() > 0) { + String contents; + try (InputStream is = delegate.open()) { + contents = IOUtils.toString(is, StandardCharsets.UTF_8); + } + return new URL(null, baseURL + URLEncoder.encode(contents, "UTF-8"), new URLStreamHandler() { + @Override protected URLConnection openConnection(URL u) throws IOException { + throw new IOException("not allowed to open " + u + " from this JVM"); + } + }); + } else { + return delegate.toExternalURL(); + } + } + + @Override public String getName() { + return delegate.getName(); + } + + @Override public URI toURI() { + return delegate.toURI(); + } + + @Override public VirtualFile getParent() { + return new NoOpenVF(delegate.getParent(), baseURL); + } + + @Override public boolean isDirectory() throws IOException { + return delegate.isDirectory(); + } + + @Override public boolean isFile() throws IOException { + return delegate.isFile(); + } + + @Override public String readLink() throws IOException { + return delegate.readLink(); + } + + @Override public boolean exists() throws IOException { + return delegate.exists(); + } + + @Override public VirtualFile[] list() throws IOException { + return Arrays.stream(delegate.list()).map(vf -> new NoOpenVF(vf, baseURL)).toArray(VirtualFile[]::new); + } + + @Override public Collection list(String includes, String excludes, boolean useDefaultExcludes) throws IOException { + return delegate.list(includes, excludes, useDefaultExcludes); + } + + @Override public VirtualFile child(String string) { + return new NoOpenVF(delegate.child(string), baseURL); + } + + @Override public long length() throws IOException { + return delegate.length(); + } + + @Override public long lastModified() throws IOException { + return delegate.lastModified(); + } + + @Override public int mode() throws IOException { + return delegate.mode(); + } + + @Override public boolean canRead() throws IOException { + return delegate.canRead(); + } + + @Override public V run(Callable clbl) throws IOException { + return delegate.run(clbl); + } + + } + +}