[JENKINS-49635] VirtualFile support

Jenkinsfile

@@ -1 +1 @@
-buildPlugin(jenkinsVersions: [null, '2.89'])
+buildPlugin()

pom.xml

@@ -64,10 +64,11 @@
     <properties>
         <revision>2.28</revision>
         <changelist>-SNAPSHOT</changelist>
-        <jenkins.version>2.60.3</jenkins.version>
+        <jenkins.version>2.121</jenkins.version>
         <java.level>8</java.level>
         <no-test-jar>false</no-test-jar>
         <workflow-support-plugin.version>2.14</workflow-support-plugin.version>
+        <useBeta>true</useBeta>
     </properties>
     <dependencies>
         <dependency>
@@ -140,5 +141,29 @@
             <artifactId>structs</artifactId>
             <version>1.14</version>
         </dependency>
+        <dependency>
+            <groupId>org.jenkins-ci.test</groupId>
+            <artifactId>docker-fixtures</artifactId>
+            <version>1.8</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.jenkins-ci.plugins</groupId>
+            <artifactId>ssh-slaves</artifactId>
+            <version>1.26</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.jenkins-ci.plugins</groupId>
+            <artifactId>jdk-tool</artifactId>
+            <version>1.0</version>
+            <scope>test</scope>
+        </dependency>
+      <dependency>
+          <groupId>org.jenkins-ci.plugins</groupId>
+          <artifactId>apache-httpcomponents-client-4-api</artifactId>
+          <version>4.5.5-2.1</version>
+          <scope>test</scope>
+      </dependency>
     </dependencies>
 </project>

src/main/java/org/jenkinsci/plugins/workflow/flow/StashManager.java

@@ -26,12 +26,16 @@
 
 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 import hudson.AbortException;
+import hudson.EnvVars;
 import hudson.Extension;
 import hudson.ExtensionList;
 import hudson.ExtensionPoint;
 import hudson.FilePath;
+import hudson.Launcher;
 import hudson.Launcher.LocalLauncher;
 import hudson.Util;
+import hudson.model.Computer;
+import hudson.model.Node;
 import hudson.model.Run;
 import hudson.model.TaskListener;
 import hudson.org.apache.tools.tar.TarInputStream;
@@ -57,6 +61,7 @@
 import org.apache.commons.io.IOUtils;
 import org.apache.tools.tar.TarEntry;
 import org.kohsuke.accmod.Restricted;
+import org.kohsuke.accmod.restrictions.Beta;
 import org.kohsuke.accmod.restrictions.DoNotUse;
 import org.kohsuke.accmod.restrictions.NoExternalUse;
 
@@ -84,20 +89,35 @@ public static void stash(@Nonnull Run<?,?> build, @Nonnull String name, @Nonnull
         stash(build, name, workspace, listener, includes, excludes, useDefaultExcludes, false);
     }
 
+    @Deprecated
+    public static void stash(@Nonnull Run<?,?> build, @Nonnull String name, @Nonnull FilePath workspace, @Nonnull TaskListener listener,
+                             @CheckForNull String includes, @CheckForNull String excludes, boolean useDefaultExcludes, boolean allowEmpty) throws IOException, InterruptedException {
+        stash(build, name, workspace, launcherFor(workspace, listener), envFor(build, workspace, listener), listener, includes, excludes, useDefaultExcludes, allowEmpty);
+    }
+
     /**
      * Saves a stash of some files from a build.
      * @param build a build to use as storage
      * @param name a simple name to assign to the stash (must follow {@link Jenkins#checkGoodName} constraints)
      * @param workspace a directory to use as a base
+     * @param launcher a way to launch processes, if required
+     * @param env environment to use when launching processes, if required
+     * @param listener a way to report progress or problems
      * @param includes a set of Ant-style file includes, separated by commas; null/blank is allowed as a synonym for {@code **} (i.e., everything)
      * @param excludes an optional set of Ant-style file excludes
      * @param useDefaultExcludes whether to use Ant default excludes
      * @param allowEmpty whether to allow an empty stash
+     * @see StashAwareArtifactManager#stash
      */
     @SuppressFBWarnings(value="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE", justification="fine if mkdirs returns false")
-    public static void stash(@Nonnull Run<?,?> build, @Nonnull String name, @Nonnull FilePath workspace, @Nonnull TaskListener listener,
+    public static void stash(@Nonnull Run<?,?> build, @Nonnull String name, @Nonnull FilePath workspace, @Nonnull Launcher launcher, @Nonnull EnvVars env, @Nonnull TaskListener listener,
                              @CheckForNull String includes, @CheckForNull String excludes, boolean useDefaultExcludes, boolean allowEmpty) throws IOException, InterruptedException {
         Jenkins.checkGoodName(name);
+        StashAwareArtifactManager saam = stashAwareArtifactManager(build);
+        if (saam != null) {
+            saam.stash(name, workspace, launcher, env, listener, includes, excludes, useDefaultExcludes, allowEmpty);
+            return;
+        }
         File storage = storage(build, name);
         storage.getParentFile().mkdirs();
         if (storage.isFile()) {
@@ -115,49 +135,88 @@ public static void stash(@Nonnull Run<?,?> build, @Nonnull String name, @Nonnull
         }
     }
 
+    @Deprecated
+    public static void unstash(@Nonnull Run<?,?> build, @Nonnull String name, @Nonnull FilePath workspace, @Nonnull TaskListener listener) throws IOException, InterruptedException {
+        unstash(build, name, workspace, launcherFor(workspace, listener), envFor(build, workspace, listener), listener);
+    }
+
     /**
      * Restores a stash of some files from a build.
      * @param build a build used as storage
      * @param name a name passed previously to {@link #stash}
      * @param workspace a directory to copy into
+     * @param launcher a way to launch processes, if required
+     * @param env environment to use when launching processes, if required
+     * @param listener a way to report progress or problems
+     * @throws AbortException in case there is no such saved stash
+     * @see StashAwareArtifactManager#unstash
      */
-    public static void unstash(@Nonnull Run<?,?> build, @Nonnull String name, @Nonnull FilePath workspace, @Nonnull TaskListener listener) throws IOException, InterruptedException {
+    public static void unstash(@Nonnull Run<?,?> build, @Nonnull String name, @Nonnull FilePath workspace, @Nonnull Launcher launcher, @Nonnull EnvVars env, @Nonnull TaskListener listener) throws IOException, InterruptedException {
         Jenkins.checkGoodName(name);
+        StashAwareArtifactManager saam = stashAwareArtifactManager(build);
+        if (saam != null) {
+            saam.unstash(name, workspace, launcher, env, listener);
+            return;
+        }
         File storage = storage(build, name);
         if (!storage.isFile()) {
             throw new AbortException("No such saved stash ‘" + name + "’");
         }
         new FilePath(storage).untar(workspace, FilePath.TarCompression.GZIP);
-        // currently nothing to print; listener is a placeholder
+    }
+
+    @Deprecated
+    public static void clearAll(@Nonnull Run<?,?> build) throws IOException {
+        try {
+            clearAll(build, TaskListener.NULL);
+        } catch (InterruptedException x) {
+            throw new IOException(x);
+        }
     }
 
     /**
      * Delete any and all stashes in a build.
      * @param build a build possibly passed to {@link #stash} in the past
+     * @param listener a way to report progress or problems
+     * @see StashAwareArtifactManager#clearAllStashes
      */
-    public static void clearAll(@Nonnull Run<?,?> build) throws IOException {
+    public static void clearAll(@Nonnull Run<?,?> build, @Nonnull TaskListener listener) throws IOException, InterruptedException {
+        StashAwareArtifactManager saam = stashAwareArtifactManager(build);
+        if (saam != null) {
+            saam.clearAllStashes(listener);
+            return;
+        }
         Util.deleteRecursive(storage(build));
     }
 
+    @Deprecated
+    public static void maybeClearAll(@Nonnull Run<?,?> build) throws IOException {
+        try {
+            maybeClearAll(build, TaskListener.NULL);
+        } catch (InterruptedException x) {
+            throw new IOException(x);
+        }
+    }
+
     /**
      * Delete any and all stashes in a build unless told otherwise.
      * {@link StashBehavior#shouldClearAll} may cancel this.
      * @param build a build possibly passed to {@link #stash} in the past
+     * @see #clearAll(Run, TaskListener)
      */
-    public static void maybeClearAll(@Nonnull Run<?,?> build) throws IOException {
+    public static void maybeClearAll(@Nonnull Run<?,?> build, @Nonnull TaskListener listener) throws IOException, InterruptedException {
         for (StashBehavior behavior : ExtensionList.lookup(StashBehavior.class)) {
             if (!behavior.shouldClearAll(build)) {
                 return;
             }
         }
-        clearAll(build);
+        clearAll(build, listener);
     }
 
     /**
-     * Copy any stashes from one build to another.
-     * @param from a build possibly passed to {@link #stash} in the past
-     * @param to a new build
+     * @deprecated without replacement; only used from {@link CopyStashesAndArtifacts} anyway
      */
+    @Deprecated
     public static void copyAll(@Nonnull Run<?,?> from, @Nonnull Run<?,?> to) throws IOException {
         File fromStorage = storage(from);
         if (!fromStorage.isDirectory()) {
@@ -166,7 +225,7 @@ public static void copyAll(@Nonnull Run<?,?> from, @Nonnull Run<?,?> to) throws
         FileUtils.copyDirectory(fromStorage, storage(to));
     }
 
-    @Restricted(DoNotUse.class) // currently just for tests
+    @Restricted(DoNotUse.class) // just for tests, and incompatible with StashAwareArtifactManager
     @SuppressFBWarnings(value="DM_DEFAULT_ENCODING", justification="test code")
     public static Map<String,Map<String,String>> stashesOf(@Nonnull Run<?,?> build) throws IOException {
         Map<String,Map<String,String>> result = new TreeMap<String,Map<String,String>>();
@@ -196,11 +255,12 @@ public static Map<String,Map<String,String>> stashesOf(@Nonnull Run<?,?> build)
         return result;
     }
 
-    private static @Nonnull File storage(@Nonnull Run<?,?> build) {
+    private static @Nonnull File storage(@Nonnull Run<?,?> build) throws IOException {
+        assert stashAwareArtifactManager(build) == null;
         return new File(build.getRootDir(), "stashes");
     }
 
-    private static @Nonnull File storage(@Nonnull Run<?,?> build, @Nonnull String name) {
+    private static @Nonnull File storage(@Nonnull Run<?,?> build, @Nonnull String name) throws IOException {
         File dir = storage(build);
         File f = new File(dir, name + SUFFIX);
         if (!f.getParentFile().equals(dir)) {
@@ -229,16 +289,91 @@ public boolean shouldClearAll(@Nonnull Run<?,?> build) {
 
     }
 
+    /**
+     * Mixin interface for an {@link ArtifactManager} which supports specialized stash behavior as well.
+     *
+     * <p> When implementing off-Jenkins artifact storage, you should NOT extend this directly but instead use the
+     * {@code JCloudsArtifactManager} in the plugin currently named {@code artifact-manager-s3}.
+     *
+     * This is dangerous to directly extend if using remote storage unless you write a very robust handling of network failures including at least a base timeout and retries.
+     * The {@code JCloudsArtifactManager} implementation supports extensibility to various cloud providers and custom stores via the {@code BlobStoreProvider} ExtensionPoint.
+     * It handles all aspects of making cloud artifact storage work smoothly in Jenkins
+     * including the {@link VirtualFile} implementation, robust network error handling, overall configuration UI, and more.
+     * <strong>Implement this interface directly at your own risk.</strong>
+     * @see <a href="https://github.com/jenkinsci/jep/blob/master/jep/202/README.adoc">JEP-202</a>
+     */
+    @Restricted(Beta.class)
+    public interface StashAwareArtifactManager /* extends ArtifactManager */ {
+
+        /** @see StashManager#stash(Run, String, FilePath, Launcher, EnvVars, TaskListener, String, String, boolean, boolean) */
+        void stash(@Nonnull String name, @Nonnull FilePath workspace, @Nonnull Launcher launcher, @Nonnull EnvVars env, @Nonnull TaskListener listener, @CheckForNull String includes, @CheckForNull String excludes, boolean useDefaultExcludes, boolean allowEmpty) throws IOException, InterruptedException;
+
+        /** @see StashManager#unstash(Run, String, FilePath, Launcher, EnvVars, TaskListener) */
+        void unstash(@Nonnull String name, @Nonnull FilePath workspace, @Nonnull Launcher launcher, @Nonnull EnvVars env, @Nonnull TaskListener listener) throws IOException, InterruptedException;
+
+        /** @see StashManager#clearAll(Run, TaskListener) */
+        void clearAllStashes(@Nonnull TaskListener listener) throws IOException, InterruptedException;
+
+        /**
+         * Copy all stashes and artifacts from one build to another.
+         * The {@link ArtifactManager} configuration will be as of the origin build.
+         * If the implementation cannot handle {@code to} for whatever reason, it may throw {@link AbortException}.
+         * @see CopyStashesAndArtifacts
+         */
+        void copyAllArtifactsAndStashes(@Nonnull Run<?,?> to, @Nonnull TaskListener listener) throws IOException, InterruptedException;
+
+    }
+
+    private static @CheckForNull StashAwareArtifactManager stashAwareArtifactManager(@Nonnull Run<?, ?> build) throws IOException {
+        ArtifactManager am = build.pickArtifactManager();
+        return am instanceof StashAwareArtifactManager ? (StashAwareArtifactManager) am : null;
+    }
+
+    @Deprecated
+    private static @Nonnull Launcher launcherFor(@Nonnull FilePath workspace, @Nonnull TaskListener listener) {
+        Computer c = workspace.toComputer();
+        if (c != null) {
+            Node n = c.getNode();
+            if (n != null) {
+                return n.createLauncher(listener);
+            } else {
+                listener.error(c.getDisplayName() + " seems to be offline");
+                return new LocalLauncher(listener);
+            }
+        } else {
+            listener.error(workspace + " seems to be offline");
+            return new LocalLauncher(listener);
+        }
+    }
+
+    @Deprecated
+    private static @Nonnull EnvVars envFor(@Nonnull Run<?, ?> build, @Nonnull FilePath workspace, @Nonnull TaskListener listener) throws IOException, InterruptedException {
+        Computer c = workspace.toComputer();
+        if (c != null) {
+            EnvVars e = c.getEnvironment();
+            e.putAll(c.buildEnvironment(listener));
+            e.putAll(build.getEnvironment(listener));
+            return e;
+        } else {
+            listener.error(workspace + " seems to be offline");
+            return new EnvVars();
+        }
+    }
+
     @Restricted(NoExternalUse.class)
     @Extension public static class CopyStashesAndArtifacts extends FlowCopier.ByRun {
 
         @Override public void copy(Run<?,?> original, Run<?,?> copy, TaskListener listener) throws IOException, InterruptedException {
-            // TODO ArtifactManager should define an optimized operation to copy from another, or VirtualFile should define copyRecursive
+            StashAwareArtifactManager saam = stashAwareArtifactManager(original);
+            if (saam != null) {
+                saam.copyAllArtifactsAndStashes(copy, listener);
+                return;
+            }
             VirtualFile srcroot = original.getArtifactManager().root();
             FilePath dstDir = createTmpDir();
             try {
                 Map<String,String> files = new HashMap<>();
-                for (String path : srcroot.list("**/*")) {
+                for (String path : srcroot.list("**/*", null, false)) {
                     files.put(path, path);
                     InputStream in = srcroot.child(path).open();
                     try {