Allow multiple LogStorage with primary and secondaries

README.md

@@ -11,6 +11,15 @@ Plugin that defines Pipeline API.
 A component of [Pipeline
 Plugin](https://plugins.jenkins.io/workflow-aggregator).
 
+# JEP-210: External log storage for Pipeline
+## Implementation
+This plugin supports [https://github.com/jenkinsci/jep/tree/master/jep/210](JEP-210) which allows using one logger at a time: the pipeline logs are written and read from the same storage. The pipeline logger is automatically selected during startup through the `@Extension` annotation `ordinal` value.
+
+## Multiple log storages
+A new "Pipeline logger" section is now configurable through UI or CasC, instead of relying only on the `@Extension ordinal` value, a specific logger can be selected.
+
+A new "Multiple loggers" log storage implementation following JEP-210 has been introduced, allowing to configure a "Primary" (for read/writes) and a "Secondary" (for writes-only) logger. This acts as a "tee" command.
+
 # Changelog
 
 * For new versions, see [GitHub Releases](https://github.com/jenkinsci/workflow-api-plugin/releases)

pom.xml

@@ -138,5 +138,10 @@
           <artifactId>apache-httpcomponents-client-4-api</artifactId>
           <scope>test</scope>
         </dependency>
+        <dependency>
+          <groupId>io.jenkins.configuration-as-code</groupId>
+          <artifactId>test-harness</artifactId>
+          <scope>test</scope>
+        </dependency>
     </dependencies>
 </project>

src/main/java/org/jenkinsci/plugins/workflow/log/FileLogStorageFactory.java

@@ -0,0 +1,37 @@
+package org.jenkinsci.plugins.workflow.log;
+
+import edu.umd.cs.findbugs.annotations.NonNull;
+import hudson.Extension;
+import hudson.model.Descriptor;
+import java.io.File;
+import org.jenkinsci.Symbol;
+import org.jenkinsci.plugins.workflow.flow.FlowExecutionOwner;
+import org.kohsuke.accmod.Restricted;
+import org.kohsuke.accmod.restrictions.Beta;
+import org.kohsuke.stapler.DataBoundConstructor;
+
+@Restricted(Beta.class)
+public class FileLogStorageFactory implements LogStorageFactory {
+
+    @DataBoundConstructor
+    public FileLogStorageFactory() {}
+
+    @Override
+    public LogStorage forBuild(@NonNull FlowExecutionOwner b) {
+        try {
+            return FileLogStorage.forFile(new File(b.getRootDir(), "log"));
+        } catch (Exception x) {
+            return new BrokenLogStorage(x);
+        }
+    }
+
+    @Extension
+    @Symbol("file")
+    public static final class DescriptorImpl extends LogStorageFactoryDescriptor<FileLogStorageFactory> {
+        @NonNull
+        @Override
+        public String getDisplayName() {
+            return "Standard file logger";
+        }
+    }
+}

src/main/java/org/jenkinsci/plugins/workflow/log/LogStorage.java

@@ -25,7 +25,6 @@
 package org.jenkinsci.plugins.workflow.log;
 
 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
-import hudson.ExtensionList;
 import hudson.console.AnnotatedLargeText;
 import hudson.console.ConsoleAnnotationOutputStream;
 import hudson.model.BuildListener;
@@ -39,6 +38,7 @@
 import java.util.logging.Logger;
 import edu.umd.cs.findbugs.annotations.NonNull;
 import org.jenkinsci.plugins.workflow.actions.LogAction;
+import org.jenkinsci.plugins.workflow.log.configuration.PipelineLoggingGlobalConfiguration;
 import org.jenkinsci.plugins.workflow.flow.FlowExecutionOwner;
 import org.jenkinsci.plugins.workflow.graph.FlowNode;
 import org.jenkinsci.plugins.workflow.steps.StepContext;
@@ -160,8 +160,9 @@
      */
     static @NonNull LogStorage of(@NonNull FlowExecutionOwner b) {
         try {
-            for (LogStorageFactory factory : ExtensionList.lookup(LogStorageFactory.class)) {
-                LogStorage storage = factory.forBuild(b);
+            PipelineLoggingGlobalConfiguration config = PipelineLoggingGlobalConfiguration.get();
+            if (config.getFactoryOrDefault() != null) {
+                LogStorage storage = config.getFactoryOrDefault().forBuild(b);
                 if (storage != null) {
                     // Pending integration with JEP-207 / JEP-212, this choice is not persisted.
                     return storage;

src/main/java/org/jenkinsci/plugins/workflow/log/LogStorageFactory.java

@@ -24,9 +24,11 @@
 
 package org.jenkinsci.plugins.workflow.log;
 
-import hudson.ExtensionPoint;
 import edu.umd.cs.findbugs.annotations.CheckForNull;
 import edu.umd.cs.findbugs.annotations.NonNull;
+import hudson.model.Describable;
+import java.util.List;
+import jenkins.model.Jenkins;
 import org.jenkinsci.plugins.workflow.flow.FlowExecutionOwner;
 import org.kohsuke.accmod.Restricted;
 import org.kohsuke.accmod.restrictions.Beta;
@@ -35,7 +37,7 @@
  * Factory interface for {@link LogStorage}.
  */
 @Restricted(Beta.class)
-public interface LogStorageFactory extends ExtensionPoint {
+public interface LogStorageFactory extends Describable<LogStorageFactory> {
 
     /**
      * Checks whether we should handle a given build.
@@ -44,4 +46,24 @@ public interface LogStorageFactory extends ExtensionPoint {
      */
     @CheckForNull LogStorage forBuild(@NonNull FlowExecutionOwner b);
 
+    default LogStorageFactoryDescriptor<?> getDescriptor() {
+        return (LogStorageFactoryDescriptor<?>) Jenkins.get().getDescriptorOrDie(this.getClass());
+    }
+
+    static List<LogStorageFactoryDescriptor<?>> all() {
+        return Jenkins.get().getDescriptorList(LogStorageFactory.class);
+    }
+
+    /**
+     * Returns the default {@link LogStorageFactory} based on the descriptor {@code @Extension#ordinal} order and the {@link LogStorageFactoryDescriptor#getDefaultInstance()} implmentations.
+     */
+    static LogStorageFactory getDefaultFactory() {
+        for (LogStorageFactoryDescriptor<?> descriptor : all()) {
+            var instance = descriptor.getDefaultInstance();
+            if (instance != null) {
+                return instance;
+            }
+        }
+        return new FileLogStorageFactory();
+    }
 }

src/main/java/org/jenkinsci/plugins/workflow/log/LogStorageFactoryDescriptor.java

@@ -0,0 +1,26 @@
+package org.jenkinsci.plugins.workflow.log;
+
+import hudson.model.Descriptor;
+
+public abstract class LogStorageFactoryDescriptor<T extends LogStorageFactory> extends Descriptor<LogStorageFactory> {
+
+    /**
+     * States if the factory descriptor is configurable as primary TeeLogStorage.
+     */
+    public boolean isConfigurableAsPrimaryTeeLogStorageFactory() {
+        return true;
+    }
+    /**
+     * States if the factory descriptor is configurable as secondary TeeLogStorage.
+     */
+    public boolean isConfigurableAsSecondaryTeeLogStorageFactory() {
+        return true;
+    }
+
+    /**
+     * Allow to define the default factory instance to use if no configuration exists
+     */
+    public LogStorageFactory getDefaultInstance() {
+        return null;
+    }
+}

src/main/java/org/jenkinsci/plugins/workflow/log/configuration/PipelineLoggingGlobalConfiguration.java

@@ -0,0 +1,78 @@
+package org.jenkinsci.plugins.workflow.log.configuration;
+
+import edu.umd.cs.findbugs.annotations.NonNull;
+import hudson.Extension;
+import hudson.ExtensionList;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.logging.Logger;
+import jenkins.model.GlobalConfiguration;
+import jenkins.model.Jenkins;
+import net.sf.json.JSONObject;
+import org.jenkinsci.Symbol;
+import org.jenkinsci.plugins.workflow.log.LogStorageFactory;
+import org.jenkinsci.plugins.workflow.log.LogStorageFactoryDescriptor;
+import org.kohsuke.accmod.Restricted;
+import org.kohsuke.accmod.restrictions.Beta;
+import org.kohsuke.accmod.restrictions.NoExternalUse;
+import org.kohsuke.stapler.DataBoundSetter;
+import org.kohsuke.stapler.StaplerRequest2;
+
+@Extension
+@Symbol("pipelineLogging")
+@Restricted(Beta.class)
+public class PipelineLoggingGlobalConfiguration extends GlobalConfiguration {
+    private static final Logger LOGGER = Logger.getLogger(PipelineLoggingGlobalConfiguration.class.getName());
+    private LogStorageFactory factory;
+
+    public PipelineLoggingGlobalConfiguration() {
+        load();
+    }
+
+    /**
+     * For configuration only. Use {@link #getFactoryOrDefault()} instead.
+     */
+    @Restricted(NoExternalUse.class)
+    public LogStorageFactory getFactory() {
+        return factory;
+    }
+
+    @DataBoundSetter
+    public void setFactory(LogStorageFactory factory) {
+        this.factory = factory;
+        save();
+    }
+
+    @Override
+    public boolean configure(StaplerRequest2 req, JSONObject json) throws FormException {
+        this.factory = null;
+        return super.configure(req, json);
+    }
+
+    public LogStorageFactory getFactoryOrDefault() {
+        if (factory == null) {
+            return LogStorageFactory.getDefaultFactory();
+        }
+        return factory;
+    }
+
+    public List<LogStorageFactoryDescriptor<?>> getLogStorageFactoryDescriptors() {
+        List<LogStorageFactoryDescriptor<?>> result = new ArrayList<>();
+        result.add(null); // offer the option to use the default factory without any explicit configuration
+        result.addAll(LogStorageFactory.all());
+        return result;
+    }
+
+    public LogStorageFactoryDescriptor<?> getDefaultFactoryDescriptor() {
+        return LogStorageFactory.getDefaultFactory().getDescriptor();
+    }
+
+    public String getDefaultFactoryPlugin() {
+        var pluginWrapper = Jenkins.get().getPluginManager().whichPlugin(LogStorageFactory.getDefaultFactory().getClass());
+        return pluginWrapper != null ? pluginWrapper.getShortName() : "unknown";
+    }
+
+    public static PipelineLoggingGlobalConfiguration get() {
+        return ExtensionList.lookupSingleton(PipelineLoggingGlobalConfiguration.class);
+    }
+}

src/main/java/org/jenkinsci/plugins/workflow/log/tee/TeeBuildListener.java

@@ -0,0 +1,77 @@
+package org.jenkinsci.plugins.workflow.log.tee;
+
+import edu.umd.cs.findbugs.annotations.NonNull;
+import hudson.model.BuildListener;
+import hudson.model.TaskListener;
+import java.io.OutputStream;
+import java.io.Serial;
+import java.util.List;
+import org.jenkinsci.plugins.workflow.log.OutputStreamTaskListener;
+
+class TeeBuildListener extends OutputStreamTaskListener.Default
+        implements BuildListener, OutputStreamTaskListener, AutoCloseable {
+
+    @Serial
+    private static final long serialVersionUID = 1L;
+
+    private final TaskListener primary;
+
+    private final List<TaskListener> secondaries;
+
+    private transient OutputStream outputStream;
+
+    TeeBuildListener(TaskListener primary, TaskListener... secondaries) {
+        if (!(primary instanceof OutputStreamTaskListener)) {
+            throw new ClassCastException("Primary is not an instance of OutputStreamTaskListener: " + primary);
+        }
+        List.of(secondaries).forEach(secondary -> {
+            if (!(secondary instanceof OutputStreamTaskListener)) {
+                throw new ClassCastException("Secondary is not an instance of OutputStreamTaskListener: " + secondary);
+            }
+        });
+        this.primary = primary;
+        this.secondaries = List.of(secondaries);
+    }
+
+    @NonNull
+    @Override
+    public synchronized OutputStream getOutputStream() {
+        if (outputStream == null) {
+            outputStream = new TeeOutputStream(
+                    OutputStreamTaskListener.getOutputStream(primary),
+                    secondaries.stream()
+                            .map(OutputStreamTaskListener::getOutputStream)
+                            .toArray(OutputStream[]::new));
+        }
+        return outputStream;
+    }
+
+    @Override
+    public void close() throws Exception {
+        getLogger().close();
+        Exception exception = null;
+        if (primary instanceof AutoCloseable) {
+            try {
+                ((AutoCloseable) primary).close();
+            } catch (Exception e) {
+                exception = e;
+            }
+        }
+        for (TaskListener secondary : secondaries) {
+            if (secondary instanceof AutoCloseable) {
+                try {
+                    ((AutoCloseable) secondary).close();
+                } catch (Exception e) {
+                    if (exception == null) {
+                        exception = e;
+                    } else {
+                        exception.addSuppressed(e);
+                    }
+                }
+            }
+        }
+        if (exception != null) {
+            throw exception;
+        }
+    }
+}

src/main/java/org/jenkinsci/plugins/workflow/log/tee/TeeLogStorage.java

@@ -0,0 +1,73 @@
+package org.jenkinsci.plugins.workflow.log.tee;
+
+import edu.umd.cs.findbugs.annotations.NonNull;
+import hudson.console.AnnotatedLargeText;
+import hudson.model.BuildListener;
+import hudson.model.TaskListener;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+import org.jenkinsci.plugins.workflow.flow.FlowExecutionOwner;
+import org.jenkinsci.plugins.workflow.graph.FlowNode;
+import org.jenkinsci.plugins.workflow.log.LogStorage;
+import org.kohsuke.accmod.Restricted;
+import org.kohsuke.accmod.restrictions.Beta;
+
+/**
+ * Advancaed implementation of log storage allowing a primary log storage for read and write; and multiple secondary log storages for writes.
+ * This behaves as a tee execution.
+ */
+@Restricted(Beta.class)
+public class TeeLogStorage implements LogStorage {
+
+    LogStorage primary;
+    List<LogStorage> secondaries = List.of();
+
+    /**
+     * Log storage allowing a primary for read/write and multiple secondaries for write only
+     * @param primary primary log storage used for read and write
+     * @param secondaries secondary log storages used for write
+     */
+    public TeeLogStorage(@NonNull LogStorage primary, LogStorage... secondaries) {
+        this.primary = primary;
+        if (secondaries != null) {
+            this.secondaries =
+                    Arrays.stream(secondaries).filter(Objects::nonNull).toList();
+        }
+    }
+
+    @NonNull
+    @Override
+    public BuildListener overallListener() throws IOException, InterruptedException {
+        List<BuildListener> secondaryListeners = new ArrayList<>();
+        for (LogStorage secondary : secondaries) {
+            secondaryListeners.add(secondary.overallListener());
+        }
+        return new TeeBuildListener(primary.overallListener(), secondaryListeners.toArray(BuildListener[]::new));
+    }
+
+    @NonNull
+    @Override
+    public TaskListener nodeListener(@NonNull FlowNode node) throws IOException, InterruptedException {
+        List<TaskListener> secondaryListeners = new ArrayList<>();
+        for (LogStorage secondary : secondaries) {
+            secondaryListeners.add(secondary.nodeListener(node));
+        }
+        return new TeeBuildListener(primary.nodeListener(node), secondaryListeners.toArray(TaskListener[]::new));
+    }
+
+    @NonNull
+    @Override
+    public AnnotatedLargeText<FlowExecutionOwner.Executable> overallLog(
+            @NonNull FlowExecutionOwner.Executable build, boolean complete) {
+        return primary.overallLog(build, complete);
+    }
+
+    @NonNull
+    @Override
+    public AnnotatedLargeText<FlowNode> stepLog(@NonNull FlowNode node, boolean complete) {
+        return primary.stepLog(node, complete);
+    }
+}