diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0834364 --- /dev/null +++ b/.gitignore @@ -0,0 +1,15 @@ +target + +# mvn hpi:run +work + +# IntelliJ IDEA project files +*.iml +*.iws +*.ipr +.idea + +# Eclipse project files +.settings +.classpath +.project diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml new file mode 100644 index 0000000..510f24f --- /dev/null +++ b/.mvn/extensions.xml @@ -0,0 +1,7 @@ + + + io.jenkins.tools.incrementals + git-changelist-maven-extension + 1.0-beta-3 + + diff --git a/.mvn/maven.config b/.mvn/maven.config new file mode 100644 index 0000000..2a0299c --- /dev/null +++ b/.mvn/maven.config @@ -0,0 +1,2 @@ +-Pconsume-incrementals +-Pmight-produce-incrementals diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..5b60d43 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1 @@ +buildPlugin(platforms: ['linux']) diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..1f76f7c --- /dev/null +++ b/pom.xml @@ -0,0 +1,126 @@ + + + 4.0.0 + + + org.jenkins-ci.plugins + plugin + 3.18 + + + + io.jenkins.plugins.external-logging + external-logging-api + External Logging API plugin + The plugin provides API to simplify external logging implementations for Jenkins + https://wiki.jenkins.io/display/JENKINS/External+Logging+API+Plugin + ${revision}${changelist} + hpi + + + 1.0-alpha-1 + -SNAPSHOT + 2.135-rc15088.42aa6febbbed + 8 + true + + + + + MIT License + https://opensource.org/licenses/MIT + + + + + scm:git:git://github.com/jenkinsci/${project.artifactId}-plugin.git + scm:git:git@github.com:jenkinsci/${project.artifactId}-plugin.git + https://github.com/jenkinsci/${project.artifactId}-plugin + ${scmTag} + + + + + org.jenkins-ci.plugins + unique-id + 2.1.1 + + + org.jenkins-ci.plugins + mask-passwords + 2.12.0 + true + + + + org.jenkins-ci.plugins.workflow + workflow-job + 2.22-rc311.5616213fbed0 + + + org.jenkins-ci.plugins.workflow + workflow-support + 2.19-rc265.3e5e4aeecfff + + + org.jenkins-ci.plugins.workflow + workflow-api + 2.29-rc219.239019e84015 + + + + + org.jenkins-ci.plugins.workflow + workflow-durable-task-step + 2.20-rc333.74dc7c303e6d + test + + + org.jenkins-ci.plugins.workflow + workflow-cps + 2.19 + test + + + org.jenkins-ci.plugins.workflow + workflow-basic-steps + 2.2 + test + + + org.jenkins-ci.plugins.workflow + workflow-step-api + 2.15 + + + org.jenkins-ci.plugins + scm-api + 2.2.6 + + + org.jenkins-ci.plugins + script-security + 1.39 + + + org.jenkins-ci.plugins + credentials-binding + 1.15 + test + + + + + + repo.jenkins-ci.org + https://repo.jenkins-ci.org/public/ + + + + + repo.jenkins-ci.org + https://repo.jenkins-ci.org/public/ + + + + diff --git a/src/main/java/io/jenkins/plugins/extlogging/api/Event.java b/src/main/java/io/jenkins/plugins/extlogging/api/Event.java new file mode 100644 index 0000000..c6e8aac --- /dev/null +++ b/src/main/java/io/jenkins/plugins/extlogging/api/Event.java @@ -0,0 +1,54 @@ +package io.jenkins.plugins.extlogging.api; + +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; + +/** + * Stores events which can be sent over the channel. + * @author Oleg Nenashev + * @since TODO + */ +public class Event { + + final String message; + final long timestamp; + final long id; + + Map data = new HashMap<>(); + + public Event(long id, String message, long timestamp) { + this.id = id; + this.message = message; + this.timestamp = timestamp; + } + + public long getId() { + return id; + } + + public String getMessage() { + return message; + } + + public long getTimestamp() { + return timestamp; + } + + public Map getData() { + return data; + } + + @Override + public String toString() { + return String.format("[%d] - %s", timestamp, message); + } + + public void setData(Map data) { + this.data = data; + } + + public String toStringWithData() { + return String.format("[%d] - %s: %s", timestamp, message, data); + } +} diff --git a/src/main/java/io/jenkins/plugins/extlogging/api/ExternalLogBrowser.java b/src/main/java/io/jenkins/plugins/extlogging/api/ExternalLogBrowser.java new file mode 100644 index 0000000..c09e3dc --- /dev/null +++ b/src/main/java/io/jenkins/plugins/extlogging/api/ExternalLogBrowser.java @@ -0,0 +1,18 @@ +package io.jenkins.plugins.extlogging.api; + +import jenkins.model.logging.LogBrowser; +import jenkins.model.logging.Loggable; + +import javax.annotation.Nonnull; + +/** + * Base abstract class for External Log Browsers. + * @author Oleg Nenashev + * @since TODO + */ +public abstract class ExternalLogBrowser extends LogBrowser { + + public ExternalLogBrowser(@Nonnull Loggable loggable) { + super(loggable); + } +} diff --git a/src/main/java/io/jenkins/plugins/extlogging/api/ExternalLogBrowserFactory.java b/src/main/java/io/jenkins/plugins/extlogging/api/ExternalLogBrowserFactory.java new file mode 100644 index 0000000..6ded090 --- /dev/null +++ b/src/main/java/io/jenkins/plugins/extlogging/api/ExternalLogBrowserFactory.java @@ -0,0 +1,27 @@ +package io.jenkins.plugins.extlogging.api; + +import hudson.ExtensionPoint; +import hudson.model.Describable; +import hudson.model.Descriptor; +import jenkins.model.Jenkins; +import jenkins.model.logging.LogBrowser; +import jenkins.model.logging.Loggable; + +import javax.annotation.CheckForNull; + +/** + * @author Oleg Nenashev + * @since TODO + */ +public abstract class ExternalLogBrowserFactory + implements Describable, ExtensionPoint { + + @CheckForNull + public abstract LogBrowser create(Loggable loggable); + + @Override + public Descriptor getDescriptor() { + return Jenkins.get().getDescriptor(getClass()); + } + +} diff --git a/src/main/java/io/jenkins/plugins/extlogging/api/ExternalLogBrowserFactoryDescriptor.java b/src/main/java/io/jenkins/plugins/extlogging/api/ExternalLogBrowserFactoryDescriptor.java new file mode 100644 index 0000000..9c6be61 --- /dev/null +++ b/src/main/java/io/jenkins/plugins/extlogging/api/ExternalLogBrowserFactoryDescriptor.java @@ -0,0 +1,13 @@ +package io.jenkins.plugins.extlogging.api; + +import hudson.model.Descriptor; + +/** + * Descriptor for {@link ExternalLogBrowserFactory} + * @author Oleg Nenashev + * @since TODO + */ +public class ExternalLogBrowserFactoryDescriptor + extends Descriptor { + +} diff --git a/src/main/java/io/jenkins/plugins/extlogging/api/ExternalLoggingEventWriter.java b/src/main/java/io/jenkins/plugins/extlogging/api/ExternalLoggingEventWriter.java new file mode 100644 index 0000000..307baa3 --- /dev/null +++ b/src/main/java/io/jenkins/plugins/extlogging/api/ExternalLoggingEventWriter.java @@ -0,0 +1,47 @@ +package io.jenkins.plugins.extlogging.api; + +import java.io.IOException; +import java.io.Serializable; +import java.io.Writer; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Implements logging of events + * @author Oleg Nenashev + * @since TODO + */ +public abstract class ExternalLoggingEventWriter extends Writer implements Serializable { + + Map metadata = new HashMap<>(); + AtomicLong messageCounter = new AtomicLong(); + + public abstract void writeEvent(Event event) throws IOException; + + public void writeMessage(String message) throws IOException { + Event event = new Event(messageCounter.getAndIncrement(), message, System.currentTimeMillis()); + event.setData(metadata); // We do not copy the entry to save performance, custom implementations may need better logic + writeEvent(event); + } + + public void addMetadataEntry(String key, Serializable value) { + metadata.put(key, value); + } + + @Override + public void write(char[] cbuf, int off, int len) throws IOException { + String message = new String(cbuf, off, len); + writeMessage(message); + } + + @Override + public void close() throws IOException { + // noop + } + + @Override + public void flush() throws IOException { + // noop + } +} diff --git a/src/main/java/io/jenkins/plugins/extlogging/api/ExternalLoggingMethod.java b/src/main/java/io/jenkins/plugins/extlogging/api/ExternalLoggingMethod.java new file mode 100644 index 0000000..be82b04 --- /dev/null +++ b/src/main/java/io/jenkins/plugins/extlogging/api/ExternalLoggingMethod.java @@ -0,0 +1,133 @@ +package io.jenkins.plugins.extlogging.api; + +import hudson.Launcher; +import hudson.model.BuildListener; +import hudson.model.Node; +import hudson.model.Run; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Collections; +import java.util.List; + +import hudson.model.TaskListener; +import io.jenkins.plugins.extlogging.api.impl.ExternalLoggingBuildListener; +import io.jenkins.plugins.extlogging.api.impl.ExternalLoggingOutputStream; +import io.jenkins.plugins.extlogging.api.impl.ExternalLoggingLauncher; +import io.jenkins.plugins.extlogging.api.impl.LoggingThroughMasterOutputStreamWrapper; +import io.jenkins.plugins.extlogging.api.util.UniqueIdHelper; +import jenkins.model.Jenkins; +import jenkins.model.logging.Loggable; +import jenkins.model.logging.LoggingMethod; + +import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; + +/** + * Implements External logging method and simplifies API. + * @author Oleg Nenashev + * @since TODO + */ +public abstract class ExternalLoggingMethod extends LoggingMethod { + + public ExternalLoggingMethod(@Nonnull Loggable loggable) { + super(loggable); + } + + //TODO: implement + @CheckForNull + @Override + public TaskListener createTaskListener() { + return null; + } + + // TODO: Implement event-based logic instead of the + @Override + public BuildListener createBuildListener() throws IOException, InterruptedException { + return new ExternalLoggingBuildListener(createWriter()); + } + + public final OutputStream createOutputStream() throws IOException, InterruptedException { + final ExternalLoggingEventWriter writer = createWriter(); + + final List sensitiveStrings; + if (getOwner() instanceof Run) { + sensitiveStrings = SensitiveStringsProvider.getAllSensitiveStrings((Run)getOwner()); + } else { + sensitiveStrings = Collections.emptyList(); + } + return ExternalLoggingOutputStream.createOutputStream(writer, sensitiveStrings); + } + + //TODO: document null + /** + * Creates Remotable wrapper. + * By default, logging happens through master unless there is a custom implementation defined. + * @return Remotable wrapper + */ + @CheckForNull + public OutputStreamWrapper createWrapper() throws IOException, InterruptedException { + //TODO: capture agent in API to allow overrides with checks + return new LoggingThroughMasterOutputStreamWrapper(createOutputStream()); + } + + /** + * Produces an event writer for the object. + * This writer is always serializable, so that it can be used on recipient and sender sides. + * @return Event writer + */ + @Nonnull + public final ExternalLoggingEventWriter createWriter() throws IOException, InterruptedException { + ExternalLoggingEventWriter writer = _createWriter(); + // Produce universal metadata + writer.addMetadataEntry("jenkinsUrl", Jenkins.get().getRootUrl()); + if (loggable instanceof Run) { + Run run = (Run) loggable; + writer.addMetadataEntry("buildNum", run.getNumber()); + writer.addMetadataEntry("jobId", UniqueIdHelper.getOrCreateId(run.getParent())); + } + return writer; + } + + /** + * Produces a base event writer for the object. + * This method will be used by {@link #createWriter()}. + * @return Event writer + */ + @Nonnull + protected abstract ExternalLoggingEventWriter _createWriter() throws IOException, InterruptedException; + + @Nonnull + @Override + public Launcher decorateLauncher(@Nonnull Launcher original, + @Nonnull Run run, @Nonnull Node node) { + if (node instanceof Jenkins) { + return new ExternalLoggingLauncher.DefaultLocalLauncher(original); + } else { + return new ExternalLoggingLauncher.DefaultRemoteLauncher(original, this); + } + } + + /** + * Produces logging engine for STDOUT. + * It will be used in the {@link ExternalLoggingLauncher} + * @return Wrapper to be used. + * {@code null} will make the wrapper to use the default stream + */ + @CheckForNull + public OutputStreamWrapper provideRemotableOutStream() throws IOException, InterruptedException { + return createWrapper(); + } + + /* + * Produces logging engine for STDERR. + * It will be used in the {@link ExternalLoggingLauncher} + * @return Wrapper to be used. + * {@code null} will make the wrapper to use the default stream + */ + @CheckForNull + public OutputStreamWrapper provideRemotableErrStream() throws IOException, InterruptedException { + return createWrapper(); + } + +} \ No newline at end of file diff --git a/src/main/java/io/jenkins/plugins/extlogging/api/ExternalLoggingMethodFactory.java b/src/main/java/io/jenkins/plugins/extlogging/api/ExternalLoggingMethodFactory.java new file mode 100644 index 0000000..7241fcc --- /dev/null +++ b/src/main/java/io/jenkins/plugins/extlogging/api/ExternalLoggingMethodFactory.java @@ -0,0 +1,26 @@ +package io.jenkins.plugins.extlogging.api; + +import hudson.ExtensionPoint; +import hudson.model.Describable; +import hudson.model.Descriptor; +import jenkins.model.Jenkins; +import jenkins.model.logging.Loggable; + +import javax.annotation.CheckForNull; + +/** + * @author Oleg Nenashev + * @since TODO + */ +public abstract class ExternalLoggingMethodFactory + implements Describable, ExtensionPoint { + + @CheckForNull + public abstract ExternalLoggingMethod create(Loggable loggable); + + @Override + public Descriptor getDescriptor() { + return Jenkins.get().getDescriptor(getClass()); + } + +} diff --git a/src/main/java/io/jenkins/plugins/extlogging/api/ExternalLoggingMethodFactoryDescriptor.java b/src/main/java/io/jenkins/plugins/extlogging/api/ExternalLoggingMethodFactoryDescriptor.java new file mode 100644 index 0000000..3527acf --- /dev/null +++ b/src/main/java/io/jenkins/plugins/extlogging/api/ExternalLoggingMethodFactoryDescriptor.java @@ -0,0 +1,13 @@ +package io.jenkins.plugins.extlogging.api; + +import hudson.model.Descriptor; + +/** + * Descriptor for {@link ExternalLoggingMethodFactory} + * @author Oleg Nenashev + * @since TODO + */ +public class ExternalLoggingMethodFactoryDescriptor + extends Descriptor { + +} diff --git a/src/main/java/io/jenkins/plugins/extlogging/api/OutputStreamWrapper.java b/src/main/java/io/jenkins/plugins/extlogging/api/OutputStreamWrapper.java new file mode 100644 index 0000000..b6d2d88 --- /dev/null +++ b/src/main/java/io/jenkins/plugins/extlogging/api/OutputStreamWrapper.java @@ -0,0 +1,14 @@ +package io.jenkins.plugins.extlogging.api; + +import org.jenkinsci.remoting.SerializableOnlyOverRemoting; + +import java.io.OutputStream; + +public interface OutputStreamWrapper extends SerializableOnlyOverRemoting { + + /** + * Produces a serializable object which can be sent over the channel + * @return Serializable output stream, e.g. {@link hudson.remoting.RemoteOutputStream} + */ + OutputStream toSerializableOutputStream(); +} \ No newline at end of file diff --git a/src/main/java/io/jenkins/plugins/extlogging/api/SensitiveStringsProvider.java b/src/main/java/io/jenkins/plugins/extlogging/api/SensitiveStringsProvider.java new file mode 100644 index 0000000..9de0cfc --- /dev/null +++ b/src/main/java/io/jenkins/plugins/extlogging/api/SensitiveStringsProvider.java @@ -0,0 +1,38 @@ +package io.jenkins.plugins.extlogging.api; + +import hudson.ExtensionList; +import hudson.ExtensionPoint; +import hudson.model.Run; + +import javax.annotation.Nonnull; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +/** + * Provides a list of sensitive variables, which should be hidden from console. + * @author Oleg Nenashev + * @since TODO + */ +public abstract class SensitiveStringsProvider implements ExtensionPoint { + + public abstract void getSensitiveStrings(@Nonnull Run run, List dest); + + public static List getAllSensitiveStrings(@Nonnull Run run) { + final ExtensionList all = all(); + if (all.isEmpty()) { + return Collections.emptyList(); + } + + ArrayList res = new ArrayList<>(); + for (SensitiveStringsProvider provider : all) { + provider.getSensitiveStrings(run, res); + } + return res; + } + + public static ExtensionList all() { + return ExtensionList.lookup(SensitiveStringsProvider.class); + } +} diff --git a/src/main/java/io/jenkins/plugins/extlogging/api/impl/DisabledExternalLogBrowserFactory.java b/src/main/java/io/jenkins/plugins/extlogging/api/impl/DisabledExternalLogBrowserFactory.java new file mode 100644 index 0000000..626d679 --- /dev/null +++ b/src/main/java/io/jenkins/plugins/extlogging/api/impl/DisabledExternalLogBrowserFactory.java @@ -0,0 +1,37 @@ +package io.jenkins.plugins.extlogging.api.impl; + +import hudson.Extension; +import io.jenkins.plugins.extlogging.api.ExternalLogBrowserFactory; +import io.jenkins.plugins.extlogging.api.ExternalLogBrowserFactoryDescriptor; +import jenkins.model.logging.LogBrowser; +import jenkins.model.logging.Loggable; +import org.jenkinsci.Symbol; +import org.kohsuke.stapler.DataBoundConstructor; + +/** + * Default Disabled implementation. + * @author Oleg Nenashev + * @since TODO + */ +public class DisabledExternalLogBrowserFactory extends ExternalLogBrowserFactory { + + @DataBoundConstructor + public DisabledExternalLogBrowserFactory() { + + } + + @Override + public LogBrowser create(Loggable loggable) { + return null; + } + + @Extension(ordinal = Float.MAX_VALUE) + @Symbol("disabled") + public static class DescriptorImpl extends ExternalLogBrowserFactoryDescriptor { + + @Override + public String getDisplayName() { + return "Disabled"; + } + } +} diff --git a/src/main/java/io/jenkins/plugins/extlogging/api/impl/DisabledExternalLoggingMethodFactory.java b/src/main/java/io/jenkins/plugins/extlogging/api/impl/DisabledExternalLoggingMethodFactory.java new file mode 100644 index 0000000..9536862 --- /dev/null +++ b/src/main/java/io/jenkins/plugins/extlogging/api/impl/DisabledExternalLoggingMethodFactory.java @@ -0,0 +1,36 @@ +package io.jenkins.plugins.extlogging.api.impl; + +import hudson.Extension; +import io.jenkins.plugins.extlogging.api.ExternalLoggingMethod; +import io.jenkins.plugins.extlogging.api.ExternalLoggingMethodFactory; +import io.jenkins.plugins.extlogging.api.ExternalLoggingMethodFactoryDescriptor; +import jenkins.model.logging.Loggable; +import org.jenkinsci.Symbol; +import org.kohsuke.stapler.DataBoundConstructor; + +/** + * @author Oleg Nenashev + * @since TODO + */ +public class DisabledExternalLoggingMethodFactory extends ExternalLoggingMethodFactory { + + @DataBoundConstructor + public DisabledExternalLoggingMethodFactory() { + + } + + @Override + public ExternalLoggingMethod create(Loggable loggable) { + return null; + } + + @Extension(ordinal = Float.MAX_VALUE) + @Symbol("disabled") + public static final class DescriptorImpl extends ExternalLoggingMethodFactoryDescriptor { + + @Override + public String getDisplayName() { + return "disabled"; + } + } +} diff --git a/src/main/java/io/jenkins/plugins/extlogging/api/impl/ExternalLoggingBuildListener.java b/src/main/java/io/jenkins/plugins/extlogging/api/impl/ExternalLoggingBuildListener.java new file mode 100644 index 0000000..bf024e5 --- /dev/null +++ b/src/main/java/io/jenkins/plugins/extlogging/api/impl/ExternalLoggingBuildListener.java @@ -0,0 +1,24 @@ +package io.jenkins.plugins.extlogging.api.impl; + +import hudson.model.BuildListener; +import io.jenkins.plugins.extlogging.api.ExternalLoggingEventWriter; + +import java.io.PrintStream; + +/** + * @author Oleg Nenashev + * @since TODO + */ +public class ExternalLoggingBuildListener implements BuildListener { + + private final ExternalLoggingEventWriter writer; + + public ExternalLoggingBuildListener(ExternalLoggingEventWriter writer) { + this.writer = writer; + } + + @Override + public PrintStream getLogger() { + return new PrintStream(new ExternalLoggingOutputStream(writer)); + } +} diff --git a/src/main/java/io/jenkins/plugins/extlogging/api/impl/ExternalLoggingGlobalConfiguration.java b/src/main/java/io/jenkins/plugins/extlogging/api/impl/ExternalLoggingGlobalConfiguration.java new file mode 100644 index 0000000..411354f --- /dev/null +++ b/src/main/java/io/jenkins/plugins/extlogging/api/impl/ExternalLoggingGlobalConfiguration.java @@ -0,0 +1,70 @@ +package io.jenkins.plugins.extlogging.api.impl; + +import hudson.Extension; +import io.jenkins.plugins.extlogging.api.ExternalLogBrowserFactory; +import io.jenkins.plugins.extlogging.api.ExternalLoggingMethodFactory; +import jenkins.model.GlobalConfiguration; +import org.jenkinsci.Symbol; +import org.kohsuke.stapler.DataBoundSetter; + +import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; + +//TODO: support global configuration +/** + * @author Oleg Nenashev + * @since TODO + */ +@Extension +@Symbol("externalLogging") +public class ExternalLoggingGlobalConfiguration extends GlobalConfiguration { + + private static final ExternalLoggingGlobalConfiguration DEFAULT = new DefaultExternalLoggingGlobalConfiguration(); + + @CheckForNull + private ExternalLoggingMethodFactory loggingMethod; + + @CheckForNull + private ExternalLogBrowserFactory logBrowser; + + @Nonnull + public static ExternalLoggingGlobalConfiguration getInstance() { + ExternalLoggingGlobalConfiguration cfg = GlobalConfiguration.all().get(ExternalLoggingGlobalConfiguration.class); + assert cfg != null : "Global configuration should be present"; + return cfg != null ? cfg : DEFAULT; + } + + public ExternalLoggingGlobalConfiguration() { + load(); + } + + @DataBoundSetter + public void setLogBrowser(@CheckForNull ExternalLogBrowserFactory logBrowser) { + this.logBrowser = logBrowser; + this.save(); + } + + @DataBoundSetter + public void setLoggingMethod(@CheckForNull ExternalLoggingMethodFactory loggingMethod) { + this.loggingMethod = loggingMethod; + this.save(); + } + + @CheckForNull + public ExternalLogBrowserFactory getLogBrowser() { + return logBrowser; + } + + @CheckForNull + public ExternalLoggingMethodFactory getLoggingMethod() { + return loggingMethod; + } + + private static final class DefaultExternalLoggingGlobalConfiguration extends ExternalLoggingGlobalConfiguration { + + public DefaultExternalLoggingGlobalConfiguration() { + // prevent config loading + } + + } +} diff --git a/src/main/java/io/jenkins/plugins/extlogging/api/impl/ExternalLoggingLauncher.java b/src/main/java/io/jenkins/plugins/extlogging/api/impl/ExternalLoggingLauncher.java new file mode 100644 index 0000000..7995edb --- /dev/null +++ b/src/main/java/io/jenkins/plugins/extlogging/api/impl/ExternalLoggingLauncher.java @@ -0,0 +1,175 @@ +/* + * The MIT License + * + * Copyright (c) 2016 Oleg Nenashev. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package io.jenkins.plugins.extlogging.api.impl; + +import hudson.Launcher; +import hudson.Proc; +import hudson.model.TaskListener; +import hudson.remoting.Channel; +import hudson.remoting.RemoteInputStream; +import io.jenkins.plugins.extlogging.api.ExternalLoggingMethod; +import io.jenkins.plugins.extlogging.api.OutputStreamWrapper; +import jenkins.model.logging.LoggingMethod; +import jenkins.security.MasterToSlaveCallable; +import org.apache.commons.io.input.NullInputStream; +import org.kohsuke.accmod.Restricted; +import org.kohsuke.accmod.restrictions.Beta; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InterruptedIOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; + +/** + * Provides {@link Launcher} implementations for External Logging. + * These implementations plug-in {@link ExternalLoggingMethod} into the on-agent executions. + * @author Oleg Nenashev + */ +@Restricted(Beta.class) +public class ExternalLoggingLauncher { + + /** + * Default local launcher, doesn't do anything. + */ + public static class DefaultLocalLauncher extends Launcher.DecoratedLauncher { + public DefaultLocalLauncher(Launcher inner) { + super(inner); + } + } + + /** + * Default remote launcher which redirects all the output and error to the stream {@link LoggingMethod} provides. + */ + public static class DefaultRemoteLauncher extends Launcher.DecoratedLauncher { + private static final NullInputStream NULL_INPUT_STREAM = new NullInputStream(0); + private final ExternalLoggingMethod loggingMethod; + + public DefaultRemoteLauncher(Launcher inner, ExternalLoggingMethod loggingMethod) { + super(inner); + this.loggingMethod = loggingMethod; + } + + @Override + public Proc launch(Launcher.ProcStarter ps) throws IOException { + final OutputStreamWrapper streamOut, streamErr; + try { + streamOut = loggingMethod.provideRemotableOutStream(); + streamErr = loggingMethod.provideRemotableErrStream(); + } catch (InterruptedException ex) { + throw new IOException(ex); + } + + // RemoteLogstashReporterStream(new CloseProofOutputStream(ps.stdout() + final OutputStream out = ps.stdout() == null ? null : (streamOut == null ? ps.stdout() : streamOut.toSerializableOutputStream()); + final OutputStream err = ps.stdout() == null ? null : (streamErr == null ? ps.stdout() : streamErr.toSerializableOutputStream()); + final InputStream in = (ps.stdin() == null || ps.stdin() == NULL_INPUT_STREAM) ? null : new RemoteInputStream(ps.stdin(), false); + final String workDir = ps.pwd() == null ? null : ps.pwd().getRemote(); + + // TODO: we do not reverse streams => the parameters + try { + final RemoteLaunchCallable callable = new RemoteLaunchCallable( + ps.cmds(), ps.masks(), ps.envs(), in, + out, err, ps.quiet(), workDir, listener); + + return new Launcher.RemoteLauncher.ProcImpl(getChannel().call(callable)); + } catch (InterruptedException e) { + throw (IOException) new InterruptedIOException().initCause(e); + } + } + + private static class RemoteLaunchCallable extends + MasterToSlaveCallable { + + private final List cmd; + private final boolean[] masks; + private final String[] env; + private final InputStream in; + private final OutputStream out; + private final OutputStream err; + private final String workDir; + private final TaskListener listener; + private final boolean quiet; + + RemoteLaunchCallable(List cmd, boolean[] masks, String[] env, InputStream in, OutputStream out, OutputStream err, boolean quiet, String workDir, TaskListener listener) { + this.cmd = new ArrayList<>(cmd); + this.masks = masks; + this.env = env; + this.in = in; + this.out = out; + this.err = err; + this.workDir = workDir; + this.listener = listener; + this.quiet = quiet; + } + + public Launcher.RemoteProcess call() throws IOException { + Launcher.ProcStarter ps = new Launcher.LocalLauncher(listener).launch(); + ps.cmds(cmd).masks(masks).envs(env).stdin(in).stdout(out).stderr(err).quiet(quiet); + if (workDir != null) { + ps.pwd(workDir); + } + + final Proc p = ps.start(); + + return Channel.current().export(Launcher.RemoteProcess.class, new Launcher.RemoteProcess() { + public int join() throws InterruptedException, IOException { + try { + return p.join(); + } finally { + // make sure I/O is delivered to the remote before we return + try { + Channel.current().syncIO(); + } catch (Throwable th) { + // this includes a failure to sync, slave.jar too old, etc + } + } + } + + public void kill() throws IOException, InterruptedException { + p.kill(); + } + + public boolean isAlive() throws IOException, InterruptedException { + return p.isAlive(); + } + + public Launcher.IOTriplet getIOtriplet() { + Launcher.IOTriplet r = new Launcher.IOTriplet(); + /* TODO: we do not need reverse? + if (reverseStdout) r.stdout = new RemoteInputStream(p.getStdout()); + if (reverseStderr) r.stderr = new RemoteInputStream(p.getStderr()); + if (reverseStdin) r.stdin = new RemoteOutputStream(p.getStdin()); + */ + return r; + + } + }); + } + + private static final long serialVersionUID = 1L; + } + } +} diff --git a/src/main/java/io/jenkins/plugins/extlogging/api/impl/ExternalLoggingMethodLocator.java b/src/main/java/io/jenkins/plugins/extlogging/api/impl/ExternalLoggingMethodLocator.java new file mode 100644 index 0000000..f8f6bce --- /dev/null +++ b/src/main/java/io/jenkins/plugins/extlogging/api/impl/ExternalLoggingMethodLocator.java @@ -0,0 +1,40 @@ +package io.jenkins.plugins.extlogging.api.impl; + +import hudson.Extension; +import io.jenkins.plugins.extlogging.api.ExternalLogBrowserFactory; +import io.jenkins.plugins.extlogging.api.ExternalLoggingMethodFactory; +import jenkins.model.logging.LogBrowser; +import jenkins.model.logging.Loggable; +import jenkins.model.logging.LoggingMethod; +import jenkins.model.logging.LoggingMethodLocator; + +import javax.annotation.CheckForNull; + +/** + * Locator which provides logging nethods and browsers from {@link ExternalLoggingGlobalConfiguration}. + * @author Oleg Nenashev + * @see ExternalLoggingGlobalConfiguration + */ +@Extension +public class ExternalLoggingMethodLocator extends LoggingMethodLocator { + + @CheckForNull + @Override + protected LoggingMethod getLoggingMethod(Loggable loggable) { + ExternalLoggingMethodFactory factory = ExternalLoggingGlobalConfiguration.getInstance().getLoggingMethod(); + if (factory != null) { + return factory.create(loggable); + } + return null; + } + + @CheckForNull + @Override + protected LogBrowser getLogBrowser(Loggable loggable) { + ExternalLogBrowserFactory factory = ExternalLoggingGlobalConfiguration.getInstance().getLogBrowser(); + if (factory != null) { + return factory.create(loggable); + } + return null; + } +} diff --git a/src/main/java/io/jenkins/plugins/extlogging/api/impl/ExternalLoggingOutputStream.java b/src/main/java/io/jenkins/plugins/extlogging/api/impl/ExternalLoggingOutputStream.java new file mode 100644 index 0000000..07f451a --- /dev/null +++ b/src/main/java/io/jenkins/plugins/extlogging/api/impl/ExternalLoggingOutputStream.java @@ -0,0 +1,50 @@ +package io.jenkins.plugins.extlogging.api.impl; + +import hudson.console.LineTransformationOutputStream; +import io.jenkins.plugins.extlogging.api.ExternalLoggingEventWriter; +import io.jenkins.plugins.extlogging.api.util.MaskSecretsOutputStream; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.Collection; +import java.util.Collections; + +/** + * @author Oleg Nenashev + * @since TODO + */ +public class ExternalLoggingOutputStream extends LineTransformationOutputStream { + + ExternalLoggingEventWriter writer; + + public ExternalLoggingOutputStream(ExternalLoggingEventWriter writer) { + this.writer = writer; + } + + @Override protected void eol(byte[] b, int len) throws IOException { + int eol = len; + while (eol > 0) { + byte c = b[eol - 1]; + if (c == '\n' || c == '\r') { + eol--; + } else { + break; + } + } + String message = new String(b, 0, eol, StandardCharsets.UTF_8); + writer.writeMessage(message); + } + + public static OutputStream createOutputStream(ExternalLoggingEventWriter writer, Collection sensitiveStrings) { + if (sensitiveStrings.isEmpty()) { + return new ExternalLoggingOutputStream(writer); + } else { + return new MaskSecretsOutputStream( + new ExternalLoggingOutputStream(writer), + sensitiveStrings, + Collections.emptyList() + ); + } + } +} diff --git a/src/main/java/io/jenkins/plugins/extlogging/api/impl/LoggingThroughMasterOutputStreamWrapper.java b/src/main/java/io/jenkins/plugins/extlogging/api/impl/LoggingThroughMasterOutputStreamWrapper.java new file mode 100644 index 0000000..316f11b --- /dev/null +++ b/src/main/java/io/jenkins/plugins/extlogging/api/impl/LoggingThroughMasterOutputStreamWrapper.java @@ -0,0 +1,27 @@ +package io.jenkins.plugins.extlogging.api.impl; + +import hudson.remoting.RemoteOutputStream; +import io.jenkins.plugins.extlogging.api.OutputStreamWrapper; + +import java.io.OutputStream; + +/** + * Default {@link OutputStreamWrapper} implementation, which sends logs through the master. + * This wrapper can be used, when {@link io.jenkins.plugins.extlogging.api.ExternalLoggingMethod} + * does not support logging from agents. + * @author Oleg Nenashev + * @since TODO + */ +public class LoggingThroughMasterOutputStreamWrapper implements OutputStreamWrapper { + + private final OutputStream ostream; + + public LoggingThroughMasterOutputStreamWrapper(OutputStream stream) { + this.ostream = stream; + } + + @Override + public OutputStream toSerializableOutputStream() { + return new RemoteOutputStream(ostream); + } +} diff --git a/src/main/java/io/jenkins/plugins/extlogging/api/integrations/MaskPasswordsSensitiveStringsProvider/MaskSensitiveStringsProvider.java b/src/main/java/io/jenkins/plugins/extlogging/api/integrations/MaskPasswordsSensitiveStringsProvider/MaskSensitiveStringsProvider.java new file mode 100644 index 0000000..4b116a7 --- /dev/null +++ b/src/main/java/io/jenkins/plugins/extlogging/api/integrations/MaskPasswordsSensitiveStringsProvider/MaskSensitiveStringsProvider.java @@ -0,0 +1,67 @@ +package io.jenkins.plugins.extlogging.api.integrations.MaskPasswordsSensitiveStringsProvider; + +import com.michelin.cio.hudson.plugins.maskpasswords.MaskPasswordsBuildWrapper; +import com.michelin.cio.hudson.plugins.maskpasswords.MaskPasswordsConfig; +import hudson.Extension; +import hudson.model.BuildableItemWithBuildWrappers; +import hudson.model.Job; +import hudson.model.Run; +import hudson.tasks.BuildWrapper; +import io.jenkins.plugins.extlogging.api.SensitiveStringsProvider; +import org.kohsuke.accmod.Restricted; +import org.kohsuke.accmod.restrictions.NoExternalUse; + +import javax.annotation.Nonnull; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * @author Oleg Nenashev + * @since TODO + */ +@Extension(optional = true) +public class MaskSensitiveStringsProvider extends SensitiveStringsProvider { + + private static final Logger LOGGER = Logger.getLogger(MaskSensitiveStringsProvider.class.getName()); + + static { + String wrapperName = MaskPasswordsBuildWrapper.class.getName(); + LOGGER.log(Level.FINEST, "Initialized SensitiveStringsProvider for {0}", wrapperName); + } + + @Override + public void getSensitiveStrings(@Nonnull Run run, List dest) { + for (MaskPasswordsBuildWrapper.VarPasswordPair pair : getVarPasswordPairs(run)) { + dest.add(pair.getPassword()); + } + } + + @Restricted(NoExternalUse.class) + public static List getVarPasswordPairs(Run build) { + List allPasswordPairs = new ArrayList<>(); + Job job = build.getParent(); + if (job instanceof BuildableItemWithBuildWrappers) { + BuildableItemWithBuildWrappers project = (BuildableItemWithBuildWrappers) job; + for (BuildWrapper wrapper : project.getBuildWrappersList()) { + if (wrapper instanceof MaskPasswordsBuildWrapper) { + MaskPasswordsBuildWrapper maskPasswordsWrapper = (MaskPasswordsBuildWrapper) wrapper; + List jobPasswordPairs = maskPasswordsWrapper.getVarPasswordPairs(); + if (jobPasswordPairs != null) { + allPasswordPairs.addAll(jobPasswordPairs); + } + + MaskPasswordsConfig config = MaskPasswordsConfig.getInstance(); + List globalPasswordPairs = config.getGlobalVarPasswordPairs(); + if (globalPasswordPairs != null) { + allPasswordPairs.addAll(globalPasswordPairs); + } + + return allPasswordPairs; + } + } + } + return allPasswordPairs; + } +} diff --git a/src/main/java/io/jenkins/plugins/extlogging/api/integrations/pipeline/ExternalPipelineLogStorage.java b/src/main/java/io/jenkins/plugins/extlogging/api/integrations/pipeline/ExternalPipelineLogStorage.java new file mode 100644 index 0000000..f4875bc --- /dev/null +++ b/src/main/java/io/jenkins/plugins/extlogging/api/integrations/pipeline/ExternalPipelineLogStorage.java @@ -0,0 +1,119 @@ +/* + * The MIT License + * + * Copyright 2016-2018 CloudBees, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package io.jenkins.plugins.extlogging.api.integrations.pipeline; + + +import hudson.console.AnnotatedLargeText; +import hudson.model.BuildListener; +import hudson.model.TaskListener; +import io.jenkins.plugins.extlogging.api.ExternalLoggingEventWriter; +import io.jenkins.plugins.extlogging.api.ExternalLoggingMethod; +import io.jenkins.plugins.extlogging.api.impl.ExternalLoggingOutputStream; +import io.jenkins.plugins.extlogging.api.SensitiveStringsProvider; +import jenkins.model.logging.LogBrowser; +import org.jenkinsci.plugins.workflow.flow.FlowExecutionOwner; +import org.jenkinsci.plugins.workflow.graph.FlowNode; +import org.jenkinsci.plugins.workflow.job.WorkflowRun; +import org.jenkinsci.plugins.workflow.log.LogStorage; + +import javax.annotation.Nonnull; +import java.io.IOException; +import java.io.PrintStream; +import java.util.Collection; +import java.util.logging.Logger; + + +/** + * Integrates remote logging with Pipeline builds using an experimental API. + */ +public class ExternalPipelineLogStorage implements LogStorage { + + private static final Logger LOGGER = + Logger.getLogger(ExternalPipelineLogStorage.class.getName()); + + private final ExternalLoggingMethod lm; + private final LogBrowser logBrowser; + private final WorkflowRun run; + + ExternalPipelineLogStorage(@Nonnull WorkflowRun run, @Nonnull ExternalLoggingMethod lm, @Nonnull LogBrowser browser) { + this.run = run; + this.lm = lm; + this.logBrowser = browser; + } + + @Nonnull + @Override + public BuildListener overallListener() throws IOException, InterruptedException { + return new PipelineListener(run, lm); + } + + @Nonnull + @Override + public TaskListener nodeListener(@Nonnull FlowNode flowNode) throws IOException, InterruptedException { + return new PipelineListener(run, flowNode, lm); + } + + @Nonnull + @Override + public AnnotatedLargeText overallLog(@Nonnull FlowExecutionOwner.Executable executable, boolean completed) { + //TODO: Handle executable? Why is it needed at all? + return logBrowser.overallLog(); + } + + @Nonnull + @Override + public AnnotatedLargeText stepLog(@Nonnull FlowNode flowNode, boolean completed) { + return logBrowser.stepLog(flowNode.getId(), completed); + } + + private static class PipelineListener implements BuildListener { + + private static final long serialVersionUID = 1; + + private final Collection sensitiveStrings; + private final ExternalLoggingEventWriter writer; + private transient PrintStream logger; + + PipelineListener(WorkflowRun run, ExternalLoggingMethod method) throws IOException, InterruptedException { + this.writer = method.createWriter(); + this.sensitiveStrings = SensitiveStringsProvider.getAllSensitiveStrings(run); + } + + PipelineListener(WorkflowRun run, FlowNode node, ExternalLoggingMethod method) throws IOException, InterruptedException { + this(run, method); + writer.addMetadataEntry("stepId", node.getId()); + } + + @Override + public PrintStream getLogger() { + if (logger == null) { + logger = new PrintStream(ExternalLoggingOutputStream.createOutputStream(writer, sensitiveStrings)); + } + return logger; + } + + } + +} diff --git a/src/main/java/io/jenkins/plugins/extlogging/api/integrations/pipeline/ExternalPipelineLogStorageFactory.java b/src/main/java/io/jenkins/plugins/extlogging/api/integrations/pipeline/ExternalPipelineLogStorageFactory.java new file mode 100644 index 0000000..5005715 --- /dev/null +++ b/src/main/java/io/jenkins/plugins/extlogging/api/integrations/pipeline/ExternalPipelineLogStorageFactory.java @@ -0,0 +1,67 @@ +package io.jenkins.plugins.extlogging.api.integrations.pipeline; + +import hudson.Extension; +import hudson.model.Queue; +import hudson.model.Run; +import io.jenkins.plugins.extlogging.api.ExternalLoggingMethod; +import io.jenkins.plugins.extlogging.api.integrations.MaskPasswordsSensitiveStringsProvider.MaskSensitiveStringsProvider; +import jenkins.model.logging.LogBrowser; +import jenkins.model.logging.LoggingMethod; +import jenkins.model.logging.LoggingMethodLocator; +import org.jenkinsci.plugins.workflow.flow.FlowExecutionOwner; +import org.jenkinsci.plugins.workflow.job.WorkflowRun; +import org.jenkinsci.plugins.workflow.log.LogStorage; +import org.jenkinsci.plugins.workflow.log.LogStorageFactory; +import org.kohsuke.accmod.Restricted; +import org.kohsuke.accmod.restrictions.NoExternalUse; + +import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; +import java.io.IOException; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Bridges Jenkins Core external logging API and Pipeline external logging API. + * @author Oleg Nenashev + * @since TODO + */ +@Restricted(NoExternalUse.class) +@Extension(optional = true) +public class ExternalPipelineLogStorageFactory implements LogStorageFactory { + + private static final Logger LOGGER = Logger.getLogger(MaskSensitiveStringsProvider.class.getName()); + + @CheckForNull + @Override + public LogStorage forBuild(@Nonnull FlowExecutionOwner flowExecutionOwner) { + final WorkflowRun run; + try { + final Queue.Executable executable = flowExecutionOwner.getExecutable(); + if (executable instanceof Run) { + Run r = (Run)executable; + if (r instanceof WorkflowRun) { + run = (WorkflowRun) r; + } else { + return null; + } + } else { + return null; + } + } catch (IOException ex) { + LOGGER.log(Level.WARNING, "Failed to locate executable for the execution owner " + flowExecutionOwner, ex); + return null; + } + + final LoggingMethod loggingMethod = LoggingMethodLocator.locate(run); + final LogBrowser browser = LoggingMethodLocator.locateBrowser(run); + + if (loggingMethod instanceof ExternalLoggingMethod) { + return new ExternalPipelineLogStorage(run, + (ExternalLoggingMethod) loggingMethod, + browser); + } + + return null; + } +} diff --git a/src/main/java/io/jenkins/plugins/extlogging/api/util/AbstractConsoleAction.java b/src/main/java/io/jenkins/plugins/extlogging/api/util/AbstractConsoleAction.java new file mode 100644 index 0000000..933c701 --- /dev/null +++ b/src/main/java/io/jenkins/plugins/extlogging/api/util/AbstractConsoleAction.java @@ -0,0 +1,37 @@ +package io.jenkins.plugins.extlogging.api.util; + +import hudson.model.Action; +import hudson.model.Run; + +/** + * Wrapper base, which is required to nest {@code buildCaption.jelly}. + * @author Oleg Nenashev + */ +public abstract class AbstractConsoleAction implements Action { + private final String jobId; + private final Run run; + + public AbstractConsoleAction(Run run) { + this.run = run; + this.jobId = UniqueIdHelper.getOrCreateId(run); + } + + @Override + public String getDisplayName() { + return "External log (" + getDataSourceDisplayName() + ")"; + } + + public Run getRun() { + return run; + } + + public String getJobId() { + return jobId; + } + + public boolean isLogUpdated() { + return run.isLogUpdated(); + } + + public abstract String getDataSourceDisplayName(); +} diff --git a/src/main/java/io/jenkins/plugins/extlogging/api/util/MaskSecretsOutputStream.java b/src/main/java/io/jenkins/plugins/extlogging/api/util/MaskSecretsOutputStream.java new file mode 100644 index 0000000..9ef713f --- /dev/null +++ b/src/main/java/io/jenkins/plugins/extlogging/api/util/MaskSecretsOutputStream.java @@ -0,0 +1,152 @@ +/* + * The MIT License + * + * Copyright (c) 2010-2011, Manufacture Francaise des Pneumatiques Michelin, + * Romain Seguy + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package io.jenkins.plugins.extlogging.api.util; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import hudson.console.LineTransformationOutputStream; +import org.apache.commons.lang.StringUtils; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.Collection; +import java.util.regex.Pattern; +import javax.annotation.CheckForNull; + +// Copied from the MaskPasswords plugin +/** + * Custom output stream which masks a predefined set of passwords. + * + * @author Romain Seguy (http://openromain.blogspot.com) + */ +public class MaskSecretsOutputStream extends LineTransformationOutputStream { + + private final static String MASKED_PASSWORD = "********"; + + private final OutputStream logger; + private final Pattern passwordsAsPattern; + + /** + * @param logger The output stream to which this {@link MaskSecretsOutputStream} + * will write to + * @param secrets A collection of {@link String}s to be masked + * @param regexes A collection of Regular Expression {@link String}s to be masked + */ + public MaskSecretsOutputStream(OutputStream logger, @CheckForNull Collection secrets, @CheckForNull Collection regexes) { + this.logger = logger; + + + if((secrets != null && secrets.size() > 0) || (regexes != null && regexes.size() > 0)) { + // passwords are aggregated into a regex which is compiled as a pattern + // for efficiency + StringBuilder regex = new StringBuilder().append('('); + + int nbMaskedPasswords = 0; + + if(secrets != null && secrets.size() > 0) { + for(String password: secrets) { + if(StringUtils.isNotEmpty(password)) { // we must not handle empty passwords + regex.append(Pattern.quote(password)); + regex.append('|'); + try { + String encodedPassword = URLEncoder.encode(password, "UTF-8"); + if (!encodedPassword.equals(password)) { + // add to masking regex + regex.append(Pattern.quote(encodedPassword)); + regex.append('|'); + } + } catch (UnsupportedEncodingException e) { + // ignore any encoding problem => status quo + } + nbMaskedPasswords++; + } + } + } + if(regexes != null && regexes.size() > 0) { + for(String user_regex: regexes) { + if(StringUtils.isNotEmpty(user_regex)) { // we must not handle empty passwords + regex.append(user_regex); + regex.append('|'); + nbMaskedPasswords++; + } + } + } + + if(nbMaskedPasswords++ >= 1) { // is there at least one password to mask? + regex.deleteCharAt(regex.length()-1); // removes the last unuseful pipe + regex.append(')'); + passwordsAsPattern = Pattern.compile(regex.toString()); + } + else { // no passwords to hide + passwordsAsPattern = null; + } + } + else { // no passwords to hide + passwordsAsPattern = null; + } + } + + /** + * @param logger The output stream to which this {@link MaskSecretsOutputStream} + * will write to + * @param passwords A collection of {@link String}s to be masked + */ + public MaskSecretsOutputStream(OutputStream logger, @CheckForNull Collection passwords) { + this(logger, passwords, null); + } + + // TODO: The logic relies on the default encoding, which may cause issues when master and agent have different encodings + @SuppressFBWarnings(value = "DM_DEFAULT_ENCODING", justification = "Open TODO item for wider rework") + @Override + protected void eol(byte[] bytes, int len) throws IOException { + String line = new String(bytes, 0, len); + if(passwordsAsPattern != null) { + line = passwordsAsPattern.matcher(line).replaceAll(MASKED_PASSWORD); + } + logger.write(line.getBytes()); + } + + /** + * {@inheritDoc} + * @throws IOException + */ + @Override + public void close() throws IOException { + super.close(); + logger.close(); + } + + /** + * {@inheritDoc} + * @throws IOException + */ + @Override + public void flush() throws IOException { + super.flush(); + logger.flush(); + } +} \ No newline at end of file diff --git a/src/main/java/io/jenkins/plugins/extlogging/api/util/UniqueIdHelper.java b/src/main/java/io/jenkins/plugins/extlogging/api/util/UniqueIdHelper.java new file mode 100644 index 0000000..5e8c05b --- /dev/null +++ b/src/main/java/io/jenkins/plugins/extlogging/api/util/UniqueIdHelper.java @@ -0,0 +1,35 @@ +package io.jenkins.plugins.extlogging.api.util; + +import hudson.model.Run; +import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; +import jenkins.model.Jenkins; +import org.jenkinsci.plugins.uniqueid.IdStore; +import org.kohsuke.accmod.Restricted; +import org.kohsuke.accmod.restrictions.NoExternalUse; + +/** + * Creates on-demand Unique IDs. + * @author Oleg Nenashev + */ +public class UniqueIdHelper { + + @CheckForNull + public static String getOrCreateId(@Nonnull Run run) { + return getOrCreateId(run.getParent()); + } + + @CheckForNull + public static String getOrCreateId(@Nonnull hudson.model.Job job) { + if (Jenkins.getInstance() == null) { + return null; + } + + String id = IdStore.getId(job); + if (id == null) { + IdStore.makeId(job); + id = IdStore.getId(job);; + } + return id; + } +} diff --git a/src/main/resources/index.jelly b/src/main/resources/index.jelly new file mode 100644 index 0000000..dd3e91d --- /dev/null +++ b/src/main/resources/index.jelly @@ -0,0 +1,5 @@ + + +
+ A Jenkins plugin to keep artifacts and Pipeline stashes in Amazon S3. +
diff --git a/src/main/resources/io/jenkins/plugins/extlogging/api/impl/ExternalLoggingGlobalConfiguration/config.jelly b/src/main/resources/io/jenkins/plugins/extlogging/api/impl/ExternalLoggingGlobalConfiguration/config.jelly new file mode 100644 index 0000000..5500e56 --- /dev/null +++ b/src/main/resources/io/jenkins/plugins/extlogging/api/impl/ExternalLoggingGlobalConfiguration/config.jelly @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/src/main/resources/io/jenkins/plugins/extlogging/api/util/AbstractConsoleAction/buildCaption.jelly b/src/main/resources/io/jenkins/plugins/extlogging/api/util/AbstractConsoleAction/buildCaption.jelly new file mode 100644 index 0000000..3def4f6 --- /dev/null +++ b/src/main/resources/io/jenkins/plugins/extlogging/api/util/AbstractConsoleAction/buildCaption.jelly @@ -0,0 +1,53 @@ + + + + + +

+ +
+ + +
+ ${%Progress}: + + + + +
+
+
+ + + ${%Console Output from} ${it.dataSourceDisplayName} +

+
+ diff --git a/src/test/java/io/jenkins/plugins/extlogging/api/FreestyleJobTest.java b/src/test/java/io/jenkins/plugins/extlogging/api/FreestyleJobTest.java new file mode 100644 index 0000000..dcf2fea --- /dev/null +++ b/src/test/java/io/jenkins/plugins/extlogging/api/FreestyleJobTest.java @@ -0,0 +1,56 @@ +package io.jenkins.plugins.extlogging.api; + +import hudson.model.FreeStyleBuild; +import hudson.model.FreeStyleProject; +import hudson.tasks.Shell; +import io.jenkins.plugins.extlogging.api.impl.ExternalLoggingGlobalConfiguration; +import io.jenkins.plugins.extlogging.api.util.MockExternalLoggingEventWriter; +import io.jenkins.plugins.extlogging.api.util.MockLogBrowserFactory; +import io.jenkins.plugins.extlogging.api.util.MockLoggingMethod; +import io.jenkins.plugins.extlogging.api.util.MockLoggingMethodFactory; +import io.jenkins.plugins.extlogging.api.util.MockLoggingTestUtil; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.jvnet.hudson.test.JenkinsRule; + +import java.io.File; + +/** + * @author Oleg Nenashev + * @since TODO + */ +public class FreestyleJobTest { + + @Rule + public JenkinsRule j = new JenkinsRule(); + + @Rule + public TemporaryFolder tmpDir = new TemporaryFolder(); + + @Test + public void spotcheck_Default() throws Exception { + FreeStyleProject project = j.createFreeStyleProject(); + project.getBuildersList().add(new Shell("echo hello")); + + FreeStyleBuild build = j.buildAndAssertSuccess(project); + j.assertLogContains("hello", build); + } + + @Test + public void spotcheck_Mock() throws Exception { + MockLoggingTestUtil.setup(tmpDir); + FreeStyleProject project = j.createFreeStyleProject(); + project.getBuildersList().add(new Shell("echo hello")); + + FreeStyleBuild build = j.buildAndAssertSuccess(project); + MockLoggingMethod lm = (MockLoggingMethod)build.getLoggingMethod(); + MockExternalLoggingEventWriter writer = lm.getWriter(); + Assert.assertTrue(writer.isEventWritten()); + j.assertLogContains("hello", build); + } + +} diff --git a/src/test/java/io/jenkins/plugins/extlogging/api/PipelineSmokeTest.java b/src/test/java/io/jenkins/plugins/extlogging/api/PipelineSmokeTest.java new file mode 100644 index 0000000..9fe0898 --- /dev/null +++ b/src/test/java/io/jenkins/plugins/extlogging/api/PipelineSmokeTest.java @@ -0,0 +1,61 @@ +package io.jenkins.plugins.extlogging.api; + +import hudson.model.Run; +import io.jenkins.plugins.extlogging.api.impl.ExternalLoggingGlobalConfiguration; +import io.jenkins.plugins.extlogging.api.util.MockExternalLoggingEventWriter; +import io.jenkins.plugins.extlogging.api.util.MockLogBrowser; +import io.jenkins.plugins.extlogging.api.util.MockLogBrowserFactory; +import io.jenkins.plugins.extlogging.api.util.MockLoggingMethod; +import io.jenkins.plugins.extlogging.api.util.MockLoggingMethodFactory; +import io.jenkins.plugins.extlogging.api.util.MockLoggingTestUtil; +import org.jenkinsci.plugins.workflow.cps.CpsFlowDefinition; +import org.jenkinsci.plugins.workflow.job.WorkflowJob; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.jvnet.hudson.test.JenkinsRule; + +import java.io.File; + +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.notNullValue; +import static org.junit.Assert.assertThat; + +/** + * @author Oleg Nenashev + * @since TODO + */ +public class PipelineSmokeTest { + + @Rule + public JenkinsRule j = new JenkinsRule(); + + @Rule + public TemporaryFolder tmpDir = new TemporaryFolder(); + + @Test + public void spotcheck_Default() throws Exception { + WorkflowJob project = j.createProject(WorkflowJob.class); + project.setDefinition(new CpsFlowDefinition("echo 'Hello'", true)); + Run build = j.buildAndAssertSuccess(project); + j.assertLogContains("Hello", build); + } + + @Test + public void spotcheck_Mock() throws Exception { + MockLoggingTestUtil.setup(tmpDir); + WorkflowJob project = j.createProject(WorkflowJob.class); + project.setDefinition(new CpsFlowDefinition("echo 'Hello'", true)); + + Run build = j.buildAndAssertSuccess(project); + assertThat(build.getLoggingMethod(), instanceOf(MockLoggingMethod.class)); + MockLoggingMethod loggingMethod = (MockLoggingMethod)build.getLoggingMethod(); + MockExternalLoggingEventWriter writer = loggingMethod.getWriter(); + // Do not try to add it. Pipeline creates separate PipelineLogListeners for each call + // Assert.assertTrue(writer.isEventWritten()); + j.assertLogContains("Hello", build); + } + +} diff --git a/src/test/java/io/jenkins/plugins/extlogging/api/util/MockExternalLoggingEventWriter.java b/src/test/java/io/jenkins/plugins/extlogging/api/util/MockExternalLoggingEventWriter.java new file mode 100644 index 0000000..c4233f7 --- /dev/null +++ b/src/test/java/io/jenkins/plugins/extlogging/api/util/MockExternalLoggingEventWriter.java @@ -0,0 +1,46 @@ +package io.jenkins.plugins.extlogging.api.util; + +import io.jenkins.plugins.extlogging.api.Event; +import io.jenkins.plugins.extlogging.api.ExternalLoggingEventWriter; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.logging.Logger; + +/** + * Mock {@link ExternalLoggingEventWriter} for testing purposes. + * @author Oleg Nenashev + * @see MockLoggingMethod + */ +public class MockExternalLoggingEventWriter extends ExternalLoggingEventWriter { + + private static final Logger LOGGER = + Logger.getLogger(MockExternalLoggingEventWriter.class.getName()); + + private final File dest; + + // Debug flags + private boolean eventWritten; + + public MockExternalLoggingEventWriter(File dest) { + this.dest = dest; + } + + @Override + public void writeEvent(Event event) throws IOException { + eventWritten = true; + FileWriter writer = new FileWriter(dest, true); + writer.write(event.toString() + "\n"); + writer.flush(); + writer.close(); + } + + public File getDest() { + return dest; + } + + public boolean isEventWritten() { + return eventWritten; + } +} diff --git a/src/test/java/io/jenkins/plugins/extlogging/api/util/MockLogBrowser.java b/src/test/java/io/jenkins/plugins/extlogging/api/util/MockLogBrowser.java new file mode 100644 index 0000000..789c972 --- /dev/null +++ b/src/test/java/io/jenkins/plugins/extlogging/api/util/MockLogBrowser.java @@ -0,0 +1,38 @@ +package io.jenkins.plugins.extlogging.api.util; + +import hudson.model.Run; +import jenkins.model.logging.Loggable; +import jenkins.model.logging.impl.FileLogBrowser; + + +import java.io.File; +import java.io.IOException; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * @author Oleg Nenashev + * @since TODO + */ +public class MockLogBrowser extends FileLogBrowser { + + private static final Logger LOGGER = + Logger.getLogger(MockLogBrowser.class.getName()); + + private File baseDir; + + public MockLogBrowser(Run run, File baseDir) { + super(run); + this.baseDir = baseDir; + } + + @Override + protected Run getOwner() { + return (Run)super.getOwner(); + } + + @Override + public File getLogFileOrFail(Loggable loggable) throws IOException { + return new File(baseDir, getOwner().getFullDisplayName() + ".txt"); + } +} diff --git a/src/test/java/io/jenkins/plugins/extlogging/api/util/MockLogBrowserFactory.java b/src/test/java/io/jenkins/plugins/extlogging/api/util/MockLogBrowserFactory.java new file mode 100644 index 0000000..bb939ff --- /dev/null +++ b/src/test/java/io/jenkins/plugins/extlogging/api/util/MockLogBrowserFactory.java @@ -0,0 +1,26 @@ +package io.jenkins.plugins.extlogging.api.util; + +import hudson.model.Run; +import io.jenkins.plugins.extlogging.api.ExternalLogBrowserFactory; +import jenkins.model.logging.LogBrowser; +import jenkins.model.logging.Loggable; + +import java.io.File; + +/** + * @author Oleg Nenashev + * @since TODO + */ +public class MockLogBrowserFactory extends ExternalLogBrowserFactory { + + private File baseDir; + + public MockLogBrowserFactory(File baseDir) { + this.baseDir = baseDir; + } + + @Override + public LogBrowser create(Loggable loggable) { + return new MockLogBrowser((Run)loggable, baseDir); + } +} diff --git a/src/test/java/io/jenkins/plugins/extlogging/api/util/MockLoggingMethod.java b/src/test/java/io/jenkins/plugins/extlogging/api/util/MockLoggingMethod.java new file mode 100644 index 0000000..01e8dc2 --- /dev/null +++ b/src/test/java/io/jenkins/plugins/extlogging/api/util/MockLoggingMethod.java @@ -0,0 +1,46 @@ +package io.jenkins.plugins.extlogging.api.util; + +import hudson.model.Run; +import io.jenkins.plugins.extlogging.api.ExternalLoggingEventWriter; +import io.jenkins.plugins.extlogging.api.ExternalLoggingMethod; +import jenkins.model.logging.LogBrowser; + +import javax.annotation.CheckForNull; +import java.io.File; + +/** + * Mock logging method for testing purposes + * @author Oleg Nenashev + * @see MockExternalLoggingEventWriter + */ +public class MockLoggingMethod extends ExternalLoggingMethod { + + private File baseDir; + transient MockExternalLoggingEventWriter writer; + + public MockLoggingMethod(Run run, File baseDir) { + super(run); + this.baseDir = baseDir; + } + + @Override + protected Run getOwner() { + return (Run)super.getOwner(); + } + + @Override + public ExternalLoggingEventWriter _createWriter() { + writer = new MockExternalLoggingEventWriter(new File(baseDir, getOwner().getFullDisplayName() + ".txt")); + return writer; + } + + @CheckForNull + public MockExternalLoggingEventWriter getWriter() { + return writer; + } + + @Override + public LogBrowser getDefaultLogBrowser() { + return new MockLogBrowser(getOwner(), baseDir); + } +} diff --git a/src/test/java/io/jenkins/plugins/extlogging/api/util/MockLoggingMethodFactory.java b/src/test/java/io/jenkins/plugins/extlogging/api/util/MockLoggingMethodFactory.java new file mode 100644 index 0000000..20c5c05 --- /dev/null +++ b/src/test/java/io/jenkins/plugins/extlogging/api/util/MockLoggingMethodFactory.java @@ -0,0 +1,29 @@ +package io.jenkins.plugins.extlogging.api.util; + +import hudson.model.Run; +import io.jenkins.plugins.extlogging.api.ExternalLoggingMethod; +import io.jenkins.plugins.extlogging.api.ExternalLoggingMethodFactory; +import jenkins.model.logging.Loggable; + +import javax.annotation.CheckForNull; +import java.io.File; +import java.util.HashMap; + +/** + * @author Oleg Nenashev + * @since TODO + */ +public class MockLoggingMethodFactory extends ExternalLoggingMethodFactory { + + private File baseDir; + + public MockLoggingMethodFactory(File baseDir) { + this.baseDir = baseDir; + } + + @CheckForNull + @Override + public ExternalLoggingMethod create(Loggable loggable) { + return new MockLoggingMethod((Run)loggable, baseDir); + } +} diff --git a/src/test/java/io/jenkins/plugins/extlogging/api/util/MockLoggingTestUtil.java b/src/test/java/io/jenkins/plugins/extlogging/api/util/MockLoggingTestUtil.java new file mode 100644 index 0000000..3d4f3db --- /dev/null +++ b/src/test/java/io/jenkins/plugins/extlogging/api/util/MockLoggingTestUtil.java @@ -0,0 +1,24 @@ +package io.jenkins.plugins.extlogging.api.util; + +import io.jenkins.plugins.extlogging.api.impl.ExternalLoggingGlobalConfiguration; +import org.junit.Before; +import org.junit.rules.TemporaryFolder; + +import javax.annotation.Nonnull; +import java.io.File; + +/** + * Test util for mock classes. + * @author Oleg Nenashev + * @see MockLogBrowser + * @see MockLoggingMethod + */ +public class MockLoggingTestUtil { + + public static void setup(@Nonnull TemporaryFolder tmpDir) throws Exception { + ExternalLoggingGlobalConfiguration cfg = ExternalLoggingGlobalConfiguration.getInstance(); + File logDir = tmpDir.newFolder("logs"); + cfg.setLoggingMethod(new MockLoggingMethodFactory(logDir)); + cfg.setLogBrowser(new MockLogBrowserFactory(logDir)); + } +}