diff --git a/pom.xml b/pom.xml index 518f0589..21b25781 100644 --- a/pom.xml +++ b/pom.xml @@ -5,12 +5,14 @@ UTF-8 true + 2.7.3 + 7 org.jenkins-ci.plugins plugin - 1.599 + 2.9 @@ -18,6 +20,11 @@ repo.jenkins-ci.org https://repo.jenkins-ci.org/public/ + + + jitpack.io + https://jitpack.io + @@ -88,7 +95,7 @@ org.mockito mockito-core - 2.1.0 + 2.13.0 jar test @@ -113,7 +120,6 @@ org.jenkins-ci.plugins mask-passwords 2.8 - true @@ -127,6 +133,19 @@ structs 1.2 + + + com.github.akostadinov + script-security-plugin + master-SNAPSHOT + + + + javax.validation + validation-api + 1.0.0.GA + provided + diff --git a/src/main/java/jenkins/plugins/logstash/LogstashBuildWrapper.java b/src/main/java/jenkins/plugins/logstash/LogstashBuildWrapper.java index 2e4a0100..f55772b1 100644 --- a/src/main/java/jenkins/plugins/logstash/LogstashBuildWrapper.java +++ b/src/main/java/jenkins/plugins/logstash/LogstashBuildWrapper.java @@ -32,13 +32,19 @@ import hudson.model.BuildableItemWithBuildWrappers; import hudson.tasks.BuildWrapper; import hudson.tasks.BuildWrapperDescriptor; +import org.jenkinsci.plugins.scriptsecurity.sandbox.groovy.SecureGroovyScript; import java.io.IOException; import java.io.OutputStream; import java.util.ArrayList; import java.util.List; +import javax.annotation.CheckForNull; + +import groovy.lang.Binding; + import org.kohsuke.stapler.DataBoundConstructor; +import org.kohsuke.stapler.DataBoundSetter; import com.michelin.cio.hudson.plugins.maskpasswords.MaskPasswordsBuildWrapper; import com.michelin.cio.hudson.plugins.maskpasswords.MaskPasswordsBuildWrapper.VarPasswordPair; @@ -46,18 +52,26 @@ /** * Build wrapper that decorates the build's logger to insert a - * {@link LogstashNote} on each output line. + * Logstash note on each output line. * * @author K Jonathan Harker */ public class LogstashBuildWrapper extends BuildWrapper { + @CheckForNull + private SecureGroovyScript secureGroovyScript; + /** * Create a new {@link LogstashBuildWrapper}. */ @DataBoundConstructor public LogstashBuildWrapper() {} + @DataBoundSetter + public void setSecureGroovyScript(@CheckForNull SecureGroovyScript script) { + this.secureGroovyScript = script != null ? script.configuringWithNonKeyItem() : null; + } + /** * {@inheritDoc} */ @@ -106,9 +120,19 @@ public DescriptorImpl getDescriptor() { return (DescriptorImpl) super.getDescriptor(); } + @CheckForNull + public SecureGroovyScript getSecureGroovyScript() { + return secureGroovyScript; + } + // Method to encapsulate calls for unit-testing LogstashWriter getLogStashWriter(AbstractBuild build, OutputStream errorStream) { - return new LogstashWriter(build, errorStream, null); + LogstashScriptProcessor processor = null; + if (secureGroovyScript != null) { + processor = new LogstashScriptProcessor(secureGroovyScript, errorStream); + } + + return new LogstashWriter(build, errorStream, null, processor); } /** diff --git a/src/main/java/jenkins/plugins/logstash/LogstashInstallation.java b/src/main/java/jenkins/plugins/logstash/LogstashInstallation.java index a806d95b..9f2161dd 100644 --- a/src/main/java/jenkins/plugins/logstash/LogstashInstallation.java +++ b/src/main/java/jenkins/plugins/logstash/LogstashInstallation.java @@ -43,6 +43,8 @@ import org.kohsuke.stapler.QueryParameter; import org.kohsuke.stapler.StaplerRequest; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + /** * POJO for storing global configurations shared between components. * @@ -65,6 +67,9 @@ public static Descriptor getLogstashDescriptor() { public static final class Descriptor extends ToolDescriptor { public IndexerType type; public SyslogFormat syslogFormat; + @SuppressFBWarnings( + value="UUF_UNUSED_PUBLIC_OR_PROTECTED_FIELD", + justification="TODO: do we need this?") public SyslogProtocol syslogProtocol; public String host; public Integer port = -1; @@ -85,6 +90,9 @@ public boolean configure(StaplerRequest req, JSONObject formData) throws FormExc } @Override + @SuppressFBWarnings( + value="NP_PARAMETER_MUST_BE_NONNULL_BUT_MARKED_AS_NULLABLE", + justification="TODO: investigate") public ToolInstallation newInstance(StaplerRequest req, JSONObject formData) throws FormException { req.bindJSON(this, formData.getJSONObject("logstash")); save(); diff --git a/src/main/java/jenkins/plugins/logstash/LogstashOutputStream.java b/src/main/java/jenkins/plugins/logstash/LogstashOutputStream.java index 929abb47..0e9a561b 100644 --- a/src/main/java/jenkins/plugins/logstash/LogstashOutputStream.java +++ b/src/main/java/jenkins/plugins/logstash/LogstashOutputStream.java @@ -35,6 +35,8 @@ import com.michelin.cio.hudson.plugins.maskpasswords.MaskPasswordsBuildWrapper.VarPasswordPair; import com.michelin.cio.hudson.plugins.maskpasswords.MaskPasswordsOutputStream; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + /** * Output stream that writes each line to the provided delegate output stream * and also sends it to an indexer for logstash to consume. @@ -61,6 +63,9 @@ public MaskPasswordsOutputStream maskPasswords(List passwords) } @Override + @SuppressFBWarnings( + value="DM_DEFAULT_ENCODING", + justification="TODO: not sure how to fix this") protected void eol(byte[] b, int len) throws IOException { delegate.write(b, 0, len); this.flush(); @@ -86,6 +91,7 @@ public void flush() throws IOException { */ @Override public void close() throws IOException { + logstash.close(); delegate.close(); super.close(); } diff --git a/src/main/java/jenkins/plugins/logstash/LogstashPayloadProcessor.java b/src/main/java/jenkins/plugins/logstash/LogstashPayloadProcessor.java new file mode 100644 index 00000000..e39f60ea --- /dev/null +++ b/src/main/java/jenkins/plugins/logstash/LogstashPayloadProcessor.java @@ -0,0 +1,50 @@ +/* + * The MIT License + * + * Copyright 2017 Red Hat inc, and individual contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package jenkins.plugins.logstash; + +import net.sf.json.JSONObject; + +/** + * Interface describing processors of persisted payload. + * + * @author Aleksandar Kostadinov + * @since 1.4.0 + */ +public interface LogstashPayloadProcessor { + /** + * Modifies a JSON payload compatible with the Logstash schema. + * + * @param payload the JSON payload that has been constructed so far. + * @return The formatted JSON object, can be null to ignore this payload. + */ + JSONObject process(JSONObject payload) throws Exception; + + /** + * Finalizes any operations, for example returns cashed lines at end of build. + * + * @return A formatted JSON object, can be null when it has nothing. + */ + JSONObject finish() throws Exception; +} diff --git a/src/main/java/jenkins/plugins/logstash/LogstashScriptProcessor.java b/src/main/java/jenkins/plugins/logstash/LogstashScriptProcessor.java new file mode 100644 index 00000000..a0cf545a --- /dev/null +++ b/src/main/java/jenkins/plugins/logstash/LogstashScriptProcessor.java @@ -0,0 +1,123 @@ +/* + * The MIT License + * + * Copyright 2017 Red Hat inc. and individual contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package jenkins.plugins.logstash; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.util.LinkedHashMap; + +import javax.annotation.Nonnull; + +import groovy.lang.Binding; + +import net.sf.json.JSONObject; + +import jenkins.model.Jenkins; +import org.jenkinsci.plugins.scriptsecurity.sandbox.groovy.SecureGroovyScript; +import org.jenkinsci.plugins.scriptsecurity.sandbox.whitelists.Whitelisted; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + +/** + * This class is handling custom groovy script processing of JSON payload. + * Each call to process executes the script provided in job configuration. + * Script is executed under the same binding each time so that it has ability + * to persist data during build execution if desired by script author. + * When build is finished, script will receive null as the payload and can + * return any cached but non-sent data back for persisting. + * The return value of script is the payload to be persisted unless null. + * + * @author Aleksandar Kostadinov + * @since 1.4.0 + */ +public class LogstashScriptProcessor implements LogstashPayloadProcessor{ + @Nonnull + private final SecureGroovyScript script; + + @Nonnull + private final OutputStream consoleOut; + + /** Groovy binding for script execution */ + @Nonnull + private final Binding binding; + + /** Classloader for script execution */ + @Nonnull + private final ClassLoader classLoader; + + public LogstashScriptProcessor(SecureGroovyScript script, OutputStream consoleOut) { + this.script = script; + this.consoleOut = consoleOut; + + // TODO: should we put variables in the binding like manager, job, etc.? + binding = new Binding(); + binding.setVariable("console", new BuildConsoleWrapper()); + + // not sure what the diff is compared to getClass().getClassLoader(); + final Jenkins jenkins = Jenkins.getInstance(); + classLoader = jenkins.getPluginManager().uberClassLoader; + } + + /** + * Helper method to allow logging to build console. + */ + @SuppressFBWarnings( + value="DM_DEFAULT_ENCODING", + justification="TODO: not sure how to fix this") + private void buildLogPrintln(Object o) throws IOException { + consoleOut.write(o.toString().getBytes()); + consoleOut.write("\n".getBytes()); + consoleOut.flush(); + } + + /* + * good examples in: + * https://github.com/jenkinsci/envinject-plugin/blob/master/src/main/java/org/jenkinsci/plugins/envinject/service/EnvInjectEnvVars.java + * https://github.com/jenkinsci/groovy-postbuild-plugin/pull/11/files + */ + @Override + public JSONObject process(JSONObject payload) throws Exception { + binding.setVariable("payload", payload); + script.evaluate(classLoader, binding); + return (JSONObject) binding.getVariable("payload"); + } + + @Override + public JSONObject finish() throws Exception { + buildLogPrintln("Tearing down Script Log Processor.."); + return process(null); + } + + /** + * Helper to allow access from sandboxed script to output messages to console. + */ + private class BuildConsoleWrapper { + @Whitelisted + public void println(Object o) throws IOException { + buildLogPrintln(o); + } + } +} diff --git a/src/main/java/jenkins/plugins/logstash/LogstashWriter.java b/src/main/java/jenkins/plugins/logstash/LogstashWriter.java index 68134d30..05d8d609 100644 --- a/src/main/java/jenkins/plugins/logstash/LogstashWriter.java +++ b/src/main/java/jenkins/plugins/logstash/LogstashWriter.java @@ -36,12 +36,15 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.exception.ExceptionUtils; +import javax.annotation.CheckForNull; import java.io.IOException; import java.io.OutputStream; import java.util.Arrays; import java.util.Date; import java.util.List; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + /** * A writer that wraps all Logstash DAOs. Handles error reporting and per build connection state. * Each call to write (one line or multiple lines) sends a Logstash payload to the DAO. @@ -49,6 +52,7 @@ * * @author Rusty Gerard * @author Liam Newman + * @author Aleksandar Kostadinov * @since 1.0.5 */ public class LogstashWriter { @@ -61,10 +65,18 @@ public class LogstashWriter { final LogstashIndexerDao dao; private boolean connectionBroken; + @CheckForNull + private final LogstashPayloadProcessor payloadProcessor; + public LogstashWriter(Run run, OutputStream error, TaskListener listener) { + this(run, error, listener, null); + } + + public LogstashWriter(Run run, OutputStream error, TaskListener listener, LogstashPayloadProcessor payloadProcessor) { this.errorStream = error != null ? error : System.err; this.build = run; this.listener = listener; + this.payloadProcessor = payloadProcessor; this.dao = this.getDaoOrNull(); if (this.dao == null) { this.jenkinsUrl = ""; @@ -149,7 +161,35 @@ String getJenkinsUrl() { * Write a list of lines to the indexer as one Logstash payload. */ private void write(List lines) { - JSONObject payload = dao.buildPayload(buildData, jenkinsUrl, lines); + write(dao.buildPayload(buildData, jenkinsUrl, lines)); + } + + /** + * Write JSONObject payload to the Logstash indexer. + * @since 1.0.5 + */ + private void write(JSONObject payload) { + if (payloadProcessor != null) { + JSONObject processedPayload = payload; + try { + processedPayload = payloadProcessor.process(payload); + } catch (Exception e) { + String msg = ExceptionUtils.getMessage(e) + "\n" + + "[logstash-plugin]: Error in payload processing.\n"; + + logErrorMessage(msg); + } + if (processedPayload != null) { writeRaw(processedPayload); } + } else { + writeRaw(payload); + } + } + + /** + * Write JSONObject payload to the Logstash indexer. + * @since 1.0.5 + */ + private void writeRaw(JSONObject payload) { try { dao.push(payload.toString()); } catch (IOException e) { @@ -181,6 +221,9 @@ private LogstashIndexerDao getDaoOrNull() { /** * Write error message to errorStream and set connectionBroken to true. */ + @SuppressFBWarnings( + value="DM_DEFAULT_ENCODING", + justification="TODO: not sure how to fix this") private void logErrorMessage(String msg) { try { connectionBroken = true; @@ -191,4 +234,23 @@ private void logErrorMessage(String msg) { ex.printStackTrace(); } } + + /** + * Signal payload processor that there will be no more lines + */ + public void close() { + if (payloadProcessor != null) { + JSONObject payload = null; + try { + // calling finish() is mandatory to avoid memory leaks + payload = payloadProcessor.finish(); + } catch (Exception e) { + String msg = ExceptionUtils.getMessage(e) + "\n" + + "[logstash-plugin]: Error with payload processor on finish.\n"; + + logErrorMessage(msg); + } + if (payload != null) writeRaw(payload); + } + } } diff --git a/src/main/java/jenkins/plugins/logstash/persistence/AbstractLogstashIndexerDao.java b/src/main/java/jenkins/plugins/logstash/persistence/AbstractLogstashIndexerDao.java index 25bd93c6..1c1ddaf8 100644 --- a/src/main/java/jenkins/plugins/logstash/persistence/AbstractLogstashIndexerDao.java +++ b/src/main/java/jenkins/plugins/logstash/persistence/AbstractLogstashIndexerDao.java @@ -24,13 +24,15 @@ package jenkins.plugins.logstash.persistence; -import java.util.Calendar; +import java.util.Date; import java.util.List; import org.apache.commons.lang.StringUtils; import net.sf.json.JSONObject; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + /** * Abstract data access object for Logstash indexers. * @@ -64,7 +66,7 @@ public JSONObject buildPayload(BuildData buildData, String jenkinsUrl, List testResultAction = null; if (action instanceof AbstractTestResultAction) { @@ -113,8 +119,8 @@ public TestData(Action action) { } } + @CheckForNull protected String result; protected String id; - protected String result; protected String projectName; protected String fullProjectName; protected String displayName; @@ -207,8 +213,14 @@ public BuildData(Run build, Date currentTime, TaskListener listener) { } } + // ISO 8601 date format + public static String formatDateIso(Date date) { + return DATE_FORMATTER.format(date); + } + private void initData(Run build, Date currentTime) { - result = build.getResult() == null ? null : build.getResult().toString(); + Result buildResult = build.getResult(); + result = buildResult == null ? null : buildResult.toString(); id = build.getId(); projectName = build.getParent().getName(); fullProjectName = build.getParent().getFullName(); @@ -224,7 +236,7 @@ private void initData(Run build, Date currentTime) { } buildDuration = currentTime.getTime() - build.getStartTimeInMillis(); - timestamp = DATE_FORMATTER.format(build.getTimestamp().getTime()); + timestamp = formatDateIso(build.getTime()); } @Override @@ -339,7 +351,7 @@ public String getTimestamp() { } public void setTimestamp(Calendar timestamp) { - this.timestamp = DATE_FORMATTER.format(timestamp.getTime()); + this.timestamp = formatDateIso(timestamp.getTime()); } public String getRootProjectName() { diff --git a/src/main/java/jenkins/plugins/logstash/persistence/ElasticSearchDao.java b/src/main/java/jenkins/plugins/logstash/persistence/ElasticSearchDao.java index 91c29df2..94297e9f 100644 --- a/src/main/java/jenkins/plugins/logstash/persistence/ElasticSearchDao.java +++ b/src/main/java/jenkins/plugins/logstash/persistence/ElasticSearchDao.java @@ -45,6 +45,8 @@ import com.google.common.collect.Range; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + /** * Elastic Search Data Access Object. * @@ -63,6 +65,9 @@ public ElasticSearchDao(String host, int port, String key, String username, Stri } // Factored for unit testing + @SuppressFBWarnings( + value="DM_DEFAULT_ENCODING", + justification="TODO: not sure how to fix this") ElasticSearchDao(HttpClientBuilder factory, String host, int port, String key, String username, String password) { super(host, port, key, username, password); @@ -127,6 +132,9 @@ public void push(String data) throws IOException { } } + @SuppressFBWarnings( + value="DM_DEFAULT_ENCODING", + justification="TODO: not sure how to fix this") private String getErrorMessage(CloseableHttpResponse response) { ByteArrayOutputStream byteStream = null; PrintStream stream = null; diff --git a/src/main/java/jenkins/plugins/logstash/persistence/IndexerDaoFactory.java b/src/main/java/jenkins/plugins/logstash/persistence/IndexerDaoFactory.java index ffd79345..76f5b0d7 100644 --- a/src/main/java/jenkins/plugins/logstash/persistence/IndexerDaoFactory.java +++ b/src/main/java/jenkins/plugins/logstash/persistence/IndexerDaoFactory.java @@ -35,6 +35,8 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.exception.ExceptionUtils; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + /** * Factory for AbstractLogstashIndexerDao objects. * @@ -74,6 +76,9 @@ public final class IndexerDaoFactory { * @return The instance of the appropriate indexer DAO, never null * @throws InstantiationException */ + @SuppressFBWarnings( + value="BX_UNBOXING_IMMEDIATELY_REBOXED", + justification="TODO: not sure how to fix this") public static synchronized LogstashIndexerDao getInstance(IndexerType type, String host, Integer port, String key, String username, String password) throws InstantiationException { if (!INDEXER_MAP.containsKey(type)) { throw new InstantiationException("[logstash-plugin]: Unknown IndexerType '" + type + "'. Did you forget to configure the plugin?"); diff --git a/src/main/java/jenkins/plugins/logstash/persistence/RabbitMqDao.java b/src/main/java/jenkins/plugins/logstash/persistence/RabbitMqDao.java index d49e2ccb..0817af33 100644 --- a/src/main/java/jenkins/plugins/logstash/persistence/RabbitMqDao.java +++ b/src/main/java/jenkins/plugins/logstash/persistence/RabbitMqDao.java @@ -32,6 +32,8 @@ import com.rabbitmq.client.Connection; import com.rabbitmq.client.ConnectionFactory; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + /** * RabbitMQ Data Access Object. * @@ -68,6 +70,9 @@ public RabbitMqDao(String host, int port, String key, String username, String pa } @Override + @SuppressFBWarnings( + value="DM_DEFAULT_ENCODING", + justification="TODO: not sure how to fix this") public void push(String data) throws IOException { Connection connection = null; Channel channel = null; diff --git a/src/main/resources/jenkins/plugins/logstash/LogstashBuildWrapper/config.jelly b/src/main/resources/jenkins/plugins/logstash/LogstashBuildWrapper/config.jelly new file mode 100644 index 00000000..4c7cbf08 --- /dev/null +++ b/src/main/resources/jenkins/plugins/logstash/LogstashBuildWrapper/config.jelly @@ -0,0 +1,7 @@ + + + + + + + diff --git a/src/main/resources/jenkins/plugins/logstash/LogstashBuildWrapper/help-script.html b/src/main/resources/jenkins/plugins/logstash/LogstashBuildWrapper/help-script.html new file mode 100644 index 00000000..27404758 --- /dev/null +++ b/src/main/resources/jenkins/plugins/logstash/LogstashBuildWrapper/help-script.html @@ -0,0 +1,37 @@ +

+ With this script you can filter, modify and/or group messages that are sent to the + configured logstash backend. The following variables are available during execution: +

    +
  • payload - JSONObject contaning the payload to be persisted.
    • +
  • console - a class to output messages to build log without persisting them. It also works in a sandbox: console.println("my logged message").
    • +
+

+

+ The script will be executed once per each line of build console output with + the variable payload set to the complete JSON payload to be + sent including message, timestamp, build url, etc. + The line text in particular will be present under the "message" + key as an array with a single string element. +

+

+ Once script completes execution, the payload variable will be read out + of current Binding and passed down to the Logstash backend to be persisted. If + payload is set to null by the script, then nothing will be + persisted. +

+

+ The script within a build will be executed always using the same Binding. + This means that variables can be saved between script invocations. +

+

+ At the end of the build a payload == null will be submitted. You can use this to + output any messages that you have cached for grouping or other purposes. +

+

+ Example script to filter out some console messages by pattern: + 


+    if (payload && payload["message"][0] =~ /my needless pattern/) {
+      payload = null
+    }
+
+

diff --git a/src/main/resources/jenkins/plugins/logstash/LogstashBuildWrapper/help.html b/src/main/resources/jenkins/plugins/logstash/LogstashBuildWrapper/help.html index fd472c59..d0e26e2b 100644 --- a/src/main/resources/jenkins/plugins/logstash/LogstashBuildWrapper/help.html +++ b/src/main/resources/jenkins/plugins/logstash/LogstashBuildWrapper/help.html @@ -1,3 +1,6 @@
-

Send individual log lines to Logstash.

+

+ Send individual log lines to Logstash. You can optionally filter and modify them + via groovy script in from advanced configuration. +

diff --git a/src/test/java/jenkins/plugins/logstash/LogstashOutputStreamTest.java b/src/test/java/jenkins/plugins/logstash/LogstashOutputStreamTest.java index 5b445916..7ee3065a 100644 --- a/src/test/java/jenkins/plugins/logstash/LogstashOutputStreamTest.java +++ b/src/test/java/jenkins/plugins/logstash/LogstashOutputStreamTest.java @@ -14,6 +14,7 @@ import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.Mockito; +import org.mockito.InOrder; import org.mockito.runners.MockitoJUnitRunner; @SuppressWarnings("resource") @@ -118,4 +119,17 @@ public void eolSuccessNoDao() throws Exception { assertEquals("Results don't match", msg, buffer.toString()); verify(mockWriter).isConnectionBroken(); } + + @Test + public void writerClosedBeforeDelegate() throws Exception { + ByteArrayOutputStream mockBuffer = Mockito.spy(buffer); + new LogstashOutputStream(mockBuffer, mockWriter).close(); + + InOrder inOrder = Mockito.inOrder(mockBuffer, mockWriter); + inOrder.verify(mockWriter).close(); + inOrder.verify(mockBuffer).close(); + + // Verify results + assertEquals("Results don't match", "", buffer.toString()); + } } diff --git a/src/test/java/jenkins/plugins/logstash/LogstashWriterTest.java b/src/test/java/jenkins/plugins/logstash/LogstashWriterTest.java index 8e643e39..8121e237 100644 --- a/src/test/java/jenkins/plugins/logstash/LogstashWriterTest.java +++ b/src/test/java/jenkins/plugins/logstash/LogstashWriterTest.java @@ -9,20 +9,26 @@ import jenkins.plugins.logstash.persistence.BuildData; import jenkins.plugins.logstash.persistence.LogstashIndexerDao; import jenkins.plugins.logstash.persistence.LogstashIndexerDao.IndexerType; +import org.jenkinsci.plugins.scriptsecurity.sandbox.groovy.SecureGroovyScript; +import org.jvnet.hudson.test.JenkinsRule; import net.sf.json.JSONObject; +import net.sf.json.JSONArray; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.Rule; import org.junit.runner.RunWith; import org.mockito.*; +import org.mockito.invocation.InvocationOnMock; import org.mockito.runners.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; import java.util.Arrays; import java.util.Collections; -import java.util.GregorianCalendar; +import java.util.Date; import java.util.List; import static org.hamcrest.core.StringContains.containsString; @@ -31,13 +37,16 @@ @RunWith(MockitoJUnitRunner.class) public class LogstashWriterTest { + @Rule public JenkinsRule j = new JenkinsRule(); + // Extension of the unit under test that avoids making calls to getInstance() to get the DAO singleton static LogstashWriter createLogstashWriter(final AbstractBuild testBuild, OutputStream error, final String url, final LogstashIndexerDao indexer, - final BuildData data) { - return new LogstashWriter(testBuild, error, null) { + final BuildData data, + final LogstashPayloadProcessor processor) { + return new LogstashWriter(testBuild, error, null, processor) { @Override LogstashIndexerDao getDao() throws InstantiationException { if (indexer == null) { @@ -66,6 +75,14 @@ String getJenkinsUrl() { }; } + static LogstashWriter createLogstashWriter(final AbstractBuild testBuild, + OutputStream error, + final String url, + final LogstashIndexerDao indexer, + final BuildData data) { + return createLogstashWriter(testBuild, error, url, indexer, data, null); + } + ByteArrayOutputStream errorBuffer; @Mock LogstashIndexerDao mockDao; @@ -87,15 +104,13 @@ public void before() throws Exception { when(mockBuild.getParent()).thenReturn(mockProject); when(mockBuild.getBuiltOn()).thenReturn(null); when(mockBuild.getNumber()).thenReturn(123456); - when(mockBuild.getTimestamp()).thenReturn(new GregorianCalendar()); + when(mockBuild.getTime()).thenReturn(new Date()); when(mockBuild.getRootBuild()).thenReturn(mockBuild); when(mockBuild.getBuildVariables()).thenReturn(Collections.emptyMap()); when(mockBuild.getSensitiveBuildVariables()).thenReturn(Collections.emptySet()); when(mockBuild.getEnvironments()).thenReturn(null); when(mockBuild.getAction(AbstractTestResultAction.class)).thenReturn(mockTestResultAction); - when(mockBuild.getLog(0)).thenReturn(Arrays.asList()); when(mockBuild.getLog(3)).thenReturn(Arrays.asList("line 1", "line 2", "line 3", "Log truncated...")); - when(mockBuild.getLog(Integer.MAX_VALUE)).thenReturn(Arrays.asList("line 1", "line 2", "line 3", "line 4")); when(mockBuild.getEnvironment(null)).thenReturn(new EnvVars()); when(mockTestResultAction.getTotalCount()).thenReturn(0); @@ -107,7 +122,15 @@ public void before() throws Exception { when(mockProject.getFullName()).thenReturn("parent/LogstashWriterTest"); when(mockDao.buildPayload(Matchers.any(BuildData.class), Matchers.anyString(), Matchers.anyListOf(String.class))) - .thenReturn(JSONObject.fromObject("{\"data\":{},\"message\":[\"test\"],\"source\":\"jenkins\",\"source_host\":\"http://my-jenkins-url\",\"@version\":1}")); + .thenAnswer(new Answer() { + @Override + public JSONObject answer(InvocationOnMock invocation) { + Object[] args = invocation.getArguments(); + JSONObject json = JSONObject.fromObject("{\"data\":{},\"message\": null,\"source\":\"jenkins\",\"source_host\":\"http://my-jenkins-url\",\"@version\":1}"); + json.element("message", args[2]); + return json; + } + }); Mockito.doNothing().when(mockDao).push(Matchers.anyString()); when(mockDao.getIndexerType()).thenReturn(IndexerType.REDIS); @@ -133,7 +156,7 @@ public void constructorSuccess() throws Exception { // Verify that the BuildData constructor is what is being called here. // This also lets us verify that in the instantiation failure cases we do not construct BuildData. verify(mockBuild).getId(); - verify(mockBuild, times(2)).getResult(); + verify(mockBuild, times(1)).getResult(); verify(mockBuild, times(2)).getParent(); verify(mockBuild, times(2)).getProject(); verify(mockBuild, times(1)).getStartTimeInMillis(); @@ -144,7 +167,7 @@ public void constructorSuccess() throws Exception { verify(mockBuild).getAction(AbstractTestResultAction.class); verify(mockBuild).getBuiltOn(); verify(mockBuild, times(2)).getNumber(); - verify(mockBuild).getTimestamp(); + verify(mockBuild).getTime(); verify(mockBuild, times(4)).getRootBuild(); verify(mockBuild).getBuildVariables(); verify(mockBuild).getSensitiveBuildVariables(); @@ -235,10 +258,55 @@ public void writeBuildLogSuccess() throws Exception { // Verify results // No error output assertEquals("Results don't match", "", errorBuffer.toString()); - verify(mockBuild).getLog(3); + String lines = JSONArray.fromObject(mockBuild.getLog(3)).toString(); + verify(mockBuild, times(2)).getLog(3); verify(mockDao).buildPayload(Matchers.eq(mockBuildData), Matchers.eq("http://my-jenkins-url"), Matchers.anyListOf(String.class)); - verify(mockDao).push("{\"data\":{},\"message\":[\"test\"],\"source\":\"jenkins\",\"source_host\":\"http://my-jenkins-url\",\"@version\":1}"); + verify(mockDao).push("{\"data\":{},\"message\":" + lines + ",\"source\":\"jenkins\",\"source_host\":\"http://my-jenkins-url\",\"@version\":1}"); + } + + @Test + public void writeProcessedSuccess() throws Exception { + String goodMsg = "test message"; + String ignoredMsg = "ignored input"; + String scriptString = + "if (payload) {\n" + + " if (payload['message'][0] =~ /" + ignoredMsg + "/) {\n" + + " payload = null\n" + + " } else {\n" + + " console.println('l');\n" + + " }\n" + + " lastPayload = payload\n" + + "} else {\n" + + " console.println('test build console message')\n" + + " payload = lastPayload\n" + + "}"; + + SecureGroovyScript script = new SecureGroovyScript(scriptString, true, null); + script.configuringWithNonKeyItem(); + LogstashScriptProcessor processor = new LogstashScriptProcessor(script, errorBuffer); + LogstashWriter writer = createLogstashWriter(mockBuild, errorBuffer, "http://my-jenkins-url", mockDao, mockBuildData, processor); + errorBuffer.reset(); + + // Unit under test + writer.write(goodMsg); + writer.write(ignoredMsg); + writer.write(goodMsg); + writer.close(); + + // Verify results + // buffer contains 2 lines logged by the script, then standard tear down message and finally test message at close + assertEquals("Results don't match", "l\nl\nTearing down Script Log Processor..\ntest build console message\n", errorBuffer.toString()); + + InOrder inOrder = Mockito.inOrder(mockDao); + + // first message is generated and pushed to DAO + inOrder.verify(mockDao).buildPayload(Matchers.eq(mockBuildData), Matchers.eq("http://my-jenkins-url"), Matchers.anyListOf(String.class)); + inOrder.verify(mockDao).push("{\"data\":{},\"message\":[\"" + goodMsg + "\"],\"source\":\"jenkins\",\"source_host\":\"http://my-jenkins-url\",\"@version\":1}"); + // now message only generated but filtered out by script thus not pushed to DAO + inOrder.verify(mockDao, times(2)).buildPayload(Matchers.eq(mockBuildData), Matchers.eq("http://my-jenkins-url"), Matchers.anyListOf(String.class)); + // the message at close time is generated by the script so no call to DAO for that + inOrder.verify(mockDao, times(2)).push("{\"data\":{},\"message\":[\"" + goodMsg + "\"],\"source\":\"jenkins\",\"source_host\":\"http://my-jenkins-url\",\"@version\":1}"); } @Test @@ -302,9 +370,10 @@ public void writeBuildLogGetLogError() throws Exception { List expectedErrorLines = Arrays.asList( "[logstash-plugin]: Unable to serialize log data.", "java.io.IOException: Unable to read log file"); - verify(mockDao).push("{\"data\":{},\"message\":[\"test\"],\"source\":\"jenkins\",\"source_host\":\"http://my-jenkins-url\",\"@version\":1}"); verify(mockDao).buildPayload(eq(mockBuildData), eq("http://my-jenkins-url"), logLinesCaptor.capture()); List actualLogLines = logLinesCaptor.getValue(); + String linesJSON = JSONArray.fromObject(actualLogLines).toString(); + verify(mockDao).push("{\"data\":{},\"message\":" + linesJSON + ",\"source\":\"jenkins\",\"source_host\":\"http://my-jenkins-url\",\"@version\":1}"); assertThat("The exception was not sent to Logstash", actualLogLines.get(0), containsString(expectedErrorLines.get(0))); assertThat("The exception was not sent to Logstash", actualLogLines.get(1), containsString(expectedErrorLines.get(1))); diff --git a/src/test/java/jenkins/plugins/logstash/persistence/BuildDataTest.java b/src/test/java/jenkins/plugins/logstash/persistence/BuildDataTest.java index 51dfb09b..857029f5 100644 --- a/src/test/java/jenkins/plugins/logstash/persistence/BuildDataTest.java +++ b/src/test/java/jenkins/plugins/logstash/persistence/BuildDataTest.java @@ -9,7 +9,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.Date; -import java.util.GregorianCalendar; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -69,10 +68,9 @@ public void before() throws Exception { when(mockBuild.getDisplayName()).thenReturn("BuildData Test"); when(mockBuild.getFullDisplayName()).thenReturn("BuildData Test #123456"); when(mockBuild.getDescription()).thenReturn("Mock project for testing BuildData"); - when(mockBuild.getProject()).thenReturn(mockProject); when(mockBuild.getParent()).thenReturn(mockProject); when(mockBuild.getNumber()).thenReturn(123456); - when(mockBuild.getTimestamp()).thenReturn(new GregorianCalendar()); + when(mockBuild.getTime()).thenReturn(new Date()); when(mockBuild.getRootBuild()).thenReturn(mockBuild); when(mockBuild.getBuildVariables()).thenReturn(Collections.emptyMap()); when(mockBuild.getSensitiveBuildVariables()).thenReturn(Collections.emptySet()); @@ -116,7 +114,7 @@ private void verifyMocks() throws Exception verify(mockProject).getFullName(); verify(mockBuild).getId(); - verify(mockBuild, times(2)).getResult(); + verify(mockBuild, times(1)).getResult(); verify(mockBuild, times(2)).getParent(); verify(mockBuild).getDisplayName(); verify(mockBuild).getFullDisplayName(); @@ -126,7 +124,7 @@ private void verifyMocks() throws Exception verify(mockBuild).getAction(AbstractTestResultAction.class); verify(mockBuild).getBuiltOn(); verify(mockBuild).getNumber(); - verify(mockBuild).getTimestamp(); + verify(mockBuild).getTime(); verify(mockBuild, times(4)).getRootBuild(); verify(mockBuild).getBuildVariables(); verify(mockBuild).getSensitiveBuildVariables();