diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 00000000..81229e8d Binary files /dev/null and b/.DS_Store differ diff --git a/README.md b/README.md index c9ff236e..fbb06986 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Jenkins Logstash Plugin Travis: [![Build Status](https://travis-ci.org/jenkinsci/logstash-plugin.svg?branch=master)](https://travis-ci.org/jenkinsci/logstash-plugin) Jenkins: [![Build Status](https://ci.jenkins.io/job/Plugins/job/logstash-plugin/job/master/badge/icon)](https://ci.jenkins.io/job/Plugins/job/logstash-plugin/job/master/) -This plugin adds support for sending a job's console log to Logstash indexers such as [Elastic Search](https://www.elastic.co/products/elasticsearch), [Logstash](https://www.elastic.co/de/products/logstash), [RabbitMQ](https://www.rabbitmq.com), [Redis](https://redis.io/) or to Syslog. +This plugin adds support for sending a job's console log to Logstash indexers such as [Logzio](https://logz.io/), [Elastic Search](https://www.elastic.co/products/elasticsearch), [Logstash](https://www.elastic.co/de/products/logstash), [RabbitMQ](https://www.rabbitmq.com), [Redis](https://redis.io/) or to Syslog. * see [Jenkins wiki](https://wiki.jenkins-ci.org/display/JENKINS/Logstash+Plugin) for detailed feature descriptions * use [JIRA](https://issues.jenkins-ci.org) to report issues / feature requests @@ -15,6 +15,7 @@ Install * Generate the `hpi` file with the command: `mvn package` * Put the `hpi` file in the directory `$JENKINS_HOME/plugins` + * Restart jenkins Configure @@ -27,6 +28,7 @@ Currently supported methods of input/output: * Redis {format => 'json_event'} * RabbitMQ {mechanism => PLAIN} * Syslog {format => cee/json ([RFC-5424](https://tools.ietf.org/html/rfc5424),[RFC-3164](https://tools.ietf.org/html/rfc3164)), protocol => UDP} +* Logz.io Pipeline ======== diff --git a/pom.xml b/pom.xml index 082e99ca..ac3ee1c6 100644 --- a/pom.xml +++ b/pom.xml @@ -78,6 +78,18 @@ amqp-client 3.3.5 + + io.logz.sender + logzio-sender + 1.0.14 + + + + com.github.wnameless + json-flattener + 0.6.0 + + com.google.code.gson gson @@ -110,10 +122,15 @@ true - org.jenkins-ci.plugins.workflow - workflow-step-api - 2.15 - true + org.jenkins-ci.plugins + script-security + 1.39 + + + org.awaitility + awaitility + 3.1.6 + test org.mockito @@ -133,18 +150,6 @@ 2.0.0-beta.5 test - - org.awaitility - awaitility - 3.0.0 - test - - - org.jenkins-ci.plugins - script-security - 1.39 - test - org.jenkins-ci.plugins mask-passwords diff --git a/src/main/java/jenkins/plugins/logstash/LogstashBuildWrapper.java b/src/main/java/jenkins/plugins/logstash/LogstashBuildWrapper.java index 81ec9f41..3d900d9b 100644 --- a/src/main/java/jenkins/plugins/logstash/LogstashBuildWrapper.java +++ b/src/main/java/jenkins/plugins/logstash/LogstashBuildWrapper.java @@ -48,60 +48,60 @@ public class LogstashBuildWrapper extends BuildWrapper { - /** - * Create a new {@link LogstashBuildWrapper}. - */ - @DataBoundConstructor - public LogstashBuildWrapper() - {} - - /** - * {@inheritDoc} - */ - @Override - public Environment setUp(AbstractBuild build, Launcher launcher, BuildListener listener) - throws IOException, InterruptedException - { - return new Environment() - { - }; - } - - @Override - public DescriptorImpl getDescriptor() - { - return (DescriptorImpl)super.getDescriptor(); - } - - /** - * Registers {@link LogstashBuildWrapper} as a {@link BuildWrapper}. - */ - @Extension - public static class DescriptorImpl extends BuildWrapperDescriptor - { - - public DescriptorImpl() - { - super(LogstashBuildWrapper.class); - load(); - } + /** + * Create a new {@link LogstashBuildWrapper}. + */ + @DataBoundConstructor + public LogstashBuildWrapper() + {} /** * {@inheritDoc} */ @Override - public String getDisplayName() + public Environment setUp(AbstractBuild build, Launcher launcher, BuildListener listener) + throws IOException, InterruptedException { - return Messages.DisplayName(); + return new Environment() + { + }; + } + + @Override + public DescriptorImpl getDescriptor() + { + return (DescriptorImpl)super.getDescriptor(); } /** - * {@inheritDoc} + * Registers {@link LogstashBuildWrapper} as a {@link BuildWrapper}. */ - @Override - public boolean isApplicable(AbstractProject item) + @Extension + public static class DescriptorImpl extends BuildWrapperDescriptor { - return false; + + public DescriptorImpl() + { + super(LogstashBuildWrapper.class); + load(); + } + + /** + * {@inheritDoc} + */ + @Override + public String getDisplayName() + { + return Messages.DisplayName(); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean isApplicable(AbstractProject item) + { + return false; + } } - } } diff --git a/src/main/java/jenkins/plugins/logstash/LogstashOutputStream.java b/src/main/java/jenkins/plugins/logstash/LogstashOutputStream.java index f6ae5de2..5a0621f2 100644 --- a/src/main/java/jenkins/plugins/logstash/LogstashOutputStream.java +++ b/src/main/java/jenkins/plugins/logstash/LogstashOutputStream.java @@ -79,6 +79,7 @@ public void flush() throws IOException { */ @Override public void close() throws IOException { + logstash.close(); delegate.close(); super.close(); } diff --git a/src/main/java/jenkins/plugins/logstash/LogstashPayloadProcessor.java b/src/main/java/jenkins/plugins/logstash/LogstashPayloadProcessor.java new file mode 100644 index 00000000..8e62e7a9 --- /dev/null +++ b/src/main/java/jenkins/plugins/logstash/LogstashPayloadProcessor.java @@ -0,0 +1,47 @@ +/* + * The MIT License + * + * Copyright 2017 Red Hat inc, and individual contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package jenkins.plugins.logstash; + +import net.sf.json.JSONObject; + +/** + * Interface describing processors of persisted payload. + */ +public interface LogstashPayloadProcessor { + /** + * Modifies a JSON payload compatible with the Logstash schema. + * + * @param payload the JSON payload that has been constructed so far. + * @return The formatted JSON object, can be null to ignore this payload. + */ + JSONObject process(JSONObject payload) throws Exception; + + /** + * Finalizes any operations, for example returns cashed lines at end of build. + * + * @return A formatted JSON object, can be null when it has nothing. + */ + JSONObject finish() throws Exception; +} diff --git a/src/main/java/jenkins/plugins/logstash/LogstashScriptProcessor.java b/src/main/java/jenkins/plugins/logstash/LogstashScriptProcessor.java new file mode 100644 index 00000000..16c1221b --- /dev/null +++ b/src/main/java/jenkins/plugins/logstash/LogstashScriptProcessor.java @@ -0,0 +1,109 @@ +/* + * The MIT License + * + * Copyright 2017 Red Hat inc. and individual contributors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package jenkins.plugins.logstash; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import groovy.lang.Binding; +import jenkins.model.Jenkins; +import net.sf.json.JSONObject; +import org.jenkinsci.plugins.scriptsecurity.sandbox.groovy.SecureGroovyScript; +import org.jenkinsci.plugins.scriptsecurity.sandbox.whitelists.Whitelisted; + +import javax.annotation.Nonnull; +import java.io.IOException; +import java.io.OutputStream; + +/** + * This class is handling custom groovy script processing of JSON payload. + * Each call to process executes the script provided in job configuration. + * Script is executed under the same binding each time so that it has ability + * to persist data during build execution if desired by script author. + * When build is finished, script will receive null as the payload and can + * return any cached but non-sent data back for persisting. + * The return value of script is the payload to be persisted unless null. + */ +public class LogstashScriptProcessor implements LogstashPayloadProcessor{ + @Nonnull + private final SecureGroovyScript script; + + @Nonnull + private final OutputStream consoleOut; + + /** Groovy binding for script execution */ + @Nonnull + private final Binding binding; + + /** Classloader for script execution */ + @Nonnull + private final ClassLoader classLoader; + + public LogstashScriptProcessor(SecureGroovyScript script, OutputStream consoleOut) { + this.script = script; + this.consoleOut = consoleOut; + + // TODO: should we put variables in the binding like manager, job, etc.? + binding = new Binding(); + binding.setVariable("console", new BuildConsoleWrapper()); + + // not sure what the diff is compared to getClass().getClassLoader(); + final Jenkins jenkins = Jenkins.getInstance(); + classLoader = jenkins.getPluginManager().uberClassLoader; + } + + /** + * Helper method to allow logging to build console. + */ + @SuppressFBWarnings( + value="DM_DEFAULT_ENCODING", + justification="TODO: not sure how to fix this") + private void buildLogPrintln(Object o) throws IOException { + consoleOut.write(o.toString().getBytes()); + consoleOut.write("\n".getBytes()); + consoleOut.flush(); + } + + @Override + public JSONObject process(JSONObject payload) throws Exception { + binding.setVariable("payload", payload); + script.evaluate(classLoader, binding); + return (JSONObject) binding.getVariable("payload"); + } + + @Override + public JSONObject finish() throws Exception { + buildLogPrintln("Tearing down Script Log Processor.."); + return process(null); + } + + /** + * Helper to allow access from sandboxed script to output messages to console. + */ + private class BuildConsoleWrapper { + @Whitelisted + public void println(Object o) throws IOException { + buildLogPrintln(o); + } + } +} diff --git a/src/main/java/jenkins/plugins/logstash/LogstashWriter.java b/src/main/java/jenkins/plugins/logstash/LogstashWriter.java index 2523814a..f5658542 100644 --- a/src/main/java/jenkins/plugins/logstash/LogstashWriter.java +++ b/src/main/java/jenkins/plugins/logstash/LogstashWriter.java @@ -35,6 +35,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.exception.ExceptionUtils; +import javax.annotation.CheckForNull; import java.io.IOException; import java.io.OutputStream; import java.nio.charset.Charset; @@ -62,11 +63,23 @@ public class LogstashWriter { private boolean connectionBroken; private final Charset charset; + @CheckForNull + private final LogstashPayloadProcessor payloadProcessor; + + public LogstashWriter(Run run, OutputStream error, TaskListener listener) { + this(run, error, listener, null); + } + public LogstashWriter(Run run, OutputStream error, TaskListener listener, Charset charset) { + this(run, error, listener, charset, null); + } + + public LogstashWriter(Run run, OutputStream error, TaskListener listener, Charset charset, LogstashPayloadProcessor payloadProcessor) { this.errorStream = error != null ? error : System.err; this.build = run; this.listener = listener; this.charset = charset; + this.payloadProcessor = payloadProcessor; this.dao = this.getDaoOrNull(); if (this.dao == null) { this.jenkinsUrl = ""; @@ -178,6 +191,21 @@ private void write(List lines) { } } + /** + * Write JSONObject payload to the Logstash indexer. + * @since 1.0.5 + */ + private void writeRaw(JSONObject payload) { + try { + dao.push(payload.toString()); + } catch (IOException e) { + String msg = "[logstash-plugin]: Failed to send log data: " + dao.getDescription() + ".\n" + + "[logstash-plugin]: No Further logs will be sent to " + dao.getDescription() + ".\n" + + ExceptionUtils.getStackTrace(e); + logErrorMessage(msg); + } + } + /** * Construct a valid indexerDao or return null. * Writes errors to errorStream if dao constructor fails. @@ -215,4 +243,23 @@ private void logErrorMessage(String msg) { } } + /** + * Signal payload processor that there will be no more lines + */ + public void close() { + if (payloadProcessor != null) { + JSONObject payload = null; + try { + // calling finish() is mandatory to avoid memory leaks + payload = payloadProcessor.finish(); + } catch (Exception e) { + String msg = ExceptionUtils.getMessage(e) + "\n" + + "[logstash-plugin]: Error with payload processor on finish.\n"; + + logErrorMessage(msg); + } + if (payload != null) writeRaw(payload); + } + } + } diff --git a/src/main/java/jenkins/plugins/logstash/configuration/Logzio.java b/src/main/java/jenkins/plugins/logstash/configuration/Logzio.java new file mode 100644 index 00000000..7c3850fa --- /dev/null +++ b/src/main/java/jenkins/plugins/logstash/configuration/Logzio.java @@ -0,0 +1,102 @@ +package jenkins.plugins.logstash.configuration; + +import hudson.Extension; +import hudson.util.FormValidation; +import hudson.util.Secret; + +import jenkins.plugins.logstash.persistence.LogzioDao; +import jenkins.plugins.logstash.Messages; + +import org.apache.commons.lang.StringUtils; +import org.kohsuke.stapler.DataBoundConstructor; +import org.kohsuke.stapler.DataBoundSetter; +import org.kohsuke.stapler.QueryParameter; + +import javax.annotation.Nonnull; +import java.util.Objects; + +public class Logzio extends LogstashIndexer +{ + private Secret token; + private String host; + + @DataBoundConstructor + public Logzio(){} + + public String getHost(){ return this.host; } + + @DataBoundSetter + public void setHost(String host){ this.host = host; } + + public String getToken() + { + return Secret.toString(token); + } + + @DataBoundSetter + public void setToken(String token) + { + this.token = Secret.fromString(token); + } + + + @Override + public boolean equals(Object obj) { + if (obj == null) + return false; + if (this == obj) + return true; + if (getClass() != obj.getClass()) + return false; + Logzio other = (Logzio) obj; + return Secret.toString(token).equals(other.getToken()) && Objects.equals(host, other.host); + } + + @Override + public int hashCode() + { + final int prime = 31; + int result = super.hashCode(); + result = prime * result + ((host == null) ? 0 : host.hashCode()); + result = prime * result + Secret.toString(token).hashCode(); + return result; + } + + @Override + public LogzioDao createIndexerInstance() { return new LogzioDao(host, Secret.toString(token)); } + + @Extension + public static class LogzioDescriptor extends LogstashIndexerDescriptor + { + @Nonnull + @Override + public String getDisplayName() + { + return "Logz.io"; + } + + @Override + public int getDefaultPort() + { + return 0; + } + + public FormValidation doCheckToken(@QueryParameter("value") String value) + { + if (StringUtils.isBlank(value)) + { + return FormValidation.error(Messages.ValueIsRequired()); + } + return FormValidation.ok(); + } + + public FormValidation doCheckHost(@QueryParameter("value") String value) + { + if (StringUtils.isBlank(value)) + { + return FormValidation.error(Messages.ValueIsRequired()); + } + return FormValidation.ok(); + } + } +} diff --git a/src/main/java/jenkins/plugins/logstash/persistence/ElasticSearchDao.java b/src/main/java/jenkins/plugins/logstash/persistence/ElasticSearchDao.java index 65f4de99..823f931a 100644 --- a/src/main/java/jenkins/plugins/logstash/persistence/ElasticSearchDao.java +++ b/src/main/java/jenkins/plugins/logstash/persistence/ElasticSearchDao.java @@ -24,7 +24,7 @@ package jenkins.plugins.logstash.persistence; -import static com.google.common.collect.Ranges.closedOpen; +import static com.google.common.collect.Range.closedOpen; import org.apache.commons.codec.binary.Base64; import org.apache.commons.lang.StringUtils; diff --git a/src/main/java/jenkins/plugins/logstash/persistence/LogzioDao.java b/src/main/java/jenkins/plugins/logstash/persistence/LogzioDao.java new file mode 100644 index 00000000..36f30499 --- /dev/null +++ b/src/main/java/jenkins/plugins/logstash/persistence/LogzioDao.java @@ -0,0 +1,109 @@ +package jenkins.plugins.logstash.persistence; + +import com.github.wnameless.json.flattener.JsonFlattener; +import com.google.gson.JsonObject; +import io.logz.sender.FormattedLogMessage; +import io.logz.sender.exceptions.LogzioParameterErrorException; +import io.logz.sender.exceptions.LogzioServerErrorException; +import net.sf.json.JSONArray; +import net.sf.json.JSONObject; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.List; +import java.util.Map; + +/** + * Logz.io Data Access Object. + * + * @author Ido Halevi + * + */ + +public class LogzioDao extends AbstractLogstashIndexerDao { + private final String TYPE = "jenkins_logstash_plugin"; + private String key; + private String host; + private LogzioHttpsClient httpsClient; + + //primary constructor used by indexer factory + public LogzioDao(String host, String key){ + this(null, host, key); + } + + // Factored for unit testing + LogzioDao(LogzioHttpsClient factory, String host, String key) { + this.host = host; + this.key = key; + try{ + this.httpsClient = factory == null ? new LogzioHttpsClient(key, host, TYPE) : factory; + }catch (LogzioParameterErrorException e) { + throw new IllegalArgumentException(e); + } + } + + @Override + public synchronized void push(String data) throws IOException { + JSONObject jsonData = JSONObject.fromObject(data); + JSONArray logMessages = jsonData.getJSONArray("message"); + if (!logMessages.isEmpty()) { + try{ + for (Object logMsg : logMessages) { + JsonObject logLine = createLogLine(jsonData, logMsg.toString()); + httpsClient.send(new FormattedLogMessage((logLine + "\n").getBytes(Charset.forName("UTF-8")))); + } + httpsClient.flush(); + }catch (LogzioServerErrorException e){ + throw new IOException(e); + } + } + } + + + private JsonObject createLogLine(JSONObject jsonData, String logMsg) { + JsonObject logLine = new JsonObject(); + + logLine.addProperty("message", logMsg); + logLine.addProperty("@timestamp", ZonedDateTime.now(ZoneOffset.UTC).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + + for (Object key : jsonData.keySet()) { + if (!key.equals("message")){ + logLine.addProperty(key.toString(), jsonData.getString(key.toString())); + } + } + + return logLine; + } + + @Override + public JSONObject buildPayload(BuildData buildData, String jenkinsUrl, List logLines) { + JSONObject payload = new JSONObject(); + payload.put("message", logLines); + payload.put("source", "jenkins"); + payload.put("source_host", jenkinsUrl); + payload.put("@buildTimestamp", buildData.getTimestamp()); + payload.put("@version", 1); + // Flatten build data - so the user will be able to use fields for visualization in Kibana. + // In addition, it makes the query much easier. + Map flattenJson = JsonFlattener.flattenAsMap(buildData.toString()); + for (Map.Entry entry : flattenJson.entrySet()) { + String key = entry.getKey().replace('.','_'); + Object value = entry.getValue(); + payload.put(key, value); + } + + return payload; + } + + @Override + public String getDescription(){ return host; } + + public String getHost(){ return host; } + + public String getKey(){ return key; } + + public String getType(){ return TYPE; } +} diff --git a/src/main/java/jenkins/plugins/logstash/persistence/LogzioHttpsClient.java b/src/main/java/jenkins/plugins/logstash/persistence/LogzioHttpsClient.java new file mode 100644 index 00000000..04462bd3 --- /dev/null +++ b/src/main/java/jenkins/plugins/logstash/persistence/LogzioHttpsClient.java @@ -0,0 +1,101 @@ +package jenkins.plugins.logstash.persistence; + +import io.logz.sender.FormattedLogMessage; +import io.logz.sender.HttpsRequestConfiguration; +import io.logz.sender.HttpsSyncSender; +import io.logz.sender.SenderStatusReporter; +import io.logz.sender.exceptions.LogzioParameterErrorException; +import io.logz.sender.exceptions.LogzioServerErrorException; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +public class LogzioHttpsClient{ + private static final int MAX_SIZE_IN_BYTES = 8 * 1024 * 1024; // 8 MB + private static final int CONNECT_TIMEOUT = 10 * 1000; + private static final int SOCKET_TIMEOUT = 10 * 1000; + private final HttpsSyncSender logzioClient; + private List messages; + private int size; + + LogzioHttpsClient(String token, String listener, String type) throws LogzioParameterErrorException { + HttpsRequestConfiguration gzipHttpsRequestConfiguration = HttpsRequestConfiguration + .builder() + .setLogzioToken(token) + .setLogzioType(type) + .setLogzioListenerUrl(listener) + .setSocketTimeout(SOCKET_TIMEOUT) + .setConnectTimeout(CONNECT_TIMEOUT) + .setCompressRequests(true) + .build(); + + logzioClient = new HttpsSyncSender(gzipHttpsRequestConfiguration, new Reporter()); + messages = Collections.synchronizedList(new ArrayList()); + size = 0; + } + + public void send(FormattedLogMessage log) throws LogzioServerErrorException { + if (size + log.getSize() > MAX_SIZE_IN_BYTES) { + sendAndReset(); + } + messages.add(log); + size += log.getSize(); + } + + private void reset(){ + size = 0; + messages.clear(); + } + + void flush() throws LogzioServerErrorException { + if(messages.size() > 0) { + sendAndReset(); + } + } + + private void sendAndReset() throws LogzioServerErrorException { + logzioClient.sendToLogzio(messages); + reset(); + } + + private static class Reporter implements SenderStatusReporter{ + private static final Logger LOGGER = Logger.getLogger(LogzioDao.class.getName()); + + private void pringLogMessage(Level level, String msg) { + LOGGER.log(level, msg); + } + + @Override + public void error(String msg) { + pringLogMessage(Level.SEVERE, "[LogzioSender]ERROR: " + msg); + } + + @Override + public void error(String msg, Throwable e) { + pringLogMessage(Level.SEVERE, "[LogzioSender]ERROR: " + msg + "\n" +e); + } + + @Override + public void warning(String msg) { + pringLogMessage(Level.WARNING, "[LogzioSender]WARNING: " + msg); + } + + @Override + public void warning(String msg, Throwable e) { + pringLogMessage(Level.WARNING, "[LogzioSender]WARNING: " + msg + "\n" + e); + } + + @Override + public void info(String msg) { + pringLogMessage(Level.INFO, "[LogzioSender]INFO: " + msg); + } + + @Override + public void info(String msg, Throwable e) { + pringLogMessage(Level.INFO, "[LogzioSender]INFO: " + msg + "\n" + e); + } + } +} diff --git a/src/main/resources/index.jelly b/src/main/resources/index.jelly index 8f22feef..147c6c6c 100644 --- a/src/main/resources/index.jelly +++ b/src/main/resources/index.jelly @@ -1,4 +1,4 @@
- Adds the possibility to push builds logs and build data to a Logstash indexer such as Redis, RabbitMQ, Elastic Search or to Syslog. + Adds the possibility to push builds logs and build data to a Logstash indexer such as Redis, RabbitMQ, Elastic Search, Logz.io or to Syslog.
diff --git a/src/main/resources/jenkins/plugins/logstash/configuration/Logzio/config.jelly b/src/main/resources/jenkins/plugins/logstash/configuration/Logzio/config.jelly new file mode 100644 index 00000000..2f26e1f1 --- /dev/null +++ b/src/main/resources/jenkins/plugins/logstash/configuration/Logzio/config.jelly @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/src/main/resources/jenkins/plugins/logstash/configuration/Logzio/help-host.jelly b/src/main/resources/jenkins/plugins/logstash/configuration/Logzio/help-host.jelly new file mode 100644 index 00000000..fe3675f4 --- /dev/null +++ b/src/main/resources/jenkins/plugins/logstash/configuration/Logzio/help-host.jelly @@ -0,0 +1,6 @@ + +
+

Logz.io listener URL.
+ If you are in the EU region insert https://listener-eu.logz.io:8071. Otherwise, use https://listener.logz.io:8071
+ You can tell which region you are in by checking your login URL

+
\ No newline at end of file diff --git a/src/main/resources/jenkins/plugins/logstash/configuration/Logzio/help-token.jelly b/src/main/resources/jenkins/plugins/logstash/configuration/Logzio/help-token.jelly new file mode 100644 index 00000000..cfb2dc00 --- /dev/null +++ b/src/main/resources/jenkins/plugins/logstash/configuration/Logzio/help-token.jelly @@ -0,0 +1,4 @@ + +
+

The Logz.io account token.

+
diff --git a/src/main/resources/jenkins/plugins/logstash/configuration/Logzio/help.jelly b/src/main/resources/jenkins/plugins/logstash/configuration/Logzio/help.jelly new file mode 100644 index 00000000..d8e01e22 --- /dev/null +++ b/src/main/resources/jenkins/plugins/logstash/configuration/Logzio/help.jelly @@ -0,0 +1,4 @@ + + + Push to Logz.io with HTTPs input. + \ No newline at end of file diff --git a/src/test/java/jenkins/plugins/logstash/LogstashConfigurationMigrationTest.java b/src/test/java/jenkins/plugins/logstash/LogstashConfigurationMigrationTest.java index 0785ffe7..61b5568d 100644 --- a/src/test/java/jenkins/plugins/logstash/LogstashConfigurationMigrationTest.java +++ b/src/test/java/jenkins/plugins/logstash/LogstashConfigurationMigrationTest.java @@ -150,5 +150,4 @@ public void rabbitMqMigration() assertThat(es.getPassword(), equalTo("pwd")); assertThat(es.getUsername(), equalTo("user")); } - } diff --git a/src/test/java/jenkins/plugins/logstash/LogstashOutputStreamTest.java b/src/test/java/jenkins/plugins/logstash/LogstashOutputStreamTest.java index 7752bf38..e7b6d994 100644 --- a/src/test/java/jenkins/plugins/logstash/LogstashOutputStreamTest.java +++ b/src/test/java/jenkins/plugins/logstash/LogstashOutputStreamTest.java @@ -1,6 +1,7 @@ package jenkins.plugins.logstash; import static org.hamcrest.core.StringContains.containsString; +import static org.hamcrest.object.HasToString.hasToString; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; import static org.mockito.Mockito.*; @@ -13,6 +14,7 @@ import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.InOrder; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; @@ -122,4 +124,17 @@ public void eolSuccessNoDao() throws Exception { assertEquals("Results don't match", msg, buffer.toString()); verify(mockWriter).isConnectionBroken(); } + + @Test + public void writerClosedBeforeDelegate() throws Exception { + ByteArrayOutputStream mockBuffer = Mockito.spy(buffer); + new LogstashOutputStream(mockBuffer, mockWriter).close(); + + InOrder inOrder = Mockito.inOrder(mockBuffer, mockWriter); + inOrder.verify(mockWriter).close(); + inOrder.verify(mockBuffer).close(); + + // Verify results + assertThat(buffer, hasToString("")); + } } diff --git a/src/test/java/jenkins/plugins/logstash/configuration/LogzioTest.java b/src/test/java/jenkins/plugins/logstash/configuration/LogzioTest.java new file mode 100644 index 00000000..0f30fd05 --- /dev/null +++ b/src/test/java/jenkins/plugins/logstash/configuration/LogzioTest.java @@ -0,0 +1,44 @@ +package jenkins.plugins.logstash.configuration; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.jvnet.hudson.test.JenkinsRule; + +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; + +public class LogzioTest{ + @Rule + public JenkinsRule j = new JenkinsRule(); + + private Logzio indexer; + private Logzio indexer2; + + @Before + public void setup(){ + indexer = new Logzio(); + indexer.setHost("https://listener.logz.io:8071"); + indexer.setToken("token"); + + indexer2 = new Logzio(); + indexer2.setHost("https://listener.logz.io:8071"); + indexer2.setToken("token"); + } + + @Test + public void sameSettingsAreEqual(){ assertThat(indexer.equals(indexer2), is(true)); } + + @Test + public void tokenChangeIsNotEqual() { + indexer.setToken("newPassword"); + assertThat(indexer.equals(indexer2), is(false)); + } + + @Test + public void hostChangeIsNotEqual() { + indexer.setHost("https://logz.io"); + assertThat(indexer.equals(indexer2), is(false)); + } + +} diff --git a/src/test/java/jenkins/plugins/logstash/persistence/LogzioDaoTest.java b/src/test/java/jenkins/plugins/logstash/persistence/LogzioDaoTest.java new file mode 100644 index 00000000..31b3c794 --- /dev/null +++ b/src/test/java/jenkins/plugins/logstash/persistence/LogzioDaoTest.java @@ -0,0 +1,162 @@ +package jenkins.plugins.logstash.persistence; + +import io.logz.sender.FormattedLogMessage; +import io.logz.sender.exceptions.LogzioServerErrorException; +import net.sf.json.JSONObject; +import net.sf.json.test.JSONAssert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.powermock.api.mockito.PowerMockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class LogzioDaoTest { + + private static final String data = "{\"a\":{\"b\":1,\"c\":2,\"d\":[false, true]},\"e\":\"f\",\"g\":2.3}"; + private static final String flat_data = "\"a_b\":1,\"a_c\":2,\"a_d[0]\":false,\"a_d[1]\":true,\"e\":\"f\",\"g\":2.3"; + private static final String EMPTY_STRING_WITH_DATA = "{\"@buildTimestamp\":\"2000-01-01\"," + flat_data + ",\"message\":[],\"source\":\"jenkins\",\"source_host\":\"http://localhost:8080/jenkins\",\"@version\":1}"; + private static final String ONE_LINE_STRING_WITH_DATA = "{\"@buildTimestamp\":\"2000-01-01\"," + flat_data + ",\"message\":[\"LINE 1\"],\"source\":\"jenkins\",\"source_host\":\"http://localhost:8080/jenkins\",\"@version\":1}"; + private static final String TWO_LINE_STRING_WITH_DATA = "{\"@buildTimestamp\":\"2000-01-01\"," + flat_data + ",\"message\":[\"LINE 1\", \"LINE 2\"],\"source\":\"jenkins\",\"source_host\":\"http://localhost:8080/jenkins\",\"@version\":1}"; + private static final String EMPTY_STRING_NO_DATA = "{\"@buildTimestamp\":\"2000-01-01\",\"message\":[],\"source\":\"jenkins\",\"source_host\":\"http://localhost:8080/jenkins\",\"@version\":1}"; + private static final String ONE_LINE_STRING_NO_DATA = "{\"@buildTimestamp\":\"2000-01-01\",\"message\":[\"LINE 1\"],\"source\":\"jenkins\",\"source_host\":\"http://localhost:8080/jenkins\",\"@version\":1}"; + private static final String TWO_LINE_STRING_NO_DATA = "{\"@buildTimestamp\":\"2000-01-01\",\"message\":[\"LINE 1\", \"LINE 2\"],\"source\":\"jenkins\",\"source_host\":\"http://localhost:8080/jenkins\",\"@version\":1}"; + private LogzioDao dao; + + @Captor private ArgumentCaptor sendArgument = ArgumentCaptor.forClass(FormattedLogMessage.class); + + @Mock private LogzioHttpsClient logzioSender; + @Mock private BuildData mockBuildData; + + private LogzioDao createDao(String host, String key) throws IllegalArgumentException { + return new LogzioDao(logzioSender, host, key); + } + + @Before + public void before() throws IllegalArgumentException, LogzioServerErrorException { + when(mockBuildData.getTimestamp()).thenReturn("2000-01-01"); + + doNothing().when(logzioSender).send(any(FormattedLogMessage.class)); + doNothing().when(logzioSender).flush(); + + dao = createDao("http://localhost:8200/", "123456789"); + + } + + @Test + public void constructorSuccess() throws IllegalArgumentException { + // Unit under test + dao = createDao("https://localhost:8201/", "123"); + + // Verify results + assertEquals("Wrong host name", "https://localhost:8201/", dao.getHost()); + assertEquals("Wrong key", "123", dao.getKey()); + } + + @Test + public void buildPayloadSuccessEmpty(){ + when(mockBuildData.toString()).thenReturn("{}"); + // Unit under test + JSONObject result = dao.buildPayload(mockBuildData, "http://localhost:8080/jenkins", new ArrayList()); + result.remove("@timestamp"); + + // Verify results + JSONAssert.assertEquals("Results don't match", JSONObject.fromObject(EMPTY_STRING_NO_DATA), result); + } + + @Test + public void buildPayloadSuccessOneLine(){ + when(mockBuildData.toString()).thenReturn("{}"); + // Unit under test + JSONObject result = dao.buildPayload(mockBuildData, "http://localhost:8080/jenkins", Collections.singletonList("LINE 1")); + result.remove("@timestamp"); + + // Verify results + JSONAssert.assertEquals("Results don't match", JSONObject.fromObject(ONE_LINE_STRING_NO_DATA), result); + } + + @Test + public void buildPayloadSuccessTwoLines(){ + when(mockBuildData.toString()).thenReturn("{}"); + // Unit under test + JSONObject result = dao.buildPayload(mockBuildData, "http://localhost:8080/jenkins", Arrays.asList("LINE 1", "LINE 2")); + result.remove("@timestamp"); + + // Verify results + JSONAssert.assertEquals("Results don't match", JSONObject.fromObject(TWO_LINE_STRING_NO_DATA), result); + } + + @Test + public void buildPayloadWithDataSuccessEmpty(){ + when(mockBuildData.toString()).thenReturn(data); + // Unit under test + JSONObject result = dao.buildPayload(mockBuildData, "http://localhost:8080/jenkins", new ArrayList()); + result.remove("@timestamp"); + + // Verify results + JSONAssert.assertEquals("Results don't match", JSONObject.fromObject(EMPTY_STRING_WITH_DATA), result); + } + + @Test + public void buildPayloadWithDataSuccessOneLine(){ + when(mockBuildData.toString()).thenReturn(data); + // Unit under test + JSONObject result = dao.buildPayload(mockBuildData, "http://localhost:8080/jenkins", Collections.singletonList("LINE 1")); + result.remove("@timestamp"); + + // Verify results + JSONAssert.assertEquals("Results don't match", JSONObject.fromObject(ONE_LINE_STRING_WITH_DATA), result); + } + + @Test + public void buildPayloadWithDataSuccessTwoLines(){ + when(mockBuildData.toString()).thenReturn(data); + // Unit under test + JSONObject result = dao.buildPayload(mockBuildData, "http://localhost:8080/jenkins", Arrays.asList("LINE 1", "LINE 2")); + result.remove("@timestamp"); + + // Verify results + JSONAssert.assertEquals("Results don't match", JSONObject.fromObject(TWO_LINE_STRING_WITH_DATA), result); + } + + @Test + public void pushNoMessage() throws IOException, LogzioServerErrorException { + // Unit under test + dao.push(EMPTY_STRING_WITH_DATA); + verify(logzioSender, never()).send(sendArgument.capture()); + verify(logzioSender, never()).flush(); + } + + @Test + public void pushOneMessage() throws IOException, LogzioServerErrorException { + // Unit under test + dao.push(ONE_LINE_STRING_WITH_DATA); + // Verify results + verify(logzioSender, times(1)).send(sendArgument.capture()); + verify(logzioSender, times(1)).flush(); + } + + @Test + public void pushMultiMessages() throws IOException, LogzioServerErrorException { + // Unit under test + dao.push(TWO_LINE_STRING_WITH_DATA); + // Verify results + verify(logzioSender, times(2)).send(sendArgument.capture()); + verify(logzioSender, times(1)).flush(); + } +} diff --git a/src/test/resources/logzio.xml b/src/test/resources/logzio.xml new file mode 100644 index 00000000..5e81bdd1 --- /dev/null +++ b/src/test/resources/logzio.xml @@ -0,0 +1,8 @@ + + + + https://listener.logz.io:8071 + {AQAAABAAAAAQAwaAxyveddM0PF+kR0dYFAymdth9PpitQnvJW0SR6JU=} + + true + \ No newline at end of file