Add Logz.io support

README.md

@@ -27,6 +27,7 @@ Currently supported methods of input/output:
 * Redis {format => 'json_event'}
 * RabbitMQ {mechanism => PLAIN}
 * Syslog {format => cee/json ([RFC-5424](https://tools.ietf.org/html/rfc5424),[RFC-3164](https://tools.ietf.org/html/rfc3164)), protocol => UDP}
+* Logz.io
 
 Pipeline
 ========

pom.xml

@@ -6,10 +6,10 @@
     <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
     <skipIntegrationTests>true</skipIntegrationTests>
     <jenkins-test-harness.version>2.23</jenkins-test-harness.version>
-    <java.level>7</java.level>
+    <java.level>8</java.level>
     <jenkins.version>2.7.4</jenkins.version>
   </properties>
-  
+
   <parent>
     <groupId>org.jenkins-ci.plugins</groupId>
     <artifactId>plugin</artifactId>
@@ -109,6 +109,17 @@
       <scope>compile</scope>
     </dependency>
 
+    <dependency>
+      <groupId>io.logz.sender</groupId>
+      <artifactId>logzio-sender</artifactId>
+      <version>1.0.10</version>
+    </dependency>
+
+    <dependency>
+      <groupId>com.github.wnameless</groupId>
+      <artifactId>json-flattener</artifactId>
+      <version>0.5.0</version>
+    </dependency>
 
     <dependency>
       <groupId>org.jenkins-ci.plugins</groupId>
@@ -196,8 +207,8 @@
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-compiler-plugin</artifactId>
         <configuration>
-          <source>1.7</source>
-          <target>1.7</target>
+          <source>1.8</source>
+          <target>1.8</target>
         </configuration>
       </plugin>
 

src/main/java/jenkins/plugins/logstash/LogstashConfiguration.java

@@ -25,6 +25,7 @@
 import jenkins.plugins.logstash.configuration.RabbitMq;
 import jenkins.plugins.logstash.configuration.Redis;
 import jenkins.plugins.logstash.configuration.Syslog;
+import jenkins.plugins.logstash.configuration.Logzio;
 import jenkins.plugins.logstash.persistence.LogstashIndexerDao;
 import jenkins.plugins.logstash.persistence.LogstashIndexerDao.IndexerType;
 import net.sf.json.JSONObject;
@@ -211,6 +212,13 @@ public void migrateData()
             logstashIndexer = syslog;
             enabled = true;
             break;
+          case LOGZIO:
+            LOGGER.log(Level.INFO, "Migrating logstash configuration for Logz.io");
+            Logzio logz = new Logzio();
+            logz.setHost(descriptor.getHost());
+            logz.setKey(descriptor.getKey());
+            logstashIndexer = logz;
+            break;
           default:
             LOGGER.log(Level.INFO, "unknown logstash Indexer type: " + type);
             enabled = false;

src/main/java/jenkins/plugins/logstash/configuration/Logzio.java

@@ -0,0 +1,120 @@
+package jenkins.plugins.logstash.configuration;
+
+import hudson.Extension;
+import hudson.util.FormValidation;
+import hudson.util.Secret;
+
+import jenkins.plugins.logstash.persistence.LogzioDao;
+import jenkins.plugins.logstash.Messages;
+
+import org.apache.commons.lang.StringUtils;
+import org.kohsuke.stapler.DataBoundConstructor;
+import org.kohsuke.stapler.DataBoundSetter;
+import org.kohsuke.stapler.QueryParameter;
+
+import javax.annotation.Nonnull;
+
+public class Logzio extends LogstashIndexer<LogzioDao>
+{
+    private Secret key;
+    private String host;
+
+    @DataBoundConstructor
+    public Logzio(){}
+
+    /*
+     * We use URL for the setter as stapler can autoconvert a string to a URL but not to a URI
+     */
+    public String getHost(){ return this.host; }
+
+    @DataBoundSetter
+    public void setHost(String host){ this.host = host; }
+
+    public String getKey()
+    {
+        return Secret.toString(key);
+    }
+
+    @DataBoundSetter
+    public void setKey(String key)
+    {
+        this.key = Secret.fromString(key);
+    }
+
+
+    @Override
+    public boolean equals(Object obj)
+    {
+        if (obj == null)
+            return false;
+        if (this == obj)
+            return true;
+        if (getClass() != obj.getClass())
+            return false;
+        Logzio other = (Logzio) obj;
+        if (!Secret.toString(key).equals(other.getKey()))
+        {
+            return false;
+        }
+        if (host == null)
+        {
+            return other.host == null;
+        }
+        else return host.equals(other.host);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        final int prime = 31;
+        int result = super.hashCode();
+        result = prime * result + ((host == null) ? 0 : host.hashCode());
+        result = prime * result + Secret.toString(key).hashCode();
+        return result;
+    }
+
+    @Override
+    public LogzioDao createIndexerInstance() { return new LogzioDao(host, Secret.toString(key)); }
+
+    @Extension
+    public static class LogzioDescriptor extends LogstashIndexerDescriptor
+    {
+        private static String EU_HOST = "https://listener-eu.logz.io:8071";
+        private static String NONEU_HOST = "https://listener.logz.io:8071";
+
+        @Nonnull
+        @Override
+        public String getDisplayName()
+        {
+            return "Logz.io";
+        }
+
+        @Override
+        public int getDefaultPort()
+        {
+            return 0;
+        }
+
+        public FormValidation doCheckKey(@QueryParameter("value") String value)
+        {
+            if (StringUtils.isBlank(value))
+            {
+                return FormValidation.error(Messages.ValueIsRequired());
+            }
+            return FormValidation.ok();
+        }
+
+        public FormValidation doCheckHost(@QueryParameter("value") String value)
+        {
+            if (StringUtils.isBlank(value))
+            {
+                return FormValidation.error(Messages.ValueIsRequired());
+            }
+            else if (!(value.equals(EU_HOST) || value.equals(NONEU_HOST))){
+                return FormValidation.error("Please verify your logz.io host is one of the two possible hosts - "
+                + EU_HOST + " or " + NONEU_HOST);
+            }
+            return FormValidation.ok();
+        }
+    }
+}

src/main/java/jenkins/plugins/logstash/persistence/LogstashIndexerDao.java

@@ -42,7 +42,8 @@ static enum IndexerType {
     REDIS,
     RABBIT_MQ,
     ELASTICSEARCH,
-    SYSLOG
+    SYSLOG,
+    LOGZIO
   }
 
   @Deprecated

src/main/java/jenkins/plugins/logstash/persistence/LogzioDao.java

@@ -0,0 +1,127 @@
+package jenkins.plugins.logstash.persistence;
+
+import com.github.wnameless.json.flattener.JsonFlattener;
+import com.google.common.io.Files;
+
+import io.logz.sender.LogzioSender;
+import io.logz.sender.SenderStatusReporter;
+import io.logz.sender.com.google.gson.JsonObject;
+import io.logz.sender.exceptions.LogzioParameterErrorException;
+
+import java.io.*;
+import java.util.*;
+import java.util.concurrent.Executors;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import jenkins.model.Jenkins;
+import jenkins.plugins.logstash.LogstashConfiguration;
+import net.sf.json.JSONArray;
+import net.sf.json.JSONObject;
+
+/**
+ * Logz.io Data Access Object.
+ *
+ * @author Ido Halevi
+ */
+
+public class LogzioDao extends AbstractLogstashIndexerDao {
+    private static final int CONNECT_TIMEOUT = 10*1000;
+    private static final int CORE_POOL_SIZE = 2;
+    private static final int DRAIN_TIMEOUT = 2;
+    private static final int FS_PERCENT_THRESHOLD = 99;
+    private static final int GC_PERSISTED_QUEUE_FILE_INTERVAL_SECOND = 30;
+    private static final int SOCKET_TIMEOUT = 10*1000;
+    private static final String TYPE = "jenkins_plugin";
+    private final LogzioSender logzioSender;
+
+    private String key;
+    private String host;
+
+    //primary constructor used by indexer factory
+    public LogzioDao(String host, String key){
+        this(null, host, key);
+    }
+
+    // Factored for unit testing
+    LogzioDao(LogzioSender factory, String host, String key){
+        this.host = host;
+        this.key = key;
+
+        // create file for sender queue
+        File fp;
+        Jenkins jenkins = Jenkins.getInstanceOrNull();
+        if (jenkins == null){
+            // for testing (mvn package)
+            fp = Files.createTempDir();
+        }else{
+            fp = new File(jenkins.getRootDir() + "/logs/logzio_plugin");
+            boolean folderExisted = fp.mkdirs() || fp.setWritable(true);
+            if (!folderExisted) {
+                throw new IllegalArgumentException("Unable to create path");
+            }
+        }
+
+        try{
+            this.logzioSender = factory == null ? LogzioSender.getOrCreateSenderByType(key, TYPE, DRAIN_TIMEOUT,
+                    FS_PERCENT_THRESHOLD, fp, host, SOCKET_TIMEOUT, CONNECT_TIMEOUT,false, null,
+                    Executors.newScheduledThreadPool(CORE_POOL_SIZE),GC_PERSISTED_QUEUE_FILE_INTERVAL_SECOND, true) : factory;
+            this.logzioSender.start();
+        }catch (LogzioParameterErrorException e){
+            throw new IllegalArgumentException(e.getMessage());
+        }
+    }
+
+    @Override
+    public void push(String data) {
+        JSONObject jsonData = JSONObject.fromObject(data);
+        JSONArray logMessages = jsonData.getJSONArray("message");
+        for (Object logMsg : logMessages) {
+            JsonObject logLine = createLogLine(jsonData, logMsg.toString());
+            this.logzioSender.send(logLine);
+        }
+    }
+
+    protected JsonObject createLogLine(JSONObject jsonData, String logMsg) {
+        JsonObject logLine = new JsonObject();
+        logLine.addProperty("message", logMsg);
+        logLine.addProperty("@timestamp", LogstashConfiguration.getInstance().getDateFormatter().format(Calendar.getInstance().getTime()));
+        for(Object key : jsonData.keySet()){
+            String keyStr = (String) key;
+            if (!keyStr.equals("message")){
+                logLine.addProperty(key.toString(), jsonData.getString(key.toString()));
+            }
+        }
+        return logLine;
+    }
+
+    @Override
+    public JSONObject buildPayload(BuildData buildData, String jenkinsUrl, List<String> logLines) {
+        JSONObject payload = new JSONObject();
+        payload.put("message", logLines);
+        payload.put("source", "jenkins");
+        payload.put("source_host", jenkinsUrl);
+        payload.put("@buildTimestamp", buildData.getTimestamp());
+        payload.put("@version", 1);
+        // flatten build data
+        Map<String, Object> flattenJson = JsonFlattener.flattenAsMap(buildData.toString());
+        for (Map.Entry<String, Object> entry : flattenJson.entrySet()) {
+            String key = entry.getKey().replace('.','_');
+            Object value = entry.getValue();
+            payload.put(key, value);
+        }
+
+        return payload;
+    }
+
+    @Override
+    public String getDescription(){ return host; }
+
+    public String getHost(){ return host; }
+
+    public String getKey(){ return key; }
+
+    public String getType(){ return TYPE; }
+
+}
+

src/main/resources/index.jelly

@@ -1,3 +1,3 @@
 <div>
-  Adds the possibility to push builds logs and build data to a Logstash indexer such as Redis, RabbitMQ, Elastic Search or to Syslog.
+  Adds the possibility to push builds logs and build data to a Logstash indexer such as Redis, RabbitMQ, Elastic Search, Logz.io or to Syslog.
 </div>

src/main/resources/jenkins/plugins/logstash/configuration/Logzio/config.jelly

@@ -0,0 +1,9 @@
+<?jelly escape-by-default='true'?>
+<j:jelly xmlns:j="jelly:core" xmlns:st="jelly:stapler" xmlns:d="jelly:define" xmlns:l="/lib/layout" xmlns:t="/lib/hudson" xmlns:f="/lib/form">
+      <f:entry title="${%Logz.io Host}" field="host">
+        <f:textbox default="https://listener.logz.io:8071"/>
+      </f:entry>
+      <f:entry title="${%Logz.io key}" field="key">
+        <f:password/>
+      </f:entry>
+</j:jelly>

src/main/resources/jenkins/plugins/logstash/configuration/Logzio/help-host.jelly

@@ -0,0 +1,5 @@
+<div>
+  <p>Logz.io listener URL.<br/>
+  If you are in the EU region insert https://listener-eu.logz.io:8071. Otherwise, use https://listener.logz.io:8071<br/>
+  You can tell which region you are in by checking your login URL</p>
+</div>

src/main/resources/jenkins/plugins/logstash/configuration/Logzio/help-key.jelly

@@ -0,0 +1,3 @@
+<div>
+  <p>The Logz.io account token.</p>
+</div>

src/main/resources/jenkins/plugins/logstash/configuration/Logzio/help.jelly

@@ -0,0 +1,4 @@
+<?jelly escape-by-default='true'?>
+<j:jelly xmlns:j="jelly:core" xmlns:st="jelly:stapler">
+  Push to Logz.io with HTTPs input.
+</j:jelly>

src/test/java/jenkins/plugins/logstash/LogstashConfigurationMigrationTest.java

@@ -30,6 +30,7 @@
 import jenkins.plugins.logstash.configuration.RabbitMq;
 import jenkins.plugins.logstash.configuration.Redis;
 import jenkins.plugins.logstash.configuration.Syslog;
+import jenkins.plugins.logstash.configuration.Logzio;
 import jenkins.plugins.logstash.persistence.LogstashIndexerDao.IndexerType;
 import jenkins.plugins.logstash.persistence.LogstashIndexerDao.SyslogFormat;
 
@@ -151,4 +152,18 @@ public void rabbitMqMigration()
     assertThat(es.getUsername(), equalTo("user"));
   }
 
+  @Test
+  public void logzioMigration()
+  {
+    when(descriptor.getType()).thenReturn(IndexerType.LOGZIO);
+    when(descriptor.getHost()).thenReturn("https://listener.logz.io:8071");
+    when(descriptor.getKey()).thenReturn("key");
+    configuration.migrateData();
+    LogstashIndexer<?> indexer = configuration.getLogstashIndexer();
+    assertThat(indexer, IsInstanceOf.instanceOf(Logzio.class));
+    assertThat(configuration.isMilliSecondTimestamps(),equalTo(false));
+    Logzio logz = (Logzio) indexer;
+    assertThat(logz.getKey(), equalTo("key"));
+    assertThat(logz.getHost(), equalTo("https://listener.logz.io:8071"));
+  }
 }