Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@

/**
* Build wrapper that decorates the build's logger to insert a
* {@link LogstashNote} on each output line.
*
* @author K Jonathan Harker
*/
Expand Down Expand Up @@ -102,13 +101,14 @@ public OutputStream decorateLogger(AbstractBuild build, OutputStream logger) {
return los;
}

@Override
public DescriptorImpl getDescriptor() {
return (DescriptorImpl) super.getDescriptor();
}

// Method to encapsulate calls for unit-testing
LogstashWriter getLogStashWriter(AbstractBuild<?, ?> build, OutputStream errorStream) {
return new LogstashWriter(build, errorStream, null);
return new LogstashWriter(build, errorStream, null, build.getCharset());
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -85,9 +85,10 @@ private boolean perform(Run<?, ?> run, TaskListener listener) {

// Method to encapsulate calls for unit-testing
LogstashWriter getLogStashWriter(Run<?, ?> run, OutputStream errorStream, TaskListener listener) {
return new LogstashWriter(run, errorStream, listener);
return new LogstashWriter(run, errorStream, listener, run.getCharset());
}

@Override
public BuildStepMonitor getRequiredMonitorService() {
// We don't call Run#getPreviousBuild() so no external synchronization between builds is required
return BuildStepMonitor.NONE;
Expand All @@ -106,6 +107,7 @@ public boolean isApplicable(@SuppressWarnings("rawtypes") Class<? extends Abstra
return true;
}

@Override
public String getDisplayName() {
return Messages.DisplayName();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ protected void eol(byte[] b, int len) throws IOException {
this.flush();

if(!logstash.isConnectionBroken()) {
String line = new String(b, 0, len).trim();
String line = new String(b, 0, len, logstash.getCharset()).trim();
line = ConsoleNote.removeNotes(line);
logstash.write(line);
}
Expand Down
17 changes: 15 additions & 2 deletions src/main/java/jenkins/plugins/logstash/LogstashWriter.java
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@

import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.Charset;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
Expand All @@ -60,21 +61,33 @@ public class LogstashWriter {
final String jenkinsUrl;
final LogstashIndexerDao dao;
private boolean connectionBroken;
private Charset charset;

public LogstashWriter(Run<?, ?> run, OutputStream error, TaskListener listener) {
public LogstashWriter(Run<?, ?> run, OutputStream error, TaskListener listener, Charset charset) {
this.errorStream = error != null ? error : System.err;
this.build = run;
this.listener = listener;
this.charset = charset;
this.dao = this.getDaoOrNull();
if (this.dao == null) {
this.jenkinsUrl = "";
this.buildData = null;
} else {
this.jenkinsUrl = getJenkinsUrl();
this.buildData = getBuildData();
dao.setCharset(charset);
}
}

/**
* gets the charset that Jenkins is using during this build.
* @return
*/
public Charset getCharset()
{
return charset;
}

/**
* Sends a logstash payload for a single line to the indexer.
* Call will be ignored if the line is empty or if the connection to the indexer is broken.
Expand Down Expand Up @@ -184,7 +197,7 @@ private LogstashIndexerDao getDaoOrNull() {
private void logErrorMessage(String msg) {
try {
connectionBroken = true;
errorStream.write(msg.getBytes());
errorStream.write(msg.getBytes(charset));
errorStream.flush();
} catch (IOException ex) {
// This should never happen, but if it does we just have to let it go.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@

package jenkins.plugins.logstash.persistence;

import java.nio.charset.Charset;
import java.util.Calendar;
import java.util.List;

Expand All @@ -43,6 +44,7 @@ public abstract class AbstractLogstashIndexerDao implements LogstashIndexerDao {
protected final String key;
protected final String username;
protected final String password;
private Charset charset;

public AbstractLogstashIndexerDao(String host, int port, String key, String username, String password) {
this.host = host;
Expand All @@ -56,6 +58,27 @@ public AbstractLogstashIndexerDao(String host, int port, String key, String user
}
}

/**
* Sets the charset used to push data to the indexer
*
*@param charset The charset to push data
*/
@Override
public void setCharset(Charset charset)
{
this.charset = charset;
}

/**
* Gets the configured charset used to push data to the indexer
*
* @return charste to push data
*/
public Charset getCharset()
{
return charset;
}

@Override
public JSONObject buildPayload(BuildData buildData, String jenkinsUrl, List<String> logLines) {
JSONObject payload = new JSONObject();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,11 @@
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;

import com.google.common.collect.Range;

Expand Down Expand Up @@ -85,7 +88,7 @@ public ElasticSearchDao(String host, int port, String key, String username, Stri
}

if (StringUtils.isNotBlank(username)) {
auth = Base64.encodeBase64String((username + ":" + StringUtils.defaultString(password)).getBytes());
auth = Base64.encodeBase64String((username + ":" + StringUtils.defaultString(password)).getBytes(StandardCharsets.UTF_8));
} else {
auth = null;
}
Expand Down Expand Up @@ -132,7 +135,7 @@ private String getErrorMessage(CloseableHttpResponse response) {
PrintStream stream = null;
try {
byteStream = new ByteArrayOutputStream();
stream = new PrintStream(byteStream);
stream = new PrintStream(byteStream, true, StandardCharsets.UTF_8.name());

try {
stream.print("HTTP error code: ");
Expand All @@ -145,7 +148,11 @@ private String getErrorMessage(CloseableHttpResponse response) {
stream.println(ExceptionUtils.getStackTrace(e));
}
stream.flush();
return byteStream.toString();
return byteStream.toString(StandardCharsets.UTF_8.name());
}
catch (UnsupportedEncodingException e)
{
return ExceptionUtils.getStackTrace(e);
} finally {
if (stream != null) {
stream.close();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
package jenkins.plugins.logstash.persistence;

import java.io.IOException;
import java.nio.charset.Charset;
import java.util.List;

import net.sf.json.JSONObject;
Expand All @@ -42,7 +43,7 @@ static enum IndexerType {
ELASTICSEARCH,
SYSLOG
}

static enum SyslogFormat {
RFC5424,
RFC3164
Expand All @@ -51,7 +52,9 @@ static enum SyslogFormat {
static enum SyslogProtocol {
UDP
}


public void setCharset(Charset charset);

String getDescription();

IndexerType getIndexerType();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ public void push(String data) throws IOException {
channel.queueDeclare(key, true, false, false, null);
}

channel.basicPublish("", key, null, data.getBytes());
channel.basicPublish("", key, null, data.getBytes(getCharset()));
} finally {
finalizeChannel(channel);
finalizeConnection(connection);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

import java.io.ByteArrayOutputStream;
import java.io.OutputStream;
import java.nio.charset.Charset;

import org.junit.After;
import org.junit.Before;
Expand All @@ -32,6 +33,7 @@ public void before() throws Exception {
buffer = new ByteArrayOutputStream();
Mockito.doNothing().when(mockWriter).write(anyString());
when(mockWriter.isConnectionBroken()).thenReturn(false);
when(mockWriter.getCharset()).thenReturn(Charset.defaultCharset());
}

@After
Expand Down Expand Up @@ -61,6 +63,7 @@ public void eolSuccess() throws Exception {
assertEquals("Results don't match", msg, buffer.toString());
verify(mockWriter).isConnectionBroken();
verify(mockWriter).write(msg);
verify(mockWriter).getCharset();
}

@Test
Expand Down Expand Up @@ -102,6 +105,7 @@ public void eolSuccessConnectionBroken() throws Exception {
//Verify calls were made to the dao logging twice, not three times.
verify(mockWriter, times(2)).write(msg);
verify(mockWriter, times(3)).isConnectionBroken();
verify(mockWriter, times(2)).getCharset();
}

@Test
Expand Down
19 changes: 18 additions & 1 deletion src/test/java/jenkins/plugins/logstash/LogstashWriterTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.Charset;
import java.util.Arrays;
import java.util.Collections;
import java.util.GregorianCalendar;
Expand All @@ -39,7 +40,7 @@ static LogstashWriter createLogstashWriter(final AbstractBuild<?, ?> testBuild,
final String url,
final LogstashIndexerDao indexer,
final BuildData data) {
return new LogstashWriter(testBuild, error, null) {
return new LogstashWriter(testBuild, error, null, testBuild.getCharset()) {
@Override
LogstashIndexerDao getDao() throws InstantiationException {
if (indexer == null) {
Expand Down Expand Up @@ -100,6 +101,7 @@ public void before() throws Exception {
when(mockBuild.getLog(3)).thenReturn(Arrays.asList("line 1", "line 2", "line 3", "Log truncated..."));
when(mockBuild.getEnvironment(null)).thenReturn(new EnvVars());
when(mockBuild.getExecutor()).thenReturn(mockExecutor);
when(mockBuild.getCharset()).thenReturn(Charset.defaultCharset());
when(mockExecutor.getOwner()).thenReturn(mockComputer);
when(mockComputer.getNode()).thenReturn(null);

Expand Down Expand Up @@ -156,6 +158,8 @@ public void constructorSuccess() throws Exception {
verify(mockBuild).getSensitiveBuildVariables();
verify(mockBuild).getEnvironments();
verify(mockBuild).getEnvironment(null);
verify(mockBuild).getCharset();
verify(mockDao).setCharset(Charset.defaultCharset());

verify(mockTestResultAction).getTotalCount();
verify(mockTestResultAction).getSkipCount();
Expand All @@ -180,6 +184,7 @@ public void constructorSuccessNoDao() throws Exception {
// Verify results
assertEquals("Results don't match", exMessage, errorBuffer.toString());
assertTrue("Connection not broken", writer.isConnectionBroken());
verify(mockBuild).getCharset();
}

@Test
Expand All @@ -196,6 +201,7 @@ public void writeSuccessNoDao() throws Exception {
// Verify results
assertEquals("Results don't match", "", errorBuffer.toString());
assertTrue("Connection not broken", writer.isConnectionBroken());
verify(mockBuild).getCharset();
}

@Test
Expand All @@ -211,6 +217,7 @@ public void writeBuildLogSuccessNoDao() throws Exception {
// Verify results
assertEquals("Results don't match", "", errorBuffer.toString());
assertTrue("Connection not broken", writer.isConnectionBroken());
verify(mockBuild).getCharset();
}

@Test
Expand All @@ -228,6 +235,9 @@ public void writeSuccess() throws Exception {

verify(mockDao).buildPayload(Matchers.eq(mockBuildData), Matchers.eq("http://my-jenkins-url"), Matchers.anyListOf(String.class));
verify(mockDao).push("{\"data\":{},\"message\":[\"test\"],\"source\":\"jenkins\",\"source_host\":\"http://my-jenkins-url\",\"@version\":1}");
verify(mockDao).setCharset(Charset.defaultCharset());
verify(mockBuild).getCharset();

}

@Test
Expand All @@ -242,9 +252,11 @@ public void writeBuildLogSuccess() throws Exception {
// No error output
assertEquals("Results don't match", "", errorBuffer.toString());
verify(mockBuild).getLog(3);
verify(mockBuild).getCharset();

verify(mockDao).buildPayload(Matchers.eq(mockBuildData), Matchers.eq("http://my-jenkins-url"), Matchers.anyListOf(String.class));
verify(mockDao).push("{\"data\":{},\"message\":[\"test\"],\"source\":\"jenkins\",\"source_host\":\"http://my-jenkins-url\",\"@version\":1}");
verify(mockDao).setCharset(Charset.defaultCharset());
}

@Test
Expand Down Expand Up @@ -289,6 +301,9 @@ public void writeSuccessConnectionBroken() throws Exception {
verify(mockDao, times(2)).push("{\"data\":{},\"message\":[\"test\"],\"source\":\"jenkins\",\"source_host\":\"http://my-jenkins-url\",\"@version\":1}");
verify(mockDao).getIndexerType();
verify(mockDao, times(2)).getDescription();
verify(mockDao).setCharset(Charset.defaultCharset());
verify(mockBuild).getCharset();

}

@Test
Expand All @@ -304,12 +319,14 @@ public void writeBuildLogGetLogError() throws Exception {

// Verify results
verify(mockBuild).getLog(3);
verify(mockBuild).getCharset();

List<String> expectedErrorLines = Arrays.asList(
"[logstash-plugin]: Unable to serialize log data.",
"java.io.IOException: Unable to read log file");
verify(mockDao).push("{\"data\":{},\"message\":[\"test\"],\"source\":\"jenkins\",\"source_host\":\"http://my-jenkins-url\",\"@version\":1}");
verify(mockDao).buildPayload(eq(mockBuildData), eq("http://my-jenkins-url"), logLinesCaptor.capture());
verify(mockDao).setCharset(Charset.defaultCharset());
List<String> actualLogLines = logLinesCaptor.getValue();

assertThat("The exception was not sent to Logstash", actualLogLines.get(0), containsString(expectedErrorLines.get(0)));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

import java.io.IOException;
import java.net.SocketException;
import java.nio.charset.Charset;

import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
Expand Down Expand Up @@ -37,6 +38,8 @@ RabbitMqDao createDao(String host, int port, String key, String username, String
verify(mockPool, atLeastOnce()).setPassword(password);
}

factory.setCharset(Charset.defaultCharset());

return factory;
}

Expand Down