diff --git a/src/main/java/org/jenkinsci/plugins/durabletask/BourneShellScript.java b/src/main/java/org/jenkinsci/plugins/durabletask/BourneShellScript.java index e93b7be3..c563c77d 100644 --- a/src/main/java/org/jenkinsci/plugins/durabletask/BourneShellScript.java +++ b/src/main/java/org/jenkinsci/plugins/durabletask/BourneShellScript.java @@ -207,8 +207,8 @@ private FilePath pidFile(FilePath ws) throws IOException, InterruptedException { return controlDir(ws).child("pid"); } - @Override public Integer exitStatus(FilePath workspace, Launcher launcher, TaskListener listener) throws IOException, InterruptedException { - Integer status = super.exitStatus(workspace, launcher, listener); + @Override protected Integer exitStatus(FilePath workspace, TaskListener listener) throws IOException, InterruptedException { + Integer status = super.exitStatus(workspace, listener); if (status != null) { LOGGER.log(Level.FINE, "found exit code {0} in {1}", new Object[] {status, controlDir}); return status; diff --git a/src/main/java/org/jenkinsci/plugins/durabletask/Controller.java b/src/main/java/org/jenkinsci/plugins/durabletask/Controller.java index e110bc49..9b155590 100644 --- a/src/main/java/org/jenkinsci/plugins/durabletask/Controller.java +++ b/src/main/java/org/jenkinsci/plugins/durabletask/Controller.java @@ -43,19 +43,35 @@ */ public abstract class Controller implements Serializable { + /** + * Begins watching the process asynchronously, so that the master may receive notification when output is available or the process has exited. + * This should be called as soon as the process is launched, and thereafter whenever reconnecting to the agent. + * You should not call {@link #writeLog} or {@link #cleanup} in this case; you do not need to call {@link #exitStatus(FilePath, Launcher)} frequently, + * though it is advisable to still call it occasionally to verify that the process is still running. + * @param workspace the workspace in use + * @param handler a remotable callback + * @param listener a remotable destination for messages + * @throws UnsupportedOperationException when this mode is not available, so you must fall back to polling {@link #writeLog} and {@link #exitStatus(FilePath, Launcher)} + */ + public void watch(@Nonnull FilePath workspace, @Nonnull Handler handler, @Nonnull TaskListener listener) throws IOException, InterruptedException, UnsupportedOperationException { + throw new UnsupportedOperationException("Asynchronous mode is not implemented in " + getClass().getName()); + } + /** * Obtains any new task log output. * Could use a serializable field to keep track of how much output has been previously written. * @param workspace the workspace in use * @param sink where to send new log output * @return true if something was written and the controller should be resaved, false if everything is idle + * @see DurableTask#charset + * @see DurableTask#defaultCharset */ public abstract boolean writeLog(FilePath workspace, OutputStream sink) throws IOException, InterruptedException; /** * Checks whether the task has finished. * @param workspace the workspace in use - * @param launcher a way to start processes + * @param launcher a way to start processes (currently unused) * @param logger a way to report special messages * @return an exit code (zero is successful), or null if the task appears to still be running */ @@ -86,8 +102,10 @@ public abstract class Controller implements Serializable { * Intended for use after {@link #exitStatus(FilePath, Launcher)} has returned a non-null status. * The result is undefined if {@link DurableTask#captureOutput} was not called before launch; generally an {@link IOException} will result. * @param workspace the workspace in use - * @param launcher a way to start processes + * @param launcher a way to start processes (currently unused) * @return the output of the process as raw bytes (may be empty but not null) + * @see DurableTask#charset + * @see DurableTask#defaultCharset */ public @Nonnull byte[] getOutput(@Nonnull FilePath workspace, @Nonnull Launcher launcher) throws IOException, InterruptedException { throw new IOException("Did not implement getOutput in " + getClass().getName()); diff --git a/src/main/java/org/jenkinsci/plugins/durabletask/DurableTask.java b/src/main/java/org/jenkinsci/plugins/durabletask/DurableTask.java index d86b5bb9..87ec264f 100644 --- a/src/main/java/org/jenkinsci/plugins/durabletask/DurableTask.java +++ b/src/main/java/org/jenkinsci/plugins/durabletask/DurableTask.java @@ -31,6 +31,8 @@ import hudson.model.AbstractDescribableImpl; import hudson.model.TaskListener; import java.io.IOException; +import java.nio.charset.Charset; +import javax.annotation.Nonnull; /** * A task which may be run asynchronously on a build node and withstand disconnection of the slave agent. @@ -53,8 +55,9 @@ public abstract class DurableTask extends AbstractDescribableImpl i public abstract Controller launch(EnvVars env, FilePath workspace, Launcher launcher, TaskListener listener) throws IOException, InterruptedException; /** - * Requests that standard output of the task be captured rather than streamed to {@link Controller#writeLog}. - * If so, you may call {@link Controller#getOutput}. + * Requests that standard output of the task be captured rather than streamed. + * If you use {@link Controller#watch}, standard output will not be sent to {@link Handler#output}; it will be included in {@link Handler#exited} instead. + * Otherwise (using polling mode), standard output will not be sent to {@link Controller#writeLog}; call {@link Controller#getOutput} to collect. * Standard error should still be streamed to the log. * Should be called prior to {@link #launch} to take effect. * @throws UnsupportedOperationException if this implementation does not support that mode @@ -63,4 +66,23 @@ public void captureOutput() throws UnsupportedOperationException { throw new UnsupportedOperationException("Capturing of output is not implemented in " + getClass().getName()); } + /** + * Requests that a specified charset be used to transcode process output. + * The encoding of {@link Controller#writeLog} and {@link Controller#getOutput} is then presumed to be UTF-8. + * If not called, no translation is performed. + * @param cs the character set in which process output is expected to be + */ + public void charset(@Nonnull Charset cs) { + // by default, ignore + } + + /** + * Requests that the node’s system charset be used to transcode process output. + * The encoding of {@link Controller#writeLog} and {@link Controller#getOutput} is then presumed to be UTF-8. + * If not called, no translation is performed. + */ + public void defaultCharset() { + // by default, ignore + } + } diff --git a/src/main/java/org/jenkinsci/plugins/durabletask/FileMonitoringTask.java b/src/main/java/org/jenkinsci/plugins/durabletask/FileMonitoringTask.java index 7ef7d3d7..9614e815 100644 --- a/src/main/java/org/jenkinsci/plugins/durabletask/FileMonitoringTask.java +++ b/src/main/java/org/jenkinsci/plugins/durabletask/FileMonitoringTask.java @@ -30,6 +30,8 @@ import hudson.Util; import hudson.model.TaskListener; import hudson.remoting.Channel; +import hudson.remoting.DaemonThreadFactory; +import hudson.remoting.NamingThreadFactory; import hudson.remoting.RemoteOutputStream; import hudson.remoting.VirtualChannel; import hudson.slaves.WorkspaceList; @@ -37,17 +39,33 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; import java.io.OutputStream; import java.io.RandomAccessFile; import java.io.StringWriter; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import java.nio.charset.Charset; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; import java.util.Collections; import java.util.Map; import java.util.TreeMap; import java.util.UUID; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; +import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; import jenkins.MasterToSlaveFileCallable; -import org.apache.commons.io.IOUtils; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.input.CountingInputStream; +import org.apache.commons.io.input.ReaderInputStream; /** * A task which forks some external command and then waits for log and status files to be updated/created. @@ -58,12 +76,19 @@ public abstract class FileMonitoringTask extends DurableTask { private static final String COOKIE = "JENKINS_SERVER_COOKIE"; + /** + * Charset name to use for transcoding, or the empty string for node system default, or null for no transcoding. + */ + private @CheckForNull String charset; + private static String cookieFor(FilePath workspace) { return "durable-" + Util.getDigestOf(workspace.getRemote()); } @Override public final Controller launch(EnvVars env, FilePath workspace, Launcher launcher, TaskListener listener) throws IOException, InterruptedException { - return launchWithCookie(workspace, launcher, listener, env, COOKIE, cookieFor(workspace)); + FileMonitoringController controller = launchWithCookie(workspace, launcher, listener, env, COOKIE, cookieFor(workspace)); + controller.charset = charset; + return controller; } protected FileMonitoringController launchWithCookie(FilePath workspace, Launcher launcher, TaskListener listener, EnvVars envVars, String cookieVariable, String cookieValue) throws IOException, InterruptedException { @@ -71,6 +96,14 @@ protected FileMonitoringController launchWithCookie(FilePath workspace, Launcher return doLaunch(workspace, launcher, listener, envVars); } + @Override public final void charset(Charset cs) { + charset = cs.name(); + } + + @Override public final void defaultCharset() { + charset = ""; + } + /** * Should start a process which sends output to {@linkplain FileMonitoringController#getLogFile(FilePath) log file} * in the workspace and finally writes its exit code to {@linkplain FileMonitoringController#getResultFile(FilePath) result file}. @@ -95,6 +128,10 @@ protected static Map escape(EnvVars envVars) { return m; } + /** + * Tails a log file and watches for an exit status file. + * Must be remotable so that {@link #watch} can transfer the implementation. + */ protected static class FileMonitoringController extends Controller { /** Absolute path of {@link #controlDir(FilePath)}. */ @@ -107,9 +144,13 @@ protected static class FileMonitoringController extends Controller { /** * Byte offset in the file that has been reported thus far. + * Only used if {@link #writeLog(FilePath, OutputStream)} is used; not used for {@link #watch}. */ private long lastLocation; + /** @see FileMonitoringTask#charset */ + private @CheckForNull String charset; + protected FileMonitoringController(FilePath ws) throws IOException, InterruptedException { // can't keep ws reference because Controller is expected to be serializable ws.mkdirs(); @@ -120,7 +161,7 @@ protected FileMonitoringController(FilePath ws) throws IOException, InterruptedE @Override public final boolean writeLog(FilePath workspace, OutputStream sink) throws IOException, InterruptedException { FilePath log = getLogFile(workspace); - Long newLocation = log.act(new WriteLog(lastLocation, new RemoteOutputStream(sink))); + Long newLocation = log.act(new WriteLog(lastLocation, new RemoteOutputStream(sink), charset)); if (newLocation != null) { LOGGER.log(Level.FINE, "copied {0} bytes from {1}", new Object[] {newLocation - lastLocation, log}); lastLocation = newLocation; @@ -132,9 +173,11 @@ protected FileMonitoringController(FilePath ws) throws IOException, InterruptedE private static class WriteLog extends MasterToSlaveFileCallable { private final long lastLocation; private final OutputStream sink; - WriteLog(long lastLocation, OutputStream sink) { + private final @CheckForNull String charset; + WriteLog(long lastLocation, OutputStream sink, String charset) { this.lastLocation = lastLocation; this.sink = sink; + this.charset = charset; } @Override public Long invoke(File f, VirtualChannel channel) throws IOException, InterruptedException { long len = f.length(); @@ -146,10 +189,14 @@ private static class WriteLog extends MasterToSlaveFileCallable { if (toRead > Integer.MAX_VALUE) { // >2Gb of output at once is unlikely throw new IOException("large reads not yet implemented"); } - // TODO is this efficient for large amounts of output? Would it be better to stream data, or return a byte[] from the callable? byte[] buf = new byte[(int) toRead]; raf.readFully(buf); - sink.write(buf); + ByteBuffer transcoded = maybeTranscode(buf, charset); + if (transcoded == null) { + sink.write(buf); + } else { + Channels.newChannel(sink).write(transcoded); + } } finally { raf.close(); } @@ -160,8 +207,14 @@ private static class WriteLog extends MasterToSlaveFileCallable { } } - // TODO would be more efficient to allow API to consolidate writeLog with exitStatus (save an RPC call) @Override public Integer exitStatus(FilePath workspace, Launcher launcher, TaskListener listener) throws IOException, InterruptedException { + return exitStatus(workspace, listener); + } + + /** + * Like {@link #exitStatus(FilePath, Launcher, TaskListener)} but not requesting a {@link Launcher}, which would not be available in {@link #watch} mode anyway. + */ + protected Integer exitStatus(FilePath workspace, TaskListener listener) throws IOException, InterruptedException { FilePath status = getResultFile(workspace); if (status.exists()) { try { @@ -175,14 +228,60 @@ private static class WriteLog extends MasterToSlaveFileCallable { } @Override public byte[] getOutput(FilePath workspace, Launcher launcher) throws IOException, InterruptedException { - // TODO could perhaps be more efficient for large files to send a MasterToSlaveFileCallable - try (InputStream is = getOutputFile(workspace).read()) { - return IOUtils.toByteArray(is); + return getOutput(workspace); + } + + /** + * Like {@link #getOutput(FilePath, Launcher)} but not requesting a {@link Launcher}, which would not be available in {@link #watch} mode anyway. + */ + protected byte[] getOutput(FilePath workspace) throws IOException, InterruptedException { + return getOutputFile(workspace).act(new MasterToSlaveFileCallable() { + @Override public byte[] invoke(File f, VirtualChannel channel) throws IOException, InterruptedException { + byte[] buf = FileUtils.readFileToByteArray(f); + ByteBuffer transcoded = maybeTranscode(buf, charset); + if (transcoded == null) { + return buf; + } else { + byte[] buf2 = new byte[transcoded.remaining()]; + transcoded.get(buf2); + return buf2; + } + } + }); + } + + /** + * Transcode process output to UTF-8 if necessary. + * @param data output presumed to be in local encoding + * @param charset a particular encoding name, or the empty string for the system default encoding, or null to skip transcoding + * @return a buffer of UTF-8 encoded data ({@link CodingErrorAction#REPLACE} is used), + * or null if not performing transcoding because it was not requested or the data was already thought to be in UTF-8 + */ + private static @CheckForNull ByteBuffer maybeTranscode(@Nonnull byte[] data, @CheckForNull String charset) { + Charset cs = transcodingCharset(charset); + if (cs == null) { + return null; + } else { + return StandardCharsets.UTF_8.encode(cs.decode(ByteBuffer.wrap(data))); + } + } + + private static @CheckForNull Charset transcodingCharset(@CheckForNull String charset) { + if (charset == null) { + return null; + } else { + Charset cs = charset.isEmpty() ? Charset.defaultCharset() : Charset.forName(charset); + if (cs.equals(StandardCharsets.UTF_8)) { // transcoding unnecessary as output was already UTF-8 + return null; + } else { // decode output in specified charset and reëncode in UTF-8 + return cs; + } } } @Override public final void stop(FilePath workspace, Launcher launcher) throws IOException, InterruptedException { launcher.kill(Collections.singletonMap(COOKIE, cookieFor(workspace))); + // TODO after 10s, if the control dir still exists, write a flag file and have the Watcher shut down (interrupting any ongoing handler.output call if possible) } @Override public void cleanup(FilePath workspace) throws IOException, InterruptedException { @@ -248,7 +347,107 @@ public FilePath getOutputFile(FilePath workspace) throws IOException, Interrupte } } + @Override public void watch(FilePath workspace, Handler handler, TaskListener listener) throws IOException, InterruptedException, ClassCastException { + workspace.actAsync(new StartWatching(this, handler, listener)); + LOGGER.log(Level.FINE, "started asynchronous watch in {0}", controlDir); + } + + /** + * File in which a last-read position is stored if {@link #watch} is used. + */ + public FilePath getLastLocationFile(FilePath workspace) throws IOException, InterruptedException { + return controlDir(workspace).child("last-location.txt"); + } + private static final long serialVersionUID = 1L; } + private static ScheduledExecutorService watchService; + private synchronized static ScheduledExecutorService watchService() { + if (watchService == null) { + // TODO 2.105+ use ClassLoaderSanityThreadFactory + watchService = new /*ErrorLogging*/ScheduledThreadPoolExecutor(5, new NamingThreadFactory(new DaemonThreadFactory(), "FileMonitoringTask watcher")); + } + return watchService; + } + + private static class StartWatching extends MasterToSlaveFileCallable { + + private static final long serialVersionUID = 1L; + + private final FileMonitoringController controller; + private final Handler handler; + private final TaskListener listener; + + StartWatching(FileMonitoringController controller, Handler handler, TaskListener listener) { + this.controller = controller; + this.handler = handler; + this.listener = listener; + } + + @Override public Void invoke(File workspace, VirtualChannel channel) throws IOException, InterruptedException { + watchService().submit(new Watcher(controller, new FilePath(workspace), handler, listener)); + return null; + } + + } + + private static class Watcher implements Runnable { + + private final FileMonitoringController controller; + private final FilePath workspace; + private final Handler handler; + private final TaskListener listener; + private final @CheckForNull Charset cs; + + Watcher(FileMonitoringController controller, FilePath workspace, Handler handler, TaskListener listener) { + this.controller = controller; + this.workspace = workspace; + this.handler = handler; + this.listener = listener; + cs = FileMonitoringController.transcodingCharset(controller.charset); + } + + @Override public void run() { + try { + Integer exitStatus = controller.exitStatus(workspace, listener); // check before collecting output, in case the process is just now finishing + long lastLocation = 0; + FilePath lastLocationFile = controller.getLastLocationFile(workspace); + if (lastLocationFile.exists()) { + lastLocation = Long.parseLong(lastLocationFile.readToString()); + } + FilePath logFile = controller.getLogFile(workspace); + long len = logFile.length(); + if (len > lastLocation) { + assert !logFile.isRemote(); + try (FileChannel ch = FileChannel.open(Paths.get(logFile.getRemote()), StandardOpenOption.READ)) { + InputStream locallyEncodedStream = Channels.newInputStream(ch.position(lastLocation)); + InputStream utf8EncodedStream = cs == null ? locallyEncodedStream : new ReaderInputStream(new InputStreamReader(locallyEncodedStream, cs), StandardCharsets.UTF_8); + CountingInputStream cis = new CountingInputStream(utf8EncodedStream); + handler.output(cis); + lastLocationFile.write(Long.toString(lastLocation + cis.getByteCount()), null); + } + } + if (exitStatus != null) { + byte[] output; + if (controller.getOutputFile(workspace).exists()) { + output = controller.getOutput(workspace); + } else { + output = null; + } + handler.exited(exitStatus, output); + controller.cleanup(workspace); + } else { + // Could use an adaptive timeout as in DurableTaskStep.Execution in polling mode, + // though less relevant here since there is no network overhead to the check. + watchService().schedule(this, 100, TimeUnit.MILLISECONDS); + } + } catch (Exception x) { + // note that LOGGER here is going to the agent log, not master log + LOGGER.log(Level.WARNING, "giving up on watching " + controller.controlDir, x); + } + } + + } + } diff --git a/src/main/java/org/jenkinsci/plugins/durabletask/Handler.java b/src/main/java/org/jenkinsci/plugins/durabletask/Handler.java new file mode 100644 index 00000000..0abf6f89 --- /dev/null +++ b/src/main/java/org/jenkinsci/plugins/durabletask/Handler.java @@ -0,0 +1,67 @@ +/* + * The MIT License + * + * Copyright 2016 CloudBees, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +package org.jenkinsci.plugins.durabletask; + +import hudson.FilePath; +import hudson.Launcher; +import hudson.remoting.VirtualChannel; +import java.io.InputStream; +import java.io.Serializable; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * A remote handler which may be sent to an agent and handle process output and results. + * If it needs to communicate with the master, you may use {@link VirtualChannel#export}. + * @see Controller#watch + */ +public abstract class Handler implements Serializable { + + /** + * Notification that new process output is available. + *

Should only be called when at least one byte is available. + * Whatever bytes are actually read will not be offered on the next call, if there is one; there is no need to close the stream. + *

There is no guarantee that output is offered in the form of complete lines of text, + * though in the typical case of line-oriented output it is likely that it will end in a newline. + *

Buffering is the responsibility of the caller, and {@link InputStream#markSupported} may be false. + * @param stream a way to read process output which has not already been handled + * @throws Exception if anything goes wrong, this watch is deactivated + */ + public abstract void output(@Nonnull InputStream stream) throws Exception; + + /** + * Notification that the process has exited or vanished. + * {@link #output} should have been called with any final uncollected output. + *

Any metadata associated with the process may be deleted after this call completes, rendering subsequent {@link Controller} calls unsatisfiable. + *

Note that unlike {@link Controller#exitStatus(FilePath, Launcher)}, no specialized {@link Launcher} is available on the agent, + * so if there are specialized techniques for determining process liveness they will not be considered here; + * you still need to occasionally poll for an exit status from the master. + * @param code the exit code, if known (0 conventionally represents success); may be negative for anomalous conditions such as a missing process + * @param output standard output captured, if {@link DurableTask#captureOutput} was called; else null + * @throws Exception if anything goes wrong, this watch is deactivated + */ + public abstract void exited(int code, @Nullable byte[] output) throws Exception; + +} diff --git a/src/test/java/org/jenkinsci/plugins/durabletask/BourneShellScriptTest.java b/src/test/java/org/jenkinsci/plugins/durabletask/BourneShellScriptTest.java index 78ff543a..caa2bf28 100644 --- a/src/test/java/org/jenkinsci/plugins/durabletask/BourneShellScriptTest.java +++ b/src/test/java/org/jenkinsci/plugins/durabletask/BourneShellScriptTest.java @@ -27,15 +27,24 @@ import hudson.EnvVars; import hudson.FilePath; import hudson.Launcher; +import hudson.model.Slave; import hudson.plugins.sshslaves.SSHLauncher; +import hudson.remoting.VirtualChannel; import hudson.slaves.DumbSlave; import hudson.slaves.OfflineCause; import hudson.util.StreamTaskListener; import hudson.util.VersionNumber; import java.io.ByteArrayOutputStream; import java.io.File; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.util.Collections; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; import java.util.logging.Level; +import jenkins.security.MasterToSlaveCallable; +import org.apache.commons.io.IOUtils; import org.apache.commons.io.output.TeeOutputStream; import static org.hamcrest.Matchers.*; import org.jenkinsci.test.acceptance.docker.Docker; @@ -168,6 +177,51 @@ public void smokeTest() throws Exception { c.cleanup(ws); } + @Issue("JENKINS-38381") + @Test public void watch() throws Exception { + Slave s = j.createOnlineSlave(); + ws = s.getWorkspaceRoot(); + launcher = s.createLauncher(listener); + DurableTask task = new BourneShellScript("set +x; for x in 1 2 3 4 5; do echo $x; sleep 1; done"); + Controller c = task.launch(new EnvVars(), ws, launcher, listener); + BlockingQueue status = new LinkedBlockingQueue<>(); + BlockingQueue output = new LinkedBlockingQueue<>(); + BlockingQueue lines = new LinkedBlockingQueue<>(); + c.watch(ws, new MockHandler(s.getChannel(), status, output, lines), listener); + assertEquals("+ set +x", lines.take()); + assertEquals(0, status.take().intValue()); + assertEquals("", output.take()); + assertEquals("[1, 2, 3, 4, 5]", lines.toString()); + task = new BourneShellScript("echo result"); + task.captureOutput(); + c = task.launch(new EnvVars(), ws, launcher, listener); + status = new LinkedBlockingQueue<>(); + output = new LinkedBlockingQueue<>(); + lines = new LinkedBlockingQueue<>(); + c.watch(ws, new MockHandler(s.getChannel(), status, output, lines), listener); + assertEquals(0, status.take().intValue()); + assertEquals("result\n", output.take()); + assertEquals("[+ echo result]", lines.toString()); + } + private static class MockHandler extends Handler { + private final BlockingQueue status; + private final BlockingQueue output; + private final BlockingQueue lines; + @SuppressWarnings("unchecked") + MockHandler(VirtualChannel channel, BlockingQueue status, BlockingQueue output, BlockingQueue lines) { + this.status = channel.export(BlockingQueue.class, status); + this.output = channel.export(BlockingQueue.class, output); + this.lines = channel.export(BlockingQueue.class, lines); + } + @Override public void output(InputStream stream) throws Exception { + lines.addAll(IOUtils.readLines(stream, StandardCharsets.UTF_8)); + } + @Override public void exited(int code, byte[] data) throws Exception { + status.add(code); + output.add(data != null ? new String(data, StandardCharsets.UTF_8) : ""); + } + } + @Issue("JENKINS-40734") @Test public void envWithShellChar() throws Exception { Controller c = new BourneShellScript("echo \"value=$MYNEWVAR\"").launch(new EnvVars("MYNEWVAR", "foo$$bar"), ws, launcher, listener); @@ -236,4 +290,241 @@ private void runOnDocker(DumbSlave s) throws Exception { runOnDocker(new DumbSlave("docker", "/home/jenkins/agent", new SimpleCommandLauncher("docker run -i --rm --name agent --init jenkinsci/slave:3.7-1 java -jar /usr/share/jenkins/slave.jar"))); } + @Issue({"JENKINS-31096", "JENKINS-38381"}) + @Test public void encoding() throws Exception { + JavaContainer container = dockerUbuntu.get(); + DumbSlave s = new DumbSlave("docker", "/home/test", new SSHLauncher(container.ipBound(22), container.port(22), "test", "test", "", "-Dfile.encoding=ISO-8859-1")); + j.jenkins.addNode(s); + j.waitOnline(s); + assertEquals("ISO-8859-1", s.getChannel().call(new DetectCharset())); + FilePath dockerWS = s.getWorkspaceRoot(); + dockerWS.child("latin").write("¡Ole!", "ISO-8859-1"); + dockerWS.child("eastern").write("Čau!", "ISO-8859-2"); + dockerWS.child("mixed").write("¡Čau → there!", "UTF-8"); + Launcher dockerLauncher = s.createLauncher(listener); + // control: no transcoding + Controller c = new BourneShellScript("cat latin").launch(new EnvVars(), dockerWS, dockerLauncher, listener); + while (c.exitStatus(dockerWS, dockerLauncher, listener) == null) { + Thread.sleep(100); + } + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + c.writeLog(dockerWS, baos); + assertEquals(0, c.exitStatus(dockerWS, dockerLauncher, listener).intValue()); + assertThat(baos.toString("ISO-8859-1"), containsString("¡Ole!")); + c.cleanup(dockerWS); + // and with output capture: + BourneShellScript dt = new BourneShellScript("cat latin"); + dt.captureOutput(); + c = dt.launch(new EnvVars(), dockerWS, dockerLauncher, listener); + while (c.exitStatus(dockerWS, dockerLauncher, listener) == null) { + Thread.sleep(100); + } + c.writeLog(dockerWS, System.err); + assertEquals(0, c.exitStatus(dockerWS, dockerLauncher, listener).intValue()); + assertEquals("¡Ole!", new String(c.getOutput(dockerWS, launcher), "ISO-8859-1")); + c.cleanup(dockerWS); + // test: specify particular charset (UTF-8) + dt = new BourneShellScript("cat mixed"); + dt.charset(StandardCharsets.UTF_8); + c = dt.launch(new EnvVars(), dockerWS, dockerLauncher, listener); + while (c.exitStatus(dockerWS, dockerLauncher, listener) == null) { + Thread.sleep(100); + } + baos = new ByteArrayOutputStream(); + c.writeLog(dockerWS, baos); + assertEquals(0, c.exitStatus(dockerWS, dockerLauncher, listener).intValue()); + assertThat(baos.toString("UTF-8"), containsString("¡Čau → there!")); + c.cleanup(dockerWS); + // and with output capture: + dt = new BourneShellScript("cat mixed"); + dt.charset(StandardCharsets.UTF_8); + dt.captureOutput(); + c = dt.launch(new EnvVars(), dockerWS, dockerLauncher, listener); + while (c.exitStatus(dockerWS, dockerLauncher, listener) == null) { + Thread.sleep(100); + } + c.writeLog(dockerWS, System.err); + assertEquals(0, c.exitStatus(dockerWS, dockerLauncher, listener).intValue()); + assertEquals("¡Čau → there!", new String(c.getOutput(dockerWS, launcher), "UTF-8")); + c.cleanup(dockerWS); + // test: specify particular charset (unrelated) + dt = new BourneShellScript("cat eastern"); + dt.charset(Charset.forName("ISO-8859-2")); + c = dt.launch(new EnvVars(), dockerWS, dockerLauncher, listener); + while (c.exitStatus(dockerWS, dockerLauncher, listener) == null) { + Thread.sleep(100); + } + baos = new ByteArrayOutputStream(); + c.writeLog(dockerWS, baos); + assertEquals(0, c.exitStatus(dockerWS, dockerLauncher, listener).intValue()); + assertThat(baos.toString("UTF-8"), containsString("Čau!")); + c.cleanup(dockerWS); + // and with output capture: + dt = new BourneShellScript("cat eastern"); + dt.charset(Charset.forName("ISO-8859-2")); + dt.captureOutput(); + c = dt.launch(new EnvVars(), dockerWS, dockerLauncher, listener); + while (c.exitStatus(dockerWS, dockerLauncher, listener) == null) { + Thread.sleep(100); + } + c.writeLog(dockerWS, System.err); + assertEquals(0, c.exitStatus(dockerWS, dockerLauncher, listener).intValue()); + assertEquals("Čau!", new String(c.getOutput(dockerWS, launcher), "UTF-8")); + c.cleanup(dockerWS); + // test: specify agent default charset + dt = new BourneShellScript("cat latin"); + dt.defaultCharset(); + c = dt.launch(new EnvVars(), dockerWS, dockerLauncher, listener); + while (c.exitStatus(dockerWS, dockerLauncher, listener) == null) { + Thread.sleep(100); + } + baos = new ByteArrayOutputStream(); + c.writeLog(dockerWS, baos); + assertEquals(0, c.exitStatus(dockerWS, dockerLauncher, listener).intValue()); + assertThat(baos.toString("UTF-8"), containsString("¡Ole!")); + c.cleanup(dockerWS); + // and with output capture: + dt = new BourneShellScript("cat latin"); + dt.defaultCharset(); + dt.captureOutput(); + c = dt.launch(new EnvVars(), dockerWS, dockerLauncher, listener); + while (c.exitStatus(dockerWS, dockerLauncher, listener) == null) { + Thread.sleep(100); + } + c.writeLog(dockerWS, System.err); + assertEquals(0, c.exitStatus(dockerWS, dockerLauncher, listener).intValue()); + assertEquals("¡Ole!", new String(c.getOutput(dockerWS, launcher), "UTF-8")); + c.cleanup(dockerWS); + // test: inappropriate charset, some replacement characters + dt = new BourneShellScript("cat mixed"); + dt.charset(StandardCharsets.US_ASCII); + c = dt.launch(new EnvVars(), dockerWS, dockerLauncher, listener); + while (c.exitStatus(dockerWS, dockerLauncher, listener) == null) { + Thread.sleep(100); + } + baos = new ByteArrayOutputStream(); + c.writeLog(dockerWS, baos); + assertEquals(0, c.exitStatus(dockerWS, dockerLauncher, listener).intValue()); + assertThat(baos.toString("UTF-8"), containsString("����au ��� there!")); + c.cleanup(dockerWS); + // and with output capture: + dt = new BourneShellScript("cat mixed"); + dt.charset(StandardCharsets.US_ASCII); + dt.captureOutput(); + c = dt.launch(new EnvVars(), dockerWS, dockerLauncher, listener); + while (c.exitStatus(dockerWS, dockerLauncher, listener) == null) { + Thread.sleep(100); + } + c.writeLog(dockerWS, System.err); + assertEquals(0, c.exitStatus(dockerWS, dockerLauncher, listener).intValue()); + assertEquals("����au ��� there!", new String(c.getOutput(dockerWS, launcher), "UTF-8")); + c.cleanup(dockerWS); + // test: using watch with particular charset + dt = new BourneShellScript("cat mixed"); + dt.charset(StandardCharsets.UTF_8); + c = dt.launch(new EnvVars(), dockerWS, dockerLauncher, listener); + BlockingQueue status = new LinkedBlockingQueue<>(); + BlockingQueue output = new LinkedBlockingQueue<>(); + BlockingQueue lines = new LinkedBlockingQueue<>(); + c.watch(dockerWS, new MockHandler(s.getChannel(), status, output, lines), listener); + assertEquals("+ cat mixed", lines.take()); + assertEquals(0, status.take().intValue()); + assertEquals("", output.take()); + assertEquals("[¡Čau → there!]", lines.toString()); + // and with output capture: + dt = new BourneShellScript("cat mixed"); + dt.charset(StandardCharsets.UTF_8); + dt.captureOutput(); + c = dt.launch(new EnvVars(), dockerWS, dockerLauncher, listener); + status = new LinkedBlockingQueue<>(); + output = new LinkedBlockingQueue<>(); + lines = new LinkedBlockingQueue<>(); + c.watch(dockerWS, new MockHandler(s.getChannel(), status, output, lines), listener); + assertEquals("+ cat mixed", lines.take()); + assertEquals(0, status.take().intValue()); + assertEquals("¡Čau → there!", output.take()); + assertEquals("[]", lines.toString()); + // with unrelated charset: + dt = new BourneShellScript("cat eastern"); + dt.charset(Charset.forName("ISO-8859-2")); + c = dt.launch(new EnvVars(), dockerWS, dockerLauncher, listener); + status = new LinkedBlockingQueue<>(); + output = new LinkedBlockingQueue<>(); + lines = new LinkedBlockingQueue<>(); + c.watch(dockerWS, new MockHandler(s.getChannel(), status, output, lines), listener); + assertEquals("+ cat eastern", lines.take()); + assertEquals(0, status.take().intValue()); + assertEquals("", output.take()); + assertEquals("[Čau!]", lines.toString()); + // and with output capture: + dt = new BourneShellScript("cat eastern"); + dt.charset(Charset.forName("ISO-8859-2")); + dt.captureOutput(); + c = dt.launch(new EnvVars(), dockerWS, dockerLauncher, listener); + status = new LinkedBlockingQueue<>(); + output = new LinkedBlockingQueue<>(); + lines = new LinkedBlockingQueue<>(); + c.watch(dockerWS, new MockHandler(s.getChannel(), status, output, lines), listener); + assertEquals("+ cat eastern", lines.take()); + assertEquals(0, status.take().intValue()); + assertEquals("Čau!", output.take()); + assertEquals("[]", lines.toString()); + // with agent default charset: + dt = new BourneShellScript("cat latin"); + dt.defaultCharset(); + c = dt.launch(new EnvVars(), dockerWS, dockerLauncher, listener); + status = new LinkedBlockingQueue<>(); + output = new LinkedBlockingQueue<>(); + lines = new LinkedBlockingQueue<>(); + c.watch(dockerWS, new MockHandler(s.getChannel(), status, output, lines), listener); + assertEquals("+ cat latin", lines.take()); + assertEquals(0, status.take().intValue()); + assertEquals("", output.take()); + assertEquals("[¡Ole!]", lines.toString()); + // and with output capture: + dt = new BourneShellScript("cat latin"); + dt.defaultCharset(); + dt.captureOutput(); + c = dt.launch(new EnvVars(), dockerWS, dockerLauncher, listener); + status = new LinkedBlockingQueue<>(); + output = new LinkedBlockingQueue<>(); + lines = new LinkedBlockingQueue<>(); + c.watch(dockerWS, new MockHandler(s.getChannel(), status, output, lines), listener); + assertEquals("+ cat latin", lines.take()); + assertEquals(0, status.take().intValue()); + assertEquals("¡Ole!", output.take()); + assertEquals("[]", lines.toString()); + // and mojibake: + dt = new BourneShellScript("cat mixed"); + dt.charset(StandardCharsets.US_ASCII); + c = dt.launch(new EnvVars(), dockerWS, dockerLauncher, listener); + status = new LinkedBlockingQueue<>(); + output = new LinkedBlockingQueue<>(); + lines = new LinkedBlockingQueue<>(); + c.watch(dockerWS, new MockHandler(s.getChannel(), status, output, lines), listener); + assertEquals("+ cat mixed", lines.take()); + assertEquals(0, status.take().intValue()); + assertEquals("", output.take()); + assertEquals("[����au ��� there!]", lines.toString()); + // and with output capture: + dt = new BourneShellScript("cat mixed"); + dt.charset(StandardCharsets.US_ASCII); + dt.captureOutput(); + c = dt.launch(new EnvVars(), dockerWS, dockerLauncher, listener); + status = new LinkedBlockingQueue<>(); + output = new LinkedBlockingQueue<>(); + lines = new LinkedBlockingQueue<>(); + c.watch(dockerWS, new MockHandler(s.getChannel(), status, output, lines), listener); + assertEquals("+ cat mixed", lines.take()); + assertEquals(0, status.take().intValue()); + assertEquals("����au ��� there!", output.take()); + assertEquals("[]", lines.toString()); + s.toComputer().disconnect(new OfflineCause.UserCause(null, null)); + } + private static class DetectCharset extends MasterToSlaveCallable { + @Override public String call() throws RuntimeException { + return Charset.defaultCharset().name(); + } + } + }