From d1d67cd2f69887eaa66e470bcd5feecfa3891096 Mon Sep 17 00:00:00 2001 From: Joerg Werner <4639399+jowerner@users.noreply.github.com> Date: Wed, 24 May 2023 13:19:51 +0200 Subject: [PATCH 1/3] - introduced Utf8Reader incl. test case - use Utf8Reader when reading properties - make sure we use UTF-8 when writing or appending to properties files --- .../com/xceptance/common/io/Utf8Reader.java | 104 ++++++++++++++++++ .../common/util/PropertiesUtils.java | 6 +- .../agentcontroller/AgentControllerImpl.java | 4 +- .../mastercontroller/ResultDownloader.java | 2 +- .../xlt/util/PropertiesIOException.java | 5 + .../xceptance/xlt/util/XltPropertiesImpl.java | 4 +- .../xceptance/common/io/Utf8ReaderTest.java | 86 +++++++++++++++ .../ConfigurationReportProviderTest.java | 2 +- 8 files changed, 204 insertions(+), 9 deletions(-) create mode 100644 src/main/java/com/xceptance/common/io/Utf8Reader.java create mode 100644 src/test/java/com/xceptance/common/io/Utf8ReaderTest.java diff --git a/src/main/java/com/xceptance/common/io/Utf8Reader.java b/src/main/java/com/xceptance/common/io/Utf8Reader.java new file mode 100644 index 000000000..d83ede140 --- /dev/null +++ b/src/main/java/com/xceptance/common/io/Utf8Reader.java @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2005-2023 Xceptance Software Technologies GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.xceptance.common.io; + +import java.io.CharArrayReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; + +/** + * A {@link Reader} implementation that reads UTF-8-encoded text from an {@link InputStream} throwing an + * {@link IOException} if the text is not valid UTF-8. + *

+ * Note: This class buffers the content of the stream in memory so it should be used for small amounts of data only. + */ +public class Utf8Reader extends Reader +{ + /** + * The reader we delegate to when actually dealing out the characters. + */ + private final CharArrayReader charArrayReader; + + /** + * Creates a new {@link Utf8Reader} instance. + * + * @param inputStream + * the stream to read the text from + * @throws IOException + * if the input stream could not be read or does not represent UTF-8-encoded text + */ + public Utf8Reader(final InputStream inputStream) throws IOException + { + final byte[] bytes = inputStream.readAllBytes(); + char[] chars; + + try + { + chars = getAsChars(bytes, StandardCharsets.UTF_8); + } + catch (final CharacterCodingException cce) + { + throw new IOException("Data does not represent UTF-8-encoded text", cce); + } + + charArrayReader = new CharArrayReader(chars); + } + + /** + * Converts the given bytes to chars according to the specified character set. + * + * @param bytes + * the input bytes + * @param charset + * the character set + * @return the corresponding chars + * @throws CharacterCodingException + * if the bytes do not represent text encoded with the given character set + */ + private static char[] getAsChars(final byte[] bytes, final Charset charset) throws CharacterCodingException + { + final CharBuffer charBuffer = charset.newDecoder().decode(ByteBuffer.wrap(bytes)); + + final char[] chars = new char[charBuffer.length()]; + charBuffer.get(chars); + + return chars; + } + + /** + * {@inheritDoc} + */ + @Override + public int read(final char[] cbuf, final int off, final int len) throws IOException + { + return charArrayReader.read(cbuf, off, len); + } + + /** + * {@inheritDoc} + */ + @Override + public void close() throws IOException + { + charArrayReader.close(); + } +} diff --git a/src/main/java/com/xceptance/common/util/PropertiesUtils.java b/src/main/java/com/xceptance/common/util/PropertiesUtils.java index 00d07712e..71300c270 100644 --- a/src/main/java/com/xceptance/common/util/PropertiesUtils.java +++ b/src/main/java/com/xceptance/common/util/PropertiesUtils.java @@ -31,6 +31,8 @@ import org.apache.commons.vfs2.FileObject; import org.apache.commons.vfs2.VFS; +import com.xceptance.common.io.Utf8Reader; + /** * The PropertiesUtils helps in dealing with properties files. * @@ -123,7 +125,7 @@ public static void loadProperties(final FileObject file, final Properties props) { ParameterCheckUtils.isReadableFile(file, "file"); } - catch(IllegalArgumentException e) + catch (IllegalArgumentException e) { throw new FileNotFoundException(file.toString()); } @@ -131,7 +133,7 @@ public static void loadProperties(final FileObject file, final Properties props) try (final InputStream is = file.getContent().getInputStream()) { - props.load(is); + props.load(new Utf8Reader(is)); } } diff --git a/src/main/java/com/xceptance/xlt/agentcontroller/AgentControllerImpl.java b/src/main/java/com/xceptance/xlt/agentcontroller/AgentControllerImpl.java index b6ff171b9..d666052c9 100644 --- a/src/main/java/com/xceptance/xlt/agentcontroller/AgentControllerImpl.java +++ b/src/main/java/com/xceptance/xlt/agentcontroller/AgentControllerImpl.java @@ -1118,14 +1118,14 @@ private static void maskFile(File inputFile, File outputFile) throws Configurati { PropertiesConfiguration config = new PropertiesConfiguration(); config.setIOFactory(new JupIOFactory()); // for better compatibility with java.util.Properties (GH#144) - try (final FileReader reader = new FileReader(inputFile)) + try (final FileReader reader = new FileReader(inputFile, StandardCharsets.UTF_8)) { config.read(reader); } config = mask(config, inputFile.getName().equals(XltConstants.SECRET_PROPERTIES_FILENAME)); final StringWriter writer = new StringWriter(); config.write(writer); - FileUtils.writeStringToFile(outputFile, writer.toString(), StandardCharsets.ISO_8859_1); + FileUtils.writeStringToFile(outputFile, writer.toString(), StandardCharsets.UTF_8); } /** diff --git a/src/main/java/com/xceptance/xlt/mastercontroller/ResultDownloader.java b/src/main/java/com/xceptance/xlt/mastercontroller/ResultDownloader.java index 4834114b4..67c7ac117 100644 --- a/src/main/java/com/xceptance/xlt/mastercontroller/ResultDownloader.java +++ b/src/main/java/com/xceptance/xlt/mastercontroller/ResultDownloader.java @@ -446,7 +446,7 @@ private boolean updateTimeData(final FileObject testPropFile) if (startTime > 0L && startTime < Long.MAX_VALUE) { try (var w = new BufferedWriter(new OutputStreamWriter(testPropFile.getContent().getOutputStream(true), - StandardCharsets.ISO_8859_1))) + StandardCharsets.UTF_8))) { w.newLine(); w.newLine(); diff --git a/src/main/java/com/xceptance/xlt/util/PropertiesIOException.java b/src/main/java/com/xceptance/xlt/util/PropertiesIOException.java index 40717b47f..313a02286 100644 --- a/src/main/java/com/xceptance/xlt/util/PropertiesIOException.java +++ b/src/main/java/com/xceptance/xlt/util/PropertiesIOException.java @@ -31,4 +31,9 @@ public PropertiesIOException(final String msg) { super(msg); } + + public PropertiesIOException(String message, Throwable cause) + { + super(message, cause); + } } diff --git a/src/main/java/com/xceptance/xlt/util/XltPropertiesImpl.java b/src/main/java/com/xceptance/xlt/util/XltPropertiesImpl.java index 2c5d65e44..2461aa3fc 100644 --- a/src/main/java/com/xceptance/xlt/util/XltPropertiesImpl.java +++ b/src/main/java/com/xceptance/xlt/util/XltPropertiesImpl.java @@ -418,9 +418,7 @@ private void process(final FileObject homeDirectory, final FileObject configDire } catch (IOException e) { - XltLogger.runTimeLogger.error(String.format("Issues loading properties from %s", fileName), e); - - throw new PropertiesIOException(String.format("Issues loading properties from %s", fileName)); + throw new PropertiesIOException(String.format("Issues loading properties from %s", fileName), e); } } diff --git a/src/test/java/com/xceptance/common/io/Utf8ReaderTest.java b/src/test/java/com/xceptance/common/io/Utf8ReaderTest.java new file mode 100644 index 000000000..e58b44e40 --- /dev/null +++ b/src/test/java/com/xceptance/common/io/Utf8ReaderTest.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2005-2023 Xceptance Software Technologies GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.xceptance.common.io; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.Reader; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; + +import junitparams.JUnitParamsRunner; +import junitparams.Parameters; + +/** + * Tests the implementation of {@link Utf8Reader}. + */ +@RunWith(JUnitParamsRunner.class) +public class Utf8ReaderTest +{ + /** + * Checks that the reader can read test texts encoded with the given encodings. + */ + @Test + @Parameters(value = + { + "UTF-8 | 日本の東京", // + "UTF-8 | ッ ツ ヅ ミ べ ボ プ", // + "UTF-8 | äöüÄÖÜßáàÁÀ", // + "UTF-8 | foobar", // + "ISO-8859-1 | foobar", // same bytes as with UTF-8 + "US-ASCII | foobar", // same bytes as with UTF-8 + }) + public void read(final String charsetName, final String text) throws IOException + { + doRead(charsetName, text); + } + + /** + * Checks that the reader throws an IOException for test texts encoded with the given encodings. + */ + @Test(expected = IOException.class) + @Parameters(value = + { + "ISO-8859-1 | äöüÄÖÜßáàÁÀ", // + "UTF-16 | äöüÄÖÜßáàÁÀ", // + "UTF-16BE | äöüÄÖÜßáàÁÀ", // + "UTF-16LE | äöüÄÖÜßáàÁÀ", // + }) + public void read_illegalEncoding(final String charsetName, final String text) throws IOException + { + doRead(charsetName, text); + } + + private void doRead(final String charsetName, final String text) throws IOException + { + // get the bytes of the text in the wanted encoding + final byte[] bytes = text.getBytes(charsetName); + + // now read the bytes in again via the Utf8Reader and check the resulting text + try (final Reader reader = new Utf8Reader(new ByteArrayInputStream(bytes))) + { + final char[] chars = new char[1024]; + final int charsRead = reader.read(chars); + + final String actualText = new String(chars, 0, charsRead); + + Assert.assertEquals(text.length(), charsRead); + Assert.assertEquals(text, actualText); + } + } +} diff --git a/src/test/java/com/xceptance/xlt/report/providers/ConfigurationReportProviderTest.java b/src/test/java/com/xceptance/xlt/report/providers/ConfigurationReportProviderTest.java index 824e861cc..a1a6aff69 100644 --- a/src/test/java/com/xceptance/xlt/report/providers/ConfigurationReportProviderTest.java +++ b/src/test/java/com/xceptance/xlt/report/providers/ConfigurationReportProviderTest.java @@ -40,7 +40,7 @@ public void testSecretPropertiesAreMaskedInTheOutput() throws IOException { final Path secretPath = testDir.resolve("config").resolve(XltConstants.SECRET_PROPERTIES_FILENAME); Files.createDirectories(secretPath.getParent()); - Files.write(secretPath, "value=Some very secret Value\n".getBytes(StandardCharsets.ISO_8859_1)); + Files.write(secretPath, "value=Some very secret Value\n".getBytes(StandardCharsets.UTF_8)); final ConfigurationReportProvider provider = new ConfigurationReportProvider(); ReportGeneratorConfiguration config = new ReportGeneratorConfiguration(); config.setReportDirectory(testDir.toFile()); From 5634c41876bb67fb3c90c92cbfe7e0e1c0fda904 Mon Sep 17 00:00:00 2001 From: Joerg Werner <4639399+jowerner@users.noreply.github.com> Date: Wed, 24 May 2023 15:10:39 +0200 Subject: [PATCH 2/3] - make use of Utf8Reader to get an exception in case the data file is not correctly UTF-8-encoded --- .../java/com/xceptance/xlt/engine/scripting/TestDataUtils.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/xceptance/xlt/engine/scripting/TestDataUtils.java b/src/main/java/com/xceptance/xlt/engine/scripting/TestDataUtils.java index 71bce0690..3853a73b6 100644 --- a/src/main/java/com/xceptance/xlt/engine/scripting/TestDataUtils.java +++ b/src/main/java/com/xceptance/xlt/engine/scripting/TestDataUtils.java @@ -39,6 +39,7 @@ import org.xml.sax.SAXException; import org.xml.sax.SAXParseException; +import com.xceptance.common.io.Utf8Reader; import com.xceptance.common.util.CsvUtils; import com.xceptance.xlt.engine.XltExecutionContext; @@ -255,7 +256,7 @@ private static Map parsePropertiesData(final InputStream is) thr try { final Properties props = new Properties(); - props.load(new InputStreamReader(is, "UTF-8")); + props.load(new Utf8Reader(is)); for (final Map.Entry entry : props.entrySet()) { From a380b0ec406aebbff0e5a79cba6a6cbf936336d0 Mon Sep 17 00:00:00 2001 From: Joerg Werner <4639399+jowerner@users.noreply.github.com> Date: Thu, 1 Jun 2023 14:46:54 +0200 Subject: [PATCH 3/3] - code and javadoc improvements --- src/main/java/com/xceptance/common/io/Utf8Reader.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/xceptance/common/io/Utf8Reader.java b/src/main/java/com/xceptance/common/io/Utf8Reader.java index d83ede140..b7ba79391 100644 --- a/src/main/java/com/xceptance/common/io/Utf8Reader.java +++ b/src/main/java/com/xceptance/common/io/Utf8Reader.java @@ -27,7 +27,9 @@ /** * A {@link Reader} implementation that reads UTF-8-encoded text from an {@link InputStream} throwing an - * {@link IOException} if the text is not valid UTF-8. + * {@link IOException} if the text is not valid UTF-8. The latter is in contrast to + * new InputStreamReader(is, StandardCharsets.UTF_8) which does not throw an exception, but simply returns + * bytes it cannot decode as funny ��� characters. *

* Note: This class buffers the content of the stream in memory so it should be used for small amounts of data only. */ @@ -49,18 +51,16 @@ public class Utf8Reader extends Reader public Utf8Reader(final InputStream inputStream) throws IOException { final byte[] bytes = inputStream.readAllBytes(); - char[] chars; try { - chars = getAsChars(bytes, StandardCharsets.UTF_8); + final char[] chars = getAsChars(bytes, StandardCharsets.UTF_8); + charArrayReader = new CharArrayReader(chars); } catch (final CharacterCodingException cce) { throw new IOException("Data does not represent UTF-8-encoded text", cce); } - - charArrayReader = new CharArrayReader(chars); } /**