#372: Read properties files as UTF-8

src/main/java/com/xceptance/common/io/Utf8Reader.java

@@ -0,0 +1,104 @@
+/*
+ * Copyright (c) 2005-2023 Xceptance Software Technologies GmbH
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.xceptance.common.io;
+
+import java.io.CharArrayReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Reader;
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.charset.CharacterCodingException;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+
+/**
+ * A {@link Reader} implementation that reads UTF-8-encoded text from an {@link InputStream} throwing an
+ * {@link IOException} if the text is not valid UTF-8.
+ * <p>
+ * Note: This class buffers the content of the stream in memory so it should be used for small amounts of data only.
+ */
+public class Utf8Reader extends Reader
+{
+    /**
+     * The reader we delegate to when actually dealing out the characters.
+     */
+    private final CharArrayReader charArrayReader;
+
+    /**
+     * Creates a new {@link Utf8Reader} instance.
+     * 
+     * @param inputStream
+     *            the stream to read the text from
+     * @throws IOException
+     *             if the input stream could not be read or does not represent UTF-8-encoded text
+     */
+    public Utf8Reader(final InputStream inputStream) throws IOException
+    {
+        final byte[] bytes = inputStream.readAllBytes();
+        char[] chars;
+
+        try
+        {
+            chars = getAsChars(bytes, StandardCharsets.UTF_8);
+        }
+        catch (final CharacterCodingException cce)
+        {
+            throw new IOException("Data does not represent UTF-8-encoded text", cce);
+        }
+
+        charArrayReader = new CharArrayReader(chars);
+    }
+
+    /**
+     * Converts the given bytes to chars according to the specified character set.
+     * 
+     * @param bytes
+     *            the input bytes
+     * @param charset
+     *            the character set
+     * @return the corresponding chars
+     * @throws CharacterCodingException
+     *             if the bytes do not represent text encoded with the given character set
+     */
+    private static char[] getAsChars(final byte[] bytes, final Charset charset) throws CharacterCodingException
+    {
+        final CharBuffer charBuffer = charset.newDecoder().decode(ByteBuffer.wrap(bytes));
+
+        final char[] chars = new char[charBuffer.length()];
+        charBuffer.get(chars);
+
+        return chars;
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public int read(final char[] cbuf, final int off, final int len) throws IOException
+    {
+        return charArrayReader.read(cbuf, off, len);
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public void close() throws IOException
+    {
+        charArrayReader.close();
+    }
+}

src/main/java/com/xceptance/common/util/PropertiesUtils.java

@@ -31,6 +31,8 @@
 import org.apache.commons.vfs2.FileObject;
 import org.apache.commons.vfs2.VFS;
 
+import com.xceptance.common.io.Utf8Reader;
+
 /**
  * The PropertiesUtils helps in dealing with properties files.
  *
@@ -123,15 +125,15 @@ public static void loadProperties(final FileObject file, final Properties props)
         {
             ParameterCheckUtils.isReadableFile(file, "file");
         }
-        catch(IllegalArgumentException e)
+        catch (IllegalArgumentException e)
         {
             throw new FileNotFoundException(file.toString());
         }
         ParameterCheckUtils.isNotNull(props, "props");
 
         try (final InputStream is = file.getContent().getInputStream())
         {
-            props.load(is);
+            props.load(new Utf8Reader(is));
         }
     }
 

src/main/java/com/xceptance/xlt/agentcontroller/AgentControllerImpl.java

@@ -1118,14 +1118,14 @@ private static void maskFile(File inputFile, File outputFile) throws Configurati
     {
         PropertiesConfiguration config = new PropertiesConfiguration();
         config.setIOFactory(new JupIOFactory()); // for better compatibility with java.util.Properties (GH#144)
-        try (final FileReader reader = new FileReader(inputFile))
+        try (final FileReader reader = new FileReader(inputFile, StandardCharsets.UTF_8))
         {
             config.read(reader);
         }
         config = mask(config, inputFile.getName().equals(XltConstants.SECRET_PROPERTIES_FILENAME));
         final StringWriter writer = new StringWriter();
         config.write(writer);
-        FileUtils.writeStringToFile(outputFile, writer.toString(), StandardCharsets.ISO_8859_1);
+        FileUtils.writeStringToFile(outputFile, writer.toString(), StandardCharsets.UTF_8);
     }
 
     /**

src/main/java/com/xceptance/xlt/engine/scripting/TestDataUtils.java

@@ -39,6 +39,7 @@
 import org.xml.sax.SAXException;
 import org.xml.sax.SAXParseException;
 
+import com.xceptance.common.io.Utf8Reader;
 import com.xceptance.common.util.CsvUtils;
 import com.xceptance.xlt.engine.XltExecutionContext;
 
@@ -255,7 +256,7 @@ private static Map<String, String> parsePropertiesData(final InputStream is) thr
         try
         {
             final Properties props = new Properties();
-            props.load(new InputStreamReader(is, "UTF-8"));
+            props.load(new Utf8Reader(is));
 
             for (final Map.Entry<Object, Object> entry : props.entrySet())
             {

src/main/java/com/xceptance/xlt/mastercontroller/ResultDownloader.java

@@ -446,7 +446,7 @@ private boolean updateTimeData(final FileObject testPropFile)
                 if (startTime > 0L && startTime < Long.MAX_VALUE)
                 {
                     try (var w = new BufferedWriter(new OutputStreamWriter(testPropFile.getContent().getOutputStream(true),
-                                                                           StandardCharsets.ISO_8859_1)))
+                                                                           StandardCharsets.UTF_8)))
                     {
                         w.newLine();
                         w.newLine();

src/main/java/com/xceptance/xlt/util/PropertiesIOException.java

@@ -31,4 +31,9 @@ public PropertiesIOException(final String msg)
     {
         super(msg);
     }
+
+    public PropertiesIOException(String message, Throwable cause)
+    {
+        super(message, cause);
+    }
 }

src/main/java/com/xceptance/xlt/util/XltPropertiesImpl.java

@@ -418,9 +418,7 @@ private void process(final FileObject homeDirectory, final FileObject configDire
             }
             catch (IOException e)
             {
-                XltLogger.runTimeLogger.error(String.format("Issues loading properties from %s", fileName), e);
-
-                throw new PropertiesIOException(String.format("Issues loading properties from %s", fileName));
+                throw new PropertiesIOException(String.format("Issues loading properties from %s", fileName), e);
             }
         }
 

src/test/java/com/xceptance/common/io/Utf8ReaderTest.java

@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2005-2023 Xceptance Software Technologies GmbH
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.xceptance.common.io;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.Reader;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import junitparams.JUnitParamsRunner;
+import junitparams.Parameters;
+
+/**
+ * Tests the implementation of {@link Utf8Reader}.
+ */
+@RunWith(JUnitParamsRunner.class)
+public class Utf8ReaderTest
+{
+    /**
+     * Checks that the reader can read test texts encoded with the given encodings.
+     */
+    @Test
+    @Parameters(value =
+        {
+            "UTF-8      | 日本の東京", //
+            "UTF-8      | ッ ツ ヅ ミ べ ボ プ", //
+            "UTF-8      | äöüÄÖÜßáàÁÀ", //
+            "UTF-8      | foobar", //
+            "ISO-8859-1 | foobar", // same bytes as with UTF-8
+            "US-ASCII   | foobar", // same bytes as with UTF-8
+    })
+    public void read(final String charsetName, final String text) throws IOException
+    {
+        doRead(charsetName, text);
+    }
+
+    /**
+     * Checks that the reader throws an IOException for test texts encoded with the given encodings.
+     */
+    @Test(expected = IOException.class)
+    @Parameters(value =
+        {
+            "ISO-8859-1 | äöüÄÖÜßáàÁÀ",  //
+            "UTF-16     | äöüÄÖÜßáàÁÀ",  //
+            "UTF-16BE   | äöüÄÖÜßáàÁÀ",  //
+            "UTF-16LE   | äöüÄÖÜßáàÁÀ",  //
+    })
+    public void read_illegalEncoding(final String charsetName, final String text) throws IOException
+    {
+        doRead(charsetName, text);
+    }
+
+    private void doRead(final String charsetName, final String text) throws IOException
+    {
+        // get the bytes of the text in the wanted encoding
+        final byte[] bytes = text.getBytes(charsetName);
+
+        // now read the bytes in again via the Utf8Reader and check the resulting text
+        try (final Reader reader = new Utf8Reader(new ByteArrayInputStream(bytes)))
+        {
+            final char[] chars = new char[1024];
+            final int charsRead = reader.read(chars);
+
+            final String actualText = new String(chars, 0, charsRead);
+
+            Assert.assertEquals(text.length(), charsRead);
+            Assert.assertEquals(text, actualText);
+        }
+    }
+}

src/test/java/com/xceptance/xlt/report/providers/ConfigurationReportProviderTest.java

@@ -40,7 +40,7 @@ public void testSecretPropertiesAreMaskedInTheOutput() throws IOException
         {
             final Path secretPath = testDir.resolve("config").resolve(XltConstants.SECRET_PROPERTIES_FILENAME);
             Files.createDirectories(secretPath.getParent());
-            Files.write(secretPath, "value=Some very secret Value\n".getBytes(StandardCharsets.ISO_8859_1));
+            Files.write(secretPath, "value=Some very secret Value\n".getBytes(StandardCharsets.UTF_8));
             final ConfigurationReportProvider provider = new ConfigurationReportProvider();
             ReportGeneratorConfiguration config = new ReportGeneratorConfiguration();
             config.setReportDirectory(testDir.toFile());