Unicode handling changes (#1252)

* first pass at unicode handling

* first pass at system-test

* updating Jenkins test image

* remove MT parts from yaml file'

* disabling unicode characaters in system-test causing Jenkins to fail

* removing comments

* one more try to remove special characters

* one more try for unicode and Jenkins

* one more try for unicode and Jenkins

* replacing lost changes and removing unicode characters from the tests

Co-authored-by: Carolyn Rountree <[email protected]>

Author: robertpatrick

Jenkinsfile

@@ -32,7 +32,7 @@ pipeline {
                 docker {
                     alwaysPull true
                     reuseNode true
-                    image 'phx.ocir.io/weblogick8s/wdt/jenkinsslave:wls12213'
+                    image 'phx.ocir.io/weblogick8s/wdt/jenkins-slave:122130'
                     args '-u jenkins -v /var/run/docker.sock:/var/run/docker.sock'
                 }
             }
@@ -57,7 +57,7 @@ pipeline {
                 docker {
                     alwaysPull true
                     reuseNode true
-                    image 'phx.ocir.io/weblogick8s/wdt/jenkinsslave:wls12213'
+                    image 'phx.ocir.io/weblogick8s/wdt/jenkins-slave:122130'
                     args '-u jenkins -v /var/run/docker.sock:/var/run/docker.sock'
                 }
             }

core/src/main/java/oracle/weblogic/deploy/json/AbstractJsonTranslator.java

@@ -29,6 +29,7 @@
 import org.python.core.PyLong;
 import org.python.core.PyObject;
 import org.python.core.PyString;
+import org.python.core.PyUnicode;
 
 /**
  * This class does the heavy-lifting of walking the parse tree and performing the conversion into a Python dictionary.
@@ -43,6 +44,8 @@ public abstract class AbstractJsonTranslator extends JSONBaseListener {
     private PyObject currentScalarValue;
     @SuppressWarnings("WeakerAccess")
     protected boolean useOrderedDict;
+    @SuppressWarnings("WeakerAccess")
+    protected boolean useUnicode;
 
     /**
      * This method triggers parsing of the JSON and conversion into the Python dictionary.
@@ -109,7 +112,7 @@ public void exitPair(JSONParser.PairContext ctx) {
                 getLogger().severe("WLSDPLY-18027", name, valueType);
                 value = Py.None;
         }
-        container.__setitem__(new PyString(name), value);
+        container.__setitem__(this.getPythonString(name), value);
     }
 
     /**
@@ -135,7 +138,7 @@ public void enterJsonObject(JSONParser.JsonObjectContext ctx) {
 
         String name = currentPairName.peek();
         PyDictionary nextDict = currentDict.peek();
-        if (name != null && nextDict != null && nextDict.has_key(new PyString(name))) {
+        if (name != null && nextDict != null && nextDict.has_key(this.getPythonString(name))) {
             String message = ExceptionHelper.getMessage("WLSDPLY-18028", name);
             ParseCancellationException ex =
                 new ParseCancellationException(message);
@@ -184,7 +187,7 @@ public void exitJsonArray(JSONParser.JsonArrayContext ctx) {
     @Override
     public void enterJsonString(JSONParser.JsonStringContext ctx) {
         String cleanString = resolveEscapeSequences(StringUtils.stripQuotes(ctx.STRING().getText()));
-        currentScalarValue = new PyString(cleanString);
+        currentScalarValue = this.getPythonString(cleanString);
         currentValueType.push(ValueType.SCALAR);
     }
 
@@ -362,6 +365,14 @@ private void addToArrayIfNeeded() {
         }
     }
 
+    private PyObject getPythonString(String text) {
+        if (this.useUnicode) {
+            return new PyUnicode(text);
+        } else {
+            return new PyString(text);
+        }
+    }
+
     private static String resolveEscapeSequences(String text) {
         String result = text;
         if (!StringUtils.isEmpty(text)) {

core/src/main/java/oracle/weblogic/deploy/json/JsonStreamTranslator.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2020, Oracle Corporation and/or its affiliates.
+ * Copyright (c) 2017, 2022, Oracle Corporation and/or its affiliates.
  * Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
  */
 package oracle.weblogic.deploy.json;
@@ -43,6 +43,23 @@ public JsonStreamTranslator(String streamFileName, InputStream jsonStream, boole
         this.streamFileName = streamFileName;
         this.jsonStream = jsonStream;
         this.useOrderedDict = useOrderedDict;
+        this.useUnicode = false;
+    }
+
+    /**
+     * The constructor used to specify ordering and unicode usage.
+     *
+     * @param streamFileName the name of the file used to create the InputStream (used only for logging purposes)
+     * @param jsonStream the input stream
+     * @param useOrderedDict whether or not to use an ordered dictionary for storing translation results
+     * @param useUnicode whether or not to use PyUnicode instead of PyString
+     */
+    public JsonStreamTranslator(String streamFileName, InputStream jsonStream,
+                                boolean useOrderedDict, boolean useUnicode) {
+        this.streamFileName = streamFileName;
+        this.jsonStream = jsonStream;
+        this.useOrderedDict = useOrderedDict;
+        this.useUnicode = useUnicode;
     }
 
     /**

core/src/main/java/oracle/weblogic/deploy/json/JsonTranslator.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2020, Oracle Corporation and/or its affiliates.
+ * Copyright (c) 2017, 2022, Oracle Corporation and/or its affiliates.
  * Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
  */
 package oracle.weblogic.deploy.json;
@@ -44,6 +44,21 @@ public JsonTranslator(String fileName) {
     public JsonTranslator(String fileName, boolean useOrdering) {
         this.jsonFile = FileUtils.validateExistingFile(fileName);
         this.useOrderedDict = useOrdering;
+        this.useUnicode = false;
+    }
+
+    /**
+     * Constructor for parsing JSON file into a Python dictionary and control ordering and unicode usage.
+     *
+     * @param fileName - the name of the existing JSON file to parse
+     * @param useOrdering - whether or not to use an ordered dictionary
+     * @param useUnicode - whether or not to use PyUnicode instead of PyString
+     * @throws IllegalArgumentException if the file name is null or does not point to a valid, existing file.
+     */
+    public JsonTranslator(String fileName, boolean useOrdering, boolean useUnicode) {
+        this.jsonFile = FileUtils.validateExistingFile(fileName);
+        this.useOrderedDict = useOrdering;
+        this.useUnicode = useUnicode;
     }
 
     /**

core/src/main/java/oracle/weblogic/deploy/yaml/AbstractYamlTranslator.java

@@ -24,6 +24,7 @@
 import org.python.core.PyLong;
 import org.python.core.PyObject;
 import org.python.core.PyString;
+import org.python.core.PyUnicode;
 
 import org.yaml.snakeyaml.DumperOptions;
 import org.yaml.snakeyaml.LoaderOptions;
@@ -35,6 +36,7 @@
 public abstract class AbstractYamlTranslator {
 
     private final boolean useOrderedDict;
+    private final boolean useUnicode;
     private final String fileName;
     private final int codePointsLimit;
 
@@ -47,9 +49,10 @@ public abstract class AbstractYamlTranslator {
     // override to write a list of documents as Python dictionaries to the YAML
     public abstract void dumpDocuments(List<?> documents) throws YamlException;
 
-    protected AbstractYamlTranslator(String fileName, boolean useOrderedDict, int codePointsLimit) {
+    protected AbstractYamlTranslator(String fileName, boolean useOrderedDict, boolean useUnicode, int codePointsLimit) {
         this.fileName = fileName;
         this.useOrderedDict = useOrderedDict;
+        this.useUnicode = useUnicode;
         this.codePointsLimit = codePointsLimit;
     }
 
@@ -268,7 +271,11 @@ private PyObject convertScalarToPythonObject(Object object) throws YamlException
             String classname = object.getClass().getName();
             switch (classname) {
                 case "java.lang.String":
-                    result = new PyString((String) object);
+                    if (this.useUnicode) {
+                        result = new PyUnicode((String) object);
+                    } else {
+                        result = new PyString((String) object);
+                    }
                     break;
 
                 case "java.lang.Boolean":

core/src/main/java/oracle/weblogic/deploy/yaml/YamlStreamTranslator.java

@@ -32,7 +32,7 @@ public class YamlStreamTranslator extends AbstractYamlTranslator {
      * @param yamlStream the input stream
      */
     public YamlStreamTranslator(String streamFileName, InputStream yamlStream) {
-        this(streamFileName, yamlStream, false);
+        this(streamFileName, yamlStream, false, false, 0);
     }
 
     /**
@@ -43,7 +43,7 @@ public YamlStreamTranslator(String streamFileName, InputStream yamlStream) {
      * @param useOrderedDict whether or not to use an ordered dictionary to maintain the order
      */
     public YamlStreamTranslator(String streamFileName, InputStream yamlStream, boolean useOrderedDict) {
-        super(streamFileName, useOrderedDict, 0);
+        super(streamFileName, useOrderedDict, false, 0);
         this.streamFileName = streamFileName;
         this.yamlStream = yamlStream;
         this.yamlOutputWriter = null;
@@ -57,21 +57,39 @@ public YamlStreamTranslator(String streamFileName, InputStream yamlStream, boole
      * @param useOrderedDict whether or not to use an ordered dictionary to maintain the order
      * @param maxCodePoints the maximum number of characters that the parser will accept
      */
-    public YamlStreamTranslator(String streamFileName, InputStream yamlStream, boolean useOrderedDict, int maxCodePoints) {
-        super(streamFileName, useOrderedDict, maxCodePoints);
+    public YamlStreamTranslator(String streamFileName, InputStream yamlStream, boolean useOrderedDict,
+                                int maxCodePoints) {
+        super(streamFileName, useOrderedDict, false, maxCodePoints);
         this.streamFileName = streamFileName;
         this.yamlStream = yamlStream;
         this.yamlOutputWriter = null;
     }
 
+    /**
+     * The constructor that allows control of everything.
+     * @param streamFileName the name of the file used to create the InputStream (used only for logging purposes)
+     * @param yamlStream the input stream
+     * @param useOrderedDict whether or not to use an ordered dictionary to maintain the order
+     * @param useUnicode whether or not to use PyUnicode instead of PyString
+     * @param maxCodePoints the maximum number of characters that the parser will accept
+     */
+    public YamlStreamTranslator(String streamFileName, InputStream yamlStream, boolean useOrderedDict,
+                                boolean useUnicode, int maxCodePoints) {
+        super(streamFileName, useOrderedDict, useUnicode, maxCodePoints);
+        this.streamFileName = streamFileName;
+        this.yamlStream = yamlStream;
+        this.yamlOutputWriter = null;
+    }
+
+
     /**
      * The constructor for writing YAML output.
      *
      * @param streamFileName   the name of the file used to create the OutputStream (used only for logging purposes)
      * @param yamlOutputWriter the Writer to use for writing the YAML output
      */
     public YamlStreamTranslator(String streamFileName, Writer yamlOutputWriter) {
-        super(streamFileName, true, 0);
+        super(streamFileName, true, false, 0);
         this.streamFileName = streamFileName;
         this.yamlStream = null;
         this.yamlOutputWriter = yamlOutputWriter;

core/src/main/java/oracle/weblogic/deploy/yaml/YamlTranslator.java

@@ -32,7 +32,7 @@ public class YamlTranslator extends AbstractYamlTranslator {
      * @throws IllegalArgumentException if the file name is null or does not point to a valid, existing file.
      */
     public YamlTranslator(String fileName) {
-        this(fileName, false, 0);
+        this(fileName, false, false, 0);
     }
 
     /**
@@ -43,7 +43,7 @@ public YamlTranslator(String fileName) {
      * @throws IllegalArgumentException if the file name is null or does not point to a valid, existing file.
      */
     public YamlTranslator(String fileName, boolean useOrderedDict) {
-        super(fileName, useOrderedDict, 0);
+        super(fileName, useOrderedDict, false, 0);
         this.yamlFile = FileUtils.validateExistingFile(fileName);
     }
 
@@ -57,7 +57,21 @@ public YamlTranslator(String fileName, boolean useOrderedDict) {
      * @throws IllegalArgumentException if the file name is null or does not point to a valid, existing file.
      */
     public YamlTranslator(String fileName, boolean useOrderedDict, int maxCodePoints) {
-        super(fileName, useOrderedDict, maxCodePoints);
+        super(fileName, useOrderedDict, false, maxCodePoints);
+        this.yamlFile = FileUtils.validateExistingFile(fileName);
+    }
+
+    /**
+     * Constructor for parsing YAML file into a Python dictionary, controlling everything.
+     *
+     * @param fileName the name of the existing YAML file to parse
+     * @param useOrderedDict whether or not to use an ordered dictionary to maintain the order
+     * @param useUnicode whether or not to use PyUnicode instead of PyString
+     * @param maxCodePoints the maximum number of code points for the input file, or zero to accept the default
+     * @throws IllegalArgumentException if the file name is null or does not point to a valid, existing file.
+     */
+    public YamlTranslator(String fileName, boolean useOrderedDict, boolean useUnicode, int maxCodePoints) {
+        super(fileName, useOrderedDict, useUnicode, maxCodePoints);
         this.yamlFile = FileUtils.validateExistingFile(fileName);
     }