Move Processor docs to docs/

Author: stasm

fluent.syntax/docs/processing.rst

@@ -0,0 +1,25 @@
+Processing
+==========
+
+.. code-block:: kotlin
+
+   package org.projectfluent.syntax.processor
+
+   import org.projectfluent.syntax.ast.*
+
+   /**
+    * Process patterns by returning new patterns with elements transformed.
+    */
+   class Processor {
+       /**
+        * "Bake" the values of StringLiterals into TextElements. This is a lossy
+        * transformation for literals which are not special in Fluent syntax.
+        */
+       fun unescapeLiteralsToText(pattern: Pattern): Pattern
+
+       /**
+        * "Un-bake" special characters into StringLiterals, which would otherwise
+        * cause syntax errors with Fluent parsers.
+        */
+       fun escapeTextToLiterals(pattern: Pattern): Pattern
+   }

fluent.syntax/docs/reference.rst

@@ -16,3 +16,4 @@ provide more fine-grained control and detail.
    ast
    visitor
    serializing
+   processing

fluent.syntax/docs/usage.rst

@@ -4,6 +4,7 @@ Using syntax
 The ``org.projectfluent:syntax`` package provides a parser, a serializer, and libraries
 for analysis and processing of Fluent files.
 
+
 Parsing
 -------
 
@@ -29,6 +30,7 @@ To create Fluent syntax from AST objects, use
    serializer.serialize(resource)
    serializer.serialize(resource.body[0])
 
+
 Analysis (Visitor)
 ------------------
 
@@ -51,6 +53,7 @@ to continue iteration.
        }
    }
 
+
 Custom Traversal (childrenOf)
 -----------------------------
 
@@ -69,3 +72,89 @@ iterate over children.
        listOf("default", "key", "span", "value"),
        variant_props.map { (name, _) -> name }.sorted().toList()
    )
+
+
+Pattern Processing
+------------------
+
+The :py:class:`syntax.processor.Processor` class can be used to transform
+patterns in a way that is friendly to localization workflows which want to
+allow text characters which are special in Fluent to be written as regular
+text.
+
+According to the Fluent syntax, characters like the curly braces must be
+enclosed in :py:class:`syntax.ast.StringLiteral` instances if they are
+supposed to be part of the translation content. Otherwise, an open curly
+brace would start a :py:class:`syntax.ast.Placeable` and likely lead to a
+syntax error.
+
+Workflows in which the support for Fluent placeables is limited may choose to
+provide their own visual cues for them. This often comes in form of visual
+placeholders which can be rearranged within a translation segment by the
+translator, but whose contents cannot be modified.
+
+In these workflows, the special meaning of the character like the curly
+braces is void; the translator is not able to insert new placeables by
+opening a curly brace anyways. Thus, for translators' convenience, the curly
+brace can be treated as a regular text character and part of the translation
+content.
+
+The :py:class:`syntax.processor.Processor`'s methods allow baking
+:py:class:`syntax.ast.StringLiteral` instances into surrounding
+:py:class:`syntax.ast.TextElement` instances, and then "un-baking" them again
+if required by the Fluent syntax. Note that all string literals are baked,
+while only some are un-baked. The processing is a lossy transformation.
+
+.. note::
+   Processed patterns are not valid Fluent AST nodes anymore and must not be
+   serialized without first un-processing them.
+
+
+Baking literals into text
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Use the :py:func:`unescapeLiteralsToText` method to bake the values of string
+literals into the surrounding text elements. This is a lossy transformation
+for literals which are not special in Fluent syntax.
+
+Examples::
+
+   →Hello, {"{-_-}"}.
+   ←Hello, {-_-}.
+
+   →{"     "}Hello, world!
+   ←     Hello, world!
+
+   →A multiline pattern:
+    {"*"} Asterisk is special
+   ←A multiline pattern:
+    * Asterisk is special
+
+   →Copyright {"\u00A9"} 2020
+   ←Copyright © 2020
+
+
+Un-baking special characters into literals
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Use the :py:func:`escapeTextToLiterals` method to un-bake special characters
+into string literals, which would otherwise cause syntax errors with Fluent
+parsers. Character sequences which might have been previously enclosed in
+string literals will not be un-baked as long as they are valid text
+characters in Fluent syntax.
+
+Examples::
+
+   →Hello, {-_-}.
+   ←Hello, {"{"}-_-{"}"}.
+
+   →     Hello, world!
+   ←{""}     Hello, world!
+
+   →A multiline pattern:
+    * Asterisk is special
+   ←A multiline pattern:
+    {"*"} Asterisk is special
+
+   →Copyright © 2020
+   ←Copyright © 2020

fluent.syntax/src/main/kotlin/org/projectfluent/syntax/processor/Processor.kt

@@ -7,53 +7,12 @@ private val special =
     """\\(([\\"])|(u[0-9a-fA-F]{4}))""".toRegex()
 
 /**
- * Processor for transforming patterns.
- *
- * This class can be used to transform patterns in a way that is friendly
- * to localization workflows which want to allow text characters which are
- * special in Fluent to be written as regular text.
- *
- * According to the Fluent syntax, characters like the curly braces must be
- * enclosed in StringLiterals if they are supposed to be part of the
- * translation content. Otherwise, an open curly brace would start a Placeable
- * and likely lead to a syntax error.
- *
- * Workflows in which the support for Fluent placeables is limited may choose
- * to provide their own visual cues for them. This often comes in form of
- * visual placeholders which can be rearranged within a translation
- * segment by the translator, but whose contents cannot be modified.
- *
- * In these workflows, the special meaning of the character like the curly
- * braces is void; the translator is not able to insert new placeables by
- * opening a curly brace anyways. Thus, for translators' convenience, the
- * curly brace can be treated as a regular text character and part of the
- * translation content.
- *
- * The Processor's methods allow baking StringLiterals into TextElements, and
- * then "un-baking" them again if required by the Fluent syntax. Note that
- * all StringLiterals are baked, while only some are un-baked. The Processor
- * is lossy.
+ * Process patterns by returning new patterns with elements transformed.
  */
 class Processor {
     /**
      * "Bake" the values of StringLiterals into TextElements. This is a lossy
      * transformation for literals which are not special in Fluent syntax.
-     *
-     * Examples:
-     *
-     *     →Hello, {"{-_-}"}.
-     *     ←Hello, {-_-}.
-     *
-     *     →{"     "}Hello, world!
-     *     ←     Hello, world!
-     *
-     *     →A multiline pattern:
-     *      {"*"} Asterisk is special
-     *     ←A multiline pattern:
-     *      * Asterisk is special
-     *
-     *     →Copyright {"\u00A9"} 2020
-     *     ←Copyright © 2020
      */
     fun unescapeLiteralsToText(pattern: Pattern): Pattern {
         val result = Pattern()
@@ -65,25 +24,7 @@ class Processor {
 
     /**
      * "Un-bake" special characters into StringLiterals, which would otherwise
-     * cause syntax errors with Fluent parsers. Character sequences which might
-     * have been previously enclosed in StringLiterals, will not be un-baked,
-     * provided they are valid text characters in Fluent syntax.
-     *
-     * Examples:
-     *
-     *     →Hello, {-_-}.
-     *     ←Hello, {"{"}-_-{"}"}.
-     *
-     *     →     Hello, world!
-     *     ←{""}     Hello, world!
-     *
-     *     →A multiline pattern:
-     *      * Asterisk is special
-     *     ←A multiline pattern:
-     *      {"*"} Asterisk is special
-     *
-     *     →Copyright © 2020
-     *     ←Copyright © 2020
+     * cause syntax errors with Fluent parsers.
      */
     fun escapeTextToLiterals(pattern: Pattern): Pattern {
         val result = Pattern()