refactor: robust whitespace handling (#83)

* feature: robust JSX whitespace handling

* cleanup

* refactor(js-macro): align whitespacing handling with js version

ref: lingui/js-lingui#1897

!BREAKING CHANGE

Cargo.toml

@@ -22,8 +22,7 @@ serde = "1"
 serde_json = "1.0.95"
 regex = "1.7.3"
 once_cell = "1.17.1"
-swc_common = "=0.33.15"
-swc_core = { version = "0.87.28", features = [
+swc_core = { version = "0.90.3", features = [
     "ecma_plugin_transform",
     "ecma_utils",
     "ecma_visit",

src/ast_utils.rs

@@ -149,6 +149,7 @@ pub fn create_key_value_prop(key: &str, value: Box<Expr>) -> PropOrSpread {
 pub fn create_import(source: JsWord, specifier: Ident) -> ModuleItem {
     ModuleItem::ModuleDecl(ModuleDecl::Import(ImportDecl {
         span: DUMMY_SP,
+        phase: ImportPhase::default(),
         specifiers: vec![
             ImportSpecifier::Named(ImportNamedSpecifier {
                 span: DUMMY_SP,

src/builder.rs

@@ -6,10 +6,6 @@ use swc_core::{
     },
 };
 
-use crate::{
-  normalize_witespaces_js::normalize_whitespaces_js,
-  normalize_witespaces_jsx::normalize_whitespaces_jsx,
-};
 use crate::tokens::{IcuChoice, CaseOrOffset, MsgToken};
 
 fn dedup_values(mut v: Vec<ValueWithPlaceholder>) -> Vec<ValueWithPlaceholder> {
@@ -55,7 +51,7 @@ pub struct MessageBuilder {
 }
 
 impl MessageBuilder {
-    pub fn parse(tokens: Vec<MsgToken>, jsx: bool) -> MessageBuilderResult {
+    pub fn parse(tokens: Vec<MsgToken>) -> MessageBuilderResult {
         let mut builder = MessageBuilder {
             message: String::new(),
             components_stack: Vec::new(),
@@ -65,15 +61,11 @@ impl MessageBuilder {
         };
 
         builder.from_tokens(tokens);
-        builder.to_args(jsx)
+        builder.to_args()
     }
 
-    pub fn to_args(mut self, jsx: bool) -> MessageBuilderResult {
-      let message_str = if jsx {
-        normalize_whitespaces_jsx(&self.message)
-      } else {
-        normalize_whitespaces_js(&self.message)
-      };
+    pub fn to_args(mut self) -> MessageBuilderResult {
+      let message_str = self.message;
 
       let message = Box::new(Expr::Lit(Lit::Str(Str {
         span: DUMMY_SP,

src/js_macro_folder.rs

@@ -24,7 +24,7 @@ impl<'a> JsMacroFolder<'a> {
     }
 
     fn create_message_descriptor_from_tokens(&mut self, tokens: Vec<MsgToken>) -> Expr {
-      let parsed = MessageBuilder::parse(tokens, false);
+      let parsed = MessageBuilder::parse(tokens);
 
       let mut props: Vec<PropOrSpread> = vec![
         create_key_value_prop("id", generate_message_id(&parsed.message_str, "").into()),
@@ -98,7 +98,7 @@ impl<'a> JsMacroFolder<'a> {
         if let Some(prop) = message_prop {
           let tokens = self.ctx.try_tokenize_expr(&prop.value).unwrap_or_else(|| Vec::new());
 
-          let parsed = MessageBuilder::parse(tokens, false);
+          let parsed = MessageBuilder::parse(tokens);
 
           if !id_prop.is_some() {
             new_props.push(

src/jsx_visitor.rs

@@ -32,6 +32,49 @@ static WORD_OPTION: Lazy<Regex> = Lazy::new(|| Regex::new(r"_(\w+)").unwrap());
 // const pluralRuleRe = /(_[\d\w]+|zero|one|two|few|many|other)/
 // const jsx2icuExactChoice = (value: string) => value.replace(/_(\d+)/, "=$1").replace(/_(\w+)/, "$1")
 
+// taken from babel repo -> packages/babel-types/src/utils/react/cleanJSXElementLiteralChild.ts
+fn clean_jsx_element_literal_child(value: &str) -> String {
+  let lines: Vec<&str> = value.split('\n').collect();
+  let mut last_non_empty_line = 0;
+
+  for (i, line) in lines.iter().enumerate() {
+    if line.trim().len() > 0 {
+      last_non_empty_line = i;
+    }
+  }
+
+  let mut result = String::new();
+
+  for (i, line) in lines.iter().enumerate() {
+    let is_first_line = i == 0;
+    let is_last_line = i == lines.len() - 1;
+    let is_last_non_empty_line = i == last_non_empty_line;
+
+    // replace rendered whitespace tabs with spaces
+    let mut trimmed_line = line.replace("\t", " ");
+
+    // trim whitespace touching a newline
+    if !is_first_line {
+      trimmed_line = trimmed_line.trim_start().to_string();
+    }
+
+    // trim whitespace touching an endline
+    if !is_last_line {
+      trimmed_line = trimmed_line.trim_end().to_string();
+    }
+
+    if !trimmed_line.is_empty() {
+      if !is_last_non_empty_line {
+        trimmed_line.push(' ');
+      }
+
+      result.push_str(&trimmed_line);
+    }
+  }
+
+  result
+}
+
 fn is_allowed_plural_option(key: &str) -> Option<JsWord> {
     if PLURAL_OPTIONS_WHITELIST.is_match(key) {
         let key = NUM_OPTION.replace(key, "=$1");
@@ -174,8 +217,9 @@ impl<'a> Visit for TransJSXVisitor<'a> {
     }
 
     fn visit_jsx_text(&mut self, el: &JSXText) {
+
         self.tokens.push(
-            MsgToken::String(el.value.to_string())
+            MsgToken::String(clean_jsx_element_literal_child(&el.raw.to_string()))
         );
     }
 

src/lib.rs

@@ -13,8 +13,6 @@ use swc_core::{
 };
 
 mod tests;
-mod normalize_witespaces_jsx;
-mod normalize_witespaces_js;
 mod builder;
 mod tokens;
 mod ast_utils;
@@ -58,7 +56,7 @@ impl LinguiMacroFolder {
             el.visit_children_with(&mut trans_visitor);
         }
 
-      let parsed = MessageBuilder::parse(trans_visitor.tokens, true);
+      let parsed = MessageBuilder::parse(trans_visitor.tokens);
       let id_attr = get_jsx_attr(&el.opening, "id");
 
       let context_attr_val = get_jsx_attr(&el.opening, "context")

src/macro_utils.rs

@@ -110,7 +110,7 @@ impl MacroCtx {
         let mut tokens: Vec<MsgToken> = Vec::with_capacity(tpl.quasis.len());
 
         for (i, tpl_element) in tpl.quasis.iter().enumerate() {
-            tokens.push(MsgToken::String(tpl_element.raw.to_string()));
+            tokens.push(MsgToken::String(tpl_element.cooked.as_ref().unwrap_or(&tpl_element.raw).to_string()));
 
             if let Some(exp) = tpl.exprs.get(i) {
                 if let Expr::Call(call) = exp.as_ref() {

src/tests/common/mod.rs

@@ -40,4 +40,3 @@ macro_rules! to {
         );
     }
 }
-

src/tests/js_t.rs

@@ -156,12 +156,46 @@ to!(
      r#"
         import { i18n } from "@lingui/core";
         i18n._({
-            id: "EfogM+",
-            message: "Multiline\nstring"
+            id: "amQF7O",
+            message: "Multiline\n           string"
         });
      "#
 );
 
+to!(
+    js_continuation_character,
+    r#"
+       import { t } from '@lingui/macro';
+         t`Multiline\
+           string`;
+    "#,
+     r#"
+        import { i18n } from "@lingui/core";
+        i18n._({
+            id: "d1nA7b",
+            message: "Multiline           string"
+        });
+     "#
+);
+to!(
+    unicode_characters_interpreted,
+    r#"
+       import { t } from '@lingui/macro';
+       t`Message \u0020`;
+       t`Bienvenue\xA0!`
+    "#,
+     r#"
+        import { i18n } from "@lingui/core";
+        i18n._({
+            id: "dZXeyN",
+            message: "Message  "
+        });
+        i18n._({
+            id: "9K3RGd",
+            message: "Bienvenue !"
+        });
+     "#
+);
 to!(
     js_support_message_descriptor_in_t_fn,
     r#"