Fix js parser

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "compiler/tests-js-parser/test262"]
+	path = compiler/tests-js-parser/test262
+	url = https://github.com/tc39/test262.git

CHANGES.md

@@ -9,7 +9,11 @@
 * Compiler: js-parser: fix support for LHS assignment target
 * Compiler: js-parser: fix parser of default export
 * Compiler: js-parser: allow 'as' as ident
-* Compiler: js-parser: fix for in rewriting
+* Compiler: js-parser: fix for-in rewriting
+* Compiler: js-parser: fix yield pretty print
+* Compiler: js-parser: fix async arrow function
+* Compiler: js-parser: fix class printing
+* Compiler: js-parser: fix #privateName
 
 # 5.5.2 (2023-12-01) - Lille
 

compiler/lib/flow_lexer.ml

@@ -224,8 +224,13 @@ let lex_error (env : Lex_env.t) loc err : Lex_env.t =
   let lex_errors_acc = (loc, err) :: env.lex_state.lex_errors_acc in
   { env with lex_state = { lex_errors_acc } }
 
-let illegal (env : Lex_env.t) (loc : Loc.t) =
-  lex_error env loc (Parse_error.Unexpected "token ILLEGAL")
+let illegal (env : Lex_env.t) (loc : Loc.t) reason =
+  let reason =
+    match reason with
+    | "" -> "token ILLEGAL"
+    | s -> s
+  in
+  lex_error env loc (Parse_error.Unexpected reason)
 
 let decode_identifier =
   let sub_lexeme lexbuf trim_start trim_end =
@@ -315,7 +320,7 @@ let decode_identifier =
     id_char env loc buf lexbuf
 
 let recover env lexbuf ~f =
-  let env = illegal env (loc_of_lexbuf env lexbuf) in
+  let env = illegal env (loc_of_lexbuf env lexbuf) "recovery" in
   Sedlexing.rollback lexbuf;
   f env lexbuf
 
@@ -364,7 +369,7 @@ let rec comment env buf lexbuf =
       lexeme_to_buffer lexbuf buf;
       comment env buf lexbuf
   | _ ->
-      let env = illegal env (loc_of_lexbuf env lexbuf) in
+      let env = illegal env (loc_of_lexbuf env lexbuf) "" in
       env
 
 let drop_line env =
@@ -385,7 +390,7 @@ let rec line_comment env buf lexbuf =
       line_comment env buf lexbuf
   | _ -> failwith "unreachable line_comment"
 
-let string_escape env lexbuf =
+let string_escape ~accept_invalid env lexbuf =
   match%sedlex lexbuf with
   | eof | '\\' ->
       let str = lexeme lexbuf in
@@ -420,11 +425,17 @@ let string_escape env lexbuf =
       let hex = String.sub str 2 (String.length str - 3) in
       let code = int_of_string ("0x" ^ hex) in
       (* 11.8.4.1 *)
-      let env = if code > 0x10FFFF then illegal env (loc_of_lexbuf env lexbuf) else env in
+      let env =
+        if code > 0x10FFFF && not accept_invalid
+        then illegal env (loc_of_lexbuf env lexbuf) "unicode escape out of range"
+        else env
+      in
       env, str
   | 'u' | 'x' | '0' .. '7' ->
       let str = lexeme lexbuf in
-      let env = illegal env (loc_of_lexbuf env lexbuf) in
+      let env =
+        if accept_invalid then env else illegal env (loc_of_lexbuf env lexbuf) ""
+      in
       env, str
   | line_terminator_sequence ->
       newline lexbuf;
@@ -450,21 +461,22 @@ let rec string_quote env q buf lexbuf =
       newline lexbuf;
       string_quote env q buf lexbuf
   | '\\' ->
-      let env, str = string_escape env lexbuf in
-      if String.equal str "" || String.get q 0 <> String.get str 0
-      then Buffer.add_string buf "\\";
+      let env, str = string_escape ~accept_invalid:false env lexbuf in
+      (match str with
+      | "'" | "\"" -> ()
+      | _ -> Buffer.add_string buf "\\");
       Buffer.add_string buf str;
       string_quote env q buf lexbuf
   | '\n' ->
       let x = lexeme lexbuf in
       Buffer.add_string buf x;
-      let env = illegal env (loc_of_lexbuf env lexbuf) in
+      let env = illegal env (loc_of_lexbuf env lexbuf) "" in
       string_quote env q buf lexbuf
   (* env, end_pos_of_lexbuf env lexbuf *)
   | eof ->
       let x = lexeme lexbuf in
       Buffer.add_string buf x;
-      let env = illegal env (loc_of_lexbuf env lexbuf) in
+      let env = illegal env (loc_of_lexbuf env lexbuf) "" in
       env
   (* match multi-char substrings that don't contain the start chars of the above patterns *)
   | Plus (Compl ("'" | '"' | '\\' | '\n' | eof)) | any ->
@@ -688,13 +700,19 @@ let token (env : Lex_env.t) lexbuf : result =
               | None -> (
                   match is_valid_identifier_name decoded with
                   | true -> env
-                  | false -> illegal env (loc_of_lexbuf env lexbuf))
-              | Some _ -> illegal env (loc_of_lexbuf env lexbuf)
+                  | false ->
+                      illegal
+                        env
+                        (loc_of_lexbuf env lexbuf)
+                        (Printf.sprintf "%S is not a valid identifier" decoded))
+              | Some _ ->
+                  (* accept keyword as ident if escaped *)
+                  env
             in
             Token (env, T_IDENTIFIER (Stdlib.Utf8_string.of_string_exn decoded, raw)))
   | eof -> Token (env, T_EOF)
   | any ->
-      let env = illegal env (loc_of_lexbuf env lexbuf) in
+      let env = illegal env (loc_of_lexbuf env lexbuf) "" in
       Token (env, T_ERROR (lexeme lexbuf))
   | _ -> failwith "unreachable token"
 
@@ -783,7 +801,7 @@ let regexp env lexbuf =
       let env, flags = regexp_body env buf lexbuf in
       Token (env, T_REGEXP (Stdlib.Utf8_string.of_string_exn (Buffer.contents buf), flags))
   | any ->
-      let env = illegal env (loc_of_lexbuf env lexbuf) in
+      let env = illegal env (loc_of_lexbuf env lexbuf) "" in
       Token (env, T_ERROR (lexeme lexbuf))
   | _ -> failwith "unreachable regexp"
 
@@ -804,12 +822,12 @@ let backquote env lexbuf =
   | '\\' ->
       let buf = Buffer.create 127 in
       Buffer.add_char buf '\\';
-      let env, str = string_escape env lexbuf in
+      let env, str = string_escape ~accept_invalid:true env lexbuf in
       Buffer.add_string buf str;
       Token (env, T_ENCAPSED_STRING (Buffer.contents buf))
   | eof -> Token (env, T_EOF)
   | _ ->
-      let env = illegal env (loc_of_lexbuf env lexbuf) in
+      let env = illegal env (loc_of_lexbuf env lexbuf) "" in
       Token (env, T_ERROR (lexeme lexbuf))
 
 let wrap f =

compiler/lib/javascript.ml

@@ -279,19 +279,24 @@ and expression =
   | ECallTemplate of expression * template * location
   | EAccess of expression * access_kind * expression
   | EDot of expression * access_kind * identifier
+  | EDotPrivate of expression * access_kind * identifier
   | ENew of expression * arguments option
   | EVar of ident
   | EFun of ident option * function_declaration
   | EClass of ident option * class_declaration
-  | EArrow of function_declaration * arrow_info
+  | EArrow of function_declaration * bool * arrow_info
   | EStr of Utf8_string.t
   | ETemplate of template
   | EArr of array_litteral
   | EBool of bool
   | ENum of Num.t
   | EObj of property_list
   | ERegexp of string * string option
-  | EYield of expression option
+  | EYield of
+      { delegate : bool
+      ; expr : expression option
+      }
+  | EPrivName of identifier
   | CoverParenthesizedExpressionAndArrowParameterList of early_error
   | CoverCallExpressionAndAsyncArrowHead of early_error
 
@@ -343,7 +348,7 @@ and statement =
   | Continue_statement of Label.t option
   | Break_statement of Label.t option
   | Return_statement of expression option
-  (* | With_statement of expression * statement *)
+  | With_statement of expression * (statement * location)
   | Labelled_statement of Label.t * (statement * location)
   | Switch_statement of
       expression * case_clause list * statement_list option * case_clause list
@@ -395,7 +400,7 @@ and class_element =
 
 and class_element_name =
   | PropName of property_name
-  | PrivName of ident
+  | PrivName of identifier
 
 and ('a, 'b) list_with_rest =
   { list : 'a list
@@ -450,8 +455,8 @@ and export =
   | ExportClass of ident * class_declaration
   | ExportNames of (ident * Utf8_string.t) list
   (* default *)
-  | ExportDefaultFun of ident * function_declaration
-  | ExportDefaultClass of ident * class_declaration
+  | ExportDefaultFun of ident option * function_declaration
+  | ExportDefaultClass of ident option * class_declaration
   | ExportDefaultExpression of expression
   (* from *)
   | ExportFrom of

compiler/lib/javascript.mli

@@ -196,11 +196,12 @@ and expression =
   | ECallTemplate of expression * template * location
   | EAccess of expression * access_kind * expression
   | EDot of expression * access_kind * identifier
+  | EDotPrivate of expression * access_kind * identifier
   | ENew of expression * arguments option
   | EVar of ident
   | EFun of ident option * function_declaration
   | EClass of ident option * class_declaration
-  | EArrow of function_declaration * arrow_info
+  | EArrow of function_declaration * bool * arrow_info
   | EStr of Utf8_string.t
   (* A UTF-8 encoded string that may contain escape sequences. *)
   | ETemplate of template
@@ -209,7 +210,11 @@ and expression =
   | ENum of Num.t
   | EObj of property_list
   | ERegexp of string * string option
-  | EYield of expression option
+  | EYield of
+      { delegate : bool
+      ; expr : expression option
+      }
+  | EPrivName of identifier
   | CoverParenthesizedExpressionAndArrowParameterList of early_error
   | CoverCallExpressionAndAsyncArrowHead of early_error
 
@@ -261,9 +266,7 @@ and statement =
   | Continue_statement of Label.t option
   | Break_statement of Label.t option
   | Return_statement of expression option
-  (*
-  | With_statement
-*)
+  | With_statement of expression * (statement * location)
   | Labelled_statement of Label.t * (statement * location)
   | Switch_statement of
       expression * case_clause list * statement_list option * case_clause list
@@ -315,7 +318,7 @@ and class_element =
 
 and class_element_name =
   | PropName of property_name
-  | PrivName of ident
+  | PrivName of identifier
 
 and ('a, 'b) list_with_rest =
   { list : 'a list
@@ -370,8 +373,8 @@ and export =
   | ExportClass of ident * class_declaration
   | ExportNames of (ident * Utf8_string.t) list
   (* default *)
-  | ExportDefaultFun of ident * function_declaration
-  | ExportDefaultClass of ident * class_declaration
+  | ExportDefaultFun of ident option * function_declaration
+  | ExportDefaultClass of ident option * class_declaration
   | ExportDefaultExpression of expression
   (* from *)
   | ExportFrom of