[MERGE chakra-core#5592 @rhuanjl] Implement RegExp dotAll flag (/s)

Merge pull request chakra-core#5592 from rhuanjl:dotall

This PR implements the es2018 RegExp /s dotAll flag.

See proposal spec here: https://tc39.github.io/proposal-regexp-dotall-flag/

Notes:
1. I've given this the flag ES9RegExDotAll but I've made it default to enabled
2. When run with the ES6RegExPrototypeProperties this passes all relevant test262 tests, without that flag several test262 tests fail (hopefully that can be enabled soon)

fixes: chakra-core#2787

cc: @mathiasbynens

Author: dilijev

lib/Common/ConfigFlagsList.h

@@ -640,6 +640,7 @@ PHASE(All)
 #define DEFAULT_CONFIG_ES6UnicodeVerbose       (true)
 #define DEFAULT_CONFIG_ES6Unscopables          (true)
 #define DEFAULT_CONFIG_ES6RegExSticky          (true)
+#define DEFAULT_CONFIG_ES2018RegExDotAll          (true)
 #ifdef COMPILE_DISABLE_ES6RegExPrototypeProperties
     // If ES6RegExPrototypeProperties needs to be disabled by compile flag, DEFAULT_CONFIG_ES6RegExPrototypeProperties should be false
     #define DEFAULT_CONFIG_ES6RegExPrototypeProperties (false)
@@ -1135,6 +1136,7 @@ FLAGPR           (Boolean, ES6, ES6Unicode             , "Enable ES6 Unicode 6.0
 FLAGPR           (Boolean, ES6, ES6UnicodeVerbose      , "Enable ES6 Unicode 6.0 verbose failure output"            , DEFAULT_CONFIG_ES6UnicodeVerbose)
 FLAGPR           (Boolean, ES6, ES6Unscopables         , "Enable ES6 With Statement Unscopables"                    , DEFAULT_CONFIG_ES6Unscopables)
 FLAGPR           (Boolean, ES6, ES6RegExSticky         , "Enable ES6 RegEx sticky flag"                             , DEFAULT_CONFIG_ES6RegExSticky)
+FLAGPR           (Boolean, ES6, ES2018RegExDotAll      , "Enable ES2018 RegEx dotAll flag"                          , DEFAULT_CONFIG_ES2018RegExDotAll)
 
 #ifndef COMPILE_DISABLE_ES6RegExPrototypeProperties
     #define COMPILE_DISABLE_ES6RegExPrototypeProperties 0

lib/Parser/RegexFlags.h

@@ -14,6 +14,7 @@ namespace UnifiedRegex
         MultilineRegexFlag  = 1 << 2,
         UnicodeRegexFlag    = 1 << 3,
         StickyRegexFlag     = 1 << 4,
-        AllRegexFlags       = (1 << 5) - 1
+        DotAllRegexFlag     = 1 << 5,
+        AllRegexFlags       = (1 << 6) - 1
     };
 }

lib/Parser/RegexParser.cpp

@@ -145,6 +145,7 @@ namespace UnifiedRegex
         , tempLocationOfRange(nullptr)
         , codePointAtTempLocation(0)
         , unicodeFlagPresent(false)
+        , dotAllFlagPresent(false)
         , caseInsensitiveFlagPresent(false)
         , positionAfterLastSurrogate(nullptr)
         , valueOfLastSurrogate(INVALID_CODEPOINT)
@@ -2758,6 +2759,16 @@ namespace UnifiedRegex
                 }
                 flags = (RegexFlags)(flags | MultilineRegexFlag);
                 break;
+            case 's':
+                if (scriptContext->GetConfig()->IsES2018RegExDotAllEnabled())
+                {
+                    if ((flags & DotAllRegexFlag) != 0)
+                    {
+                        Fail(JSERR_RegExpSyntax);
+                    }
+                    flags = (RegexFlags)(flags | DotAllRegexFlag);
+                    break;
+                }
             case 'u':
                 // If we don't have unicode enabled, fall through to default
                 if (scriptContext->GetConfig()->IsES6UnicodeExtensionsEnabled())
@@ -2832,12 +2843,15 @@ namespace UnifiedRegex
                 Fail(JSERR_RegExpSyntax);
             this->unicodeFlagPresent = (flags & UnifiedRegex::UnicodeRegexFlag) == UnifiedRegex::UnicodeRegexFlag;
             this->caseInsensitiveFlagPresent = (flags & UnifiedRegex::IgnoreCaseRegexFlag) == UnifiedRegex::IgnoreCaseRegexFlag;
+            this->dotAllFlagPresent = (flags & UnifiedRegex::DotAllRegexFlag) == UnifiedRegex::DotAllRegexFlag;
             Assert(!this->unicodeFlagPresent || scriptContext->GetConfig()->IsES6UnicodeExtensionsEnabled());
+            Assert(!this->dotAllFlagPresent || scriptContext->GetConfig()->IsES2018RegExDotAllEnabled());
         }
         else
         {
             this->unicodeFlagPresent = false;
             this->caseInsensitiveFlagPresent = false;
+            this->dotAllFlagPresent = false;
         }
 
         // If this HR has been set, that means we have an earlier failure than the one caught above.
@@ -2891,6 +2905,7 @@ namespace UnifiedRegex
         Options(flags);
         this->unicodeFlagPresent = (flags & UnifiedRegex::UnicodeRegexFlag) == UnifiedRegex::UnicodeRegexFlag;
         this->caseInsensitiveFlagPresent = (flags & UnifiedRegex::IgnoreCaseRegexFlag) == UnifiedRegex::IgnoreCaseRegexFlag;
+        this->dotAllFlagPresent = (flags & UnifiedRegex::DotAllRegexFlag) == UnifiedRegex::DotAllRegexFlag;
         Assert(!this->unicodeFlagPresent || scriptContext->GetConfig()->IsES6UnicodeExtensionsEnabled());
 
         // If this HR has been set, that means we have an earlier failure than the one caught above.
@@ -2946,6 +2961,7 @@ namespace UnifiedRegex
         Options(dummyFlags);
         this->unicodeFlagPresent = (dummyFlags & UnifiedRegex::UnicodeRegexFlag) == UnifiedRegex::UnicodeRegexFlag;
         this->caseInsensitiveFlagPresent = (dummyFlags & UnifiedRegex::IgnoreCaseRegexFlag) == UnifiedRegex::IgnoreCaseRegexFlag;
+        this->dotAllFlagPresent = (dummyFlags & UnifiedRegex::DotAllRegexFlag) == UnifiedRegex::DotAllRegexFlag;
         outTotalEncodedChars = Chars<EncodedChar>::OSB(next, input);
         outTotalChars = Pos();
 
@@ -3101,7 +3117,14 @@ namespace UnifiedRegex
             switch (cc)
             {
             case '.':
-                standardChars->SetNonNewline(ctAllocator, partialPrefixSetNode->set);
+                if (this->dotAllFlagPresent)
+                {
+                    standardChars->SetFullSet(ctAllocator, partialPrefixSetNode->set);
+                }
+                else
+                {
+                    standardChars->SetNonNewline(ctAllocator, partialPrefixSetNode->set);
+                }
                 break;
             case 'S':
                 standardChars->SetNonWhitespace(ctAllocator, partialPrefixSetNode->set);
@@ -3137,7 +3160,14 @@ namespace UnifiedRegex
             switch (cc)
             {
             case '.':
-                standardChars->SetNonNewline(ctAllocator, setNode->set);
+                if (this->dotAllFlagPresent)
+                {
+                    standardChars->SetFullSet(ctAllocator, setNode->set);
+                }
+                else
+                {
+                    standardChars->SetNonNewline(ctAllocator, setNode->set);
+                }
                 break;
             case 'S':
                 standardChars->SetNonWhitespace(ctAllocator, setNode->set);

lib/Parser/RegexParser.h

@@ -107,6 +107,7 @@ namespace UnifiedRegex
         SurrogatePairTracker* currentSurrogatePairNode;
         bool unicodeFlagPresent;
         bool caseInsensitiveFlagPresent;
+        bool dotAllFlagPresent;
 
         // The following two variables are used to determine if the the surrogate pair has been encountered
         // First holds the temporary location, second holds the value of the codepoint

lib/Parser/RegexPattern.cpp

@@ -93,6 +93,11 @@ namespace UnifiedRegex
         return (rep.unified.program->flags & IgnoreCaseRegexFlag) != 0;
     }
 
+    bool RegexPattern::IsDotAll() const
+    {
+        return GetScriptContext()->GetConfig()->IsES2018RegExDotAllEnabled() && (rep.unified.program->flags & DotAllRegexFlag) != 0;
+    }
+
     bool RegexPattern::IsGlobal() const
     {
         return (rep.unified.program->flags & GlobalRegexFlag) != 0;
@@ -195,6 +200,8 @@ namespace UnifiedRegex
             w->Print(_u("g"));
         if (IsMultiline())
             w->Print(_u("m"));
+        if (IsDotAll())
+            w->Print(_u("s"));
         if (IsUnicode())
             w->Print(_u("u"));
         if (IsSticky())

lib/Parser/RegexPattern.h

@@ -62,6 +62,7 @@ namespace UnifiedRegex
         bool IsIgnoreCase() const;
         bool IsGlobal() const;
         bool IsMultiline() const;
+        bool IsDotAll() const;
         bool IsUnicode() const;
         bool IsSticky() const;
         bool WasLastMatchSuccessful() const;

lib/Parser/RegexRuntime.cpp

@@ -5755,9 +5755,10 @@ namespace UnifiedRegex
         w->Print(_u("flags:        "));
         if ((flags & GlobalRegexFlag) != 0) w->Print(_u("global "));
         if ((flags & MultilineRegexFlag) != 0) w->Print(_u("multiline "));
-        if ((flags & IgnoreCaseRegexFlag) != 0) w->Print(_u("ignorecase"));
-        if ((flags & UnicodeRegexFlag) != 0) w->Print(_u("unicode"));
-        if ((flags & StickyRegexFlag) != 0) w->Print(_u("sticky"));
+        if ((flags & IgnoreCaseRegexFlag) != 0) w->Print(_u("ignorecase "));
+        if ((flags & DotAllRegexFlag) != 0) w->Print(_u("dotAll "));
+        if ((flags & UnicodeRegexFlag) != 0) w->Print(_u("unicode "));
+        if ((flags & StickyRegexFlag) != 0) w->Print(_u("sticky "));
         w->EOL();
         w->PrintEOL(_u("numGroups:    %d"), numGroups);
         w->PrintEOL(_u("numLoops:     %d"), numLoops);

lib/Parser/StandardChars.cpp

@@ -261,6 +261,11 @@ END {
         set.SetNotRanges(setAllocator, numNewlinePairs, newlineStr);
     }
 
+    void StandardChars<char16>::SetFullSet(ArenaAllocator* setAllocator, CharSet<Char> &set)
+    {
+        set.SetNotRanges(allocator, 0, nullptr);
+    }
+
     CharSet<char16>* StandardChars<char16>::GetFullSet()
     {
         if (fullSet == 0)

lib/Parser/StandardChars.h

@@ -323,6 +323,7 @@ namespace UnifiedRegex
         void SetNonWordIUChars(ArenaAllocator* setAllocator, CharSet<Char> &set);
         void SetNewline(ArenaAllocator* setAllocator, CharSet<Char> &set);
         void SetNonNewline(ArenaAllocator* setAllocator, CharSet<Char> &set);
+        void SetFullSet(ArenaAllocator* setAllocator, CharSet<Char> &set);
 
         CharSet<Char>* GetFullSet();
         CharSet<Char>* GetEmptySet();

lib/Runtime/Base/JnDirectFields.h

@@ -422,6 +422,7 @@ ENTRY(compile)
 ENTRY(global)
 ENTRY(lastIndex)
 ENTRY(multiline)
+ENTRY(dotAll)
 ENTRY(ignoreCase)
 ENTRY(unicode)
 ENTRY(sticky)