New Bytecode regen process:

1. Use python on windows and macOS/Linux
2. Suport multiple files of BuiltIns
3. Move file structure from ch.cpp into python
4. Delete defunct .cmd and .ps1 scripts
5. Delete defunct tests
6. Add testing via new python script and wire into CI
7. Add a noJit build on macOS to ensure noJit bytecode is tested

Author: rhuanjl

.gitattributes

@@ -3,6 +3,7 @@
 test/**/*.js -crlf
 test/es6/HTMLComments.js binary diff=cpp
 *.wasm binary
+lib/**/*.js eol=lf diff=cpp
 *.cpp text eol=lf diff=cpp
 *.h text eol=lf diff=cpp
 *.inl text eol=lf diff=cpp

CMakeLists.txt

@@ -466,6 +466,9 @@ if(NO_JIT_SH)
     unset(NO_JIT_SH CACHE)      # don't cache
     unset(BuildJIT CACHE)       # also clear it just in case
     add_definitions(-DDISABLE_JIT=1)
+elseif(DISABLE_JIT)
+    set(BuildJIT 0)
+    add_definitions(-DDISABLE_JIT=1)
 else()
     set(BuildJIT 1)
 endif()

RegenAllByteCode.cmd

@@ -50,6 +50,12 @@ jobs:
           build_type: 'Release'
           do_test: false
           libtype_flag: ''
+        OSX.noJit:
+          image_name: 'macOS-latest'
+          deps: 'brew install ninja icu4c'
+          build_type: 'Debug'
+          do_test: true
+          libtype_flag: '-DSTATIC_LIBRARY=ON -DDISABLE_JIT=1'
         OSX.Debug:
           image_name: 'macOS-latest'
           deps: 'brew install ninja icu4c'

RegenAllByteCodeNoBuild.cmd

@@ -196,97 +196,62 @@ HANDLE GetFileHandle(LPCWSTR filename)
     return GetStdHandle(STD_OUTPUT_HANDLE);
 }
 
-HRESULT CreateLibraryByteCodeHeader(LPCSTR contentsRaw, JsFinalizeCallback contentsRawFinalizeCallback, DWORD lengthBytes, LPCWSTR bcFullPath, LPCSTR libraryNameNarrow)
+HRESULT CreateLibraryByteCode(const char* contentsRaw)
 {
-    HANDLE bcFileHandle = nullptr;
     JsValueRef bufferVal;
     BYTE *bcBuffer = nullptr;
     unsigned int bcBufferSize = 0;
-    DWORD written;
-    // For validating the header file against the library file
-    auto outputStr =
-        "//-------------------------------------------------------------------------------------------------------\n"
-        "// Copyright (C) Microsoft. All rights reserved.\n"
-        "// Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.\n"
-        "//-------------------------------------------------------------------------------------------------------\n"
-        "#if 0\n";
-
-    std::string normalizedContentStr;
-    char* nextToken = nullptr;
-    char* token = strtok_s((char*)contentsRaw, "\r", &nextToken);
-    while (token)
-    {
-        normalizedContentStr.append(token);
-        token = strtok_s(nullptr, "\r", &nextToken);
-    }
-    // We no longer need contentsRaw, so call the finalizer for it if one was provided
-    if (contentsRawFinalizeCallback != nullptr)
-    {
-        contentsRawFinalizeCallback((void*)contentsRaw);
-    }
-
-    const char* normalizedContent = normalizedContentStr.c_str();
-    // We still need contentsRaw after this, so pass a null finalizeCallback into it
-    HRESULT hr = GetSerializedBuffer(normalizedContent, nullptr, &bufferVal);
-
-    IfFailedGoLabel((hr), ErrorRunFinalize);
-
-    IfJsrtErrorHRLabel(ChakraRTInterface::JsGetArrayBufferStorage(bufferVal, &bcBuffer, &bcBufferSize), ErrorRunFinalize);
-
-    bcFileHandle = GetFileHandle(bcFullPath);
-    IfFalseGo(bcFileHandle != INVALID_HANDLE_VALUE && bcFileHandle != nullptr);
-
-    IfFalseGoLabel(WriteFile(bcFileHandle, outputStr, (DWORD)strlen(outputStr), &written, nullptr), ErrorRunFinalize);
-    IfFalseGoLabel(WriteFile(bcFileHandle, normalizedContent, (DWORD)normalizedContentStr.size(), &written, nullptr), ErrorRunFinalize);
-    outputStr = "\n#endif\n";
+    HRESULT hr = E_FAIL;
+    
+    // Windows can't do the below with printf - so use windows API on windows but printf on posix
+    #ifdef _WIN32
+        HANDLE out = GetStdHandle(STD_OUTPUT_HANDLE);
+        DWORD written = 0;
+        #define print_format(format, element, size) \
+        { \
+            auto scratchLen = size; \
+            char scratch[size]; \
+            int len = _snprintf_s(scratch, scratchLen, _countof(scratch), format, element); \
+            IfFalseGo(WriteFile(out, scratch, (DWORD)(len), &written, nullptr)); \
+        }
+        #define print(text) \
+            WriteFile(out, text, (DWORD)strlen(text), &written, nullptr);
+    #else
+        #define print_format(format, element, size) printf(format, element)
+        #define print printf
+    #endif
 
-    IfFalseGo(WriteFile(bcFileHandle, outputStr, (DWORD)strlen(outputStr), &written, nullptr));
+    // Generate the bytecode, free the original buffer then retrieve the generated bytecode
+    IfFailGo(GetSerializedBuffer(contentsRaw, WScriptJsrt::FinalizeFree, &bufferVal));
+    IfFailGo(ChakraRTInterface::JsGetArrayBufferStorage(bufferVal, &bcBuffer, &bcBufferSize));
 
     // Write out the bytecode
-    outputStr = "namespace Js\n{\n    const char Library_Bytecode_";
-    IfFalseGo(WriteFile(bcFileHandle, outputStr, (DWORD)strlen(outputStr), &written, nullptr));
-    IfFalseGo(WriteFile(bcFileHandle, libraryNameNarrow, (DWORD)strlen(libraryNameNarrow), &written, nullptr));
-    outputStr = "[] = {\n/* 00000000 */";
-    IfFalseGo(WriteFile(bcFileHandle, outputStr, (DWORD)strlen(outputStr), &written, nullptr));
+    print("[] = {\n/* 00000000 */");
 
     for (unsigned int i = 0; i < bcBufferSize; i++)
     {
-        char scratch[6];
-        auto scratchLen = sizeof(scratch);
-        int num = _snprintf_s(scratch, scratchLen, _countof(scratch), " 0x%02X", bcBuffer[i]);
-        Assert(num == 5);
-        IfFalseGo(WriteFile(bcFileHandle, scratch, (DWORD)(scratchLen - 1), &written, nullptr));
-
-        // Add a comma and a space if this is not the last item
+        print_format(" 0x%02X", bcBuffer[i], 6);
+        // Add a comma if this is not the last item
         if (i < bcBufferSize - 1)
         {
-            char commaSpace[2];
-            _snprintf_s(commaSpace, sizeof(commaSpace), _countof(commaSpace), ",");  // close quote, new line, offset and open quote
-            IfFalseGo(WriteFile(bcFileHandle, commaSpace, (DWORD)strlen(commaSpace), &written, nullptr));
+            print(",");
         }
 
         // Add a line break every 16 scratches, primarily so the compiler doesn't complain about the string being too long.
         // Also, won't add for the last scratch
         if (i % 16 == 15 && i < bcBufferSize - 1)
         {
-            char offset[17];
-            int actualLen = _snprintf_s(offset, sizeof(offset), _countof(offset), "\n/* %08X */", i + 1);  // close quote, new line, offset and open quote
-            IfFalseGo(WriteFile(bcFileHandle, offset, actualLen, &written, nullptr));
+            print_format("\n/* %08X */", i + 1, 17);
         }
     }
-    outputStr = "};\n\n";
-    IfFalseGo(WriteFile(bcFileHandle, outputStr, (DWORD)strlen(outputStr), &written, nullptr));
+    print("};\n\n");
 
-    outputStr = "}\n";
-    IfFalseGo(WriteFile(bcFileHandle, outputStr, (DWORD)strlen(outputStr), &written, nullptr));
+    #undef print
+    #undef print_format
 
-ErrorRunFinalize:
-Error:
-    if (bcFileHandle != nullptr)
-    {
-        CloseHandle(bcFileHandle);
-    }
+    hr = S_OK;
 
+Error:
     return hr;
 }
 
@@ -933,19 +898,7 @@ HRESULT ExecuteTest(const char* fileName)
         len = strlen(fullPath);
         if (HostConfigFlags::flags.GenerateLibraryByteCodeHeaderIsEnabled)
         {
-
-            if (HostConfigFlags::flags.GenerateLibraryByteCodeHeader != nullptr)
-            {
-                if (wcslen(HostConfigFlags::flags.GenerateLibraryByteCodeHeader) == 0)
-                {
-                    HostConfigFlags::flags.GenerateLibraryByteCodeHeader = nullptr;
-                }
-            }
-            CHAR libraryName[_MAX_PATH];
-            CHAR ext[_MAX_EXT];
-            _splitpath_s(fullPath, NULL, 0, NULL, 0, libraryName, _countof(libraryName), ext, _countof(ext));
-
-            IfFailGo(CreateLibraryByteCodeHeader(fileContents, WScriptJsrt::FinalizeFree, lengthBytes, HostConfigFlags::flags.GenerateLibraryByteCodeHeader, libraryName));
+            IfFailGo(CreateLibraryByteCode(fileContents));
         }
         else if (HostConfigFlags::flags.SerializedIsEnabled)
         {