Refactor ziputils

docs/rgp-lua/finenv-properties.md

@@ -82,6 +82,18 @@ end
 
 ---
 
+#### EmbeddedLuaOSUtils\* (read-only property)
+
+If this property is true, it signifies that the `luaosutils` library is embedded in this version
+of _RGP Lua_ and can be successfully accessed with a `require` statement. You are also guaranteed
+that the minimum version of the embedded `luaosutils` is `2.2.0`.
+
+```lua
+local luaosutils = finenv.EmbeddedLuaOSUtils and require('luaosutils')
+```
+
+---
+
 #### EndUndoBlock\* (function)
 
 Ends the currently active Undo/Redo block in Finale (if any). Finale will only store Undo/Redo blocks that contain edit changes to the documents. These calls cannot be nested. If your script will make further changes to the document after this call, it should call `StartNewUndoBlock()` again before making them. Otherwise, Finale's Undo stack could become corrupted.

src/document_options_to_musescore.lua

@@ -4,8 +4,8 @@ function plugindef()
     finaleplugin.NoStore = true
     finaleplugin.Author = "Robert Patterson"
     finaleplugin.Copyright = "CC0 https://creativecommons.org/publicdomain/zero/1.0/"
-    finaleplugin.Version = "1.0.1"
-    finaleplugin.Date = "October 6, 2024"
+    finaleplugin.Version = "1.0.2"
+    finaleplugin.Date = "October 20, 2024"
     finaleplugin.CategoryTags = "Document"
     finaleplugin.MinJWLuaVersion = 0.75
     finaleplugin.Notes = [[
@@ -54,11 +54,10 @@ end
 
 -- luacheck: ignore 11./global_dialog
 
-local text = require("luaosutils").text
-
 local mixin = require("library.mixin")
 local enigma_string = require("library.enigma_string")
 local utils = require("library.utils")
+local client = require("library.client")
 
 do_folder = do_folder or false
 
@@ -68,7 +67,7 @@ local MUSX_EXTENSION <const> = ".musx"
 local MUS_EXTENSION <const> = ".mus"
 local TEXT_EXTENSION <const> = ".mss"
 local PART_EXTENSION <const> = ".part" .. TEXT_EXTENSION
-local MSS_VERSION <const> = "4.40"
+local MSS_VERSION <const> = "4.50"
 
 -- hard-coded scaling values
 local EVPU_PER_INCH <const> = 288
@@ -380,6 +379,7 @@ function write_note_related_prefs(style_element)
     set_element_text(style_element, "graceNoteMag", size_prefs.GraceNoteSize / 100)
     set_element_text(style_element, "concertPitch", part_scope_prefs.DisplayInConcertPitch)
     set_element_text(style_element, "multiVoiceRestTwoSpaceOffset", math.abs(layer_one_prefs.RestOffset) >= 4)
+    set_element_text(style_element, "mergeMatchingRests", misc_prefs.ConsolidateRestsAcrossLayers)
 end
 
 function write_smart_shape_prefs(style_element)
@@ -783,7 +783,7 @@ function document_options_to_musescore()
         end
         local selected_directory = select_directory()
         if selected_directory then
-            logfile_path = text.convert_encoding(selected_directory, text.get_utf8_codepage(), text.get_default_codepage()) .. LOGFILE_NAME
+            logfile_path = client.encode_with_client_codepage(selected_directory) .. LOGFILE_NAME
             local file <close> = io.open(logfile_path, "w")
             if not file then
                 error("unable to create logfile " .. logfile_path)

src/library/client.lua

@@ -164,20 +164,19 @@ end
 --[[
 % encode_with_client_codepage
 
-If the client supports LuaOSUtils, the filepath is encoded from UTF-8 to the current client
-encoding. On macOS, this is always also UTF-8, so the situation where the string may be re-encoded
-is only on Windows. (Recent versions of Windows also allow UTF-8 as the client encoding, so it may
+If the client supports `luaosutils`, the filepath is encoded from utf8 to the current client
+encoding. On macOS, this is always also utf8, so the situation where the string may be re-encoded
+is only on Windows. (Recent versions of Windows also allow utf8 as the client encoding, so it may
 not be re-encoded even on Windows.)
 
-If LuaOSUtils is not available, the string is returned unchanged.
+If `luaosutils` is not available, the string is returned unchanged.
 
 A primary use-case for this function is filepaths. Windows requires 8-bit filepaths to be encoded
 with the client codepage.
 
-@ input_string (string) the UTF-encoded string to re-encode
-: (string) the string re-encoded with the clieng codepage
+@ input_string (string) the utf8-encoded string to re-encode
+: (string) the string re-encoded with the client codepage
 ]]
-
 function client.encode_with_client_codepage(input_string)
     if client.supports("luaosutils") then
         local text = require("luaosutils").text
@@ -188,4 +187,56 @@ function client.encode_with_client_codepage(input_string)
     return input_string
 end
 
+--[[
+% encode_with_utf8_codepage
+
+If the client supports `luaosutils`, the filepath is encoded from the current client encoding
+to utf8. On macOS, the client encoding is always also utf8, so the situation where the string may
+be re-encoded is only on Windows. (Recent versions of Windows also allow utf8 as the client encoding, so it may
+not be re-encoded even on Windows.)
+
+If `luaosutils` is not available, the string is returned unchanged.
+
+A primary use-case for this function is filepaths. Windows requires 8-bit filepaths to be encoded
+with the client codepage.
+
+@ input_string (string) the client-encoded string to re-encode
+: (string) the string re-encoded with the utf8 codepage
+]]
+function client.encode_with_utf8_codepage(input_string)
+    if client.supports("luaosutils") then
+        local text = require("luaosutils").text
+        if text and text.get_default_codepage() ~= text.get_utf8_codepage() then
+            return text.convert_encoding(input_string, text.get_default_codepage(), text.get_utf8_codepage())
+        end
+    end
+    return input_string
+end
+
+--[[
+% execute
+
+If the client supports `luaosutils`, the command is executed using `luaosutils.execute`. Otherwise it uses `io.popen`.
+In either case, the output from the command is returned.
+
+Starting with v0.67, this function throws an error if the script is not trusted or has not set
+`finaleplugin.ExecuteExternalCode` to `true`.
+
+@ command (string) The command to execute encoded with **client encoding**.
+: (string) The `stdout` from the command, in whatever encoding it generated.
+]]
+function client.execute(command)
+    if client.supports("luaosutils") then
+        local process = require("luaosutils").process
+        if process then
+            return process.execute(command)
+        end
+    end
+    local handle = io.popen(command)
+    if not handle then return nil end
+    local retval = handle:read("*a")
+    handle:close()
+    return retval
+end
+
 return client

src/library/enigmaxml.lua

@@ -0,0 +1,106 @@
+--[[
+$module enigmaxml
+
+EnigmaXML is the underlying file format of a Finale `.musx` file. It is undocumented
+by MakeMusic and must be extracted from the `.musx` file. There is an effort to document
+it underway at the [EnigmaXML Documentation](https://github.com/Project-Attacca/enigmaxml-documentation)
+repository.
+]] --
+local enigmaxml = {}
+
+local utils = require("library.utils")
+local client = require("library.client")
+local ziputils = require("library.ziputils")
+
+-- symmetrical encryption/decryption function for EnigmaXML
+local function crypt_enigmaxml_buffer(buffer)
+    -- do not use <const> because this library must be loadable by Lua 5.2 (JW Lua)
+    local INITIAL_STATE = 0x28006D45 -- this value was determined empirically
+    local state = INITIAL_STATE
+    local result = {}
+
+    for i = 1, #buffer do
+        -- BSD rand()
+        if (i - 1) % 0x20000 == 0 then
+            state = INITIAL_STATE
+        end
+        state = (state * 0x41c64e6d + 0x3039) & 0xFFFFFFFF  -- Simulate 32-bit overflow
+        local upper = state >> 16
+        local c = upper + math.floor(upper / 255)
+
+        local byte = string.byte(buffer, i)
+        byte = byte ~ (c & 0xFF)  -- XOR operation on the byte
+
+        table.insert(result, string.char(byte))
+    end
+
+    return table.concat(result)
+end
+
+--[[
+% extract_enigmaxml
+
+This function extracts the EnigmaXML buffer from a `.musx` file. Note that it does not work with Finale's
+older `.mus` format.
+
+@ filepath (string) utf8-encoded file path to a `.musx` file.
+: (string) buffer of EnigmaXml data extracted from the `.musx`. (The xml declaration specifies the encoding, but expect it to be utf8.)
+]]
+function enigmaxml.extract_enigmaxml(filepath)
+    local not_supported_message
+    if finenv.TrustedMode == finenv.TrustedModeType.UNTRUSTED then
+        not_supported_message = "enigmaxml.extract_enigmaxml must run in Trusted mode."
+    elseif not finaleplugin.ExecuteExternalCode then
+        not_supported_message = "enigmaxml.extract_enigmaxml must have finaleplugin.ExecuteExternalCode set to true."
+    end
+    if not_supported_message then
+        error(not_supported_message, 2)
+    end
+    local _, _, extension = utils.split_file_path(filepath)
+    if extension ~= ".musx" then
+        error(filepath .. " is not a .musx file.", 2)
+    end
+
+    -- Steps to extract:
+    --      Unzip the `.musx` (which is a `.zip` archive in disguise)
+    --      Run the `score.dat` file through `crypt_enigmaxml_buffer` to get a gzip archive of the EnigmaXML file.
+    --      Gunzip the extracted EnigmaXML gzip archive into a string and return it.
+
+    local os_filepath = client.encode_with_client_codepage(filepath)
+    local output_dir, zipcommand = ziputils.calc_temp_output_path(os_filepath)
+    if not client.execute(zipcommand) then
+        error(zipcommand .. " failed")
+    end
+
+    -- do not use <close> because this library must be loadable by Lua 5.2 (JW Lua)
+    local file = io.open(output_dir .. "/score.dat", "rb")
+    if not file then
+        error("unable to read " .. output_dir .. "/score.dat")
+    end
+    local buffer = file:read("*all")
+    file:close()
+
+    local delcommand = ziputils.calc_rmdir_command(output_dir)
+    client.execute(delcommand)
+
+    buffer = crypt_enigmaxml_buffer(buffer)
+    if ziputils.calc_is_gzip(buffer) then
+        local gzip_path = ziputils.calc_temp_output_path()
+        local gzip_file = io.open(gzip_path, "wb")
+        if not gzip_file then
+            error("unable to create " .. gzip_file)
+        end
+        gzip_file:write(buffer)
+        gzip_file:close()
+        local gunzip_command = ziputils.calc_gunzip_command(gzip_path)
+        buffer = client.execute(gunzip_command)
+        client.execute(ziputils.calc_delete_file_command(gzip_path))
+        if not buffer or buffer == "" then
+            error(gunzip_command .. " failed")
+        end
+    end
+
+    return buffer
+end
+
+return enigmaxml

src/library/general_library.lua

@@ -330,7 +330,7 @@ the .json files for each font. The table is in the format:
 ]]
 
 function library.get_smufl_font_list()
-    local osutils = finenv.EmbeddedLuaOSUtils and require("luaosutils")
+    local osutils = client.supports("luaosutils") and require("luaosutils")
     local font_names = {}
     local add_to_table = function(for_user)
         local smufl_directory = calc_smufl_directory(for_user)
@@ -339,13 +339,9 @@ function library.get_smufl_font_list()
             if osutils then
                 return osutils.process.list_dir(smufl_directory, options)
             end
-            -- Starting in 0.67, io.popen may fail due to being untrusted.
+            -- Starting in 0.67, execute may fail due to being untrusted.
             local cmd = finenv.UI():IsOnWindows() and "dir " or "ls "
-            local handle = io.popen(cmd .. options .. " \"" .. smufl_directory .. "\"")
-            if not handle then return "" end
-            local retval = handle:read("*a")
-            handle:close()
-            return retval
+            return client.execute(cmd .. options .. " \"" .. smufl_directory .. "\"") or ""
         end
         local is_font_available = function(dir)
             local fc_dir = finale.FCString()

src/library/utils.lua

@@ -533,6 +533,7 @@ function utils.eachfile(directory_path, recursive)
     fcstr:AssureEndingPathDelimiter()
     directory_path = fcstr.LuaString
 
+    -- direcly call text.convert_encoding to avoid dependency on library.utils
     local lfs_directory_path = text.convert_encoding(directory_path, text.get_utf8_codepage(), text.get_default_codepage())
 
     return coroutine.wrap(function()