diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c4e00f1c..0d5df924 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,7 +28,7 @@ jobs: - uses: erlef/setup-beam@v1 with: otp-version: ${{ matrix.erlang_version }} - gleam-version: "1.6.0" + gleam-version: "1.9.0" - run: gleam test --target erlang - run: gleam format --check src test @@ -44,7 +44,7 @@ jobs: - uses: erlef/setup-beam@v1 with: otp-version: "27.0" - gleam-version: "1.6.0" + gleam-version: "1.9.0" - uses: actions/setup-node@v4 with: node-version: ${{ matrix.node_version }} @@ -62,7 +62,7 @@ jobs: - uses: erlef/setup-beam@v1 with: otp-version: "27.0" - gleam-version: "1.6.0" + gleam-version: "1.9.0" - uses: oven-sh/setup-bun@v2 with: bun-version: ${{ matrix.bun_version }} @@ -80,7 +80,7 @@ jobs: - uses: erlef/setup-beam@v1 with: otp-version: "27.0" - gleam-version: "1.6.0" + gleam-version: "1.9.0" - uses: denoland/setup-deno@v1 with: deno-version: ${{ matrix.deno_version }} diff --git a/CHANGELOG.md b/CHANGELOG.md index f2c48b79..0ba3ef4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## Unreleased + +- The minimum supported Gleam version has been increased to 1.9.0. +- The functions in the `bit_array` module now support unaligned bit arrays on + the JavaScript target. + ## v0.56.0 - 2025-03-09 - The decode API can now index into the first 8 elements of lists. @@ -8,9 +14,6 @@ - The performance of `dict.is_empty` has been improved. - The `flip` function in the `function` module has been deprecated. - -## v0.54.0 - 2025-02-04 - - The `uri` module gains the `empty` value, representing an empty URI which equivalent to `""`. diff --git a/gleam.toml b/gleam.toml index 4763985d..58caf857 100644 --- a/gleam.toml +++ b/gleam.toml @@ -1,6 +1,6 @@ name = "gleam_stdlib" version = "0.56.0" -gleam = ">= 0.32.0" +gleam = ">= 1.9.0" licences = ["Apache-2.0"] description = "A standard library for the Gleam programming language" @@ -11,6 +11,4 @@ links = [ ] [javascript.deno] -allow_read = [ - "./", -] +allow_read = ["./"] diff --git a/src/gleam/bit_array.gleam b/src/gleam/bit_array.gleam index df75be59..8dbefb50 100644 --- a/src/gleam/bit_array.gleam +++ b/src/gleam/bit_array.gleam @@ -13,22 +13,20 @@ pub fn from_string(x: String) -> BitArray /// Returns an integer which is the number of bits in the bit array. /// @external(erlang, "erlang", "bit_size") -pub fn bit_size(x: BitArray) -> Int { - byte_size(x) * 8 -} +@external(javascript, "../gleam_stdlib.mjs", "bit_array_bit_size") +pub fn bit_size(x: BitArray) -> Int /// Returns an integer which is the number of bytes in the bit array. /// @external(erlang, "erlang", "byte_size") -@external(javascript, "../gleam_stdlib.mjs", "length") +@external(javascript, "../gleam_stdlib.mjs", "bit_array_byte_size") pub fn byte_size(x: BitArray) -> Int /// Pads a bit array with zeros so that it is a whole number of bytes. /// @external(erlang, "gleam_stdlib", "bit_array_pad_to_bytes") -pub fn pad_to_bytes(x: BitArray) -> BitArray { - x -} +@external(javascript, "../gleam_stdlib.mjs", "bit_array_pad_to_bytes") +pub fn pad_to_bytes(x: BitArray) -> BitArray /// Creates a new bit array by joining two bit arrays. /// @@ -228,7 +226,6 @@ fn inspect_loop(input: BitArray, accumulator: String) -> String { /// // -> Eq /// ``` /// -@external(javascript, "../gleam_stdlib.mjs", "bit_array_compare") pub fn compare(a: BitArray, with b: BitArray) -> order.Order { case a, b { <>, <> -> @@ -257,6 +254,7 @@ pub fn compare(a: BitArray, with b: BitArray) -> order.Order { } @external(erlang, "gleam_stdlib", "bit_array_to_int_and_size") +@external(javascript, "../gleam_stdlib.mjs", "bit_array_to_int_and_size") fn bit_array_to_int_and_size(a: BitArray) -> #(Int, Int) /// Checks whether the first `BitArray` starts with the second one. diff --git a/src/gleam_stdlib.mjs b/src/gleam_stdlib.mjs index 700a3620..c96b48ed 100644 --- a/src/gleam_stdlib.mjs +++ b/src/gleam_stdlib.mjs @@ -7,6 +7,7 @@ import { UtfCodepoint, stringBits, toBitArray, + bitArraySlice, NonEmpty, CustomType, } from "./gleam.mjs"; @@ -316,8 +317,50 @@ export function bit_array_from_string(string) { return toBitArray([stringBits(string)]); } +export function bit_array_bit_size(bit_array) { + return bit_array.bitSize; +} + +export function bit_array_byte_size(bit_array) { + return bit_array.byteSize; +} + +export function bit_array_pad_to_bytes(bit_array) { + const trailingBitsCount = bit_array.bitSize % 8; + + // If the bit array is a whole number of bytes it can be returned unchanged + if (trailingBitsCount === 0) { + return bit_array; + } + + const finalByte = bit_array.byteAt(bit_array.byteSize - 1); + + // The required final byte has its unused trailing bits set to zero + const unusedBitsCount = 8 - trailingBitsCount; + const correctFinalByte = (finalByte >> unusedBitsCount) << unusedBitsCount; + + // If the unused bits in the final byte are already set to zero then the + // existing buffer can be re-used, avoiding a copy + if (finalByte === correctFinalByte) { + return new BitArray( + bit_array.rawBuffer, + bit_array.byteSize * 8, + bit_array.bitOffset, + ); + } + + // Copy the bit array into a new aligned buffer and set the correct final byte + const buffer = new Uint8Array(bit_array.byteSize); + for (let i = 0; i < buffer.length - 1; i++) { + buffer[i] = bit_array.byteAt(i); + } + buffer[buffer.length - 1] = correctFinalByte; + + return new BitArray(buffer); +} + export function bit_array_concat(bit_arrays) { - return toBitArray(bit_arrays.toArray().map((b) => b.buffer)); + return toBitArray(bit_arrays.toArray()); } export function console_log(term) { @@ -333,9 +376,25 @@ export function crash(message) { } export function bit_array_to_string(bit_array) { + // If the bit array isn't a whole number of bytes then return an error + if (bit_array.bitSize % 8 !== 0) { + return new Error(Nil); + } + try { const decoder = new TextDecoder("utf-8", { fatal: true }); - return new Ok(decoder.decode(bit_array.buffer)); + + if (bit_array.bitOffset === 0) { + return new Ok(decoder.decode(bit_array.rawBuffer)); + } else { + // The input data isn't aligned, so copy it into a new aligned buffer so + // that TextDecoder can be used + const buffer = new Uint8Array(bit_array.byteSize); + for (let i = 0; i < buffer.length; i++) { + buffer[i] = bit_array.byteAt(i); + } + return new Ok(decoder.decode(buffer)); + } } catch { return new Error(Nil); } @@ -415,14 +474,12 @@ export function random_uniform() { export function bit_array_slice(bits, position, length) { const start = Math.min(position, position + length); const end = Math.max(position, position + length); - if (start < 0 || end > bits.length) return new Error(Nil); - const byteOffset = bits.buffer.byteOffset + start; - const buffer = new Uint8Array( - bits.buffer.buffer, - byteOffset, - Math.abs(length), - ); - return new Ok(new BitArray(buffer)); + + if (start < 0 || end * 8 > bits.bitSize) { + return new Error(Nil); + } + + return new Ok(bitArraySlice(bits, start * 8, end * 8)); } export function codepoint(int) { @@ -522,16 +579,20 @@ let b64TextDecoder; export function encode64(bit_array, padding) { b64TextDecoder ??= new TextDecoder(); - const bytes = bit_array.buffer; + bit_array = bit_array_pad_to_bytes(bit_array); - const m = bytes.length; + const m = bit_array.byteSize; const k = m % 3; const n = Math.floor(m / 3) * 4 + (k && k + 1); const N = Math.ceil(m / 3) * 4; const encoded = new Uint8Array(N); for (let i = 0, j = 0; j < m; i += 4, j += 3) { - const y = (bytes[j] << 16) + (bytes[j + 1] << 8) + (bytes[j + 2] | 0); + const y = + (bit_array.byteAt(j) << 16) + + (bit_array.byteAt(j + 1) << 8) + + (bit_array.byteAt(j + 2) | 0); + encoded[i] = b64EncodeLookup[y >> 18]; encoded[i + 1] = b64EncodeLookup[(y >> 12) & 0x3f]; encoded[i + 2] = b64EncodeLookup[(y >> 6) & 0x3f]; @@ -804,7 +865,7 @@ export function inspect(v) { if (Array.isArray(v)) return `#(${v.map(inspect).join(", ")})`; if (v instanceof List) return inspectList(v); if (v instanceof UtfCodepoint) return inspectUtfCodepoint(v); - if (v instanceof BitArray) return inspectBitArray(v); + if (v instanceof BitArray) return `<<${bit_array_inspect(v, "")}>>`; if (v instanceof CustomType) return inspectCustomType(v); if (v instanceof Dict) return inspectDict(v); if (v instanceof Set) return `//js(Set(${[...v].map(inspect).join(", ")}))`; @@ -895,19 +956,26 @@ export function inspectList(list) { return `[${list.toArray().map(inspect).join(", ")}]`; } -export function inspectBitArray(bits) { - return `<<${Array.from(bits.buffer).join(", ")}>>`; -} - export function inspectUtfCodepoint(codepoint) { return `//utfcodepoint(${String.fromCodePoint(codepoint.value)})`; } export function base16_encode(bit_array) { + const trailingBitsCount = bit_array.bitSize % 8; + let result = ""; - for (const byte of bit_array.buffer) { + + for (let i = 0; i < bit_array.byteSize; i++) { + let byte = bit_array.byteAt(i); + + if (i === bit_array.byteSize - 1 && trailingBitsCount !== 0) { + const unusedBitsCount = 8 - trailingBitsCount; + byte = (byte >> unusedBitsCount) << unusedBitsCount; + } + result += byte.toString(16).padStart(2, "0").toUpperCase(); } + return result; } @@ -923,38 +991,53 @@ export function base16_decode(string) { } export function bit_array_inspect(bits, acc) { - return `${acc}${[...bits.buffer].join(", ")}`; -} + if (bits.bitSize === 0) { + return acc; + } -export function bit_array_compare(first, second) { - for (let i = 0; i < first.length; i++) { - if (i >= second.length) { - return new Gt(); // first has more items - } - const f = first.buffer[i]; - const s = second.buffer[i]; - if (f > s) { - return new Gt(); - } - if (f < s) { - return new Lt(); - } + for (let i = 0; i < bits.byteSize - 1; i++) { + acc += bits.byteAt(i).toString(); + acc += ", "; } - // This means that either first did not have any items - // or all items in first were equal to second. - if (first.length === second.length) { - return new Eq(); + + if (bits.byteSize * 8 === bits.bitSize) { + acc += bits.byteAt(bits.byteSize - 1).toString(); + } else { + const trailingBitsCount = bits.bitSize % 8; + acc += bits.byteAt(bits.byteSize - 1) >> (8 - trailingBitsCount); + acc += `:size(${trailingBitsCount})`; } - return new Lt(); // second has more items + + return acc; +} + +export function bit_array_to_int_and_size(bits) { + const trailingBitsCount = bits.bitSize % 8; + const unusedBitsCount = trailingBitsCount === 0 ? 0 : 8 - trailingBitsCount; + + return [bits.byteAt(0) >> unusedBitsCount, bits.bitSize]; } export function bit_array_starts_with(bits, prefix) { - if (prefix.length > bits.length) { + if (prefix.bitSize > bits.bitSize) { return false; } - for (let i = 0; i < prefix.length; i++) { - if (bits.buffer[i] !== prefix.buffer[i]) { + // Check any whole bytes + const byteCount = Math.trunc(prefix.bitSize / 8); + for (let i = 0; i < byteCount; i++) { + if (bits.byteAt(i) !== prefix.byteAt(i)) { + return false; + } + } + + // Check any trailing bits at the end of the prefix + if (prefix.bitSize % 8 !== 0) { + const unusedBitsCount = 8 - (prefix.bitSize % 8); + if ( + bits.byteAt(byteCount) >> unusedBitsCount !== + prefix.byteAt(byteCount) >> unusedBitsCount + ) { return false; } } diff --git a/test/gleam/bit_array_test.gleam b/test/gleam/bit_array_test.gleam index 638a8b2d..1d0c6224 100644 --- a/test/gleam/bit_array_test.gleam +++ b/test/gleam/bit_array_test.gleam @@ -1,5 +1,4 @@ import gleam/bit_array -@target(erlang) import gleam/order import gleam/result import gleam/should @@ -17,12 +16,7 @@ pub fn bit_size_test() { bit_array.bit_size(<<0:-8>>) |> should.equal(0) -} -// This test is target specific since it's using non byte-aligned BitArrays -// and those are not supported on the JavaScript target. -@target(erlang) -pub fn bit_size_erlang_only_test() { bit_array.bit_size(<<0:1>>) |> should.equal(1) @@ -42,12 +36,7 @@ pub fn byte_size_test() { bit_array.byte_size(<<0, 1, 2, 3, 4>>) |> should.equal(5) -} -// This test is target specific since it's using non byte-aligned BitArrays -// and those are not supported on the JavaScript target. -@target(erlang) -pub fn byte_size_erlang_only_test() { bit_array.byte_size(<<1, 2, 3:6>>) |> should.equal(3) } @@ -64,12 +53,7 @@ pub fn pad_to_bytes_test() { <<0xAB, 0x12>> |> bit_array.pad_to_bytes |> should.equal(<<0xAB, 0x12>>) -} -// This test is target specific since it's using non byte-aligned BitArrays -// and those are not supported on the JavaScript target. -@target(erlang) -pub fn pad_to_bytes_erlang_only_test() { <<1:1>> |> bit_array.pad_to_bytes |> should.equal(<<0x80>>) @@ -81,6 +65,11 @@ pub fn pad_to_bytes_erlang_only_test() { <<0xAB, 0x12, 3:3>> |> bit_array.pad_to_bytes |> should.equal(<<0xAB, 0x12, 0x60>>) + + let assert <> = <<0xAB, 0xFF>> + a + |> bit_array.pad_to_bytes + |> should.equal(<<0xAB, 0xF0>>) } pub fn not_equal_test() { @@ -100,12 +89,7 @@ pub fn append_test() { <<1, 2>> |> bit_array.append(<<3, 4>>) |> should.equal(<<1, 2, 3, 4>>) -} -// This test is target specific since it's using non byte-aligned BitArrays -// and those are not supported on the JavaScript target. -@target(erlang) -pub fn append_erlang_only_test() { <<1, 2:4>> |> bit_array.append(<<3>>) |> should.equal(<<1, 2:4, 3>>) @@ -119,12 +103,7 @@ pub fn concat_test() { [<<1, 2>>, <<3>>, <<4>>] |> bit_array.concat |> should.equal(<<1, 2, 3, 4>>) -} -// This test is target specific since it's using non byte-aligned BitArrays -// and those are not supported on the JavaScript target. -@target(erlang) -pub fn concat_erlang_only_test() { [<<-1:32>>, <<0:1>>, <<0:0>>] |> bit_array.concat |> should.equal(<<255, 255, 255, 255, 0:1>>) @@ -187,12 +166,7 @@ pub fn slice_test() { |> bit_array.slice(1, 1) |> result.try(bit_array.slice(_, 0, 1)) |> should.equal(Ok(<<"b":utf8>>)) -} -// This test is target specific since it's using non byte-aligned BitArrays -// and those are not supported on the JavaScript target. -@target(erlang) -pub fn slice_erlang_only_test() { <<0, 1, 2:7>> |> bit_array.slice(0, 3) |> should.equal(Error(Nil)) @@ -222,15 +196,15 @@ pub fn to_string_test() { <<65_535>> |> bit_array.to_string |> should.equal(Error(Nil)) -} -// This test is target specific since it's using non byte-aligned BitArrays -// and those are not supported on the JavaScript target. -@target(erlang) -pub fn to_string_erlang_only_test() { <<"ø":utf8, 50:4>> |> bit_array.to_string |> should.equal(Error(Nil)) + + let assert <<_:3, x:bits>> = <<0:3, "ø":utf8>> + x + |> bit_array.to_string + |> should.equal(Ok("ø")) } pub fn is_utf8_test() { @@ -283,12 +257,7 @@ pub fn base64_encode_test() { "QUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFB", 1024 * 32, )) -} -// This test is target specific since it's using non byte-aligned BitArrays -// and those are not supported on the JavaScript target. -@target(erlang) -pub fn base64_erlang_only_encode_test() { <<-1:7>> |> bit_array.base64_encode(True) |> should.equal("/g==") @@ -374,7 +343,7 @@ pub fn decode64_crash_regression_1_test() { |> should.equal(Error(Nil)) } -pub fn base16_test() { +pub fn base16_encode_test() { bit_array.base16_encode(<<"":utf8>>) |> should.equal("") @@ -398,12 +367,7 @@ pub fn base16_test() { bit_array.base16_encode(<<161, 178, 195, 212, 229, 246, 120, 145>>) |> should.equal("A1B2C3D4E5F67891") -} -// This test is target specific since it's using non byte-aligned BitArrays -// and those are not supported on the JavaScript target. -@target(erlang) -pub fn base16_encode_erlang_only_test() { <<-1:7>> |> bit_array.base16_encode() |> should.equal("FE") @@ -464,12 +428,7 @@ pub fn inspect_test() { bit_array.inspect(<<0, 20, 0x20, 255>>) |> should.equal("<<0, 20, 32, 255>>") -} -// This test is target specific since it's using non byte-aligned BitArrays -// and those are not supported on the JavaScript target. -@target(erlang) -pub fn inspect_erlang_only_test() { bit_array.inspect(<<4:5>>) |> should.equal("<<4:size(5)>>") @@ -480,7 +439,6 @@ pub fn inspect_erlang_only_test() { |> should.equal("<<182, 1:size(1)>>") } -@target(erlang) pub fn compare_test() { bit_array.compare(<<4:5>>, <<4:5>>) |> should.equal(order.Eq) @@ -551,12 +509,7 @@ pub fn starts_with_test() { bit_array.starts_with(<<0, 1, 2>>, <<1>>) |> should.be_false -} -// This test is target specific since it's using non byte-aligned BitArrays -// and those are not supported on the JavaScript target. -@target(erlang) -pub fn starts_with_erlang_only_test() { bit_array.starts_with(<<1:1>>, <<1:1>>) |> should.be_true diff --git a/test/gleam/bytes_tree_test.gleam b/test/gleam/bytes_tree_test.gleam index 3f549f15..e7dd0f07 100644 --- a/test/gleam/bytes_tree_test.gleam +++ b/test/gleam/bytes_tree_test.gleam @@ -18,7 +18,6 @@ pub fn tree_test() { |> should.equal(4) } -@target(erlang) pub fn tree_unaligned_bit_arrays_test() { let data = bytes_tree.from_bit_array(<<-1:5>>) @@ -84,7 +83,6 @@ pub fn concat_bit_arrays_test() { |> should.equal(<<"hey":utf8>>) } -@target(erlang) pub fn concat_unaligned_bit_arrays_test() { bytes_tree.concat_bit_arrays([<<-1:4>>, <<-1:5>>, <<-1:3>>, <<-2:2>>]) |> bytes_tree.to_bit_array diff --git a/test/gleam/dict_test.gleam b/test/gleam/dict_test.gleam index d9b8822d..b2789c2a 100644 --- a/test/gleam/dict_test.gleam +++ b/test/gleam/dict_test.gleam @@ -256,11 +256,9 @@ fn list_to_map(list) { fn grow_and_shrink_map(initial_size, final_size) { range(0, initial_size, []) |> list_to_map - |> list.fold( - range(final_size, initial_size, []), - _, - fn(map, item) { dict.delete(map, item) }, - ) + |> list.fold(range(final_size, initial_size, []), _, fn(map, item) { + dict.delete(map, item) + }) } // maps should be equal even if the insert/removal order was different diff --git a/test/gleam/uri_test.gleam b/test/gleam/uri_test.gleam index 1e997188..b24c28e3 100644 --- a/test/gleam/uri_test.gleam +++ b/test/gleam/uri_test.gleam @@ -342,11 +342,29 @@ pub fn empty_query_to_string_test() { } const percent_codec_fixtures = [ - #(" ", "%20"), #(",", "%2C"), #(";", "%3B"), #(":", "%3A"), #("!", "!"), - #("?", "%3F"), #("'", "'"), #("(", "("), #(")", ")"), #("[", "%5B"), - #("@", "%40"), #("/", "%2F"), #("\\", "%5C"), #("&", "%26"), #("#", "%23"), - #("=", "%3D"), #("~", "~"), #("ñ", "%C3%B1"), #("-", "-"), #("_", "_"), - #(".", "."), #("*", "*"), #("+", "+"), + #(" ", "%20"), + #(",", "%2C"), + #(";", "%3B"), + #(":", "%3A"), + #("!", "!"), + #("?", "%3F"), + #("'", "'"), + #("(", "("), + #(")", ")"), + #("[", "%5B"), + #("@", "%40"), + #("/", "%2F"), + #("\\", "%5C"), + #("&", "%26"), + #("#", "%23"), + #("=", "%3D"), + #("~", "~"), + #("ñ", "%C3%B1"), + #("-", "-"), + #("_", "_"), + #(".", "."), + #("*", "*"), + #("+", "+"), #("100% great+fun", "100%25%20great+fun"), ]