diff --git a/ruby/red-arrow-format/lib/arrow-format/array.rb b/ruby/red-arrow-format/lib/arrow-format/array.rb index bc79edf9eb0..bddcf7c9af8 100644 --- a/ruby/red-arrow-format/lib/arrow-format/array.rb +++ b/ruby/red-arrow-format/lib/arrow-format/array.rb @@ -537,7 +537,18 @@ class DurationArray < TemporalArray end class VariableSizeBinaryArray < Array - def initialize(size, validity_buffer, offsets_buffer, values_buffer) + include BufferAlignable + + def initialize(*args) + if args.size == 1 + args = build_data(args.first) + elsif args.size != 4 + raise ArgumentError, + "wrong number of arguments (given #{args.size}, expected 1 or 4)" + end + + size, validity_buffer, offsets_buffer, values_buffer = args + super(self.class.type, size, validity_buffer) @offsets_buffer = offsets_buffer @values_buffer = values_buffer @@ -577,6 +588,48 @@ def to_a end private + def build_data(data) + type = self.class.type + + n = 0 + validity_buffer_builder = nil + + values = +"".b + offsets = [0] + + data.each_with_index do |value, i| + if value.nil? + validity_buffer_builder ||= SparseBitmapBuilder.new + validity_buffer_builder.unset(i) + + offsets << values.bytesize + else + values.append_as_bytes(value) + offsets << values.bytesize + end + + n += 1 + end + + validity_buffer = validity_buffer_builder&.finish(n) + + offsets_data = offsets.pack("#{type.offset_pack_template}*") + pad!(offsets_data, buffer_padding_size(offsets_data)) + offsets_data.freeze + offsets_buffer = IO::Buffer.for(offsets_data) + + pad!(values, buffer_padding_size(values)) + values.freeze + values_buffer = IO::Buffer.for(values) + + [ + n, + validity_buffer, + offsets_buffer, + values_buffer, + ] + end + def offset_size IO::Buffer.size_of(@type.offset_buffer_type) end diff --git a/ruby/red-arrow-format/lib/arrow-format/type.rb b/ruby/red-arrow-format/lib/arrow-format/type.rb index 6711f995fdb..7205619f31f 100644 --- a/ruby/red-arrow-format/lib/arrow-format/type.rb +++ b/ruby/red-arrow-format/lib/arrow-format/type.rb @@ -704,6 +704,10 @@ def offset_buffer_type :s32 # TODO: big endian support end + def offset_pack_template + "l" + end + def encoding Encoding::ASCII_8BIT end @@ -732,6 +736,10 @@ def offset_buffer_type :s64 # TODO: big endian support end + def offset_pack_template + "q" + end + def encoding Encoding::ASCII_8BIT end @@ -760,6 +768,10 @@ def offset_buffer_type :s32 # TODO: big endian support end + def offset_pack_template + "l" + end + def encoding Encoding::UTF_8 end @@ -788,6 +800,10 @@ def offset_buffer_type :s64 # TODO: big endian support end + def offset_pack_template + "q" + end + def encoding Encoding::UTF_8 end diff --git a/ruby/red-arrow-format/test/test-binary-array.rb b/ruby/red-arrow-format/test/test-binary-array.rb new file mode 100644 index 00000000000..bcc122ccb04 --- /dev/null +++ b/ruby/red-arrow-format/test/test-binary-array.rb @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class TestBinaryArray < Test::Unit::TestCase + sub_test_case("#initialize") do + def test_no_null + values = ["\x00\x01".b, "\xff\x10".b] + assert_equal(values, + ArrowFormat::BinaryArray.new(values).to_a) + end + + def test_mixed + values = ["\x00\x01".b, nil, "\xff\x10".b] + assert_equal(values, + ArrowFormat::BinaryArray.new(values).to_a) + end + end + + sub_test_case("#==") do + def test_no_slice + values = ["\x00\x01".b, nil, "\xff\x10".b] + array1 = ArrowFormat::BinaryArray.new(values) + array2 = ArrowFormat::BinaryArray.new(values) + assert_equal(array1, array2) + end + + def test_sliced + values = ["\x00\x01".b, nil, "\xff\x10".b] + array1 = ArrowFormat::BinaryArray.new(values) + array2 = ArrowFormat::BinaryArray.new(["".b, *values, "".b]) + assert_equal(array1, array2.slice(1, 3)) + end + + def test_sliced_different_content + values = ["\x00\x01".b, nil, "\xff\x10".b] + array1 = ArrowFormat::BinaryArray.new(values) + array2 = ArrowFormat::BinaryArray.new(["".b, "".b, *values, "".b]) + assert_not_equal(array1, array2.slice(1, 3)) + end + end +end diff --git a/ruby/red-arrow-format/test/test-large-binary-array.rb b/ruby/red-arrow-format/test/test-large-binary-array.rb new file mode 100644 index 00000000000..67e55dd1b68 --- /dev/null +++ b/ruby/red-arrow-format/test/test-large-binary-array.rb @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class TestLargeBinaryArray < Test::Unit::TestCase + sub_test_case("#initialize") do + def test_no_null + values = ["\x00\x01".b, "\xff\x10".b] + assert_equal(values, + ArrowFormat::LargeBinaryArray.new(values).to_a) + end + + def test_mixed + values = ["\x00\x01".b, nil, "\xff\x10".b] + assert_equal(values, + ArrowFormat::LargeBinaryArray.new(values).to_a) + end + end + + sub_test_case("#==") do + def test_no_slice + values = ["\x00\x01".b, nil, "\xff\x10".b] + array1 = ArrowFormat::LargeBinaryArray.new(values) + array2 = ArrowFormat::LargeBinaryArray.new(values) + assert_equal(array1, array2) + end + + def test_sliced + values = ["\x00\x01".b, nil, "\xff\x10".b] + array1 = ArrowFormat::LargeBinaryArray.new(values) + array2 = ArrowFormat::LargeBinaryArray.new(["".b, *values, "".b]) + assert_equal(array1, array2.slice(1, 3)) + end + + def test_sliced_different_content + values = ["\x00\x01".b, nil, "\xff\x10".b] + array1 = ArrowFormat::LargeBinaryArray.new(values) + array2 = ArrowFormat::LargeBinaryArray.new(["".b, "".b, *values, "".b]) + assert_not_equal(array1, array2.slice(1, 3)) + end + end +end diff --git a/ruby/red-arrow-format/test/test-large-utf8-array.rb b/ruby/red-arrow-format/test/test-large-utf8-array.rb new file mode 100644 index 00000000000..1ee4a399451 --- /dev/null +++ b/ruby/red-arrow-format/test/test-large-utf8-array.rb @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class TestLargeUTF8Array < Test::Unit::TestCase + sub_test_case("#initialize") do + def test_no_null + values = ["Hello", "World"] + assert_equal(values, + ArrowFormat::LargeUTF8Array.new(values).to_a) + end + + def test_mixed + values = ["Hello", nil, "World"] + assert_equal(values, + ArrowFormat::LargeUTF8Array.new(values).to_a) + end + end + + sub_test_case("#==") do + def test_no_slice + values = ["Hello", nil, "World"] + array1 = ArrowFormat::LargeUTF8Array.new(values) + array2 = ArrowFormat::LargeUTF8Array.new(values) + assert_equal(array1, array2) + end + + def test_sliced + values = ["Hello", nil, "World"] + array1 = ArrowFormat::LargeUTF8Array.new(values) + array2 = ArrowFormat::LargeUTF8Array.new(["", *values, ""]) + assert_equal(array1, array2.slice(1, 3)) + end + + def test_sliced_different_content + values = ["Hello", nil, "World"] + array1 = ArrowFormat::LargeUTF8Array.new(values) + array2 = ArrowFormat::LargeUTF8Array.new(["", "", *values, ""]) + assert_not_equal(array1, array2.slice(1, 3)) + end + end +end diff --git a/ruby/red-arrow-format/test/test-utf8-array.rb b/ruby/red-arrow-format/test/test-utf8-array.rb new file mode 100644 index 00000000000..2970d77793e --- /dev/null +++ b/ruby/red-arrow-format/test/test-utf8-array.rb @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class TestUTF8Array < Test::Unit::TestCase + sub_test_case("#initialize") do + def test_no_null + values = ["Hello", "World"] + assert_equal(values, + ArrowFormat::UTF8Array.new(values).to_a) + end + + def test_mixed + values = ["Hello", nil, "World"] + assert_equal(values, + ArrowFormat::UTF8Array.new(values).to_a) + end + end + + sub_test_case("#==") do + def test_no_slice + values = ["Hello", nil, "World"] + array1 = ArrowFormat::UTF8Array.new(values) + array2 = ArrowFormat::UTF8Array.new(values) + assert_equal(array1, array2) + end + + def test_sliced + values = ["Hello", nil, "World"] + array1 = ArrowFormat::UTF8Array.new(values) + array2 = ArrowFormat::UTF8Array.new(["", *values, ""]) + assert_equal(array1, array2.slice(1, 3)) + end + + def test_sliced_different_content + values = ["Hello", nil, "World"] + array1 = ArrowFormat::UTF8Array.new(values) + array2 = ArrowFormat::UTF8Array.new(["", "", *values, ""]) + assert_not_equal(array1, array2.slice(1, 3)) + end + end +end