Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion python/pyarrow/src/arrow/python/python_to_arrow.cc
Original file line number Diff line number Diff line change
Expand Up @@ -908,8 +908,20 @@ class PyListConverter : public ListConverter<T, PyConverter, PyConverterTrait> {

Status AppendNdarray(PyObject* value) {
PyArrayObject* ndarray = reinterpret_cast<PyArrayObject*>(value);
OwnedRef flattened;
if (PyArray_NDIM(ndarray) != 1) {
Comment thread
aboderinsamuel marked this conversation as resolved.
return Status::Invalid("Can only convert 1-dimensional array values");
// GH-49644: a fixed-size list (e.g. the storage of a fixed-shape tensor)
// can be built from a multi-dimensional array by flattening it in C
// order. The total number of elements must still match the list size,
// which the builder validates below. Variable-sized lists remain
// restricted to 1-dimensional values to avoid ambiguity.
if (this->list_type_->id() != Type::FIXED_SIZE_LIST) {
return Status::Invalid("Can only convert 1-dimensional array values");
Comment thread
rok marked this conversation as resolved.
Outdated
}
Comment thread
aboderinsamuel marked this conversation as resolved.
flattened.reset(PyArray_Ravel(ndarray, NPY_CORDER));
Comment thread
AlenkaF marked this conversation as resolved.
Outdated
RETURN_IF_PYERROR();
value = flattened.obj();
ndarray = reinterpret_cast<PyArrayObject*>(value);
}
if (PyArray_ISBYTESWAPPED(ndarray)) {
Comment thread
rok marked this conversation as resolved.
Outdated
// TODO
Expand Down
26 changes: 26 additions & 0 deletions python/pyarrow/tests/test_array.py
Original file line number Diff line number Diff line change
Expand Up @@ -2924,6 +2924,32 @@ def test_array_from_invalid_dim_raises():
pa.array(arr0d)


@pytest.mark.numpy
def test_fixed_size_list_from_multidim_ndarray():
# GH-49644: a fixed-size list can be built from multi-dimensional ndarray
# elements by flattening them in C order.
Comment thread
rok marked this conversation as resolved.
Outdated
arr = pa.array([np.array([[1, 2, 3]], dtype=np.int64),
np.array([[4, 5, 6]], dtype=np.int64)],
type=pa.list_(pa.int64(), 3))
assert arr.type == pa.list_(pa.int64(), 3)
assert arr.to_pylist() == [[1, 2, 3], [4, 5, 6]]
Comment thread
aboderinsamuel marked this conversation as resolved.

# A non-trivial 2D shape confirms values are flattened in C (row-major) order
arr = pa.array([np.array([[1, 2], [3, 4]], dtype=np.int64)],
type=pa.list_(pa.int64(), 4))
assert arr.to_pylist() == [[1, 2, 3, 4]]

# The flattened length must still match the fixed size
with pytest.raises(pa.lib.ArrowInvalid):
pa.array([np.array([[1, 2], [3, 4]], dtype=np.int64)],
type=pa.list_(pa.int64(), 3))

# Variable-sized lists still require 1-dimensional values
with pytest.raises(pa.lib.ArrowInvalid, match="1-dimensional"):
Comment thread
rok marked this conversation as resolved.
Outdated
pa.array([np.array([[1, 2, 3]], dtype=np.int64)],
type=pa.list_(pa.int64()))


@pytest.mark.numpy
def test_array_from_strided_bool():
# ARROW-6325
Expand Down
42 changes: 42 additions & 0 deletions python/pyarrow/tests/test_extension_type.py
Original file line number Diff line number Diff line change
Expand Up @@ -1730,6 +1730,48 @@ def test_tensor_array_from_numpy(np_type_str):
pa.FixedShapeTensorArray.from_numpy_ndarray(arr, dim_names=[0, 1])


@pytest.mark.numpy
@pytest.mark.parametrize("np_type_str", ("int8", "int64", "float32"))
def test_tensor_array_from_list_of_ndarrays(np_type_str):
# GH-49644: build a fixed-shape-tensor array from a list of individual
# (multi-dimensional) ndarrays, not only from a single stacked ndarray.
Comment thread
rok marked this conversation as resolved.
Outdated
np_dtype = np.dtype(np_type_str)
tensor_type = pa.fixed_shape_tensor(pa.from_numpy_dtype(np_dtype), (2, 3))

elements = [
np.arange(6, dtype=np_dtype).reshape(2, 3),
np.arange(6, 12, dtype=np_dtype).reshape(2, 3),
]
result = pa.array(elements, type=tensor_type)
assert isinstance(result, pa.FixedShapeTensorArray)
assert result.type == tensor_type
assert len(result) == 2

# Must match the existing from_numpy_ndarray path on the same data
expected = pa.FixedShapeTensorArray.from_numpy_ndarray(np.stack(elements))
assert result.storage.equals(expected.storage)

# Each element round-trips back to the original ndarray (with its shape)
for scalar, original in zip(result, elements):
np.testing.assert_array_equal(scalar.to_numpy(), original)

# Higher-dimensional tensors work too
tensor_3d = pa.fixed_shape_tensor(pa.from_numpy_dtype(np_dtype), (2, 2, 3))
elements_3d = [np.arange(12, dtype=np_dtype).reshape(2, 2, 3)]
result_3d = pa.array(elements_3d, type=tensor_3d)
assert result_3d.type == tensor_3d
np.testing.assert_array_equal(result_3d[0].to_numpy(), elements_3d[0])

# None elements are allowed
result_with_null = pa.array([elements[0], None], type=tensor_type)
assert result_with_null.null_count == 1
assert result_with_null[1].as_py() is None

Comment thread
rok marked this conversation as resolved.
# A flattened size that doesn't match the tensor shape is rejected
with pytest.raises(pa.lib.ArrowInvalid):
pa.array([np.arange(8, dtype=np_dtype).reshape(2, 4)], type=tensor_type)


@pytest.mark.numpy
@pytest.mark.parametrize("tensor_type", (
pa.fixed_shape_tensor(pa.int8(), [2, 2, 3]),
Expand Down