Skip to content

Commit fea605c

Browse files
authored
[Variant] Fix cast logic for Variant to Arrow for DataType::Null (#8825)
# Which issue does this PR close? - Closes #8810 . # What changes are included in this PR? Fix the logic for `VariantToNullArrowRowBuilder` so that it respects the `cast` option # Are these changes tested? added test # Are there any user-facing changes? Noe
1 parent 6675204 commit fea605c

File tree

2 files changed

+51
-8
lines changed

2 files changed

+51
-8
lines changed

parquet-variant-compute/src/variant_get.rs

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -319,6 +319,7 @@ mod test {
319319
use arrow::compute::CastOptions;
320320
use arrow::datatypes::DataType::{Int16, Int32, Int64};
321321
use arrow::datatypes::i256;
322+
use arrow::util::display::FormatOptions;
322323
use arrow_schema::DataType::{Boolean, Float32, Float64, Int8};
323324
use arrow_schema::{DataType, Field, FieldRef, Fields, IntervalUnit, TimeUnit};
324325
use chrono::DateTime;
@@ -1039,6 +1040,43 @@ mod test {
10391040
arrow::array::NullArray::new(3)
10401041
);
10411042

1043+
perfectly_shredded_variant_array_fn!(perfectly_shredded_null_variant_array_with_int, || {
1044+
Int32Array::from(vec![Some(32), Some(64), Some(48)])
1045+
});
1046+
1047+
// We append null values if type miss match happens in safe mode
1048+
perfectly_shredded_to_arrow_primitive_test!(
1049+
get_variant_perfectly_shredded_null_with_type_missmatch_in_safe_mode,
1050+
DataType::Null,
1051+
perfectly_shredded_null_variant_array_with_int,
1052+
arrow::array::NullArray::new(3)
1053+
);
1054+
1055+
// We'll return an error if type miss match happens in strict mode
1056+
#[test]
1057+
fn get_variant_perfectly_shredded_null_as_null_with_type_missmatch_in_strict_mode() {
1058+
let array = perfectly_shredded_null_variant_array_with_int();
1059+
let field = Field::new("typed_value", DataType::Null, true);
1060+
let options = GetOptions::new()
1061+
.with_as_type(Some(FieldRef::from(field)))
1062+
.with_cast_options(CastOptions {
1063+
safe: false,
1064+
format_options: FormatOptions::default(),
1065+
});
1066+
1067+
let result = variant_get(&array, options);
1068+
1069+
assert!(result.is_err());
1070+
let error_msg = format!("{}", result.unwrap_err());
1071+
assert!(
1072+
error_msg
1073+
.contains("Cast error: Failed to extract primitive of type Null from variant Int32(32) at path VariantPath([])"),
1074+
"Expected=[Cast error: Failed to extract primitive of type Null from variant Int32(32) at path VariantPath([])],\
1075+
Got error message=[{}]",
1076+
error_msg
1077+
);
1078+
}
1079+
10421080
perfectly_shredded_variant_array_fn!(perfectly_shredded_decimal4_variant_array, || {
10431081
Decimal32Array::from(vec![Some(12345), Some(23400), Some(-12342)])
10441082
.with_precision_and_scale(5, 2)

parquet-variant-compute/src/variant_to_arrow.rs

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -696,24 +696,29 @@ impl VariantToBinaryVariantArrowRowBuilder {
696696
}
697697
}
698698

699-
struct FakeNullBuilder(NullArray);
699+
#[derive(Default)]
700+
struct FakeNullBuilder {
701+
item_count: usize,
702+
}
700703

701704
impl FakeNullBuilder {
702-
fn new(capacity: usize) -> Self {
703-
Self(NullArray::new(capacity))
705+
fn append_value(&mut self, _: ()) {
706+
self.item_count += 1;
707+
}
708+
709+
fn append_null(&mut self) {
710+
self.item_count += 1;
704711
}
705-
fn append_value<T>(&mut self, _: T) {}
706-
fn append_null(&mut self) {}
707712

708713
fn finish(self) -> NullArray {
709-
self.0
714+
NullArray::new(self.item_count)
710715
}
711716
}
712717

713718
define_variant_to_primitive_builder!(
714719
struct VariantToNullArrowRowBuilder<'a>
715-
|capacity| -> FakeNullBuilder { FakeNullBuilder::new(capacity) },
716-
|_value| Some(Variant::Null),
720+
|_capacity| -> FakeNullBuilder { FakeNullBuilder::default() },
721+
|value| value.as_null(),
717722
type_name: "Null"
718723
);
719724

0 commit comments

Comments
 (0)