Skip to content

Support ListView/BinaryView/RunEndEncoded types in integration test JSON parser #9888

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
paleolimbot wants to merge 5 commits into
base: main
Choose a base branch
from
+560 −39
Open

Support ListView/BinaryView/RunEndEncoded types in integration test JSON parser #9888

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
2a3e2b6
support in the datatype module
paleolimbot May 4, 2026
4452f13
maybe decode arrays
paleolimbot May 4, 2026
dd57ca0
maybe fix child parsing issues
paleolimbot May 4, 2026
34e8b38
check inlined key
paleolimbot May 4, 2026
20dc35f
add test json
paleolimbot May 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
239 changes: 239 additions & 0 deletions arrow-integration-test/data/integration.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -319,6 +319,89 @@
"children": []
}
]
},
{
"name": "utf8views",
"type": {
"name": "utf8view"
},
"nullable": true,
"children": []
},
{
"name": "binaryviews",
"type": {
"name": "binaryview"
},
"nullable": true,
"children": []
},
{
"name": "listviews",
"type": {
"name": "listview"
},
"nullable": true,
"children": [
{
"name": "item",
"type": {
"name": "int",
"isSigned": true,
"bitWidth": 32
},
"nullable": true,
"children": []
}
]
},
{
"name": "largelistviews",
"type": {
"name": "largelistview"
},
"nullable": true,
"children": [
{
"name": "item",
"type": {
"name": "int",
"isSigned": true,
"bitWidth": 32
},
"nullable": true,
"children": []
}
]
},
{
"name": "runendencoded",
"type": {
"name": "runendencoded"
},
"nullable": true,
"children": [
{
"name": "run_ends",
"type": {
"name": "int",
"isSigned": true,
"bitWidth": 16
},
"nullable": false,
"children": []
},
{
"name": "values",
"type": {
"name": "int",
"isSigned": true,
"bitWidth": 32
},
"nullable": true,
"children": []
}
]
}
]
},
Expand Down Expand Up @@ -801,6 +884,162 @@
]
}
]
},
{
"name": "utf8views",
"count": 3,
"VALIDITY": [
1,
0,
1
],
"VIEWS": [
{
"SIZE": 5,
"INLINED": "hello"
},
{
"SIZE": 0,
"INLINED": ""
},
{
"SIZE": 19,
"PREFIX_HEX": "74686973",
"BUFFER_INDEX": 0,
"OFFSET": 0
}
],
"VARIADIC_DATA_BUFFERS": ["74686973206973206E6F7420696E6C696E6564"]
},
{
"name": "binaryviews",
"count": 3,
"VALIDITY": [
1,
1,
0
],
"VIEWS": [
{
"SIZE": 2,
"INLINED": "F34D"
},
{
"SIZE": 16,
"PREFIX_HEX": "00010203",
"BUFFER_INDEX": 0,
"OFFSET": 0
},
{
"SIZE": 0,
"INLINED": ""
}
],
"VARIADIC_DATA_BUFFERS": ["000102030405060708090A0B0C0D0E0F"]
},
{
"name": "listviews",
"count": 3,
"VALIDITY": [
1,
0,
1
],
"OFFSET": [
0,
2,
2
],
"SIZE": [
2,
0,
3
],
"children": [
{
"name": "item",
"count": 5,
"VALIDITY": [
1,
1,
1,
0,
1
],
"DATA": [
1,
2,
3,
4,
5
]
}
]
},
{
"name": "largelistviews",
"count": 3,
"VALIDITY": [
1,
1,
0
],
"OFFSET": [
"0",
"2",
"3"
],
"SIZE": [
"2",
"1",
"0"
],
"children": [
{
"name": "item",
"count": 3,
"VALIDITY": [
1,
0,
1
],
"DATA": [
10,
20,
30
]
}
]
},
{
"name": "runendencoded",
"count": 3,
"children": [
{
"name": "run_ends",
"count": 2,
"VALIDITY": [
1,
1
],
"DATA": [
2,
3
]
},
{
"name": "values",
"count": 2,
"VALIDITY": [
1,
0
],
"DATA": [
100,
200
]
}
]
}
]
}
Expand Down
29 changes: 22 additions & 7 deletions arrow-integration-test/src/datatype.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ pub fn data_type_from_json(json: &serde_json::Value) -> Result<DataType> {
Some(s) if s == "bool" => Ok(DataType::Boolean),
Some(s) if s == "binary" => Ok(DataType::Binary),
Some(s) if s == "largebinary" => Ok(DataType::LargeBinary),
Some(s) if s == "binaryview" => Ok(DataType::BinaryView),
Some(s) if s == "utf8view" => Ok(DataType::Utf8View),
Some(s) if s == "utf8" => Ok(DataType::Utf8),
Some(s) if s == "largeutf8" => Ok(DataType::LargeUtf8),
Some(s) if s == "fixedsizebinary" => {
Expand Down Expand Up @@ -182,6 +184,14 @@ pub fn data_type_from_json(json: &serde_json::Value) -> Result<DataType> {
// return a largelist with any type as its child isn't defined in the map
Ok(DataType::LargeList(default_field))
}
Some(s) if s == "listview" => {
// return a listview with any type as its child isn't defined in the map
Ok(DataType::ListView(default_field))
}
Some(s) if s == "largelistview" => {
// return a large listview with any type as its child isn't defined in the map
Ok(DataType::LargeListView(default_field))
}
Some(s) if s == "fixedsizelist" => {
// return a list with any type as its child isn't defined in the map
if let Some(Value::Number(size)) = map.get("listSize") {
Expand All @@ -199,6 +209,13 @@ pub fn data_type_from_json(json: &serde_json::Value) -> Result<DataType> {
// return an empty `struct` type as its children aren't defined in the map
Ok(DataType::Struct(Fields::empty()))
}
Some(s) if s == "runendencoded" => {
// return a run end encoded with placeholder types as children aren't defined in the map
Ok(DataType::RunEndEncoded(
Arc::new(Field::new("run_ends", DataType::Int32, false)),
default_field,
))
}
Some(s) if s == "map" => {
if let Some(Value::Bool(keys_sorted)) = map.get("keysSorted") {
// Return a map with an empty type as its children aren't defined in the map
Expand Down Expand Up @@ -271,19 +288,17 @@ pub fn data_type_to_json(data_type: &DataType) -> serde_json::Value {
DataType::LargeUtf8 => json!({"name": "largeutf8"}),
DataType::Binary => json!({"name": "binary"}),
DataType::LargeBinary => json!({"name": "largebinary"}),
DataType::BinaryView | DataType::Utf8View => {
unimplemented!("BinaryView/Utf8View not implemented")
}
DataType::BinaryView => json!({"name": "binaryview"}),
DataType::Utf8View => json!({"name": "utf8view"}),
DataType::FixedSizeBinary(byte_width) => {
json!({"name": "fixedsizebinary", "byteWidth": byte_width})
}
DataType::Struct(_) => json!({"name": "struct"}),
DataType::Union(_, _) => json!({"name": "union"}),
DataType::List(_) => json!({ "name": "list"}),
DataType::LargeList(_) => json!({ "name": "largelist"}),
DataType::ListView(_) | DataType::LargeListView(_) => {
unimplemented!("ListView/LargeListView not implemented")
}
DataType::ListView(_) => json!({ "name": "listview"}),
DataType::LargeListView(_) => json!({ "name": "largelistview"}),
DataType::FixedSizeList(_, length) => {
json!({"name":"fixedsizelist", "listSize": length})
}
Expand Down Expand Up @@ -352,7 +367,7 @@ pub fn data_type_to_json(data_type: &DataType) -> serde_json::Value {
DataType::Map(_, keys_sorted) => {
json!({"name": "map", "keysSorted": keys_sorted})
}
DataType::RunEndEncoded(_, _) => todo!(),
DataType::RunEndEncoded(_, _) => json!({"name": "runendencoded"}),
}
}

Expand Down
Loading
Loading