From 9e517a68ea1563a232bb025d8b6ed5f0898e3415 Mon Sep 17 00:00:00 2001 From: Michael Kleen Date: Thu, 9 Jul 2026 20:48:01 +0200 Subject: [PATCH 1/2] feat: Support List type in approx_distinct --- .../src/approx_distinct.rs | 2 + .../sqllogictest/test_files/aggregate.slt | 37 +++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/datafusion/functions-aggregate/src/approx_distinct.rs b/datafusion/functions-aggregate/src/approx_distinct.rs index 0e35b47d643c2..ae0d22206aa80 100644 --- a/datafusion/functions-aggregate/src/approx_distinct.rs +++ b/datafusion/functions-aggregate/src/approx_distinct.rs @@ -800,6 +800,7 @@ impl AggregateUDFImpl for ApproxDistinct { | DataType::Binary | DataType::BinaryView | DataType::FixedSizeBinary(_) + | DataType::List(_) | DataType::LargeBinary => Box::new(HLLAccumulator::new()), DataType::Null => { Box::new(NoopAccumulator::new(ScalarValue::UInt64(Some(0)))) @@ -871,6 +872,7 @@ fn is_hll_groups_type(data_type: &DataType) -> bool { | DataType::BinaryView | DataType::FixedSizeBinary(_) | DataType::LargeBinary + | DataType::List(_) ) } diff --git a/datafusion/sqllogictest/test_files/aggregate.slt b/datafusion/sqllogictest/test_files/aggregate.slt index c5970bde9c954..9c11b1568bf18 100644 --- a/datafusion/sqllogictest/test_files/aggregate.slt +++ b/datafusion/sqllogictest/test_files/aggregate.slt @@ -1969,6 +1969,43 @@ SELECT g, approx_distinct(arrow_cast(arrow_cast(s, 'Binary'), 'FixedSizeBinary(1 4 1 +# List +statement ok +CREATE TABLE approx_distinct_list_test (g INT, l INT[]) AS VALUES + (1, [1, 2]), (1, [1, 2]), (1, [3, 4]), + (2, [5, 6]), (2, NULL), + (3, NULL), (3, NULL), + (4, [7, 8]); + +# List non-grouped +query I +SELECT approx_distinct(l) FROM approx_distinct_list_test WHERE g = 1; +---- +2 + +# List grouped +# Group 1 -> {[1,2],[3,4]}=2, +# Group 2 -> {[5,6]}=1 (NULL excluded), +# Group 3 -> all null=0, +# Group 4 -> {[7,8]}=1 +query II +SELECT g, approx_distinct(l) FROM approx_distinct_list_test GROUP BY g ORDER BY g; +---- +1 2 +2 1 +3 0 +4 1 + +# The non-group path must agree with the grouped path on +# the same data, and distinct lists across groups are still counted overall. +query I +SELECT approx_distinct(l) FROM approx_distinct_list_test; +---- +4 + +statement ok +DROP TABLE approx_distinct_list_test; + # Integers (Int32): group 1 -> {10,20}=2, group 2 -> {30,40}=2, group 3 -> 0, group 4 -> {50}=1 query II SELECT g, approx_distinct(i) FROM approx_distinct_group_test GROUP BY g ORDER BY g; From 53919e3f820d9510ad3face1d3bf180868bab5c3 Mon Sep 17 00:00:00 2001 From: Michael Kleen Date: Sat, 11 Jul 2026 08:21:46 +0200 Subject: [PATCH 2/2] Add support for LargeList, FixedSizeList, ListView, LargeListView --- .../src/approx_distinct.rs | 8 ++ .../sqllogictest/test_files/aggregate.slt | 89 +++++++++++++++++++ 2 files changed, 97 insertions(+) diff --git a/datafusion/functions-aggregate/src/approx_distinct.rs b/datafusion/functions-aggregate/src/approx_distinct.rs index ae0d22206aa80..2b261422384ba 100644 --- a/datafusion/functions-aggregate/src/approx_distinct.rs +++ b/datafusion/functions-aggregate/src/approx_distinct.rs @@ -801,6 +801,10 @@ impl AggregateUDFImpl for ApproxDistinct { | DataType::BinaryView | DataType::FixedSizeBinary(_) | DataType::List(_) + | DataType::LargeList(_) + | DataType::FixedSizeList(_, _) + | DataType::ListView(_) + | DataType::LargeListView(_) | DataType::LargeBinary => Box::new(HLLAccumulator::new()), DataType::Null => { Box::new(NoopAccumulator::new(ScalarValue::UInt64(Some(0)))) @@ -873,6 +877,10 @@ fn is_hll_groups_type(data_type: &DataType) -> bool { | DataType::FixedSizeBinary(_) | DataType::LargeBinary | DataType::List(_) + | DataType::LargeList(_) + | DataType::FixedSizeList(_, _) + | DataType::ListView(_) + | DataType::LargeListView(_) ) } diff --git a/datafusion/sqllogictest/test_files/aggregate.slt b/datafusion/sqllogictest/test_files/aggregate.slt index 9c11b1568bf18..e07c9e1b734ab 100644 --- a/datafusion/sqllogictest/test_files/aggregate.slt +++ b/datafusion/sqllogictest/test_files/aggregate.slt @@ -2003,6 +2003,95 @@ SELECT approx_distinct(l) FROM approx_distinct_list_test; ---- 4 +# LargeList non-grouped +query I +SELECT approx_distinct(arrow_cast(l, 'LargeList(Int32)')) FROM approx_distinct_list_test WHERE g = 1; +---- +2 + +# LargeList grouped +query II +SELECT g, approx_distinct(arrow_cast(l, 'LargeList(Int32)')) FROM approx_distinct_list_test GROUP BY g ORDER BY g; +---- +1 2 +2 1 +3 0 +4 1 + +# The non-group path must agree with the grouped path on +# the same data, and distinct lists across groups are still counted overall. +query I +SELECT approx_distinct(arrow_cast(l, 'LargeList(Int32)')) FROM approx_distinct_list_test; +---- +4 + +# ListView non-grouped +query I +SELECT approx_distinct(arrow_cast(l, 'ListView(Int32)')) FROM approx_distinct_list_test WHERE g = 1; +---- +2 + +# ListView grouped +query II +SELECT g, approx_distinct(arrow_cast(l, 'ListView(Int32)')) FROM approx_distinct_list_test GROUP BY g ORDER BY g; +---- +1 2 +2 1 +3 0 +4 1 + +# The non-group path must agree with the grouped path on +# the same data, and distinct lists across groups are still counted overall. +query I +SELECT approx_distinct(arrow_cast(l, 'ListView(Int32)')) FROM approx_distinct_list_test; +---- +4 + +# LargeListView non-grouped +query I +SELECT approx_distinct(arrow_cast(l, 'LargeListView(Int32)')) FROM approx_distinct_list_test WHERE g = 1; +---- +2 + +# LargeListView grouped +query II +SELECT g, approx_distinct(arrow_cast(l, 'LargeListView(Int32)')) FROM approx_distinct_list_test GROUP BY g ORDER BY g; +---- +1 2 +2 1 +3 0 +4 1 + +# The non-group path must agree with the grouped path on +# the same data, and distinct lists across groups are still counted overall. +query I +SELECT approx_distinct(arrow_cast(l, 'LargeListView(Int32)')) FROM approx_distinct_list_test; +---- +4 + + +# FixedSizeList non-grouped +query I +SELECT approx_distinct(arrow_cast(l, 'FixedSizeList(2, Int32)')) FROM approx_distinct_list_test WHERE g = 1; +---- +2 + +# FixedSizeList grouped +query II +SELECT g, approx_distinct(arrow_cast(l, 'FixedSizeList(2, Int32)')) FROM approx_distinct_list_test GROUP BY g ORDER BY g; +---- +1 2 +2 1 +3 0 +4 1 + +# The non-group path must agree with the grouped path on +# the same data, and distinct lists across groups are still counted overall. +query I +SELECT approx_distinct(arrow_cast(l, 'FixedSizeList(2, Int32)')) FROM approx_distinct_list_test; +---- +4 + statement ok DROP TABLE approx_distinct_list_test;