diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala index e5bee2dc202..b84628087cf 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/utils/CHExpressionUtil.scala @@ -200,6 +200,7 @@ object CHExpressionUtil { REGR_INTERCEPT -> DefaultValidator(), REGR_SXY -> DefaultValidator(), BITMAP_CONSTRUCT_AGG -> DefaultValidator(), + BITMAP_OR_AGG -> DefaultValidator(), TO_UTC_TIMESTAMP -> UtcTimestampValidator(), FROM_UTC_TIMESTAMP -> UtcTimestampValidator(), STACK -> DefaultValidator(), diff --git a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc index 767af98c760..3548038226e 100644 --- a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc +++ b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.cc @@ -1328,7 +1328,8 @@ bool SubstraitToVeloxPlanValidator::validate(const ::substrait::AggregateRel& ag "regr_sxy", "regr_replacement", "bitmap_construct_agg", - "bitmapaggregator"}; + "bitmapaggregator", + "bitmap_or_agg"}; auto udafFuncs = UdfLoader::getInstance()->getRegisteredUdafNames(); diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index 3c48f1205b0..cffd433c4c8 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -90,8 +90,9 @@ class ClickHouseTestSettings extends BackendTestSettings { // Exception. .exclude("column pruning - non-readable file") enableSuite[GlutenBitmapExpressionsQuerySuite] - // bitmap_construct_agg is not supported natively in CH backend. + // bitmap_construct_agg and bitmap_or_agg are not supported natively in CH backend. .excludeCH("bitmap_construct_agg routes to native") + .excludeCH("bitmap_or_agg routes to native") enableSuite[GlutenBitwiseExpressionsSuite] enableSuite[GlutenBloomFilterAggregateQuerySuite] .excludeCH("Test bloom_filter_agg and might_contain") diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenBitmapExpressionsQuerySuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenBitmapExpressionsQuerySuite.scala index 97072bfeff7..816ba0b35dd 100644 --- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenBitmapExpressionsQuerySuite.scala +++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenBitmapExpressionsQuerySuite.scala @@ -18,6 +18,7 @@ package org.apache.spark.sql import org.apache.gluten.execution.HashAggregateExecBaseTransformer +import org.apache.spark.sql.catalyst.expressions.BitmapOrAgg import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper class GlutenBitmapExpressionsQuerySuite @@ -37,4 +38,23 @@ class GlutenBitmapExpressionsQuerySuite "Expected native HashAggregateExecBaseTransformer in plan" ) } + + test("bitmap_or_agg routes to native") { + val df = spark.sql( + "SELECT bitmap_or_agg(bm) FROM (" + + "SELECT bitmap_construct_agg(bitmap_bit_position(col)) AS bm " + + "FROM values (1L), (2L), (3L) AS t(col)" + + ") sub") + df.collect() + val nativeBitmapOrAggs = collectWithSubqueries(df.queryExecution.executedPlan) { + case h: HashAggregateExecBaseTransformer + if h.aggregateExpressions.exists( + _.aggregateFunction.isInstanceOf[BitmapOrAgg]) => + h + } + assert( + nativeBitmapOrAggs.nonEmpty, + "Expected native HashAggregateExecBaseTransformer with bitmap_or_agg in plan" + ) + } } diff --git a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index 9c9a4c76998..4b21f08ea76 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -89,8 +89,9 @@ class ClickHouseTestSettings extends BackendTestSettings { // Exception. .exclude("column pruning - non-readable file") enableSuite[GlutenBitmapExpressionsQuerySuite] - // bitmap_construct_agg is not supported natively in CH backend. + // bitmap_construct_agg and bitmap_or_agg are not supported natively in CH backend. .excludeCH("bitmap_construct_agg routes to native") + .excludeCH("bitmap_or_agg routes to native") enableSuite[GlutenBitwiseExpressionsSuite] enableSuite[GlutenBloomFilterAggregateQuerySuite] .excludeCH("Test bloom_filter_agg and might_contain") diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenBitmapExpressionsQuerySuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenBitmapExpressionsQuerySuite.scala index 97072bfeff7..816ba0b35dd 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenBitmapExpressionsQuerySuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenBitmapExpressionsQuerySuite.scala @@ -18,6 +18,7 @@ package org.apache.spark.sql import org.apache.gluten.execution.HashAggregateExecBaseTransformer +import org.apache.spark.sql.catalyst.expressions.BitmapOrAgg import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper class GlutenBitmapExpressionsQuerySuite @@ -37,4 +38,23 @@ class GlutenBitmapExpressionsQuerySuite "Expected native HashAggregateExecBaseTransformer in plan" ) } + + test("bitmap_or_agg routes to native") { + val df = spark.sql( + "SELECT bitmap_or_agg(bm) FROM (" + + "SELECT bitmap_construct_agg(bitmap_bit_position(col)) AS bm " + + "FROM values (1L), (2L), (3L) AS t(col)" + + ") sub") + df.collect() + val nativeBitmapOrAggs = collectWithSubqueries(df.queryExecution.executedPlan) { + case h: HashAggregateExecBaseTransformer + if h.aggregateExpressions.exists( + _.aggregateFunction.isInstanceOf[BitmapOrAgg]) => + h + } + assert( + nativeBitmapOrAggs.nonEmpty, + "Expected native HashAggregateExecBaseTransformer with bitmap_or_agg in plan" + ) + } } diff --git a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala index 9c9a4c76998..4b21f08ea76 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala @@ -89,8 +89,9 @@ class ClickHouseTestSettings extends BackendTestSettings { // Exception. .exclude("column pruning - non-readable file") enableSuite[GlutenBitmapExpressionsQuerySuite] - // bitmap_construct_agg is not supported natively in CH backend. + // bitmap_construct_agg and bitmap_or_agg are not supported natively in CH backend. .excludeCH("bitmap_construct_agg routes to native") + .excludeCH("bitmap_or_agg routes to native") enableSuite[GlutenBitwiseExpressionsSuite] enableSuite[GlutenBloomFilterAggregateQuerySuite] .excludeCH("Test bloom_filter_agg and might_contain") diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenBitmapExpressionsQuerySuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenBitmapExpressionsQuerySuite.scala index 97072bfeff7..816ba0b35dd 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenBitmapExpressionsQuerySuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenBitmapExpressionsQuerySuite.scala @@ -18,6 +18,7 @@ package org.apache.spark.sql import org.apache.gluten.execution.HashAggregateExecBaseTransformer +import org.apache.spark.sql.catalyst.expressions.BitmapOrAgg import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper class GlutenBitmapExpressionsQuerySuite @@ -37,4 +38,23 @@ class GlutenBitmapExpressionsQuerySuite "Expected native HashAggregateExecBaseTransformer in plan" ) } + + test("bitmap_or_agg routes to native") { + val df = spark.sql( + "SELECT bitmap_or_agg(bm) FROM (" + + "SELECT bitmap_construct_agg(bitmap_bit_position(col)) AS bm " + + "FROM values (1L), (2L), (3L) AS t(col)" + + ") sub") + df.collect() + val nativeBitmapOrAggs = collectWithSubqueries(df.queryExecution.executedPlan) { + case h: HashAggregateExecBaseTransformer + if h.aggregateExpressions.exists( + _.aggregateFunction.isInstanceOf[BitmapOrAgg]) => + h + } + assert( + nativeBitmapOrAggs.nonEmpty, + "Expected native HashAggregateExecBaseTransformer with bitmap_or_agg in plan" + ) + } } diff --git a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala index dd5c3a188b6..3482a083b39 100644 --- a/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala +++ b/shims/common/src/main/scala/org/apache/gluten/expression/ExpressionNames.scala @@ -34,6 +34,7 @@ object ExpressionNames { final val BLOOM_FILTER_AGG = "bloom_filter_agg" final val BITMAP_CONSTRUCT_AGG = "bitmap_construct_agg" final val BITMAP_AGGREGATOR = "bitmapaggregator" + final val BITMAP_OR_AGG = "bitmap_or_agg" final val VAR_SAMP = "var_samp" final val VAR_POP = "var_pop" final val BIT_AND_AGG = "bit_and" diff --git a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala index 28c1bb177a8..2d97e63b023 100644 --- a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala +++ b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala @@ -81,7 +81,8 @@ class Spark35Shims extends SparkShims { Sig[RegrIntercept](ExpressionNames.REGR_INTERCEPT), Sig[RegrSXY](ExpressionNames.REGR_SXY), Sig[RegrReplacement](ExpressionNames.REGR_REPLACEMENT), - Sig[BitmapConstructAgg](ExpressionNames.BITMAP_CONSTRUCT_AGG) + Sig[BitmapConstructAgg](ExpressionNames.BITMAP_CONSTRUCT_AGG), + Sig[BitmapOrAgg](ExpressionNames.BITMAP_OR_AGG) ) } diff --git a/shims/spark40/src/main/scala/org/apache/gluten/sql/shims/spark40/Spark40Shims.scala b/shims/spark40/src/main/scala/org/apache/gluten/sql/shims/spark40/Spark40Shims.scala index 6363be33038..b903458d349 100644 --- a/shims/spark40/src/main/scala/org/apache/gluten/sql/shims/spark40/Spark40Shims.scala +++ b/shims/spark40/src/main/scala/org/apache/gluten/sql/shims/spark40/Spark40Shims.scala @@ -86,7 +86,8 @@ class Spark40Shims extends SparkShims { Sig[RegrIntercept](ExpressionNames.REGR_INTERCEPT), Sig[RegrSXY](ExpressionNames.REGR_SXY), Sig[RegrReplacement](ExpressionNames.REGR_REPLACEMENT), - Sig[BitmapConstructAgg](ExpressionNames.BITMAP_CONSTRUCT_AGG) + Sig[BitmapConstructAgg](ExpressionNames.BITMAP_CONSTRUCT_AGG), + Sig[BitmapOrAgg](ExpressionNames.BITMAP_OR_AGG) ) } diff --git a/shims/spark41/src/main/scala/org/apache/gluten/sql/shims/spark41/Spark41Shims.scala b/shims/spark41/src/main/scala/org/apache/gluten/sql/shims/spark41/Spark41Shims.scala index 226c2953893..4a8e7aacf9b 100644 --- a/shims/spark41/src/main/scala/org/apache/gluten/sql/shims/spark41/Spark41Shims.scala +++ b/shims/spark41/src/main/scala/org/apache/gluten/sql/shims/spark41/Spark41Shims.scala @@ -85,7 +85,8 @@ class Spark41Shims extends SparkShims { Sig[RegrIntercept](ExpressionNames.REGR_INTERCEPT), Sig[RegrSXY](ExpressionNames.REGR_SXY), Sig[RegrReplacement](ExpressionNames.REGR_REPLACEMENT), - Sig[BitmapConstructAgg](ExpressionNames.BITMAP_CONSTRUCT_AGG) + Sig[BitmapConstructAgg](ExpressionNames.BITMAP_CONSTRUCT_AGG), + Sig[BitmapOrAgg](ExpressionNames.BITMAP_OR_AGG) ) }