Skip to content

[fix] disable gluten internal 'round' function and rely on Bolt's RoundFunction implementation #18

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
markjin1990 wants to merge 1 commit into
base: add_bolt_backend
Choose a base branch
from
+22 −24
Open

[fix] disable gluten internal 'round' function and rely on Bolt's RoundFunction implementation #18

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 8 additions & 7 deletions cpp/bolt/operators/functions/RegistrationAllFunctions.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,13 +49,14 @@ namespace gluten {
namespace {

void registerFunctionOverwrite() {
bolt::functions::registerUnaryNumeric<RoundFunction>({"round"});
bolt::registerFunction<RoundFunction, int8_t, int8_t, int32_t>({"round"});
bolt::registerFunction<RoundFunction, int16_t, int16_t, int32_t>({"round"});
bolt::registerFunction<RoundFunction, int32_t, int32_t, int32_t>({"round"});
bolt::registerFunction<RoundFunction, int64_t, int64_t, int32_t>({"round"});
bolt::registerFunction<RoundFunction, double, double, int32_t>({"round"});
bolt::registerFunction<RoundFunction, float, float, int32_t>({"round"});
// Disable gluten internal round function
// bolt::functions::registerUnaryNumeric<RoundFunction>({"round"});
// bolt::registerFunction<RoundFunction, int8_t, int8_t, int32_t>({"round"});
// bolt::registerFunction<RoundFunction, int16_t, int16_t, int32_t>({"round"});
// bolt::registerFunction<RoundFunction, int32_t, int32_t, int32_t>({"round"});
// bolt::registerFunction<RoundFunction, int64_t, int64_t, int32_t>({"round"});
// bolt::registerFunction<RoundFunction, double, double, int32_t>({"round"});
// bolt::registerFunction<RoundFunction, float, float, int32_t>({"round"});

auto kRowConstructorWithNull = RowConstructorWithNullCallToSpecialForm::kRowConstructorWithNull;
bolt::exec::registerVectorFunction(
Expand Down
11 changes: 0 additions & 11 deletions cpp/bolt/substrait/SubstraitToBoltPlanValidator.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,23 +130,12 @@ bool SubstraitToBoltPlanValidator::validateRound(
return false;
}

// Bolt has different result with Spark on negative scale.
auto typeCase = arguments[1].value().literal().literal_type_case();
switch (typeCase) {
case ::substrait::Expression_Literal::LiteralTypeCase::kI32: {
int32_t scale = arguments[1].value().literal().i32();
if (scale < 0) {
LOG_VALIDATION_MSG("Round scale validation failed: scale " + std::to_string(scale) + " is negative.");
return false;
}
return true;
}
case ::substrait::Expression_Literal::LiteralTypeCase::kI64: {
int64_t scale = arguments[1].value().literal().i64();
if (scale < 0) {
LOG_VALIDATION_MSG("Round scale validation failed: scale " + std::to_string(scale) + " is negative.");
return false;
}
return true;
}
default:
Expand Down
12 changes: 10 additions & 2 deletions cpp/bolt/tests/SparkFunctionTest.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,13 +67,21 @@ class SparkFunctionTest : public SparkFunctionBaseTest {

template <typename T>
std::vector<std::tuple<T, int32_t, T>> testRoundWithDecFloatAndDoubleData() {
return {{1.122112, 0, 1}, {1.129, 1, 1.1}, {1.129, 2, 1.13}, {1.0 / 3, 0, 0.0},
std::vector<std::tuple<T, int32_t, T>> data =
{{1.122112, 0, 1}, {1.129, 1, 1.1}, {1.129, 2, 1.13}, {1.0 / 3, 0, 0.0},
{1.0 / 3, 1, 0.3}, {1.0 / 3, 2, 0.33}, {1.0 / 3, 6, 0.333333}, {-1.122112, 0, -1},
{-1.129, 1, -1.1}, {-1.129, 2, -1.13}, {-1.129, 2, -1.13}, {-1.0 / 3, 0, 0.0},
{-1.0 / 3, 1, -0.3}, {-1.0 / 3, 2, -0.33}, {-1.0 / 3, 6, -0.333333}, {1.0, -1, 0.0},
{0.0, -2, 0.0}, {-1.0, -3, 0.0}, {11111.0, -1, 11110.0}, {11111.0, -2, 11100.0},
{11111.0, -3, 11000.0}, {11111.0, -4, 10000.0}, {0.575, 2, 0.58}, {0.574, 2, 0.57},
{-0.575, 2, -0.58}, {-0.574, 2, -0.57}};
{-0.575, 2, -0.58}, {-0.574, 2, -0.57}, {102.825291, 6, 102.825291}, {10.424817, 6, 10.424817},
{-82.737209, 6, -82.737209}, {-123.106541, 6, -123.106541}};
// To be consistent with Spark
if constexpr (std::is_same_v<T, float>) {
data[22] = {0.575, 2, 0.57};
data[24] = {-0.575, 2, -0.57};
}
return data;
}

template <typename T>
Expand Down
2 changes: 1 addition & 1 deletion ...src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ class GlutenMathExpressionsSuite extends MathExpressionsSuite with GlutenTestsTr
Seq(0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 3.0, 3.1, 3.14, 3.142, 3.1416, 3.14159, 3.141593)

val floatResults: Seq[Float] =
Seq(0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 3.0f, 3.1f, 3.14f, 3.142f, 3.1415f, 3.1415f, 3.1415f)
Seq(0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 3.0f, 3.1f, 3.14f, 3.141f, 3.1415f, 3.1415f, 3.1415f)

val bRoundFloatResults: Seq[Float] =
Seq(0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 3.0f, 3.1f, 3.14f, 3.141f, 3.1415f, 3.1415f, 3.1415f)
Expand Down
2 changes: 1 addition & 1 deletion ...src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ class GlutenMathExpressionsSuite extends MathExpressionsSuite with GlutenTestsTr
Seq(0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 3.0, 3.1, 3.14, 3.142, 3.1416, 3.14159, 3.141593)

val floatResults: Seq[Float] =
Seq(0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 3.0f, 3.1f, 3.14f, 3.142f, 3.1415f, 3.1415f, 3.1415f)
Seq(0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 3.0f, 3.1f, 3.14f, 3.141f, 3.1415f, 3.1415f, 3.1415f)

val bRoundFloatResults: Seq[Float] =
Seq(0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 3.0f, 3.1f, 3.14f, 3.141f, 3.1415f, 3.1415f, 3.1415f)
Expand Down
2 changes: 1 addition & 1 deletion ...src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ class GlutenMathExpressionsSuite extends MathExpressionsSuite with GlutenTestsTr
Seq(0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 3.0, 3.1, 3.14, 3.142, 3.1416, 3.14159, 3.141593)

val floatResults: Seq[Float] =
Seq(0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 3.0f, 3.1f, 3.14f, 3.142f, 3.1415f, 3.1415f, 3.1415f)
Seq(0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 3.0f, 3.1f, 3.14f, 3.141f, 3.1415f, 3.1415f, 3.1415f)

val bRoundFloatResults: Seq[Float] =
Seq(0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 3.0f, 3.1f, 3.14f, 3.141f, 3.1415f, 3.1415f, 3.1415f)
Expand Down
2 changes: 1 addition & 1 deletion ...src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenMathExpressionsSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ class GlutenMathExpressionsSuite extends MathExpressionsSuite with GlutenTestsTr
Seq(0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 3.0, 3.1, 3.14, 3.142, 3.1416, 3.14159, 3.141593)

val floatResults: Seq[Float] =
Seq(0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 3.0f, 3.1f, 3.14f, 3.142f, 3.1415f, 3.1415f, 3.1415f)
Seq(0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 3.0f, 3.1f, 3.14f, 3.141f, 3.1415f, 3.1415f, 3.1415f)

val bRoundFloatResults: Seq[Float] =
Seq(0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 3.0f, 3.1f, 3.14f, 3.141f, 3.1415f, 3.1415f, 3.1415f)
Expand Down
Loading