Skip to content

[GLUTEN-11630][VL] Enable iceberg tests #11641

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
jinchengchenghh wants to merge 5 commits into from
Closed

[GLUTEN-11630][VL] Enable iceberg tests #11641

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
432e4fd
Enable iceberg tests
rui-mo Feb 19, 2026
978a3b9
fix use Iceberg connector
jinchengchenghh Feb 23, 2026
58c091e
fix table scan connector
jinchengchenghh Feb 24, 2026
77a3041
add header
jinchengchenghh Feb 24, 2026
cb20a14
remove iceberg test from enhanced branch
jinchengchenghh Feb 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .github/workflows/velox_backend_enhanced.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ jobs:
java -version
export SPARK_HOME=/opt/shims/spark34/spark_home/
ls -l $SPARK_HOME
$MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Piceberg-test -Pdelta -Phudi \
$MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Pdelta -Phudi \
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTest \
-DargLine="-Dspark.test.home=$SPARK_HOME"
- name: Upload test report
Expand Down Expand Up @@ -185,7 +185,7 @@ jobs:
java -version
export SPARK_HOME=/opt/shims/spark35/spark_home/
ls -l $SPARK_HOME
$MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi \
$MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Pdelta -Phudi \
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTest \
-DargLine="-Dspark.test.home=$SPARK_HOME"
- name: Upload test report
Expand Down Expand Up @@ -237,7 +237,7 @@ jobs:
export JAVA_HOME=/usr/lib/jvm/java-17-openjdk
export PATH=$JAVA_HOME/bin:$PATH
java -version
$MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \
$MVN_CMD clean test -Pspark-3.5 -Pjava-17 -Pbackends-velox -Pdelta -Phudi -Ppaimon -Pspark-ut \
-DargLine="-Dspark.test.home=/opt/shims/spark35/spark_home/" \
-DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest
- name: Upload test report
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/velox_backend_x86.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -821,7 +821,7 @@ jobs:
java -version
export SPARK_HOME=/opt/shims/spark34/spark_home/
ls -l $SPARK_HOME
$MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Pdelta -Phudi -Ppaimon -Pspark-ut \
$MVN_CMD clean test -Pspark-3.4 -Pjava-17 -Pbackends-velox -Piceberg -Piceberg-test -Pdelta -Phudi -Ppaimon -Pspark-ut \
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest \
-DargLine="-Dspark.test.home=$SPARK_HOME"
- name: Upload test report
Expand Down
3 changes: 0 additions & 3 deletions backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxIcebergSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,4 @@
*/
package org.apache.gluten.execution

import org.apache.gluten.tags.SkipTest

@SkipTest
class VeloxIcebergSuite extends IcebergSuite
3 changes: 0 additions & 3 deletions ...ends-velox/src-iceberg/test/scala/org/apache/gluten/execution/VeloxTPCHIcebergSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,13 @@
*/
package org.apache.gluten.execution

import org.apache.gluten.tags.SkipTest

import org.apache.spark.SparkConf
import org.apache.spark.sql.functions.col

import org.apache.iceberg.spark.SparkWriteOptions

import java.io.File

@SkipTest
class VeloxTPCHIcebergSuite extends VeloxTPCHSuite {
protected val tpchBasePath: String =
getClass.getResource("/").getPath + "../../../src/test/resources"
Expand Down
9 changes: 8 additions & 1 deletion cpp/velox/compute/VeloxBackend.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
#include "velox/common/file/FileSystems.h"
#include "velox/connectors/hive/BufferedInputBuilder.h"
#include "velox/connectors/hive/HiveConnector.h"
#include "velox/connectors/hive/iceberg/IcebergConnector.h"
#include "velox/connectors/hive/HiveDataSource.h"
#include "operators/plannodes/RowVectorStream.h"
#include "velox/connectors/hive/storage_adapters/abfs/RegisterAbfsFileSystem.h" // @manual
Expand Down Expand Up @@ -317,7 +318,13 @@ void VeloxBackend::initConnector(const std::shared_ptr<velox::config::ConfigBase
}
velox::connector::registerConnector(
std::make_shared<velox::connector::hive::HiveConnector>(kHiveConnectorId, hiveConf, ioExecutor_.get()));

IcebergConnectorFactory icebergFactory;
auto icebergConnector = icebergFactory.newConnector(

@jinchengchenghh jinchengchenghh Feb 24, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If the IcebergConnectorFactory is reverted, this PR cannot compile successfully CC @PingLiuPing

kIcebergConnectorId,
hiveConf,
ioExecutor_.get());
connector::registerConnector(icebergConnector);

// Register value-stream connector for runtime iterator-based inputs
velox::connector::registerConnector(std::make_shared<ValueStreamConnector>(kIteratorConnectorId, hiveConf));

Expand Down
7 changes: 3 additions & 4 deletions cpp/velox/compute/WholeStageResultIterator.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,18 +155,16 @@ WholeStageResultIterator::WholeStageResultIterator(

std::shared_ptr<velox::connector::ConnectorSplit> split;
if (auto icebergSplitInfo = std::dynamic_pointer_cast<IcebergSplitInfo>(scanInfo)) {
// Set Iceberg split.
std::unordered_map<std::string, std::string> customSplitInfo{{"table_format", "hive-iceberg"}};
auto deleteFiles = icebergSplitInfo->deleteFilesVec[idx];
split = std::make_shared<velox::connector::hive::iceberg::HiveIcebergSplit>(
kHiveConnectorId,
kIcebergConnectorId,
paths[idx],
format,
starts[idx],
lengths[idx],
partitionKeys,
std::nullopt,
customSplitInfo,
std::unordered_map<std::string, std::string>(),
nullptr,
true,
deleteFiles,
Expand Down Expand Up @@ -215,6 +213,7 @@ WholeStageResultIterator::WholeStageResultIterator(
std::shared_ptr<velox::core::QueryCtx> WholeStageResultIterator::createNewVeloxQueryCtx() {
std::unordered_map<std::string, std::shared_ptr<velox::config::ConfigBase>> connectorConfigs;
connectorConfigs[kHiveConnectorId] = createHiveConnectorSessionConfig(veloxCfg_);
connectorConfigs[kIcebergConnectorId] = connectorConfigs[kHiveConnectorId];
std::shared_ptr<velox::core::QueryCtx> ctx = velox::core::QueryCtx::create(
nullptr,
facebook::velox::core::QueryConfig{getQueryContextConf()},
Expand Down
1 change: 1 addition & 0 deletions cpp/velox/config/VeloxConfig.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ const std::string kVeloxMemReclaimMaxWaitMs = "spark.gluten.sql.columnar.backend
const uint64_t kVeloxMemReclaimMaxWaitMsDefault = 3600000; // 60min

const std::string kHiveConnectorId = "test-hive";
const std::string kIcebergConnectorId = "test-iceberg";
const std::string kVeloxCacheEnabled = "spark.gluten.sql.columnar.backend.velox.cacheEnabled";

const std::string kExprMaxCompiledRegexes = "spark.gluten.sql.columnar.backend.velox.maxCompiledRegexes";
Expand Down
15 changes: 15 additions & 0 deletions cpp/velox/substrait/SubstraitToVeloxPlan.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
#include "utils/ConfigExtractor.h"
#include "utils/VeloxWriterUtils.h"

#include "compute/iceberg/IcebergPlanConverter.h"
#include "config.pb.h"
#include "config/GlutenConfig.h"
#include "config/VeloxConfig.h"
Expand Down Expand Up @@ -54,6 +55,16 @@ bool useCudfTableHandle(const std::vector<std::shared_ptr<SplitInfo>>& splitInfo
#endif
}

bool isIcebergSplit(const std::vector<std::shared_ptr<SplitInfo>>& splitInfos) {
if (splitInfos.empty()) {
return false;
}
if (auto icebergSplitInfo = std::dynamic_pointer_cast<IcebergSplitInfo>(splitInfos[0])) {
return true;
}
return false;
}

core::SortOrder toSortOrder(const ::substrait::SortField& sortField) {
switch (sortField.direction()) {
case ::substrait::SortField_SortDirection_SORT_DIRECTION_ASC_NULLS_FIRST:
Expand Down Expand Up @@ -1466,6 +1477,10 @@ core::PlanNodePtr SubstraitToVeloxPlanConverter::toVeloxPlan(const ::substrait::
connectorId = kCudfHiveConnectorId;
#endif
}
if (isIcebergSplit(splitInfos_)) {
connectorId = kIcebergConnectorId;
}

common::SubfieldFilters subfieldFilters;
tableHandle = std::make_shared<connector::hive::HiveTableHandle>(
connectorId, "hive_table", filterPushdownEnabled, std::move(subfieldFilters), remainingFilter, tableSchema);
Expand Down