From cc97e526529a9cfd59e4f391ebd4c5fc4b23f1a2 Mon Sep 17 00:00:00 2001 From: glutenperfbot Date: Mon, 17 Nov 2025 21:46:56 +0000 Subject: [PATCH 1/5] [GLUTEN-6887][VL] Daily Update Velox Version (dft-2025_11_17) Upstream Velox's New Commits: 46e984109 by Kevin Wilfong, fix: Don't corrupt the FlatVector if resize fails to allocate memory (#15495) 7d2979d2c by Xiaoxuan Meng, refactor: nimble tablet reader code refactor (#15514) 974e77c12 by duanmeng, docs: Add Multi-Round LocalMerge blog article (#15086) 579f13d35 by James Gill, refactor(geo): Implement RTree SpatialIndex for faster spatial joins (#15492) 349dad31f by Artem Selishchev, Add allocation records to the local memory exceeds exception in debug mode (#14830) a7c281582 by James Gill, test(geo): Increase scale of spatial join benchmarks (#15473) 24008217f by Abhinav Mukherjee, Add map_except to Velox as UDF (#15075) ae9f70851 by Wei He, fix: Fix StringView remapping in transferOrCopyTo() when string buffers are copied (#15489) e4d688078 by Pramod Satya, docs: Update Presto functions coverage map (#15502) 7576f4e56 by Ping Liu, feat: Add Iceberg partition name generator (#15461) c0d9747f4 by Chengcheng Jin, feat(cudf): Support CudfVector childrenSize (#15410) f2be8cfe5 by Chengcheng Jin, fix(cudf): Fix mergeRowVector when only one vector (#15494) Signed-off-by: glutenperfbot --- ep/build-velox/src/get-velox.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ep/build-velox/src/get-velox.sh b/ep/build-velox/src/get-velox.sh index d8340948c2a..7f29308c7ba 100755 --- a/ep/build-velox/src/get-velox.sh +++ b/ep/build-velox/src/get-velox.sh @@ -18,8 +18,8 @@ set -exu CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd) VELOX_REPO=https://github.com/IBM/velox.git -VELOX_BRANCH=oap-2025_11_15 -VELOX_ENHANCED_BRANCH=ibm-2025_11_15 +VELOX_BRANCH=dft-2025_11_17 +VELOX_ENHANCED_BRANCH=ibm-2025_11_17 VELOX_HOME="" RUN_SETUP_SCRIPT=ON ENABLE_ENHANCED_FEATURES=OFF From ddec845a7ea128d62066cc2842a14925bf9c3758 Mon Sep 17 00:00:00 2001 From: binwei yang Date: Mon, 17 Nov 2025 16:31:49 -0800 Subject: [PATCH 2/5] fix for 15416 --- cpp/velox/compute/WholeStageResultIterator.cc | 2 +- cpp/velox/jni/VeloxJniWrapper.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/velox/compute/WholeStageResultIterator.cc b/cpp/velox/compute/WholeStageResultIterator.cc index 16a865ac312..38794b81106 100644 --- a/cpp/velox/compute/WholeStageResultIterator.cc +++ b/cpp/velox/compute/WholeStageResultIterator.cc @@ -675,7 +675,7 @@ std::shared_ptr WholeStageResultIterator::createConne // The semantics of reading as lower case is opposite with case-sensitive. configs[velox::connector::hive::HiveConfig::kFileColumnNamesReadAsLowerCaseSession] = !veloxCfg_->get(kCaseSensitive, false) ? "true" : "false"; - configs[velox::connector::hive::HiveConfig::kPartitionPathAsLowerCaseSession] = "false"; + configs[velox::connector::hive::HiveConfig::kPartitionPathAsLowerCaseSession] = "true"; configs[velox::parquet::WriterOptions::kParquetSessionWriteTimestampUnit] = "6"; configs[velox::connector::hive::HiveConfig::kReadTimestampUnitSession] = "6"; configs[velox::connector::hive::HiveConfig::kMaxPartitionsPerWritersSession] = diff --git a/cpp/velox/jni/VeloxJniWrapper.cc b/cpp/velox/jni/VeloxJniWrapper.cc index a9694608083..b48f628551f 100644 --- a/cpp/velox/jni/VeloxJniWrapper.cc +++ b/cpp/velox/jni/VeloxJniWrapper.cc @@ -560,7 +560,7 @@ Java_org_apache_gluten_datasource_VeloxDataSourceJniWrapper_splitBlockByPartitio const auto numRows = inputRowVector->size(); connector::hive::PartitionIdGenerator idGen{ - asRowType(inputRowVector->type()), partitionColIndicesVec, 128, pool.get(), true}; + asRowType(inputRowVector->type()), partitionColIndicesVec, 128, pool.get()}; raw_vector partitionIds{}; idGen.run(inputRowVector, partitionIds); GLUTEN_CHECK(partitionIds.size() == numRows, "Mismatched number of partition ids"); From 59d019124df9863856f20e3118304a4cd486e3be Mon Sep 17 00:00:00 2001 From: binwei yang Date: Mon, 17 Nov 2025 16:42:54 -0800 Subject: [PATCH 3/5] fix --- cpp/velox/jni/VeloxJniWrapper.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/velox/jni/VeloxJniWrapper.cc b/cpp/velox/jni/VeloxJniWrapper.cc index b48f628551f..c4a76738c1a 100644 --- a/cpp/velox/jni/VeloxJniWrapper.cc +++ b/cpp/velox/jni/VeloxJniWrapper.cc @@ -559,8 +559,8 @@ Java_org_apache_gluten_datasource_VeloxDataSourceJniWrapper_splitBlockByPartitio const auto inputRowVector = veloxBatch->getRowVector(); const auto numRows = inputRowVector->size(); - connector::hive::PartitionIdGenerator idGen{ - asRowType(inputRowVector->type()), partitionColIndicesVec, 128, pool.get()}; + connector::hive::PartitionIdGenerator idGen( + asRowType(inputRowVector->type()), partitionColIndicesVec, 128, pool.get()); raw_vector partitionIds{}; idGen.run(inputRowVector, partitionIds); GLUTEN_CHECK(partitionIds.size() == numRows, "Mismatched number of partition ids"); From cd25526a570ba3c9fbe301041cd593ce098ff8f0 Mon Sep 17 00:00:00 2001 From: binwei yang Date: Mon, 17 Nov 2025 18:31:40 -0800 Subject: [PATCH 4/5] fix for enhanced features tmporarily --- cpp/velox/jni/VeloxJniWrapper.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cpp/velox/jni/VeloxJniWrapper.cc b/cpp/velox/jni/VeloxJniWrapper.cc index c4a76738c1a..c18832f9be8 100644 --- a/cpp/velox/jni/VeloxJniWrapper.cc +++ b/cpp/velox/jni/VeloxJniWrapper.cc @@ -560,7 +560,12 @@ Java_org_apache_gluten_datasource_VeloxDataSourceJniWrapper_splitBlockByPartitio const auto numRows = inputRowVector->size(); connector::hive::PartitionIdGenerator idGen( - asRowType(inputRowVector->type()), partitionColIndicesVec, 128, pool.get()); + asRowType(inputRowVector->type()), partitionColIndicesVec, 128, pool.get() +#ifdef GLUTEN_ENABLE_ENHANCED_FEATURES + , + true +#endif + ); raw_vector partitionIds{}; idGen.run(inputRowVector, partitionIds); GLUTEN_CHECK(partitionIds.size() == numRows, "Mismatched number of partition ids"); From 865c84f628f783ebed5f2c0f1e222244f3396958 Mon Sep 17 00:00:00 2001 From: binwei yang Date: Mon, 17 Nov 2025 21:52:36 -0800 Subject: [PATCH 5/5] set case sensitive to false --- cpp/velox/compute/WholeStageResultIterator.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/velox/compute/WholeStageResultIterator.cc b/cpp/velox/compute/WholeStageResultIterator.cc index 38794b81106..16a865ac312 100644 --- a/cpp/velox/compute/WholeStageResultIterator.cc +++ b/cpp/velox/compute/WholeStageResultIterator.cc @@ -675,7 +675,7 @@ std::shared_ptr WholeStageResultIterator::createConne // The semantics of reading as lower case is opposite with case-sensitive. configs[velox::connector::hive::HiveConfig::kFileColumnNamesReadAsLowerCaseSession] = !veloxCfg_->get(kCaseSensitive, false) ? "true" : "false"; - configs[velox::connector::hive::HiveConfig::kPartitionPathAsLowerCaseSession] = "true"; + configs[velox::connector::hive::HiveConfig::kPartitionPathAsLowerCaseSession] = "false"; configs[velox::parquet::WriterOptions::kParquetSessionWriteTimestampUnit] = "6"; configs[velox::connector::hive::HiveConfig::kReadTimestampUnitSession] = "6"; configs[velox::connector::hive::HiveConfig::kMaxPartitionsPerWritersSession] =