From 8448e40c8752ad738a4c7dce6aaf2f5dc285b657 Mon Sep 17 00:00:00 2001 From: glutenperfbot Date: Thu, 16 Jul 2026 00:39:32 +0000 Subject: [PATCH 1/7] [GLUTEN-6887][VL] Daily Update Velox Version (dft-2026_07_16) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream Velox's New Commits: 8bece69ac by Linsong Wang, Merge branch 'ci-fix-pr' into staging-rebase-pr 669c9f2e4 by wforget, [OAP] Change SpillPartitionId::kMaxSpillLevel to 7 a6e61e605 by Ke Jia, [OAP]feat: Implement se/dser method for HashTable f0b511b59 by Yuan, [OAP] [11771] Fix smj result mismatch issue in semi, anit and full outer join 7ab4ec425 by Rui Mo, feat: Allow subfield rename and deletion for Parquet format 53a990726 by Zhenyuan Zhao, fix(dwrf): Bound PATCHED_BASE gap chain to avoid OOB read (#18106) e1dd48045 by Hongze Zhang, refactor: Split `dwio/common/Statistics.h` to `.h` and `.cpp` 5219cafbd by oerling, feat(torchwave): Free intermediate tensors at last use (#18119) d8ceba11c by Wechar Yu, fix(spark): Support VARBINARY type in to_json function 1861f170f by Krishna Pai, fix: Revert folly double/float formatting goldens (#18116) ea4ae633b by Apurva Kumar, feat: Replace (union) an existing deletion vector on repeated mutation (#18117) a3c8b8657 by Zhenyuan Zhao, fix: Fix test failure (#18103) 949c7309b by Deepak Majeti, feat(parquet): Add support for Timestamp type rowgroup filtering 0b01ba0ad by Deepak Majeti, fix(build): FBThrift warnings for parquet.thrift 1e7952ade by Apurva Kumar, test: NIMBLE Iceberg field-id end-to-end write+read test (#18056) 332b91f61 by PRASHANT GOLASH, perf: Use DirectBufferedInput for Nimble (enable loadQuantum) (#17948) db15b91d3 by Liangcai Li, feat(cudf): Add Spark substring support fe6aae45d by Reema, fix(cudf): Fix multi column `hash_with_seed` on GPU 049fe62d6 by Philo He, fix: Enable ANSI-compliant cast from VARCHAR to DECIMAL c59294974 by Amit Dutta, fix(window): Fail gracefully on oversized window partitions (#18102) efb391ea4 by Richard Barnes, Update test goldens for folly double/float string formatting change d12dbd60e by jkhaliqi, feat: Add CompressionKind_LZ4_HADOOP 7f6858340 by Daniel Bauer, fix(cudf): Support ms/s timestamp scalars on GPU 4991078e0 by Ping Liu, docs: Scope BMI/BMI2/F16C requirements to x86 4920e8bf3 by Ismaël Mejía, perf(dwio): Cross-platform bit-unpacking and level conversion 4cf8f85db by Ismaël Mejía, fix(hive): Wire file-handle-expiration-duration-ms to SimpleLRUCache (#18072) c1287a247 by meta-codesync[bot], misc: Add Thrift AllowLegacyMissingUris annotation to Velox files (#18012) c759d48ad by Raymond Lin, fix(dwio): Correct flat-map prefetch eligibility (#18038) 039614eda by Ismaël Mejía, fix(dwio): Advance pointers in unpackNaive after decoding loop a8c470526 by Durvesh Pilankar, docs: Fix duplicated words and typos affccb3dc by Kyle Hubert, fix(ucx): Preserve shared exchange client on operator close (#17752) d1f09a430 by binwei yang, upgrade to gcc-133 + spark-3.5 build 6841cd103 by binwei yang, remove spark3.4 7531da59e by binwei yang, add cache key e962632aa by Yuan, fix: Enable CI on IBM repo Signed-off-by: glutenperfbot --- ep/build-velox/src/get-velox.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ep/build-velox/src/get-velox.sh b/ep/build-velox/src/get-velox.sh index 080b10b0a38..3b7294127e0 100755 --- a/ep/build-velox/src/get-velox.sh +++ b/ep/build-velox/src/get-velox.sh @@ -18,8 +18,8 @@ set -exu CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd) VELOX_REPO=https://github.com/IBM/velox.git -VELOX_BRANCH=dft-2026_07_10 -VELOX_ENHANCED_BRANCH=ibm-2026_07_10 +VELOX_BRANCH=dft-2026_07_16 +VELOX_ENHANCED_BRANCH=ibm-2026_07_16 VELOX_HOME="" RUN_SETUP_SCRIPT=ON ENABLE_ENHANCED_FEATURES=OFF From 30c1d3e8a6f4201548d5a4f3757b5d9840f93f21 Mon Sep 17 00:00:00 2001 From: BInwei Yang Date: Thu, 16 Jul 2026 20:25:27 -0700 Subject: [PATCH 2/7] Add support for existing deletion vectors in IcebergWriter --- cpp/velox/compute/iceberg/IcebergWriter.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cpp/velox/compute/iceberg/IcebergWriter.cc b/cpp/velox/compute/iceberg/IcebergWriter.cc index ea838ec9d12..04e35743d89 100644 --- a/cpp/velox/compute/iceberg/IcebergWriter.cc +++ b/cpp/velox/compute/iceberg/IcebergWriter.cc @@ -161,7 +161,9 @@ std::shared_ptr createIcebergInsertTableHandle( const std::unordered_map serdeParameters; auto writeKind = connector::hive::iceberg::IcebergInsertTableHandle::WriteKind::kData; return std::make_shared( - columnHandles, locationHandle, fileFormat, spec, compressionKind, serdeParameters, writeKind, fileNameGenerator); + columnHandles, locationHandle, fileFormat, spec, compressionKind, serdeParameters, writeKind, + std::unordered_map{}, + fileNameGenerator); } } // namespace From e5d54b7484e99547fd99fe712f7285883d1ff960 Mon Sep 17 00:00:00 2001 From: BInwei Yang Date: Thu, 16 Jul 2026 21:33:39 -0700 Subject: [PATCH 3/7] Format parameters for IcebergInsertTableHandle constructor --- cpp/velox/compute/iceberg/IcebergWriter.cc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cpp/velox/compute/iceberg/IcebergWriter.cc b/cpp/velox/compute/iceberg/IcebergWriter.cc index 04e35743d89..9a1427266a9 100644 --- a/cpp/velox/compute/iceberg/IcebergWriter.cc +++ b/cpp/velox/compute/iceberg/IcebergWriter.cc @@ -161,7 +161,13 @@ std::shared_ptr createIcebergInsertTableHandle( const std::unordered_map serdeParameters; auto writeKind = connector::hive::iceberg::IcebergInsertTableHandle::WriteKind::kData; return std::make_shared( - columnHandles, locationHandle, fileFormat, spec, compressionKind, serdeParameters, writeKind, + columnHandles, + locationHandle, + fileFormat, + spec, + compressionKind, + serdeParameters, + writeKind, std::unordered_map{}, fileNameGenerator); } From 33fe817ab749e399f1a6214a5f5315ea77886d71 Mon Sep 17 00:00:00 2001 From: BInwei Yang Date: Thu, 16 Jul 2026 23:16:48 -0700 Subject: [PATCH 4/7] Update IcebergWriter.cc --- cpp/velox/compute/iceberg/IcebergWriter.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cpp/velox/compute/iceberg/IcebergWriter.cc b/cpp/velox/compute/iceberg/IcebergWriter.cc index 9a1427266a9..08327dafca8 100644 --- a/cpp/velox/compute/iceberg/IcebergWriter.cc +++ b/cpp/velox/compute/iceberg/IcebergWriter.cc @@ -168,7 +168,9 @@ std::shared_ptr createIcebergInsertTableHandle( compressionKind, serdeParameters, writeKind, - std::unordered_map{}, + std::unordered_map< + std::string, + facebook::velox::connector::hive::iceberg::IcebergInsertTableHandle::ExistingDeletionVector>{}, fileNameGenerator); } From 2638cd6c31373ec170670a1f36c5c1418ca8e059 Mon Sep 17 00:00:00 2001 From: BInwei Yang Date: Thu, 16 Jul 2026 23:18:33 -0700 Subject: [PATCH 5/7] Fix type in IcebergWriter for ExistingDeletionVector --- cpp/velox/compute/iceberg/IcebergWriter.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cpp/velox/compute/iceberg/IcebergWriter.cc b/cpp/velox/compute/iceberg/IcebergWriter.cc index 08327dafca8..76d7bb93c48 100644 --- a/cpp/velox/compute/iceberg/IcebergWriter.cc +++ b/cpp/velox/compute/iceberg/IcebergWriter.cc @@ -168,9 +168,7 @@ std::shared_ptr createIcebergInsertTableHandle( compressionKind, serdeParameters, writeKind, - std::unordered_map< - std::string, - facebook::velox::connector::hive::iceberg::IcebergInsertTableHandle::ExistingDeletionVector>{}, + std::unordered_map{}, fileNameGenerator); } From 98ef06943d2eed584b13d85202147ad77feb1f17 Mon Sep 17 00:00:00 2001 From: Rong Ma Date: Fri, 17 Jul 2026 11:46:22 +0100 Subject: [PATCH 6/7] remove unused profile --- pom.xml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/pom.xml b/pom.xml index f74955d0613..e4c4abf050f 100644 --- a/pom.xml +++ b/pom.xml @@ -2307,14 +2307,5 @@ ${platform}_${arch} - - disaggregated-shuffle - - false - - - gluten-disaggregated-shuffle - - From 58a1831d9dd579bc898223dab2553a5f75c6e996 Mon Sep 17 00:00:00 2001 From: BInwei Yang Date: Sat, 18 Jul 2026 14:10:39 -0700 Subject: [PATCH 7/7] Update VeloxWriterUtils.cc --- cpp/velox/utils/VeloxWriterUtils.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/velox/utils/VeloxWriterUtils.cc b/cpp/velox/utils/VeloxWriterUtils.cc index 0196aa57575..52551787c25 100644 --- a/cpp/velox/utils/VeloxWriterUtils.cc +++ b/cpp/velox/utils/VeloxWriterUtils.cc @@ -88,7 +88,7 @@ std::shared_ptr makeParquetWriteOp // please make sure `brotli` is enabled when compiling throw GlutenException("Gluten+velox does not support write parquet using brotli."); } else if (boost::iequals(compressionCodecStr, "lz4")) { - compressionCodec = CompressionKind::CompressionKind_LZ4; + compressionCodec = CompressionKind::CompressionKind_LZ4_HADOOP; } else if (boost::iequals(compressionCodecStr, "zstd")) { compressionCodec = CompressionKind::CompressionKind_ZSTD; auto codecOptions = std::make_shared();