From 1bb1005ba5c73548f6589d0c4500c60078ef13fa Mon Sep 17 00:00:00 2001 From: glutenperfbot Date: Sat, 27 Jun 2026 07:32:34 +0000 Subject: [PATCH 01/12] [GLUTEN-6887][VL] Daily Update Velox Version (dft-2026_06_27) Upstream Velox's New Commits: 14de1ea1a by Zac Wen, fix: Reject nullAllowed filters in index lookup extraction (#17928) 72b003f55 by Ping Liu, docs: Refactor hive connector config properties (#17872) a49069bf8 by Hazmi, feat(iceberg): Support customize file name generator for IcebergInsertTableHandle (#17437) 8fc595046 by Masha Basmanova, refactor(test): Build DuckDbQueryRunner's database lazily (#17939) 9e63ccfe0 by Zac Wen, fix: Handle exclusive floating-point bounds in extractRangeBounds (#17916) f259ed2de by sungwoo-XCENA, build(cxl): Add the VELOX_ENABLE_CXL build flag (#17935) 83a17d6fd by Jimmy Lu, build: Add BloomFilter.cpp to velox_common_base CMake sources (#17946) f13ddcb8c by Hongze Zhang, feat(spark): Use BloomFilterView in BloomFilterMightContainFunction to avoid copy (#16609) Signed-off-by: glutenperfbot --- ep/build-velox/src/get-velox.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ep/build-velox/src/get-velox.sh b/ep/build-velox/src/get-velox.sh index 637860db37d..81112e8fe22 100755 --- a/ep/build-velox/src/get-velox.sh +++ b/ep/build-velox/src/get-velox.sh @@ -18,8 +18,8 @@ set -exu CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd) VELOX_REPO=https://github.com/IBM/velox.git -VELOX_BRANCH=dft-2026_06_12 -VELOX_ENHANCED_BRANCH=ibm-2026_06_12 +VELOX_BRANCH=dft-2026_06_27 +VELOX_ENHANCED_BRANCH=ibm-2026_06_27 VELOX_HOME="" RUN_SETUP_SCRIPT=ON ENABLE_ENHANCED_FEATURES=OFF From dc3c0be92675af6e5041cd1d599aabb9c5f968de Mon Sep 17 00:00:00 2001 From: Yuan Zhou Date: Sun, 28 Jun 2026 09:36:27 +0100 Subject: [PATCH 02/12] fix compile Signed-off-by: Yuan Zhou --- .../writer/VeloxColumnarBatchWriter.cc | 10 +++++-- cpp/velox/substrait/SubstraitToVeloxPlan.cc | 2 +- cpp/velox/utils/ConfigExtractor.cc | 8 +++--- cpp/velox/utils/VeloxWriterUtils.cc | 28 +++++++++++-------- cpp/velox/utils/VeloxWriterUtils.h | 2 +- 5 files changed, 29 insertions(+), 21 deletions(-) diff --git a/cpp/velox/operators/writer/VeloxColumnarBatchWriter.cc b/cpp/velox/operators/writer/VeloxColumnarBatchWriter.cc index b12c0472ac5..598199f19b3 100644 --- a/cpp/velox/operators/writer/VeloxColumnarBatchWriter.cc +++ b/cpp/velox/operators/writer/VeloxColumnarBatchWriter.cc @@ -33,11 +33,15 @@ arrow::Status VeloxColumnarBatchWriter::initWriter(const facebook::velox::RowTyp auto localWriteFile = std::make_unique(path_, false, true); auto sink = std::make_unique(std::move(localWriteFile), path_); - facebook::velox::parquet::WriterOptions writerOptions; + facebook::velox::dwio::common::WriterOptions writerOptions; + facebook::velox::parquet::ParquetWriterOptions parquetWriterOptions; + writerOptions.memoryPool = pool_.get(); writerOptions.compressionKind = facebook::velox::common::CompressionKind::CompressionKind_SNAPPY; - writerOptions.batchSize = batchSize_; - + // set parquet specific options + parquetWriterOptions.batchSize = batchSize_; + writerOptions.formatSpecificOptions = + std::make_shared(parquetWriterOptions); writer_ = std::make_unique(std::move(sink), writerOptions, rowType); return arrow::Status::OK(); } diff --git a/cpp/velox/substrait/SubstraitToVeloxPlan.cc b/cpp/velox/substrait/SubstraitToVeloxPlan.cc index 2d7e65734ba..39363af4367 100644 --- a/cpp/velox/substrait/SubstraitToVeloxPlan.cc +++ b/cpp/velox/substrait/SubstraitToVeloxPlan.cc @@ -878,7 +878,7 @@ core::PlanNodePtr SubstraitToVeloxPlanConverter::toVeloxPlan(const ::substrait:: GLUTEN_CHECK(formatShortName == "parquet", "Unsupported file write format: " + formatShortName); dwio::common::FileFormat fileFormat = dwio::common::FileFormat::PARQUET; - const std::shared_ptr writerOptions = makeParquetWriteOption(writeConfs); + const std::shared_ptr writerOptions = makeParquetWriteOption(writeConfs); // Spark's default compression code is snappy. const auto& compressionKind = writerOptions->compressionKind.value_or(common::CompressionKind::CompressionKind_SNAPPY); diff --git a/cpp/velox/utils/ConfigExtractor.cc b/cpp/velox/utils/ConfigExtractor.cc index bc226541a8b..c8886e74ebc 100644 --- a/cpp/velox/utils/ConfigExtractor.cc +++ b/cpp/velox/utils/ConfigExtractor.cc @@ -25,7 +25,7 @@ #include "utils/Macros.h" #include "velox/connectors/hive/HiveConfig.h" #include "velox/connectors/hive/storage_adapters/s3fs/S3Config.h" -#include "velox/dwio/parquet/writer/WriterConfig.h" +#include "velox/dwio/parquet/common/ParquetConfig.h" namespace gluten { @@ -229,7 +229,7 @@ std::shared_ptr createHiveConnectorSessionC configs[facebook::velox::connector::hive::HiveConfig::kFileColumnNamesReadAsLowerCaseSession] = !conf->get(kCaseSensitive, false) ? "true" : "false"; configs[facebook::velox::connector::hive::HiveConfig::kPartitionPathAsLowerCaseSession] = "false"; - configs[facebook::velox::parquet::WriterConfig::kParquetSessionWriteTimestampUnit] = std::string("6"); + configs[std::string(facebook::velox::parquet::ParquetConfig::kWriterTimestampUnitSession)] = std::string("6"); configs[facebook::velox::connector::hive::HiveConfig::kReadTimestampUnitSession] = std::string("6"); configs[facebook::velox::connector::hive::HiveConfig::kMaxPartitionsPerWritersSession] = conf->get(kMaxPartitions, "10000"); @@ -243,7 +243,7 @@ std::shared_ptr createHiveConnectorSessionC conf->get(kAllowInt32Narrowing, true) ? "true" : "false"; configs[facebook::velox::connector::hive::HiveConfig::kOrcUseColumnNamesSession] = conf->get(kOrcUseColumnNames, true) ? "true" : "false"; - configs[facebook::velox::parquet::WriterConfig::kParquetSessionWritePageSize] = + configs[std::string(facebook::velox::parquet::ParquetConfig::kWriterPageSizeSession)] = conf->get(kWriteParquetPageSizeBytes, "1MB"); overwriteVeloxConf(conf.get(), configs, kDynamicBackendConfPrefix); @@ -307,7 +307,7 @@ std::shared_ptr createHiveConnectorConfig( // read as UTC hiveConfMap[facebook::velox::connector::hive::HiveConfig::kReadTimestampPartitionValueAsLocalTime] = "false"; - hiveConfMap[facebook::velox::connector::hive::HiveConfig::kParquetNullStructForMissingFields] = "true"; + hiveConfMap[std::string(facebook::velox::parquet::ParquetConfig::kNullStructIfAllFieldsMissingSession)] = "true"; overwriteVeloxConf(conf.get(), hiveConfMap, kStaticBackendConfPrefix); return std::make_shared(std::move(hiveConfMap)); diff --git a/cpp/velox/utils/VeloxWriterUtils.cc b/cpp/velox/utils/VeloxWriterUtils.cc index 5616cd94812..bbb29071c71 100644 --- a/cpp/velox/utils/VeloxWriterUtils.cc +++ b/cpp/velox/utils/VeloxWriterUtils.cc @@ -38,7 +38,8 @@ const int32_t kGzipWindowBits4k = 12; const int32_t kZSTDDefaultCompressionLevel = 3; } // namespace -std::unique_ptr makeParquetWriteOption(const std::unordered_map& sparkConfs) { +std::shared_ptr makeParquetWriteOption( + const std::unordered_map& sparkConfs) { int64_t maxRowGroupBytes = 134217728; // 128MB int64_t maxRowGroupRows = 100000000; // 100M if (auto it = sparkConfs.find(kParquetBlockSize); it != sparkConfs.end()) { @@ -47,8 +48,10 @@ std::unique_ptr makeParquetWriteOption(const std::unordered_mapsecond); } - auto writeOption = std::make_unique(); - writeOption->parquetWriteTimestampUnit = TimestampPrecision::kMicroseconds /*micro*/; + + auto writeOption = std::make_shared(); + auto parquetOptions = std::make_shared(); + parquetOptions->parquetWriteTimestampUnit = TimestampPrecision::kMicroseconds /*micro*/; bool writeLegacyParquetFormat = false; if (auto it = sparkConfs.find(kParquetStoreDecimalAsInteger); it != sparkConfs.end()) { writeLegacyParquetFormat = boost::iequals(it->second, "true"); @@ -62,7 +65,7 @@ std::unique_ptr makeParquetWriteOption(const std::unordered_mapenableStoreDecimalAsInteger = !writeLegacyParquetFormat; + parquetOptions->enableStoreDecimalAsInteger = !writeLegacyParquetFormat; auto compressionCodec = CompressionKind::CompressionKind_SNAPPY; if (auto it = sparkConfs.find(kParquetCompressionCodec); it != sparkConfs.end()) { auto compressionCodecStr = it->second; @@ -76,7 +79,7 @@ std::unique_ptr makeParquetWriteOption(const std::unordered_map(); codecOptions->windowBits = kGzipWindowBits4k; - writeOption->codecOptions = std::move(codecOptions); + parquetOptions->codecOptions = std::move(codecOptions); } } } else if (boost::iequals(compressionCodecStr, "lzo")) { @@ -92,7 +95,7 @@ std::unique_ptr makeParquetWriteOption(const std::unordered_mapsecond) : kZSTDDefaultCompressionLevel; codecOptions->compressionLevel = compressionLevel; - writeOption->codecOptions = std::move(codecOptions); + parquetOptions->codecOptions = std::move(codecOptions); } else if (boost::iequals(compressionCodecStr, "uncompressed")) { compressionCodec = CompressionKind::CompressionKind_NONE; } else if (boost::iequals(compressionCodecStr, "none")) { @@ -104,28 +107,29 @@ std::unique_ptr makeParquetWriteOption(const std::unordered_map(maxRowGroupRows, maxRowGroupBytes, [&]() { return false; }); }; if (auto it = sparkConfs.find(kSessionTimezone); it != sparkConfs.end()) { - writeOption->parquetWriteTimestampTimeZone = normalizeSessionTimezone(it->second); + parquetOptions->parquetWriteTimestampTimeZone = normalizeSessionTimezone(it->second); } - writeOption->arrowMemoryPool = + parquetOptions->arrowMemoryPool = getDefaultMemoryManager()->getOrCreateArrowMemoryPool("VeloxParquetWrite.ArrowMemoryPool"); if (auto it = sparkConfs.find(kParquetDataPageSize); it != sparkConfs.end()) { auto dataPageSize = std::stoll(it->second); - writeOption->dataPageSize = dataPageSize; + parquetOptions->dataPageSize = dataPageSize; } if (auto it = sparkConfs.find(kParquetWriterVersion); it != sparkConfs.end()) { auto parquetVersion = it->second; if (boost::iequals(parquetVersion, "v2")) { - writeOption->useParquetDataPageV2 = true; + parquetOptions->useParquetDataPageV2 = true; } } if (auto it = sparkConfs.find(kParquetEnableDictionary); it != sparkConfs.end()) { auto enableDictionary = it->second; if (boost::iequals(enableDictionary, "true")) { - writeOption->enableDictionary = true; + parquetOptions->enableDictionary = true; } else { - writeOption->enableDictionary = false; + parquetOptions->enableDictionary = false; } } + writeOption->formatSpecificOptions = std::move(parquetOptions); return writeOption; } diff --git a/cpp/velox/utils/VeloxWriterUtils.h b/cpp/velox/utils/VeloxWriterUtils.h index 65fe2b2c621..56bd5b2e272 100644 --- a/cpp/velox/utils/VeloxWriterUtils.h +++ b/cpp/velox/utils/VeloxWriterUtils.h @@ -23,7 +23,7 @@ namespace gluten { -std::unique_ptr makeParquetWriteOption( +std::shared_ptr makeParquetWriteOption( const std::unordered_map& sparkConfs); } // namespace gluten From afbfc3b6a14855dfa55082d10a0444ee0e661f04 Mon Sep 17 00:00:00 2001 From: Hazmi Date: Tue, 5 May 2026 15:15:01 +0000 Subject: [PATCH 03/12] [VL] Refactor Gluten to use upstream Velox --- .../enhanced/VeloxIcebergSuite.scala | 3 +- .../gluten/utils/VeloxBloomFilterTest.java | 20 +++--- cpp/velox/CMakeLists.txt | 6 +- cpp/velox/compute/VeloxBackend.cc | 8 +++ cpp/velox/compute/VeloxBackend.h | 4 ++ cpp/velox/compute/VeloxConnectorIds.h | 2 + cpp/velox/compute/VeloxRuntime.cc | 15 ++++- cpp/velox/compute/VeloxRuntime.h | 7 -- cpp/velox/compute/WholeStageResultIterator.cc | 3 +- cpp/velox/compute/iceberg/IcebergWriter.cc | 66 +++++++++++-------- cpp/velox/compute/iceberg/IcebergWriter.h | 5 +- cpp/velox/config/VeloxConfig.h | 2 + cpp/velox/jni/VeloxJniWrapper.cc | 10 --- cpp/velox/substrait/SubstraitToVeloxPlan.cc | 9 ++- .../substrait/SubstraitToVeloxPlanValidator.h | 6 +- cpp/velox/utils/VeloxWriterUtils.cc | 2 +- .../spark/source/IcebergWriteUtil.scala | 16 +++-- 17 files changed, 112 insertions(+), 72 deletions(-) diff --git a/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/enhanced/VeloxIcebergSuite.scala b/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/enhanced/VeloxIcebergSuite.scala index ddd57dca466..0ec53f4bab0 100644 --- a/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/enhanced/VeloxIcebergSuite.scala +++ b/backends-velox/src-iceberg/test/scala/org/apache/gluten/execution/enhanced/VeloxIcebergSuite.scala @@ -327,8 +327,7 @@ class VeloxIcebergSuite extends IcebergSuite { val lastExecId = statusStore.executionsList().last.executionId val executionMetrics = statusStore.executionMetrics(lastExecId) - // TODO: fix https://github.com/apache/gluten/issues/11510 - assert(executionMetrics(metrics("numWrittenFiles").id).toLong == 0) + assert(executionMetrics(metrics("numWrittenFiles").id).toLong == 1) } } diff --git a/backends-velox/src/test/java/org/apache/gluten/utils/VeloxBloomFilterTest.java b/backends-velox/src/test/java/org/apache/gluten/utils/VeloxBloomFilterTest.java index 351b97ee112..f8cafc79085 100644 --- a/backends-velox/src/test/java/org/apache/gluten/utils/VeloxBloomFilterTest.java +++ b/backends-velox/src/test/java/org/apache/gluten/utils/VeloxBloomFilterTest.java @@ -47,16 +47,16 @@ public void testMalformed() { buf.putInt(0); // size TaskResources$.MODULE$.runUnsafe( () -> { - final BloomFilter filter = VeloxBloomFilter.readFrom(buf.array()); - Assert.assertThrows( - "Bloom-filter is not initialized", - RuntimeException.class, - new ThrowingRunnable() { - @Override - public void run() throws Throwable { - filter.mightContainLong(0); - } - }); + RuntimeException exception = + Assert.assertThrows( + RuntimeException.class, + new ThrowingRunnable() { + @Override + public void run() throws Throwable { + VeloxBloomFilter.readFrom(buf.array()); + } + }); + Assert.assertTrue(exception.getMessage().contains("Invalid BloomFilter size: 0")); return null; }); } diff --git a/cpp/velox/CMakeLists.txt b/cpp/velox/CMakeLists.txt index 4ec3f9f0f2e..eea15ed3019 100644 --- a/cpp/velox/CMakeLists.txt +++ b/cpp/velox/CMakeLists.txt @@ -230,10 +230,8 @@ if(ENABLE_GPU) memory/GpuBufferColumnarBatch.cc) endif() -if(ENABLE_ENHANCED_FEATURES) - list(APPEND VELOX_SRCS compute/iceberg/IcebergFormat.cc - compute/iceberg/IcebergWriter.cc) -endif() +list(APPEND VELOX_SRCS compute/iceberg/IcebergFormat.cc + compute/iceberg/IcebergWriter.cc) if(BUILD_TESTS OR BUILD_BENCHMARKS) set(BUILD_TEST_UTILS ON) diff --git a/cpp/velox/compute/VeloxBackend.cc b/cpp/velox/compute/VeloxBackend.cc index aa77d0ffe7a..4ba7dd7a739 100644 --- a/cpp/velox/compute/VeloxBackend.cc +++ b/cpp/velox/compute/VeloxBackend.cc @@ -56,6 +56,7 @@ #include "velox/connectors/hive/BufferedInputBuilder.h" #include "velox/connectors/hive/HiveConnector.h" #include "velox/connectors/hive/HiveDataSource.h" +#include "velox/connectors/hive/iceberg/IcebergConnector.h" #include "velox/connectors/hive/storage_adapters/abfs/RegisterAbfsFileSystem.h" // @manual #include "velox/connectors/hive/storage_adapters/gcs/RegisterGcsFileSystem.h" // @manual #include "velox/connectors/hive/storage_adapters/hdfs/HdfsFileSystem.h" @@ -380,6 +381,13 @@ std::shared_ptr VeloxBackend::createDelta return std::make_shared(connectorId, hiveConnectorConfig_, ioExecutor); } +std::shared_ptr VeloxBackend::createIcebergConnector( + const std::string& connectorId, + folly::Executor* ioExecutor) const { + return std::make_shared( + connectorId, hiveConnectorConfig_, ioExecutor); +} + std::shared_ptr VeloxBackend::createValueStreamConnector( const std::string& connectorId, bool dynamicFilterEnabled) const { diff --git a/cpp/velox/compute/VeloxBackend.h b/cpp/velox/compute/VeloxBackend.h index 2796ce20c3c..5597ca67e6a 100644 --- a/cpp/velox/compute/VeloxBackend.h +++ b/cpp/velox/compute/VeloxBackend.h @@ -74,6 +74,10 @@ class VeloxBackend { const std::string& connectorId, folly::Executor* ioExecutor) const; + std::shared_ptr createIcebergConnector( + const std::string& connectorId, + folly::Executor* ioExecutor) const; + std::shared_ptr createDeltaConnector( const std::string& connectorId, folly::Executor* ioExecutor) const; diff --git a/cpp/velox/compute/VeloxConnectorIds.h b/cpp/velox/compute/VeloxConnectorIds.h index a0e37ba8b60..73b526229fb 100644 --- a/cpp/velox/compute/VeloxConnectorIds.h +++ b/cpp/velox/compute/VeloxConnectorIds.h @@ -23,10 +23,12 @@ namespace gluten { struct VeloxConnectorIds { std::string hive; + std::string iceberg; std::string delta; std::string iterator; std::string cudfHive; bool hiveRegistered{false}; + bool icebergRegistered{false}; bool deltaRegistered{false}; bool iteratorRegistered{false}; bool cudfHiveRegistered{false}; diff --git a/cpp/velox/compute/VeloxRuntime.cc b/cpp/velox/compute/VeloxRuntime.cc index f13430bd0c5..4099111dfcd 100644 --- a/cpp/velox/compute/VeloxRuntime.cc +++ b/cpp/velox/compute/VeloxRuntime.cc @@ -227,6 +227,7 @@ std::string makeScopedConnectorId(const std::string& base, uint64_t runtimeId) { VeloxConnectorIds makeScopedConnectorIds(uint64_t runtimeId) { return VeloxConnectorIds{ .hive = makeScopedConnectorId(kHiveConnectorId, runtimeId), + .iceberg = makeScopedConnectorId(kIcebergConnectorId, runtimeId), .delta = makeScopedConnectorId(delta::DeltaConnectorFactory::kDeltaConnectorName, runtimeId), .iterator = makeScopedConnectorId(kIteratorConnectorId, runtimeId), .cudfHive = makeScopedConnectorId(kCudfHiveConnectorId, runtimeId)}; @@ -290,6 +291,14 @@ void VeloxRuntime::registerConnectors() { velox::connector::hasConnector(connectorIds_.hive), "Scoped hive connector not found after registration: " + connectorIds_.hive); + connectorIds_.icebergRegistered = + velox::connector::registerConnector(backend->createIcebergConnector(connectorIds_.iceberg, ioExecutor_.get())); + GLUTEN_CHECK( + connectorIds_.icebergRegistered, "Failed to register scoped Iceberg connector: " + connectorIds_.iceberg); + GLUTEN_CHECK( + velox::connector::hasConnector(connectorIds_.iceberg), + "Scoped Iceberg connector not found after registration: " + connectorIds_.iceberg); + connectorIds_.deltaRegistered = velox::connector::registerConnector(backend->createDeltaConnector(connectorIds_.delta, ioExecutor_.get())); GLUTEN_CHECK(connectorIds_.deltaRegistered, "Failed to register scoped delta connector: " + connectorIds_.delta); @@ -340,6 +349,10 @@ void VeloxRuntime::unregisterConnectors() { velox::connector::unregisterConnector(connectorIds_.hive); connectorIds_.hiveRegistered = false; } + if (connectorIds_.icebergRegistered) { + velox::connector::unregisterConnector(connectorIds_.iceberg); + connectorIds_.icebergRegistered = false; + } } void VeloxRuntime::parsePlan(const uint8_t* data, int32_t size) { @@ -518,7 +531,6 @@ std::shared_ptr VeloxRuntime::createRow2ColumnarConverte return std::make_shared(cSchema, veloxPool); } -#ifdef GLUTEN_ENABLE_ENHANCED_FEATURES std::shared_ptr VeloxRuntime::createIcebergWriter( RowTypePtr rowType, int32_t format, @@ -546,7 +558,6 @@ std::shared_ptr VeloxRuntime::createIcebergWriter( veloxPool, connectorPool); } -#endif std::shared_ptr VeloxRuntime::createShuffleWriter( int32_t numPartitions, diff --git a/cpp/velox/compute/VeloxRuntime.h b/cpp/velox/compute/VeloxRuntime.h index c6ee1c462c6..3f53653e5ba 100644 --- a/cpp/velox/compute/VeloxRuntime.h +++ b/cpp/velox/compute/VeloxRuntime.h @@ -20,9 +20,7 @@ #include "WholeStageResultIterator.h" #include "compute/Runtime.h" #include "compute/VeloxConnectorIds.h" -#ifdef GLUTEN_ENABLE_ENHANCED_FEATURES #include "iceberg/IcebergWriter.h" -#endif #include #include "memory/VeloxMemoryManager.h" #include "operators/serializer/VeloxColumnarBatchSerializer.h" @@ -30,10 +28,7 @@ #include "operators/writer/VeloxParquetDataSource.h" #include "shuffle/ShuffleReader.h" #include "shuffle/ShuffleWriter.h" - -#ifdef GLUTEN_ENABLE_ENHANCED_FEATURES #include "IcebergNestedField.pb.h" -#endif namespace gluten { @@ -77,7 +72,6 @@ class VeloxRuntime final : public Runtime { std::shared_ptr createRow2ColumnarConverter(struct ArrowSchema* cSchema) override; -#ifdef GLUTEN_ENABLE_ENHANCED_FEATURES std::shared_ptr createIcebergWriter( RowTypePtr rowType, int32_t format, @@ -89,7 +83,6 @@ class VeloxRuntime final : public Runtime { std::shared_ptr spec, const gluten::IcebergNestedField& protoField, const std::unordered_map& sparkConfs); -#endif std::shared_ptr createShuffleWriter( int numPartitions, diff --git a/cpp/velox/compute/WholeStageResultIterator.cc b/cpp/velox/compute/WholeStageResultIterator.cc index b3d9f480c1c..29213c2e0e2 100644 --- a/cpp/velox/compute/WholeStageResultIterator.cc +++ b/cpp/velox/compute/WholeStageResultIterator.cc @@ -202,7 +202,7 @@ WholeStageResultIterator::WholeStageResultIterator( std::unordered_map customSplitInfo{{"table_format", "hive-iceberg"}}; auto deleteFiles = icebergSplitInfo->deleteFilesVec[idx]; split = std::make_shared( - connectorIds_.hive, + connectorIds_.iceberg, paths[idx], format, starts[idx], @@ -284,6 +284,7 @@ std::shared_ptr WholeStageResultIterator::createNewVeloxQ std::unordered_map> connectorConfigs; auto hiveSessionConfig = createHiveConnectorSessionConfig(veloxCfg_); connectorConfigs[connectorIds_.hive] = hiveSessionConfig; + connectorConfigs[connectorIds_.iceberg] = hiveSessionConfig; connectorConfigs[connectorIds_.delta] = hiveSessionConfig; connectorConfigs[connectorIds_.iterator] = hiveSessionConfig; #ifdef GLUTEN_ENABLE_GPU diff --git a/cpp/velox/compute/iceberg/IcebergWriter.cc b/cpp/velox/compute/iceberg/IcebergWriter.cc index e248fe4d279..ea838ec9d12 100644 --- a/cpp/velox/compute/iceberg/IcebergWriter.cc +++ b/cpp/velox/compute/iceberg/IcebergWriter.cc @@ -17,8 +17,10 @@ #include "IcebergWriter.h" +#include "IcebergNestedField.pb.h" #include "IcebergPartitionSpec.pb.h" #include "compute/ProtobufUtils.h" +#include "compute/VeloxBackend.h" #include "compute/iceberg/IcebergFormat.h" #include "config/VeloxConfig.h" #include "utils/ConfigExtractor.h" @@ -44,6 +46,7 @@ class GlutenIcebergFileNameGenerator : public connector::hive::FileNameGenerator std::optional bucketId, const std::shared_ptr insertTableHandle, const connector::ConnectorQueryCtx& connectorQueryCtx, + uint32_t maxNumBuckets, bool commitRequired) const override { auto targetFileName = insertTableHandle->locationHandle()->targetFileName(); if (targetFileName.empty()) { @@ -99,9 +102,9 @@ class GlutenIcebergFileNameGenerator : public connector::hive::FileNameGenerator mutable int32_t fileCount_; }; -iceberg::IcebergNestedField convertToIcebergNestedField(const gluten::IcebergNestedField& protoField) { - IcebergNestedField result; - result.id = protoField.id(); +parquet::ParquetFieldId convertToIcebergNestedField(const gluten::IcebergNestedField& protoField) { + parquet::ParquetFieldId result; + result.fieldId = protoField.id(); // Recursively convert children result.children.reserve(protoField.children_size()); @@ -121,7 +124,7 @@ std::shared_ptr createIcebergInsertTableHandle( int64_t taskId, const std::string& operationId, std::shared_ptr spec, - const iceberg::IcebergNestedField& nestedField, + const parquet::ParquetFieldId& nestedField, facebook::velox::memory::MemoryPool* pool) { std::vector> columnHandles; @@ -139,14 +142,12 @@ std::shared_ptr createIcebergInsertTableHandle( columnNames.at(i), connector::hive::HiveColumnHandle::ColumnType::kPartitionKey, columnTypes.at(i), - columnTypes.at(i), nestedField.children[i])); } else { columnHandles.push_back(std::make_shared( columnNames.at(i), connector::hive::HiveColumnHandle::ColumnType::kRegular, columnTypes.at(i), - columnTypes.at(i), nestedField.children[i])); } } @@ -157,18 +158,10 @@ std::shared_ptr createIcebergInsertTableHandle( std::shared_ptr locationHandle = std::make_shared( outputDirectoryPath, outputDirectoryPath, connector::hive::LocationHandle::TableType::kExisting); - const std::vector sortedBy; const std::unordered_map serdeParameters; + auto writeKind = connector::hive::iceberg::IcebergInsertTableHandle::WriteKind::kData; return std::make_shared( - columnHandles, - locationHandle, - spec, - pool, - fileFormat, - sortedBy, - compressionKind, - serdeParameters, - fileNameGenerator); + columnHandles, locationHandle, fileFormat, spec, compressionKind, serdeParameters, writeKind, fileNameGenerator); } } // namespace @@ -200,20 +193,36 @@ IcebergWriter::IcebergWriter( connectorSessionProperties_ = createHiveConnectorSessionConfig(veloxCfg); connectorConfig_ = std::make_shared(createHiveConnectorConfig(veloxCfg)); + std::unordered_map> connectorConfigs; + connectorConfigs[kHiveConnectorId] = connectorSessionProperties_; + auto queryConfigBase = + std::make_shared(std::unordered_map(sparkConfs)); + queryCtx_ = facebook::velox::core::QueryCtx::create( + nullptr, + facebook::velox::core::QueryConfig{facebook::velox::core::QueryConfig::ConfigTag{}, queryConfigBase}, + connectorConfigs, + nullptr, // cache + pool_, + nullptr, // spillExecutor + "IcebergWriter"); + + auto expressionEvaluator = + std::make_unique(queryCtx_.get(), pool_.get()); + connectorQueryCtx_ = std::make_unique( pool_.get(), connectorPool_.get(), connectorSessionProperties_.get(), nullptr, common::PrefixSortConfig(), - nullptr, + std::move(expressionEvaluator), nullptr, "query.IcebergDataSink", "task.IcebergDataSink", "planNodeId.IcebergDataSink", 0, ""); - + auto icebergConfig = std::make_shared(veloxCfg); dataSink_ = std::make_unique( rowType_, createIcebergInsertTableHandle( @@ -229,24 +238,29 @@ IcebergWriter::IcebergWriter( pool_.get()), connectorQueryCtx_.get(), facebook::velox::connector::CommitStrategy::kNoCommit, - connectorConfig_); + connectorConfig_, + icebergConfig); } void IcebergWriter::write(const VeloxColumnarBatch& batch) { auto inputRowVector = batch.getRowVector(); auto inputRowType = asRowType(inputRowVector->type()); - if (inputRowType->size() != rowType_->size()) { - const auto& children = inputRowVector->children(); - std::vector dataColumns(children.begin() + 1, children.begin() + 1 + rowType_->size()); + const auto& children = inputRowVector->children(); - auto filteredRowVector = std::make_shared( - pool_.get(), rowType_, inputRowVector->nulls(), inputRowVector->size(), std::move(dataColumns)); + std::vector dataColumns; + dataColumns.reserve(rowType_->size()); - dataSink_->appendData(filteredRowVector); + if (inputRowType->size() != rowType_->size()) { + dataColumns.insert(dataColumns.end(), children.begin() + 1, children.begin() + 1 + rowType_->size()); } else { - dataSink_->appendData(inputRowVector); + dataColumns.insert(dataColumns.end(), children.begin(), children.end()); } + + auto rowVector = std::make_shared( + pool_.get(), rowType_, inputRowVector->nulls(), inputRowVector->size(), std::move(dataColumns)); + + dataSink_->appendData(rowVector); } std::vector IcebergWriter::commit() { diff --git a/cpp/velox/compute/iceberg/IcebergWriter.h b/cpp/velox/compute/iceberg/IcebergWriter.h index 2fa13dcd698..0ab3a803608 100644 --- a/cpp/velox/compute/iceberg/IcebergWriter.h +++ b/cpp/velox/compute/iceberg/IcebergWriter.h @@ -47,7 +47,7 @@ class IcebergWriter { int64_t taskId, const std::string& operationId, std::shared_ptr spec, - const gluten::IcebergNestedField& field, + const IcebergNestedField& field, const std::unordered_map& sparkConfs, std::shared_ptr memoryPool, std::shared_ptr connectorPool); @@ -60,7 +60,7 @@ class IcebergWriter { private: facebook::velox::RowTypePtr rowType_; - const facebook::velox::connector::hive::iceberg::IcebergNestedField field_; + const facebook::velox::parquet::ParquetFieldId field_; int32_t partitionId_; int64_t taskId_; std::string operationId_; @@ -69,6 +69,7 @@ class IcebergWriter { std::shared_ptr connectorConfig_; std::shared_ptr connectorSessionProperties_; + std::shared_ptr queryCtx_; std::unique_ptr connectorQueryCtx_; std::unique_ptr dataSink_; diff --git a/cpp/velox/config/VeloxConfig.h b/cpp/velox/config/VeloxConfig.h index d88c4361938..b6a9f54e806 100644 --- a/cpp/velox/config/VeloxConfig.h +++ b/cpp/velox/config/VeloxConfig.h @@ -116,6 +116,8 @@ const std::string kVeloxMemReclaimMaxWaitMs = "spark.gluten.sql.columnar.backend const uint64_t kVeloxMemReclaimMaxWaitMsDefault = 3600000; // 60min const std::string kHiveConnectorId = "test-hive"; +const std::string kIcebergConnectorId = "test-iceberg"; + const std::string kVeloxCacheEnabled = "spark.gluten.sql.columnar.backend.velox.cacheEnabled"; const std::string kExprMaxCompiledRegexes = "spark.gluten.sql.columnar.backend.velox.maxCompiledRegexes"; diff --git a/cpp/velox/jni/VeloxJniWrapper.cc b/cpp/velox/jni/VeloxJniWrapper.cc index a06c948c0fe..369d6716ed4 100644 --- a/cpp/velox/jni/VeloxJniWrapper.cc +++ b/cpp/velox/jni/VeloxJniWrapper.cc @@ -52,10 +52,6 @@ #include "utils/GpuBufferBatchResizer.h" #endif -#ifdef GLUTEN_ENABLE_ENHANCED_FEATURES -#include "IcebergNestedField.pb.h" -#endif - using namespace gluten; using namespace facebook; @@ -844,11 +840,7 @@ Java_org_apache_gluten_vectorized_UnifflePartitionWriterJniWrapper_createPartiti JNIEXPORT jboolean JNICALL Java_org_apache_gluten_config_ConfigJniWrapper_isEnhancedFeaturesEnabled( // NOLINT JNIEnv* env, jclass) { -#ifdef GLUTEN_ENABLE_ENHANCED_FEATURES return true; -#else - return false; -#endif } #ifdef GLUTEN_ENABLE_GPU @@ -867,7 +859,6 @@ JNIEXPORT jboolean JNICALL Java_org_apache_gluten_cudf_VeloxCudfPlanValidatorJni } #endif -#ifdef GLUTEN_ENABLE_ENHANCED_FEATURES JNIEXPORT jlong JNICALL Java_org_apache_gluten_execution_IcebergWriteJniWrapper_init( // NOLINT JNIEnv* env, jobject wrapper, @@ -955,7 +946,6 @@ JNIEXPORT jobject JNICALL Java_org_apache_gluten_execution_IcebergWriteJniWrappe JNI_METHOD_END(nullptr) } -#endif JNIEXPORT jlong JNICALL Java_org_apache_gluten_vectorized_HashJoinBuilder_nativeBuild( // NOLINT JNIEnv* env, diff --git a/cpp/velox/substrait/SubstraitToVeloxPlan.cc b/cpp/velox/substrait/SubstraitToVeloxPlan.cc index 39363af4367..c88dbcbe9b0 100644 --- a/cpp/velox/substrait/SubstraitToVeloxPlan.cc +++ b/cpp/velox/substrait/SubstraitToVeloxPlan.cc @@ -21,6 +21,7 @@ #include "VariantToVectorConverter.h" #include "compute/delta/DeltaConnector.h" #include "compute/delta/DeltaSplitInfo.h" +#include "compute/iceberg/IcebergPlanConverter.h" #include "jni/JniHashTable.h" #include "operators/hashjoin/HashTableBuilder.h" #include "operators/plannodes/RowVectorStream.h" @@ -878,7 +879,8 @@ core::PlanNodePtr SubstraitToVeloxPlanConverter::toVeloxPlan(const ::substrait:: GLUTEN_CHECK(formatShortName == "parquet", "Unsupported file write format: " + formatShortName); dwio::common::FileFormat fileFormat = dwio::common::FileFormat::PARQUET; - const std::shared_ptr writerOptions = makeParquetWriteOption(writeConfs); + const std::shared_ptr writerOptions = + makeParquetWriteOption(writeConfs); // Spark's default compression code is snappy. const auto& compressionKind = writerOptions->compressionKind.value_or(common::CompressionKind::CompressionKind_SNAPPY); @@ -1584,7 +1586,10 @@ core::PlanNodePtr SubstraitToVeloxPlanConverter::toVeloxPlan(const ::substrait:: connector::ConnectorTableHandlePtr tableHandle; auto remainingFilter = readRel.has_filter() ? exprConverter_->toVeloxExpr(readRel.filter(), baseSchema) : nullptr; auto connectorId = isDeltaSplitInfo(splitInfo) ? connectorIds_.delta : connectorIds_.hive; - if (connectorId == connectorIds_.hive && useCudfTableHandle(splitInfos_) && + if (std::dynamic_pointer_cast(splitInfo)) { + connectorId = connectorIds_.iceberg; + } + if ((connectorId == connectorIds_.hive || connectorId == connectorIds_.iceberg) && useCudfTableHandle(splitInfos_) && veloxCfg_->get(kCudfEnableTableScan, kCudfEnableTableScanDefault) && veloxCfg_->get(kCudfEnabled, kCudfEnabledDefault)) { #ifdef GLUTEN_ENABLE_GPU diff --git a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.h b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.h index ca9f6c4012b..e9c4f9879e2 100644 --- a/cpp/velox/substrait/SubstraitToVeloxPlanValidator.h +++ b/cpp/velox/substrait/SubstraitToVeloxPlanValidator.h @@ -41,7 +41,11 @@ class SubstraitToVeloxPlanValidator { pool, veloxCfg_.get(), std::vector>{}, - VeloxConnectorIds{.hive = kHiveConnectorId, .iterator = kIteratorConnectorId, .cudfHive = kCudfHiveConnectorId}, + VeloxConnectorIds{ + .hive = kHiveConnectorId, + .iceberg = kIcebergConnectorId, + .iterator = kIteratorConnectorId, + .cudfHive = kCudfHiveConnectorId}, std::nullopt, std::nullopt, true); diff --git a/cpp/velox/utils/VeloxWriterUtils.cc b/cpp/velox/utils/VeloxWriterUtils.cc index bbb29071c71..0196aa57575 100644 --- a/cpp/velox/utils/VeloxWriterUtils.cc +++ b/cpp/velox/utils/VeloxWriterUtils.cc @@ -39,7 +39,7 @@ const int32_t kZSTDDefaultCompressionLevel = 3; } // namespace std::shared_ptr makeParquetWriteOption( - const std::unordered_map& sparkConfs) { + const std::unordered_map& sparkConfs) { int64_t maxRowGroupBytes = 134217728; // 128MB int64_t maxRowGroupRows = 100000000; // 100M if (auto it = sparkConfs.find(kParquetBlockSize); it != sparkConfs.end()) { diff --git a/gluten-iceberg/src/main/scala/org/apache/iceberg/spark/source/IcebergWriteUtil.scala b/gluten-iceberg/src/main/scala/org/apache/iceberg/spark/source/IcebergWriteUtil.scala index 3f7ab278d0b..18e7153e9c7 100644 --- a/gluten-iceberg/src/main/scala/org/apache/iceberg/spark/source/IcebergWriteUtil.scala +++ b/gluten-iceberg/src/main/scala/org/apache/iceberg/spark/source/IcebergWriteUtil.scala @@ -34,9 +34,7 @@ object IcebergWriteUtil { } private lazy val writePropertiesField = { - val field = classOf[SparkWrite].getDeclaredField("writeProperties") - field.setAccessible(true) - field + optionalField(classOf[SparkWrite], "writeProperties") } private lazy val writeConfField = { @@ -89,7 +87,9 @@ object IcebergWriteUtil { } def getWriteProperty(write: Write): java.util.Map[String, String] = { - writePropertiesField.get(write).asInstanceOf[java.util.Map[String, String]] + writePropertiesField + .map(_.get(write).asInstanceOf[java.util.Map[String, String]]) + .getOrElse(java.util.Map.of()) } def getWriteConf(write: Write): SparkWriteConf = { @@ -128,4 +128,12 @@ object IcebergWriteUtil { commit } + private def optionalField(cls: Class[_], name: String): Option[java.lang.reflect.Field] = + try { + val f = cls.getDeclaredField(name) + f.setAccessible(true) + Some(f) + } catch { + case _: NoSuchFieldException => None + } } From 722bf4fb85c95d9785d208354bbda70c0f10d5ef Mon Sep 17 00:00:00 2001 From: Yuan Date: Mon, 29 Jun 2026 13:38:55 +0100 Subject: [PATCH 04/12] fix Signed-off-by: Yuan --- cpp/velox/utils/ConfigExtractor.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cpp/velox/utils/ConfigExtractor.cc b/cpp/velox/utils/ConfigExtractor.cc index c8886e74ebc..b6e52ca957b 100644 --- a/cpp/velox/utils/ConfigExtractor.cc +++ b/cpp/velox/utils/ConfigExtractor.cc @@ -245,6 +245,7 @@ std::shared_ptr createHiveConnectorSessionC conf->get(kOrcUseColumnNames, true) ? "true" : "false"; configs[std::string(facebook::velox::parquet::ParquetConfig::kWriterPageSizeSession)] = conf->get(kWriteParquetPageSizeBytes, "1MB"); + configs[std::string(facebook::velox::parquet::ParquetConfig::kNullStructIfAllFieldsMissingSession)] = "true"; overwriteVeloxConf(conf.get(), configs, kDynamicBackendConfPrefix); return std::make_shared(std::move(configs)); @@ -307,8 +308,6 @@ std::shared_ptr createHiveConnectorConfig( // read as UTC hiveConfMap[facebook::velox::connector::hive::HiveConfig::kReadTimestampPartitionValueAsLocalTime] = "false"; - hiveConfMap[std::string(facebook::velox::parquet::ParquetConfig::kNullStructIfAllFieldsMissingSession)] = "true"; - overwriteVeloxConf(conf.get(), hiveConfMap, kStaticBackendConfPrefix); return std::make_shared(std::move(hiveConfMap)); } From 41ece5c5e4aac33f10811854d657be050525668b Mon Sep 17 00:00:00 2001 From: Hazmi Date: Mon, 29 Jun 2026 18:09:42 +0000 Subject: [PATCH 05/12] Temp branch --- ep/build-velox/src/get-velox.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ep/build-velox/src/get-velox.sh b/ep/build-velox/src/get-velox.sh index 81112e8fe22..0f1175ef45e 100755 --- a/ep/build-velox/src/get-velox.sh +++ b/ep/build-velox/src/get-velox.sh @@ -18,8 +18,8 @@ set -exu CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd) VELOX_REPO=https://github.com/IBM/velox.git -VELOX_BRANCH=dft-2026_06_27 -VELOX_ENHANCED_BRANCH=ibm-2026_06_27 +VELOX_BRANCH=tempfix +VELOX_ENHANCED_BRANCH=tempfix VELOX_HOME="" RUN_SETUP_SCRIPT=ON ENABLE_ENHANCED_FEATURES=OFF From ef3eb2ef0345b9de8b09a10eae4f50d882750cc5 Mon Sep 17 00:00:00 2001 From: Rui Mo Date: Tue, 30 Jun 2026 10:10:33 +0100 Subject: [PATCH 06/12] Fix --- cpp/velox/utils/ConfigExtractor.cc | 13 ++++++++++--- ep/build-velox/src/get-velox.sh | 4 ++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/cpp/velox/utils/ConfigExtractor.cc b/cpp/velox/utils/ConfigExtractor.cc index b6e52ca957b..39c9ebd78e7 100644 --- a/cpp/velox/utils/ConfigExtractor.cc +++ b/cpp/velox/utils/ConfigExtractor.cc @@ -219,6 +219,11 @@ void getAbfsHiveConfig( #endif } +std::string parquetSessionProperty(std::string_view key) { + return facebook::velox::dwio::common::formatConfigPrefix(facebook::velox::dwio::common::FileFormat::PARQUET, "_") + + std::string(key); +} + } // namespace std::shared_ptr createHiveConnectorSessionConfig( @@ -229,7 +234,8 @@ std::shared_ptr createHiveConnectorSessionC configs[facebook::velox::connector::hive::HiveConfig::kFileColumnNamesReadAsLowerCaseSession] = !conf->get(kCaseSensitive, false) ? "true" : "false"; configs[facebook::velox::connector::hive::HiveConfig::kPartitionPathAsLowerCaseSession] = "false"; - configs[std::string(facebook::velox::parquet::ParquetConfig::kWriterTimestampUnitSession)] = std::string("6"); + configs[parquetSessionProperty(facebook::velox::parquet::ParquetConfig::kWriterTimestampUnitSession)] = + std::string("6"); configs[facebook::velox::connector::hive::HiveConfig::kReadTimestampUnitSession] = std::string("6"); configs[facebook::velox::connector::hive::HiveConfig::kMaxPartitionsPerWritersSession] = conf->get(kMaxPartitions, "10000"); @@ -243,9 +249,10 @@ std::shared_ptr createHiveConnectorSessionC conf->get(kAllowInt32Narrowing, true) ? "true" : "false"; configs[facebook::velox::connector::hive::HiveConfig::kOrcUseColumnNamesSession] = conf->get(kOrcUseColumnNames, true) ? "true" : "false"; - configs[std::string(facebook::velox::parquet::ParquetConfig::kWriterPageSizeSession)] = + configs[parquetSessionProperty(facebook::velox::parquet::ParquetConfig::kWriterPageSizeSession)] = conf->get(kWriteParquetPageSizeBytes, "1MB"); - configs[std::string(facebook::velox::parquet::ParquetConfig::kNullStructIfAllFieldsMissingSession)] = "true"; + configs[parquetSessionProperty(facebook::velox::parquet::ParquetConfig::kNullStructIfAllFieldsMissingSession)] = + "true"; overwriteVeloxConf(conf.get(), configs, kDynamicBackendConfPrefix); return std::make_shared(std::move(configs)); diff --git a/ep/build-velox/src/get-velox.sh b/ep/build-velox/src/get-velox.sh index 0f1175ef45e..81112e8fe22 100755 --- a/ep/build-velox/src/get-velox.sh +++ b/ep/build-velox/src/get-velox.sh @@ -18,8 +18,8 @@ set -exu CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd) VELOX_REPO=https://github.com/IBM/velox.git -VELOX_BRANCH=tempfix -VELOX_ENHANCED_BRANCH=tempfix +VELOX_BRANCH=dft-2026_06_27 +VELOX_ENHANCED_BRANCH=ibm-2026_06_27 VELOX_HOME="" RUN_SETUP_SCRIPT=ON ENABLE_ENHANCED_FEATURES=OFF From 075e447e778ab565b37481a2154ad8bff1fa08f3 Mon Sep 17 00:00:00 2001 From: Hazmi Date: Tue, 30 Jun 2026 09:51:04 +0000 Subject: [PATCH 07/12] Fixed iceberg java support --- .../org/apache/iceberg/spark/source/IcebergWriteUtil.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gluten-iceberg/src/main/scala/org/apache/iceberg/spark/source/IcebergWriteUtil.scala b/gluten-iceberg/src/main/scala/org/apache/iceberg/spark/source/IcebergWriteUtil.scala index 18e7153e9c7..83ecb07037c 100644 --- a/gluten-iceberg/src/main/scala/org/apache/iceberg/spark/source/IcebergWriteUtil.scala +++ b/gluten-iceberg/src/main/scala/org/apache/iceberg/spark/source/IcebergWriteUtil.scala @@ -89,7 +89,7 @@ object IcebergWriteUtil { def getWriteProperty(write: Write): java.util.Map[String, String] = { writePropertiesField .map(_.get(write).asInstanceOf[java.util.Map[String, String]]) - .getOrElse(java.util.Map.of()) + .getOrElse(java.util.Collections.emptyMap[String, String]()) } def getWriteConf(write: Write): SparkWriteConf = { From bafb64d9f32836e4dcf5225f3e65d8456beceab9 Mon Sep 17 00:00:00 2001 From: Yuan Date: Tue, 30 Jun 2026 11:59:17 +0100 Subject: [PATCH 08/12] fix Signed-off-by: Yuan --- cpp/velox/utils/ConfigExtractor.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/velox/utils/ConfigExtractor.cc b/cpp/velox/utils/ConfigExtractor.cc index 39c9ebd78e7..f398c2588d3 100644 --- a/cpp/velox/utils/ConfigExtractor.cc +++ b/cpp/velox/utils/ConfigExtractor.cc @@ -25,6 +25,7 @@ #include "utils/Macros.h" #include "velox/connectors/hive/HiveConfig.h" #include "velox/connectors/hive/storage_adapters/s3fs/S3Config.h" +#include "velox/dwio/common/Options.h" #include "velox/dwio/parquet/common/ParquetConfig.h" namespace gluten { From e2c694c7257a3b4387413e82253a70412360b0f3 Mon Sep 17 00:00:00 2001 From: Yuan Date: Tue, 30 Jun 2026 14:41:50 +0100 Subject: [PATCH 09/12] fix gpu build Signed-off-by: Yuan --- cpp/velox/substrait/SubstraitToVeloxPlan.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/velox/substrait/SubstraitToVeloxPlan.cc b/cpp/velox/substrait/SubstraitToVeloxPlan.cc index c88dbcbe9b0..6a4544a2a11 100644 --- a/cpp/velox/substrait/SubstraitToVeloxPlan.cc +++ b/cpp/velox/substrait/SubstraitToVeloxPlan.cc @@ -782,7 +782,7 @@ std::shared_ptr makeCudfHiveInsertTableHandle( columnHandles.push_back(std::make_shared( tableColumnNames.at(i), tableColumnTypes.at(i), - cudf::data_type{cudf_velox::veloxToCudfTypeId(tableColumnTypes.at(i))})); + cudf_velox::veloxToCudfDataType(tableColumnTypes.at(i)))); } return std::make_shared( From b47e017c19d548025d4d42b704c33b18dee13a5a Mon Sep 17 00:00:00 2001 From: Yuan Date: Tue, 30 Jun 2026 15:05:52 +0100 Subject: [PATCH 10/12] remove run_setupscripts and arrow build Signed-off-by: Yuan --- .github/workflows/velox_backend_x86.yml | 2 +- dev/ci-velox-buildshared-centos-9.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/velox_backend_x86.yml b/.github/workflows/velox_backend_x86.yml index ad3cf96ecf8..3b97c86b3a5 100644 --- a/.github/workflows/velox_backend_x86.yml +++ b/.github/workflows/velox_backend_x86.yml @@ -1178,7 +1178,7 @@ jobs: mkdir -p /work/.ccache cd /work - bash dev/builddeps-veloxbe.sh --run_setup_script=ON --build_arrow=OFF --build_tests=ON --build_benchmarks=ON --enable_gpu=ON + bash dev/builddeps-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF --build_tests=ON --build_benchmarks=ON --enable_gpu=ON rm -rf ep/build-velox/build/velox_ep build/mvn clean package -Pbackends-velox -Pspark-3.4 -DskipTests ccache -s diff --git a/dev/ci-velox-buildshared-centos-9.sh b/dev/ci-velox-buildshared-centos-9.sh index 846765df7e1..2d2fa8c9bc9 100755 --- a/dev/ci-velox-buildshared-centos-9.sh +++ b/dev/ci-velox-buildshared-centos-9.sh @@ -27,5 +27,5 @@ fi export VELOX_BUILD_SHARED=ON -./dev/builddeps-veloxbe.sh --run_setup_script=ON --build_arrow=ON --build_tests=ON \ +./dev/builddeps-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF --build_tests=ON \ --build_examples=ON --build_benchmarks=ON From 3ca52415c8709eab2147e13fdaf6f5c0d8405ee2 Mon Sep 17 00:00:00 2001 From: Hazmi Date: Tue, 30 Jun 2026 15:08:30 +0000 Subject: [PATCH 11/12] Fix codestyle --- cpp/velox/substrait/SubstraitToVeloxPlan.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cpp/velox/substrait/SubstraitToVeloxPlan.cc b/cpp/velox/substrait/SubstraitToVeloxPlan.cc index 6a4544a2a11..5bd499582b8 100644 --- a/cpp/velox/substrait/SubstraitToVeloxPlan.cc +++ b/cpp/velox/substrait/SubstraitToVeloxPlan.cc @@ -780,9 +780,7 @@ std::shared_ptr makeCudfHiveInsertTableHandle( for (int i = 0; i < tableColumnNames.size(); ++i) { columnHandles.push_back(std::make_shared( - tableColumnNames.at(i), - tableColumnTypes.at(i), - cudf_velox::veloxToCudfDataType(tableColumnTypes.at(i)))); + tableColumnNames.at(i), tableColumnTypes.at(i), cudf_velox::veloxToCudfDataType(tableColumnTypes.at(i)))); } return std::make_shared( From 0e9e1bf09b44e9e33785a23d8581175405648336 Mon Sep 17 00:00:00 2001 From: Yuan Date: Tue, 30 Jun 2026 19:02:44 +0100 Subject: [PATCH 12/12] fix Signed-off-by: Yuan --- cpp/velox/utils/GpuBufferBatchResizer.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/velox/utils/GpuBufferBatchResizer.cc b/cpp/velox/utils/GpuBufferBatchResizer.cc index e896442dde3..fc4ed60cba5 100644 --- a/cpp/velox/utils/GpuBufferBatchResizer.cc +++ b/cpp/velox/utils/GpuBufferBatchResizer.cc @@ -152,7 +152,7 @@ std::shared_ptr makeCudfTable( DispatchColumn dispatch{stream, cudf::get_current_device_resource_ref(), buffers, numRows}; for (const auto& colType : type->children()) { auto res = VELOX_DYNAMIC_SCALAR_TYPE_DISPATCH( - dispatch.readFlatColumn, colType->kind(), cudf_velox::veloxToCudfTypeId(colType)); + dispatch.readFlatColumn, colType->kind(), cudf_velox::veloxToCudfDataType(colType).id()); cudfColumns.emplace_back(std::move(res)); } auto cudfTable = std::make_unique(std::move(cudfColumns));