From 1125de48d342aa853c22bd4ce8f18fa83d539f20 Mon Sep 17 00:00:00 2001 From: wecharyu Date: Thu, 11 Dec 2025 22:59:23 +0800 Subject: [PATCH] [GLUTEN-11283][VL] Respect spark default zstd compression level --- cpp/velox/utils/VeloxWriterUtils.cc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cpp/velox/utils/VeloxWriterUtils.cc b/cpp/velox/utils/VeloxWriterUtils.cc index 9a9922423b2..fc4b810fb8f 100644 --- a/cpp/velox/utils/VeloxWriterUtils.cc +++ b/cpp/velox/utils/VeloxWriterUtils.cc @@ -35,7 +35,8 @@ using namespace facebook::velox::common; namespace { const int32_t kGzipWindowBits4k = 12; -} +const int32_t kZSTDDefaultCompressionLevel = 3; +} // namespace std::unique_ptr makeParquetWriteOption(const std::unordered_map& sparkConfs) { int64_t maxRowGroupBytes = 134217728; // 128MB @@ -73,12 +74,11 @@ std::unique_ptr makeParquetWriteOption(const std::unordered_mapsecond); - auto codecOptions = std::make_shared(); - codecOptions->compression_level = compressionLevel; - writeOption->codecOptions = std::move(codecOptions); - } + auto codecOptions = std::make_shared(); + auto it = sparkConfs.find(kParquetZSTDCompressionLevel); + auto compressionLevel = it != sparkConfs.end() ? std::stoi(it->second) : kZSTDDefaultCompressionLevel; + codecOptions->compression_level = compressionLevel; + writeOption->codecOptions = std::move(codecOptions); } else if (boost::iequals(compressionCodecStr, "uncompressed")) { compressionCodec = CompressionKind::CompressionKind_NONE; } else if (boost::iequals(compressionCodecStr, "none")) {