diff --git a/cpp/velox/utils/VeloxWriterUtils.cc b/cpp/velox/utils/VeloxWriterUtils.cc index 9a9922423b2..fc4b810fb8f 100644 --- a/cpp/velox/utils/VeloxWriterUtils.cc +++ b/cpp/velox/utils/VeloxWriterUtils.cc @@ -35,7 +35,8 @@ using namespace facebook::velox::common; namespace { const int32_t kGzipWindowBits4k = 12; -} +const int32_t kZSTDDefaultCompressionLevel = 3; +} // namespace std::unique_ptr makeParquetWriteOption(const std::unordered_map& sparkConfs) { int64_t maxRowGroupBytes = 134217728; // 128MB @@ -73,12 +74,11 @@ std::unique_ptr makeParquetWriteOption(const std::unordered_mapsecond); - auto codecOptions = std::make_shared(); - codecOptions->compression_level = compressionLevel; - writeOption->codecOptions = std::move(codecOptions); - } + auto codecOptions = std::make_shared(); + auto it = sparkConfs.find(kParquetZSTDCompressionLevel); + auto compressionLevel = it != sparkConfs.end() ? std::stoi(it->second) : kZSTDDefaultCompressionLevel; + codecOptions->compression_level = compressionLevel; + writeOption->codecOptions = std::move(codecOptions); } else if (boost::iequals(compressionCodecStr, "uncompressed")) { compressionCodec = CompressionKind::CompressionKind_NONE; } else if (boost::iequals(compressionCodecStr, "none")) {